diff --git a/.env.example b/.env.example index 295d143..7c3523f 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,7 @@ GEMINI_API_KEY= +EDINET_API_KEY= +JQUANTS_REFRESH_TOKEN= +JQUANTS_API_KEY= +INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS= APP_ENV=local LOG_LEVEL=INFO diff --git a/src/investment_assistant/edinet/ingest.py b/src/investment_assistant/edinet/ingest.py index 1ddda4a..e24d24c 100644 --- a/src/investment_assistant/edinet/ingest.py +++ b/src/investment_assistant/edinet/ingest.py @@ -43,6 +43,7 @@ ) from investment_assistant.edinet.registry import EdinetTarget from investment_assistant.financials import compare_financials, load_financials +from investment_assistant.financials.dividend_quality import normalize_dividend_points from investment_assistant.financials.models import FinancialPoint from investment_assistant.ingestion.fetcher import reject_path_traversal from investment_assistant.observability import get_logger @@ -170,6 +171,7 @@ def ingest_targets( for p in _load_existing_points(csv_path) ] merged = dedupe_points([*deduped, *existing]) + merged, dividend_quality = normalize_dividend_points(merged) summary: dict[str, object] = { "output_dir": str(base_dir), "scanned_dates": scanned_dates, @@ -179,6 +181,7 @@ def ingest_targets( "cached_count": cached, "financial_points": len(deduped), "financial_points_total": len(merged), + "dividend_quality": dividend_quality, "results": results, } if merged: diff --git a/src/investment_assistant/financials/__init__.py b/src/investment_assistant/financials/__init__.py index 0088d1c..4f26e7d 100644 --- a/src/investment_assistant/financials/__init__.py +++ b/src/investment_assistant/financials/__init__.py @@ -1,5 +1,15 @@ """Cross-company financial comparison (non-advisory).""" +from investment_assistant.financials.current_yield import ( + CURRENT_YIELD_COLUMNS, + DEFAULT_CURRENT_YIELDS_CSV, + CurrentYieldFact, + CurrentYieldReconciliation, + current_yields_to_csv_text, + load_current_yields, + parse_current_yields_csv, + reconcile_current_yield, +) from investment_assistant.financials.evidence import ( build_financial_evidence, dividend_evidence_text, @@ -16,12 +26,20 @@ ) __all__ = [ + "CURRENT_YIELD_COLUMNS", "DISCLAIMER", + "DEFAULT_CURRENT_YIELDS_CSV", "FINANCIAL_COLUMNS", + "CurrentYieldFact", + "CurrentYieldReconciliation", "FinancialPoint", "build_financial_evidence", "compare_financials", + "current_yields_to_csv_text", "dividend_evidence_text", + "load_current_yields", "load_financials", + "parse_current_yields_csv", + "reconcile_current_yield", "ticker_from_source", ] diff --git a/src/investment_assistant/financials/current_yield.py b/src/investment_assistant/financials/current_yield.py new file mode 100644 index 0000000..2645eb8 --- /dev/null +++ b/src/investment_assistant/financials/current_yield.py @@ -0,0 +1,286 @@ +"""Current dividend/yield reconciliation for portfolio income estimates. + +EDINET dividend facts are historical filing values. They are useful evidence, +but they are not always on the same basis as a current market price because +stock splits and forecast dividend revisions can happen after the filing. This +module keeps that boundary explicit: current income uses a current/forecast +dividend fact when available, while EDINET remains the fallback and evidence. +""" + +from __future__ import annotations + +import csv +import io +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path + +CURRENT_YIELD_COLUMNS: tuple[str, ...] = ( + "ticker", + "name", + "current_dividend_per_share", + "current_price", + "yield_pct", + "as_of", + "source_ref", + "provider_id", + "note", +) + +DEFAULT_CURRENT_YIELDS_CSV = Path("local_docs/market/current_yields.csv") +CURRENT_YIELD_REVIEW_THRESHOLD_PCT = 5.0 + + +@dataclass(frozen=True) +class CurrentYieldFact: + ticker: str + name: str = "" + current_dividend_per_share: float | None = None + current_price: float | None = None + yield_pct: float | None = None + as_of: str = "" + source_ref: str = "" + provider_id: str = "user_csv" + note: str = "" + + def to_dict(self) -> dict[str, object]: + return {key: value for key, value in asdict(self).items() if value not in (None, "")} + + +@dataclass(frozen=True) +class CurrentYieldReconciliation: + ticker: str + name: str = "" + status: str = "not_available" + income_source: str = "not_available" + current_dividend_per_share: float | None = None + current_price: float | None = None + income_yield_pct: float | None = None + edinet_dividend_per_share: float | None = None + edinet_implied_yield_pct: float | None = None + correction_factor: float | None = None + source_ref: str = "" + provider_id: str = "" + warnings: tuple[str, ...] = () + formula: str = "" + + def to_dict(self) -> dict[str, object]: + return {key: value for key, value in asdict(self).items() if value not in (None, "", ())} + + +def load_current_yields( + path: str | Path | None = DEFAULT_CURRENT_YIELDS_CSV, +) -> dict[str, CurrentYieldFact]: + """Load current dividend/yield facts from CSV. + + Missing files are treated as an empty overlay so local production/dev flows + can opt in without forcing sample data. + """ + + if path is None: + return {} + csv_path = Path(path) + if not csv_path.is_file(): + return {} + return parse_current_yields_csv(csv_path.read_text(encoding="utf-8-sig")) + + +def parse_current_yields_csv(text: str) -> dict[str, CurrentYieldFact]: + """Parse current dividend/yield facts keyed by ticker.""" + + reader = csv.DictReader(io.StringIO(text)) + facts: dict[str, CurrentYieldFact] = {} + for row in reader: + fact = current_yield_fact_from_row(row) + if fact is not None: + facts[fact.ticker] = fact + return facts + + +def current_yield_fact_from_row(row: Mapping[str, object]) -> CurrentYieldFact | None: + ticker = _text(row.get("ticker") or row.get("code") or row.get("security_code")) + if not ticker: + return None + current_price = _optional_float( + row.get("current_price") or row.get("price") or row.get("market_price") + ) + yield_pct = _optional_float(row.get("yield_pct") or row.get("current_yield_pct")) + dividend = _optional_float( + row.get("current_dividend_per_share") + or row.get("forecast_dividend_per_share") + or row.get("dividend_per_share") + ) + if dividend is None and current_price is not None and yield_pct is not None: + dividend = current_price * yield_pct / 100.0 + if yield_pct is None and dividend is not None and current_price is not None: + yield_pct = dividend / current_price * 100.0 + return CurrentYieldFact( + ticker=ticker, + name=_text(row.get("name")), + current_dividend_per_share=_positive_or_none(dividend), + current_price=_positive_or_none(current_price), + yield_pct=_positive_or_none(yield_pct), + as_of=_text(row.get("as_of") or row.get("date") or row.get("last_updated")), + source_ref=_text(row.get("source_ref") or row.get("source") or row.get("url")), + provider_id=_text(row.get("provider_id") or row.get("provider")) or "user_csv", + note=_text(row.get("note")), + ) + + +def reconcile_current_yield( + *, + ticker: str, + name: str = "", + edinet_dividend_per_share: float | None = None, + current_price: float | None = None, + fact: CurrentYieldFact | None = None, + review_threshold_pct: float = CURRENT_YIELD_REVIEW_THRESHOLD_PCT, +) -> CurrentYieldReconciliation: + """Reconcile EDINET historical DPS with current dividend facts. + + A current/forecast dividend fact wins because it is price-date compatible. + EDINET is still returned as a fallback and as an audit comparison. + """ + + price = _positive_or_none(current_price) + if price is None and fact is not None: + price = fact.current_price + edinet_dps = _positive_or_none(edinet_dividend_per_share) + edinet_yield = _yield_pct(edinet_dps, price) + warnings: list[str] = [] + + if fact is not None and fact.current_dividend_per_share is not None: + dividend = fact.current_dividend_per_share + income_yield = _yield_pct(dividend, price) or fact.yield_pct + correction_factor = None + if edinet_dps is not None and dividend > 0 and not _close(edinet_dps, dividend): + correction_factor = round(edinet_dps / dividend, 6) + if ( + edinet_yield is not None + and income_yield is not None + and abs(edinet_yield - income_yield) >= max(1.0, income_yield * 0.25) + ): + warnings.append("edinet_current_basis_mismatch_adjusted") + return CurrentYieldReconciliation( + ticker=ticker, + name=fact.name or name, + status="current_fact", + income_source="current_dividend_per_share", + current_dividend_per_share=round(dividend, 6), + current_price=round(price, 6) if price is not None else None, + income_yield_pct=round(income_yield, 6) if income_yield is not None else None, + edinet_dividend_per_share=edinet_dps, + edinet_implied_yield_pct=edinet_yield, + correction_factor=correction_factor, + source_ref=fact.source_ref, + provider_id=fact.provider_id, + warnings=tuple(warnings), + formula="current_dividend_per_share / current_price * 100", + ) + + if edinet_dps is None: + return CurrentYieldReconciliation( + ticker=ticker, + name=name, + status="not_available", + income_source="not_available", + current_price=round(price, 6) if price is not None else None, + warnings=("current_dividend_missing",), + formula="current dividend fact or EDINET dividend per share required", + ) + + if edinet_yield is not None and edinet_yield >= review_threshold_pct: + warnings.append("edinet_current_basis_review") + return CurrentYieldReconciliation( + ticker=ticker, + name=name, + status="edinet_fallback_review" if warnings else "edinet_fallback", + income_source="edinet_latest_dividend_per_share", + current_dividend_per_share=round(edinet_dps, 6), + current_price=round(price, 6) if price is not None else None, + income_yield_pct=edinet_yield, + edinet_dividend_per_share=edinet_dps, + edinet_implied_yield_pct=edinet_yield, + warnings=tuple(warnings), + formula="edinet_latest_dividend_per_share / current_price * 100", + ) + + +def current_yields_to_csv_text(facts: Sequence[CurrentYieldFact]) -> str: + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=list(CURRENT_YIELD_COLUMNS), lineterminator="\n") + writer.writeheader() + for fact in facts: + writer.writerow( + { + "ticker": fact.ticker, + "name": fact.name, + "current_dividend_per_share": _format_number(fact.current_dividend_per_share), + "current_price": _format_number(fact.current_price), + "yield_pct": _format_number(fact.yield_pct), + "as_of": fact.as_of, + "source_ref": fact.source_ref, + "provider_id": fact.provider_id, + "note": fact.note, + } + ) + return output.getvalue() + + +def merge_current_yield_facts( + existing: Iterable[CurrentYieldFact], + incoming: Iterable[CurrentYieldFact], +) -> list[CurrentYieldFact]: + facts = {fact.ticker: fact for fact in existing} + facts.update({fact.ticker: fact for fact in incoming}) + return [facts[ticker] for ticker in sorted(facts)] + + +def _yield_pct(dividend_per_share: float | None, price: float | None) -> float | None: + dividend = _positive_or_none(dividend_per_share) + market_price = _positive_or_none(price) + if dividend is None or market_price is None: + return None + return round(dividend / market_price * 100.0, 6) + + +def _optional_float(value: object) -> float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int | float): + number = float(value) + return number if math.isfinite(number) else None + if isinstance(value, str): + text = value.strip().replace(",", "") + if not text: + return None + try: + number = float(text) + except ValueError: + return None + return number if math.isfinite(number) else None + return None + + +def _positive_or_none(value: float | None) -> float | None: + if value is None or isinstance(value, bool): + return None + number = float(value) + if not math.isfinite(number) or number <= 0: + return None + return number + + +def _text(value: object) -> str: + return str(value or "").strip() + + +def _format_number(value: float | None) -> str: + if value is None: + return "" + return str(int(value)) if value == int(value) else str(round(value, 6)) + + +def _close(left: float, right: float) -> bool: + return abs(left - right) <= max(0.01, abs(right) * 0.01) diff --git a/src/investment_assistant/financials/dividend_quality.py b/src/investment_assistant/financials/dividend_quality.py new file mode 100644 index 0000000..3108a55 --- /dev/null +++ b/src/investment_assistant/financials/dividend_quality.py @@ -0,0 +1,312 @@ +"""Dividend per-share validation and unit correction. + +The financial pipeline stores dividends as yen per share. EDINET filings and +manual CSVs can still carry obvious unit mistakes after extraction or copy/paste +work, most commonly 10x/100x values. This module keeps the correction +deterministic and conservative: it only changes a value when a previous fiscal +year or an extreme price-based yield makes the unit error clear. +""" + +from __future__ import annotations + +import csv +import io +import math +from collections.abc import Iterable, Sequence +from dataclasses import asdict, dataclass, replace + +from investment_assistant.financials.models import FINANCIAL_COLUMNS, FinancialPoint + +DEFAULT_MAX_REASONABLE_YIELD_PCT = 15.0 +DEFAULT_EXTREME_YIELD_PCT = 50.0 +_UNIT_FACTORS = (100.0, 10.0) + + +@dataclass(frozen=True) +class DividendQualityCheck: + """One validation/correction event for a dividend per-share value.""" + + ticker: str + fiscal_year: int + original_value: float + checked_value: float + status: str + code: str + message: str + previous_value: float | None = None + price: float | None = None + original_yield_pct: float | None = None + checked_yield_pct: float | None = None + correction_factor: float | None = None + + def to_dict(self) -> dict[str, object]: + return { + key: value + for key, value in asdict(self).items() + if value is not None + } + + +def normalize_dividend_per_share( + value: float, + *, + ticker: str = "", + fiscal_year: int = 0, + previous_value: float | None = None, + price: float | None = None, + max_reasonable_yield_pct: float = DEFAULT_MAX_REASONABLE_YIELD_PCT, + extreme_yield_pct: float = DEFAULT_EXTREME_YIELD_PCT, +) -> tuple[float, DividendQualityCheck | None]: + """Return a validated dividend-per-share value and optional audit check.""" + + original = float(value) + if original < 0: + return 0.0, DividendQualityCheck( + ticker=ticker, + fiscal_year=fiscal_year, + original_value=original, + checked_value=0.0, + status="corrected", + code="dividend_negative_clamped", + message="Negative dividend per share was clamped to 0.", + previous_value=previous_value, + price=price, + original_yield_pct=_yield_pct(original, price), + checked_yield_pct=_yield_pct(0.0, price), + ) + + previous = _positive(previous_value) + market_price = _positive(price) + + if previous is not None: + corrected = _correction_from_previous( + original, + previous, + market_price, + max_reasonable_yield_pct, + ) + if corrected is not None: + value_after, factor = corrected + return value_after, DividendQualityCheck( + ticker=ticker, + fiscal_year=fiscal_year, + original_value=original, + checked_value=value_after, + status="corrected", + code="dividend_unit_scale_corrected", + message=( + "Dividend per share looked 10x/100x larger than the prior " + "accepted fiscal-year value and was unit-normalized." + ), + previous_value=previous, + price=market_price, + original_yield_pct=_yield_pct(original, market_price), + checked_yield_pct=_yield_pct(value_after, market_price), + correction_factor=factor, + ) + + original_yield = _yield_pct(original, market_price) + if original_yield is not None: + corrected = _correction_from_extreme_yield( + original, + market_price, + max_reasonable_yield_pct=max_reasonable_yield_pct, + extreme_yield_pct=extreme_yield_pct, + ) + if corrected is not None: + value_after, factor = corrected + return value_after, DividendQualityCheck( + ticker=ticker, + fiscal_year=fiscal_year, + original_value=original, + checked_value=value_after, + status="corrected", + code="dividend_yield_unit_scale_corrected", + message=( + "Dividend per share implied an extreme yield; a 10x/100x " + "unit correction produced a plausible yield." + ), + previous_value=previous, + price=market_price, + original_yield_pct=original_yield, + checked_yield_pct=_yield_pct(value_after, market_price), + correction_factor=factor, + ) + if original_yield > max_reasonable_yield_pct: + return original, DividendQualityCheck( + ticker=ticker, + fiscal_year=fiscal_year, + original_value=original, + checked_value=original, + status="warn", + code="dividend_yield_high_review", + message=( + "Dividend per share implies a high yield; verify the source " + "before relying on this value." + ), + previous_value=previous, + price=market_price, + original_yield_pct=original_yield, + checked_yield_pct=original_yield, + ) + + return original, None + + +def normalize_dividend_points( + points: Sequence[FinancialPoint], +) -> tuple[list[FinancialPoint], dict[str, object]]: + """Normalize dividend values within each ticker's fiscal-year series.""" + + indexed = list(enumerate(points)) + corrected_by_index: dict[int, FinancialPoint] = {} + checks: list[DividendQualityCheck] = [] + + by_ticker: dict[str, list[tuple[int, FinancialPoint]]] = {} + for index, point in indexed: + by_ticker.setdefault(point.ticker, []).append((index, point)) + + for ticker_points in by_ticker.values(): + previous: float | None = None + for index, point in sorted(ticker_points, key=lambda item: item[1].fiscal_year): + value, check = normalize_dividend_per_share( + point.dividend_per_share, + ticker=point.ticker, + fiscal_year=point.fiscal_year, + previous_value=previous, + ) + if check is not None: + checks.append(check) + corrected_by_index[index] = ( + replace(point, dividend_per_share=value) + if value != point.dividend_per_share + else point + ) + if value > 0: + previous = value + + normalized = [corrected_by_index.get(index, point) for index, point in indexed] + return normalized, dividend_quality_summary(checks) + + +def dividend_quality_summary( + checks: Iterable[DividendQualityCheck], +) -> dict[str, object]: + """Summarize dividend validation events for API payloads.""" + + check_list = list(checks) + corrected = [check for check in check_list if check.status == "corrected"] + warnings = [check for check in check_list if check.status == "warn"] + status = "ok" + if corrected: + status = "corrected" + elif warnings: + status = "warn" + return { + "status": status, + "checked_rule": "dividend_per_share unit/yield sanity", + "max_reasonable_yield_pct": DEFAULT_MAX_REASONABLE_YIELD_PCT, + "extreme_yield_pct": DEFAULT_EXTREME_YIELD_PCT, + "corrected_count": len(corrected), + "warning_count": len(warnings), + "checks": [check.to_dict() for check in check_list], + } + + +def financial_points_to_csv_text(points: Sequence[FinancialPoint]) -> str: + """Serialize financial points as the canonical financials.csv format.""" + + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=list(FINANCIAL_COLUMNS), lineterminator="\n") + writer.writeheader() + for point in sorted(points, key=lambda p: (p.ticker, p.fiscal_year)): + writer.writerow( + { + "ticker": point.ticker, + "name": point.name, + "fiscal_year": str(point.fiscal_year), + "operating_cf": _format_number(point.operating_cf), + "equity_ratio": _format_number(point.equity_ratio), + "dividend_per_share": _format_number(point.dividend_per_share), + "payout_policy": point.payout_policy, + } + ) + return output.getvalue() + + +def _correction_from_previous( + original: float, + previous: float, + price: float | None, + max_reasonable_yield_pct: float, +) -> tuple[float, float] | None: + if original <= 0 or previous <= 0: + return None + original_gap = _log_gap(original, previous) + candidates: list[tuple[float, float, float]] = [] + for factor in _UNIT_FACTORS: + if original < previous * factor * 0.65: + continue + scaled = original / factor + if scaled <= 0: + continue + scaled_yield = _yield_pct(scaled, price) + if scaled_yield is not None and scaled_yield > max_reasonable_yield_pct: + continue + scaled_gap = _log_gap(scaled, previous) + if scaled_gap < original_gap / 3: + candidates.append((scaled_gap, scaled, factor)) + if not candidates: + return None + _, value, factor = min(candidates, key=lambda item: item[0]) + return value, factor + + +def _correction_from_extreme_yield( + original: float, + price: float | None, + *, + max_reasonable_yield_pct: float, + extreme_yield_pct: float, +) -> tuple[float, float] | None: + original_yield = _yield_pct(original, price) + if original_yield is None or original_yield < extreme_yield_pct: + return None + candidates: list[tuple[float, float, float]] = [] + for factor in _UNIT_FACTORS: + scaled = original / factor + scaled_yield = _yield_pct(scaled, price) + if scaled > 0 and scaled_yield is not None and scaled_yield <= max_reasonable_yield_pct: + candidates.append((scaled_yield, scaled, factor)) + if not candidates: + return None + # Keep the largest still-plausible yield so 800/1000 becomes 80 -> 8%, + # while 4100/1000 becomes 410% -> 4.1%. + _, value, factor = max(candidates, key=lambda item: item[0]) + return value, factor + + +def _yield_pct(dividend_per_share: float, price: float | None) -> float | None: + price_value = _positive(price) + if price_value is None: + return None + return round(dividend_per_share / price_value * 100.0, 6) + + +def _positive(value: float | None) -> float | None: + if value is None: + return None + if isinstance(value, bool): + return None + number = float(value) + if not math.isfinite(number) or number <= 0: + return None + return number + + +def _log_gap(a: float, b: float) -> float: + return abs(math.log(a / b)) + + +def _format_number(value: float) -> str: + return str(int(value)) if value == int(value) else str(value) diff --git a/src/investment_assistant/financials/evidence.py b/src/investment_assistant/financials/evidence.py index 5756a32..273fc44 100644 --- a/src/investment_assistant/financials/evidence.py +++ b/src/investment_assistant/financials/evidence.py @@ -11,6 +11,7 @@ import re from pathlib import Path +from investment_assistant.financials.dividend_quality import normalize_dividend_points from investment_assistant.financials.loader import compare_financials, load_financials DEFAULT_FINANCIALS_CSV = "local_docs/edinet/financials.csv" @@ -33,7 +34,8 @@ def load_comparison(csv_path: str | Path = DEFAULT_FINANCIALS_CSV) -> dict[str, if not path.is_file(): return None try: - return compare_financials(load_financials(path)) + points, _ = normalize_dividend_points(load_financials(path)) + return compare_financials(points) except (ValueError, OSError): return None diff --git a/src/investment_assistant/investment/__init__.py b/src/investment_assistant/investment/__init__.py index 59d0185..929ab22 100644 --- a/src/investment_assistant/investment/__init__.py +++ b/src/investment_assistant/investment/__init__.py @@ -3,6 +3,7 @@ from investment_assistant.investment.analysis import analyze_portfolio from investment_assistant.investment.candidates import screen_candidates from investment_assistant.investment.detail import build_investment_detail +from investment_assistant.investment.file_import import convert_holding_file_payload from investment_assistant.investment.loader import ( fund_profile_csv_template, fund_profiles_from_payload, @@ -20,6 +21,7 @@ "audit_investment_report", "build_investment_detail", "build_investment_monthly_report", + "convert_holding_file_payload", "fund_profile_csv_template", "fund_profiles_from_payload", "holding_csv_template", diff --git a/src/investment_assistant/investment/analysis.py b/src/investment_assistant/investment/analysis.py index 17e1a17..5c25e1e 100644 --- a/src/investment_assistant/investment/analysis.py +++ b/src/investment_assistant/investment/analysis.py @@ -6,7 +6,14 @@ from datetime import UTC, datetime from pathlib import Path +from investment_assistant.financials.current_yield import ( + DEFAULT_CURRENT_YIELDS_CSV, + CurrentYieldFact, + load_current_yields, + reconcile_current_yield, +) from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV, load_comparison +from investment_assistant.investment.edinet import build_edinet_summary from investment_assistant.investment.models import DISCLAIMER, InvestmentHolding from investment_assistant.investment.provider_policy import ProviderPolicy, provider_policy @@ -22,6 +29,7 @@ def analyze_portfolio( holdings: Sequence[InvestmentHolding], *, financials_csv: str | Path = DEFAULT_FINANCIALS_CSV, + current_yields_csv: str | Path | None = DEFAULT_CURRENT_YIELDS_CSV, runtime_mode: str = "development", ) -> dict[str, object]: """Analyze user-provided holdings without LLMs or recommendations.""" @@ -32,6 +40,7 @@ def analyze_portfolio( generated_at_dt = datetime.now(UTC) generated_at = generated_at_dt.isoformat() companies = _company_index(financials_csv) + current_yields = load_current_yields(current_yields_csv) financials_metadata = _financials_metadata(financials_csv, generated_at_dt) rows: list[dict[str, object]] = [] evidence: list[dict[str, object]] = [] @@ -53,7 +62,20 @@ def analyze_portfolio( market_value = holding.quantity * price cost_basis = holding.quantity * holding.avg_cost company = companies.get(holding.ticker_or_fund_code) - annual_income, income_source = _annual_income(holding, company) + current_yield_fact = current_yields.get(holding.ticker_or_fund_code) + edinet_dps = _number((company or {}).get("latest_dividend_per_share")) + yield_reconciliation = reconcile_current_yield( + ticker=holding.ticker_or_fund_code, + name=holding.name, + edinet_dividend_per_share=edinet_dps, + current_price=price, + fact=current_yield_fact, + ) + annual_income, income_source = _annual_income( + holding, + company, + current_yield=current_yield_fact, + ) provider_id = _holding_provider_id(holding) policy = provider_policy(provider_id, runtime_mode=runtime_mode) pnl = market_value - cost_basis @@ -69,10 +91,17 @@ def analyze_portfolio( "unrealized_pnl_pct": round(pnl / cost_basis * 100.0, 2) if cost_basis else 0.0, "annual_income_estimate": round(annual_income, 2), "annual_income_source": income_source, + "current_yield_reconciliation": yield_reconciliation.to_dict(), "income_yield_pct": round(annual_income / market_value * 100.0, 2) if market_value else 0.0, } + if company is not None: + row["edinet_summary"] = build_edinet_summary( + company, + financials_csv=financials_csv, + generated_at=generated_at, + ) row_data_alerts = _holding_data_alerts( holding=holding, generated_at=generated_at_dt, @@ -85,6 +114,7 @@ def analyze_portfolio( row=row, market_value=market_value, income_source=income_source, + yield_reconciliation=yield_reconciliation.to_dict(), ) row["data_alerts"] = row_data_alerts row["income_alerts"] = row_income_alerts @@ -128,11 +158,14 @@ def analyze_portfolio( { "claim_key": f"holding.{holding.ticker_or_fund_code}.annual_income", "source_type": income_source, - "source_ref": str(financials_csv) - if income_source == "edinet_latest_dividend_per_share" - else holding.source, + "source_ref": _income_source_ref( + income_source=income_source, + holding=holding, + financials_csv=financials_csv, + current_yield=current_yield_fact, + ), "metric_key": "annual_income_estimate", - "formula": "quantity * dividend_or_distribution_per_unit", + "formula": _income_formula(income_source), "last_updated": generated_at, "note": "Income estimate is deterministic and not a future guarantee.", } @@ -171,6 +204,12 @@ def analyze_portfolio( "asset_mix": _share_map(asset_mix, total_market), "tax_wrapper_mix": _share_map(tax_wrapper_mix, total_market), "nisa": nisa, + "edinet_covered_holdings": sum(1 for row in rows if row.get("edinet_summary")), + "edinet_source_ref": str(financials_csv), + "current_yields_source_ref": str(current_yields_csv) + if current_yields_csv is not None + else None, + "current_yield_overlay_count": len(current_yields), "data_quality": _data_quality_summary(data_alerts), "income_quality": _income_quality_summary(income_alerts), } @@ -241,12 +280,24 @@ def _company_index(financials_csv: str | Path) -> dict[str, dict[str, object]]: def _annual_income( - holding: InvestmentHolding, company: dict[str, object] | None + holding: InvestmentHolding, + company: dict[str, object] | None, + *, + current_yield: CurrentYieldFact | None = None, ) -> tuple[float, str]: if holding.annual_income is not None: return max(holding.annual_income, 0.0), "user_annual_income" if holding.distribution_per_unit is not None: return max(holding.distribution_per_unit, 0.0) * holding.quantity, "user_distribution" + if ( + holding.asset_type == "stock" + and current_yield is not None + and current_yield.current_dividend_per_share is not None + ): + return ( + current_yield.current_dividend_per_share * holding.quantity, + "current_dividend_per_share", + ) if holding.asset_type == "stock" and company is not None: dps = _number(company.get("latest_dividend_per_share")) if dps is not None: @@ -254,6 +305,28 @@ def _annual_income( return 0.0, "not_available" +def _income_source_ref( + *, + income_source: str, + holding: InvestmentHolding, + financials_csv: str | Path, + current_yield: CurrentYieldFact | None, +) -> str: + if income_source == "edinet_latest_dividend_per_share": + return str(financials_csv) + if income_source == "current_dividend_per_share" and current_yield is not None: + return current_yield.source_ref or current_yield.provider_id + return holding.source + + +def _income_formula(income_source: str) -> str: + if income_source == "current_dividend_per_share": + return "quantity * current_dividend_per_share" + if income_source == "edinet_latest_dividend_per_share": + return "quantity * edinet_latest_dividend_per_share" + return "quantity * dividend_or_distribution_per_unit" + + def _holding_data_alerts( *, holding: InvestmentHolding, @@ -350,6 +423,7 @@ def _income_alerts( row: Mapping[str, object], market_value: float, income_source: str, + yield_reconciliation: Mapping[str, object] | None = None, ) -> list[dict[str, object]]: alerts: list[dict[str, object]] = [] base: dict[str, object] = { @@ -394,6 +468,22 @@ def _income_alerts( ), } ) + warnings = yield_reconciliation.get("warnings") if yield_reconciliation is not None else None + if isinstance(warnings, (tuple, list)) and "edinet_current_basis_review" in warnings: + alerts.append( + { + **base, + "level": "warn", + "code": "current_yield_basis_review", + "field": "annual_income_estimate", + "value": row.get("annual_income_estimate"), + "message": ( + "EDINET dividend is a historical filing value and implies a high " + "current yield; add a current dividend/forecast CSV fact before " + "using it as current yield." + ), + } + ) income_yield_pct = _number(row.get("income_yield_pct")) or 0.0 if market_value > 0 and income_yield_pct >= _HIGH_INCOME_YIELD_PCT: alerts.append( diff --git a/src/investment_assistant/investment/candidates.py b/src/investment_assistant/investment/candidates.py index d8ffcbe..afe852d 100644 --- a/src/investment_assistant/investment/candidates.py +++ b/src/investment_assistant/investment/candidates.py @@ -3,13 +3,40 @@ from __future__ import annotations from collections.abc import Sequence +from datetime import UTC, datetime from pathlib import Path -from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV +from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV, load_comparison +from investment_assistant.investment.edinet import build_edinet_summary from investment_assistant.investment.models import DISCLAIMER, CandidateScreen, FundProfile from investment_assistant.investment.provider_policy import provider_policy from investment_assistant.scoring.stock import run_stock_scoring +FUND_SCORE_WEIGHTS: dict[str, float] = { + "expense_ratio": 0.35, + "nisa_eligible": 0.25, + "diversification": 0.30, + "distribution_policy": 0.10, +} + +_FUND_ASSET_CLASS_DIVERSIFICATION_HINTS: dict[str, float] = { + "global_equity": 0.90, + "balanced": 0.85, + "bond": 0.75, + "domestic_equity": 0.65, + "theme": 0.35, + "unknown": 0.50, +} + +_FUND_DISTRIBUTION_POLICY_SCORES: dict[str, float] = { + "reinvest": 1.0, + "accumulating": 1.0, + "no_distribution": 1.0, + "distribution": 0.70, + "monthly_distribution": 0.45, + "unknown": 0.50, +} + def screen_candidates( *, @@ -23,6 +50,10 @@ def screen_candidates( items: list[dict[str, object]] = [] blocked_providers: list[dict[str, object]] = [] asset_types = set(screen.asset_types) + generated_at = datetime.now(UTC).isoformat() + financials_source_ref = str(financials_csv) + companies = _company_index(financials_csv) + fund_scoring_model = _fund_scoring_model() if "stock" in asset_types: stock_result = run_stock_scoring( @@ -34,7 +65,16 @@ def screen_candidates( limit=None, ) for row in _rows(stock_result.get("results")): - items.append(_stock_candidate(row, screen)) + code = str(row.get("ticker") or "") + items.append( + _stock_candidate( + row, + screen, + financials_source_ref=financials_source_ref, + generated_at=generated_at, + company=companies.get(code), + ) + ) if "fund" in asset_types: for fund in funds: @@ -42,6 +82,7 @@ def screen_candidates( if not policy.production_allowed: blocked_providers.append(policy.to_dict()) continue + score_details = _fund_score_details(fund) if ( screen.max_expense_ratio is not None and fund.expense_ratio > screen.max_expense_ratio @@ -51,17 +92,27 @@ def screen_candidates( continue if ( screen.min_diversification_score is not None - and fund.diversification_score is not None - and fund.diversification_score < screen.min_diversification_score + and _float(score_details.get("diversification_score")) + < screen.min_diversification_score ): continue - items.append(_fund_candidate(fund, screen, policy.to_dict())) + items.append( + _fund_candidate( + fund, + screen, + policy.to_dict(), + score_details=score_details, + generated_at=generated_at, + ) + ) items = _sort(items, screen.sort_by) if screen.limit is not None: items = items[: max(screen.limit, 0)] return { "available": True, + "generated_at": generated_at, + "financials_source_ref": financials_source_ref, "screen": { "asset_types": list(screen.asset_types), "exclude_dividend_cut": screen.exclude_dividend_cut, @@ -75,6 +126,7 @@ def screen_candidates( "results": items, "count": len(items), "blocked_providers": blocked_providers, + "fund_scoring_model": fund_scoring_model, "non_advisory_boundary": ( "条件に一致した比較対象の提示のみです。買付・売却・保有継続を推奨しません。" ), @@ -108,7 +160,14 @@ def screen_from_values( ) -def _stock_candidate(row: dict[str, object], screen: CandidateScreen) -> dict[str, object]: +def _stock_candidate( + row: dict[str, object], + screen: CandidateScreen, + *, + financials_source_ref: str, + generated_at: str, + company: dict[str, object] | None, +) -> dict[str, object]: metrics = row.get("metrics") metric_map = metrics if isinstance(metrics, dict) else {} conditions = ["EDINET財務データあり"] @@ -116,6 +175,16 @@ def _stock_candidate(row: dict[str, object], screen: CandidateScreen) -> dict[st conditions.append("減配履歴なし") if screen.min_equity_ratio is not None: conditions.append(f"自己資本比率 {screen.min_equity_ratio:g}% 以上") + evidence = [ + { + "claim_key": f"candidate.{row.get('ticker')}.edinet_financials", + "source_type": "edinet_financials", + "metric_key": "dividend/equity/operating_cf", + "source_ref": financials_source_ref, + "formula": "EDINET由来財務データを決定論ルールで集計", + "last_updated": generated_at, + } + ] return { "asset_type": "stock", "code": row.get("ticker"), @@ -123,18 +192,24 @@ def _stock_candidate(row: dict[str, object], screen: CandidateScreen) -> dict[st "score": row.get("total_score"), "matched_conditions": conditions, "metrics": metric_map, - "evidence": [ - { - "source_type": "edinet_financials", - "metric_key": "dividend/equity/operating_cf", - "source_ref": str(DEFAULT_FINANCIALS_CSV), - } - ], + "edinet_summary": build_edinet_summary( + company, + financials_csv=financials_source_ref, + generated_at=generated_at, + ) + if company is not None + else None, + "evidence": evidence, } def _fund_candidate( - fund: FundProfile, screen: CandidateScreen, policy: dict[str, object] + fund: FundProfile, + screen: CandidateScreen, + policy: dict[str, object], + *, + score_details: dict[str, object], + generated_at: str, ) -> dict[str, object]: conditions = ["投信プロファイル入力あり"] if screen.max_expense_ratio is not None: @@ -147,32 +222,166 @@ def _fund_candidate( "asset_type": "fund", "code": fund.fund_code, "name": fund.name, - "score": fund.diversification_score, + "score": score_details["score"], "matched_conditions": conditions, "metrics": { "asset_class": fund.asset_class, "expense_ratio": fund.expense_ratio, "distribution_policy": fund.distribution_policy, "nisa_eligible": fund.nisa_eligible, - "diversification_score": fund.diversification_score, + "diversification_score": score_details["diversification_score"], + "diversification_source": score_details["diversification_source"], + "calculated_score": score_details["score"], + "score_model": score_details["model_version"], }, + "scoring_model": _fund_scoring_model(), + "score_breakdown": score_details["breakdown"], "provider_policy": policy, "evidence": [ { + "claim_key": f"candidate.{fund.fund_code}.fund_profile_score", "source_type": "fund_profile", "metric_key": "expense_ratio/nisa_eligible/diversification_score", "source_ref": fund.provider_id, + "formula": score_details["formula"], + "last_updated": generated_at, } ], } +def _fund_scoring_model() -> dict[str, object]: + return { + "model_version": "fund_weighted_v1", + "formula": "sum(weight * normalized_score)", + "weights": [ + { + "key": "expense_ratio", + "label": "低コスト性", + "weight": FUND_SCORE_WEIGHTS["expense_ratio"], + }, + { + "key": "nisa_eligible", + "label": "NISA適合", + "weight": FUND_SCORE_WEIGHTS["nisa_eligible"], + }, + { + "key": "diversification", + "label": "分散度", + "weight": FUND_SCORE_WEIGHTS["diversification"], + }, + { + "key": "distribution_policy", + "label": "分配方針", + "weight": FUND_SCORE_WEIGHTS["distribution_policy"], + }, + ], + "note": ( + "投信スコアは条件比較のための機械集計です。" + "売買推奨や将来リターン予測ではありません。" + ), + } + + +def _fund_score_details(fund: FundProfile) -> dict[str, object]: + diversification, diversification_source = _fund_diversification_score(fund) + normalized_scores = { + "expense_ratio": _expense_ratio_score(fund.expense_ratio), + "nisa_eligible": 1.0 if fund.nisa_eligible else 0.0, + "diversification": diversification, + "distribution_policy": _distribution_policy_score(fund.distribution_policy), + } + raw_values = { + "expense_ratio": f"{fund.expense_ratio:g}%", + "nisa_eligible": "対象" if fund.nisa_eligible else "対象外", + "diversification": f"{diversification:g} ({diversification_source})", + "distribution_policy": fund.distribution_policy, + } + formulas = { + "expense_ratio": "max(0, 1 - 信託報酬(%) / 1.0)", + "nisa_eligible": "NISA対象なら1、対象外なら0", + "diversification": "入力値。未入力時は資産クラスの保守的な目安値", + "distribution_policy": "再投資/無分配を1、分配型を0.7、毎月分配を0.45で正規化", + } + labels = { + "expense_ratio": "低コスト性", + "nisa_eligible": "NISA適合", + "diversification": "分散度", + "distribution_policy": "分配方針", + } + breakdown: list[dict[str, object]] = [] + total = 0.0 + for key, weight in FUND_SCORE_WEIGHTS.items(): + normalized_score = normalized_scores[key] + contribution = weight * normalized_score + total += contribution + breakdown.append( + { + "key": key, + "label": labels[key], + "weight": weight, + "raw_value": raw_values[key], + "normalized_score": round(normalized_score, 6), + "contribution": round(contribution, 6), + "formula": formulas[key], + } + ) + return { + "model_version": "fund_weighted_v1", + "formula": "sum(weight * normalized_score)", + "score": round(total, 6), + "diversification_score": diversification, + "diversification_source": diversification_source, + "breakdown": breakdown, + } + + +def _fund_diversification_score(fund: FundProfile) -> tuple[float, str]: + if fund.diversification_score is not None: + return _clamp01(fund.diversification_score), "user_input" + asset_class = fund.asset_class.strip().lower() or "unknown" + return ( + _FUND_ASSET_CLASS_DIVERSIFICATION_HINTS.get(asset_class, 0.50), + f"asset_class:{asset_class}", + ) + + +def _expense_ratio_score(expense_ratio: float) -> float: + return _clamp01(1.0 - max(expense_ratio, 0.0) / 1.0) + + +def _distribution_policy_score(policy: str) -> float: + normalized = policy.strip().lower() or "unknown" + return _FUND_DISTRIBUTION_POLICY_SCORES.get(normalized, 0.50) + + +def _clamp01(value: float) -> float: + return min(1.0, max(0.0, value)) + + def _rows(value: object) -> list[dict[str, object]]: if not isinstance(value, list): return [] return [item for item in value if isinstance(item, dict)] +def _company_index(financials_csv: str | Path) -> dict[str, dict[str, object]]: + comparison = load_comparison(financials_csv) + if comparison is None: + return {} + companies = comparison.get("companies") + if not isinstance(companies, list): + return {} + out: dict[str, dict[str, object]] = {} + for company in companies: + if not isinstance(company, dict): + continue + ticker = str(company.get("ticker") or "").strip() + if ticker: + out[ticker] = company + return out + + def _sort(items: list[dict[str, object]], sort_by: str) -> list[dict[str, object]]: if sort_by == "expense_ratio": return sorted(items, key=lambda item: _expense_key(item)) @@ -183,6 +392,10 @@ def _sort(items: list[dict[str, object]], sort_by: str) -> list[dict[str, object def _score(item: dict[str, object]) -> float: value = item.get("score") + return _float(value) + + +def _float(value: object) -> float: if isinstance(value, bool): return 0.0 if isinstance(value, int | float): diff --git a/src/investment_assistant/investment/data_catalog.py b/src/investment_assistant/investment/data_catalog.py new file mode 100644 index 0000000..45e41b6 --- /dev/null +++ b/src/investment_assistant/investment/data_catalog.py @@ -0,0 +1,292 @@ +"""Data catalog for the local investment assistant workspace.""" + +from __future__ import annotations + +import csv +from collections.abc import Callable, Iterable +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from investment_assistant.financials.current_yield import ( + DEFAULT_CURRENT_YIELDS_CSV, + load_current_yields, +) +from investment_assistant.financials.dividend_quality import normalize_dividend_points +from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV +from investment_assistant.financials.loader import load_financials +from investment_assistant.investment.universe import ( + DEFAULT_JPX_LISTED_ISSUES_PATH, + load_jpx_listed_issues, +) +from investment_assistant.portfolio.bar_store import ( + DEFAULT_DAILY_BARS_CSV, + load_daily_bars, +) +from investment_assistant.portfolio.price_store import ( + DEFAULT_CURRENT_PRICES_CSV, + load_current_prices, +) + +JsonDict = dict[str, Any] +DEFAULT_COMPANY_MASTER_CSV = Path("local_docs/company_master/company_master.csv") + + +def build_data_catalog( + *, + financials_csv: str | Path = DEFAULT_FINANCIALS_CSV, + jpx_listed_path: str | Path = DEFAULT_JPX_LISTED_ISSUES_PATH, + company_master_path: str | Path = DEFAULT_COMPANY_MASTER_CSV, + market_prices_path: str | Path = DEFAULT_CURRENT_PRICES_CSV, + daily_bars_path: str | Path = DEFAULT_DAILY_BARS_CSV, + current_yields_path: str | Path = DEFAULT_CURRENT_YIELDS_CSV, + stale_after_days: int = 7, +) -> JsonDict: + """Return a single status view over all local data files.""" + + stale_days = max(int(stale_after_days), 1) + datasets = [ + _financials_dataset(Path(financials_csv), stale_days), + _jpx_listed_dataset(Path(jpx_listed_path), stale_days), + _company_master_dataset(Path(company_master_path), stale_days), + _market_prices_dataset(Path(market_prices_path), stale_days), + _daily_bars_dataset(Path(daily_bars_path), stale_days), + _current_yields_dataset(Path(current_yields_path), stale_days), + ] + by_key = {str(item["key"]): item for item in datasets} + missing = [item for item in datasets if item["status"] == "missing"] + stale = [item for item in datasets if item["status"] == "stale"] + invalid = [item for item in datasets if item["status"] == "invalid"] + return { + "available": True, + "status": "needs_attention" if missing or stale or invalid else "ready", + "generated_at": datetime.now(UTC).isoformat(), + "canonical_paths": { + "financials_csv": str(financials_csv), + "jpx_listed_path": str(jpx_listed_path), + "company_master_path": str(company_master_path), + "market_prices_path": str(market_prices_path), + "daily_bars_path": str(daily_bars_path), + "current_yields_path": str(current_yields_path), + }, + "datasets": datasets, + "by_key": by_key, + "summary": { + "dataset_count": len(datasets), + "ready_count": sum(1 for item in datasets if item["status"] == "ready"), + "missing_count": len(missing), + "stale_count": len(stale), + "invalid_count": len(invalid), + }, + "next_actions": _next_actions(datasets), + "auto_trading": False, + "call_real_api": False, + } + + +def _financials_dataset(path: Path, stale_after_days: int) -> JsonDict: + def reader() -> JsonDict: + points, _ = normalize_dividend_points(load_financials(path)) + tickers = {point.ticker for point in points} + latest_year = max((point.fiscal_year for point in points), default=None) + return { + "row_count": len(points), + "company_count": len(tickers), + "latest_fiscal_year": latest_year, + } + + return _dataset( + key="financials", + label="EDINET財務", + path=path, + stale_after_days=stale_after_days, + reader=reader, + purpose="候補抽出、保有分析、レポート根拠", + ) + + +def _jpx_listed_dataset(path: Path, stale_after_days: int) -> JsonDict: + def reader() -> JsonDict: + issues = load_jpx_listed_issues(path) + return { + "row_count": len(issues), + "prime_count": sum(1 for issue in issues if issue.is_prime), + "as_of": _max_text(issue.as_of for issue in issues), + } + + return _dataset( + key="jpx_listed", + label="JPX上場一覧", + path=path, + stale_after_days=stale_after_days, + reader=reader, + purpose="証券コード検索、東証プライム選択、会社マスター作成", + ) + + +def _company_master_dataset(path: Path, stale_after_days: int) -> JsonDict: + def reader() -> JsonDict: + rows = _read_csv_rows(path) + return { + "row_count": len(rows), + "prime_count": sum( + 1 + for row in rows + if "プライム" + in str(row.get("market_segment") or row.get("market_segment_label") or "") + ), + "financials_count": sum(1 for row in rows if _truthy(row.get("has_financials"))), + } + + return _dataset( + key="company_master", + label="会社マスター", + path=path, + stale_after_days=stale_after_days, + reader=reader, + purpose="銘柄選択、日経225/東証プライム/財務取得状況の統合表示", + ) + + +def _market_prices_dataset(path: Path, stale_after_days: int) -> JsonDict: + def reader() -> JsonDict: + facts = load_current_prices(path) + return { + "row_count": len(facts), + "ticker_count": len(facts), + "latest_as_of": _max_text(fact.as_of for fact in facts.values()), + "providers": sorted({fact.provider_id for fact in facts.values() if fact.provider_id}), + } + + return _dataset( + key="market_prices", + label="株価スナップショット", + path=path, + stale_after_days=1, + reader=reader, + purpose="試算画面の株価初期値と取得失敗時のフォールバック", + ) + + +def _daily_bars_dataset(path: Path, stale_after_days: int) -> JsonDict: + def reader() -> JsonDict: + facts = load_daily_bars(path) + tickers = {fact.ticker for fact in facts} + return { + "row_count": len(facts), + "ticker_count": len(tickers), + "latest_as_of": _max_text(fact.date for fact in facts), + "providers": sorted({fact.provider_id for fact in facts if fact.provider_id}), + } + + return _dataset( + key="daily_bars", + label="株価四本値・出来高", + path=path, + stale_after_days=1, + reader=reader, + purpose="日次OHLCV、調整後終値、出来高、売買代金による履歴分析", + ) + + +def _current_yields_dataset(path: Path, stale_after_days: int) -> JsonDict: + def reader() -> JsonDict: + facts = load_current_yields(path) + return { + "row_count": len(facts), + "ticker_count": len(facts), + "latest_as_of": _max_text(fact.as_of for fact in facts.values()), + "providers": sorted({fact.provider_id for fact in facts.values() if fact.provider_id}), + } + + return _dataset( + key="current_yields", + label="現在配当・利回り", + path=path, + stale_after_days=stale_after_days, + reader=reader, + purpose="EDINET配当と現在株価の単位ずれ補正", + ) + + +def _dataset( + *, + key: str, + label: str, + path: Path, + stale_after_days: int, + reader: Callable[[], JsonDict], + purpose: str, +) -> JsonDict: + base: JsonDict = { + "key": key, + "label": label, + "path": str(path), + "purpose": purpose, + "stale_after_days": stale_after_days, + "auto_trading": False, + } + if not path.is_file(): + return { + **base, + "available": False, + "status": "missing", + "modified_at": None, + "age_days": None, + "row_count": 0, + } + stat = path.stat() + modified_at = datetime.fromtimestamp(stat.st_mtime, UTC) + age_days = (datetime.now(UTC) - modified_at).total_seconds() / 86400 + try: + extra = reader() + except (OSError, ValueError, csv.Error) as exc: + return { + **base, + "available": False, + "status": "invalid", + "modified_at": modified_at.isoformat(), + "age_days": round(age_days, 2), + "row_count": 0, + "error": f"{type(exc).__name__}: {exc}", + } + return { + **base, + "available": True, + "status": "stale" if age_days > stale_after_days else "ready", + "modified_at": modified_at.isoformat(), + "age_days": round(age_days, 2), + **extra, + } + + +def _read_csv_rows(path: Path) -> list[dict[str, str]]: + with path.open(newline="", encoding="utf-8-sig") as handle: + return [dict(row) for row in csv.DictReader(handle)] + + +def _truthy(value: object) -> bool: + return str(value or "").strip().lower() in {"1", "true", "yes", "y", "on"} + + +def _max_text(values: Iterable[object]) -> str: + strings = [str(value).strip() for value in values if str(value or "").strip()] + return max(strings) if strings else "" + + +def _next_actions(datasets: list[JsonDict]) -> list[str]: + actions: list[str] = [] + by_key = {str(item["key"]): item for item in datasets} + if by_key["jpx_listed"]["status"] == "missing": + actions.append("JPX公式データを取得して、東証プライムを選択できる状態にする。") + if by_key["financials"]["status"] == "missing": + actions.append("EDINETまたは手動CSVで財務データを作成する。") + if by_key["company_master"]["status"] in {"missing", "stale"}: + actions.append("会社マスターを更新して、会社情報と財務取得状況をそろえる。") + if by_key["market_prices"]["status"] in {"missing", "stale"}: + actions.append("J-Quantsまたは許可済み価格ソースで株価を更新する。") + if by_key["daily_bars"]["status"] in {"missing", "stale"}: + actions.append("J-Quantsの日次OHLCVを更新して、出来高と価格推移を確認できる状態にする。") + if by_key["current_yields"]["status"] == "missing": + actions.append("必要な銘柄だけ現在配当・利回りデータを追加する。") + return actions diff --git a/src/investment_assistant/investment/detail.py b/src/investment_assistant/investment/detail.py index 74ecac9..f811c4f 100644 --- a/src/investment_assistant/investment/detail.py +++ b/src/investment_assistant/investment/detail.py @@ -10,8 +10,10 @@ from datetime import UTC, datetime from pathlib import Path +from investment_assistant.financials.current_yield import DEFAULT_CURRENT_YIELDS_CSV from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV, load_comparison from investment_assistant.investment.analysis import analyze_portfolio +from investment_assistant.investment.edinet import build_edinet_summary from investment_assistant.investment.models import DISCLAIMER, FundProfile, InvestmentHolding @@ -22,6 +24,7 @@ def build_investment_detail( holdings: Sequence[InvestmentHolding] = (), funds: Sequence[FundProfile] = (), financials_csv: str | Path = DEFAULT_FINANCIALS_CSV, + current_yields_csv: str | Path | None = DEFAULT_CURRENT_YIELDS_CSV, ) -> dict[str, object]: """Build a non-advisory detail view for a single code.""" @@ -47,7 +50,11 @@ def build_investment_detail( holding_summary: dict[str, object] | None = None holding_rows: list[dict[str, object]] = [] if matching_holdings: - analysis = analyze_portfolio(holdings, financials_csv=financials_csv) + analysis = analyze_portfolio( + holdings, + financials_csv=financials_csv, + current_yields_csv=current_yields_csv, + ) all_rows = analysis.get("holdings") if isinstance(all_rows, list): holding_rows = [ @@ -104,6 +111,15 @@ def build_investment_detail( } ) metrics.extend(_financial_metrics(company, claim_key, generated_at)) + edinet_summary = ( + build_edinet_summary( + company, + financials_csv=financials_csv, + generated_at=generated_at, + ) + if company is not None + else None + ) if fund is not None: claim_key = f"fund.{normalized_code}.profile" @@ -130,6 +146,7 @@ def build_investment_detail( "holding_summary": holding_summary, "holdings": holding_rows, "financials": company, + "edinet_summary": edinet_summary, "fund_profile": fund.to_dict() if fund is not None else None, "metrics": _dedupe_metrics(metrics), "sections": _sections( diff --git a/src/investment_assistant/investment/edinet.py b/src/investment_assistant/investment/edinet.py new file mode 100644 index 0000000..0768b6c --- /dev/null +++ b/src/investment_assistant/investment/edinet.py @@ -0,0 +1,64 @@ +"""Small EDINET summary helpers for investment MVP payloads.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from pathlib import Path + +TREND_LABELS: dict[str, str] = { + "increasing": "増加傾向", + "declining": "減少傾向", + "flat": "横ばい", + "mixed": "増減混在", + "insufficient": "データ不足", +} + + +def trend_label(value: object) -> str: + """Return a Japanese label for a normalized financial trend value.""" + + key = str(value or "insufficient") + return TREND_LABELS.get(key, key) + + +def build_edinet_summary( + company: Mapping[str, object], + *, + financials_csv: str | Path, + generated_at: str, +) -> dict[str, object]: + """Build a compact, display-ready EDINET financial summary.""" + + years = _list(company.get("years")) + cut_years = _list(company.get("dividend_cut_years")) + dividend_series = _list(company.get("dividend_series")) + dividend_trend = str(company.get("dividend_trend") or "insufficient") + operating_cf_trend = str(company.get("operating_cf_trend") or "insufficient") + equity_ratio_trend = str(company.get("equity_ratio_trend") or "insufficient") + return { + "source_type": "edinet_financials", + "source_ref": str(financials_csv), + "ticker": str(company.get("ticker") or ""), + "name": str(company.get("name") or ""), + "latest_fiscal_year": company.get("latest_fiscal_year"), + "latest_operating_cf": company.get("latest_operating_cf"), + "latest_equity_ratio": company.get("latest_equity_ratio"), + "latest_dividend_per_share": company.get("latest_dividend_per_share"), + "dividend_cut_years": cut_years, + "dividend_trend": dividend_trend, + "dividend_trend_label": trend_label(dividend_trend), + "operating_cf_trend": operating_cf_trend, + "operating_cf_trend_label": trend_label(operating_cf_trend), + "equity_ratio_trend": equity_ratio_trend, + "equity_ratio_trend_label": trend_label(equity_ratio_trend), + "periods": len(years) if years else len(dividend_series), + "payout_policy": company.get("payout_policy"), + "last_updated": generated_at, + "note": "EDINET由来の取得済み財務CSVを機械集計した比較材料です。", + } + + +def _list(value: object) -> list[object]: + if not isinstance(value, Sequence) or isinstance(value, str | bytes): + return [] + return list(value) diff --git a/src/investment_assistant/investment/file_import.py b/src/investment_assistant/investment/file_import.py new file mode 100644 index 0000000..7840bb5 --- /dev/null +++ b/src/investment_assistant/investment/file_import.py @@ -0,0 +1,347 @@ +"""File-to-CSV conversion helpers for investment holdings. + +The converter is intentionally conservative: it only normalizes uploaded files +into the existing holdings CSV contract, then delegates validation to the same +deterministic loader used by the rest of the MVP. +""" + +from __future__ import annotations + +import base64 +import binascii +import csv +import html as html_lib +import io +import re +import zlib +from collections.abc import Mapping, Sequence +from html.parser import HTMLParser + +from investment_assistant.investment.loader import validate_holdings_payload + +_TEXT_ENCODINGS = ("utf-8-sig", "utf-8", "cp932", "shift_jis", "euc_jp") +_PDF_STREAM_RE = re.compile(rb"stream\r?\n(.*?)\r?\nendstream", re.DOTALL) +_PDF_LITERAL_RE = re.compile(rb"\((?:\\.|[^\\()])*\)", re.DOTALL) +_PDF_HEX_RE = re.compile(rb"(?(?!>)") + + +def convert_holding_file_payload(payload: Mapping[str, object]) -> dict[str, object]: + """Convert an uploaded CSV/HTML/PDF holdings file into validated CSV text.""" + + filename = str(payload.get("filename") or "uploaded").strip() or "uploaded" + content_type = str(payload.get("content_type") or "").strip() + raw = _payload_bytes(payload) + detected_format = _detect_format(filename, content_type, raw) + warnings: list[dict[str, object]] = [] + encoding: str | None = None + + if detected_format in {"csv", "tsv", "text"}: + csv_text, encoding = _decode_text(raw) + if detected_format == "tsv": + warnings.append(_warning("tsv_converted", "TSV input was normalized as CSV.")) + elif detected_format == "html": + html_text, encoding = _decode_text(raw) + csv_text = _csv_from_html(html_text, warnings) + elif detected_format == "pdf": + pdf_text = _extract_pdf_text(raw) + warnings.append( + _warning( + "pdf_text_best_effort", + "PDF text extraction is best-effort. Review the converted CSV before analysis.", + ) + ) + csv_text = _csv_from_text(pdf_text, warnings) + else: + raise ValueError(f"Unsupported holdings file type: {filename or content_type}") + + validation = validate_holdings_payload({"csv_text": csv_text}) + validation_warnings = validation.get("warnings") + input_warnings = list(warnings) + if isinstance(validation_warnings, list): + input_warnings.extend(item for item in validation_warnings if isinstance(item, dict)) + + return { + "available": True, + "filename": filename, + "content_type": content_type, + "detected_format": detected_format, + "detected_encoding": encoding, + "csv_text": csv_text, + "validation": validation, + "valid": validation.get("valid") is True, + "count": validation.get("count", 0), + "holdings": validation.get("holdings", []), + "warnings": warnings, + "input_warnings": input_warnings, + "auto_trading": False, + "call_real_api": False, + } + + +def _payload_bytes(payload: Mapping[str, object]) -> bytes: + raw_base64 = payload.get("content_base64") + if isinstance(raw_base64, str) and raw_base64.strip(): + try: + return base64.b64decode(raw_base64, validate=True) + except (ValueError, binascii.Error) as exc: + raise ValueError("content_base64 must be valid base64") from exc + + text = payload.get("text") + if isinstance(text, str) and text: + return text.encode("utf-8") + + csv_text = payload.get("csv_text") + if isinstance(csv_text, str) and csv_text: + return csv_text.encode("utf-8") + + raise ValueError("content_base64, text, or csv_text is required") + + +def _detect_format(filename: str, content_type: str, raw: bytes) -> str: + lowered_name = filename.lower() + lowered_type = content_type.lower() + sniff = raw[:256].lstrip().lower() + if ( + lowered_name.endswith(".pdf") + or lowered_type == "application/pdf" + or raw.startswith(b"%PDF") + ): + return "pdf" + if ( + lowered_name.endswith((".html", ".htm")) + or "html" in lowered_type + or sniff.startswith((b" tuple[str, str]: + for encoding in _TEXT_ENCODINGS: + try: + return data.decode(encoding), encoding + except UnicodeDecodeError: + continue + return data.decode("utf-8", errors="replace"), "utf-8-replace" + + +def _csv_from_html(text: str, warnings: list[dict[str, object]]) -> str: + parser = _HtmlTableParser() + parser.feed(text) + for index, table in enumerate(parser.tables, start=1): + candidate = _csv_from_table(table) + if candidate and _valid_holding_csv(candidate): + warnings.append( + _warning( + "html_table_converted", + f"HTML table #{index} was converted to holdings CSV.", + ) + ) + return candidate + + fallback_text = _html_to_text(text) + return _csv_from_text(fallback_text, warnings) + + +def _csv_from_table(table: Sequence[Sequence[str]]) -> str | None: + rows = [[cell.strip() for cell in row] for row in table if any(cell.strip() for cell in row)] + for header_index in range(max(0, min(3, len(rows) - 1))): + candidate = _rows_to_csv(rows[header_index], rows[header_index + 1 :]) + if _valid_holding_csv(candidate): + return candidate + return None + + +def _csv_from_text(text: str, warnings: list[dict[str, object]]) -> str: + cleaned = text.strip().lstrip("\ufeff") + if _valid_holding_csv(cleaned): + return cleaned + + lines = [line.strip() for line in cleaned.splitlines() if line.strip()] + for start in range(min(10, len(lines))): + candidate = "\n".join(lines[start:]) + if _valid_holding_csv(candidate): + warnings.append( + _warning( + "text_table_converted", + "Delimited text was converted to holdings CSV.", + ) + ) + return candidate + + raise ValueError( + "Could not find a holdings table. Include asset_type, ticker_or_fund_code, " + "name, quantity, and avg_cost columns." + ) + + +def _rows_to_csv(header: Sequence[str], rows: Sequence[Sequence[str]]) -> str: + output = io.StringIO() + writer = csv.writer(output, lineterminator="\n") + writer.writerow(list(header)) + width = len(header) + for row in rows: + cells = list(row[:width]) + if len(cells) < width: + cells.extend([""] * (width - len(cells))) + if any(str(cell).strip() for cell in cells): + writer.writerow(cells) + return output.getvalue() + + +def _valid_holding_csv(text: str) -> bool: + return validate_holdings_payload({"csv_text": text}).get("valid") is True + + +def _html_to_text(text: str) -> str: + parser = _HtmlTextParser() + parser.feed(text) + parser.close() + return "\n".join(line for line in parser.lines if line.strip()) + + +def _extract_pdf_text(data: bytes) -> str: + chunks = [data] + for match in _PDF_STREAM_RE.finditer(data): + stream = match.group(1).strip(b"\r\n") + chunks.append(stream) + try: + chunks.append(zlib.decompress(stream)) + except zlib.error: + continue + + strings: list[str] = [] + for chunk in chunks: + strings.extend(_decode_pdf_literal(item) for item in _PDF_LITERAL_RE.findall(chunk)) + strings.extend(_decode_pdf_hex(item) for item in _PDF_HEX_RE.findall(chunk)) + + extracted = "\n".join(item.strip() for item in strings if item.strip()) + if extracted: + return extracted + text, _encoding = _decode_text(data) + return text + + +def _decode_pdf_literal(token: bytes) -> str: + body = token[1:-1] + out = bytearray() + index = 0 + while index < len(body): + value = body[index] + if value != 0x5C: # backslash + out.append(value) + index += 1 + continue + index += 1 + if index >= len(body): + break + escaped = body[index] + index += 1 + if escaped in b"nrtbf": + escaped_chars = { + ord("n"): 10, + ord("r"): 13, + ord("t"): 9, + ord("b"): 8, + ord("f"): 12, + } + out.append(escaped_chars[escaped]) + elif escaped in b"\r\n": + if escaped == 13 and index < len(body) and body[index] == 10: + index += 1 + elif 48 <= escaped <= 55: + octal = bytes([escaped]) + for _ in range(2): + if index < len(body) and 48 <= body[index] <= 55: + octal += bytes([body[index]]) + index += 1 + out.append(int(octal, 8)) + else: + out.append(escaped) + return _decode_pdf_text_bytes(bytes(out)) + + +def _decode_pdf_hex(token: bytes) -> str: + cleaned = re.sub(rb"\s+", b"", token) + if len(cleaned) % 2: + cleaned += b"0" + try: + return _decode_pdf_text_bytes(bytes.fromhex(cleaned.decode("ascii"))) + except ValueError: + return "" + + +def _decode_pdf_text_bytes(data: bytes) -> str: + if data.startswith(b"\xfe\xff"): + try: + return data[2:].decode("utf-16-be") + except UnicodeDecodeError: + return "" + text, _encoding = _decode_text(data) + return text + + +def _warning(code: str, message: str, *, level: str = "info") -> dict[str, object]: + return {"level": level, "code": code, "message": message} + + +class _HtmlTableParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.tables: list[list[list[str]]] = [] + self._table_stack: list[list[list[str]]] = [] + self._row: list[str] | None = None + self._cell: list[str] | None = None + + def handle_starttag(self, tag: str, _attrs: list[tuple[str, str | None]]) -> None: + if tag == "table": + self._table_stack.append([]) + elif tag == "tr" and self._table_stack: + self._row = [] + elif tag in {"td", "th"} and self._row is not None: + self._cell = [] + + def handle_data(self, data: str) -> None: + if self._cell is not None: + self._cell.append(data) + + def handle_endtag(self, tag: str) -> None: + if tag in {"td", "th"} and self._cell is not None and self._row is not None: + self._row.append(html_lib.unescape(" ".join(self._cell).strip())) + self._cell = None + elif tag == "tr" and self._row is not None and self._table_stack: + self._table_stack[-1].append(self._row) + self._row = None + elif tag == "table" and self._table_stack: + self.tables.append(self._table_stack.pop()) + + +class _HtmlTextParser(HTMLParser): + def __init__(self) -> None: + super().__init__(convert_charrefs=True) + self.lines: list[str] = [] + self._parts: list[str] = [] + + def handle_data(self, data: str) -> None: + text = data.strip() + if text: + self._parts.append(text) + + def handle_endtag(self, tag: str) -> None: + if tag in {"br", "p", "div", "li", "tr", "td", "th", "h1", "h2", "h3"}: + self._flush() + + def close(self) -> None: + self._flush() + super().close() + + def _flush(self) -> None: + if self._parts: + self.lines.append(" ".join(self._parts)) + self._parts = [] diff --git a/src/investment_assistant/investment/jpx_excel.py b/src/investment_assistant/investment/jpx_excel.py new file mode 100644 index 0000000..045b7e2 --- /dev/null +++ b/src/investment_assistant/investment/jpx_excel.py @@ -0,0 +1,112 @@ +"""Optional local conversion for JPX legacy Excel files. + +JPX publishes the listed-issues table as a legacy ``.xls`` file. The core app +stays dependency-light, so this helper uses installed desktop Excel only when +the user runs the single-user local PWA on Windows. It is a convenience path, +not a server requirement. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + + +class JpxExcelConversionError(RuntimeError): + """Raised when a local Excel conversion could not be completed.""" + + +def convert_legacy_xls_to_csv_with_excel( + xls_path: str | Path, + csv_path: str | Path, + *, + timeout_seconds: int = 120, +) -> str: + """Convert a JPX legacy ``.xls`` file to UTF-8 CSV via local Excel COM.""" + + if os.name != "nt": + raise JpxExcelConversionError("Excel conversion is only available on Windows.") + + powershell = shutil.which("powershell.exe") + if powershell is None: + raise JpxExcelConversionError("powershell.exe was not found.") + + source = Path(xls_path).resolve() + target = Path(csv_path).resolve() + if not source.is_file(): + raise JpxExcelConversionError(f"Excel source file was not found: {source}") + target.parent.mkdir(parents=True, exist_ok=True) + + script = _conversion_script() + script_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + "w", + suffix=".ps1", + delete=False, + encoding="utf-8", + ) as handle: + handle.write(script) + script_path = Path(handle.name) + + completed = subprocess.run( + [ + powershell, + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(script_path), + "-Source", + str(source), + "-Output", + str(target), + ], + check=False, + capture_output=True, + text=True, + timeout=timeout_seconds, + ) + except subprocess.TimeoutExpired as exc: + raise JpxExcelConversionError("Excel conversion timed out.") from exc + finally: + if script_path is not None: + script_path.unlink(missing_ok=True) + + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout or "").strip() + raise JpxExcelConversionError( + "Excel conversion failed." + (f" Detail: {detail}" if detail else "") + ) + if not target.is_file(): + raise JpxExcelConversionError("Converted CSV was not created.") + return str(target) + + +def _conversion_script() -> str: + return r""" +param( + [Parameter(Mandatory=$true)][string]$Source, + [Parameter(Mandatory=$true)][string]$Output +) +$ErrorActionPreference = "Stop" +$excel = New-Object -ComObject Excel.Application +$excel.Visible = $false +$excel.DisplayAlerts = $false +try { + $workbook = $excel.Workbooks.Open($Source) + try { + $xlCSVUTF8 = 62 + $workbook.SaveAs($Output, $xlCSVUTF8) + } finally { + $workbook.Close($false) + [System.Runtime.InteropServices.Marshal]::ReleaseComObject($workbook) | Out-Null + } +} finally { + $excel.Quit() + [System.Runtime.InteropServices.Marshal]::ReleaseComObject($excel) | Out-Null +} +""" diff --git a/src/investment_assistant/investment/loader.py b/src/investment_assistant/investment/loader.py index 71ddf51..a3cc6a6 100644 --- a/src/investment_assistant/investment/loader.py +++ b/src/investment_assistant/investment/loader.py @@ -21,6 +21,95 @@ ) _ROW_ERROR_RE = re.compile(r"^Row (?P\d+): (?P.+)$") +_HOLDING_MINIMUM_COLUMNS: tuple[str, ...] = ( + "asset_type", + "ticker_or_fund_code", + "name", + "quantity", + "avg_cost", +) +_CSV_HEADER_ALIASES: dict[str, str] = { + "assettype": "asset_type", + "asset_type": "asset_type", + "資産種別": "asset_type", + "商品種別": "asset_type", + "種別": "asset_type", + "ticker": "ticker_or_fund_code", + "code": "ticker_or_fund_code", + "securitycode": "ticker_or_fund_code", + "ticker_or_fund_code": "ticker_or_fund_code", + "銘柄コード": "ticker_or_fund_code", + "証券コード": "ticker_or_fund_code", + "コード": "ticker_or_fund_code", + "ファンドコード": "ticker_or_fund_code", + "fund_code": "fund_code", + "name": "name", + "銘柄名": "name", + "名称": "name", + "商品名": "name", + "ファンド名": "name", + "quantity": "quantity", + "qty": "quantity", + "shares": "quantity", + "units": "quantity", + "数量": "quantity", + "保有数量": "quantity", + "株数": "quantity", + "口数": "quantity", + "avgcost": "avg_cost", + "avg_cost": "avg_cost", + "averagecost": "avg_cost", + "取得単価": "avg_cost", + "平均取得単価": "avg_cost", + "平均単価": "avg_cost", + "買付単価": "avg_cost", + "currentprice": "current_price", + "current_price": "current_price", + "price": "current_price", + "現在価格": "current_price", + "現在値": "current_price", + "時価": "current_price", + "評価単価": "current_price", + "accounttype": "account_type", + "account_type": "account_type", + "口座": "account_type", + "口座区分": "account_type", + "預り区分": "account_type", + "taxwrapper": "tax_wrapper", + "tax_wrapper": "tax_wrapper", + "税区分": "tax_wrapper", + "NISA区分": "tax_wrapper", + "nisa区分": "tax_wrapper", + "source": "source", + "入力元": "source", + "データ元": "source", + "annualincome": "annual_income", + "annual_income": "annual_income", + "年間配当": "annual_income", + "年間分配金": "annual_income", + "distributionperunit": "distribution_per_unit", + "distribution_per_unit": "distribution_per_unit", + "1口分配金": "distribution_per_unit", + "一口分配金": "distribution_per_unit", + "dataprovider": "data_provider", + "data_provider": "data_provider", + "provider": "data_provider", + "priceasof": "price_as_of", + "price_as_of": "price_as_of", + "価格日": "price_as_of", + "基準日": "price_as_of", + "expense_ratio": "expense_ratio", + "信託報酬": "expense_ratio", + "asset_class": "asset_class", + "資産クラス": "asset_class", + "distribution_policy": "distribution_policy", + "分配方針": "distribution_policy", + "nisa_eligible": "nisa_eligible", + "NISA対象": "nisa_eligible", + "provider_id": "provider_id", + "diversification_score": "diversification_score", + "分散度": "diversification_score", +} def holdings_from_payload(payload: Mapping[str, object]) -> list[InvestmentHolding]: @@ -51,7 +140,7 @@ def validate_holdings_payload(payload: Mapping[str, object]) -> dict[str, object base = _validation_base( kind="holdings", columns=HOLDING_TEMPLATE_COLUMNS, - required_columns=HOLDING_COLUMNS, + required_columns=_HOLDING_MINIMUM_COLUMNS, optional_columns=HOLDING_OPTIONAL_COLUMNS, recommended_columns=HOLDING_RECOMMENDED_COLUMNS, ) @@ -290,7 +379,7 @@ def load_holdings_csv(path: str | Path) -> list[InvestmentHolding]: def load_holdings_csv_text(text: str) -> list[InvestmentHolding]: - rows = _read_rows(text, required=HOLDING_COLUMNS) + rows = _read_rows(text, required=_HOLDING_MINIMUM_COLUMNS) holdings = [_holding_from_mapping(row, row=index) for index, row in enumerate(rows, start=2)] if not holdings: raise ValueError("Holding CSV must contain at least one row.") @@ -310,12 +399,21 @@ def load_funds_csv_text(text: str) -> list[FundProfile]: def _read_rows(text: str, *, required: tuple[str, ...]) -> list[dict[str, str]]: - reader = csv.DictReader(io.StringIO(text.strip())) - fieldnames = set(reader.fieldnames or []) + reader = _dict_reader(text) + field_map = _csv_field_map(reader.fieldnames or []) + fieldnames = set(field_map.values()) missing = [column for column in required if column not in fieldnames] if missing: raise ValueError(f"Missing required CSV columns: {', '.join(missing)}") - return [dict(row) for row in reader] + rows: list[dict[str, str]] = [] + for row in reader: + normalized: dict[str, str] = {} + for key, value in row.items(): + mapped = field_map.get(str(key or "")) + if mapped and mapped not in normalized: + normalized[mapped] = str(value or "").strip() + rows.append(normalized) + return rows def _payload_holding_fieldnames(payload: Mapping[str, object]) -> set[str] | None: @@ -340,8 +438,35 @@ def _payload_holding_fieldnames(payload: Mapping[str, object]) -> set[str] | Non def _csv_fieldnames(text: str) -> set[str]: - reader = csv.DictReader(io.StringIO(text.strip())) - return {str(field) for field in (reader.fieldnames or [])} + reader = _dict_reader(text) + return set(_csv_field_map(reader.fieldnames or []).values()) + + +def _dict_reader(text: str) -> csv.DictReader[str]: + cleaned = text.strip().lstrip("\ufeff") + try: + dialect = csv.Sniffer().sniff(cleaned[:4096], delimiters=",\t;") + except csv.Error: + dialect = csv.excel + return csv.DictReader(io.StringIO(cleaned), dialect=dialect) + + +def _csv_field_map(fieldnames: Sequence[str | None]) -> dict[str, str]: + out: dict[str, str] = {} + for field in fieldnames: + if field is None: + continue + original = str(field).strip().lstrip("\ufeff") + if not original: + continue + out[original] = _canonical_csv_header(original) + return out + + +def _canonical_csv_header(value: str) -> str: + stripped = value.strip().lstrip("\ufeff") + compact = re.sub(r"[\s　_\-・/（）()\[\]]+", "", stripped).lower() + return _CSV_HEADER_ALIASES.get(compact) or _CSV_HEADER_ALIASES.get(stripped) or stripped def _write_csv(columns: tuple[str, ...], rows: Sequence[Mapping[str, object]]) -> str: @@ -491,10 +616,18 @@ def _asset_type(value: str) -> str: "jp_stock": "stock", "japan_stock": "stock", "equity": "stock", + "stock": "stock", + "株式": "stock", + "国内株式": "stock", + "内国株式": "stock", + "日本株": "stock", + "日本株式": "stock", "mutual_fund": "fund", "investment_trust": "fund", + "fund": "fund", "投信": "fund", - "日本株": "stock", + "投資信託": "fund", + "ファンド": "fund", } return aliases.get(normalized, normalized) @@ -524,6 +657,9 @@ def _required_float( def _optional_float(value: object, *, row: int, column: str) -> float | None: text = _text(value, default="") + if not text: + return None + text = _numeric_text(text) if not text: return None try: @@ -538,6 +674,18 @@ def _text(value: object, *, default: str) -> str: return str(value).strip() +def _numeric_text(value: str) -> str: + text = value.strip() + if text in {"-", "ー", "―", "N/A", "n/a", "なし"}: + return "" + text = text.replace(",", "").replace("，", "") + text = text.replace("￥", "").replace("¥", "") + for suffix in ("円", "株", "口", " shares", " share", " units", " unit"): + if text.endswith(suffix): + text = text[: -len(suffix)] + return text.strip() + + def _bool(value: object) -> bool: if isinstance(value, bool): return value diff --git a/src/investment_assistant/investment/operators.py b/src/investment_assistant/investment/operators.py new file mode 100644 index 0000000..b14516c --- /dev/null +++ b/src/investment_assistant/investment/operators.py @@ -0,0 +1,234 @@ +"""Human-readable operator catalog for deterministic investment workflows.""" + +from __future__ import annotations + +from datetime import UTC, datetime + +from investment_assistant.investment.candidates import FUND_SCORE_WEIGHTS +from investment_assistant.rag.search import ( + DEFAULT_HYBRID_ALPHA, + DEFAULT_MAX_PER_SOURCE, + DEFAULT_RRF_K, +) +from investment_assistant.scoring.stock import StockScoreWeights + + +def operator_catalog() -> dict[str, object]: + """Return formulas and boundaries used by the investment-only MVP. + + This catalog is intentionally static and deterministic. It gives the UI a + single place to explain what is calculated by rules, what is retrieved as + evidence, and what is never automated. + """ + + stock_weights = StockScoreWeights().normalized() + return { + "generated_at": datetime.now(UTC).isoformat(), + "version": "investment_operator_catalog_v1", + "auto_trading": False, + "call_real_api": False, + "non_advisory_boundary": ( + "候補抽出、スコア、RAG検索は比較材料の提示に限定します。" + "売買推奨、断定的な投資判断、自動注文は行いません。" + ), + "groups": [ + { + "key": "portfolio_analysis", + "label": "保有分析", + "purpose": ( + "手入力またはCSVの保有データから、評価額、損益、集中度、" + "NISA利用額を再現可能に集計する。" + ), + "operators": [ + { + "key": "market_value", + "label": "評価額", + "formula": "quantity * (current_price or avg_cost)", + "inputs": ["quantity", "current_price", "avg_cost"], + "output": "market_value", + }, + { + "key": "unrealized_pnl", + "label": "評価損益", + "formula": "market_value - quantity * avg_cost", + "inputs": ["quantity", "avg_cost", "current_price"], + "output": "unrealized_pnl", + }, + { + "key": "position_share", + "label": "集中度", + "formula": "holding.market_value / portfolio.market_value", + "inputs": ["holding.market_value", "portfolio.market_value"], + "output": "share_pct", + }, + { + "key": "nisa_used_cost_basis", + "label": "NISA利用額", + "formula": "sum(quantity * avg_cost where tax_wrapper starts with nisa)", + "inputs": ["quantity", "avg_cost", "tax_wrapper"], + "output": "nisa.used_cost_basis", + }, + ], + }, + { + "key": "stock_scoring", + "label": "日本株スコア", + "purpose": "EDINET由来の財務CSVを、透明な重み付きルールで比較材料に変換する。", + "model_version": "stock_score_balanced_v1", + "formula": "sum(normalized_component * normalized_weight)", + "weights": [ + { + "key": "dividend_level", + "label": "配当水準", + "weight": round(stock_weights.dividend_level, 4), + }, + { + "key": "dividend_trend", + "label": "配当トレンド", + "weight": round(stock_weights.dividend_trend, 4), + }, + { + "key": "dividend_safety", + "label": "減配耐性", + "weight": round(stock_weights.dividend_safety, 4), + }, + { + "key": "equity_ratio", + "label": "自己資本比率", + "weight": round(stock_weights.equity_ratio, 4), + }, + { + "key": "operating_cf", + "label": "営業CFトレンド", + "weight": round(stock_weights.operating_cf, 4), + }, + ], + "operators": [ + { + "key": "exclude_dividend_cut", + "label": "減配除外", + "formula": "cut_count == 0 when enabled", + "inputs": ["dividend_series"], + "output": "filter_pass", + }, + { + "key": "min_equity_ratio", + "label": "自己資本比率しきい値", + "formula": "latest_equity_ratio >= threshold", + "inputs": ["latest_equity_ratio", "threshold"], + "output": "filter_pass", + }, + ], + }, + { + "key": "fund_scoring", + "label": "投信プロファイル", + "purpose": ( + "ユーザー入力または契約済みproviderの投信プロファイルを" + "比較材料としてスコア化する。" + ), + "model_version": "fund_weighted_v1", + "formula": "sum(weight * normalized_score)", + "weights": [ + { + "key": key, + "label": _fund_weight_label(key), + "weight": round(value, 4), + } + for key, value in FUND_SCORE_WEIGHTS.items() + ], + "operators": [ + { + "key": "expense_ratio_score", + "label": "信託報酬スコア", + "formula": "max(0, 1 - expense_ratio_percent / 1.0)", + "inputs": ["expense_ratio"], + "output": "normalized_score", + }, + { + "key": "nisa_eligible_score", + "label": "NISA対象", + "formula": "1 if nisa_eligible else 0", + "inputs": ["nisa_eligible"], + "output": "normalized_score", + }, + { + "key": "diversification_score", + "label": "分散度", + "formula": "user diversification_score or conservative asset_class hint", + "inputs": ["diversification_score", "asset_class"], + "output": "normalized_score", + }, + ], + }, + { + "key": "rag_search", + "label": "RAG検索", + "purpose": "ローカル文書から根拠候補を探し、LLMに渡す前の出典と順位を可視化する。", + "operators": [ + { + "key": "query_decomposition", + "label": "クエリ分解", + "formula": "original query + separator phrases + useful tokens", + "inputs": ["query"], + "output": "query_variants", + }, + { + "key": "hybrid_blend", + "label": "ハイブリッド検索", + "formula": ( + f"{DEFAULT_HYBRID_ALPHA} * semantic_score + " + f"{1 - DEFAULT_HYBRID_ALPHA} * lexical_score" + ), + "inputs": ["BM25/keyword score", "embedding cosine score"], + "output": "blended_score", + }, + { + "key": "reciprocal_rank_fusion", + "label": "RRF順位統合", + "formula": f"sum(1 / ({DEFAULT_RRF_K} + rank))", + "inputs": ["ranked results per query"], + "output": "fused_score", + }, + { + "key": "source_diversity", + "label": "出典分散", + "formula": f"max_per_source <= {DEFAULT_MAX_PER_SOURCE}", + "inputs": ["source", "near-duplicate fingerprint"], + "output": "selected_context", + }, + ], + }, + { + "key": "report_evidence", + "label": "レポート根拠", + "purpose": "重要KPIに計算式、出典、最終更新、免責を結びつけて公開前検算を行う。", + "operators": [ + { + "key": "claim_evidence_check", + "label": "claim-evidence検査", + "formula": "all important KPI claim_keys must have evidence rows", + "inputs": ["report.kpis", "report.evidence"], + "output": "audit_status", + }, + { + "key": "disclaimer_check", + "label": "免責検査", + "formula": "report.disclaimer exists and auto_trading is false", + "inputs": ["report"], + "output": "audit_status", + }, + ], + }, + ], + } + + +def _fund_weight_label(key: str) -> str: + labels = { + "expense_ratio": "低コスト性", + "nisa_eligible": "NISA対象", + "diversification": "分散度", + "distribution_policy": "分配方針", + } + return labels.get(key, key) diff --git a/src/investment_assistant/investment/provider_policy.py b/src/investment_assistant/investment/provider_policy.py index 6bf6bd8..1cad678 100644 --- a/src/investment_assistant/investment/provider_policy.py +++ b/src/investment_assistant/investment/provider_policy.py @@ -9,13 +9,21 @@ RUNTIME_MODE_ENV = "INVESTMENT_ASSISTANT_RUNTIME_MODE" CONTRACTED_PROVIDERS_ENV = "INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS" -_ALWAYS_ALLOWED = {"edinet", "manual", "user_csv", "user_input", "contracted"} +_ALWAYS_ALLOWED = { + "edinet", + "manual", + "user_csv", + "user_input", + "yahoo_finance_manual", + "contracted", +} _DEFAULT_LEDGER_PROVIDERS = ( "edinet", "user_csv", "manual", "stooq_public_csv", "yfinance", + "yahoo_finance_manual", "jquants", "alpha_vantage", "contracted", @@ -46,6 +54,11 @@ "primary_use": "Research and prototype market data", "recommended_use": "development_only", }, + "yahoo_finance_manual": { + "category": "user_supplied_market_data", + "primary_use": "User-entered Yahoo Finance quote CSV for personal local use", + "recommended_use": "manual_single_user_only", + }, "jquants": { "category": "market_data_api", "primary_use": "Japanese market, financial, and dividend data", diff --git a/src/investment_assistant/investment/reporting.py b/src/investment_assistant/investment/reporting.py index 225fc63..a05bfdd 100644 --- a/src/investment_assistant/investment/reporting.py +++ b/src/investment_assistant/investment/reporting.py @@ -6,6 +6,7 @@ from datetime import UTC, datetime from pathlib import Path +from investment_assistant.financials.current_yield import DEFAULT_CURRENT_YIELDS_CSV from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV from investment_assistant.investment.analysis import analyze_portfolio from investment_assistant.investment.models import DISCLAIMER, InvestmentHolding @@ -18,6 +19,7 @@ def build_investment_monthly_report( candidates: Sequence[dict[str, object]] = (), target_result: Mapping[str, object] | None = None, financials_csv: str | Path = DEFAULT_FINANCIALS_CSV, + current_yields_csv: str | Path | None = DEFAULT_CURRENT_YIELDS_CSV, runtime_mode: str = "development", ) -> dict[str, object]: """Build a non-advisory monthly report from computed facts.""" @@ -25,6 +27,7 @@ def build_investment_monthly_report( analysis = analyze_portfolio( holdings, financials_csv=financials_csv, + current_yields_csv=current_yields_csv, runtime_mode=runtime_mode, ) summary = analysis["summary"] @@ -55,6 +58,7 @@ def build_investment_monthly_report( "note": "条件一致の比較候補であり、推奨ではありません。", } ) + evidence.extend(_candidate_evidence_rows(item, generated_at)) target = _mapping(target_result.get("target")) if target_result is not None else None target_summary = _mapping(target_result.get("summary")) if target_result is not None else None target_concentration = ( @@ -329,7 +333,9 @@ def _formula(key: str) -> str: formulas = { "market_value": "数量 × 現在価格（未入力時は取得単価）", "unrealized_pnl": "評価額 - 取得額", - "annual_income_estimate": "ユーザー入力分配金、またはEDINET最新1株配当 × 数量", + "annual_income_estimate": ( + "ユーザー入力分配金、現在配当/予想配当CSV、またはEDINET最新1株配当 × 数量" + ), "nisa_remaining": "18,000,000円 - NISA口座の取得額合計", "concentration_top_weight": "最大保有銘柄の評価額 ÷ ポートフォリオ評価額", "concentration_hhi": "各保有比率の2乗和", @@ -409,6 +415,56 @@ def _evidence_rows(value: object) -> list[dict[str, object]]: return [item for item in value if isinstance(item, dict)] +def _candidate_evidence_rows( + candidate: Mapping[str, object], + generated_at: str, +) -> list[dict[str, object]]: + rows: list[dict[str, object]] = [] + raw_rows = candidate.get("evidence") + if isinstance(raw_rows, list): + for index, row in enumerate(raw_rows): + if not isinstance(row, Mapping): + continue + source_type = row.get("source_type") + if source_type != "edinet_financials": + continue + code = str(candidate.get("code") or f"row{index}") + rows.append( + { + "claim_key": str( + row.get("claim_key") or f"candidate.{code}.edinet_financials" + ), + "source_type": "edinet_financials", + "source_ref": row.get("source_ref"), + "metric_key": row.get("metric_key") or "dividend/equity/operating_cf", + "formula": row.get("formula") + or "EDINET-derived financials CSV used in candidate screen", + "last_updated": row.get("last_updated") or generated_at, + "note": "候補抽出で参照したEDINET由来財務データです。推奨ではありません。", + } + ) + summary = candidate.get("edinet_summary") + if isinstance(summary, Mapping): + code = str(candidate.get("code") or summary.get("ticker") or "") + rows.append( + { + "claim_key": f"candidate.{code}.edinet_summary", + "source_type": "edinet_financials", + "source_ref": summary.get("source_ref"), + "metric_key": "latest_fiscal_year/equity_ratio/dividend", + "formula": "latest EDINET financial row summarized for report evidence", + "last_updated": summary.get("last_updated") or generated_at, + "note": ( + f"FY{summary.get('latest_fiscal_year')}: " + f"自己資本比率 {summary.get('latest_equity_ratio')}%, " + f"1株配当 {summary.get('latest_dividend_per_share')}, " + f"営業CF {summary.get('operating_cf_trend_label')}" + ), + } + ) + return rows + + def _claim_keys( evidence: Sequence[dict[str, object]], *, diff --git a/src/investment_assistant/investment/universe.py b/src/investment_assistant/investment/universe.py new file mode 100644 index 0000000..bfc48c3 --- /dev/null +++ b/src/investment_assistant/investment/universe.py @@ -0,0 +1,398 @@ +"""Market universe helpers for non-advisory stock selection. + +This module deliberately separates *security selection metadata* from market +prices or index weights. JPX market segment data and Nikkei 225 membership are +used only to help the user narrow a comparison universe; they are not trading +signals. +""" + +from __future__ import annotations + +import csv +import io +import re +from dataclasses import asdict, dataclass +from pathlib import Path + +from investment_assistant.edinet.registry import build_edinet_targets_from_registry +from investment_assistant.financials import compare_financials, load_financials +from investment_assistant.financials.dividend_quality import normalize_dividend_points +from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV + +DEFAULT_NIKKEI225_REGISTRY = "examples/source_registry_nikkei225_edinet.yaml" +DEFAULT_JPX_LISTED_ISSUES_PATH = "local_docs/jpx/listed_issues.csv" +JPX_LISTED_ISSUES_PAGE_URL = "https://www.jpx.co.jp/markets/statistics-equities/misc/01.html" +JPX_LISTED_ISSUES_FILE_URL = ( + "https://www.jpx.co.jp/markets/statistics-equities/misc/" + "tvdivq0000001vg2-att/data_j.xls" +) +JPX_DATA_PORTAL_URL = "https://clientportal.jpx.co.jp/" +NIKKEI225_COMPONENTS_URL = "https://indexes.nikkei.co.jp/en/nkave/index/component?idx=nk225" + +_CODE_RE = re.compile(r"^[0-9A-Za-z]{4}") + +_CODE_COLUMNS = ("コード", "code", "local code", "銘柄コード", "security code") +_NAME_COLUMNS = ("銘柄名", "name", "issue name", "company name", "銘柄") +_MARKET_COLUMNS = ("市場・商品区分", "market segment", "market", "市場区分") +_SECTOR_COLUMNS = ("33業種区分", "sector", "33 sector", "業種") +_DATE_COLUMNS = ("日付", "date", "as_of", "基準日") + + +@dataclass(frozen=True) +class ListedIssue: + code: str + name: str + market_segment: str + sector: str = "" + as_of: str = "" + source_ref: str = "" + + @property + def is_prime(self) -> bool: + return is_prime_segment(self.market_segment) + + def to_dict(self) -> dict[str, object]: + payload = asdict(self) + raw_segment = self.market_segment + display_segment = display_market_segment(raw_segment) + payload["market_segment_raw"] = raw_segment + payload["market_segment"] = display_segment + payload["market_segment_label"] = display_segment + payload["is_prime"] = self.is_prime + return payload + + +def source_manifest() -> dict[str, object]: + """Return official source references and licensing cautions.""" + + return { + "jpx_listed_issues": { + "label": "JPX 東証上場銘柄一覧", + "page_url": JPX_LISTED_ISSUES_PAGE_URL, + "file_url": JPX_LISTED_ISSUES_FILE_URL, + "data_portal_url": JPX_DATA_PORTAL_URL, + "usage": ( + "市場区分による銘柄選択補助のみ。" + "価格・指数ウェイト・再配布用途には使いません。" + ), + }, + "nikkei225_components": { + "label": "Nikkei 225 Components", + "page_url": NIKKEI225_COMPONENTS_URL, + "usage": "日経225構成銘柄フラグの表示のみ。指数データやウェイトの再配布は扱いません。", + }, + "non_advisory_boundary": ( + "市場区分と指数構成フラグは比較対象を絞るための表示です。" + "買付・売却・保有継続を推奨しません。" + ), + "auto_trading": False, + "call_real_api": False, + } + + +def jpx_listed_issue_template() -> dict[str, object]: + csv_text = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,7203,トヨタ自動車,プライム（国内株式）,輸送用機器\n" + "2026-05-31,8306,三菱ＵＦＪフィナンシャル・グループ,プライム（国内株式）,銀行業\n" + "2026-05-31,9999,サンプルスタンダード,スタンダード（国内株式）,サービス業\n" + ) + return { + "kind": "jpx_listed_issues", + "csv_text": csv_text, + "required_columns": ["コード", "銘柄名", "市場・商品区分"], + "optional_columns": ["日付", "33業種区分"], + "sources": source_manifest(), + "auto_trading": False, + "call_real_api": False, + } + + +def parse_jpx_listed_issues_text(text: str, *, source_ref: str = "") -> list[ListedIssue]: + """Parse JPX listed issue data exported as CSV/TSV. + + The official JPX monthly file is distributed as legacy ``.xls``. To keep the + app dependency-free, this parser accepts the same table after it has been + exported or copied as CSV/TSV. + """ + + normalized = text.strip() + if not normalized: + raise ValueError("JPX listed issue data is empty.") + delimiter = _detect_delimiter(normalized) + reader = csv.DictReader(io.StringIO(normalized), delimiter=delimiter) + headers = [str(name or "").strip() for name in (reader.fieldnames or [])] + code_col = _find_header(headers, _CODE_COLUMNS) + name_col = _find_header(headers, _NAME_COLUMNS) + market_col = _find_header(headers, _MARKET_COLUMNS) + sector_col = _find_header(headers, _SECTOR_COLUMNS) + date_col = _find_header(headers, _DATE_COLUMNS) + missing = [ + label + for label, column in ( + ("コード", code_col), + ("銘柄名", name_col), + ("市場・商品区分", market_col), + ) + if column is None + ] + if missing: + raise ValueError(f"JPX listed issue data is missing columns: {', '.join(missing)}") + + issues: list[ListedIssue] = [] + for row in reader: + code = normalize_security_code(row.get(code_col or "")) + if not code: + continue + issues.append( + ListedIssue( + code=code, + name=str(row.get(name_col or "") or "").strip(), + market_segment=str(row.get(market_col or "") or "").strip(), + sector=str(row.get(sector_col or "") or "").strip() if sector_col else "", + as_of=str(row.get(date_col or "") or "").strip() if date_col else "", + source_ref=source_ref, + ) + ) + if not issues: + raise ValueError("JPX listed issue data has no usable rows.") + return issues + + +def load_jpx_listed_issues(path: str | Path = DEFAULT_JPX_LISTED_ISSUES_PATH) -> list[ListedIssue]: + csv_path = Path(path) + if not csv_path.is_file(): + return [] + return parse_jpx_listed_issues_text( + csv_path.read_text(encoding="utf-8"), + source_ref=str(csv_path), + ) + + +def write_jpx_listed_issues(issues: list[ListedIssue], path: str | Path) -> str: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter( + handle, + fieldnames=["日付", "コード", "銘柄名", "市場・商品区分", "33業種区分"], + ) + writer.writeheader() + for issue in issues: + writer.writerow( + { + "日付": issue.as_of, + "コード": issue.code, + "銘柄名": issue.name, + "市場・商品区分": issue.market_segment, + "33業種区分": issue.sector, + } + ) + return str(target) + + +def build_market_universe( + *, + financials_csv: str | Path = DEFAULT_FINANCIALS_CSV, + jpx_listed_path: str | Path = DEFAULT_JPX_LISTED_ISSUES_PATH, + nikkei225_registry: str | Path = DEFAULT_NIKKEI225_REGISTRY, + query: str = "", + scope: str = "prime", + limit: int = 50, +) -> dict[str, object]: + listed = {issue.code: issue for issue in load_jpx_listed_issues(jpx_listed_path)} + nikkei = nikkei225_index(nikkei225_registry) + financials = _financials_index(financials_csv) + + codes = set(financials) | set(listed) | set(nikkei) + rows = [_universe_row(code, financials, listed, nikkei) for code in codes] + scoped = [_row for _row in rows if _in_scope(_row, scope)] + searched = [_row for _row in scoped if _matches_query(_row, query)] + searched.sort(key=_universe_sort_key) + clipped = searched[: max(limit, 1)] + prime_available = bool(listed) + return { + "available": True, + "scope": scope, + "query": query, + "count": len(clipped), + "total_count": len(searched), + "securities": clipped, + "universe": clipped, + "jpx_listed_available": prime_available, + "jpx_listed_count": len(listed), + "nikkei225_count": len(nikkei), + "financials_available": bool(financials), + "financials_count": len(financials), + "sources": source_manifest(), + "hint": _scope_hint(scope, prime_available), + "auto_trading": False, + "call_real_api": False, + } + + +def nikkei225_index( + registry_path: str | Path = DEFAULT_NIKKEI225_REGISTRY, +) -> dict[str, dict[str, object]]: + try: + targets = build_edinet_targets_from_registry(registry_path) + except (OSError, ValueError): + return {} + return { + target.ticker: { + "ticker": target.ticker, + "name": target.company or target.name, + "source_ref": str(registry_path), + } + for target in targets + } + + +def normalize_security_code(value: object) -> str: + text = str(value or "").strip() + if text.endswith(".0"): + text = text[:-2] + match = _CODE_RE.match(text) + return match.group(0).upper() if match else "" + + +def is_prime_segment(value: object) -> bool: + text = str(value or "").strip().lower() + return "プライム" in text or "prime" in text + + +def display_market_segment(value: object) -> str: + """Return a user-facing market segment label without mutating source data.""" + + text = str(value or "").strip() + if not text: + return "未取込" + return text.replace("内国株式", "国内株式") + + +def _detect_delimiter(text: str) -> str: + first = text.splitlines()[0] if text.splitlines() else "" + if "\t" in first: + return "\t" + try: + return csv.Sniffer().sniff(text[:2048], delimiters=",\t;").delimiter + except csv.Error: + return "," + + +def _find_header(headers: list[str], aliases: tuple[str, ...]) -> str | None: + lowered = {header.lower().strip(): header for header in headers} + for alias in aliases: + hit = lowered.get(alias.lower()) + if hit is not None: + return hit + for header in headers: + normalized = header.lower().strip() + if any(alias.lower() in normalized for alias in aliases): + return header + return None + + +def _financials_index(path: str | Path) -> dict[str, dict[str, object]]: + csv_path = Path(path) + if not csv_path.is_file(): + return {} + try: + points, _ = normalize_dividend_points(load_financials(csv_path)) + comparison = compare_financials(points) + except (OSError, ValueError): + return {} + companies = comparison.get("companies") + if not isinstance(companies, list): + return {} + out: dict[str, dict[str, object]] = {} + for company in companies: + if not isinstance(company, dict): + continue + code = normalize_security_code(company.get("ticker")) + if code: + out[code] = company + return out + + +def _universe_row( + code: str, + financials: dict[str, dict[str, object]], + listed: dict[str, ListedIssue], + nikkei: dict[str, dict[str, object]], +) -> dict[str, object]: + financial = financials.get(code, {}) + listed_issue = listed.get(code) + nikkei_issue = nikkei.get(code, {}) + name = ( + str(financial.get("name") or "").strip() + or (listed_issue.name if listed_issue else "") + or str(nikkei_issue.get("name") or "").strip() + ) + market_segment = listed_issue.market_segment if listed_issue else "" + display_segment = display_market_segment(market_segment) + return { + "ticker": code, + "code": code, + "name": name, + "market_segment": display_segment, + "market_segment_raw": market_segment, + "market_segment_label": display_segment, + "sector": listed_issue.sector if listed_issue else "", + "is_prime": listed_issue.is_prime if listed_issue else False, + "is_nikkei225": code in nikkei, + "has_financials": code in financials, + "latest_fiscal_year": financial.get("latest_fiscal_year"), + "latest_equity_ratio": financial.get("latest_equity_ratio"), + "latest_dividend_per_share": financial.get("latest_dividend_per_share"), + "dividend_cut_years": financial.get("dividend_cut_years"), + "operating_cf_trend": financial.get("operating_cf_trend"), + "source_ref": financial.get("source_ref") or "", + "jpx_source_ref": listed_issue.source_ref if listed_issue else "", + "nikkei225_source_ref": nikkei_issue.get("source_ref") or "", + } + + +def _in_scope(row: dict[str, object], scope: str) -> bool: + normalized = scope.strip().lower() + if normalized in {"prime", "tse_prime", "tosho_prime"}: + return bool(row.get("is_prime")) + if normalized in {"nikkei225", "nikkei_225", "n225"}: + return bool(row.get("is_nikkei225")) + if normalized in {"domestic", "domestic_stock", "domestic_stocks", "japan_stocks"}: + segment = " ".join( + str(row.get(key) or "") + for key in ("market_segment", "market_segment_raw", "market_segment_label") + ).lower() + return "国内株式" in segment or "内国株式" in segment or "domestic stock" in segment + if normalized in {"financials", "edinet", "financials_available"}: + return bool(row.get("has_financials")) + return True + + +def _matches_query(row: dict[str, object], query: str) -> bool: + needle = query.strip().lower() + if not needle: + return True + haystack = " ".join( + str(row.get(key) or "") + for key in ("ticker", "name", "market_segment", "market_segment_raw", "sector") + ).lower() + return needle in haystack + + +def _universe_sort_key(row: dict[str, object]) -> tuple[int, int, str]: + return ( + 0 if row.get("is_prime") else 1, + 0 if row.get("is_nikkei225") else 1, + str(row.get("ticker") or ""), + ) + + +def _scope_hint(scope: str, jpx_available: bool) -> str: + if scope.strip().lower() in {"prime", "tse_prime", "tosho_prime"} and not jpx_available: + return ( + "東証プライムを選ぶにはJPX上場銘柄一覧データの取込が必要です。" + "Dataタブで公式JPXファイルを取得し、CSV/TSVとして保存してください。" + ) + return "市場区分と日経225フラグは比較対象の絞り込み用です。投資助言ではありません。" diff --git a/src/investment_assistant/jquants/__init__.py b/src/investment_assistant/jquants/__init__.py new file mode 100644 index 0000000..599833a --- /dev/null +++ b/src/investment_assistant/jquants/__init__.py @@ -0,0 +1,21 @@ +"""J-Quants API integration helpers.""" + +from investment_assistant.jquants.client import ( + API_KEY_ENV_VAR, + BASE_URL_ENV_VAR, + DEFAULT_BASE_URL, + REFRESH_TOKEN_ENV_VAR, + JQuantsApiError, + JQuantsClient, + normalize_equity_code, +) + +__all__ = [ + "API_KEY_ENV_VAR", + "BASE_URL_ENV_VAR", + "DEFAULT_BASE_URL", + "REFRESH_TOKEN_ENV_VAR", + "JQuantsApiError", + "JQuantsClient", + "normalize_equity_code", +] diff --git a/src/investment_assistant/jquants/client.py b/src/investment_assistant/jquants/client.py new file mode 100644 index 0000000..d3b8d0d --- /dev/null +++ b/src/investment_assistant/jquants/client.py @@ -0,0 +1,513 @@ +"""Minimal J-Quants API v2 client. + +The official v2 docs use an API key issued from the dashboard and pass it in +the ``x-api-key`` header. This client intentionally supports only read-only +market data endpoints used by the local single-user investment tool. +""" + +from __future__ import annotations + +import json +import os +import re +from collections.abc import Callable, Iterable, Mapping +from datetime import UTC, datetime, timedelta +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +JsonDict = dict[str, Any] +FetchJson = Callable[[str, Mapping[str, str], Mapping[str, str]], JsonDict] + +DEFAULT_BASE_URL = "https://api.jquants.com/v2" +BASE_URL_ENV_VAR = "JQUANTS_API_BASE_URL" +API_KEY_ENV_VAR = "JQUANTS_API_KEY" +REFRESH_TOKEN_ENV_VAR = "JQUANTS_REFRESH_TOKEN" +USER_AGENT = "investment-assistant/0.1 (+jquants-local-single-user)" + + +class JQuantsApiError(RuntimeError): + """Raised when J-Quants configuration or API access fails.""" + + +class JQuantsClient: + """Small API-key based J-Quants v2 client.""" + + def __init__( + self, + *, + api_key: str | None = None, + base_url: str | None = None, + fetch_json: FetchJson | None = None, + timeout_seconds: float = 20.0, + ) -> None: + self.api_key = (api_key or _env_api_key()).strip() + self.base_url = (base_url or os.getenv(BASE_URL_ENV_VAR) or DEFAULT_BASE_URL).rstrip("/") + self.fetch_json = fetch_json + self.timeout_seconds = timeout_seconds + + def daily_bars( + self, + code: str, + *, + date: str | None = None, + from_date: str | None = None, + to_date: str | None = None, + pagination_key: str | None = None, + ) -> JsonDict: + """Fetch stock OHLC rows from ``/v2/equities/bars/daily``.""" + + return self._daily_bars_page( + code=normalize_equity_code(code), + date=date, + from_date=from_date, + to_date=to_date, + pagination_key=pagination_key, + ) + + def _daily_bars_page( + self, + *, + code: str | None = None, + date: str | None = None, + from_date: str | None = None, + to_date: str | None = None, + pagination_key: str | None = None, + ) -> JsonDict: + query = { + "code": code, + "date": _compact_date(date), + "from": _compact_date(from_date), + "to": _compact_date(to_date), + "pagination_key": pagination_key, + } + payload = self._request_json("/equities/bars/daily", query) + return { + "rows": _extract_rows(payload), + "pagination_key": payload.get("pagination_key"), + "source_endpoint": "/v2/equities/bars/daily", + "raw": payload, + } + + def _daily_bars_with_subscription_fallback( + self, + code: str | None, + *, + date: str | None, + from_date: str | None, + to_date: str | None, + lookback_days: int, + ) -> JsonDict: + try: + return self._daily_bars_page( + code=code, + date=date, + from_date=from_date, + to_date=to_date, + ) + except JQuantsApiError as exc: + retry_window = None if date else _subscription_retry_window(str(exc), lookback_days) + if retry_window is None: + raise + retry_from, retry_to = retry_window + result = self._daily_bars_page( + code=code, + from_date=retry_from, + to_date=retry_to, + ) + result["subscription_window_used"] = { + "from": retry_from, + "to": retry_to, + } + return result + + def fetch_latest_prices( + self, + tickers: Iterable[str], + *, + date: str | None = None, + lookback_days: int = 14, + ) -> JsonDict: + """Fetch latest available close price for each ticker.""" + + prices: dict[str, float | None] = {} + notes: dict[str, str] = {} + as_of: dict[str, str] = {} + today = datetime.now(UTC).date() + from_date = today - timedelta(days=max(lookback_days, 1)) + for raw in tickers: + ticker = str(raw).strip() + if not ticker or ticker in prices: + continue + errors: list[str] = [] + candidates = candidate_equity_codes(ticker) + try: + result: JsonDict | None = None + for code in candidates: + try: + result = self._daily_bars_with_subscription_fallback( + code, + date=date, + from_date=None if date else from_date.isoformat(), + to_date=None if date else today.isoformat(), + lookback_days=lookback_days, + ) + price, row_date = _latest_close(result["rows"]) + if price is not None: + break + errors.append(f"{code}: no_close_price_returned") + except JQuantsApiError as exc: + errors.append(f"{code}: {exc}") + if result is None: + raise JQuantsApiError("; ".join(errors) or "no J-Quants response") + price, row_date = _latest_close(result["rows"]) + prices[ticker] = price + if row_date: + as_of[ticker] = row_date + if isinstance(result.get("subscription_window_used"), dict): + window = result["subscription_window_used"] + notes[ticker] = ( + f"subscription_window_used:{window.get('from')}~{window.get('to')}" + ) + if price is None: + notes[ticker] = "; ".join(errors) or "no_close_price_returned" + except JQuantsApiError as exc: + prices[ticker] = None + notes[ticker] = str(exc) + return { + "prices": prices, + "notes": notes, + "as_of": as_of, + "source": "https://api.jquants.com/v2/equities/bars/daily", + "provider_id": "jquants", + "auto_trading": False, + "call_real_api": True, + } + + def fetch_daily_bars_bulk( + self, + tickers: Iterable[str], + *, + date: str | None = None, + lookback_days: int = 30, + ) -> JsonDict: + """Fetch OHLCV rows in page batches, then filter to the requested tickers. + + J-Quants' daily bars endpoint can return many issues for a date/range. + Universe updates should therefore avoid one request per ticker whenever + possible; that shape hits provider rate limits quickly. + """ + + from investment_assistant.portfolio.bar_store import ( + daily_bar_from_jquants_row, + summarize_daily_bars, + visible_ticker, + ) + + wanted = {visible_ticker(ticker) for ticker in tickers if str(ticker or "").strip()} + today = datetime.now(UTC).date() + from_date = today - timedelta(days=max(lookback_days, 1)) + source = "https://api.jquants.com/v2/equities/bars/daily" + rows: list[JsonDict] = [] + notes: dict[str, str] = {} + pages_fetched = 0 + pagination_key: str | None = None + subscription_window_used: dict[str, str] | None = None + + try: + while True: + try: + result = self._daily_bars_page( + date=date, + from_date=None if date else from_date.isoformat(), + to_date=None if date else today.isoformat(), + pagination_key=pagination_key, + ) + except JQuantsApiError as exc: + retry_window = None if date else _subscription_retry_window( + str(exc), lookback_days + ) + if retry_window is None: + raise + retry_from, retry_to = retry_window + subscription_window_used = {"from": retry_from, "to": retry_to} + rows = [] + pages_fetched = 0 + pagination_key = None + while True: + result = self._daily_bars_page( + from_date=retry_from, + to_date=retry_to, + pagination_key=pagination_key, + ) + rows.extend(result["rows"]) + pages_fetched += 1 + pagination_key = _pagination_key(result) + if not pagination_key: + break + break + rows.extend(result["rows"]) + pages_fetched += 1 + pagination_key = _pagination_key(result) + if not pagination_key: + break + except JQuantsApiError as exc: + notes["bulk"] = str(exc) + + bar_facts = [ + fact + for row in rows + if (ticker := visible_ticker(row.get("Code") or row.get("code"))) in wanted + if ( + fact := daily_bar_from_jquants_row( + row, + fallback_ticker=ticker, + provider_id="jquants", + source_ref=source, + ) + ) + is not None + ] + matched = {fact.ticker for fact in bar_facts} + missing = sorted(wanted - matched) + if subscription_window_used: + notes["bulk"] = ( + "subscription_window_used:" + f"{subscription_window_used.get('from')}~{subscription_window_used.get('to')}" + ) + if missing and len(wanted) <= 50 and "bulk" not in notes: + notes["missing_tickers"] = ",".join(missing) + return { + "bars": [fact.to_dict() for fact in bar_facts], + "summary": summarize_daily_bars(bar_facts), + "notes": notes, + "tried_codes": {"bulk": ["date_range"]}, + "source": source, + "provider_id": "jquants", + "fetch_mode": "bulk_date_range", + "pages_fetched": pages_fetched, + "rows_returned": len(rows), + "matched_ticker_count": len(matched), + "missing_ticker_count": len(missing), + "auto_trading": False, + "call_real_api": True, + } + + def fetch_daily_bars( + self, + tickers: Iterable[str], + *, + date: str | None = None, + lookback_days: int = 30, + ) -> JsonDict: + """Fetch normalized daily OHLCV rows for each ticker.""" + + from investment_assistant.portfolio.bar_store import ( + daily_bar_from_jquants_row, + summarize_daily_bars, + ) + + bar_facts: list[Any] = [] + notes: dict[str, str] = {} + tried_codes: dict[str, list[str]] = {} + today = datetime.now(UTC).date() + from_date = today - timedelta(days=max(lookback_days, 1)) + source = "https://api.jquants.com/v2/equities/bars/daily" + for raw in tickers: + ticker = str(raw).strip() + if not ticker: + continue + errors: list[str] = [] + candidates = candidate_equity_codes(ticker) + tried_codes[ticker] = list(candidates) + for code in candidates: + try: + result = self._daily_bars_with_subscription_fallback( + code, + date=date, + from_date=None if date else from_date.isoformat(), + to_date=None if date else today.isoformat(), + lookback_days=lookback_days, + ) + except JQuantsApiError as exc: + errors.append(f"{code}: {exc}") + continue + rows = result["rows"] + normalized = [ + fact + for row in rows + if ( + fact := daily_bar_from_jquants_row( + row, + fallback_ticker=ticker, + provider_id="jquants", + source_ref=source, + ) + ) + is not None + ] + if normalized: + bar_facts.extend(normalized) + if isinstance(result.get("subscription_window_used"), dict): + window = result["subscription_window_used"] + notes[ticker] = ( + f"subscription_window_used:{window.get('from')}~{window.get('to')}" + ) + break + errors.append(f"{code}: no_daily_bars_returned") + if errors and not any(str(fact.ticker) == ticker for fact in bar_facts): + notes[ticker] = "; ".join(errors) + return { + "bars": [fact.to_dict() for fact in bar_facts], + "summary": summarize_daily_bars(bar_facts), + "notes": notes, + "tried_codes": tried_codes, + "source": source, + "provider_id": "jquants", + "auto_trading": False, + "call_real_api": True, + } + + def _request_json(self, path: str, params: Mapping[str, str | None]) -> JsonDict: + if not self.api_key: + raise JQuantsApiError( + f"{API_KEY_ENV_VAR} is not configured. Set it from the Data tab or .env." + ) + cleaned = {key: value for key, value in params.items() if value} + headers = { + "Accept": "application/json", + "User-Agent": USER_AGENT, + "x-api-key": self.api_key, + } + if self.fetch_json is not None: + return self.fetch_json(path, cleaned, headers) + url = f"{self.base_url}{path}" + if cleaned: + url = f"{url}?{urlencode(cleaned)}" + request = Request(url, headers=headers, method="GET") + try: + with urlopen(request, timeout=self.timeout_seconds) as response: + raw = response.read() + except HTTPError as exc: + raise JQuantsApiError(_http_error_message(exc, path, cleaned)) from exc + except URLError as exc: + raise JQuantsApiError(f"J-Quants API connection failed: {exc.reason}") from exc + try: + parsed = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise JQuantsApiError("J-Quants API returned invalid JSON") from exc + if not isinstance(parsed, dict): + raise JQuantsApiError("J-Quants API returned unexpected payload") + return parsed + + +def normalize_equity_code(code: str) -> str: + """Normalize a visible 4-digit Japanese ticker to J-Quants' 5-digit code.""" + + raw = str(code or "").strip().upper() + digits = "".join(ch for ch in raw if ch.isdigit()) + if len(digits) == 4: + return f"{digits}0" + return digits or raw + + +def candidate_equity_codes(code: str) -> tuple[str, ...]: + """Return J-Quants code candidates for visible Japanese security codes.""" + + raw = str(code or "").strip().upper() + digits = "".join(ch for ch in raw if ch.isdigit()) + primary = normalize_equity_code(code) + candidates: list[str] = [] + for candidate in (primary, digits): + if candidate and candidate not in candidates: + candidates.append(candidate) + if len(digits) == 5 and digits.endswith("0"): + visible = digits[:4] + if visible not in candidates: + candidates.append(visible) + if not digits and raw and raw not in candidates: + candidates.append(raw) + return tuple(candidates) + + +def _env_api_key() -> str: + return os.getenv(API_KEY_ENV_VAR, "").strip() or os.getenv(REFRESH_TOKEN_ENV_VAR, "").strip() + + +def _compact_date(value: str | None) -> str | None: + if value is None: + return None + text = str(value).strip() + if not text: + return None + return text.replace("-", "") + + +def _subscription_retry_window(message: str, lookback_days: int) -> tuple[str, str] | None: + match = re.search(r"(\d{4}-\d{2}-\d{2})\s*~\s*(\d{4}-\d{2}-\d{2})", message) + if not match: + return None + try: + covered_to = datetime.fromisoformat(match.group(2)).date() + except ValueError: + return None + covered_from = covered_to - timedelta(days=max(lookback_days, 1)) + return covered_from.isoformat(), covered_to.isoformat() + + +def _extract_rows(payload: JsonDict) -> list[JsonDict]: + for key in ("daily_bars", "daily_quotes", "bars", "data", "items", "equities"): + value = payload.get(key) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + return [] + + +def _pagination_key(payload: JsonDict) -> str | None: + value = payload.get("pagination_key") or payload.get("next_page_token") + text = str(value or "").strip() + return text or None + + +def _latest_close(rows: Iterable[JsonDict]) -> tuple[float | None, str | None]: + candidates: list[tuple[str, float]] = [] + for row in rows: + date = str(row.get("Date") or row.get("date") or "") + value = _positive_float( + row.get("Close") + or row.get("close") + or row.get("AdjustmentClose") + or row.get("adjustment_close") + or row.get("AdjC") + or row.get("C") + ) + if value is not None: + candidates.append((date, value)) + if not candidates: + return None, None + row_date, price = sorted(candidates, key=lambda item: item[0])[-1] + return price, row_date or None + + +def _positive_float(value: object) -> float | None: + try: + number = float(value) # type: ignore[arg-type] + except (TypeError, ValueError): + return None + return number if number > 0 else None + + +def _http_error_message(exc: HTTPError, path: str, params: Mapping[str, str]) -> str: + try: + raw = exc.read(500) + except Exception: # noqa: BLE001 - HTTPError body is best-effort diagnostics only + raw = b"" + detail = raw.decode("utf-8", errors="replace").strip() + safe_params = ", ".join(f"{key}={value}" for key, value in params.items() if key != "x-api-key") + message = f"J-Quants API returned HTTP {exc.code} for {path}" + if safe_params: + message = f"{message} ({safe_params})" + if detail: + message = f"{message}: {detail[:300]}" + return message diff --git a/src/investment_assistant/portfolio/bar_store.py b/src/investment_assistant/portfolio/bar_store.py new file mode 100644 index 0000000..d93203a --- /dev/null +++ b/src/investment_assistant/portfolio/bar_store.py @@ -0,0 +1,306 @@ +"""Local OHLCV snapshot store for daily market bars. + +Daily bars are useful for deterministic analysis such as volatility, recent +range, liquidity checks, and price freshness. They are not trading signals by +themselves, and this module deliberately stores only a small normalized local +snapshot for the single-user app. +""" + +from __future__ import annotations + +import csv +import io +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from investment_assistant.portfolio.price_store import MarketPriceFact + +DAILY_BAR_COLUMNS: tuple[str, ...] = ( + "ticker", + "date", + "open", + "high", + "low", + "close", + "volume", + "trading_value", + "adjustment_factor", + "adjusted_open", + "adjusted_high", + "adjusted_low", + "adjusted_close", + "adjusted_volume", + "upper_limit_hit", + "lower_limit_hit", + "provider_id", + "source_ref", +) + +DEFAULT_DAILY_BARS_CSV = Path("local_docs/market/daily_bars.csv") + + +@dataclass(frozen=True) +class DailyBarFact: + ticker: str + date: str + open: float | None = None + high: float | None = None + low: float | None = None + close: float | None = None + volume: float | None = None + trading_value: float | None = None + adjustment_factor: float | None = None + adjusted_open: float | None = None + adjusted_high: float | None = None + adjusted_low: float | None = None + adjusted_close: float | None = None + adjusted_volume: float | None = None + upper_limit_hit: str = "" + lower_limit_hit: str = "" + provider_id: str = "user_csv" + source_ref: str = "" + + def to_dict(self) -> dict[str, object]: + return {key: value for key, value in asdict(self).items() if value not in ("", None)} + + +def load_daily_bars(path: str | Path | None = DEFAULT_DAILY_BARS_CSV) -> list[DailyBarFact]: + if path is None: + return [] + csv_path = Path(path) + if not csv_path.is_file(): + return [] + return parse_daily_bars_csv(csv_path.read_text(encoding="utf-8-sig")) + + +def parse_daily_bars_csv(text: str) -> list[DailyBarFact]: + reader = csv.DictReader(io.StringIO(text)) + facts: list[DailyBarFact] = [] + for row in reader: + fact = daily_bar_fact_from_row(row) + if fact is not None: + facts.append(fact) + return facts + + +def daily_bar_fact_from_row(row: Mapping[str, object]) -> DailyBarFact | None: + ticker = _text(row.get("ticker") or row.get("code") or row.get("security_code")) + date = _text(row.get("date") or row.get("Date")) + if not ticker or not date: + return None + return DailyBarFact( + ticker=ticker, + date=date, + open=_optional_float(row.get("open") or row.get("O")), + high=_optional_float(row.get("high") or row.get("H")), + low=_optional_float(row.get("low") or row.get("L")), + close=_optional_float(row.get("close") or row.get("C")), + volume=_optional_float(row.get("volume") or row.get("Vo")), + trading_value=_optional_float(row.get("trading_value") or row.get("Va")), + adjustment_factor=_optional_float(row.get("adjustment_factor") or row.get("AdjFactor")), + adjusted_open=_optional_float(row.get("adjusted_open") or row.get("AdjO")), + adjusted_high=_optional_float(row.get("adjusted_high") or row.get("AdjH")), + adjusted_low=_optional_float(row.get("adjusted_low") or row.get("AdjL")), + adjusted_close=_optional_float(row.get("adjusted_close") or row.get("AdjC")), + adjusted_volume=_optional_float(row.get("adjusted_volume") or row.get("AdjVo")), + upper_limit_hit=_text(row.get("upper_limit_hit") or row.get("UL")), + lower_limit_hit=_text(row.get("lower_limit_hit") or row.get("LL")), + provider_id=_text(row.get("provider_id") or row.get("provider")) or "user_csv", + source_ref=_text(row.get("source_ref") or row.get("source") or row.get("url")), + ) + + +def daily_bar_from_jquants_row( + row: Mapping[str, Any], + *, + fallback_ticker: str, + provider_id: str = "jquants", + source_ref: str = "https://api.jquants.com/v2/equities/bars/daily", +) -> DailyBarFact | None: + ticker = _text(row.get("Code") or row.get("code") or fallback_ticker) + date = _text(row.get("Date") or row.get("date")) + if not ticker or not date: + return None + return DailyBarFact( + ticker=visible_ticker(ticker), + date=date, + open=_optional_float(row.get("O") or row.get("Open") or row.get("open")), + high=_optional_float(row.get("H") or row.get("High") or row.get("high")), + low=_optional_float(row.get("L") or row.get("Low") or row.get("low")), + close=_optional_float(row.get("C") or row.get("Close") or row.get("close")), + volume=_optional_float(row.get("Vo") or row.get("Volume") or row.get("volume")), + trading_value=_optional_float(row.get("Va") or row.get("TradingValue")), + adjustment_factor=_optional_float(row.get("AdjFactor") or row.get("AdjustmentFactor")), + adjusted_open=_optional_float(row.get("AdjO") or row.get("AdjustmentOpen")), + adjusted_high=_optional_float(row.get("AdjH") or row.get("AdjustmentHigh")), + adjusted_low=_optional_float(row.get("AdjL") or row.get("AdjustmentLow")), + adjusted_close=_optional_float(row.get("AdjC") or row.get("AdjustmentClose")), + adjusted_volume=_optional_float(row.get("AdjVo") or row.get("AdjustmentVolume")), + upper_limit_hit=_text(row.get("UL")), + lower_limit_hit=_text(row.get("LL")), + provider_id=provider_id, + source_ref=source_ref, + ) + + +def merge_daily_bars( + existing: Iterable[DailyBarFact], + incoming: Iterable[DailyBarFact], +) -> list[DailyBarFact]: + facts = {(fact.ticker, fact.date): fact for fact in existing} + facts.update({(fact.ticker, fact.date): fact for fact in incoming}) + return [facts[key] for key in sorted(facts)] + + +def filter_daily_bars( + facts: Iterable[DailyBarFact], + *, + tickers: Iterable[str], + limit_per_ticker: int, +) -> list[DailyBarFact]: + wanted = {visible_ticker(ticker) for ticker in tickers if str(ticker or "").strip()} + grouped: dict[str, list[DailyBarFact]] = {} + for fact in facts: + if wanted and fact.ticker not in wanted: + continue + grouped.setdefault(fact.ticker, []).append(fact) + out: list[DailyBarFact] = [] + for ticker in sorted(grouped): + rows = sorted(grouped[ticker], key=lambda fact: fact.date)[-max(limit_per_ticker, 1) :] + out.extend(rows) + return out + + +def daily_bars_to_csv_text(facts: Sequence[DailyBarFact]) -> str: + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=list(DAILY_BAR_COLUMNS), lineterminator="\n") + writer.writeheader() + for fact in facts: + writer.writerow({column: _csv_value(getattr(fact, column)) for column in DAILY_BAR_COLUMNS}) + return output.getvalue() + + +def save_daily_bars( + facts: Sequence[DailyBarFact], + path: str | Path = DEFAULT_DAILY_BARS_CSV, +) -> str: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(daily_bars_to_csv_text(facts), encoding="utf-8") + return str(target) + + +def summarize_daily_bars(facts: Iterable[DailyBarFact]) -> dict[str, object]: + grouped: dict[str, list[DailyBarFact]] = {} + for fact in facts: + grouped.setdefault(fact.ticker, []).append(fact) + summaries: dict[str, dict[str, object]] = {} + for ticker, rows in grouped.items(): + ordered = sorted(rows, key=lambda fact: fact.date) + closes = [value for row in ordered if (value := _price_value(row)) is not None] + volumes = [row.volume for row in ordered if row.volume is not None] + latest = ordered[-1] if ordered else None + summaries[ticker] = { + "ticker": ticker, + "bar_count": len(ordered), + "latest_date": latest.date if latest else "", + "latest_close": _price_value(latest) if latest else None, + "latest_volume": latest.volume if latest else None, + "range_high": max((row.high for row in ordered if row.high is not None), default=None), + "range_low": min((row.low for row in ordered if row.low is not None), default=None), + "return_pct": _return_pct(closes), + "average_volume": round(sum(volumes) / len(volumes), 6) if volumes else None, + "formula": "return_pct = latest adjusted_close / first adjusted_close - 1", + } + return { + "available": bool(summaries), + "tickers": summaries, + "auto_trading": False, + } + + +def latest_price_facts_from_bars( + facts: Iterable[DailyBarFact], + *, + source_ref: str = str(DEFAULT_DAILY_BARS_CSV), +) -> list[MarketPriceFact]: + """Convert the latest bar close per ticker into current price facts.""" + + from investment_assistant.portfolio.price_store import MarketPriceFact + + latest_by_ticker: dict[str, DailyBarFact] = {} + for fact in facts: + price = _price_value(fact) + if price is None or price <= 0: + continue + current = latest_by_ticker.get(fact.ticker) + if current is None or fact.date > current.date: + latest_by_ticker[fact.ticker] = fact + + price_facts: list[MarketPriceFact] = [] + for ticker in sorted(latest_by_ticker): + fact = latest_by_ticker[ticker] + price = _price_value(fact) + if price is None or price <= 0: + continue + price_facts.append( + MarketPriceFact( + ticker=ticker, + price=price, + as_of=fact.date, + provider_id=fact.provider_id, + source_ref=source_ref or fact.source_ref, + note="synced_from_daily_bars", + ) + ) + return price_facts + + +def visible_ticker(value: object) -> str: + text = str(value or "").strip().upper() + digits = "".join(ch for ch in text if ch.isdigit()) + if len(digits) == 5 and digits.endswith("0"): + return digits[:4] + return digits or text + + +def _price_value(row: DailyBarFact | None) -> float | None: + if row is None: + return None + return row.adjusted_close or row.close + + +def _return_pct(values: Sequence[float]) -> float | None: + if len(values) < 2 or values[0] <= 0: + return None + return round((values[-1] / values[0] - 1.0) * 100.0, 6) + + +def _optional_float(value: object) -> float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int | float): + number = float(value) + return number if math.isfinite(number) else None + text = str(value).strip().replace(",", "") + if not text or text.lower() in {"nan", "none", "null", "-"}: + return None + try: + number = float(text) + except ValueError: + return None + return number if math.isfinite(number) else None + + +def _text(value: object) -> str: + return str(value or "").strip() + + +def _csv_value(value: object) -> object: + if isinstance(value, float): + return str(int(value)) if value == int(value) else str(round(value, 6)) + return "" if value is None else value diff --git a/src/investment_assistant/portfolio/price_store.py b/src/investment_assistant/portfolio/price_store.py new file mode 100644 index 0000000..177fd27 --- /dev/null +++ b/src/investment_assistant/portfolio/price_store.py @@ -0,0 +1,217 @@ +"""Local market price snapshot store. + +The app fetches prices on demand from an allowed provider, but the UI should not +go blank after a reload or a temporary provider failure. This module keeps the +latest locally observed prices in a small CSV file. It is an audit helper only: +it does not redistribute market data or make trading decisions. +""" + +from __future__ import annotations + +import csv +import io +import math +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import asdict, dataclass +from pathlib import Path + +MARKET_PRICE_COLUMNS: tuple[str, ...] = ( + "ticker", + "price", + "as_of", + "provider_id", + "source_ref", + "note", +) + +DEFAULT_CURRENT_PRICES_CSV = Path("local_docs/market/current_prices.csv") +DEFAULT_YAHOO_PRICE_INBOX_CSV = Path("local_docs/market/yahoo_prices_inbox.csv") + + +@dataclass(frozen=True) +class MarketPriceFact: + ticker: str + price: float + as_of: str = "" + provider_id: str = "user_csv" + source_ref: str = "" + note: str = "" + + def to_dict(self) -> dict[str, object]: + return {key: value for key, value in asdict(self).items() if value not in ("", None)} + + +def load_current_prices( + path: str | Path | None = DEFAULT_CURRENT_PRICES_CSV, +) -> dict[str, MarketPriceFact]: + """Load the local current price overlay keyed by ticker.""" + + if path is None: + return {} + csv_path = Path(path) + if not csv_path.is_file(): + return {} + return parse_current_prices_csv(csv_path.read_text(encoding="utf-8-sig")) + + +def parse_current_prices_csv(text: str) -> dict[str, MarketPriceFact]: + facts: dict[str, MarketPriceFact] = {} + for row in _dict_rows(text): + fact = market_price_fact_from_row(row) + if fact is not None: + facts[fact.ticker] = fact + return facts + + +def market_price_fact_from_row(row: Mapping[str, object]) -> MarketPriceFact | None: + ticker = normalize_ticker( + row.get("ticker") + or row.get("code") + or row.get("security_code") + or row.get("symbol") + or row.get("Symbol") + or row.get("銘柄コード") + or row.get("コード") + ) + price = _positive_float( + row.get("price") + or row.get("current_price") + or row.get("regularMarketPrice") + or row.get("Regular Market Price") + or row.get("last_price") + or row.get("Last Price") + or row.get("close") + or row.get("Close") + or row.get("現在値") + or row.get("終値") + ) + if not ticker or price is None: + return None + return MarketPriceFact( + ticker=ticker, + price=price, + as_of=_text( + row.get("as_of") + or row.get("date") + or row.get("Date") + or row.get("price_as_of") + or row.get("timestamp") + ), + provider_id=_text(row.get("provider_id") or row.get("provider")) or "user_csv", + source_ref=_text(row.get("source_ref") or row.get("source") or row.get("url")), + note=_text(row.get("note")), + ) + + +def merge_market_price_facts( + existing: Iterable[MarketPriceFact], + incoming: Iterable[MarketPriceFact], +) -> list[MarketPriceFact]: + facts = {fact.ticker: fact for fact in existing} + facts.update({fact.ticker: fact for fact in incoming}) + return [facts[ticker] for ticker in sorted(facts)] + + +def current_prices_to_csv_text(facts: Sequence[MarketPriceFact]) -> str: + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=list(MARKET_PRICE_COLUMNS), lineterminator="\n") + writer.writeheader() + for fact in facts: + writer.writerow( + { + "ticker": fact.ticker, + "price": _format_number(fact.price), + "as_of": fact.as_of, + "provider_id": fact.provider_id, + "source_ref": fact.source_ref, + "note": fact.note, + } + ) + return output.getvalue() + + +def save_current_prices( + facts: Sequence[MarketPriceFact], + path: str | Path = DEFAULT_CURRENT_PRICES_CSV, +) -> str: + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(current_prices_to_csv_text(facts), encoding="utf-8") + return str(target) + + +def facts_from_price_response( + tickers: Iterable[str], + *, + prices: Mapping[str, object], + as_of: Mapping[str, object] | None = None, + provider_id: str = "unknown", + source_ref: str = "", + notes: Mapping[str, object] | None = None, +) -> list[MarketPriceFact]: + """Build storable facts from a provider response.""" + + facts: list[MarketPriceFact] = [] + seen: set[str] = set() + for raw in tickers: + ticker = normalize_ticker(raw) + if not ticker or ticker in seen: + continue + seen.add(ticker) + price = _positive_float(prices.get(ticker)) + if price is None: + continue + facts.append( + MarketPriceFact( + ticker=ticker, + price=price, + as_of=_text((as_of or {}).get(ticker)), + provider_id=provider_id, + source_ref=source_ref, + note=_text((notes or {}).get(ticker)), + ) + ) + return facts + + +def normalize_ticker(value: object) -> str: + text = str(value or "").strip().upper() + if text.endswith(".T"): + return text[:-2] + return text + + +def _positive_float(value: object) -> float | None: + if isinstance(value, bool) or value is None: + return None + if isinstance(value, int | float): + number = float(value) + return number if math.isfinite(number) and number > 0 else None + text = str(value).strip().replace(",", "") + if not text: + return None + try: + number = float(text) + except ValueError: + return None + return number if math.isfinite(number) and number > 0 else None + + +def _text(value: object) -> str: + return str(value or "").strip() + + +def _format_number(value: float) -> str: + return str(int(value)) if value == int(value) else str(round(value, 6)) + + +def _dict_rows(text: str) -> list[dict[str, str]]: + cleaned = text.strip() + if not cleaned: + return [] + sample = cleaned[:2048] + try: + dialect = csv.Sniffer().sniff(sample, delimiters=",\t;") + except csv.Error: + dialect = csv.excel_tab if "\t" in sample else csv.excel + return list(csv.DictReader(io.StringIO(cleaned), dialect=dialect)) diff --git a/src/investment_assistant/portfolio/prices.py b/src/investment_assistant/portfolio/prices.py index 332b6b0..77246a1 100644 --- a/src/investment_assistant/portfolio/prices.py +++ b/src/investment_assistant/portfolio/prices.py @@ -69,6 +69,8 @@ def fetch_prices( url = template.format(ticker=ticker.lower()) try: prices[ticker] = parse_close(fetcher(url)) + if prices[ticker] is None: + notes[ticker] = "no_close_price_returned" except Exception as exc: # noqa: BLE001 - one bad ticker must not abort the batch _logger.warning("price fetch failed ticker=%s error=%s", ticker, type(exc).__name__) prices[ticker] = None diff --git a/src/investment_assistant/portfolio/simulator.py b/src/investment_assistant/portfolio/simulator.py index 1c30177..e534a0e 100644 --- a/src/investment_assistant/portfolio/simulator.py +++ b/src/investment_assistant/portfolio/simulator.py @@ -19,6 +19,12 @@ from dataclasses import dataclass from pathlib import Path +from investment_assistant.financials.current_yield import ( + DEFAULT_CURRENT_YIELDS_CSV, + CurrentYieldFact, + load_current_yields, + reconcile_current_yield, +) from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV, load_comparison DISCLAIMER = ( @@ -99,12 +105,15 @@ class _Prepared: shares_fixed: float amount_fixed: float nisa: bool + dividend_source: str + current_yield: dict[str, object] | None def build_universe( financials_csv: str | Path = DEFAULT_FINANCIALS_CSV, *, prices: dict[str, float] | None = None, + current_yields_csv: str | Path | None = DEFAULT_CURRENT_YIELDS_CSV, ) -> list[dict[str, object]]: """List the EDINET universe with current/conservative dividend + safety. @@ -114,22 +123,43 @@ def build_universe( prices = prices or {} companies = _index_companies(load_comparison(financials_csv)) + current_yields = load_current_yields(current_yields_csv) rows: list[dict[str, object]] = [] for ticker, company in sorted(companies.items()): series = company.get("dividend_series") band = dividend_band(series if isinstance(series, list) else []) latest = _num(company.get("latest_dividend_per_share")) - conservative = band["lower"] if band else latest price = _num(prices.get(ticker)) + current_fact = current_yields.get(ticker) + reconciliation = reconcile_current_yield( + ticker=ticker, + name=str(company.get("name") or ""), + edinet_dividend_per_share=latest, + current_price=price, + fact=current_fact, + ) + current_dps = ( + current_fact.current_dividend_per_share + if current_fact is not None and current_fact.current_dividend_per_share is not None + else latest + ) + conservative = ( + current_dps + if current_fact is not None + else (band["lower"] if band else latest) + ) rows.append( { "ticker": ticker, "name": company.get("name"), "price": round(price) if price > 0 else None, - "dividend_latest": latest, + "dividend_latest": current_dps, + "dividend_latest_edinet": latest, "dividend_conservative": conservative, - "yield_latest": round(latest / price, 4) if price > 0 else None, + "yield_latest": round(current_dps / price, 4) if price > 0 else None, "yield_conservative": round(conservative / price, 4) if price > 0 else None, + "yield_basis": reconciliation.status, + "current_yield_reconciliation": reconciliation.to_dict(), "safety": estimate_safety(company), "band": band, "periods": len(series) if isinstance(series, list) else 0, @@ -150,6 +180,7 @@ def simulate_portfolio( optimization: str = "none", dividend_basis: str = "conservative", financials_csv: str | Path = DEFAULT_FINANCIALS_CSV, + current_yields_csv: str | Path | None = DEFAULT_CURRENT_YIELDS_CSV, lot_default: int = 100, ) -> dict[str, object]: """Build a portfolio and project dividend income (conservative by default). @@ -165,7 +196,7 @@ def simulate_portfolio( budget = max(0.0, float(budget)) mode, optimize, basis, years = _normalize(auto_weight, optimization, dividend_basis, years) - prepared = _prepare_universe(holdings, financials_csv, lot_default) + prepared = _prepare_universe(holdings, financials_csv, lot_default, current_yields_csv) if not prepared: return _unavailable() @@ -196,6 +227,7 @@ def plan_for_target_dividend( optimization: str = "none", dividend_basis: str = "conservative", financials_csv: str | Path = DEFAULT_FINANCIALS_CSV, + current_yields_csv: str | Path | None = DEFAULT_CURRENT_YIELDS_CSV, lot_default: int = 100, net_target: bool = False, ) -> dict[str, object]: @@ -214,7 +246,7 @@ def plan_for_target_dividend( target = max(0.0, float(target_annual_dividend)) mode, optimize, basis, years = _normalize(auto_weight, optimization, dividend_basis, years) - prepared = _prepare_universe(holdings, financials_csv, lot_default) + prepared = _prepare_universe(holdings, financials_csv, lot_default, current_yields_csv) if not prepared: return _unavailable() @@ -387,12 +419,16 @@ def _normalize( def _prepare_universe( - holdings: list[dict[str, object]], financials_csv: str | Path, lot_default: int + holdings: list[dict[str, object]], + financials_csv: str | Path, + lot_default: int, + current_yields_csv: str | Path | None = DEFAULT_CURRENT_YIELDS_CSV, ) -> list[_Prepared]: """Load financials and build priced holdings, dropping any without a price.""" by_ticker = _index_companies(load_comparison(financials_csv)) - prepared = [_prepare_holding(h, by_ticker, lot_default) for h in holdings] + current_yields = load_current_yields(current_yields_csv) + prepared = [_prepare_holding(h, by_ticker, lot_default, current_yields) for h in holdings] return [h for h in prepared if h.price > 0] @@ -418,24 +454,52 @@ def _index_companies(comparison: dict[str, object] | None) -> dict[str, dict[str def _prepare_holding( - holding: dict[str, object], by_ticker: dict[str, dict[str, object]], lot_default: int + holding: dict[str, object], + by_ticker: dict[str, dict[str, object]], + lot_default: int, + current_yields: dict[str, CurrentYieldFact] | None = None, ) -> _Prepared: ticker = str(holding.get("ticker") or "").strip() company = by_ticker.get(ticker) series = (company or {}).get("dividend_series") band = dividend_band(series if isinstance(series, list) else []) + price = _num(holding.get("price")) + current_yields = current_yields or {} + current_fact = current_yields.get(ticker) override = holding.get("dividend_per_share") if override is not None: latest = _num(override) + dividend_source = "user_dividend_per_share" + reconciliation = None + elif current_fact is not None and current_fact.current_dividend_per_share is not None: + latest = current_fact.current_dividend_per_share + dividend_source = "current_dividend_per_share" + band = _flat_band(latest) + reconciliation = reconcile_current_yield( + ticker=ticker, + name=str(holding.get("name") or (company or {}).get("name") or ""), + edinet_dividend_per_share=_num((company or {}).get("latest_dividend_per_share")), + current_price=price, + fact=current_fact, + ).to_dict() else: latest = _num((company or {}).get("latest_dividend_per_share")) - conservative = band["lower"] if band else latest + dividend_source = "edinet_latest_dividend_per_share" + reconciliation = reconcile_current_yield( + ticker=ticker, + name=str(holding.get("name") or (company or {}).get("name") or ""), + edinet_dividend_per_share=latest, + current_price=price, + ).to_dict() + conservative = latest if dividend_source == "current_dividend_per_share" else ( + band["lower"] if band else latest + ) lot = int(_num(holding.get("lot"))) or lot_default return _Prepared( ticker=ticker, name=str(holding.get("name") or (company or {}).get("name") or ""), - price=_num(holding.get("price")), + price=price, dps_latest=latest, dps_conservative=conservative, band=band, @@ -444,6 +508,8 @@ def _prepare_holding( shares_fixed=_num(holding.get("shares")), amount_fixed=_num(holding.get("amount")), nisa=bool(holding.get("nisa")), + dividend_source=dividend_source, + current_yield=reconciliation, ) @@ -624,6 +690,7 @@ def _allocate( "invested": round(invested), "dividend_per_share": dps, "dividend_per_share_latest": holding.dps_latest, + "dividend_source": holding.dividend_source, "annual_dividend": round(annual), "annual_dividend_net": round(annual - tax), "dividend_tax": round(tax), @@ -632,6 +699,7 @@ def _allocate( "annual_band_lower": round(shares * float(band.get("lower", dps))), "annual_band_upper": round(shares * float(band.get("upper", holding.dps_latest))), "yield": round((annual / invested) if invested > 0 else 0.0, 4), + "current_yield_reconciliation": holding.current_yield, "safety": round(holding.safety, 4), "band": holding.band, } @@ -702,3 +770,8 @@ def _num(value: object) -> float: except ValueError: return 0.0 return 0.0 + + +def _flat_band(value: float) -> dict[str, float]: + rounded = round(max(float(value), 0.0), 2) + return {"mean": rounded, "std": 0.0, "upper": rounded, "lower": rounded} diff --git a/src/investment_assistant/portfolio/yahoo_market.py b/src/investment_assistant/portfolio/yahoo_market.py new file mode 100644 index 0000000..51eab65 --- /dev/null +++ b/src/investment_assistant/portfolio/yahoo_market.py @@ -0,0 +1,441 @@ +"""Yahoo! Finance OHLCV and market-fundamental acquisition.""" + +from __future__ import annotations + +import csv +import io +import json +import math +import re +from collections.abc import Iterable, Mapping, Sequence +from datetime import UTC, datetime +from html import unescape +from pathlib import Path +from typing import Protocol + +from investment_assistant.ingestion.fetcher import SafeFetcher, reject_path_traversal +from investment_assistant.portfolio.bar_store import ( + DEFAULT_DAILY_BARS_CSV, + DailyBarFact, + latest_price_facts_from_bars, + load_daily_bars, + merge_daily_bars, + save_daily_bars, + summarize_daily_bars, +) +from investment_assistant.portfolio.price_store import ( + DEFAULT_CURRENT_PRICES_CSV, + load_current_prices, + merge_market_price_facts, + save_current_prices, +) + +YAHOO_CHART_URL_TEMPLATE = ( + "https://query1.finance.yahoo.com/v8/finance/chart/" + "{ticker}.T?range={range_}&interval={interval}" +) +YAHOO_QUOTE_URL_TEMPLATE = "https://query1.finance.yahoo.com/v7/finance/quote?symbols={symbols}" +YAHOO_JAPAN_QUOTE_URL_TEMPLATE = "https://finance.yahoo.co.jp/quote/{ticker}.T" +DEFAULT_YAHOO_FUNDAMENTALS_CSV = Path("local_docs/market/yahoo_financials.csv") +ALLOWED_RANGES = frozenset({"5d", "1mo", "3mo", "6mo", "1y", "2y", "5y"}) +ALLOWED_INTERVALS = frozenset({"1d", "1wk", "1mo"}) +FUNDAMENTAL_COLUMNS = ( + "ticker", "name", "price", "per", "pbr", "dps", "dividend_yield", + "dividend_yield_percent", "eps", "market_cap", "as_of", "provider_id", "source_ref", +) +_NUMERIC_FIELDS = frozenset(FUNDAMENTAL_COLUMNS[2:10]) +_QUOTE_FIELDS = { + "regularMarketPrice": "price", "trailingPE": "per", "priceToBook": "pbr", + "trailingAnnualDividendRate": "dps", + "trailingAnnualDividendYield": "dividend_yield", + "epsTrailingTwelveMonths": "eps", "marketCap": "market_cap", +} +_HTML_FIELDS = {"PER": "per", "PBR": "pbr", "1株配当": "dps", "EPS": "eps"} +_PRICE_PATTERNS = ( + r"ポートフォリオに追加([0-9][0-9,]*(?:\.\d+)?)前日比", + r"([0-9][0-9,]*(?:\.\d+)?)前日比", + r"現在値([0-9][0-9,]*(?:\.\d+)?)", +) + + +class YahooMarketError(RuntimeError): + """Yahoo returned no usable, policy-allowed response.""" + + +class _Document(Protocol): + @property + def allowed_by_robots(self) -> bool: ... + + @property + def status_code(self) -> int | None: ... + + @property + def html(self) -> str: ... + + @property + def source(self) -> str: ... + + +class _Fetcher(Protocol): + def fetch_document(self, url: str) -> _Document: ... + + +def normalize_tickers(tickers: Iterable[object]) -> list[str]: + output: list[str] = [] + seen: set[str] = set() + for raw in tickers: + ticker = str(raw or "").strip().upper() + ticker = ticker[:-2] if ticker.endswith(".T") else ticker + if ticker and ticker not in seen: + seen.add(ticker) + output.append(ticker) + return output + + +def parse_yahoo_chart(json_text: str, *, ticker: str, source_ref: str) -> list[DailyBarFact]: + root = _json_dict(json_text) + result = _first_dict(_list(_dict(root.get("chart")).get("result"))) + timestamps = _list(result.get("timestamp")) + indicators = _dict(result.get("indicators")) + quote = _first_dict(_list(indicators.get("quote"))) + adjusted = _first_dict(_list(indicators.get("adjclose"))) + offset = _number(_dict(result.get("meta")).get("gmtoffset")) or 0.0 + if not timestamps or not quote: + return [] + bars: list[DailyBarFact] = [] + for index, raw_timestamp in enumerate(timestamps): + timestamp = _number(raw_timestamp) + values = tuple(_at(quote.get(key), index) for key in ("open", "high", "low", "close")) + if timestamp is None or all(value is None for value in values): + continue + bars.append( + DailyBarFact( + ticker=ticker, + date=datetime.fromtimestamp(int(timestamp + offset), tz=UTC).date().isoformat(), + open=values[0], high=values[1], low=values[2], close=values[3], + volume=_at(quote.get("volume"), index), + adjusted_close=_at(adjusted.get("adjclose"), index), + provider_id="yahoo_finance", source_ref=source_ref, + ) + ) + return bars + + +def parse_yahoo_quote(json_text: str) -> dict[str, dict[str, object]]: + root = _json_dict(json_text) + items = _list(_dict(root.get("quoteResponse")).get("result")) + output: dict[str, dict[str, object]] = {} + for raw_item in items: + item = _dict(raw_item) + ticker = str(item.get("symbol") or "").strip().upper() + ticker = ticker[:-2] if ticker.endswith(".T") else ticker + if not ticker: + continue + row: dict[str, object] = { + "ticker": ticker, "provider_id": "yahoo_finance", "source_ref": "yahoo_v7_quote" + } + name = item.get("longName") or item.get("shortName") + if isinstance(name, str) and name.strip(): + row["name"] = name.strip() + for source, target in _QUOTE_FIELDS.items(): + if (value := _number(item.get(source))) is not None: + row[target] = value + if (yield_value := _number(row.get("dividend_yield"))) is not None: + row["dividend_yield_percent"] = yield_value * 100.0 + output[ticker] = row + return output + + +def parse_yahoo_japan_html(html_text: str, *, ticker: str) -> dict[str, object]: + plain = _clean_html(html_text) + row: dict[str, object] = { + "ticker": ticker, + "provider_id": "yahoo_finance", + "source_ref": YAHOO_JAPAN_QUOTE_URL_TEMPLATE.format(ticker=ticker), + } + title = re.search(r"(.*?)【", html_text, flags=re.DOTALL) + if title and (name := _clean_html(title.group(1))): + row["name"] = name + if (price := _extract_price(plain)) is not None: + row["price"] = price + for block in re.findall(r"<dl\b[^>]*>.*?</dl>", html_text, flags=re.DOTALL): + block_text = _clean_html(block) + value_text = _dl_value(block) or block_text + for label, key in _HTML_FIELDS.items(): + value = _first_number(value_text) + if ( + block_text.startswith(label) + and value is not None + and (key not in {"per", "pbr", "eps"} or value > 0) + ): + row[key] = value + if block_text.startswith("配当利回り") and (value := _first_number(value_text)) is not None: + row.update(dividend_yield_percent=value, dividend_yield=value / 100.0) + if block_text.startswith("時価総額") and (value := _first_number(value_text)) is not None: + row["market_cap"] = _scale_market_cap(value, block_text) + return row + + +def load_yahoo_fundamentals( + path: str | Path = DEFAULT_YAHOO_FUNDAMENTALS_CSV, +) -> dict[str, dict[str, object]]: + csv_path = Path(path) + if not csv_path.is_file(): + return {} + output: dict[str, dict[str, object]] = {} + with csv_path.open(newline="", encoding="utf-8-sig") as handle: + for raw in csv.DictReader(handle): + ticker = str(raw.get("ticker") or "").strip() + if not ticker: + continue + row: dict[str, object] = {"ticker": ticker} + for key, value in raw.items(): + if key and value and key != "ticker": + row[key] = _number(value) if key in _NUMERIC_FIELDS else value + output[ticker] = row + return output + + +def save_yahoo_fundamentals( + rows: Mapping[str, Mapping[str, object]], + path: str | Path = DEFAULT_YAHOO_FUNDAMENTALS_CSV, +) -> str: + target = reject_path_traversal(path) + target.parent.mkdir(parents=True, exist_ok=True) + output = io.StringIO() + writer = csv.DictWriter(output, fieldnames=list(FUNDAMENTAL_COLUMNS), lineterminator="\n") + writer.writeheader() + for ticker in sorted(rows): + writer.writerow({key: _csv(ticker if key == "ticker" else rows[ticker].get(key)) + for key in FUNDAMENTAL_COLUMNS}) + target.write_text(output.getvalue(), encoding="utf-8-sig") + return str(target) + + +def refresh_yahoo_market( + tickers: Iterable[object], *, range_: str = "1mo", interval: str = "1d", + fetch_ohlcv: bool = True, fetch_fundamentals: bool = True, + daily_bars_path: str | Path = DEFAULT_DAILY_BARS_CSV, + current_prices_path: str | Path = DEFAULT_CURRENT_PRICES_CSV, + fundamentals_path: str | Path = DEFAULT_YAHOO_FUNDAMENTALS_CSV, + fetcher: _Fetcher | None = None, +) -> dict[str, object]: + resolved = normalize_tickers(tickers) + if not resolved: + raise ValueError("at least one ticker is required") + if range_ not in ALLOWED_RANGES or interval not in ALLOWED_INTERVALS: + raise ValueError("unsupported Yahoo range or interval") + if not fetch_ohlcv and not fetch_fundamentals: + raise ValueError("enable fetch_ohlcv and/or fetch_fundamentals") + client: _Fetcher = fetcher or SafeFetcher(timeout_seconds=20.0) + failures: dict[str, list[str]] = {ticker: [] for ticker in resolved} + fetched: dict[str, list[DailyBarFact]] = {} + sources: dict[str, str] = {} + + if fetch_ohlcv: + for ticker in resolved: + url = YAHOO_CHART_URL_TEMPLATE.format( + ticker=ticker.lower(), range_=range_, interval=interval + ) + try: + body, source = _fetch(client, url) + bars = parse_yahoo_chart(body, ticker=ticker, source_ref=url) + if bars: + fetched[ticker], sources[ticker] = bars, source + else: + failures[ticker].append("ohlcv_empty") + except YahooMarketError as exc: + failures[ticker].append(f"ohlcv:{exc}") + + all_bars = [bar for ticker_bars in fetched.values() for bar in ticker_bars] + bars_path, prices_path = str(daily_bars_path), str(current_prices_path) + if all_bars: + bars_path = save_daily_bars( + merge_daily_bars(load_daily_bars(daily_bars_path), all_bars), daily_bars_path + ) + price_facts = latest_price_facts_from_bars(all_bars, source_ref=bars_path) + stored = load_current_prices(current_prices_path) + prices_path = save_current_prices( + merge_market_price_facts(stored.values(), price_facts), current_prices_path + ) + + latest = {ticker: value for ticker, bars in fetched.items() + if (value := _latest_close(bars)) is not None} + fundamentals = ( + _fetch_fundamentals(client, resolved, latest, failures) + if fetch_fundamentals + else {} + ) + fundamentals_path_text = str(fundamentals_path) + if fundamentals: + merged = load_yahoo_fundamentals(fundamentals_path) + merged.update(fundamentals) + fundamentals_path_text = save_yahoo_fundamentals(merged, fundamentals_path) + + errors = {ticker: messages for ticker, messages in failures.items() if messages} + completed = max(len(fetched), len(fundamentals)) + status = "blocked" if completed == 0 else "partial" if errors else "completed" + return { + "status": status, "provider_id": "yahoo_finance", "requested_count": len(resolved), + "tickers": resolved, "ohlcv_ticker_count": len(fetched), "ohlcv_row_count": len(all_bars), + "fundamentals_ticker_count": len(fundamentals), + "latest_bars": [_summary(ticker, bars) for ticker, bars in sorted(fetched.items())], + "fundamentals": [fundamentals[ticker] for ticker in sorted(fundamentals)], + "errors": errors, "fetch_sources": sources, + "saved": {"daily_bars_path": bars_path, "current_prices_path": prices_path, + "fundamentals_path": fundamentals_path_text}, + "ohlcv_summary": summarize_daily_bars(all_bars), + "policy": {"personal_use_only": True, "robots_checked": True, "rate_limited": True, + "redistribution": False, "auto_trading": False}, + "auto_trading": False, "call_real_api": True, + } + + +def _fetch_fundamentals( + client: _Fetcher, tickers: Sequence[str], latest: Mapping[str, float], + failures: dict[str, list[str]], +) -> dict[str, dict[str, object]]: + output: dict[str, dict[str, object]] = {} + for start in range(0, len(tickers), 40): + batch = tickers[start:start + 40] + url = YAHOO_QUOTE_URL_TEMPLATE.format(symbols=",".join(f"{ticker}.T" for ticker in batch)) + try: + body, _ = _fetch(client, url) + parsed = parse_yahoo_quote(body) + for parsed_row in parsed.values(): + parsed_row["source_ref"] = url + output.update(parsed) + except YahooMarketError: + pass + for ticker in tickers: + row = output.get(ticker) + if row is None or not any(key in row for key in ("per", "pbr", "dps", "eps", "market_cap")): + url = YAHOO_JAPAN_QUOTE_URL_TEMPLATE.format(ticker=ticker) + try: + html, _ = _fetch(client, url) + html_row = parse_yahoo_japan_html(html, ticker=ticker) + if len(html_row) > 3: + row = _merge(row, html_row) + output[ticker] = row + else: + failures[ticker].append("fundamentals_empty") + except YahooMarketError as exc: + failures[ticker].append(f"fundamentals:{exc}") + if row is not None: + if "price" not in row and ticker in latest: + row["price"] = latest[ticker] + row["source_ref"] = f"{row.get('source_ref', '')}+chart_close" + row["as_of"] = datetime.now(UTC).date().isoformat() + return output + + +def _fetch(client: _Fetcher, url: str) -> tuple[str, str]: + document = client.fetch_document(url) + if not document.allowed_by_robots: + raise YahooMarketError(document.source or "robots_blocked") + if document.status_code is not None and document.status_code >= 400: + raise YahooMarketError(f"http_{document.status_code}") + if not document.html.strip(): + raise YahooMarketError("empty_response") + return document.html, document.source + + +def _merge(current: dict[str, object] | None, incoming: Mapping[str, object]) -> dict[str, object]: + output = dict(current or {}) + prior = str(output.get("source_ref") or "") + for key, value in incoming.items(): + if key not in output or output[key] in (None, ""): + output[key] = value + source = str(incoming.get("source_ref") or "") + output["source_ref"] = "+".join(value for value in (prior, source) if value) + return output + + +def _summary(ticker: str, bars: Sequence[DailyBarFact]) -> dict[str, object]: + bar = max(bars, key=lambda item: item.date) + return {"ticker": ticker, "date": bar.date, "open": bar.open, "high": bar.high, + "low": bar.low, "close": bar.close, "adjusted_close": bar.adjusted_close, + "volume": bar.volume, "bar_count": len(bars), "provider_id": bar.provider_id, + "source_ref": bar.source_ref} + + +def _latest_close(bars: Sequence[DailyBarFact]) -> float | None: + for bar in sorted(bars, key=lambda item: item.date, reverse=True): + value = bar.adjusted_close or bar.close + if value is not None and value > 0: + return float(value) + return None + + +def _json_dict(text: str) -> dict[str, object]: + try: + return _dict(json.loads(text)) + except (TypeError, ValueError): + return {} + + +def _dict(value: object) -> dict[str, object]: + return {str(key): item for key, item in value.items()} if isinstance(value, dict) else {} + + +def _list(value: object) -> list[object]: + return list(value) if isinstance(value, list) else [] + + +def _first_dict(values: Sequence[object]) -> dict[str, object]: + return _dict(values[0]) if values else {} + + +def _at(value: object, index: int) -> float | None: + return _number(value[index]) if isinstance(value, list) and index < len(value) else None + + +def _number(value: object) -> float | None: + if isinstance(value, bool) or value is None: + return None + try: + result = float(str(value).strip().replace(",", "")) + except ValueError: + return None + return result if math.isfinite(result) else None + + +def _clean_html(value: str) -> str: + value = re.sub(r"<!--.*?-->", "", value, flags=re.DOTALL) + return re.sub(r"\s+", "", unescape(re.sub(r"<[^>]+>", "", value))) + + +def _dl_value(block: str) -> str: + match = re.search(r"<dd\b[^>]*>(.*?)</dd>", block, flags=re.DOTALL) + return _clean_html(match.group(1)) if match else "" + + +def _first_number(text: str) -> float | None: + match = re.search(r"[-+]?\d[\d,]*(?:\.\d+)?", re.sub(r"\([^)]*\)", "", text)) + return _number(match.group(0)) if match else None + + +def _extract_price(text: str) -> float | None: + for pattern in _PRICE_PATTERNS: + match = re.search(pattern, text) + if match and (value := _number(match.group(1))) is not None and value > 0: + return value + return None + + +def _scale_market_cap(value: float, text: str) -> float: + if "百万円" in text: + return value * 1_000_000 + if "億円" in text: + return value * 100_000_000 + if "兆円" in text: + return value * 1_000_000_000_000 + return value + + +def _csv(value: object) -> str: + if value is None: + return "" + if isinstance(value, float): + return str(int(value)) if value.is_integer() else str(round(value, 8)) + return str(value) diff --git a/src/investment_assistant/rag/search.py b/src/investment_assistant/rag/search.py index c68d46f..feceec7 100644 --- a/src/investment_assistant/rag/search.py +++ b/src/investment_assistant/rag/search.py @@ -3,6 +3,7 @@ from __future__ import annotations import math +import re from dataclasses import dataclass, field, replace from investment_assistant.rag.embeddings import Embedder, HashingEmbedder, cosine @@ -12,8 +13,24 @@ _CONTEXT_METADATA_KEYS = ("source_url", "fetched_at", "status_code", "content_type") DEFAULT_MAX_CONTEXT_CHARS = 10000 DEFAULT_HYBRID_ALPHA = 0.5 +DEFAULT_RRF_K = 60 +DEFAULT_MAX_PER_SOURCE = 3 +DEFAULT_QUERY_VARIANTS = 4 # Near-duplicate threshold (token Jaccard) for diversity selection. _DUPLICATE_JACCARD = 0.85 +_QUERY_SPLIT_RE = re.compile(r"[\n\r,，、/／]+|\s+(?:and|or)\s+", re.IGNORECASE) +_QUERY_STOPWORDS = { + "about", + "and", + "for", + "or", + "the", + "with", + "について", + "とは", + "です", + "ます", +} @dataclass(frozen=True) @@ -28,6 +45,45 @@ class SearchResult: metadata: dict[str, str] = field(default_factory=dict) +def decompose_query(query: str, *, max_queries: int = DEFAULT_QUERY_VARIANTS) -> list[str]: + """Return deterministic query variants for recall-oriented local search. + + This is intentionally lightweight and local: no LLM call, no external API. + The original query stays first, then separator-based phrases and useful + tokens are added until ``max_queries`` is reached. + """ + + if max_queries <= 0: + return [] + normalized = " ".join(query.split()) + if not normalized: + return [] + + variants: list[str] = [] + + def add(value: str) -> None: + phrase = value.strip(" 　\t") + if len(phrase) < 2: + return + if phrase.lower() in _QUERY_STOPWORDS: + return + if phrase not in variants: + variants.append(phrase) + + add(normalized) + for part in _QUERY_SPLIT_RE.split(normalized): + add(part) + if len(variants) >= max_queries: + return variants[:max_queries] + + for token in tokenize(normalized): + if len(token) >= 2: + add(token) + if len(variants) >= max_queries: + break + return variants[:max_queries] + + def search_chunks(store: RagStore, *, query: str, limit: int = 5) -> list[SearchResult]: """Search chunks using FTS5 BM25 ranking, falling back to keyword scoring. @@ -115,6 +171,112 @@ def hybrid_search( return _dedupe_by_text(ranked)[:limit] +def enhanced_search( + store: RagStore, + *, + query: str, + limit: int = 5, + hybrid: bool = True, + alpha: float = DEFAULT_HYBRID_ALPHA, + query_expansion: bool = True, + max_queries: int = DEFAULT_QUERY_VARIANTS, + rrf_k: int = DEFAULT_RRF_K, + max_per_source: int = DEFAULT_MAX_PER_SOURCE, +) -> dict[str, object]: + """Search with deterministic query expansion, RRF fusion, and diagnostics. + + The enhanced path is designed for investor-facing evidence lookup: higher + recall from multiple query variants, transparent reciprocal-rank fusion, and + source diversity. It still returns passages only; it does not make + investment recommendations. + """ + + if limit <= 0: + return { + "query": query, + "queries": [], + "results": [], + "diagnostics": _search_diagnostics( + mode="enhanced_hybrid" if hybrid else "enhanced_lexical", + query_count=0, + candidate_count=0, + limit=limit, + alpha=alpha, + rrf_k=rrf_k, + max_per_source=max_per_source, + ), + } + + queries = decompose_query(query, max_queries=max_queries) if query_expansion else [query] + queries = [item for item in queries if tokenize(item)] + if not queries: + return { + "query": query, + "queries": [], + "results": [], + "diagnostics": _search_diagnostics( + mode="enhanced_hybrid" if hybrid else "enhanced_lexical", + query_count=0, + candidate_count=0, + limit=limit, + alpha=alpha, + rrf_k=rrf_k, + max_per_source=max_per_source, + ), + } + + pool_limit = max(limit * 4, 12) + fused_scores: dict[str, float] = {} + best_results: dict[str, SearchResult] = {} + hit_counts: dict[str, int] = {} + + for subquery in queries: + ranked = ( + hybrid_search(store, query=subquery, limit=pool_limit, alpha=alpha) + if hybrid + else search_chunks(store, query=subquery, limit=pool_limit) + ) + for rank, result in enumerate(ranked, start=1): + fused_scores[result.chunk_id] = fused_scores.get(result.chunk_id, 0.0) + 1.0 / ( + max(rrf_k, 1) + rank + ) + hit_counts[result.chunk_id] = hit_counts.get(result.chunk_id, 0) + 1 + current = best_results.get(result.chunk_id) + if current is None or result.score > current.score: + best_results[result.chunk_id] = result + + fused: list[SearchResult] = [] + for chunk_id, score in fused_scores.items(): + base = best_results[chunk_id] + fused.append( + replace( + base, + score=round(score, 6), + metadata={ + **base.metadata, + "matched_query_count": str(hit_counts.get(chunk_id, 1)), + "ranking_method": "reciprocal_rank_fusion", + }, + ) + ) + fused.sort(key=lambda result: (-result.score, result.source, result.chunk_index)) + selected = diversify_results(fused, limit=limit, max_per_source=max_per_source) + return { + "query": query, + "queries": queries, + "results": selected, + "diagnostics": _search_diagnostics( + mode="enhanced_hybrid" if hybrid else "enhanced_lexical", + query_count=len(queries), + candidate_count=len(fused), + limit=limit, + alpha=alpha, + rrf_k=rrf_k, + max_per_source=max_per_source, + ), + } + + def _min_max_normalize(scores: dict[str, float]) -> dict[str, float]: if not scores: return {} @@ -279,6 +441,59 @@ def _format_context_header(index: int, result: SearchResult) -> str: return f"{base} {metadata}" if metadata else base +def _search_diagnostics( + *, + mode: str, + query_count: int, + candidate_count: int, + limit: int, + alpha: float, + rrf_k: int, + max_per_source: int, +) -> dict[str, object]: + return { + "mode": mode, + "query_count": query_count, + "candidate_count": candidate_count, + "limit": limit, + "hybrid_alpha": alpha if "hybrid" in mode else None, + "rrf_k": rrf_k, + "max_per_source": max_per_source, + "operators": [ + { + "key": "query_decomposition", + "label": "クエリ分解", + "formula": "original query + separator phrases + useful tokens", + "purpose": "同じ意図を複数の検索語で拾い、RAGの取りこぼしを減らす", + }, + { + "key": "hybrid_blend" if "hybrid" in mode else "lexical_search", + "label": "検索スコア", + "formula": "alpha * semantic_score + (1 - alpha) * lexical_score" + if "hybrid" in mode + else "BM25 or keyword_count", + "purpose": "語句一致と意味類似を分けて扱い、数値は候補抽出に使わない", + }, + { + "key": "reciprocal_rank_fusion", + "label": "順位統合", + "formula": f"sum(1 / ({max(rrf_k, 1)} + rank))", + "purpose": "複数クエリの上位結果を安定して統合する", + }, + { + "key": "diversity_cap", + "label": "出典分散", + "formula": ( + f"near_duplicate_jaccard < {_DUPLICATE_JACCARD}; " + f"max_per_source <= {max_per_source}" + ), + "purpose": "同じ出典や近い文章だけで根拠欄が埋まるのを防ぐ", + }, + ], + "non_advisory_boundary": "検索結果は根拠候補の提示のみ。売買判断や自動売買には使わない。", + } + + def _score_chunk(chunk: StoredChunk, terms: list[str]) -> int: text = chunk.text.lower() return sum(text.count(term) for term in terms) diff --git a/src/investment_assistant/webapi/__init__.py b/src/investment_assistant/webapi/__init__.py index 05e38cc..53cde59 100644 --- a/src/investment_assistant/webapi/__init__.py +++ b/src/investment_assistant/webapi/__init__.py @@ -6,6 +6,35 @@ Gemini API calls. """ -from investment_assistant.webapi.service import ApiError, available_routes, handle_api +from investment_assistant.webapi.service import ( + ApiError, + JsonDict, +) +from investment_assistant.webapi.service import ( + available_routes as _core_available_routes, +) +from investment_assistant.webapi.service import ( + handle_api as _core_handle_api, +) +from investment_assistant.webapi.yahoo_market import ( + available_yahoo_market_routes, + handle_yahoo_market_api, +) -__all__ = ["ApiError", "available_routes", "handle_api"] + +def handle_api( + method: str, + path: str, + body: JsonDict | None = None, +) -> tuple[int, JsonDict]: + yahoo_result = handle_yahoo_market_api(method, path, body) + if yahoo_result is not None: + return yahoo_result + return _core_handle_api(method, path, body) + + +def available_routes() -> list[str]: + return sorted({*_core_available_routes(), *available_yahoo_market_routes()}) + + +__all__ = ["ApiError", "JsonDict", "available_routes", "handle_api"] diff --git a/src/investment_assistant/webapi/server.py b/src/investment_assistant/webapi/server.py index e1d500b..dbc2a7e 100644 --- a/src/investment_assistant/webapi/server.py +++ b/src/investment_assistant/webapi/server.py @@ -7,7 +7,7 @@ from pathlib import Path from investment_assistant.observability import configure_logging, get_logger -from investment_assistant.webapi.service import JsonDict, handle_api +from investment_assistant.webapi import JsonDict, handle_api _logger = get_logger("webapi.server") diff --git a/src/investment_assistant/webapi/service.py b/src/investment_assistant/webapi/service.py index f18c726..3274493 100644 --- a/src/investment_assistant/webapi/service.py +++ b/src/investment_assistant/webapi/service.py @@ -2,6 +2,7 @@ from __future__ import annotations +import csv import os import re import tempfile @@ -15,6 +16,7 @@ compare_financials, load_financials, ) +from investment_assistant.financials.current_yield import DEFAULT_CURRENT_YIELDS_CSV from investment_assistant.financials.evidence import ( DEFAULT_FINANCIALS_CSV, build_financial_evidence, @@ -33,6 +35,75 @@ Handler = Callable[[JsonDict], JsonDict] _REAL_API_ENV = "INVESTMENT_ASSISTANT_WEB_REAL_API" _REAL_API_RUNTIME_ENABLED = False +_EDINET_API_KEY_RUNTIME_SET = False +_JQUANTS_API_KEY_ENV_VAR = "JQUANTS_API_KEY" +_JQUANTS_REFRESH_TOKEN_ENV_VAR = "JQUANTS_REFRESH_TOKEN" +_JQUANTS_PROVIDER_ID = "jquants" +_JQUANTS_API_KEY_RUNTIME_SET = False +_JQUANTS_CONTRACT_RUNTIME_ACK = False +_JQUANTS_CAPABILITIES: tuple[JsonDict, ...] = ( + { + "key": "equities_master", + "label": "上場銘柄マスター", + "endpoint": "/v2/equities/master", + "status": "planned", + "use": "銘柄名・市場区分・業種などの銘柄検索をJ-Quantsへ差し替え可能", + }, + { + "key": "equities_bars_daily", + "label": "株価四本値・出来高", + "endpoint": "/v2/equities/bars/daily", + "status": "implemented", + "use": "シミュレーションと保有分析の現在価格補完に利用", + "local_endpoint": "/api/market/bars", + "data_items": [ + "日付", + "始値", + "高値", + "安値", + "終値", + "出来高", + "売買代金", + "調整後四本値", + "調整後出来高", + ], + "can_do": [ + "終値を現在価格として補完する", + "指定期間の価格推移と出来高を保存する", + "調整後終値ベースの騰落率を計算する", + "期間高値・安値・平均出来高を確認する", + "配当利回りや保有損益の株価基準日をそろえる", + ], + "not_doing": [ + "売買推奨", + "自動売買", + "将来リターンの断定予測", + "第三者へのJ-Quants生データ再配布", + ], + "storage": "local_docs/market/daily_bars.csv", + }, + { + "key": "financial_summary", + "label": "決算短信サマリー/財務情報", + "endpoint": "/v2/fins/*", + "status": "planned", + "use": "EDINET財務の補完・速報値・配当情報の比較に利用", + }, + { + "key": "dividends", + "label": "配当金データ", + "endpoint": "/v2/fins/*", + "status": "planned", + "use": "配当/分配金見込みと利回り検算に利用", + }, + { + "key": "market_activity", + "label": "信用残・空売り・投資部門別", + "endpoint": "/v2/markets/*", + "status": "planned", + "use": "需給リスクや市場参加者動向の参考指標に利用", + }, +) class ApiError(Exception): @@ -63,6 +134,134 @@ def _health(_: JsonDict) -> JsonDict: return {"status": "ok", "service": "investment-assistant", "auto_trading": False} +def _edinet_status(_: JsonDict) -> JsonDict: + from investment_assistant.edinet.client import API_KEY_ENV_VAR + + env_configured_before_dotenv = bool(os.getenv(API_KEY_ENV_VAR, "").strip()) + dotenv_loaded = _ensure_env_from_dotenv(API_KEY_ENV_VAR) + configured = bool(os.getenv(API_KEY_ENV_VAR, "").strip()) + return { + "api_key_configured": configured, + "api_key_env_var": API_KEY_ENV_VAR, + "api_key_source": _edinet_api_key_source( + configured=configured, + env_configured_before_dotenv=env_configured_before_dotenv, + dotenv_loaded=dotenv_loaded, + ), + "default_registry": "examples/source_registry_nikkei225_edinet.yaml", + "default_output_dir": "local_docs/edinet", + "default_financials_csv": DEFAULT_FINANCIALS_CSV, + "structured_refresh_requires_key": True, + "fallback_without_key": "official_disclosure_scrape_only", + "auto_trading": False, + "call_real_api": False, + } + + +def _edinet_api_key_set(body: JsonDict) -> JsonDict: + from investment_assistant.edinet.client import API_KEY_ENV_VAR + + global _EDINET_API_KEY_RUNTIME_SET + + value = str(body.get("api_key") or "").strip() + if value: + os.environ[API_KEY_ENV_VAR] = value + _EDINET_API_KEY_RUNTIME_SET = True + configured = bool(os.getenv(API_KEY_ENV_VAR, "").strip()) + return { + "api_key_configured": configured, + "api_key_env_var": API_KEY_ENV_VAR, + "api_key_source": "runtime_input" if _EDINET_API_KEY_RUNTIME_SET else "missing", + "request_api_key_applied": bool(value), + "auto_trading": False, + "call_real_api": False, + } + + +def _jquants_status(_: JsonDict) -> JsonDict: + from investment_assistant.investment.provider_policy import ( + CONTRACTED_PROVIDERS_ENV, + provider_policy, + ) + + env_configured_before_dotenv = bool( + os.getenv(_JQUANTS_REFRESH_TOKEN_ENV_VAR, "").strip() + or os.getenv(_JQUANTS_API_KEY_ENV_VAR, "").strip() + ) + dotenv_loaded = _ensure_env_from_dotenv( + _JQUANTS_REFRESH_TOKEN_ENV_VAR + ) or _ensure_env_from_dotenv(_JQUANTS_API_KEY_ENV_VAR) + configured = bool( + os.getenv(_JQUANTS_REFRESH_TOKEN_ENV_VAR, "").strip() + or os.getenv(_JQUANTS_API_KEY_ENV_VAR, "").strip() + ) + _ensure_env_from_dotenv(CONTRACTED_PROVIDERS_ENV) + policy = provider_policy(_JQUANTS_PROVIDER_ID, runtime_mode="production") + return { + "api_key_configured": configured, + "api_key_env_var": _JQUANTS_REFRESH_TOKEN_ENV_VAR, + "legacy_api_key_env_var": _JQUANTS_API_KEY_ENV_VAR, + "api_key_source": _jquants_api_key_source( + configured=configured, + env_configured_before_dotenv=env_configured_before_dotenv, + dotenv_loaded=dotenv_loaded, + ), + "contract_acknowledged": _JQUANTS_CONTRACT_RUNTIME_ACK + or _is_runtime_contracted_provider(_JQUANTS_PROVIDER_ID), + "provider_policy": policy.to_dict(), + "production_allowed": policy.production_allowed, + "auth_method": "v2_api_key_header", + "auth_header": "x-api-key", + "capabilities": list(_JQUANTS_CAPABILITIES), + "official_docs": { + "usage": "https://jpx-jquants.com/en/help/usage", + "quickstart": "https://jpx-jquants.com/en/spec/quickstart", + "daily_bars": "https://jpx-jquants.com/en/spec/eq-bars-daily", + "listed_master": "https://jpx-jquants.com/en/spec/eq-master", + "data_update": "https://jpx-jquants.com/en/spec/data-update", + }, + "request_api_key_applied": False, + "auto_trading": False, + "call_real_api": False, + } + + +def _jquants_api_key_set(body: JsonDict) -> JsonDict: + global _JQUANTS_API_KEY_RUNTIME_SET, _JQUANTS_CONTRACT_RUNTIME_ACK + + value = str(body.get("api_key") or body.get("refresh_token") or "").strip() + persist_local = _as_bool(body.get("persist_local"), False) + if value: + os.environ[_JQUANTS_REFRESH_TOKEN_ENV_VAR] = value + os.environ[_JQUANTS_API_KEY_ENV_VAR] = value + _JQUANTS_API_KEY_RUNTIME_SET = True + if _as_bool(body.get("contract_acknowledged"), False): + _mark_runtime_contracted_provider(_JQUANTS_PROVIDER_ID) + _JQUANTS_CONTRACT_RUNTIME_ACK = True + persisted_path: str | None = None + if persist_local and value: + from investment_assistant.investment.provider_policy import CONTRACTED_PROVIDERS_ENV + + providers = os.getenv(CONTRACTED_PROVIDERS_ENV, "") + persisted_path = _upsert_dotenv_values( + Path(".env.local"), + { + _JQUANTS_REFRESH_TOKEN_ENV_VAR: value, + _JQUANTS_API_KEY_ENV_VAR: value, + CONTRACTED_PROVIDERS_ENV: providers, + }, + ) + status = _jquants_status({}) + status["request_api_key_applied"] = bool(value) + status["request_contract_acknowledged"] = _as_bool( + body.get("contract_acknowledged"), False + ) + status["persisted_local"] = bool(persisted_path) + if persisted_path: + status["persisted_path"] = persisted_path + return status + + def _budget(_: JsonDict) -> JsonDict: from dataclasses import asdict @@ -121,15 +320,61 @@ def _rag_stats(body: JsonDict) -> JsonDict: def _rag_search(body: JsonDict) -> JsonDict: + from dataclasses import asdict + from typing import cast + + from investment_assistant.rag.search import SearchResult, enhanced_search + from investment_assistant.rag.store import RagStore + query = _require_str(body, "query") - results = cli.run_rag_search( + db_path = str(body.get("db_path") or DEFAULT_RAG_DB_PATH) + limit = _as_int(body.get("limit"), 5) + hybrid = _as_bool(body.get("hybrid"), True) + alpha = _as_float(body.get("alpha"), 0.5) + enhanced = _as_bool(body.get("enhanced"), True) + if not enhanced: + results = cli.run_rag_search( + query=query, + db_path=db_path, + limit=limit, + hybrid=hybrid, + alpha=alpha, + ) + return { + "query": query, + "queries": [query], + "results": results, + "diagnostics": { + "mode": "legacy_hybrid" if hybrid else "legacy_lexical", + "query_count": 1, + "candidate_count": len(results), + "limit": limit, + "hybrid_alpha": alpha if hybrid else None, + "operators": [], + "non_advisory_boundary": ( + "検索結果は根拠候補の提示のみ。" + "売買判断や自動売買には使わない。" + ), + }, + } + payload = enhanced_search( + RagStore(db_path), query=query, - db_path=str(body.get("db_path") or DEFAULT_RAG_DB_PATH), - limit=_as_int(body.get("limit"), 5), - hybrid=bool(body.get("hybrid", False)), - alpha=_as_float(body.get("alpha"), 0.5), + limit=limit, + hybrid=hybrid, + alpha=alpha, + query_expansion=_as_bool(body.get("query_expansion"), True), + max_queries=_as_int(body.get("max_queries"), 4), + rrf_k=_as_int(body.get("rrf_k"), 60), + max_per_source=_as_int(body.get("max_per_source"), 3), ) - return {"query": query, "results": results} + search_results = cast(list[SearchResult], payload["results"]) + return { + **payload, + "results": [asdict(result) for result in search_results], + "auto_trading": False, + "call_real_api": False, + } def _rag_answer_context(body: JsonDict) -> JsonDict: @@ -407,8 +652,105 @@ def _fetch_job_auto(body: JsonDict) -> JsonDict: } +def _financials_refresh(body: JsonDict) -> JsonDict: + """Refresh financial data through the safest available official path. + + Structured financial metrics come from EDINET API CSV/XBRL, because the + report and screening engine need deterministic, auditable values. When the + EDINET API key is not configured, we still run the official disclosure-page + scraping path for RAG grounding, but we intentionally do not claim that the + structured ``financials.csv`` was updated. + """ + + from investment_assistant.edinet.client import API_KEY_ENV_VAR + + _ensure_env_from_dotenv(API_KEY_ENV_VAR) + registry_path = str( + body.get("registry_path") or "examples/source_registry_nikkei225_edinet.yaml" + ) + output_dir = str(body.get("output_dir") or "local_docs/edinet") + financials_csv = str(Path(output_dir) / "financials.csv") + db_path = str(body.get("db_path") or DEFAULT_RAG_DB_PATH) + index_after_fetch = _as_bool(body.get("index_after_fetch"), True) + + if os.getenv(API_KEY_ENV_VAR, "").strip(): + result = _edinet_ingest( + { + **body, + "registry_path": registry_path, + "output_dir": output_dir, + "db_path": db_path, + "index_after_fetch": index_after_fetch, + } + ) + result["mode"] = "edinet_api" + result["api_key_configured"] = True + result["financials_updated"] = bool(result.get("financials_csv")) + result["financials_csv"] = str(result.get("financials_csv") or financials_csv) + result["official_sources"] = [ + { + "label": "EDINET API v2", + "url": "https://disclosure2.edinet-fsa.go.jp/", + "purpose": "有価証券報告書等の公式CSV/XBRL取得", + } + ] + result["hint"] = ( + "EDINET公式APIのCSV/XBRLから財務データを更新しました。" + "候補抽出、詳細、レポートはこのCSVを参照します。" + ) + return result + + scrape_result = _fetch_job_auto( + { + "sources": _default_disclosure_sources(), + "db_path": db_path, + "index_path": "local_docs", + "index_after_fetch": index_after_fetch, + } + ) + return { + "mode": "disclosure_scrape_only", + "api_key_configured": False, + "financials_updated": False, + "financials_csv": financials_csv, + "scrape": scrape_result, + "official_sources": [ + { + "label": "EDINET 閲覧サイト", + "url": "https://disclosure2.edinet-fsa.go.jp/", + "purpose": "開示ページの確認とRAG根拠取得", + }, + { + "label": "TDnet", + "url": "https://www.release.tdnet.info/inbs/I_main_00.html", + "purpose": "適時開示ページの確認とRAG根拠取得", + }, + { + "label": "JPX 東証上場銘柄一覧", + "url": "https://www.jpx.co.jp/markets/statistics-equities/misc/01.html", + "purpose": "市場区分と銘柄名の確認", + }, + ], + "hint": ( + "EDINET API KEYがバックエンドに未設定のため、構造化された財務CSVは更新していません。" + "公式ページの取得とRAG登録だけを実行しました。" + "財務数値の更新はAPI KEY設定後に再実行してください。" + ), + "auto_trading": False, + "call_real_api": False, + } + + +def _financials_refresh_async(body: JsonDict) -> JsonDict: + job_id = JOBS.start("financials-refresh", lambda: _financials_refresh(body)) + return {"job_id": job_id, "status": "running", "kind": "financials-refresh"} + + def _edinet_ingest(body: JsonDict) -> JsonDict: + from investment_assistant.edinet.client import API_KEY_ENV_VAR + + _ensure_env_from_dotenv(API_KEY_ENV_VAR) registry_path = str( body.get("registry_path") or "examples/source_registry_edinet_sample.yaml" ) @@ -430,6 +772,12 @@ def _edinet_ingest(body: JsonDict) -> JsonDict: ) +def _operators_catalog(_: JsonDict) -> JsonDict: + from investment_assistant.investment.operators import operator_catalog + + return operator_catalog() + + def _edinet_ingest_async(body: JsonDict) -> JsonDict: """Start an EDINET ingest in the background and return a job id to poll. @@ -524,6 +872,7 @@ def _portfolio_inputs(body: JsonDict) -> tuple[list[JsonDict], JsonDict]: "optimization": str(body.get("optimization") or "none"), "dividend_basis": str(body.get("dividend_basis") or "conservative"), "financials_csv": str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV), + "current_yields_csv": str(body.get("current_yields_csv") or DEFAULT_CURRENT_YIELDS_CSV), } return holdings, common @@ -549,6 +898,7 @@ def _portfolio_target(body: JsonDict) -> JsonDict: def _portfolio_universe(body: JsonDict) -> JsonDict: + from investment_assistant.investment.universe import build_market_universe from investment_assistant.portfolio.simulator import build_universe raw_prices = body.get("prices") @@ -557,14 +907,1208 @@ def _portfolio_universe(body: JsonDict) -> JsonDict: if isinstance(raw_prices, dict) else None ) - universe = build_universe( - str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV), prices=prices + scope = str(body.get("scope") or "financials") + financials_csv = str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV) + financials_available = Path(financials_csv).is_file() + financial_rows: list[dict[str, object]] = [] + financials_error = "" + if financials_available: + try: + financial_rows = build_universe( + financials_csv, + prices=prices, + current_yields_csv=str( + body.get("current_yields_csv") or DEFAULT_CURRENT_YIELDS_CSV + ), + ) + except FileNotFoundError: + financials_available = False + except ValueError as exc: + financials_available = False + financials_error = str(exc) + + market = build_market_universe( + financials_csv=financials_csv, + jpx_listed_path=str(body.get("jpx_listed_path") or "local_docs/jpx/listed_issues.csv"), + query="", + scope="all", + limit=10000, + ) + raw_market_rows = market.get("securities") + market_rows = { + str(row.get("ticker") or row.get("code") or ""): row + for row in (raw_market_rows if isinstance(raw_market_rows, list) else []) + if isinstance(row, dict) + } + financial_by_ticker = { + str(row.get("ticker") or ""): row + for row in financial_rows + if isinstance(row, dict) and str(row.get("ticker") or "") + } + enriched: list[dict[str, object]] = [] + for ticker in sorted(set(market_rows) | set(financial_by_ticker)): + row = financial_by_ticker.get(ticker) + meta = market_rows.get(ticker, {}) + market_segment = ( + meta.get("market_segment_label") + or meta.get("market_segment") + or "未取込" + ) + base_row: dict[str, object] = ( + dict(row) + if row is not None + else { + "ticker": ticker, + "name": meta.get("name") or "", + "price": None, + "dividend_latest": None, + "dividend_latest_edinet": None, + "dividend_conservative": None, + "yield_latest": None, + "yield_conservative": None, + "yield_basis": "manual_required", + "current_yield_reconciliation": None, + "safety": 0.0, + "band": None, + "periods": 0, + } + ) + enriched_row = { + **base_row, + "market_segment": market_segment, + "market_segment_raw": meta.get("market_segment_raw", ""), + "market_segment_label": market_segment, + "sector": meta.get("sector", ""), + "is_prime": bool(meta.get("is_prime")), + "is_nikkei225": bool(meta.get("is_nikkei225")), + "has_financials": row is not None, + "selection_mode": "financials" if row is not None else "manual_input", + } + if _market_scope_matches(enriched_row, scope): + enriched.append(enriched_row) + enriched.sort( + key=lambda row: ( + 0 if row.get("has_financials") else 1, + -_as_float(row.get("safety"), 0.0), + str(row.get("ticker") or ""), + ) + ) + market_hint = str(market.get("hint") or "") + normalized_scope = scope.strip().lower() + if not financials_available and normalized_scope in { + "financials", + "edinet", + "financials_available", + }: + market_hint = ( + "財務データがまだ作成されていません。DataタブでEDINET取得/手動保存を行うか、" + "東証プライム財務の自動更新を実行してください。" + ) + elif not financials_available and enriched: + market_hint = ( + "財務データは未取得ですが、JPXの市場区分一覧から銘柄を選択できます。" + "配当・安全性は手入力、またはEDINET財務データ取得後に反映します。" + ) + elif financials_error: + market_hint = f"{market_hint} 財務データの読込エラー: {financials_error}".strip() + return { + "available": bool(enriched), + "universe": enriched, + "count": len(enriched), + "scope": scope, + "source_ref": financials_csv, + "market_sources": market.get("sources"), + "jpx_listed_available": market.get("jpx_listed_available"), + "financials_available": financials_available, + "financials_count": len(financial_rows), + "market_count": len(market_rows), + "hint": market_hint + or ( + "財務データがまだ作成されていません。DataタブでEDINET取得/手動保存を行うか、" + "JPX上場銘柄一覧を取得してください。" + ), + "auto_trading": False, + "call_real_api": False, + } + + +def _market_universe(body: JsonDict) -> JsonDict: + from investment_assistant.investment.universe import build_market_universe + + return build_market_universe( + financials_csv=str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV), + jpx_listed_path=str(body.get("jpx_listed_path") or "local_docs/jpx/listed_issues.csv"), + nikkei225_registry=str( + body.get("nikkei225_registry") or "examples/source_registry_nikkei225_edinet.yaml" + ), + query=str(body.get("query") or ""), + scope=str(body.get("scope") or body.get("universe") or "prime"), + limit=max(_as_int(body.get("limit"), 50), 1), + ) + + +def _jpx_listed_template(_: JsonDict) -> JsonDict: + from investment_assistant.investment.universe import jpx_listed_issue_template + + return jpx_listed_issue_template() + + +def _jpx_listed_import(body: JsonDict) -> JsonDict: + from investment_assistant.ingestion.fetcher import reject_path_traversal + from investment_assistant.investment.universe import ( + DEFAULT_JPX_LISTED_ISSUES_PATH, + parse_jpx_listed_issues_text, + source_manifest, + write_jpx_listed_issues, + ) + + text = str(body.get("csv_text") or body.get("text") or "") + source_ref = "screen_input" + path_value = str(body.get("path") or "").strip() + if not text.strip() and path_value: + path = Path(path_value) + if path.suffix.lower() == ".xls": + raise ApiError( + "JPX公式ファイルは旧Excel形式です。Excel等でCSV/TSVに変換してから取り込んでください。" + ) + text = path.read_text(encoding="utf-8") + source_ref = str(path) + if not text.strip(): + raise ApiError("JPX上場銘柄一覧データを貼り付けるか、CSV/TSV path を指定してください。") + + issues = parse_jpx_listed_issues_text(text, source_ref=source_ref) + output_path = str(body.get("output_path") or DEFAULT_JPX_LISTED_ISSUES_PATH) + saved_path: str | None = None + if _as_bool(body.get("save"), True): + target = reject_path_traversal(output_path) + saved_path = write_jpx_listed_issues(issues, target) + prime_count = sum(1 for issue in issues if issue.is_prime) + return { + "available": True, + "count": len(issues), + "prime_count": prime_count, + "saved": saved_path is not None, + "saved_path": saved_path, + "sample": [issue.to_dict() for issue in issues[:20]], + "sources": source_manifest(), + "disclaimer": "市場区分は銘柄選択補助です。投資助言や売買推奨ではありません。", + "auto_trading": False, + "call_real_api": False, + } + + +def _jpx_listed_download(body: JsonDict) -> JsonDict: + from investment_assistant.ingestion.fetcher import SafeFetcher, reject_path_traversal + from investment_assistant.investment.universe import ( + JPX_LISTED_ISSUES_FILE_URL, + JPX_LISTED_ISSUES_PAGE_URL, + source_manifest, + ) + + url = str(body.get("url") or JPX_LISTED_ISSUES_FILE_URL) + output_path = str(body.get("output_path") or "local_docs/jpx/data_j.xls") + fetcher = SafeFetcher(timeout_seconds=30.0) + decision = fetcher.robots.can_fetch(url) + if not decision.allowed: + return { + "available": False, + "downloaded": False, + "source_url": url, + "robots_url": decision.robots_url, + "reason": decision.reason, + "hint": "robots.txtで許可されていないため自動取得しません。", + "sources": source_manifest(), + "auto_trading": False, + "call_real_api": False, + } + response = fetcher.transport.get( + url, + timeout_seconds=30.0, + user_agent=fetcher.user_agent, + ) + if response.status_code >= 400: + raise ApiError(f"JPXファイル取得に失敗しました: status={response.status_code}") + target = reject_path_traversal(output_path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(response.body) + is_legacy_xls = response.body.startswith(bytes.fromhex("d0cf11e0a1b11ae1")) + return { + "available": True, + "downloaded": True, + "saved_path": str(target), + "bytes": len(response.body), + "status_code": response.status_code, + "source_url": url, + "source_page_url": JPX_LISTED_ISSUES_PAGE_URL, + "robots_url": decision.robots_url, + "parse_supported": not is_legacy_xls, + "file_format": "legacy_xls" if is_legacy_xls else "text_or_unknown", + "hint": ( + "公式ファイルを取得しました。旧Excel形式のため、Excel等でCSV/TSVに変換して" + "市場区分データとして保存してください。" + if is_legacy_xls + else "取得ファイルを市場区分データとして取り込めます。" + ), + "sources": source_manifest(), + "auto_trading": False, + "call_real_api": False, + } + + +def _jpx_listed_download_import(body: JsonDict) -> JsonDict: + from investment_assistant.ingestion.fetcher import reject_path_traversal + from investment_assistant.investment.jpx_excel import ( + JpxExcelConversionError, + convert_legacy_xls_to_csv_with_excel, + ) + from investment_assistant.investment.universe import ( + DEFAULT_JPX_LISTED_ISSUES_PATH, + source_manifest, + ) + + raw_path = str(body.get("path") or "").strip() + output_path = str(body.get("output_path") or DEFAULT_JPX_LISTED_ISSUES_PATH) + converted_output_path = str( + body.get("converted_output_path") or "local_docs/jpx/data_j_converted.csv" + ) + downloaded = False + download_result: JsonDict | None = None + source_path: Path + + if raw_path: + source_path = reject_path_traversal(raw_path) + else: + download_result = _jpx_listed_download( + { + "url": body.get("url"), + "output_path": body.get("download_output_path") or "local_docs/jpx/data_j.xls", + } + ) + if not download_result.get("downloaded"): + return { + "available": False, + "downloaded": False, + "converted": False, + "imported": False, + "download": download_result, + "sources": source_manifest(), + "hint": str( + download_result.get("hint") + or "JPX公式ファイルを取得できませんでした。" + ), + "auto_trading": False, + "call_real_api": False, + } + downloaded = True + source_path = reject_path_traversal(str(download_result.get("saved_path") or "")) + + import_path = source_path + converted = False + conversion_error: str | None = None + if source_path.suffix.lower() == ".xls": + try: + import_path = reject_path_traversal(converted_output_path) + convert_legacy_xls_to_csv_with_excel(source_path, import_path) + converted = True + except JpxExcelConversionError as exc: + conversion_error = str(exc) + return { + "available": False, + "downloaded": downloaded, + "converted": False, + "imported": False, + "download": download_result, + "source_path": str(source_path), + "converted_path": str(import_path), + "conversion_error": conversion_error, + "sources": source_manifest(), + "hint": ( + "JPX公式ファイルは取得できましたが、Excel自動変換に失敗しました。" + "Excel等でCSV/TSVへ変換してから手動取込してください。" + ), + "auto_trading": False, + "call_real_api": False, + } + + imported = _jpx_listed_import( + { + "path": str(import_path), + "output_path": output_path, + "save": _as_bool(body.get("save"), True), + } + ) + return { + "available": True, + "downloaded": downloaded, + "converted": converted, + "imported": True, + "download": download_result, + "source_path": str(source_path), + "converted_path": str(import_path) if converted else None, + "saved_path": imported.get("saved_path"), + "count": imported.get("count"), + "prime_count": imported.get("prime_count"), + "sample": imported.get("sample"), + "sources": imported.get("sources"), + "disclaimer": imported.get("disclaimer"), + "hint": ( + "JPX公式ファイルを取得し、市場区分データへ反映しました。" + if downloaded + else "JPX市場区分データを反映しました。" + ), + "auto_trading": False, + "call_real_api": False, + } + + +def _companies_master_status(body: JsonDict) -> JsonDict: + path = Path(str(body.get("path") or "local_docs/company_master/company_master.csv")) + if not path.is_file(): + return { + "available": False, + "path": str(path), + "count": 0, + "hint": "会社情報マスターがまだありません。Dataタブで会社情報を取得してください。", + "auto_trading": False, + "call_real_api": False, + } + rows = _read_company_master_rows(path) + modified_at = datetime.fromtimestamp(path.stat().st_mtime, UTC).isoformat() + return { + "available": True, + "path": str(path), + "count": len(rows), + "company_count": sum(1 for row in rows if row.get("is_company") == "true"), + "domestic_stock_count": sum( + 1 for row in rows if row.get("entity_type") == "domestic_stock" + ), + "prime_count": sum(1 for row in rows if row.get("is_prime") == "true"), + "nikkei225_count": sum(1 for row in rows if row.get("is_nikkei225") == "true"), + "financials_count": sum(1 for row in rows if row.get("has_financials") == "true"), + "modified_at": modified_at, + "sample": rows[:20], + "auto_trading": False, + "call_real_api": False, + } + + +def _companies_master_refresh(body: JsonDict) -> JsonDict: + from investment_assistant.edinet.registry import build_edinet_targets_from_registry + from investment_assistant.ingestion.fetcher import reject_path_traversal + from investment_assistant.investment.universe import ( + DEFAULT_JPX_LISTED_ISSUES_PATH, + DEFAULT_NIKKEI225_REGISTRY, + load_jpx_listed_issues, + source_manifest, + ) + + jpx_listed_path = str(body.get("jpx_listed_path") or DEFAULT_JPX_LISTED_ISSUES_PATH) + output_path = str(body.get("output_path") or "local_docs/company_master/company_master.csv") + financials_csv = str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV) + nikkei225_registry = str(body.get("nikkei225_registry") or DEFAULT_NIKKEI225_REGISTRY) + jpx_refresh: JsonDict | None = None + if _as_bool(body.get("refresh_jpx"), False): + jpx_refresh = _jpx_listed_download_import( + { + "output_path": jpx_listed_path, + "save": True, + } + ) + + issues = load_jpx_listed_issues(jpx_listed_path) + if not issues: + return { + "available": False, + "saved": False, + "path": output_path, + "jpx_listed_path": jpx_listed_path, + "count": 0, + "jpx_refresh": jpx_refresh, + "sources": source_manifest(), + "hint": ( + "JPX上場銘柄一覧が見つかりません。" + "先に公式データ取得またはCSV取込を行ってください。" + ), + "auto_trading": False, + "call_real_api": False, + } + + financial_periods = _financials_periods_by_ticker(financials_csv) + try: + nikkei225 = { + _normalize_ticker(target.ticker) + for target in build_edinet_targets_from_registry(nikkei225_registry) + } + except (OSError, ValueError): + nikkei225 = set() + + rows = [ + _company_master_row( + issue, + financial_periods=financial_periods, + nikkei225=nikkei225, + ) + for issue in sorted(issues, key=lambda item: item.code) + ] + target = reject_path_traversal(output_path) + target.parent.mkdir(parents=True, exist_ok=True) + _write_company_master_rows(rows, target) + return { + "available": True, + "saved": True, + "path": str(target), + "jpx_listed_path": jpx_listed_path, + "financials_csv": financials_csv, + "nikkei225_registry": nikkei225_registry, + "count": len(rows), + "company_count": sum(1 for row in rows if row["is_company"]), + "domestic_stock_count": sum( + 1 for row in rows if row["entity_type"] == "domestic_stock" + ), + "prime_count": sum(1 for row in rows if row["is_prime"]), + "nikkei225_count": sum(1 for row in rows if row["is_nikkei225"]), + "financials_count": sum(1 for row in rows if row["has_financials"]), + "jpx_refresh": jpx_refresh, + "sample": rows[:20], + "sources": source_manifest(), + "hint": ( + "JPX公式の上場銘柄一覧を会社情報マスターとして保存しました。" + "売買推奨ではなく、銘柄検索・補完・レポートの基礎データとして使います。" + ), + "auto_trading": False, + "call_real_api": False, + } + + +def _financials_periods_by_ticker(financials_csv: str) -> dict[str, int]: + path = Path(financials_csv) + if not path.is_file(): + return {} + periods: dict[str, set[int]] = {} + try: + for point in load_financials(path): + ticker = _normalize_ticker(point.ticker) + if ticker: + periods.setdefault(ticker, set()).add(point.fiscal_year) + except (OSError, ValueError): + return {} + return {ticker: len(years) for ticker, years in periods.items()} + + +def _company_master_row( + issue: Any, + *, + financial_periods: dict[str, int], + nikkei225: set[str], +) -> JsonDict: + ticker = _normalize_ticker(getattr(issue, "code", "")) + segment_raw = str(getattr(issue, "market_segment", "") or "") + segment = _display_market_segment(segment_raw) + entity_type = _company_entity_type(segment_raw) + periods = financial_periods.get(ticker, 0) + return { + "ticker": ticker, + "name": str(getattr(issue, "name", "") or ""), + "market_segment": segment, + "market_segment_raw": segment_raw, + "sector": str(getattr(issue, "sector", "") or ""), + "as_of": str(getattr(issue, "as_of", "") or ""), + "entity_type": entity_type, + "is_company": entity_type in {"domestic_stock", "foreign_stock"}, + "is_domestic_stock": entity_type == "domestic_stock", + "is_prime": _is_prime_segment(segment_raw), + "is_standard": _is_standard_segment(segment_raw), + "is_growth": _is_growth_segment(segment_raw), + "is_nikkei225": ticker in nikkei225, + "has_financials": periods > 0, + "financial_periods": periods, + "source_ref": str(getattr(issue, "source_ref", "") or ""), + } + + +def _company_entity_type(segment: str) -> str: + text = str(segment or "") + lowered = text.lower() + if "ETF" in text or "ETN" in text or "etf" in lowered or "etn" in lowered: + return "etf_etn" + if "REIT" in text or "reit" in lowered: + return "reit" + if "外国株式" in text or "foreign stock" in lowered: + return "foreign_stock" + if _is_domestic_stock_segment(text): + return "domestic_stock" + if "PRO Market" in text or "pro market" in lowered: + return "pro_market" + return "other" + + +def _display_market_segment(segment: str) -> str: + return str(segment or "").replace("内国株式", "国内株式") + + +_COMPANY_MASTER_COLUMNS = [ + "ticker", + "name", + "market_segment", + "market_segment_raw", + "sector", + "as_of", + "entity_type", + "is_company", + "is_domestic_stock", + "is_prime", + "is_standard", + "is_growth", + "is_nikkei225", + "has_financials", + "financial_periods", + "source_ref", +] + + +def _write_company_master_rows(rows: list[JsonDict], path: Path) -> None: + with path.open("w", newline="", encoding="utf-8-sig") as handle: + writer = csv.DictWriter(handle, fieldnames=_COMPANY_MASTER_COLUMNS) + writer.writeheader() + for row in rows: + writer.writerow( + { + column: _company_master_csv_value(row.get(column)) + for column in _COMPANY_MASTER_COLUMNS + } + ) + + +def _read_company_master_rows(path: Path) -> list[JsonDict]: + with path.open(newline="", encoding="utf-8-sig") as handle: + return [dict(row) for row in csv.DictReader(handle)] + + +def _company_master_csv_value(value: object) -> object: + if isinstance(value, bool): + return "true" if value else "false" + return value + + +def _financials_prime_registry(body: JsonDict) -> JsonDict: + return _financials_jpx_registry( + { + **body, + "scope": "prime", + "registry_path": body.get("registry_path") + or "local_docs/edinet/source_registry_tse_prime_edinet.yaml", + } + ) + + +def _financials_listed_registry(body: JsonDict) -> JsonDict: + return _financials_jpx_registry( + { + **body, + "scope": body.get("scope") or "domestic_stocks", + "registry_path": body.get("registry_path") + or "local_docs/edinet/source_registry_all_domestic_edinet.yaml", + } + ) + + +def _financials_jpx_registry(body: JsonDict) -> JsonDict: + from investment_assistant.ingestion.fetcher import reject_path_traversal + from investment_assistant.investment.universe import ( + DEFAULT_JPX_LISTED_ISSUES_PATH, + load_jpx_listed_issues, + source_manifest, ) - return {"universe": universe, "count": len(universe)} + + jpx_listed_path = str(body.get("jpx_listed_path") or DEFAULT_JPX_LISTED_ISSUES_PATH) + output_path = str( + body.get("registry_path") or "local_docs/edinet/source_registry_all_domestic_edinet.yaml" + ) + scope = str(body.get("scope") or "domestic_stocks") + scope_label = _financial_registry_scope_label(scope) + max_targets = _as_int(body.get("max_targets"), 0) + max_periods = max(_as_int(body.get("max_periods"), 1), 1) + source_issues = load_jpx_listed_issues(jpx_listed_path) + issues = [ + issue + for issue in source_issues + if _issue_matches_financial_registry_scope(issue, scope) + ] + prime_count = sum(1 for issue in source_issues if bool(issue.is_prime)) + issues.sort(key=lambda issue: issue.code) + selected = issues[:max_targets] if max_targets > 0 else issues + if not selected: + return { + "available": False, + "saved": False, + "registry_path": output_path, + "jpx_listed_path": jpx_listed_path, + "scope": scope, + "registry_scope_label": scope_label, + "count": 0, + "eligible_count": 0, + "total_source_count": len(source_issues), + "scope_total_count": 0, + "total_prime_count": prime_count, + "sources": source_manifest(), + "hint": ( + f"{scope_label}の銘柄が見つかりません。先にJPX公式の東証上場銘柄一覧を取得し、" + "市場区分データとして保存してください。" + ), + "auto_trading": False, + "call_real_api": False, + } + + registry_text = _edinet_registry_yaml( + selected, + max_periods=max_periods, + title=f"{scope_label} EDINET financials registry", + ) + saved_path: str | None = None + if _as_bool(body.get("save"), True): + target = reject_path_traversal(output_path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(registry_text, encoding="utf-8") + saved_path = str(target) + payload: JsonDict = { + "available": True, + "saved": saved_path is not None, + "registry_path": saved_path or output_path, + "jpx_listed_path": jpx_listed_path, + "scope": scope, + "registry_scope_label": scope_label, + "count": len(selected), + "eligible_count": len(issues), + "scope_total_count": len(issues), + "total_source_count": len(source_issues), + "total_prime_count": prime_count, + "max_periods": max_periods, + "sample": [issue.to_dict() for issue in selected[:20]], + "sources": source_manifest(), + "hint": ( + f"{scope_label}からEDINET取得用registryを生成しました。" + "このregistryを使って財務CSVを更新できます。" + ), + "auto_trading": False, + "call_real_api": False, + } + if _as_bool(body.get("include_csv_text"), False) or not saved_path: + payload["csv_text"] = registry_text + return payload + + +def _financials_prime_refresh(body: JsonDict) -> JsonDict: + return _financials_jpx_refresh( + { + **body, + "scope": "prime", + "registry_path": body.get("registry_path") + or "local_docs/edinet/source_registry_tse_prime_edinet.yaml", + } + ) + + +def _financials_listed_refresh(body: JsonDict) -> JsonDict: + return _financials_jpx_refresh( + { + **body, + "scope": body.get("scope") or "domestic_stocks", + "registry_path": body.get("registry_path") + or "local_docs/edinet/source_registry_all_domestic_edinet.yaml", + } + ) + + +def _financials_jpx_refresh(body: JsonDict) -> JsonDict: + registry = _financials_jpx_registry( + { + **body, + "save": True, + } + ) + if not registry.get("available"): + return registry + result = _financials_refresh( + { + **body, + "registry_path": registry.get("registry_path"), + } + ) + result["prime_registry"] = { + "registry_path": registry.get("registry_path"), + "scope": registry.get("scope"), + "registry_scope_label": registry.get("registry_scope_label"), + "count": registry.get("count"), + "eligible_count": registry.get("eligible_count"), + "total_prime_count": registry.get("total_prime_count"), + "jpx_listed_path": registry.get("jpx_listed_path"), + } + result["jpx_registry"] = result["prime_registry"] + if result.get("financials_updated") is not False: + result["hint"] = ( + f"{registry.get('registry_scope_label')}のregistryを生成し、" + "EDINET公式APIから財務データを更新しました。" + "以後の保有分析・候補抽出・シミュレーションは更新後CSVを参照します。" + ) + return result + + +def _financials_prime_refresh_async(body: JsonDict) -> JsonDict: + job_id = JOBS.start("financials-prime-refresh", lambda: _financials_prime_refresh(body)) + return {"job_id": job_id, "status": "running", "kind": "financials-prime-refresh"} + + +def _financials_listed_refresh_async(body: JsonDict) -> JsonDict: + job_id = JOBS.start("financials-listed-refresh", lambda: _financials_listed_refresh(body)) + return {"job_id": job_id, "status": "running", "kind": "financials-listed-refresh"} + + +def _financials_missing_registry(body: JsonDict) -> JsonDict: + from investment_assistant.edinet.registry import build_edinet_targets_from_registry + from investment_assistant.ingestion.fetcher import reject_path_traversal + + registry_path = str( + body.get("registry_path") or "local_docs/edinet/source_registry_all_domestic_edinet.yaml" + ) + output_dir = str(body.get("output_dir") or "local_docs/edinet") + financials_csv = str( + body.get("financials_csv") or Path(output_dir) / "financials.csv" + ) + missing_registry_path = str( + body.get("missing_registry_path") + or body.get("output_path") + or "local_docs/edinet/source_registry_missing_edinet.yaml" + ) + max_targets = _as_int(body.get("max_targets"), 200) + max_periods = max(_as_int(body.get("max_periods"), 1), 1) + + targets = build_edinet_targets_from_registry(registry_path) + existing_tickers = _financials_existing_tickers(financials_csv) + missing = [ + target + for target in targets + if _normalize_ticker(target.ticker) not in existing_tickers + ] + selected = missing[:max_targets] if max_targets > 0 else missing + + registry_text = _edinet_targets_registry_yaml( + selected, + max_periods=max_periods, + title="Missing EDINET financials registry", + ) + saved_path: str | None = None + if _as_bool(body.get("save"), True): + target_path = reject_path_traversal(missing_registry_path) + target_path.parent.mkdir(parents=True, exist_ok=True) + target_path.write_text(registry_text, encoding="utf-8") + saved_path = str(target_path) + + payload: JsonDict = { + "available": bool(targets), + "saved": saved_path is not None, + "registry_path": saved_path or missing_registry_path, + "base_registry_path": registry_path, + "financials_csv": financials_csv, + "financials_csv_exists": Path(financials_csv).is_file(), + "registry_count": len(targets), + "existing_count": len(existing_tickers), + "missing_count": len(missing), + "count": len(selected), + "max_targets": max_targets, + "max_periods": max_periods, + "sample": [_edinet_target_to_dict(target) for target in selected[:20]], + "hint": ( + "未取得の証券コードだけを抽出しました。" + "このregistryを使うと、既存の財務CSVを壊さず差分だけ補完できます。" + ), + "auto_trading": False, + "call_real_api": False, + } + if _as_bool(body.get("include_csv_text"), False) or not saved_path: + payload["csv_text"] = registry_text + return payload + + +def _financials_missing_refresh(body: JsonDict) -> JsonDict: + registry = _financials_missing_registry({**body, "save": True}) + if not registry.get("available"): + registry["financials_updated"] = False + registry["hint"] = ( + "補完元のregistryが見つからないか、対象が空です。" + "先に全社registryを生成してください。" + ) + return registry + if _as_int(registry.get("count"), 0) <= 0: + registry["financials_updated"] = False + registry["mode"] = "missing_already_complete" + registry["hint"] = ( + "未取得の証券コードはありません。" + "現在の財務CSVは対象registryをすべてカバーしています。" + ) + return registry + + result = _financials_refresh( + { + **body, + "registry_path": registry.get("registry_path"), + } + ) + result["missing_registry"] = { + "registry_path": registry.get("registry_path"), + "base_registry_path": registry.get("base_registry_path"), + "financials_csv": registry.get("financials_csv"), + "registry_count": registry.get("registry_count"), + "existing_count": registry.get("existing_count"), + "missing_count": registry.get("missing_count"), + "count": registry.get("count"), + "max_targets": registry.get("max_targets"), + } + if result.get("financials_updated") is not False: + result["hint"] = ( + "未取得の証券コードだけを対象にEDINET財務データを補完しました。" + "取得後の財務CSVは既存データとマージされています。" + ) + return result + + +def _financials_missing_refresh_async(body: JsonDict) -> JsonDict: + job_id = JOBS.start("financials-missing-refresh", lambda: _financials_missing_refresh(body)) + return {"job_id": job_id, "status": "running", "kind": "financials-missing-refresh"} + + +def _financials_missing_sources(body: JsonDict) -> JsonDict: + missing = _financials_missing_registry( + { + **body, + "save": _as_bool(body.get("save_registry"), True), + } + ) + sample = missing.get("sample") + sample_targets = ( + [item for item in sample if isinstance(item, dict)] + if isinstance(sample, list) + else [] + ) + output_dir = str(body.get("evidence_output_dir") or "local_docs/disclosure") + sources = _missing_official_evidence_sources( + sample_targets, + output_dir=output_dir, + preview_chars=_as_int(body.get("preview_chars"), 800), + ) + return { + "available": bool(missing.get("available")), + "mode": "source_agnostic_missing_plan", + "registry": missing, + "sources": sources, + "sources_count": len(sources), + "structured_sources": [ + { + "name": "EDINET API", + "role": "structured_financial_csv", + "requires_api_key": True, + "status": "preferred_for_deterministic_metrics", + }, + { + "name": "user_financial_csv", + "role": "structured_financial_csv", + "requires_api_key": False, + "status": "manual_or_verified_import", + }, + ], + "evidence_sources": [ + { + "name": "TDnet", + "role": "official_disclosure_rag", + "structured_csv": False, + }, + { + "name": "JPX", + "role": "market_segment_and_listing_evidence", + "structured_csv": False, + }, + { + "name": "issuer_ir_pdf_html", + "role": "manual_url_or_file_evidence", + "structured_csv": False, + }, + ], + "manual_inputs": { + "csv": "Dataタブの手動EDINET財務データに貼り付け/保存", + "pdf_html": "URL取得またはテキスト化してRAG登録", + }, + "hint": ( + "補完元はEDINETだけに限定しません。" + "数値CSVはEDINET APIまたは確認済みCSVで補完し、" + "TDnet/JPX/企業IR/PDF/HTMLはRAG根拠として補完します。" + ), + "auto_trading": False, + "call_real_api": False, + } + + +def _financials_missing_evidence_refresh(body: JsonDict) -> JsonDict: + plan = _financials_missing_sources(body) + sources = plan.get("sources") + source_list = ( + [item for item in sources if isinstance(item, dict)] + if isinstance(sources, list) + else [] + ) + if not source_list: + return { + **plan, + "status": "blocked", + "evidence_updated": False, + "financials_updated": False, + "hint": "非EDINET補完に使える公式ソースがありません。補完元を確認してください。", + } + fetch_result = _fetch_job_auto( + { + "sources": source_list, + "db_path": str(body.get("db_path") or DEFAULT_RAG_DB_PATH), + "index_path": str(body.get("index_path") or "local_docs"), + "index_after_fetch": _as_bool(body.get("index_after_fetch"), True), + } + ) + return { + **plan, + "mode": "official_disclosure_rag", + "status": fetch_result.get("status"), + "evidence_updated": fetch_result.get("status") == "completed", + "financials_updated": False, + "fetch": fetch_result, + "hint": ( + "EDINETに限らず、TDnet/JPXなど公式開示ソースをRAG根拠として取得しました。" + "構造化財務CSVの補完は、EDINET APIまたは確認済みCSV取込で行ってください。" + ), + "auto_trading": False, + "call_real_api": False, + } + + +def _financials_missing_evidence_refresh_async(body: JsonDict) -> JsonDict: + job_id = JOBS.start( + "financials-missing-evidence-refresh", + lambda: _financials_missing_evidence_refresh(body), + ) + return { + "job_id": job_id, + "status": "running", + "kind": "financials-missing-evidence-refresh", + } + + +def _financials_missing_auto_refresh(body: JsonDict) -> JsonDict: + from investment_assistant.edinet.client import API_KEY_ENV_VAR + + _ensure_env_from_dotenv(API_KEY_ENV_VAR) + if os.getenv(API_KEY_ENV_VAR, "").strip(): + result = _financials_missing_refresh(body) + result["fallback_sources_available"] = True + result["source_strategy"] = "structured_edinet_api" + return result + + result = _financials_missing_evidence_refresh(body) + result["source_strategy"] = "official_disclosure_rag" + return result + + +def _financials_missing_auto_refresh_async(body: JsonDict) -> JsonDict: + job_id = JOBS.start( + "financials-missing-auto-refresh", + lambda: _financials_missing_auto_refresh(body), + ) + return { + "job_id": job_id, + "status": "running", + "kind": "financials-missing-auto-refresh", + } + + +def _missing_official_evidence_sources( + sample_targets: list[JsonDict], + *, + output_dir: str, + preview_chars: int, +) -> list[JsonDict]: + tickers = [str(item.get("ticker") or "").strip() for item in sample_targets] + companies = [ + str(item.get("company") or item.get("name") or "").strip() + for item in sample_targets + ] + query_tail = " ".join([*tickers[:20], *companies[:10]]).strip() + base = Path(output_dir) + return [ + { + "name": "tdnet_missing_financials", + "url": "https://www.release.tdnet.info/inbs/I_main_00.html", + "output_path": str(base / "missing_tdnet_disclosures.txt"), + "query_hint": f"TDnet 決算短信 配当 修正 {query_tail}".strip(), + "extract_text": True, + "include_metadata": True, + "preview_chars": preview_chars, + }, + { + "name": "jpx_listing_evidence", + "url": "https://www.jpx.co.jp/markets/statistics-equities/misc/01.html", + "output_path": str(base / "missing_jpx_listing_evidence.txt"), + "query_hint": f"JPX 上場銘柄一覧 市場区分 {query_tail}".strip(), + "extract_text": True, + "include_metadata": True, + "preview_chars": preview_chars, + }, + ] + + +def _financials_existing_tickers(financials_csv: str) -> set[str]: + path = Path(financials_csv) + if not path.is_file(): + return set() + try: + return {_normalize_ticker(point.ticker) for point in load_financials(path)} + except (OSError, ValueError): + return set() + + +def _normalize_ticker(value: object) -> str: + return str(value or "").strip().upper() + + +def _edinet_target_to_dict(target: Any) -> JsonDict: + return { + "ticker": getattr(target, "ticker", ""), + "company": getattr(target, "company", None), + "name": getattr(target, "name", ""), + "max_periods": getattr(target, "max_periods", 1), + } + + +def _issue_matches_financial_registry_scope(issue: Any, scope: str) -> bool: + normalized = scope.strip().lower() + segment = str(issue.market_segment or "") + if normalized in {"prime", "tse_prime", "tosho_prime"}: + return bool(issue.is_prime) + if normalized in {"domestic_stocks", "all_domestic", "listed_companies", "all"}: + return _is_domestic_stock_segment(segment) + if normalized in {"standard", "tse_standard"}: + return _is_standard_segment(segment) and _is_domestic_stock_segment(segment) + if normalized in {"growth", "tse_growth"}: + return _is_growth_segment(segment) and _is_domestic_stock_segment(segment) + return _is_domestic_stock_segment(segment) + + +def _is_domestic_stock_segment(segment: str) -> bool: + text = str(segment or "") + normalized = text.lower() + has_domestic_stock = ( + "内国株式" in text + or "国内株式" in text + or "domestic stock" in normalized + or "domestic common stock" in normalized + ) + return ( + has_domestic_stock + and ( + _is_prime_segment(segment) + or _is_standard_segment(segment) + or _is_growth_segment(segment) + ) + ) + + +def _is_prime_segment(segment: str) -> bool: + text = str(segment or "") + normalized = text.lower() + return "プライム" in text or "prime" in normalized + + +def _is_standard_segment(segment: str) -> bool: + text = str(segment or "") + normalized = text.lower() + return "スタンダード" in text or "standard" in normalized + + +def _is_growth_segment(segment: str) -> bool: + text = str(segment or "") + normalized = text.lower() + return "グロース" in text or "growth" in normalized + + +def _financial_registry_scope_label(scope: str) -> str: + normalized = scope.strip().lower() + if normalized in {"prime", "tse_prime", "tosho_prime"}: + return "東証プライム" + if normalized in {"standard", "tse_standard"}: + return "東証スタンダード" + if normalized in {"growth", "tse_growth"}: + return "東証グロース" + return "全社（国内株式）" + + +def _edinet_registry_yaml( + issues: list[Any], + *, + max_periods: int, + title: str, +) -> str: + lines = [ + f"# {title}", + "# Generated from JPX listed issue data. Used only for EDINET public API ingestion.", + "sources:", + ] + for issue in issues: + name = f"{issue.code}_{issue.name}".strip("_") + lines.extend( + [ + f" - name: {_yaml_scalar(name)}", + f" ticker: {_yaml_scalar(issue.code)}", + f" company: {_yaml_scalar(issue.name)}", + ' source_type: "public_api"', + ' provider: "edinet"', + " allowed: true", + f" max_periods: {max_periods}", + ] + ) + return "\n".join(lines) + "\n" + + +def _edinet_targets_registry_yaml( + targets: list[Any], + *, + max_periods: int, + title: str, +) -> str: + lines = [ + f"# {title}", + "# Generated from an EDINET registry diff. Used only for missing public API ingestion.", + "sources:", + ] + for target in targets: + ticker = str(getattr(target, "ticker", "") or "").strip() + company = str(getattr(target, "company", "") or "").strip() + name = str(getattr(target, "name", "") or f"{ticker}_{company}".strip("_")) + lines.extend( + [ + f" - name: {_yaml_scalar(name)}", + f" ticker: {_yaml_scalar(ticker)}", + f" company: {_yaml_scalar(company)}", + ' source_type: "public_api"', + ' provider: "edinet"', + " allowed: true", + f" max_periods: {max_periods}", + ] + ) + return "\n".join(lines) + "\n" def _market_prices(body: JsonDict) -> JsonDict: - from investment_assistant.investment.provider_policy import ensure_provider_allowed + from investment_assistant.investment.provider_policy import ( + ensure_provider_allowed, + provider_policy, + ) + from investment_assistant.jquants.client import JQuantsApiError, JQuantsClient + from investment_assistant.portfolio.bar_store import DEFAULT_DAILY_BARS_CSV + from investment_assistant.portfolio.price_store import DEFAULT_CURRENT_PRICES_CSV from investment_assistant.portfolio.prices import fetch_prices raw = body.get("tickers") @@ -578,12 +2122,542 @@ def _market_prices(body: JsonDict) -> JsonDict: try: policy = ensure_provider_allowed(provider_id, runtime_mode=runtime_mode) except ValueError as exc: + if _as_bool(body.get("allow_cache_on_policy_block"), False): + policy = provider_policy(provider_id, runtime_mode=runtime_mode) + result = { + "prices": {ticker: None for ticker in tickers}, + "notes": {ticker: str(exc) for ticker in tickers}, + "provider_id": provider_id, + "auto_trading": False, + "call_real_api": False, + "provider_blocked": True, + } + _apply_market_price_store( + result, + tickers, + provider_id=provider_id, + path=Path(str(body.get("price_store_path") or DEFAULT_CURRENT_PRICES_CSV)), + daily_bars_path=Path(str(body.get("daily_bars_path") or DEFAULT_DAILY_BARS_CSV)), + ) + price_store = result.get("price_store") + cache_used = ( + price_store.get("cache_used") + if isinstance(price_store, dict) + else [] + ) + result["provider_policy"] = policy.to_dict() + result["price_store_cache_hit"] = bool(cache_used) + return result raise ApiError(str(exc), status=400) from exc - result = fetch_prices(tickers) + if provider_id.strip().lower() == _JQUANTS_PROVIDER_ID: + try: + result = JQuantsClient().fetch_latest_prices( + tickers, + date=str(body.get("date") or "").strip() or None, + lookback_days=_as_int(body.get("lookback_days"), 14), + ) + except JQuantsApiError as exc: + raise ApiError(str(exc), status=400) from exc + else: + result = fetch_prices(tickers) + result["provider_id"] = provider_id + result["auto_trading"] = False + result["call_real_api"] = provider_id.strip().lower() != "user_csv" + if _as_bool(body.get("use_price_store"), True): + _apply_market_price_store( + result, + tickers, + provider_id=provider_id, + path=Path(str(body.get("price_store_path") or DEFAULT_CURRENT_PRICES_CSV)), + daily_bars_path=Path(str(body.get("daily_bars_path") or DEFAULT_DAILY_BARS_CSV)), + ) result["provider_policy"] = policy.to_dict() return result +def _market_prices_import(body: JsonDict) -> JsonDict: + from investment_assistant.investment.provider_policy import provider_policy + from investment_assistant.portfolio.price_store import ( + DEFAULT_CURRENT_PRICES_CSV, + MarketPriceFact, + load_current_prices, + merge_market_price_facts, + parse_current_prices_csv, + save_current_prices, + ) + + csv_text = str(body.get("csv_text") or "").strip() + if not csv_text: + raise ApiError("csv_text is required") + provider_id = str(body.get("provider_id") or "yahoo_finance_manual").strip() + source_ref = str(body.get("source_ref") or "Yahoo Finance manual input").strip() + path = Path(str(body.get("price_store_path") or DEFAULT_CURRENT_PRICES_CSV)) + parsed = list(parse_current_prices_csv(csv_text).values()) + if provider_id or source_ref: + parsed = [ + MarketPriceFact( + ticker=fact.ticker, + price=fact.price, + as_of=fact.as_of, + provider_id=provider_id or fact.provider_id, + source_ref=source_ref or fact.source_ref, + note=fact.note or "manual_price_import", + ) + for fact in parsed + ] + if not parsed: + raise ApiError("no valid price rows found") + existing = load_current_prices(path) + merged = merge_market_price_facts(existing.values(), parsed) + saved_path = save_current_prices(merged, path) + return { + "available": True, + "count": len(parsed), + "tickers": [fact.ticker for fact in parsed], + "saved_path": saved_path, + "total_count": len(merged), + "provider_id": provider_id, + "provider_policy": provider_policy(provider_id, runtime_mode="production").to_dict(), + "auto_trading": False, + "call_real_api": False, + } + + +def _market_prices_import_file(body: JsonDict) -> JsonDict: + from investment_assistant.ingestion.fetcher import reject_path_traversal + from investment_assistant.portfolio.price_store import ( + DEFAULT_CURRENT_PRICES_CSV, + DEFAULT_YAHOO_PRICE_INBOX_CSV, + ) + + source_path = reject_path_traversal( + str(body.get("path") or body.get("input_path") or DEFAULT_YAHOO_PRICE_INBOX_CSV) + ) + if not source_path.is_file(): + if _as_bool(body.get("allow_missing"), True): + return { + "available": False, + "status": "missing", + "input_path": str(source_path), + "price_store_path": str(body.get("price_store_path") or DEFAULT_CURRENT_PRICES_CSV), + "provider_id": str(body.get("provider_id") or "yahoo_finance_manual"), + "auto_trading": False, + "call_real_api": False, + } + raise ApiError(f"price import file not found: {source_path}", status=404) + + nested = dict(body) + nested["csv_text"] = source_path.read_text(encoding="utf-8-sig") + nested.setdefault("provider_id", "yahoo_finance_manual") + nested.setdefault("source_ref", str(source_path)) + result = _market_prices_import(nested) + result["status"] = "imported" + result["input_path"] = str(source_path) + return result + + +def _market_bars(body: JsonDict) -> JsonDict: + from investment_assistant.investment.provider_policy import ( + ensure_provider_allowed, + provider_policy, + ) + from investment_assistant.jquants.client import JQuantsClient + from investment_assistant.portfolio.bar_store import DEFAULT_DAILY_BARS_CSV + from investment_assistant.portfolio.price_store import DEFAULT_CURRENT_PRICES_CSV + + raw = body.get("tickers") + tickers = [str(t) for t in raw] if isinstance(raw, list) else [] + provider_id = str(body.get("provider_id") or _JQUANTS_PROVIDER_ID) + runtime_mode = str( + body.get("runtime_mode") + or os.getenv("INVESTMENT_ASSISTANT_RUNTIME_MODE") + or "development" + ) + store_path = Path(str(body.get("bar_store_path") or DEFAULT_DAILY_BARS_CSV)) + try: + policy = ensure_provider_allowed(provider_id, runtime_mode=runtime_mode) + except ValueError as exc: + if _as_bool(body.get("allow_cache_on_policy_block"), False): + policy = provider_policy(provider_id, runtime_mode=runtime_mode) + result = { + "bars": [], + "notes": {ticker: str(exc) for ticker in tickers}, + "provider_id": provider_id, + "provider_blocked": True, + "auto_trading": False, + "call_real_api": False, + } + _apply_daily_bar_store( + result, + tickers, + path=store_path, + price_path=Path(str(body.get("price_store_path") or DEFAULT_CURRENT_PRICES_CSV)), + lookback_days=_as_int(body.get("lookback_days"), 30), + ) + result["provider_policy"] = policy.to_dict() + return result + raise ApiError(str(exc), status=400) from exc + + if provider_id.strip().lower() != _JQUANTS_PROVIDER_ID: + raise ApiError("daily OHLCV bars are currently implemented for J-Quants only") + + client = JQuantsClient() + fetch_date = str(body.get("date") or "").strip() or None + fetch_lookback_days = _as_int(body.get("lookback_days"), 30) + if _as_bool(body.get("bulk"), False): + result = client.fetch_daily_bars_bulk( + tickers, + date=fetch_date, + lookback_days=fetch_lookback_days, + ) + else: + result = client.fetch_daily_bars( + tickers, + date=fetch_date, + lookback_days=fetch_lookback_days, + ) + if _as_bool(body.get("use_bar_store"), True): + _apply_daily_bar_store( + result, + tickers, + path=store_path, + price_path=Path(str(body.get("price_store_path") or DEFAULT_CURRENT_PRICES_CSV)), + lookback_days=_as_int(body.get("lookback_days"), 30), + ) + result["provider_policy"] = policy.to_dict() + return result + + +def _market_bars_universe(body: JsonDict) -> JsonDict: + from investment_assistant.investment.universe import build_market_universe + + scope = str(body.get("scope") or body.get("universe") or "prime") + max_tickers = _as_int(body.get("max_tickers"), 0) + limit = max_tickers if max_tickers > 0 else 10_000 + universe = build_market_universe( + financials_csv=str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV), + jpx_listed_path=str(body.get("jpx_listed_path") or "local_docs/jpx/listed_issues.csv"), + nikkei225_registry=str( + body.get("nikkei225_registry") or "examples/source_registry_nikkei225_edinet.yaml" + ), + query=str(body.get("query") or ""), + scope=scope, + limit=limit, + ) + raw_securities = universe.get("securities") + rows = [ + row + for row in (raw_securities if isinstance(raw_securities, list) else []) + if isinstance(row, dict) and str(row.get("ticker") or "").strip() + ] + tickers = [str(row.get("ticker")) for row in rows] + universe_total = _as_int(universe.get("total_count"), len(tickers)) + jpx_listed_count = _as_int(universe.get("jpx_listed_count"), 0) + nikkei225_count = _as_int(universe.get("nikkei225_count"), 0) + financials_count = _as_int(universe.get("financials_count"), 0) + if not tickers: + return { + "available": False, + "scope": scope, + "selected_count": 0, + "universe_total_count": universe_total, + "hint": universe.get("hint") or "対象銘柄がありません。", + "auto_trading": False, + "call_real_api": False, + } + + selection = { + "scope": scope, + "selected_count": len(tickers), + "universe_total_count": universe_total, + "jpx_listed_count": jpx_listed_count, + "nikkei225_count": nikkei225_count, + "financials_count": financials_count, + "tickers_sample": tickers[:20], + "auto_trading": False, + } + if _as_bool(body.get("preview_only"), False): + return { + "available": True, + "selection": selection, + "tickers": tickers, + "sources": universe.get("sources"), + "auto_trading": False, + "call_real_api": False, + } + + nested = dict(body) + nested["tickers"] = tickers + nested.setdefault("provider_id", _JQUANTS_PROVIDER_ID) + nested.setdefault("runtime_mode", "production") + nested.setdefault("allow_cache_on_policy_block", True) + nested.setdefault("bulk", True) + result = _market_bars(nested) + result["selection"] = selection + return result + + +def _market_bars_universe_async(body: JsonDict) -> JsonDict: + scope = str(body.get("scope") or body.get("universe") or "prime") + job_id = JOBS.start("market-bars-universe", lambda: _market_bars_universe(body)) + return { + "job_id": job_id, + "status": "running", + "kind": "market-bars-universe", + "scope": scope, + "auto_trading": False, + } + + +def _apply_market_price_store( + result: JsonDict, + tickers: list[str], + *, + provider_id: str, + path: Path, + daily_bars_path: Path | None = None, +) -> None: + from investment_assistant.portfolio.bar_store import ( + filter_daily_bars, + latest_price_facts_from_bars, + load_daily_bars, + ) + from investment_assistant.portfolio.price_store import ( + MarketPriceFact, + facts_from_price_response, + load_current_prices, + merge_market_price_facts, + normalize_ticker, + save_current_prices, + ) + + raw_prices = result.get("prices") + prices: dict[str, object] = dict(raw_prices) if isinstance(raw_prices, dict) else {} + raw_as_of = result.get("as_of") + as_of: dict[str, object] = dict(raw_as_of) if isinstance(raw_as_of, dict) else {} + raw_notes = result.get("notes") + notes: dict[str, object] = dict(raw_notes) if isinstance(raw_notes, dict) else {} + source_ref = str(result.get("source") or "") + cached = load_current_prices(path) + fetched_facts = facts_from_price_response( + tickers, + prices=prices, + as_of=as_of, + provider_id=provider_id, + source_ref=source_ref, + notes=notes, + ) + bar_price_facts: dict[str, MarketPriceFact] = {} + if daily_bars_path is not None: + cached_bars = load_daily_bars(daily_bars_path) + visible_bars = filter_daily_bars(cached_bars, tickers=tickers, limit_per_ticker=1) + bar_price_facts = { + fact.ticker: fact + for fact in latest_price_facts_from_bars( + visible_bars, + source_ref=str(daily_bars_path), + ) + } + + cache_used: list[str] = [] + daily_bar_used: list[str] = [] + daily_bar_facts_used: list[MarketPriceFact] = [] + for raw in tickers: + ticker = normalize_ticker(raw) + if not ticker: + continue + if prices.get(ticker) is not None or prices.get(str(raw)) is not None: + continue + fact = cached.get(ticker) + note = "using_cached_price" + if fact is None: + fact = bar_price_facts.get(ticker) + note = "using_daily_bar_close" + if fact is None: + continue + prices[str(raw)] = fact.price + if fact.as_of: + as_of[str(raw)] = fact.as_of + previous_note = str(notes.get(str(raw)) or "").strip() + notes[str(raw)] = f"{previous_note}; {note}" if previous_note else note + if note == "using_daily_bar_close": + daily_bar_used.append(ticker) + daily_bar_facts_used.append(fact) + else: + cache_used.append(ticker) + + saved_path: str | None = None + total_count = len(cached) + facts_to_save = fetched_facts + daily_bar_facts_used + if facts_to_save: + merged = merge_market_price_facts(cached.values(), facts_to_save) + saved_path = save_current_prices(merged, path) + total_count = len(merged) + + result["prices"] = prices + result["as_of"] = as_of + result["notes"] = notes + result["price_store"] = { + "path": str(path), + "saved": bool(saved_path), + "saved_path": saved_path, + "saved_count": len(facts_to_save), + "cache_used": cache_used, + "daily_bar_cache_used": daily_bar_used, + "daily_bars_path": str(daily_bars_path) if daily_bars_path else "", + "cached_count_before": len(cached), + "total_count": total_count, + "auto_trading": False, + } + + +def _apply_daily_bar_store( + result: JsonDict, + tickers: list[str], + *, + path: Path, + price_path: Path | None, + lookback_days: int, +) -> None: + from investment_assistant.portfolio.bar_store import ( + daily_bar_fact_from_row, + filter_daily_bars, + latest_price_facts_from_bars, + load_daily_bars, + merge_daily_bars, + save_daily_bars, + summarize_daily_bars, + ) + from investment_assistant.portfolio.price_store import ( + load_current_prices, + merge_market_price_facts, + save_current_prices, + ) + + raw_bars = result.get("bars") + fetched = [ + fact + for item in (raw_bars if isinstance(raw_bars, list) else []) + if isinstance(item, dict) and (fact := daily_bar_fact_from_row(item)) is not None + ] + cached = load_daily_bars(path) + if fetched: + merged = merge_daily_bars(cached, fetched) + saved_path = save_daily_bars(merged, path) + visible = filter_daily_bars(merged, tickers=tickers, limit_per_ticker=lookback_days) + result["bars"] = [fact.to_dict() for fact in visible] + result["summary"] = summarize_daily_bars(visible) + cache_used: list[str] = [] + saved = True + total_count = len(merged) + else: + visible = filter_daily_bars(cached, tickers=tickers, limit_per_ticker=lookback_days) + result["bars"] = [fact.to_dict() for fact in visible] + result["summary"] = summarize_daily_bars(visible) + cache_used = sorted({fact.ticker for fact in visible}) + saved_path = None + saved = False + total_count = len(cached) + price_sync: dict[str, object] = { + "saved": False, + "saved_path": None, + "saved_count": 0, + "path": str(price_path) if price_path else "", + } + if price_path is not None and visible: + price_facts = latest_price_facts_from_bars(visible, source_ref=str(path)) + if price_facts: + current_prices = load_current_prices(price_path) + merged_prices = merge_market_price_facts(current_prices.values(), price_facts) + price_saved_path = save_current_prices(merged_prices, price_path) + price_sync = { + "saved": True, + "saved_path": price_saved_path, + "saved_count": len(price_facts), + "path": str(price_path), + "tickers": [fact.ticker for fact in price_facts], + } + result["bar_store"] = { + "path": str(path), + "saved": saved, + "saved_path": saved_path, + "saved_count": len(fetched), + "cache_used": cache_used, + "price_sync": price_sync, + "cached_count_before": len(cached), + "total_count": total_count, + "auto_trading": False, + } + + +def _data_status(body: JsonDict) -> JsonDict: + from investment_assistant.financials.current_yield import DEFAULT_CURRENT_YIELDS_CSV + from investment_assistant.investment.data_catalog import ( + DEFAULT_COMPANY_MASTER_CSV, + build_data_catalog, + ) + from investment_assistant.investment.universe import DEFAULT_JPX_LISTED_ISSUES_PATH + from investment_assistant.portfolio.bar_store import DEFAULT_DAILY_BARS_CSV + from investment_assistant.portfolio.price_store import DEFAULT_CURRENT_PRICES_CSV + + catalog = build_data_catalog( + financials_csv=str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV), + jpx_listed_path=str(body.get("jpx_listed_path") or DEFAULT_JPX_LISTED_ISSUES_PATH), + company_master_path=str(body.get("company_master_path") or DEFAULT_COMPANY_MASTER_CSV), + market_prices_path=str(body.get("market_prices_path") or DEFAULT_CURRENT_PRICES_CSV), + daily_bars_path=str(body.get("daily_bars_path") or DEFAULT_DAILY_BARS_CSV), + current_yields_path=str(body.get("current_yields_path") or DEFAULT_CURRENT_YIELDS_CSV), + stale_after_days=max(_as_int(body.get("stale_after_days"), 7), 1), + ) + catalog["jquants"] = _jquants_status({}) + return catalog + + +def _market_current_yields_import(body: JsonDict) -> JsonDict: + from investment_assistant.financials.current_yield import ( + CurrentYieldFact, + current_yield_fact_from_row, + current_yields_to_csv_text, + load_current_yields, + merge_current_yield_facts, + parse_current_yields_csv, + ) + from investment_assistant.ingestion.fetcher import reject_path_traversal + + path = reject_path_traversal(str(body.get("path") or DEFAULT_CURRENT_YIELDS_CSV)) + incoming_facts: list[CurrentYieldFact] = [] + csv_text = str(body.get("csv_text") or "").strip() + if csv_text: + incoming_facts.extend(parse_current_yields_csv(csv_text).values()) + raw_rows = body.get("rows") + if isinstance(raw_rows, list): + for item in raw_rows: + if isinstance(item, dict): + fact = current_yield_fact_from_row(item) + if fact is not None: + incoming_facts.append(fact) + if not incoming_facts: + raise ApiError("current yield CSV rows are required", status=400) + + existing = ( + load_current_yields(path).values() + if _as_bool(body.get("merge_existing"), True) + else [] + ) + merged = merge_current_yield_facts(existing, incoming_facts) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(current_yields_to_csv_text(merged), encoding="utf-8") + return { + "available": True, + "saved_path": str(path), + "count": len(merged), + "imported_count": len(incoming_facts), + "facts": [fact.to_dict() for fact in merged], + "auto_trading": False, + "call_real_api": False, + } + + def _provider_policy_ledger(body: JsonDict) -> JsonDict: from investment_assistant.investment.provider_policy import provider_policy_ledger @@ -606,7 +2680,248 @@ def _portfolio_performance(body: JsonDict) -> JsonDict: def _financials_compare(body: JsonDict) -> JsonDict: path = str(body.get("path") or "examples/financials_sample.csv") - return compare_financials(load_financials(path)) + from investment_assistant.financials.dividend_quality import normalize_dividend_points + + points, dividend_quality = normalize_dividend_points(load_financials(path)) + result = compare_financials(points) + result["dividend_quality"] = dividend_quality + return result + + +def _financials_status(body: JsonDict) -> JsonDict: + path = Path(str(body.get("path") or body.get("financials_csv") or DEFAULT_FINANCIALS_CSV)) + stale_after_days = max(_as_int(body.get("stale_after_days"), 7), 1) + if not path.is_file(): + return { + "available": False, + "status": "missing", + "path": str(path), + "point_count": 0, + "company_count": 0, + "latest_fiscal_year": None, + "modified_at": None, + "age_days": None, + "stale_after_days": stale_after_days, + "hint": ( + "財務データがまだありません。" + "DataタブでEDINET取得または手動保存を行ってください。" + ), + "auto_trading": False, + "call_real_api": False, + } + try: + from investment_assistant.financials.dividend_quality import normalize_dividend_points + + points, dividend_quality = normalize_dividend_points(load_financials(path)) + comparison = compare_financials(points) + except (ValueError, OSError) as exc: + return { + "available": False, + "status": "invalid", + "path": str(path), + "point_count": 0, + "company_count": 0, + "latest_fiscal_year": None, + "modified_at": None, + "age_days": None, + "stale_after_days": stale_after_days, + "hint": f"財務データを読み込めません: {type(exc).__name__}: {exc}", + "auto_trading": False, + "call_real_api": False, + } + stat = path.stat() + modified_at = datetime.fromtimestamp(stat.st_mtime, UTC) + age_days = (datetime.now(UTC) - modified_at).total_seconds() / 86400 + companies = comparison.get("companies") + rows = companies if isinstance(companies, list) else [] + latest_years = [ + _as_int(row.get("latest_fiscal_year"), 0) + for row in rows + if isinstance(row, dict) + ] + is_stale = age_days > stale_after_days + return { + "available": True, + "status": "stale" if is_stale else "fresh", + "path": str(path), + "point_count": len(points), + "company_count": len(rows), + "latest_fiscal_year": max(latest_years) if latest_years else None, + "modified_at": modified_at.isoformat(), + "age_days": round(age_days, 2), + "stale_after_days": stale_after_days, + "hint": ( + "更新推奨です。Dataタブで最新7日取得またはバックフィルを実行してください。" + if is_stale + else "財務データは利用可能です。必要に応じてDataタブから更新できます。" + ), + "dividend_quality": dividend_quality, + "auto_trading": False, + "call_real_api": False, + } + + +def _financials_import(body: JsonDict) -> JsonDict: + from investment_assistant.financials.dividend_quality import ( + financial_points_to_csv_text, + normalize_dividend_points, + ) + from investment_assistant.financials.models import FINANCIAL_COLUMNS + from investment_assistant.ingestion.fetcher import reject_path_traversal + + raw_csv_text = body.get("csv_text") + csv_text = raw_csv_text if isinstance(raw_csv_text, str) else "" + save = _as_bool(body.get("save"), False) + cleanup: Path | None = None + source = "path" + source_ref: str + normalized_csv: str + + if csv_text.strip(): + if len(csv_text) > _MAX_MANUAL_TEXT_CHARS: + raise ApiError(f"csv_text is too long: max {_MAX_MANUAL_TEXT_CHARS} characters") + normalized_csv = csv_text.strip() + "\n" + with tempfile.NamedTemporaryFile( + "w", + suffix=".csv", + delete=False, + encoding="utf-8", + ) as handle: + handle.write(normalized_csv) + source_ref = handle.name + cleanup = Path(source_ref) + source = "csv_text" + else: + source_ref = _require_str(body, "path") + normalized_csv = Path(source_ref).read_text(encoding="utf-8") + + try: + points = load_financials(source_ref) + finally: + if cleanup is not None: + cleanup.unlink(missing_ok=True) + + points, dividend_quality = normalize_dividend_points(points) + normalized_csv = financial_points_to_csv_text(points) + comparison = compare_financials(points) + saved_path: str | None = None + output_path = str(body.get("output_path") or DEFAULT_FINANCIALS_CSV) + if save: + target = reject_path_traversal(output_path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(normalized_csv, encoding="utf-8") + saved_path = str(target) + + companies = comparison.get("companies") + company_count = len(companies) if isinstance(companies, list) else 0 + return { + "available": True, + "source": source, + "source_ref": None if source == "csv_text" else source_ref, + "saved": save, + "saved_path": saved_path, + "financials_csv": saved_path or (None if source == "csv_text" else source_ref), + "columns": list(FINANCIAL_COLUMNS), + "count": len(points), + "company_count": company_count, + "comparison": comparison, + "dividend_quality": dividend_quality, + "disclaimer": comparison.get("disclaimer"), + "auto_trading": False, + "call_real_api": False, + } + + +def _financials_securities(body: JsonDict) -> JsonDict: + from investment_assistant.financials.dividend_quality import normalize_dividend_points + from investment_assistant.investment.universe import build_market_universe + + path = str( + body.get("financials_csv") + or body.get("path") + or DEFAULT_FINANCIALS_CSV + ) + query = str(body.get("query") or "").strip().lower() + limit = max(_as_int(body.get("limit"), 20), 1) + if _as_bool(body.get("include_market_universe"), False) or body.get("jpx_listed_path"): + universe = build_market_universe( + financials_csv=path, + jpx_listed_path=str(body.get("jpx_listed_path") or "local_docs/jpx/listed_issues.csv"), + nikkei225_registry=str( + body.get("nikkei225_registry") + or "examples/source_registry_nikkei225_edinet.yaml" + ), + query=query, + scope=str(body.get("scope") or "prime"), + limit=limit, + ) + rows = universe.get("securities") + securities = rows if isinstance(rows, list) else [] + if securities: + return { + "available": True, + "query": query, + "source_ref": path, + "count": len(securities), + "securities": securities, + "market_universe_used": True, + "jpx_listed_available": universe.get("jpx_listed_available"), + "hint": universe.get("hint"), + "auto_trading": False, + "call_real_api": False, + } + if not Path(path).is_file(): + return { + "available": False, + "query": query, + "source_ref": path, + "count": 0, + "securities": [], + "hint": ( + "財務データが見つかりません。DataタブでEDINET取得/手動保存を行うか、" + "上部の財務データをサンプルデータに切り替えてください。" + ), + "auto_trading": False, + "call_real_api": False, + } + points, dividend_quality = normalize_dividend_points(load_financials(path)) + comparison = compare_financials(points) + companies = comparison.get("companies") + rows = companies if isinstance(companies, list) else [] + matches: list[dict[str, object]] = [] + for company in rows: + if not isinstance(company, dict): + continue + ticker = str(company.get("ticker") or "") + name = str(company.get("name") or "") + haystack = f"{ticker} {name}".lower() + if query and query not in haystack: + continue + matches.append( + { + "ticker": ticker, + "code": ticker, + "name": name, + "latest_fiscal_year": company.get("latest_fiscal_year"), + "latest_equity_ratio": company.get("latest_equity_ratio"), + "latest_dividend_per_share": company.get("latest_dividend_per_share"), + "dividend_cut_years": company.get("dividend_cut_years"), + "operating_cf_trend": company.get("operating_cf_trend"), + "source_ref": path, + } + ) + if len(matches) >= limit: + break + return { + "available": True, + "query": query, + "source_ref": path, + "count": len(matches), + "securities": matches, + "dividend_quality": dividend_quality, + "auto_trading": False, + "call_real_api": False, + } def _holdings_import(body: JsonDict) -> JsonDict: @@ -639,6 +2954,12 @@ def _holdings_validate(body: JsonDict) -> JsonDict: return validate_holdings_payload(body) +def _holdings_file_convert(body: JsonDict) -> JsonDict: + from investment_assistant.investment import convert_holding_file_payload + + return convert_holding_file_payload(body) + + def _holdings_template(body: JsonDict) -> JsonDict: from investment_assistant.investment import holding_csv_template @@ -665,6 +2986,7 @@ def _portfolio_analyze(body: JsonDict) -> JsonDict: return analyze_portfolio( holdings_from_payload(body), financials_csv=str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV), + current_yields_csv=str(body.get("current_yields_csv") or DEFAULT_CURRENT_YIELDS_CSV), runtime_mode=str(body.get("runtime_mode") or "development"), ) @@ -683,6 +3005,7 @@ def _investment_detail(body: JsonDict) -> JsonDict: holdings=holdings, funds=fund_profiles_from_payload(body), financials_csv=str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV), + current_yields_csv=str(body.get("current_yields_csv") or DEFAULT_CURRENT_YIELDS_CSV), ) @@ -746,12 +3069,14 @@ def _investment_monthly_report(body: JsonDict) -> JsonDict: optimization=str(body.get("optimization") or "balanced"), dividend_basis=str(body.get("dividend_basis") or "conservative"), financials_csv=financials_csv, + current_yields_csv=str(body.get("current_yields_csv") or DEFAULT_CURRENT_YIELDS_CSV), ) report = build_investment_monthly_report( holdings, candidates=candidates, target_result=target_result, financials_csv=financials_csv, + current_yields_csv=str(body.get("current_yields_csv") or DEFAULT_CURRENT_YIELDS_CSV), runtime_mode=str(body.get("runtime_mode") or "development"), ) if _as_bool(body.get("save_history"), True): @@ -1127,6 +3452,110 @@ def _real_api_decision(body: JsonDict) -> tuple[bool, str | None]: return False, "real API is not enabled" +def _ensure_env_from_dotenv(key: str) -> bool: + if os.getenv(key, "").strip(): + return True + for env_path in (Path(".env"), Path(".env.local")): + if not env_path.is_file(): + continue + try: + for line in env_path.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + name, value = stripped.split("=", 1) + if name.strip() != key: + continue + cleaned = value.strip().strip('"').strip("'") + if cleaned: + os.environ[key] = cleaned + return True + except OSError: + continue + return False + + +def _upsert_dotenv_values(path: Path, values: dict[str, str]) -> str: + existing_lines: list[str] = [] + if path.is_file(): + existing_lines = path.read_text(encoding="utf-8").splitlines() + remaining = {key: value for key, value in values.items() if value.strip()} + out: list[str] = [] + for line in existing_lines: + stripped = line.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + out.append(line) + continue + name, _ = stripped.split("=", 1) + key = name.strip() + if key in remaining: + out.append(f'{key}="{_escape_dotenv_value(remaining.pop(key))}"') + else: + out.append(line) + for key in sorted(remaining): + out.append(f'{key}="{_escape_dotenv_value(remaining[key])}"') + path.write_text("\n".join(out).rstrip() + "\n", encoding="utf-8") + return str(path) + + +def _escape_dotenv_value(value: str) -> str: + return value.replace("\\", "\\\\").replace('"', '\\"') + + +def _edinet_api_key_source( + *, + configured: bool, + env_configured_before_dotenv: bool, + dotenv_loaded: bool, +) -> str: + if not configured: + return "missing" + if _EDINET_API_KEY_RUNTIME_SET: + return "runtime_input" + if env_configured_before_dotenv: + return "process_env" + if dotenv_loaded: + return "dotenv" + return "unknown" + + +def _jquants_api_key_source( + *, + configured: bool, + env_configured_before_dotenv: bool, + dotenv_loaded: bool, +) -> str: + if not configured: + return "missing" + if _JQUANTS_API_KEY_RUNTIME_SET: + return "runtime_input" + if env_configured_before_dotenv: + return "process_env" + if dotenv_loaded: + return "dotenv" + return "unknown" + + +def _is_runtime_contracted_provider(provider_id: str) -> bool: + from investment_assistant.investment.provider_policy import CONTRACTED_PROVIDERS_ENV + + normalized = provider_id.strip().lower() + raw = os.getenv(CONTRACTED_PROVIDERS_ENV, "") + return normalized in {item.strip().lower() for item in raw.split(",") if item.strip()} + + +def _mark_runtime_contracted_provider(provider_id: str) -> None: + from investment_assistant.investment.provider_policy import CONTRACTED_PROVIDERS_ENV + + normalized = provider_id.strip().lower() + if not normalized: + return + raw = os.getenv(CONTRACTED_PROVIDERS_ENV, "") + providers = {item.strip().lower() for item in raw.split(",") if item.strip()} + providers.add(normalized) + os.environ[CONTRACTED_PROVIDERS_ENV] = ",".join(sorted(providers)) + + def _require_str(body: JsonDict, key: str) -> str: value = body.get(key) if not isinstance(value, str) or not value.strip(): @@ -1144,6 +3573,38 @@ def _require_sources(body: JsonDict) -> list[Any]: return sources +def _default_disclosure_sources() -> list[JsonDict]: + return [ + { + "name": "edinet_portal", + "url": "https://disclosure2.edinet-fsa.go.jp/", + "output_path": "local_docs/disclosure/edinet_portal.txt", + "query_hint": "EDINET 有価証券報告書 半期報告書 四半期報告書 財務諸表", + "extract_text": True, + "include_metadata": True, + "preview_chars": 500, + }, + { + "name": "tdnet_portal", + "url": "https://www.release.tdnet.info/inbs/I_main_00.html", + "output_path": "local_docs/disclosure/tdnet_portal.txt", + "query_hint": "TDnet 適時開示 決算短信 配当 予想 修正", + "extract_text": True, + "include_metadata": True, + "preview_chars": 500, + }, + { + "name": "jpx_listed_issues", + "url": "https://www.jpx.co.jp/markets/statistics-equities/misc/01.html", + "output_path": "local_docs/disclosure/jpx_listed_issues.txt", + "query_hint": "JPX 東証上場銘柄一覧 プライム スタンダード グロース", + "extract_text": True, + "include_metadata": True, + "preview_chars": 500, + }, + ] + + def _run_fetch_job_sources(sources: list[Any], *, dry_run: bool) -> JsonDict: yaml_text = _sources_to_yaml(sources) with tempfile.NamedTemporaryFile( @@ -1199,6 +3660,17 @@ def _as_bool(value: object, default: bool = False) -> bool: return default +def _market_scope_matches(row: JsonDict, scope: str) -> bool: + normalized = scope.strip().lower() + if normalized in {"prime", "tse_prime", "tosho_prime"}: + return bool(row.get("is_prime")) + if normalized in {"nikkei225", "nikkei_225", "n225"}: + return bool(row.get("is_nikkei225")) + if normalized in {"financials", "edinet", "financials_available"}: + return bool(row.get("has_financials")) + return True + + def _as_int(value: object, default: int) -> int: if isinstance(value, bool) or not isinstance(value, int | float | str): return default @@ -1274,6 +3746,8 @@ def _yaml_scalar(value: object) -> str: _ROUTES: dict[tuple[str, str], Handler] = { ("GET", "/api/health"): _health, + ("GET", "/api/data/status"): _data_status, + ("POST", "/api/data/status"): _data_status, ("GET", "/api/budget"): _budget, ("GET", "/api/runtime/real-api"): _runtime_real_api_status, ("POST", "/api/runtime/real-api"): _runtime_real_api_set, @@ -1281,6 +3755,8 @@ def _yaml_scalar(value: object) -> str: ("POST", "/api/rag/search"): _rag_search, ("POST", "/api/rag/answer-context"): _rag_answer_context, ("POST", "/api/rag/answer"): _rag_answer, + ("GET", "/api/operators/catalog"): _operators_catalog, + ("POST", "/api/operators/catalog"): _operators_catalog, ("POST", "/api/orchestrate"): _orchestrate, ("POST", "/api/rag/index-dir"): _rag_index_dir, ("POST", "/api/manual-doc/save"): _manual_doc_save, @@ -1292,11 +3768,26 @@ def _yaml_scalar(value: object) -> str: ("POST", "/api/portfolio/simulate"): _portfolio_simulate, ("POST", "/api/portfolio/target"): _portfolio_target, ("POST", "/api/portfolio/universe"): _portfolio_universe, + ("POST", "/api/market/universe"): _market_universe, + ("POST", "/api/market/jpx-listed/template"): _jpx_listed_template, + ("POST", "/api/market/jpx-listed/import"): _jpx_listed_import, + ("POST", "/api/market/jpx-listed/download"): _jpx_listed_download, + ("POST", "/api/market/jpx-listed/download-import"): _jpx_listed_download_import, + ("GET", "/api/companies/status"): _companies_master_status, + ("POST", "/api/companies/status"): _companies_master_status, + ("POST", "/api/companies/refresh"): _companies_master_refresh, ("POST", "/api/market/prices"): _market_prices, + ("POST", "/api/market/prices/import"): _market_prices_import, + ("POST", "/api/market/prices/import-file"): _market_prices_import_file, + ("POST", "/api/market/bars"): _market_bars, + ("POST", "/api/market/bars/universe"): _market_bars_universe, + ("POST", "/api/market/bars/universe-async"): _market_bars_universe_async, + ("POST", "/api/market/current-yields/import"): _market_current_yields_import, ("POST", "/api/providers/policy"): _provider_policy_ledger, ("POST", "/api/portfolio/performance"): _portfolio_performance, ("POST", "/api/holdings/import"): _holdings_import, ("POST", "/api/holdings/validate"): _holdings_validate, + ("POST", "/api/holdings/file/convert"): _holdings_file_convert, ("POST", "/api/holdings/template"): _holdings_template, ("POST", "/api/funds/validate"): _funds_validate, ("POST", "/api/funds/template"): _funds_template, @@ -1313,12 +3804,41 @@ def _yaml_scalar(value: object) -> str: ("POST", "/api/reports/investment-monthly/history/verify"): _investment_report_history_verify, ("POST", "/api/reports/investment-monthly/history/compare"): _investment_report_history_compare, ("POST", "/api/financials/compare"): _financials_compare, + ("GET", "/api/financials/status"): _financials_status, + ("POST", "/api/financials/status"): _financials_status, + ("POST", "/api/financials/import"): _financials_import, + ("POST", "/api/financials/prime-registry"): _financials_prime_registry, + ("POST", "/api/financials/prime-refresh"): _financials_prime_refresh, + ("POST", "/api/financials/prime-refresh-async"): _financials_prime_refresh_async, + ("POST", "/api/financials/listed-registry"): _financials_listed_registry, + ("POST", "/api/financials/listed-refresh"): _financials_listed_refresh, + ("POST", "/api/financials/listed-refresh-async"): _financials_listed_refresh_async, + ("POST", "/api/financials/missing-registry"): _financials_missing_registry, + ("POST", "/api/financials/missing-refresh"): _financials_missing_refresh, + ("POST", "/api/financials/missing-refresh-async"): _financials_missing_refresh_async, + ("POST", "/api/financials/missing-sources"): _financials_missing_sources, + ("POST", "/api/financials/missing-evidence-refresh"): _financials_missing_evidence_refresh, + ("POST", "/api/financials/missing-evidence-refresh-async"): ( + _financials_missing_evidence_refresh_async + ), + ("POST", "/api/financials/missing-auto-refresh"): _financials_missing_auto_refresh, + ("POST", "/api/financials/missing-auto-refresh-async"): ( + _financials_missing_auto_refresh_async + ), + ("POST", "/api/financials/refresh"): _financials_refresh, + ("POST", "/api/financials/refresh-async"): _financials_refresh_async, + ("POST", "/api/financials/securities"): _financials_securities, ("POST", "/api/cache/maintenance"): _cache_maintenance, ("POST", "/api/fetch-job/dry-run"): lambda body: _fetch_job(body, dry_run=True), ("POST", "/api/fetch-job/run"): lambda body: _fetch_job(body, dry_run=False), ("POST", "/api/fetch-job/auto"): _fetch_job_auto, ("POST", "/api/edinet/ingest"): _edinet_ingest, ("POST", "/api/edinet/ingest-async"): _edinet_ingest_async, + ("GET", "/api/edinet/status"): _edinet_status, + ("POST", "/api/edinet/api-key"): _edinet_api_key_set, + ("GET", "/api/jquants/status"): _jquants_status, + ("POST", "/api/jquants/status"): _jquants_status, + ("POST", "/api/jquants/api-key"): _jquants_api_key_set, ("POST", "/api/jobs/status"): _job_status, ("POST", "/api/storage/prune"): _storage_prune, ("POST", "/api/knowledge/diff"): _knowledge_diff, diff --git a/src/investment_assistant/webapi/yahoo_market.py b/src/investment_assistant/webapi/yahoo_market.py new file mode 100644 index 0000000..c618be9 --- /dev/null +++ b/src/investment_assistant/webapi/yahoo_market.py @@ -0,0 +1,265 @@ +"""HTTP-facing routes for configurable Yahoo! Finance market-data refreshes.""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from investment_assistant.financials.evidence import DEFAULT_FINANCIALS_CSV +from investment_assistant.investment.universe import ( + DEFAULT_JPX_LISTED_ISSUES_PATH, + DEFAULT_NIKKEI225_REGISTRY, + build_market_universe, +) +from investment_assistant.portfolio.bar_store import DEFAULT_DAILY_BARS_CSV, load_daily_bars +from investment_assistant.portfolio.price_store import ( + DEFAULT_CURRENT_PRICES_CSV, + load_current_prices, +) +from investment_assistant.portfolio.yahoo_market import ( + ALLOWED_INTERVALS, + ALLOWED_RANGES, + DEFAULT_YAHOO_FUNDAMENTALS_CSV, + load_yahoo_fundamentals, + normalize_tickers, + refresh_yahoo_market, +) + +JsonDict = dict[str, Any] +_MAX_AUTO_TICKERS = 200 +_MAX_CUSTOM_TICKERS = 50 +_ROUTES = { + ("GET", "/api/market/yahoo/status"), + ("POST", "/api/market/yahoo/status"), + ("POST", "/api/market/yahoo/refresh"), +} + + +def handle_yahoo_market_api( + method: str, + path: str, + body: JsonDict | None = None, +) -> tuple[int, JsonDict] | None: + """Handle Yahoo routes or return ``None`` when the core router should continue.""" + + normalized = (method.upper(), path.rstrip("/") or "/") + if normalized not in _ROUTES: + return None + try: + if normalized[1] == "/api/market/yahoo/status": + return 200, yahoo_market_status(body or {}) + return 200, yahoo_market_refresh(body or {}) + except (ValueError, KeyError, OSError) as exc: + return 400, {"error": f"{type(exc).__name__}: {exc}"} + + +def available_yahoo_market_routes() -> list[str]: + return sorted(f"{method} {path}" for method, path in _ROUTES) + + +def yahoo_market_refresh(body: JsonDict) -> JsonDict: + mode = str(body.get("mode") or "auto").strip().lower() + tickers, selection = _resolve_tickers(body, mode=mode) + range_ = str(body.get("range") or "1mo").strip() + interval = str(body.get("interval") or "1d").strip() + if range_ not in ALLOWED_RANGES: + raise ValueError(f"range must be one of: {', '.join(sorted(ALLOWED_RANGES))}") + if interval not in ALLOWED_INTERVALS: + raise ValueError(f"interval must be one of: {', '.join(sorted(ALLOWED_INTERVALS))}") + + result = refresh_yahoo_market( + tickers, + range_=range_, + interval=interval, + fetch_ohlcv=_as_bool(body.get("fetch_ohlcv"), True), + fetch_fundamentals=_as_bool(body.get("fetch_fundamentals"), True), + daily_bars_path=str(body.get("daily_bars_path") or DEFAULT_DAILY_BARS_CSV), + current_prices_path=str( + body.get("current_prices_path") or DEFAULT_CURRENT_PRICES_CSV + ), + fundamentals_path=str( + body.get("fundamentals_path") or DEFAULT_YAHOO_FUNDAMENTALS_CSV + ), + ) + result["mode"] = mode + result["selection"] = selection + result["configuration"] = { + "range": range_, + "interval": interval, + "fetch_ohlcv": _as_bool(body.get("fetch_ohlcv"), True), + "fetch_fundamentals": _as_bool(body.get("fetch_fundamentals"), True), + } + return result + + +def yahoo_market_status(body: JsonDict) -> JsonDict: + daily_bars_path = Path(str(body.get("daily_bars_path") or DEFAULT_DAILY_BARS_CSV)) + current_prices_path = Path( + str(body.get("current_prices_path") or DEFAULT_CURRENT_PRICES_CSV) + ) + fundamentals_path = Path( + str(body.get("fundamentals_path") or DEFAULT_YAHOO_FUNDAMENTALS_CSV) + ) + bars = load_daily_bars(daily_bars_path) + prices = load_current_prices(current_prices_path) + fundamentals = load_yahoo_fundamentals(fundamentals_path) + yahoo_bars = [bar for bar in bars if bar.provider_id == "yahoo_finance"] + yahoo_prices = [ + fact for fact in prices.values() if fact.provider_id == "yahoo_finance" + ] + datasets = [ + _dataset_status( + "ohlcv", + "株価四本値・出来高", + daily_bars_path, + row_count=len(yahoo_bars), + ticker_count=len({bar.ticker for bar in yahoo_bars}), + ), + _dataset_status( + "current_prices", + "現在価格", + current_prices_path, + row_count=len(yahoo_prices), + ticker_count=len(yahoo_prices), + ), + _dataset_status( + "fundamentals", + "市場財務指標", + fundamentals_path, + row_count=len(fundamentals), + ticker_count=len(fundamentals), + ), + ] + ready_count = sum(1 for item in datasets if item["status"] == "ready") + return { + "status": "ready" if ready_count == len(datasets) else "partial", + "provider_id": "yahoo_finance", + "datasets": datasets, + "summary": { + "ready_count": ready_count, + "missing_count": len(datasets) - ready_count, + }, + "policy": { + "personal_use_only": True, + "robots_checked": True, + "rate_limited": True, + "redistribution": False, + "auto_trading": False, + }, + "auto_trading": False, + "call_real_api": False, + } + + +def _resolve_tickers(body: JsonDict, *, mode: str) -> tuple[list[str], JsonDict]: + if mode == "custom": + tickers = _ticker_list(body.get("tickers")) + if not tickers: + raise ValueError("custom mode requires tickers") + if len(tickers) > _MAX_CUSTOM_TICKERS: + raise ValueError(f"custom mode supports at most {_MAX_CUSTOM_TICKERS} tickers") + return tickers, { + "mode": "custom", + "selected_count": len(tickers), + "tickers_sample": tickers[:20], + } + if mode != "auto": + raise ValueError("mode must be auto or custom") + + max_tickers = _bounded_int(body.get("max_tickers"), default=20, maximum=_MAX_AUTO_TICKERS) + scope = str(body.get("scope") or "nikkei225").strip().lower() + universe = build_market_universe( + financials_csv=str(body.get("financials_csv") or DEFAULT_FINANCIALS_CSV), + jpx_listed_path=str( + body.get("jpx_listed_path") or DEFAULT_JPX_LISTED_ISSUES_PATH + ), + nikkei225_registry=str( + body.get("nikkei225_registry") or DEFAULT_NIKKEI225_REGISTRY + ), + query=str(body.get("query") or ""), + scope=scope, + limit=max_tickers, + ) + raw_rows = universe.get("securities") + rows = ( + [row for row in raw_rows if isinstance(row, dict)] + if isinstance(raw_rows, list) + else [] + ) + tickers = normalize_tickers( + row.get("ticker") or row.get("code") or "" for row in rows + ) + if not tickers: + raise ValueError( + "automatic selection found no tickers; prepare JPX/EDINET data or use custom mode" + ) + return tickers, { + "mode": "auto", + "scope": scope, + "selected_count": len(tickers), + "universe_total_count": _as_int(universe.get("total_count"), len(tickers)), + "jpx_listed_count": _as_int(universe.get("jpx_listed_count"), 0), + "nikkei225_count": _as_int(universe.get("nikkei225_count"), 0), + "financials_count": _as_int(universe.get("financials_count"), 0), + "tickers_sample": tickers[:20], + "hint": str(universe.get("hint") or ""), + } + + +def _ticker_list(value: object) -> list[str]: + if isinstance(value, str): + raw = [item for item in re.split(r"[\s,、，]+", value) if item] + elif isinstance(value, list): + raw = [str(item) for item in value] + else: + raw = [] + return normalize_tickers(raw) + + +def _as_int(value: object, default: int) -> int: + if isinstance(value, bool) or not isinstance(value, int | float | str): + return default + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _bounded_int(value: object, *, default: int, maximum: int) -> int: + return min(max(_as_int(value, default), 1), maximum) + + +def _as_bool(value: object, default: bool) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "true", "yes", "on"}: + return True + if lowered in {"0", "false", "no", "off"}: + return False + return default + + +def _dataset_status( + key: str, + label: str, + path: Path, + *, + row_count: int, + ticker_count: int, +) -> JsonDict: + modified_at: str | None = None + if path.is_file(): + modified_at = datetime.fromtimestamp(path.stat().st_mtime, UTC).isoformat() + return { + "key": key, + "label": label, + "status": "ready" if row_count > 0 else "missing", + "path": str(path), + "row_count": row_count, + "ticker_count": ticker_count, + "modified_at": modified_at, + } diff --git a/tests/unit/test_dividend_quality.py b/tests/unit/test_dividend_quality.py new file mode 100644 index 0000000..e71eb18 --- /dev/null +++ b/tests/unit/test_dividend_quality.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from investment_assistant.financials.dividend_quality import ( + financial_points_to_csv_text, + normalize_dividend_per_share, + normalize_dividend_points, +) +from investment_assistant.financials.models import FinancialPoint + + +def _point(ticker: str, year: int, dps: float) -> FinancialPoint: + return FinancialPoint( + ticker=ticker, + name="sample", + fiscal_year=year, + operating_cf=0.0, + equity_ratio=0.0, + dividend_per_share=dps, + payout_policy="", + ) + + +def test_normalize_dividend_per_share_corrects_previous_year_unit_jump() -> None: + value, check = normalize_dividend_per_share( + 4100.0, + ticker="8306", + fiscal_year=2025, + previous_value=41.0, + ) + + assert value == 41.0 + assert check is not None + assert check.status == "corrected" + assert check.correction_factor == 100.0 + assert check.code == "dividend_unit_scale_corrected" + + +def test_normalize_dividend_per_share_warns_high_yield_without_clear_unit_fix() -> None: + value, check = normalize_dividend_per_share( + 180.0, + ticker="9999", + fiscal_year=2025, + price=1000.0, + ) + + assert value == 180.0 + assert check is not None + assert check.status == "warn" + assert check.code == "dividend_yield_high_review" + assert check.original_yield_pct == 18.0 + + +def test_normalize_dividend_per_share_corrects_extreme_price_yield() -> None: + value, check = normalize_dividend_per_share( + 4100.0, + ticker="7203", + fiscal_year=2025, + price=1000.0, + ) + + assert value == 41.0 + assert check is not None + assert check.status == "corrected" + assert check.checked_yield_pct == 4.1 + + +def test_normalize_dividend_points_preserves_order_and_reports_summary() -> None: + points = [ + _point("B", 2024, 30.0), + _point("A", 2024, 40.0), + _point("A", 2025, 4000.0), + ] + + normalized, summary = normalize_dividend_points(points) + + assert [point.ticker for point in normalized] == ["B", "A", "A"] + assert [point.dividend_per_share for point in normalized] == [30.0, 40.0, 40.0] + assert summary["status"] == "corrected" + assert summary["corrected_count"] == 1 + assert summary["warning_count"] == 0 + + +def test_financial_points_to_csv_text_uses_canonical_columns() -> None: + text = financial_points_to_csv_text([_point("8306", 2024, 41.0)]) + + assert text.splitlines()[0] == ( + "ticker,name,fiscal_year,operating_cf,equity_ratio," + "dividend_per_share,payout_policy" + ) + assert "8306,sample,2024,0,0,41," in text diff --git a/tests/unit/test_edinet_ingest.py b/tests/unit/test_edinet_ingest.py index 2b40bbf..d51a572 100644 --- a/tests/unit/test_edinet_ingest.py +++ b/tests/unit/test_edinet_ingest.py @@ -345,6 +345,60 @@ def test_ingest_corrects_split_unadjusted_dividend_from_summary(tmp_path: Path) assert nintendo["dividend_trend"] == "increasing" +def test_ingest_corrects_obvious_dividend_unit_jump_without_summary(tmp_path: Path) -> None: + client = _FakeEdinetClient( + { + "2026-06-08": [ + { + "docID": "S100Y25", + "secCode": "83060", + "filerName": "三菱UFJ", + "docTypeCode": "120", + "docDescription": "有価証券報告書", + "periodEnd": "2025-03-31", + "submitDateTime": "2025-06-21 09:00", + "csvFlag": "1", + }, + { + "docID": "S100Y24", + "secCode": "83060", + "filerName": "三菱UFJ", + "docTypeCode": "120", + "docDescription": "有価証券報告書", + "periodEnd": "2024-03-31", + "submitDateTime": "2024-06-21 09:00", + "csvFlag": "1", + }, + ], + }, + archives={ + "S100Y25": _csv_zip(dps="4100.0"), + "S100Y24": _csv_zip(dps="41.0"), + }, + ) + target = EdinetTarget( + name="8306", ticker="8306", company="MUFG", doc_types=("120",), max_periods=2 + ) + + result = ingest_targets( + client=client, # type: ignore[arg-type] + targets=[target], + dates=["2026-06-08"], + output_dir=tmp_path / "edinet", + ) + + comparison = result["comparison"] + assert isinstance(comparison, dict) + companies = comparison["companies"] + assert isinstance(companies, list) + mufg = companies[0] + assert mufg["dividend_series"] == [41.0, 41.0] + quality = result["dividend_quality"] + assert isinstance(quality, dict) + assert quality["status"] == "corrected" + assert quality["corrected_count"] == 1 + + def test_split_correction_reaches_history_only_years(tmp_path: Path) -> None: # FY2023 survives only in durable history as a pre-split ¥1700 value (its # filing was pruned). This run fetches only the FY2024 filing, whose 5-year diff --git a/tests/unit/test_investment_mvp.py b/tests/unit/test_investment_mvp.py index da3a144..cad976a 100644 --- a/tests/unit/test_investment_mvp.py +++ b/tests/unit/test_investment_mvp.py @@ -57,6 +57,18 @@ def _financials(tmp_path: Path) -> Path: return path +def _current_yields(tmp_path: Path) -> Path: + path = tmp_path / "current_yields.csv" + path.write_text( + "ticker,name,current_dividend_per_share,current_price,yield_pct,as_of," + "source_ref,provider_id,note\n" + "9433,KDDI,80,2500,3.2,2026-06-15," + "user_verified_current_dividend,user_csv,current price basis\n", + encoding="utf-8", + ) + return path + + def test_analyze_mixed_stock_and_fund_portfolio(tmp_path: Path) -> None: holdings = holdings_from_payload({"csv_text": HOLDINGS_CSV}) result = analyze_portfolio(holdings, financials_csv=_financials(tmp_path)) @@ -85,6 +97,106 @@ def test_analyze_mixed_stock_and_fund_portfolio(tmp_path: Path) -> None: assert "投資助言" in str(result["disclaimer"]) +def test_analyze_portfolio_prefers_current_yield_overlay(tmp_path: Path) -> None: + financials = tmp_path / "financials.csv" + financials.write_text( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "9433,KDDI,2025,1000,68,145,stable\n", + encoding="utf-8", + ) + holdings_csv = ( + "asset_type,ticker_or_fund_code,name,quantity,avg_cost,account_type,tax_wrapper," + "source,current_price,annual_income,distribution_per_unit\n" + "stock,9433,KDDI,100,2400,tokutei,taxable,user_csv,2500,,\n" + ) + + result = analyze_portfolio( + holdings_from_payload({"csv_text": holdings_csv}), + financials_csv=financials, + current_yields_csv=_current_yields(tmp_path), + ) + + summary = result["summary"] + assert isinstance(summary, dict) + assert summary["annual_income_estimate"] == 8000.0 + assert summary["income_yield_pct"] == 3.2 + rows = result["holdings"] + assert isinstance(rows, list) + row = rows[0] + assert isinstance(row, dict) + assert row["annual_income_source"] == "current_dividend_per_share" + reconciliation = row["current_yield_reconciliation"] + assert isinstance(reconciliation, dict) + assert reconciliation["edinet_implied_yield_pct"] == 5.8 + assert reconciliation["income_yield_pct"] == 3.2 + assert "edinet_current_basis_mismatch_adjusted" in reconciliation["warnings"] + evidence = result["evidence"] + assert isinstance(evidence, list) + annual_income_evidence = next( + item + for item in evidence + if isinstance(item, dict) and item.get("claim_key") == "holding.9433.annual_income" + ) + assert annual_income_evidence["formula"] == "quantity * current_dividend_per_share" + + +def test_analyze_portfolio_flags_edinet_current_yield_basis_review(tmp_path: Path) -> None: + financials = tmp_path / "financials.csv" + financials.write_text( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "9433,KDDI,2025,1000,68,145,stable\n", + encoding="utf-8", + ) + holdings_csv = ( + "asset_type,ticker_or_fund_code,name,quantity,avg_cost,account_type,tax_wrapper," + "source,current_price,annual_income,distribution_per_unit\n" + "stock,9433,KDDI,100,2400,tokutei,taxable,user_csv,2500,,\n" + ) + + result = analyze_portfolio( + holdings_from_payload({"csv_text": holdings_csv}), + financials_csv=financials, + current_yields_csv=tmp_path / "missing_current_yields.csv", + ) + + summary = result["summary"] + assert isinstance(summary, dict) + income_quality = summary["income_quality"] + assert isinstance(income_quality, dict) + assert income_quality["status"] == "warn" + alerts = income_quality["alerts"] + assert isinstance(alerts, list) + alert_codes = {str(alert.get("code")) for alert in alerts if isinstance(alert, dict)} + assert "current_yield_basis_review" in alert_codes + + +def test_holdings_import_accepts_japanese_broker_csv_headers(tmp_path: Path) -> None: + holdings_csv = ( + "資産種別,証券コード,銘柄名,数量,平均取得単価,口座区分,NISA区分,現在価格\n" + "国内株式,9433,KDDI,\"100株\",\"2,400円\",特定,課税,\"2,500円\"\n" + ) + + result = analyze_portfolio( + holdings_from_payload({"csv_text": holdings_csv}), + financials_csv=_financials(tmp_path), + ) + + summary = result["summary"] + assert isinstance(summary, dict) + assert summary["holdings_count"] == 1 + assert summary["market_value"] == 250000.0 + assert summary["cost_basis"] == 240000.0 + rows = result["holdings"] + assert isinstance(rows, list) + row = rows[0] + assert isinstance(row, dict) + assert row["asset_type"] == "stock" + assert row["ticker_or_fund_code"] == "9433" + assert row["account_type"] == "特定" + assert row["tax_wrapper"] == "課税" + assert row["source"] == "user_input" + + def test_analyze_portfolio_flags_nisa_cap_usage(tmp_path: Path) -> None: holdings_csv = ( "asset_type,ticker_or_fund_code,name,quantity,avg_cost,account_type,tax_wrapper," @@ -227,6 +339,16 @@ def test_candidate_screen_returns_condition_matches_not_recommendations(tmp_path codes = {str(item["code"]) for item in result["results"]} # type: ignore[index] assert {"8306", "F001"} <= codes assert "9999" not in codes and "F999" not in codes + fund_candidate = next( + item + for item in result["results"] # type: ignore[index] + if isinstance(item, dict) and item.get("code") == "F001" + ) + assert fund_candidate["score"] == 0.943 + assert fund_candidate["metrics"]["score_model"] == "fund_weighted_v1" + assert len(fund_candidate["score_breakdown"]) == 4 + assert fund_candidate["evidence"][0]["claim_key"] == "candidate.F001.fund_profile_score" + assert result["fund_scoring_model"]["formula"] == "sum(weight * normalized_score)" assert result["auto_trading"] is False assert "買い推奨" not in str(result) assert "売り推奨" not in str(result) diff --git a/tests/unit/test_investment_samples_smoke.py b/tests/unit/test_investment_samples_smoke.py index f73ae56..6554cf1 100644 --- a/tests/unit/test_investment_samples_smoke.py +++ b/tests/unit/test_investment_samples_smoke.py @@ -31,6 +31,8 @@ def test_investment_sample_csvs_drive_full_api_smoke() -> None: assert analysis["summary"]["market_value"] == 2_514_000.0 assert analysis["summary"]["cost_basis"] == 2_160_000.0 assert analysis["summary"]["annual_income_estimate"] == 12_450.0 + assert analysis["summary"]["edinet_covered_holdings"] == 2 + assert analysis["summary"]["edinet_source_ref"] == str(FINANCIALS_SAMPLE) status, candidates = handle_api( "POST", @@ -53,6 +55,9 @@ def test_investment_sample_csvs_drive_full_api_smoke() -> None: assert "9999" not in codes assert "FND999" not in codes assert candidates["auto_trading"] is False + stock_rows = [item for item in candidates["results"] if item["asset_type"] == "stock"] + assert stock_rows[0]["edinet_summary"]["source_ref"] == str(FINANCIALS_SAMPLE) + assert stock_rows[0]["evidence"][0]["source_ref"] == str(FINANCIALS_SAMPLE) status, report = handle_api( "POST", diff --git a/tests/unit/test_jquants_client.py b/tests/unit/test_jquants_client.py new file mode 100644 index 0000000..5ba954c --- /dev/null +++ b/tests/unit/test_jquants_client.py @@ -0,0 +1,239 @@ +from __future__ import annotations + +from collections.abc import Mapping + +from investment_assistant.jquants.client import ( + JQuantsApiError, + JQuantsClient, + candidate_equity_codes, + normalize_equity_code, +) + + +def test_normalize_equity_code_appends_issue_type_digit() -> None: + assert normalize_equity_code("8306") == "83060" + assert normalize_equity_code("86970") == "86970" + + +def test_candidate_equity_codes_include_visible_and_issue_type_codes() -> None: + assert candidate_equity_codes("9433") == ("94330", "9433") + assert candidate_equity_codes("9433.T") == ("94330", "9433") + assert candidate_equity_codes("86970") == ("86970", "8697") + + +def test_fetch_latest_prices_uses_v2_api_key_and_latest_close() -> None: + captured: dict[str, object] = {} + + def fake_fetch( + path: str, + params: Mapping[str, str], + headers: Mapping[str, str], + ) -> dict[str, object]: + captured["path"] = path + captured["params"] = dict(params) + captured["headers"] = dict(headers) + return { + "bars": [ + {"Code": "83060", "Date": "2026-06-12", "Close": 1200}, + {"Code": "83060", "Date": "2026-06-15", "Close": 1234.5}, + ], + } + + client = JQuantsClient(api_key="unit-key", fetch_json=fake_fetch) + + result = client.fetch_latest_prices(["8306"], lookback_days=3) + + assert result["prices"] == {"8306": 1234.5} + assert result["as_of"] == {"8306": "2026-06-15"} + assert captured["path"] == "/equities/bars/daily" + assert captured["params"]["code"] == "83060" # type: ignore[index] + assert captured["headers"]["x-api-key"] == "unit-key" # type: ignore[index] + assert result["auto_trading"] is False + + +def test_fetch_latest_prices_falls_back_to_visible_four_digit_code() -> None: + tried: list[str] = [] + + def fake_fetch( + path: str, + params: Mapping[str, str], + headers: Mapping[str, str], + ) -> dict[str, object]: + tried.append(params["code"]) + if params["code"] == "94330": + raise JQuantsApiError("J-Quants API returned HTTP 400 for test") + return { + "daily_bars": [ + {"Code": "9433", "Date": "2026-06-12", "Close": 4970}, + ], + } + + client = JQuantsClient(api_key="unit-key", fetch_json=fake_fetch) + + result = client.fetch_latest_prices(["9433"], lookback_days=3) + + assert tried == ["94330", "9433"] + assert result["prices"] == {"9433": 4970.0} + assert result["as_of"] == {"9433": "2026-06-12"} + + +def test_fetch_latest_prices_retries_inside_subscription_window() -> None: + tried: list[dict[str, str]] = [] + + def fake_fetch( + path: str, + params: Mapping[str, str], + headers: Mapping[str, str], + ) -> dict[str, object]: + _ = path, headers + tried.append(dict(params)) + if params.get("to") != "20260324": + raise JQuantsApiError( + "Your subscription covers the following dates: 2024-03-24 ~ 2026-03-24." + ) + return { + "bars": [ + {"Code": "72030", "Date": "2026-03-24", "Close": 2890}, + ], + } + + client = JQuantsClient(api_key="unit-key", fetch_json=fake_fetch) + + result = client.fetch_latest_prices(["7203"], lookback_days=5) + + assert tried[0]["code"] == "72030" + assert tried[1]["from"] == "20260319" + assert tried[1]["to"] == "20260324" + assert result["prices"] == {"7203": 2890.0} + assert result["as_of"] == {"7203": "2026-03-24"} + assert "subscription_window_used" in result["notes"]["7203"] # type: ignore[index] + + +def test_fetch_daily_bars_returns_normalized_ohlcv_summary() -> None: + captured: dict[str, object] = {} + + def fake_fetch( + path: str, + params: Mapping[str, str], + headers: Mapping[str, str], + ) -> dict[str, object]: + captured["path"] = path + captured["params"] = dict(params) + captured["headers"] = dict(headers) + return { + "data": [ + { + "Date": "2026-06-12", + "Code": "94330", + "O": 4900, + "H": 5010, + "L": 4890, + "C": 4970, + "Vo": 1200000, + "Va": 5964000000, + "AdjC": 4970, + }, + { + "Date": "2026-06-15", + "Code": "94330", + "O": 4970, + "H": 5020, + "L": 4950, + "C": 5000, + "Vo": 1300000, + "Va": 6500000000, + "AdjC": 5000, + }, + ], + } + + client = JQuantsClient(api_key="unit-key", fetch_json=fake_fetch) + + result = client.fetch_daily_bars(["9433"], lookback_days=5) + + assert captured["path"] == "/equities/bars/daily" + assert captured["params"]["code"] == "94330" # type: ignore[index] + assert len(result["bars"]) == 2 + assert result["bars"][0]["ticker"] == "9433" + assert result["bars"][0]["volume"] == 1200000.0 + summary = result["summary"]["tickers"]["9433"] # type: ignore[index] + assert summary["latest_close"] == 5000.0 + assert summary["return_pct"] == 0.603622 + + +def test_fetch_daily_bars_bulk_uses_date_range_and_pagination() -> None: + calls: list[dict[str, str]] = [] + + def fake_fetch( + path: str, + params: Mapping[str, str], + headers: Mapping[str, str], + ) -> dict[str, object]: + _ = path, headers + calls.append(dict(params)) + if "pagination_key" not in params: + return { + "data": [ + {"Date": "2026-06-15", "Code": "72030", "C": 3271, "Vo": 100}, + {"Date": "2026-06-15", "Code": "99990", "C": 100, "Vo": 10}, + ], + "pagination_key": "next-page", + } + return { + "data": [ + {"Date": "2026-06-15", "Code": "94330", "C": 2677.5, "Vo": 200}, + ], + } + + client = JQuantsClient(api_key="unit-key", fetch_json=fake_fetch) + + result = client.fetch_daily_bars_bulk(["7203", "9433"], lookback_days=3) + + assert len(calls) == 2 + assert "code" not in calls[0] + assert calls[1]["pagination_key"] == "next-page" + assert result["fetch_mode"] == "bulk_date_range" + assert result["pages_fetched"] == 2 + assert result["rows_returned"] == 3 + assert result["matched_ticker_count"] == 2 + assert [row["ticker"] for row in result["bars"]] == ["7203", "9433"] + + +def test_fetch_daily_bars_retries_inside_subscription_window() -> None: + tried: list[dict[str, str]] = [] + + def fake_fetch( + path: str, + params: Mapping[str, str], + headers: Mapping[str, str], + ) -> dict[str, object]: + _ = path, headers + tried.append(dict(params)) + if params.get("to") != "20260324": + raise JQuantsApiError( + "Your subscription covers the following dates: 2024-03-24 ~ 2026-03-24." + ) + return { + "data": [ + { + "Date": "2026-03-24", + "Code": "94330", + "O": 4800, + "H": 5010, + "L": 4790, + "C": 4970, + "Vo": 1200000, + "AdjC": 4970, + }, + ], + } + + client = JQuantsClient(api_key="unit-key", fetch_json=fake_fetch) + + result = client.fetch_daily_bars(["9433"], lookback_days=7) + + assert tried[1]["from"] == "20260317" + assert tried[1]["to"] == "20260324" + assert len(result["bars"]) == 1 + assert result["bars"][0]["date"] == "2026-03-24" + assert "subscription_window_used" in result["notes"]["9433"] # type: ignore[index] diff --git a/tests/unit/test_portfolio_simulator.py b/tests/unit/test_portfolio_simulator.py index 394bff5..1ce032a 100644 --- a/tests/unit/test_portfolio_simulator.py +++ b/tests/unit/test_portfolio_simulator.py @@ -23,6 +23,18 @@ def _csv(tmp_path: Path, rows: str) -> Path: return path +def _current_yields(tmp_path: Path) -> Path: + path = tmp_path / "current_yields.csv" + path.write_text( + "ticker,name,current_dividend_per_share,current_price,yield_pct,as_of," + "source_ref,provider_id,note\n" + "9433,KDDI,80,2500,3.2,2026-06-15," + "user_verified_current_dividend,user_csv,current price basis\n", + encoding="utf-8", + ) + return path + + def test_dividend_band_bollinger() -> None: band = dividend_band([30.0, 40.0, 50.0]) assert band is not None @@ -96,6 +108,30 @@ def test_simulate_conservative_below_latest_with_history(tmp_path: Path) -> None assert alloc["dividend_per_share"] < 50.0 # conservative band lower +def test_simulator_prefers_current_yield_overlay_for_prediction(tmp_path: Path) -> None: + csv = _csv(tmp_path, "9433,KDDI,2025,1000,68,145,stable\n") + out = simulate_portfolio( + budget=250_000, + holdings=[{"ticker": "9433", "price": 2500}], + auto_weight="equal", + dividend_basis="latest", + financials_csv=str(csv), + current_yields_csv=_current_yields(tmp_path), + ) + + alloc = out["allocations"][0] # type: ignore[index] + assert alloc["dividend_per_share_latest"] == 80.0 + assert alloc["dividend_source"] == "current_dividend_per_share" + assert alloc["annual_dividend"] == 8000 + assert alloc["annual_band_lower"] == 8000 + assert alloc["annual_band_upper"] == 8000 + assert alloc["yield"] == 0.032 + summary = out["summary"] + assert summary["portfolio_yield_latest"] == 0.032 # type: ignore[index] + projection = out["projection"] + assert projection["nominal"][0] == 8000 # type: ignore[index] + + def test_cash_min_never_overspends_with_fractional_price() -> None: # Fractional prices make lot cost non-integer; cash_min must round costs up / # budget down so the real invested can never exceed the budget (cash_left>=0). @@ -145,6 +181,21 @@ def test_build_universe_sorts_by_safety(tmp_path: Path) -> None: assert row["yield_latest"] is not None +def test_build_universe_uses_current_yield_overlay(tmp_path: Path) -> None: + csv = _csv(tmp_path, "9433,KDDI,2025,1000,68,145,stable\n") + universe = build_universe( + str(csv), + prices={"9433": 2500}, + current_yields_csv=_current_yields(tmp_path), + ) + + row = next(r for r in universe if r["ticker"] == "9433") + assert row["dividend_latest"] == 80.0 + assert row["dividend_latest_edinet"] == 145.0 + assert row["yield_latest"] == 0.032 + assert row["yield_basis"] == "current_fact" + + def test_optimize_cash_min_minimises_leftover() -> None: # Budget 1,200,000 with lots of 700,000 and 600,000: weight-floor would buy # one 700k lot (cash 500k), but cash_min buys two 600k lots (cash 0). @@ -369,6 +420,16 @@ def fake(url: str) -> str: assert prices["8306"] == 123.0 and prices["9432"] == 123.0 +def test_fetch_prices_records_missing_close_note() -> None: + def fake(url: str) -> str: + _ = url + return "Symbol,Date,Time,Open,High,Low,Close,Volume\nX.JP,N/D,N/D,N/D,N/D,N/D,N/D,N/D\n" + + out = fetch_prices(["7203"], fetch=fake) + assert out["prices"]["7203"] is None # type: ignore[index] + assert out["notes"]["7203"] == "no_close_price_returned" # type: ignore[index] + + def test_fetch_prices_records_errors() -> None: def boom(url: str) -> str: raise RuntimeError("net") diff --git a/tests/unit/test_rag.py b/tests/unit/test_rag.py index a9e203f..7e2fa57 100644 --- a/tests/unit/test_rag.py +++ b/tests/unit/test_rag.py @@ -2,7 +2,12 @@ from investment_assistant.cli import run_rag_index_dir from investment_assistant.rag.chunker import chunk_text, load_document -from investment_assistant.rag.search import build_answer_context, search_chunks +from investment_assistant.rag.search import ( + build_answer_context, + decompose_query, + enhanced_search, + search_chunks, +) from investment_assistant.rag.store import RagStore @@ -84,6 +89,59 @@ def test_search_chunks_scores_and_limits_results(tmp_path) -> None: assert "投資判断" in results[0].text +def test_decompose_query_keeps_original_and_separator_phrases() -> None: + variants = decompose_query("DOE, payout policy and dividend", max_queries=4) + + assert variants[0] == "DOE, payout policy and dividend" + assert "DOE" in variants + assert any("payout" in variant for variant in variants) + + +def test_enhanced_search_returns_rrf_diagnostics(tmp_path) -> None: + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text( + "DOE policy and payout ratio are shown in the dividend policy.", + encoding="utf-8", + ) + second.write_text( + "Capital allocation memo covers operating cash flow and dividends.", + encoding="utf-8", + ) + store = RagStore(tmp_path / "rag.sqlite") + for path in (first, second): + document = load_document(path) + store.upsert_document( + document, + chunk_text( + source=document.source, + text=document.text, + content_hash=document.content_hash, + max_chars=120, + overlap_chars=0, + ), + ) + + payload = enhanced_search( + store, + query="DOE, payout ratio", + limit=2, + hybrid=False, + query_expansion=True, + max_queries=3, + ) + + results = payload["results"] + diagnostics = payload["diagnostics"] + assert isinstance(results, list) + assert results + assert isinstance(diagnostics, dict) + assert diagnostics["mode"] == "enhanced_lexical" + assert diagnostics["query_count"] >= 2 + assert diagnostics["candidate_count"] >= 1 + assert results[0].metadata["ranking_method"] == "reciprocal_rank_fusion" + + def test_build_answer_context_formats_citations(tmp_path) -> None: path = tmp_path / "memo.md" path.write_text("自動売買は行いません。", encoding="utf-8") diff --git a/tests/unit/test_webapi.py b/tests/unit/test_webapi.py index 6fa2d8a..25c0618 100644 --- a/tests/unit/test_webapi.py +++ b/tests/unit/test_webapi.py @@ -1,5 +1,7 @@ from __future__ import annotations +import base64 +import json from pathlib import Path from investment_assistant.rag.chunker import chunk_text, load_document @@ -39,6 +41,7 @@ def test_edinet_ingest_route_is_registered_and_routed(monkeypatch) -> None: from investment_assistant.webapi import service assert "POST /api/edinet/ingest" in available_routes() + assert "GET /api/edinet/status" in available_routes() captured: dict[str, object] = {} @@ -58,6 +61,1126 @@ def fake_ingest(**kwargs: object) -> dict[str, object]: assert captured["days"] == 5 +def test_edinet_status_reports_api_key_configuration(monkeypatch) -> None: + from investment_assistant.webapi import service + + monkeypatch.setattr(service, "_EDINET_API_KEY_RUNTIME_SET", False) + monkeypatch.setenv("EDINET_API_KEY", "dummy-key") + + status, payload = handle_api("GET", "/api/edinet/status") + + assert status == 200 + assert payload["api_key_configured"] is True + assert payload["api_key_source"] == "process_env" + assert payload["default_financials_csv"] == "local_docs/edinet/financials.csv" + assert payload["auto_trading"] is False + + +def test_edinet_api_key_can_be_set_for_runtime_without_echo(monkeypatch) -> None: + from investment_assistant.webapi import service + + monkeypatch.setattr(service, "_EDINET_API_KEY_RUNTIME_SET", False) + monkeypatch.delenv("EDINET_API_KEY", raising=False) + + status, payload = handle_api( + "POST", + "/api/edinet/api-key", + {"api_key": "runtime-secret"}, + ) + + assert status == 200 + assert payload["api_key_configured"] is True + assert payload["api_key_source"] == "runtime_input" + assert payload["request_api_key_applied"] is True + assert "runtime-secret" not in str(payload) + + +def test_jquants_status_reports_runtime_key_without_echo(tmp_path: Path, monkeypatch) -> None: + from investment_assistant.webapi import service + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service, "_JQUANTS_API_KEY_RUNTIME_SET", False) + monkeypatch.setattr(service, "_JQUANTS_CONTRACT_RUNTIME_ACK", False) + monkeypatch.delenv("JQUANTS_REFRESH_TOKEN", raising=False) + monkeypatch.delenv("JQUANTS_API_KEY", raising=False) + monkeypatch.delenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", raising=False) + + status, missing = handle_api("GET", "/api/jquants/status") + + assert status == 200 + assert missing["api_key_configured"] is False + assert missing["production_allowed"] is False + assert "GET /api/jquants/status" in available_routes() + + status, payload = handle_api( + "POST", + "/api/jquants/api-key", + {"api_key": "unit-test-jquants-token"}, + ) + + assert status == 200 + assert payload["api_key_configured"] is True + assert payload["api_key_source"] == "runtime_input" + assert payload["contract_acknowledged"] is False + assert payload["production_allowed"] is False + assert "unit-test-jquants-token" not in json.dumps(payload, ensure_ascii=False) + + +def test_jquants_contract_ack_unlocks_provider_policy(tmp_path: Path, monkeypatch) -> None: + from investment_assistant.webapi import service + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service, "_JQUANTS_API_KEY_RUNTIME_SET", False) + monkeypatch.setattr(service, "_JQUANTS_CONTRACT_RUNTIME_ACK", False) + monkeypatch.delenv("JQUANTS_REFRESH_TOKEN", raising=False) + monkeypatch.delenv("JQUANTS_API_KEY", raising=False) + monkeypatch.delenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", raising=False) + + status, payload = handle_api( + "POST", + "/api/jquants/api-key", + { + "refresh_token": "unit-test-jquants-token", + "contract_acknowledged": True, + }, + ) + + assert status == 200 + assert payload["api_key_configured"] is True + assert payload["contract_acknowledged"] is True + assert payload["production_allowed"] is True + assert "unit-test-jquants-token" not in json.dumps(payload, ensure_ascii=False) + + status, ledger = handle_api( + "POST", + "/api/providers/policy", + {"runtime_mode": "production", "provider_ids": ["jquants"]}, + ) + + assert status == 200 + assert ledger["providers"][0]["runtime_decision"] == "allowed" + + +def test_jquants_api_key_can_be_persisted_to_local_dotenv( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.webapi import service + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(service, "_JQUANTS_API_KEY_RUNTIME_SET", False) + monkeypatch.setattr(service, "_JQUANTS_CONTRACT_RUNTIME_ACK", False) + monkeypatch.delenv("JQUANTS_REFRESH_TOKEN", raising=False) + monkeypatch.delenv("JQUANTS_API_KEY", raising=False) + monkeypatch.delenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", raising=False) + + status, payload = handle_api( + "POST", + "/api/jquants/api-key", + { + "api_key": "unit-test-jquants-token", + "contract_acknowledged": True, + "persist_local": True, + }, + ) + + dotenv = (tmp_path / ".env.local").read_text(encoding="utf-8") + assert status == 200 + assert payload["persisted_local"] is True + assert payload["persisted_path"] == ".env.local" + assert "unit-test-jquants-token" not in json.dumps(payload, ensure_ascii=False) + assert 'JQUANTS_API_KEY="unit-test-jquants-token"' in dotenv + assert 'JQUANTS_REFRESH_TOKEN="unit-test-jquants-token"' in dotenv + assert 'INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS="jquants"' in dotenv + + +def test_financials_refresh_with_edinet_key_updates_structured_csv( + monkeypatch, + tmp_path: Path, +) -> None: + from investment_assistant.webapi import service + + monkeypatch.setenv("EDINET_API_KEY", "dummy-key") + captured: dict[str, object] = {} + + def fake_ingest(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return { + "ingested_count": 1, + "targets_count": 225, + "financials_csv": str(tmp_path / "edinet" / "financials.csv"), + "results": [], + } + + monkeypatch.setattr(service.cli, "run_edinet_ingest", fake_ingest) + + status, payload = handle_api( + "POST", + "/api/financials/refresh", + { + "registry_path": "examples/source_registry_nikkei225_edinet.yaml", + "output_dir": str(tmp_path / "edinet"), + "days": 7, + "db_path": str(tmp_path / "rag.sqlite"), + }, + ) + + assert status == 200 + assert payload["mode"] == "edinet_api" + assert payload["api_key_configured"] is True + assert payload["financials_updated"] is True + assert payload["financials_csv"] == str(tmp_path / "edinet" / "financials.csv") + assert captured["days"] == 7 + assert "dummy-key" not in str(payload) + + +def test_financials_refresh_without_key_scrapes_only_official_pages( + monkeypatch, + tmp_path: Path, +) -> None: + from investment_assistant.webapi import service + + monkeypatch.delenv("EDINET_API_KEY", raising=False) + monkeypatch.chdir(tmp_path) + calls: list[bool] = [] + + def fake_run_fetch_job(*, path, dry_run: bool, preview_chars: int = 500): + _ = path, preview_chars + calls.append(dry_run) + return { + "results": [ + { + "name": "edinet_portal", + "url": "https://disclosure2.edinet-fsa.go.jp/", + "output_path": "local_docs/disclosure/edinet_portal.txt", + "fetch": { + "allowed_by_robots": True, + "source": "dry_run" if dry_run else "network", + "saved_path": None + if dry_run + else "local_docs/disclosure/edinet_portal.txt", + }, + } + ] + } + + def fake_index_dir(*, path, db_path): + assert path == "local_docs" + assert db_path == str(tmp_path / "rag.sqlite") + return {"files_indexed": 1, "chunks_indexed": 2} + + monkeypatch.setattr(service.cli, "run_fetch_job", fake_run_fetch_job) + monkeypatch.setattr(service.cli, "run_rag_index_dir", fake_index_dir) + + status, payload = handle_api( + "POST", + "/api/financials/refresh", + { + "output_dir": str(tmp_path / "edinet"), + "db_path": str(tmp_path / "rag.sqlite"), + }, + ) + + assert status == 200 + assert calls == [True, False] + assert payload["mode"] == "disclosure_scrape_only" + assert payload["api_key_configured"] is False + assert payload["financials_updated"] is False + assert payload["financials_csv"] == str(tmp_path / "edinet" / "financials.csv") + assert payload["scrape"]["allowed_sources_count"] == 1 + assert "EDINET API KEY" in payload["hint"] + + +def test_prime_registry_generates_edinet_targets_from_jpx_listed(tmp_path: Path) -> None: + from investment_assistant.edinet.registry import build_edinet_targets_from_registry + + listed = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,7203,トヨタ自動車,プライム（内国株式）,輸送用機器\n" + "2026-05-31,8001,伊藤忠商事,プライム（内国株式）,卸売業\n" + "2026-05-31,9999,サンプルスタンダード,スタンダード（内国株式）,サービス業\n" + ) + listed_path = tmp_path / "listed_issues.csv" + listed_path.write_text(listed, encoding="utf-8") + registry_path = tmp_path / "source_registry_tse_prime_edinet.yaml" + + status, payload = handle_api( + "POST", + "/api/financials/prime-registry", + { + "jpx_listed_path": str(listed_path), + "registry_path": str(registry_path), + "max_targets": 10, + }, + ) + + assert status == 200 + assert payload["available"] is True + assert payload["count"] == 2 + assert payload["total_prime_count"] == 2 + assert registry_path.is_file() + targets = build_edinet_targets_from_registry(registry_path) + assert [target.ticker for target in targets] == ["7203", "8001"] + assert targets[0].company == "トヨタ自動車" + + +def test_listed_registry_generates_domestic_stock_targets_from_jpx_listed( + tmp_path: Path, +) -> None: + from investment_assistant.edinet.registry import build_edinet_targets_from_registry + from investment_assistant.investment.universe import ListedIssue, write_jpx_listed_issues + + listed_path = tmp_path / "listed_issues.csv" + write_jpx_listed_issues( + [ + ListedIssue("7203", "トヨタ自動車", "プライム（内国株式）", "輸送用機器"), + ListedIssue("9999", "標準サンプル", "スタンダード（内国株式）", "サービス業"), + ListedIssue("4478", "成長サンプル", "グロース（国内株式）", "情報・通信業"), + ListedIssue("1306", "ETFサンプル", "ETF・ETN", "ETF"), + ListedIssue("8951", "REITサンプル", "REIT・ベンチャーファンド", "REIT"), + ListedIssue("9998", "外国株サンプル", "プライム（外国株式）", "小売業"), + ], + listed_path, + ) + registry_path = tmp_path / "source_registry_all_domestic_edinet.yaml" + + status, payload = handle_api( + "POST", + "/api/financials/listed-registry", + { + "jpx_listed_path": str(listed_path), + "registry_path": str(registry_path), + "max_targets": 0, + }, + ) + + assert status == 200 + assert payload["available"] is True + assert payload["scope"] == "domestic_stocks" + assert payload["count"] == 3 + assert payload["eligible_count"] == 3 + assert payload["scope_total_count"] == 3 + assert payload["total_prime_count"] == 2 + assert registry_path.is_file() + targets = build_edinet_targets_from_registry(registry_path) + assert [target.ticker for target in targets] == ["4478", "7203", "9999"] + + +def test_missing_registry_generates_only_uncovered_financial_targets( + tmp_path: Path, +) -> None: + from investment_assistant.edinet.registry import build_edinet_targets_from_registry + from investment_assistant.investment.universe import ListedIssue, write_jpx_listed_issues + + listed_path = tmp_path / "listed_issues.csv" + write_jpx_listed_issues( + [ + ListedIssue("7203", "Toyota", "Prime Market (Domestic Stock)", "Transportation"), + ListedIssue("9999", "Standard Sample", "Standard Market (Domestic Stock)", "Services"), + ListedIssue("4478", "Growth Sample", "Growth Market (Domestic Stock)", "IT"), + ], + listed_path, + ) + base_registry_path = tmp_path / "all_domestic.yaml" + status, _ = handle_api( + "POST", + "/api/financials/listed-registry", + { + "jpx_listed_path": str(listed_path), + "registry_path": str(base_registry_path), + "max_targets": 0, + }, + ) + assert status == 200 + + financials_csv = tmp_path / "financials.csv" + financials_csv.write_text( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "7203,Toyota,2025,100,55,75,stable\n", + encoding="utf-8", + ) + missing_registry_path = tmp_path / "missing.yaml" + + status, payload = handle_api( + "POST", + "/api/financials/missing-registry", + { + "registry_path": str(base_registry_path), + "financials_csv": str(financials_csv), + "missing_registry_path": str(missing_registry_path), + "max_targets": 0, + }, + ) + + assert status == 200 + assert payload["available"] is True + assert payload["registry_count"] == 3 + assert payload["existing_count"] == 1 + assert payload["missing_count"] == 2 + assert payload["count"] == 2 + targets = build_edinet_targets_from_registry(missing_registry_path) + assert [target.ticker for target in targets] == ["4478", "9999"] + + +def test_prime_refresh_generates_registry_and_delegates_to_financials_refresh( + monkeypatch, + tmp_path: Path, +) -> None: + from investment_assistant.webapi import service + + listed = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,8001,伊藤忠商事,プライム（内国株式）,卸売業\n" + ) + listed_path = tmp_path / "listed_issues.csv" + listed_path.write_text(listed, encoding="utf-8") + registry_path = tmp_path / "prime.yaml" + captured: dict[str, object] = {} + + def fake_refresh(body: dict[str, object]) -> dict[str, object]: + captured.update(body) + return { + "mode": "edinet_api", + "financials_updated": True, + "financials_csv": str(tmp_path / "edinet" / "financials.csv"), + "ingested_count": 1, + "targets_count": 1, + } + + monkeypatch.setattr(service, "_financials_refresh", fake_refresh) + + status, payload = handle_api( + "POST", + "/api/financials/prime-refresh", + { + "jpx_listed_path": str(listed_path), + "registry_path": str(registry_path), + "output_dir": str(tmp_path / "edinet"), + "days": 7, + }, + ) + + assert status == 200 + assert payload["financials_updated"] is True + assert captured["registry_path"] == str(registry_path) + assert registry_path.is_file() + assert payload["prime_registry"]["count"] == 1 + + +def test_listed_refresh_generates_registry_and_delegates_to_financials_refresh( + monkeypatch, + tmp_path: Path, +) -> None: + from investment_assistant.investment.universe import ListedIssue, write_jpx_listed_issues + from investment_assistant.webapi import service + + listed_path = tmp_path / "listed_issues.csv" + write_jpx_listed_issues( + [ + ListedIssue("7203", "トヨタ自動車", "プライム（内国株式）", "輸送用機器"), + ListedIssue("9999", "標準サンプル", "スタンダード（内国株式）", "サービス業"), + ListedIssue("1306", "ETFサンプル", "ETF・ETN", "ETF"), + ], + listed_path, + ) + registry_path = tmp_path / "all_domestic.yaml" + captured: dict[str, object] = {} + + def fake_refresh(body: dict[str, object]) -> dict[str, object]: + captured.update(body) + return { + "mode": "edinet_api", + "financials_updated": True, + "financials_csv": str(tmp_path / "edinet" / "financials.csv"), + "ingested_count": 2, + "targets_count": 2, + } + + monkeypatch.setattr(service, "_financials_refresh", fake_refresh) + + status, payload = handle_api( + "POST", + "/api/financials/listed-refresh", + { + "jpx_listed_path": str(listed_path), + "registry_path": str(registry_path), + "output_dir": str(tmp_path / "edinet"), + "days": 7, + "max_targets": 0, + }, + ) + + assert status == 200 + assert payload["financials_updated"] is True + assert captured["registry_path"] == str(registry_path) + assert registry_path.is_file() + assert payload["jpx_registry"]["count"] == 2 + assert payload["jpx_registry"]["scope"] == "domestic_stocks" + + +def test_missing_refresh_delegates_only_missing_targets( + monkeypatch, + tmp_path: Path, +) -> None: + from investment_assistant.edinet.registry import build_edinet_targets_from_registry + from investment_assistant.investment.universe import ListedIssue, write_jpx_listed_issues + from investment_assistant.webapi import service + + listed_path = tmp_path / "listed_issues.csv" + write_jpx_listed_issues( + [ + ListedIssue("7203", "Toyota", "Prime Market (Domestic Stock)", "Transportation"), + ListedIssue("9999", "Standard Sample", "Standard Market (Domestic Stock)", "Services"), + ], + listed_path, + ) + base_registry_path = tmp_path / "all_domestic.yaml" + status, _ = handle_api( + "POST", + "/api/financials/listed-registry", + { + "jpx_listed_path": str(listed_path), + "registry_path": str(base_registry_path), + "max_targets": 0, + }, + ) + assert status == 200 + + financials_csv = tmp_path / "financials.csv" + financials_csv.write_text( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "7203,Toyota,2025,100,55,75,stable\n", + encoding="utf-8", + ) + missing_registry_path = tmp_path / "missing.yaml" + captured: dict[str, object] = {} + + def fake_refresh(body: dict[str, object]) -> dict[str, object]: + captured.update(body) + return { + "mode": "edinet_api", + "financials_updated": True, + "financials_csv": str(financials_csv), + "ingested_count": 1, + "targets_count": 1, + } + + monkeypatch.setattr(service, "_financials_refresh", fake_refresh) + + status, payload = handle_api( + "POST", + "/api/financials/missing-refresh", + { + "registry_path": str(base_registry_path), + "financials_csv": str(financials_csv), + "missing_registry_path": str(missing_registry_path), + "max_targets": 0, + "output_dir": str(tmp_path), + }, + ) + + assert status == 200 + assert payload["financials_updated"] is True + assert captured["registry_path"] == str(missing_registry_path) + assert payload["missing_registry"]["missing_count"] == 1 + targets = build_edinet_targets_from_registry(missing_registry_path) + assert [target.ticker for target in targets] == ["9999"] + + +def test_missing_sources_plan_includes_non_edinet_evidence_sources(tmp_path: Path) -> None: + from investment_assistant.investment.universe import ListedIssue, write_jpx_listed_issues + + listed_path = tmp_path / "listed_issues.csv" + write_jpx_listed_issues( + [ + ListedIssue("7203", "Toyota", "Prime Market (Domestic Stock)", "Transportation"), + ListedIssue("9999", "Standard Sample", "Standard Market (Domestic Stock)", "Services"), + ], + listed_path, + ) + base_registry_path = tmp_path / "all_domestic.yaml" + status, _ = handle_api( + "POST", + "/api/financials/listed-registry", + { + "jpx_listed_path": str(listed_path), + "registry_path": str(base_registry_path), + "max_targets": 0, + }, + ) + assert status == 200 + + financials_csv = tmp_path / "financials.csv" + financials_csv.write_text( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "7203,Toyota,2025,100,55,75,stable\n", + encoding="utf-8", + ) + + status, payload = handle_api( + "POST", + "/api/financials/missing-sources", + { + "registry_path": str(base_registry_path), + "financials_csv": str(financials_csv), + "missing_registry_path": str(tmp_path / "missing.yaml"), + "max_targets": 0, + }, + ) + + assert status == 200 + assert payload["mode"] == "source_agnostic_missing_plan" + assert payload["registry"]["missing_count"] == 1 + source_names = {source["name"] for source in payload["sources"]} + assert {"tdnet_missing_financials", "jpx_listing_evidence"} <= source_names + evidence_names = {source["name"] for source in payload["evidence_sources"]} + assert {"TDnet", "JPX", "issuer_ir_pdf_html"} <= evidence_names + + +def test_missing_auto_refresh_without_edinet_key_uses_official_evidence( + monkeypatch, + tmp_path: Path, +) -> None: + from investment_assistant.investment.universe import ListedIssue, write_jpx_listed_issues + from investment_assistant.webapi import service + + monkeypatch.delenv("EDINET_API_KEY", raising=False) + monkeypatch.setattr(service, "_ensure_env_from_dotenv", lambda _key: False) + listed_path = tmp_path / "listed_issues.csv" + write_jpx_listed_issues( + [ListedIssue("9999", "Standard Sample", "Standard Market (Domestic Stock)", "Services")], + listed_path, + ) + base_registry_path = tmp_path / "all_domestic.yaml" + status, _ = handle_api( + "POST", + "/api/financials/listed-registry", + { + "jpx_listed_path": str(listed_path), + "registry_path": str(base_registry_path), + "max_targets": 0, + }, + ) + assert status == 200 + captured: dict[str, object] = {} + + def fake_fetch_job_auto(body: dict[str, object]) -> dict[str, object]: + captured.update(body) + return { + "status": "completed", + "allowed_sources_count": len(body.get("sources", [])), + } + + monkeypatch.setattr(service, "_fetch_job_auto", fake_fetch_job_auto) + + status, payload = handle_api( + "POST", + "/api/financials/missing-auto-refresh", + { + "registry_path": str(base_registry_path), + "financials_csv": str(tmp_path / "financials.csv"), + "missing_registry_path": str(tmp_path / "missing.yaml"), + "max_targets": 1, + "index_after_fetch": False, + }, + ) + + assert status == 200 + assert payload["source_strategy"] == "official_disclosure_rag" + assert payload["evidence_updated"] is True + assert payload["financials_updated"] is False + assert len(captured["sources"]) == 2 + + +def test_edinet_status_reads_local_dotenv_without_exposing_key( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.webapi import service + + monkeypatch.setattr(service, "_EDINET_API_KEY_RUNTIME_SET", False) + monkeypatch.delenv("EDINET_API_KEY", raising=False) + monkeypatch.chdir(tmp_path) + (tmp_path / ".env").write_text("EDINET_API_KEY=from-dotenv\n", encoding="utf-8") + + status, payload = handle_api("GET", "/api/edinet/status") + + assert status == 200 + assert payload["api_key_configured"] is True + assert payload["api_key_source"] == "dotenv" + assert "from-dotenv" not in str(payload) + + +def test_financials_import_saves_manual_csv_and_searches_securities( + tmp_path: Path, +) -> None: + csv_text = ( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "8306,MUFG,2023,1000,45,40,stable\n" + "8306,MUFG,2024,1200,48,45,stable\n" + "7203,Toyota,2024,3000,55,60,stable\n" + ) + output_path = tmp_path / "financials.csv" + + status, imported = handle_api( + "POST", + "/api/financials/import", + { + "csv_text": csv_text, + "save": True, + "output_path": str(output_path), + }, + ) + + assert status == 200 + assert imported["saved_path"] == str(output_path) + assert imported["count"] == 3 + assert imported["company_count"] == 2 + assert imported["auto_trading"] is False + assert output_path.is_file() + + status, searched = handle_api( + "POST", + "/api/financials/securities", + {"financials_csv": str(output_path), "query": "8306"}, + ) + + assert status == 200 + assert searched["count"] == 1 + assert searched["securities"][0]["ticker"] == "8306" + assert searched["securities"][0]["name"] == "MUFG" + + +def test_financials_import_corrects_dividend_unit_jump_before_saving( + tmp_path: Path, +) -> None: + csv_text = ( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "8306,MUFG,2024,1200,48,41,stable\n" + "8306,MUFG,2025,1300,49,4100,stable\n" + ) + output_path = tmp_path / "financials.csv" + + status, imported = handle_api( + "POST", + "/api/financials/import", + {"csv_text": csv_text, "save": True, "output_path": str(output_path)}, + ) + + assert status == 200 + quality = imported["dividend_quality"] + assert isinstance(quality, dict) + assert quality["status"] == "corrected" + assert quality["corrected_count"] == 1 + assert "4100" not in output_path.read_text(encoding="utf-8") + company = imported["comparison"]["companies"][0] # type: ignore[index] + assert company["dividend_series"] == [41.0, 41.0] + + +def test_financials_status_reports_saved_data(tmp_path: Path) -> None: + csv_text = ( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "8306,MUFG,2023,1000,45,40,stable\n" + "8306,MUFG,2024,1200,48,45,stable\n" + "7203,Toyota,2024,3000,55,60,stable\n" + ) + output_path = tmp_path / "financials.csv" + output_path.write_text(csv_text, encoding="utf-8") + + status, payload = handle_api( + "POST", + "/api/financials/status", + {"path": str(output_path), "stale_after_days": 3650}, + ) + + assert status == 200 + assert payload["available"] is True + assert payload["status"] == "fresh" + assert payload["path"] == str(output_path) + assert payload["point_count"] == 3 + assert payload["company_count"] == 2 + assert payload["latest_fiscal_year"] == 2024 + assert payload["modified_at"] + + +def test_financials_status_reports_missing_data(tmp_path: Path) -> None: + missing = tmp_path / "missing-financials.csv" + + status, payload = handle_api( + "POST", + "/api/financials/status", + {"path": str(missing)}, + ) + + assert status == 200 + assert payload["available"] is False + assert payload["status"] == "missing" + assert payload["point_count"] == 0 + assert payload["company_count"] == 0 + assert "財務データ" in payload["hint"] + + +def test_financials_securities_missing_csv_returns_empty_result(tmp_path: Path) -> None: + missing = tmp_path / "missing-financials.csv" + + status, payload = handle_api( + "POST", + "/api/financials/securities", + {"financials_csv": str(missing), "query": "7203"}, + ) + + assert status == 200 + assert payload["available"] is False + assert payload["count"] == 0 + assert payload["securities"] == [] + assert payload["source_ref"] == str(missing) + assert "財務データ" in payload["hint"] + + +def test_portfolio_universe_missing_csv_returns_empty_result(tmp_path: Path) -> None: + missing = tmp_path / "missing-financials.csv" + + status, payload = handle_api( + "POST", + "/api/portfolio/universe", + {"financials_csv": str(missing), "scope": "financials"}, + ) + + assert status == 200 + assert payload["available"] is False + assert payload["count"] == 0 + assert payload["universe"] == [] + assert payload["source_ref"] == str(missing) + assert "財務データ" in payload["hint"] + + +def test_portfolio_universe_can_select_prime_rows_without_financials( + tmp_path: Path, +) -> None: + listed = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,8001,伊藤忠商事,プライム（内国株式）,卸売業\n" + "2026-05-31,9999,サンプルスタンダード,スタンダード（内国株式）,サービス業\n" + ) + listed_path = tmp_path / "listed_issues.csv" + listed_path.write_text(listed, encoding="utf-8") + empty_registry = tmp_path / "nikkei.yaml" + empty_registry.write_text("sources: []\n", encoding="utf-8") + + status, payload = handle_api( + "POST", + "/api/portfolio/universe", + { + "financials_csv": str(tmp_path / "missing-financials.csv"), + "jpx_listed_path": str(listed_path), + "nikkei225_registry": str(empty_registry), + "scope": "prime", + }, + ) + + assert status == 200 + assert payload["available"] is True + assert payload["count"] == 1 + assert payload["financials_available"] is False + assert payload["universe"][0]["ticker"] == "8001" + assert payload["universe"][0]["market_segment"] == "プライム（国内株式）" + assert payload["universe"][0]["has_financials"] is False + assert payload["universe"][0]["selection_mode"] == "manual_input" + assert payload["universe"][0]["yield_basis"] == "manual_required" + assert "JPX" in payload["hint"] + + +def test_jpx_listed_import_and_market_universe_prime_scope(tmp_path: Path) -> None: + financials = tmp_path / "financials.csv" + financials.write_text( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "7203,Toyota,2024,1000,55,60,stable\n" + "8306,MUFG,2024,1200,48,45,stable\n" + "9999,Standard Sample,2024,90,18,10,unstable\n", + encoding="utf-8", + ) + listed = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,7203,トヨタ自動車,プライム（内国株式）,輸送用機器\n" + "2026-05-31,8306,三菱ＵＦＪフィナンシャル・グループ,プライム（内国株式）,銀行業\n" + "2026-05-31,9999,サンプルスタンダード,スタンダード（内国株式）,サービス業\n" + ) + listed_path = tmp_path / "listed_issues.csv" + + status, imported = handle_api( + "POST", + "/api/market/jpx-listed/import", + {"csv_text": listed, "output_path": str(listed_path), "save": True}, + ) + + assert status == 200 + assert imported["count"] == 3 + assert imported["prime_count"] == 2 + assert listed_path.is_file() + + status, universe = handle_api( + "POST", + "/api/market/universe", + { + "financials_csv": str(financials), + "jpx_listed_path": str(listed_path), + "scope": "prime", + "query": "", + "limit": 10, + }, + ) + + assert status == 200 + codes = {str(item["ticker"]) for item in universe["securities"]} + assert codes == {"7203", "8306"} + rows = {str(item["ticker"]): item for item in universe["securities"]} + assert rows["8306"]["is_prime"] is True + assert rows["8306"]["is_nikkei225"] is True + assert rows["8306"]["has_financials"] is True + assert rows["8306"]["market_segment"] == "プライム（国内株式）" + assert rows["8306"]["market_segment_raw"] == "プライム（内国株式）" + assert rows["8306"]["market_segment_label"] == "プライム（国内株式）" + assert universe["auto_trading"] is False + assert universe["call_real_api"] is False + + +def test_companies_refresh_builds_company_master_from_jpx_listed( + tmp_path: Path, +) -> None: + from investment_assistant.investment.universe import ListedIssue, write_jpx_listed_issues + + listed_path = tmp_path / "listed_issues.csv" + write_jpx_listed_issues( + [ + ListedIssue("7203", "Toyota", "Prime Market (Domestic Stock)", "Transportation"), + ListedIssue("9999", "Standard Sample", "Standard Market (Domestic Stock)", "Services"), + ListedIssue("1306", "ETF Sample", "ETF・ETN", "ETF"), + ListedIssue("8951", "REIT Sample", "REIT・ベンチャーファンド", "REIT"), + ListedIssue("9998", "Foreign Sample", "Prime Market (Foreign Stock)", "Retail"), + ], + listed_path, + ) + financials = tmp_path / "financials.csv" + financials.write_text( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "7203,Toyota,2024,1000,55,60,stable\n", + encoding="utf-8", + ) + nikkei = tmp_path / "nikkei.yaml" + nikkei.write_text( + "sources:\n" + ' - name: "7203"\n' + ' ticker: "7203"\n' + ' company: "Toyota"\n' + ' source_type: "public_api"\n' + ' provider: "edinet"\n' + " allowed: true\n", + encoding="utf-8", + ) + output_path = tmp_path / "company_master.csv" + + status, payload = handle_api( + "POST", + "/api/companies/refresh", + { + "jpx_listed_path": str(listed_path), + "financials_csv": str(financials), + "nikkei225_registry": str(nikkei), + "output_path": str(output_path), + }, + ) + + assert status == 200 + assert payload["available"] is True + assert payload["count"] == 5 + assert payload["company_count"] == 3 + assert payload["domestic_stock_count"] == 2 + assert payload["financials_count"] == 1 + assert payload["nikkei225_count"] == 1 + assert output_path.read_bytes().startswith(b"\xef\xbb\xbf") + rows = {str(row["ticker"]): row for row in payload["sample"]} + assert rows["7203"]["entity_type"] == "domestic_stock" + assert rows["7203"]["has_financials"] is True + assert rows["1306"]["entity_type"] == "etf_etn" + assert rows["8951"]["entity_type"] == "reit" + + status, status_payload = handle_api("POST", "/api/companies/status", {"path": str(output_path)}) + assert status == 200 + assert status_payload["available"] is True + assert status_payload["count"] == 5 + assert status_payload["company_count"] == 3 + + +def test_market_universe_includes_prime_non_nikkei_without_financials(tmp_path: Path) -> None: + listed = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,7203,トヨタ自動車,プライム（内国株式）,輸送用機器\n" + "2026-05-31,8001,伊藤忠商事,プライム（内国株式）,卸売業\n" + ) + listed_path = tmp_path / "listed_issues.csv" + listed_path.write_text(listed, encoding="utf-8") + nikkei_registry = tmp_path / "nikkei.yaml" + nikkei_registry.write_text( + "sources:\n" + ' - name: "7203"\n' + ' ticker: "7203"\n' + ' company: "Toyota"\n' + ' source_type: "public_api"\n' + ' provider: "edinet"\n' + " allowed: true\n", + encoding="utf-8", + ) + + status, universe = handle_api( + "POST", + "/api/market/universe", + { + "financials_csv": str(tmp_path / "missing.csv"), + "jpx_listed_path": str(listed_path), + "nikkei225_registry": str(nikkei_registry), + "scope": "prime", + "limit": 10, + }, + ) + + assert status == 200 + rows = {str(item["ticker"]): item for item in universe["securities"]} + assert {"7203", "8001"} <= set(rows) + assert rows["8001"]["is_prime"] is True + assert rows["8001"]["is_nikkei225"] is False + assert rows["8001"]["has_financials"] is False + assert rows["8001"]["market_segment"] == "プライム（国内株式）" + + +def test_market_universe_domestic_stocks_excludes_etf_and_reit(tmp_path: Path) -> None: + listed = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,7203,トヨタ自動車,プライム（国内株式）,輸送用機器\n" + "2026-05-31,1306,ETFサンプル,ETF・ETN,ETF\n" + "2026-05-31,8951,REITサンプル,REIT・ベンチャーファンド,REIT\n" + ) + listed_path = tmp_path / "listed_issues.csv" + listed_path.write_text(listed, encoding="utf-8") + + status, universe = handle_api( + "POST", + "/api/market/universe", + { + "financials_csv": str(tmp_path / "missing.csv"), + "jpx_listed_path": str(listed_path), + "nikkei225_registry": str(tmp_path / "missing_nikkei.yaml"), + "scope": "domestic_stocks", + "limit": 10, + }, + ) + + assert status == 200 + assert [item["ticker"] for item in universe["securities"]] == ["7203"] + assert universe["total_count"] == 1 + + +def test_financials_securities_can_search_jpx_prime_rows_without_financials( + tmp_path: Path, +) -> None: + listed = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,8001,伊藤忠商事,プライム（内国株式）,卸売業\n" + ) + listed_path = tmp_path / "listed_issues.csv" + listed_path.write_text(listed, encoding="utf-8") + empty_registry = tmp_path / "nikkei.yaml" + empty_registry.write_text("sources: []\n", encoding="utf-8") + + status, payload = handle_api( + "POST", + "/api/financials/securities", + { + "financials_csv": str(tmp_path / "missing.csv"), + "jpx_listed_path": str(listed_path), + "nikkei225_registry": str(empty_registry), + "query": "8001", + "scope": "prime", + }, + ) + + assert status == 200 + assert payload["available"] is True + assert payload["market_universe_used"] is True + assert payload["securities"][0]["ticker"] == "8001" + assert payload["securities"][0]["has_financials"] is False + + +def test_jpx_listed_download_import_accepts_csv_path(tmp_path: Path) -> None: + listed = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,8001,伊藤忠商事,プライム（内国株式）,卸売業\n" + "2026-05-31,1305,ETF Sample,ETF・ETN,-\n" + ) + source = tmp_path / "data_j.csv" + source.write_text(listed, encoding="utf-8") + output_path = tmp_path / "listed_issues.csv" + + status, payload = handle_api( + "POST", + "/api/market/jpx-listed/download-import", + {"path": str(source), "output_path": str(output_path), "save": True}, + ) + + assert status == 200 + assert payload["downloaded"] is False + assert payload["converted"] is False + assert payload["imported"] is True + assert payload["count"] == 2 + assert payload["prime_count"] == 1 + assert payload["sample"][0]["market_segment"] == "プライム（国内株式）" + assert payload["sample"][0]["market_segment_raw"] == "プライム（内国株式）" + assert output_path.is_file() + + +def test_jpx_listed_download_import_converts_legacy_xls( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.investment import jpx_excel + + source = tmp_path / "data_j.xls" + source.write_bytes(bytes.fromhex("d0cf11e0a1b11ae1")) + converted_text = ( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,8001,伊藤忠商事,プライム（内国株式）,卸売業\n" + ) + output_path = tmp_path / "listed_issues.csv" + converted_path = tmp_path / "converted.csv" + + def fake_convert(xls_path: object, csv_path: object, *, timeout_seconds: int = 120) -> str: + assert Path(str(xls_path)) == source + _ = timeout_seconds + Path(str(csv_path)).write_text(converted_text, encoding="utf-8") + return str(csv_path) + + monkeypatch.setattr(jpx_excel, "convert_legacy_xls_to_csv_with_excel", fake_convert) + + status, payload = handle_api( + "POST", + "/api/market/jpx-listed/download-import", + { + "path": str(source), + "converted_output_path": str(converted_path), + "output_path": str(output_path), + "save": True, + }, + ) + + assert status == 200 + assert payload["converted"] is True + assert payload["converted_path"] == str(converted_path) + assert payload["count"] == 1 + assert payload["prime_count"] == 1 + assert payload["sample"][0]["market_segment"] == "プライム（国内株式）" + assert payload["sample"][0]["market_segment_raw"] == "プライム（内国株式）" + assert output_path.is_file() + + def test_rag_stats_endpoint_reports_db_contents(tmp_path) -> None: db = tmp_path / "rag.sqlite" @@ -86,6 +1209,22 @@ def test_rag_search_endpoint(tmp_path) -> None: assert "投資判断" in payload["results"][0]["text"] + assert payload["diagnostics"]["mode"] == "enhanced_hybrid" + assert payload["queries"] + assert payload["auto_trading"] is False + + +def test_operator_catalog_endpoint_exposes_formulas() -> None: + status, payload = handle_api("GET", "/api/operators/catalog") + + assert status == 200 + assert payload["auto_trading"] is False + groups = {group["key"]: group for group in payload["groups"]} + assert "rag_search" in groups + assert "fund_scoring" in groups + assert groups["fund_scoring"]["formula"] == "sum(weight * normalized_score)" + + def test_rag_search_requires_query() -> None: status, payload = handle_api("POST", "/api/rag/search", {}) assert status == 400 @@ -272,9 +1411,12 @@ def test_available_routes_lists_endpoints() -> None: assert "GET /api/health" in routes assert "POST /api/rag/search" in routes assert "POST /api/rag/stats" in routes + assert "GET /api/operators/catalog" in routes assert "POST /api/manual-doc/save" in routes assert "POST /api/fetch-job/auto" in routes assert "POST /api/fetch-job/dry-run" in routes + assert "POST /api/financials/refresh" in routes + assert "POST /api/financials/refresh-async" in routes def test_sources_to_yaml_roundtrips_with_loader(tmp_path) -> None: @@ -395,6 +1537,69 @@ def test_investment_mvp_routes_import_analyze_screen_and_report(tmp_path: Path) assert status == 200 assert guided_import["input_warnings"] == [] + csv_bytes = ( + "asset_type,ticker_or_fund_code,name,quantity,avg_cost\n" + "stock,9433,ＫＤＤＩ,100,2400\n" + ).encode("cp932") + status, converted_csv = handle_api( + "POST", + "/api/holdings/file/convert", + { + "filename": "holdings.csv", + "content_type": "text/csv", + "content_base64": base64.b64encode(csv_bytes).decode("ascii"), + }, + ) + assert status == 200 + assert converted_csv["valid"] is True + assert converted_csv["detected_encoding"] == "cp932" + assert converted_csv["holdings"][0]["ticker_or_fund_code"] == "9433" + assert converted_csv["holdings"][0]["name"] == "ＫＤＤＩ" + + html_table = """ + <html><body><table> + <tr><th>asset_type</th><th>ticker_or_fund_code</th><th>name</th><th>quantity</th><th>avg_cost</th></tr> + <tr><td>stock</td><td>9433</td><td>KDDI</td><td>100</td><td>2400</td></tr> + </table></body></html> + """ + status, converted_html = handle_api( + "POST", + "/api/holdings/file/convert", + { + "filename": "holdings.html", + "content_type": "text/html", + "content_base64": base64.b64encode(html_table.encode("utf-8")).decode("ascii"), + }, + ) + assert status == 200 + assert converted_html["valid"] is True + assert converted_html["detected_format"] == "html" + assert "ticker_or_fund_code" in converted_html["csv_text"] + assert converted_html["holdings"][0]["name"] == "KDDI" + + pdf_bytes = ( + b"%PDF-1.4\n" + b"1 0 obj\n<<>>\nstream\n" + b"BT\n" + b"(asset_type,ticker_or_fund_code,name,quantity,avg_cost) Tj\n" + b"(stock,9433,KDDI,100,2400) Tj\n" + b"ET\n" + b"endstream\nendobj\n%%EOF\n" + ) + status, converted_pdf = handle_api( + "POST", + "/api/holdings/file/convert", + { + "filename": "holdings.pdf", + "content_type": "application/pdf", + "content_base64": base64.b64encode(pdf_bytes).decode("ascii"), + }, + ) + assert status == 200 + assert converted_pdf["valid"] is True + assert converted_pdf["detected_format"] == "pdf" + assert converted_pdf["holdings"][0]["ticker_or_fund_code"] == "9433" + status, analysis = handle_api( "POST", "/api/portfolio/analyze", @@ -402,6 +1607,8 @@ def test_investment_mvp_routes_import_analyze_screen_and_report(tmp_path: Path) ) assert status == 200 assert analysis["summary"]["market_value"] == 670000.0 + assert analysis["summary"]["edinet_covered_holdings"] == 1 + assert analysis["summary"]["edinet_source_ref"] == str(financials) assert analysis["summary"]["nisa"]["status"] == "ok" assert analysis["summary"]["nisa"]["alerts"] == [] assert analysis["summary"]["data_quality"]["missing_timestamp_count"] == 2 @@ -441,6 +1648,12 @@ def test_investment_mvp_routes_import_analyze_screen_and_report(tmp_path: Path) assert status == 200 assert candidates["count"] >= 2 assert candidates["auto_trading"] is False + stock_candidate = next( + item for item in candidates["results"] if item["asset_type"] == "stock" + ) + assert stock_candidate["edinet_summary"]["source_ref"] == str(financials) + assert stock_candidate["edinet_summary"]["latest_dividend_per_share"] == 45.0 + assert stock_candidate["evidence"][0]["source_ref"] == str(financials) status, detail = handle_api( "POST", @@ -458,6 +1671,8 @@ def test_investment_mvp_routes_import_analyze_screen_and_report(tmp_path: Path) assert detail["asset_type"] == "stock" assert detail["metrics"] assert detail["evidence"] + assert detail["edinet_summary"]["source_ref"] == str(financials) + assert detail["edinet_summary"]["latest_fiscal_year"] == 2024 assert detail["auto_trading"] is False status, report = handle_api( @@ -498,6 +1713,8 @@ def test_investment_mvp_routes_import_analyze_screen_and_report(tmp_path: Path) } assert "portfolio.target.required_budget" in evidence_keys assert "portfolio.concentration.current" in evidence_keys + assert "candidate.8306.edinet_financials" in evidence_keys + assert "candidate.8306.edinet_summary" in evidence_keys status, audit = handle_api( "POST", @@ -749,6 +1966,583 @@ def test_market_prices_reject_uncontracted_provider_in_production() -> None: assert "not allowed in production" in payload["error"] +def test_market_prices_can_use_cache_when_provider_is_policy_blocked( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.webapi import service + + monkeypatch.delenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", raising=False) + monkeypatch.setattr(service, "_JQUANTS_CONTRACT_RUNTIME_ACK", False) + store_path = tmp_path / "current_prices.csv" + store_path.write_text( + "ticker,price,as_of,provider_id,source_ref,note\n" + "9433,4970,2026-06-15,jquants,unit,\n", + encoding="utf-8", + ) + + status, payload = handle_api( + "POST", + "/api/market/prices", + { + "tickers": ["9433"], + "provider_id": "jquants", + "runtime_mode": "production", + "price_store_path": str(store_path), + "allow_cache_on_policy_block": True, + }, + ) + + assert status == 200 + assert payload["provider_blocked"] is True + assert payload["call_real_api"] is False + assert payload["prices"] == {"9433": 4970.0} + assert payload["price_store"]["cache_used"] == ["9433"] + + +def test_market_prices_uses_jquants_provider_when_contracted(monkeypatch) -> None: + from investment_assistant.jquants.client import JQuantsClient + + monkeypatch.setenv("JQUANTS_API_KEY", "unit-key") + monkeypatch.setenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", "jquants") + + def fake_prices( + self: JQuantsClient, + tickers: list[str], + *, + date: str | None = None, + lookback_days: int = 14, + ) -> dict[str, object]: + assert tickers == ["8306"] + assert date is None + assert lookback_days == 5 + return { + "prices": {"8306": 1234.0}, + "as_of": {"8306": "2026-06-15"}, + "provider_id": "jquants", + "auto_trading": False, + "call_real_api": True, + } + + monkeypatch.setattr(JQuantsClient, "fetch_latest_prices", fake_prices) + + status, payload = handle_api( + "POST", + "/api/market/prices", + { + "tickers": ["8306"], + "provider_id": "jquants", + "runtime_mode": "production", + "lookback_days": 5, + }, + ) + + assert status == 200 + assert payload["prices"] == {"8306": 1234.0} + assert payload["provider_id"] == "jquants" + assert payload["provider_policy"]["production_allowed"] is True + assert payload["auto_trading"] is False + + +def test_market_prices_saves_and_reuses_local_price_store( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.portfolio import prices as price_module + + store_path = tmp_path / "current_prices.csv" + + def first_fetch(tickers: list[str]) -> dict[str, object]: + assert tickers == ["9433"] + return { + "prices": {"9433": 4970.0}, + "notes": {}, + "source": "unit-price-source", + } + + monkeypatch.setattr(price_module, "fetch_prices", first_fetch) + status, payload = handle_api( + "POST", + "/api/market/prices", + { + "tickers": ["9433"], + "provider_id": "stooq_public_csv", + "runtime_mode": "development", + "price_store_path": str(store_path), + }, + ) + + assert status == 200 + assert payload["prices"] == {"9433": 4970.0} + assert payload["price_store"]["saved"] is True + assert payload["price_store"]["saved_count"] == 1 + assert store_path.is_file() + + def missing_fetch(tickers: list[str]) -> dict[str, object]: + assert tickers == ["9433"] + return { + "prices": {"9433": None}, + "notes": {"9433": "provider_miss"}, + "source": "unit-price-source", + } + + monkeypatch.setattr(price_module, "fetch_prices", missing_fetch) + status, cached = handle_api( + "POST", + "/api/market/prices", + { + "tickers": ["9433"], + "provider_id": "stooq_public_csv", + "runtime_mode": "development", + "price_store_path": str(store_path), + }, + ) + + assert status == 200 + assert cached["prices"] == {"9433": 4970.0} + assert cached["price_store"]["saved"] is False + assert cached["price_store"]["cache_used"] == ["9433"] + assert "using_cached_price" in cached["notes"]["9433"] + + +def test_market_prices_import_accepts_yahoo_style_paste(tmp_path: Path) -> None: + store_path = tmp_path / "current_prices.csv" + csv_text = ( + "Symbol\tRegular Market Price\tDate\tsource_ref\n" + "7203.T\t2890\t2026-03-24\tYahoo Finance manual check\n" + "9433.T\t4970\t2026-03-24\tYahoo Finance manual check\n" + ) + + status, payload = handle_api( + "POST", + "/api/market/prices/import", + { + "csv_text": csv_text, + "provider_id": "yahoo_finance_manual", + "price_store_path": str(store_path), + }, + ) + + saved = store_path.read_text(encoding="utf-8") + assert status == 200 + assert payload["count"] == 2 + assert payload["tickers"] == ["7203", "9433"] + assert payload["provider_policy"]["production_allowed"] is True + assert payload["auto_trading"] is False + assert payload["call_real_api"] is False + assert "7203,2890" in saved + assert "9433,4970" in saved + assert ".T" not in saved + + +def test_market_prices_import_file_reads_daily_inbox(tmp_path: Path) -> None: + store_path = tmp_path / "current_prices.csv" + inbox = tmp_path / "yahoo_prices_inbox.csv" + inbox.write_text( + "Symbol,Regular Market Price,Date\n" + "7203.T,2890,2026-03-24\n", + encoding="utf-8", + ) + + status, payload = handle_api( + "POST", + "/api/market/prices/import-file", + { + "path": str(inbox), + "price_store_path": str(store_path), + }, + ) + + assert status == 200 + assert payload["status"] == "imported" + assert payload["input_path"] == str(inbox) + assert payload["tickers"] == ["7203"] + assert "7203,2890" in store_path.read_text(encoding="utf-8") + + +def test_market_prices_import_file_can_report_missing_inbox(tmp_path: Path) -> None: + missing = tmp_path / "missing.csv" + + status, payload = handle_api( + "POST", + "/api/market/prices/import-file", + { + "path": str(missing), + "allow_missing": True, + }, + ) + + assert status == 200 + assert payload["available"] is False + assert payload["status"] == "missing" + assert payload["input_path"] == str(missing) + assert payload["auto_trading"] is False + + +def test_market_prices_falls_back_to_daily_bars_when_current_price_is_missing( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.webapi import service + + monkeypatch.delenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", raising=False) + monkeypatch.setattr(service, "_JQUANTS_CONTRACT_RUNTIME_ACK", False) + store_path = tmp_path / "current_prices.csv" + bars_path = tmp_path / "daily_bars.csv" + bars_path.write_text( + "ticker,date,open,high,low,close,volume,trading_value,adjustment_factor," + "adjusted_open,adjusted_high,adjusted_low,adjusted_close,adjusted_volume," + "upper_limit_hit,lower_limit_hit,provider_id,source_ref\n" + "9433,2026-06-12,4900,5010,4890,4970,1200000,,1,4900,5010,4890,4970," + "1200000,0,0,jquants,unit\n" + "9433,2026-06-15,4970,5020,4950,5000,1300000,,1,4970,5020,4950,5000," + "1300000,0,0,jquants,unit\n", + encoding="utf-8", + ) + + status, payload = handle_api( + "POST", + "/api/market/prices", + { + "tickers": ["9433"], + "provider_id": "jquants", + "runtime_mode": "production", + "price_store_path": str(store_path), + "daily_bars_path": str(bars_path), + "allow_cache_on_policy_block": True, + }, + ) + + assert status == 200 + assert payload["provider_blocked"] is True + assert payload["prices"] == {"9433": 5000.0} + assert payload["as_of"] == {"9433": "2026-06-15"} + assert payload["price_store"]["daily_bar_cache_used"] == ["9433"] + assert "using_daily_bar_close" in payload["notes"]["9433"] + assert store_path.read_text(encoding="utf-8").count("9433") == 1 + assert "5000" in store_path.read_text(encoding="utf-8") + + +def test_market_bars_uses_jquants_and_saves_daily_ohlcv( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.jquants.client import JQuantsClient + + monkeypatch.setenv("JQUANTS_API_KEY", "unit-key") + monkeypatch.setenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", "jquants") + store_path = tmp_path / "daily_bars.csv" + price_store_path = tmp_path / "current_prices.csv" + + def fake_bars( + self: JQuantsClient, + tickers: list[str], + *, + date: str | None = None, + lookback_days: int = 30, + ) -> dict[str, object]: + assert tickers == ["9433"] + assert date is None + assert lookback_days == 10 + return { + "bars": [ + { + "ticker": "9433", + "date": "2026-06-12", + "open": 4900.0, + "high": 5010.0, + "low": 4890.0, + "close": 4970.0, + "volume": 1200000.0, + "adjusted_close": 4970.0, + "provider_id": "jquants", + }, + { + "ticker": "9433", + "date": "2026-06-15", + "open": 4970.0, + "high": 5020.0, + "low": 4950.0, + "close": 5000.0, + "volume": 1300000.0, + "adjusted_close": 5000.0, + "provider_id": "jquants", + }, + ], + "summary": {}, + "notes": {}, + "source": "unit-jquants-bars", + "provider_id": "jquants", + "auto_trading": False, + "call_real_api": True, + } + + monkeypatch.setattr(JQuantsClient, "fetch_daily_bars", fake_bars) + status, payload = handle_api( + "POST", + "/api/market/bars", + { + "tickers": ["9433"], + "provider_id": "jquants", + "runtime_mode": "production", + "lookback_days": 10, + "bar_store_path": str(store_path), + "price_store_path": str(price_store_path), + }, + ) + + assert status == 200 + assert len(payload["bars"]) == 2 + assert payload["bar_store"]["saved"] is True + assert payload["bar_store"]["price_sync"]["saved"] is True + assert payload["bar_store"]["price_sync"]["tickers"] == ["9433"] + assert payload["summary"]["tickers"]["9433"]["latest_close"] == 5000.0 + assert store_path.is_file() + assert "5000" in price_store_path.read_text(encoding="utf-8") + + +def test_market_bars_universe_selects_prime_and_saves_ohlcv( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.jquants.client import JQuantsClient + + monkeypatch.setenv("JQUANTS_API_KEY", "unit-key") + monkeypatch.setenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", "jquants") + listed_path = tmp_path / "listed_issues.csv" + listed_path.write_text( + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + "2026-05-31,7203,トヨタ自動車,プライム（国内株式）,輸送用機器\n" + "2026-05-31,9433,KDDI,プライム（国内株式）,情報・通信業\n" + "2026-05-31,9999,標準サンプル,スタンダード（国内株式）,サービス業\n", + encoding="utf-8", + ) + nikkei_path = tmp_path / "nikkei.yaml" + nikkei_path.write_text("sources: []\n", encoding="utf-8") + bar_store_path = tmp_path / "daily_bars.csv" + price_store_path = tmp_path / "current_prices.csv" + + def fake_bars_bulk( + self: JQuantsClient, + tickers: list[str], + *, + date: str | None = None, + lookback_days: int = 30, + ) -> dict[str, object]: + _ = self, date, lookback_days + assert tickers == ["7203", "9433"] + return { + "bars": [ + { + "ticker": "7203", + "date": "2026-03-24", + "open": 3321.0, + "high": 3336.0, + "low": 3258.0, + "close": 3271.0, + "volume": 16565700.0, + "adjusted_close": 3271.0, + "provider_id": "jquants", + }, + { + "ticker": "9433", + "date": "2026-03-24", + "open": 2679.5, + "high": 2701.0, + "low": 2669.5, + "close": 2677.5, + "volume": 7094500.0, + "adjusted_close": 2677.5, + "provider_id": "jquants", + }, + ], + "summary": {}, + "notes": {}, + "source": "unit-jquants-bars", + "provider_id": "jquants", + "auto_trading": False, + "call_real_api": True, + } + + monkeypatch.setattr(JQuantsClient, "fetch_daily_bars_bulk", fake_bars_bulk) + + status, payload = handle_api( + "POST", + "/api/market/bars/universe", + { + "scope": "prime", + "jpx_listed_path": str(listed_path), + "nikkei225_registry": str(nikkei_path), + "financials_csv": str(tmp_path / "missing_financials.csv"), + "bar_store_path": str(bar_store_path), + "price_store_path": str(price_store_path), + "runtime_mode": "production", + }, + ) + + assert status == 200 + assert payload["selection"]["scope"] == "prime" + assert payload["selection"]["selected_count"] == 2 + assert payload["selection"]["universe_total_count"] == 2 + assert payload["bar_store"]["saved"] is True + assert payload["bar_store"]["price_sync"]["tickers"] == ["7203", "9433"] + assert "7203" in bar_store_path.read_text(encoding="utf-8") + assert "2677.5" in price_store_path.read_text(encoding="utf-8") + + +def test_market_bars_can_use_cache_when_provider_is_policy_blocked( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.webapi import service + + monkeypatch.delenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", raising=False) + monkeypatch.setattr(service, "_JQUANTS_CONTRACT_RUNTIME_ACK", False) + store_path = tmp_path / "daily_bars.csv" + store_path.write_text( + "ticker,date,open,high,low,close,volume,trading_value,adjustment_factor," + "adjusted_open,adjusted_high,adjusted_low,adjusted_close,adjusted_volume," + "upper_limit_hit,lower_limit_hit,provider_id,source_ref\n" + "9433,2026-06-15,4970,5020,4950,5000,1300000,,1,4970,5020,4950,5000," + "1300000,0,0,jquants,unit\n", + encoding="utf-8", + ) + + status, payload = handle_api( + "POST", + "/api/market/bars", + { + "tickers": ["9433"], + "provider_id": "jquants", + "runtime_mode": "production", + "bar_store_path": str(store_path), + "allow_cache_on_policy_block": True, + }, + ) + + assert status == 200 + assert payload["provider_blocked"] is True + assert payload["call_real_api"] is False + assert payload["bar_store"]["cache_used"] == ["9433"] + assert payload["summary"]["tickers"]["9433"]["latest_volume"] == 1300000.0 + + +def test_jquants_status_loads_contracted_provider_from_dotenv( + tmp_path: Path, + monkeypatch, +) -> None: + from investment_assistant.webapi import service + + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("JQUANTS_REFRESH_TOKEN", raising=False) + monkeypatch.delenv("JQUANTS_API_KEY", raising=False) + monkeypatch.delenv("INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS", raising=False) + monkeypatch.setattr(service, "_JQUANTS_API_KEY_RUNTIME_SET", False) + monkeypatch.setattr(service, "_JQUANTS_CONTRACT_RUNTIME_ACK", False) + (tmp_path / ".env.local").write_text( + "JQUANTS_REFRESH_TOKEN=fake-token\n" + "INVESTMENT_ASSISTANT_CONTRACTED_PROVIDERS=jquants\n", + encoding="utf-8", + ) + + status, payload = handle_api("GET", "/api/jquants/status", {}) + + assert status == 200 + assert payload["api_key_configured"] is True + assert payload["api_key_source"] == "dotenv" + assert payload["contract_acknowledged"] is True + assert payload["production_allowed"] is True + assert payload["provider_policy"]["commercial_use"] == "allowed_if_contracted" + + +def test_data_status_summarizes_canonical_local_sources(tmp_path: Path) -> None: + financials = tmp_path / "financials.csv" + financials.write_text( + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + "9433,KDDI,2025,1000,45,145,stable\n", + encoding="utf-8", + ) + listed = tmp_path / "listed_issues.csv" + listed.write_text( + "date,code,name,market segment,33 sector\n" + "2026-05-31,9433,KDDI,Prime Market (Domestic Stock),Information\n", + encoding="utf-8", + ) + company_master = tmp_path / "company_master.csv" + company_master.write_text( + "ticker,name,market_segment,has_financials\n" + "9433,KDDI,プライム（国内株式）,true\n", + encoding="utf-8", + ) + prices = tmp_path / "current_prices.csv" + prices.write_text( + "ticker,price,as_of,provider_id,source_ref,note\n" + "9433,4970,2026-06-15,jquants,unit,\n", + encoding="utf-8", + ) + bars = tmp_path / "daily_bars.csv" + bars.write_text( + "ticker,date,open,high,low,close,volume,trading_value,adjustment_factor," + "adjusted_open,adjusted_high,adjusted_low,adjusted_close,adjusted_volume," + "upper_limit_hit,lower_limit_hit,provider_id,source_ref\n" + "9433,2026-06-15,4970,5020,4950,5000,1300000,,1,4970,5020,4950,5000," + "1300000,0,0,jquants,unit\n", + encoding="utf-8", + ) + + status, payload = handle_api( + "POST", + "/api/data/status", + { + "financials_csv": str(financials), + "jpx_listed_path": str(listed), + "company_master_path": str(company_master), + "market_prices_path": str(prices), + "daily_bars_path": str(bars), + "current_yields_path": str(tmp_path / "missing_yields.csv"), + "stale_after_days": 9999, + }, + ) + + assert status == 200 + assert payload["by_key"]["financials"]["company_count"] == 1 + assert payload["by_key"]["jpx_listed"]["prime_count"] == 1 + assert payload["by_key"]["company_master"]["financials_count"] == 1 + assert payload["by_key"]["market_prices"]["ticker_count"] == 1 + assert payload["by_key"]["daily_bars"]["ticker_count"] == 1 + assert payload["by_key"]["current_yields"]["status"] == "missing" + assert payload["auto_trading"] is False + + +def test_market_current_yields_import_saves_current_dividend_overlay(tmp_path: Path) -> None: + output_path = tmp_path / "current_yields.csv" + status, payload = handle_api( + "POST", + "/api/market/current-yields/import", + { + "path": str(output_path), + "rows": [ + { + "ticker": "9433", + "name": "KDDI", + "current_dividend_per_share": 80, + "current_price": 2500, + "yield_pct": 3.2, + "as_of": "2026-06-15", + "source_ref": "user_verified_current_dividend", + } + ], + }, + ) + + assert status == 200 + assert payload["available"] is True + assert payload["saved_path"] == str(output_path) + assert payload["count"] == 1 + assert output_path.is_file() + assert "current_dividend_per_share" in output_path.read_text(encoding="utf-8") + + def test_provider_policy_ledger_route_reports_runtime_decisions() -> None: status, payload = handle_api( "POST", diff --git a/tests/unit/test_yahoo_market_refresh.py b/tests/unit/test_yahoo_market_refresh.py new file mode 100644 index 0000000..4c49e29 --- /dev/null +++ b/tests/unit/test_yahoo_market_refresh.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from investment_assistant.portfolio.yahoo_market import ( + parse_yahoo_chart, + parse_yahoo_japan_html, + refresh_yahoo_market, +) +from investment_assistant.webapi import available_routes +from investment_assistant.webapi import yahoo_market as yahoo_api + + +class _Document: + allowed_by_robots = True + status_code = 200 + source = "network" + + def __init__(self, html: str) -> None: + self.html = html + + +class _Fetcher: + def fetch_document(self, url: str) -> _Document: + if "/v8/finance/chart/" in url: + return _Document( + json.dumps( + { + "chart": { + "result": [ + { + "timestamp": [1718409600], + "meta": {"gmtoffset": 32400}, + "indicators": { + "quote": [ + { + "open": [100.0], + "high": [110.0], + "low": [90.0], + "close": [105.0], + "volume": [1234], + } + ], + "adjclose": [{"adjclose": [104.0]}], + }, + } + ] + } + } + ) + ) + if "/v7/finance/quote" in url: + return _Document( + json.dumps( + { + "quoteResponse": { + "result": [ + { + "symbol": "7203.T", + "longName": "Toyota", + "regularMarketPrice": 105.0, + "trailingPE": 10.0, + "priceToBook": 1.2, + "trailingAnnualDividendYield": 0.03, + } + ] + } + } + ) + ) + return _Document( + "<html><title>MUFG【8306】" + "
PER
12.3倍
" + "
PBR
1.1倍
" + "
配当利回り
3.5%
" + "
時価総額
10兆円
" + "現在値1234前日比" + ) + + +def test_parse_yahoo_chart_normalizes_daily_bar() -> None: + payload = json.dumps( + { + "chart": { + "result": [ + { + "timestamp": [1718409600], + "meta": {"gmtoffset": 32400}, + "indicators": { + "quote": [ + { + "open": [100], + "high": [110], + "low": [90], + "close": [105], + "volume": [0], + } + ], + "adjclose": [{"adjclose": [104]}], + }, + } + ] + } + } + ) + + bars = parse_yahoo_chart(payload, ticker="7203", source_ref="chart") + + assert len(bars) == 1 + assert bars[0].ticker == "7203" + assert bars[0].date == "2024-06-15" + assert bars[0].adjusted_close == 104.0 + assert bars[0].volume == 0.0 + assert bars[0].provider_id == "yahoo_finance" + + +def test_parse_yahoo_japan_html_extracts_market_metrics() -> None: + html = ( + "Sample【9999】" + "
PER
12.3倍
" + "
PBR
1.1倍
" + "
配当利回り
3.5%
" + "
時価総額
10兆円
" + "現在値1234前日比" + ) + + row = parse_yahoo_japan_html(html, ticker="9999") + + assert row["name"] == "Sample" + assert row["price"] == 1234.0 + assert row["per"] == 12.3 + assert row["pbr"] == 1.1 + assert row["dividend_yield"] == 0.035 + assert row["market_cap"] == 10_000_000_000_000.0 + + +def test_refresh_yahoo_market_saves_bars_prices_and_fundamentals(tmp_path: Path) -> None: + bars_path = tmp_path / "daily_bars.csv" + prices_path = tmp_path / "current_prices.csv" + fundamentals_path = tmp_path / "yahoo_financials.csv" + + result = refresh_yahoo_market( + ["7203", "8306.T"], + daily_bars_path=bars_path, + current_prices_path=prices_path, + fundamentals_path=fundamentals_path, + fetcher=_Fetcher(), + ) + + assert result["status"] == "completed" + assert result["ohlcv_ticker_count"] == 2 + assert result["fundamentals_ticker_count"] == 2 + assert bars_path.is_file() + assert prices_path.is_file() + assert fundamentals_path.is_file() + assert "yahoo_finance" in bars_path.read_text(encoding="utf-8") + assert "Toyota" in fundamentals_path.read_text(encoding="utf-8-sig") + + +def test_refresh_yahoo_market_uses_default_fetcher(monkeypatch, tmp_path: Path) -> None: + monkeypatch.setattr( + "investment_assistant.portfolio.yahoo_market.SafeFetcher", + lambda **_kwargs: _Fetcher(), + ) + + result = refresh_yahoo_market( + ["7203"], + fetch_fundamentals=False, + daily_bars_path=tmp_path / "daily_bars.csv", + current_prices_path=tmp_path / "current_prices.csv", + ) + + assert result["status"] == "completed" + assert result["ohlcv_ticker_count"] == 1 + + +def test_yahoo_routes_are_registered() -> None: + routes = available_routes() + + assert "POST /api/market/yahoo/refresh" in routes + assert "GET /api/market/yahoo/status" in routes + + +def test_yahoo_custom_refresh_routes_configuration(monkeypatch, tmp_path: Path) -> None: + captured: dict[str, object] = {} + + def fake_refresh(tickers: object, **kwargs: object) -> dict[str, object]: + captured["tickers"] = tickers + captured.update(kwargs) + return {"status": "completed", "requested_count": 2} + + monkeypatch.setattr(yahoo_api, "refresh_yahoo_market", fake_refresh) + + status, payload = yahoo_api.handle_yahoo_market_api( + "POST", + "/api/market/yahoo/refresh", + { + "mode": "custom", + "tickers": "7203, 8306.T", + "range": "3mo", + "interval": "1wk", + "fetch_ohlcv": True, + "fetch_fundamentals": False, + "daily_bars_path": str(tmp_path / "bars.csv"), + }, + ) or (0, {}) + + assert status == 200 + assert payload["mode"] == "custom" + assert captured["tickers"] == ["7203", "8306"] + assert captured["range_"] == "3mo" + assert captured["interval"] == "1wk" + assert captured["fetch_fundamentals"] is False + + +def test_yahoo_auto_refresh_resolves_market_universe(monkeypatch) -> None: + captured: dict[str, object] = {} + + def fake_universe(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return { + "securities": [{"ticker": "9432"}, {"ticker": "7203"}], + "total_count": 225, + "nikkei225_count": 225, + } + + def fake_refresh(tickers: object, **kwargs: object) -> dict[str, object]: + return {"status": "completed", "tickers": tickers, **kwargs} + + monkeypatch.setattr(yahoo_api, "build_market_universe", fake_universe) + monkeypatch.setattr(yahoo_api, "refresh_yahoo_market", fake_refresh) + + status, payload = yahoo_api.handle_yahoo_market_api( + "POST", + "/api/market/yahoo/refresh", + {"mode": "auto", "scope": "nikkei225", "max_tickers": 20}, + ) or (0, {}) + + assert status == 200 + assert payload["tickers"] == ["9432", "7203"] + assert payload["selection"]["scope"] == "nikkei225" + assert captured["limit"] == 20 + + +def test_yahoo_custom_refresh_rejects_empty_tickers() -> None: + status, payload = yahoo_api.handle_yahoo_market_api( + "POST", + "/api/market/yahoo/refresh", + {"mode": "custom", "tickers": []}, + ) or (0, {}) + + assert status == 400 + assert "requires tickers" in str(payload["error"]) diff --git a/web/public/reset.html b/web/public/reset.html new file mode 100644 index 0000000..35f317e --- /dev/null +++ b/web/public/reset.html @@ -0,0 +1,105 @@ + + + + + + InvestAssist Reset + + + +
+

表示キャッシュをリセットしています

+

Service Worker と古いキャッシュを確認しています。

+
+ アプリを開く + +
+
+ + + diff --git a/web/public/sw.js b/web/public/sw.js index 202321c..b62c0f9 100644 --- a/web/public/sw.js +++ b/web/public/sw.js @@ -2,6 +2,12 @@ // Network-first for navigations/assets so the dashboard stays fresh online and // still opens offline from cache. API (POST, /api/) is never intercepted. const CACHE = "ia-shell-v2"; +const IS_DEV_SERVER = self.location.port === "5173"; + +async function clearAllCaches() { + const keys = await caches.keys(); + await Promise.all(keys.map((key) => caches.delete(key))); +} self.addEventListener("install", () => { self.skipWaiting(); @@ -10,14 +16,20 @@ self.addEventListener("install", () => { self.addEventListener("activate", (event) => { event.waitUntil( (async () => { + if (IS_DEV_SERVER) { + await clearAllCaches(); + await self.registration.unregister(); + return; + } const keys = await caches.keys(); - await Promise.all(keys.filter((k) => k !== CACHE).map((k) => caches.delete(k))); + await Promise.all(keys.filter((key) => key !== CACHE).map((key) => caches.delete(key))); await self.clients.claim(); })(), ); }); self.addEventListener("fetch", (event) => { + if (IS_DEV_SERVER) return; const req = event.request; if (req.method !== "GET") return; const url = new URL(req.url); diff --git a/web/src/App.tsx b/web/src/App.tsx index 6478eae..16c11c7 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -56,6 +56,16 @@ type CsvDraft = Record; const DEFAULT_RAG_DB_PATH = ".cache/investment_assistant/rag.sqlite"; const CANDIDATE_SCREEN_PRESETS_STORAGE_KEY = "investment_assistant.candidate_screen_presets.v1"; +const FINANCIALS_CSV_STORAGE_KEY = + "investment_assistant.financials_csv_path.v1"; +const EXPERIENCE_MODE_STORAGE_KEY = + "investment_assistant.experience_mode.v1"; +const AI_CHAT_TARGET_SOURCE_STORAGE_KEY = + "investment_assistant.ai_chat.target_source.v1"; +const AI_CHAT_SELECTED_TICKER_STORAGE_KEY = + "investment_assistant.ai_chat.selected_ticker.v1"; +const AI_CHAT_SELECTED_SECURITY_NAME_STORAGE_KEY = + "investment_assistant.ai_chat.selected_security_name.v1"; const TARGET_SOURCE_OPTIONS: TargetSourceOption[] = [ { @@ -84,22 +94,121 @@ const TARGET_SOURCE_OPTIONS: TargetSourceOption[] = [ ]; const TABS = [ - { id: "dashboard", label: "Dashboard" }, - { id: "holdings", label: "Holdings" }, - { id: "candidates", label: "Candidates" }, - { id: "detail", label: "Detail" }, - { id: "simulate", label: "Simulate" }, - { id: "report", label: "Report" }, - { id: "evidence", label: "Evidence" }, + { id: "dashboard", label: "概要" }, + { id: "data", label: "データ" }, + { id: "holdings", label: "保有" }, + { id: "candidates", label: "候補" }, + { id: "simulate", label: "試算" }, + { id: "report", label: "レポート" }, + { id: "detail", label: "詳細" }, + { id: "answer", label: "AIチャット" }, + { id: "evidence", label: "根拠" }, ] as const; +type ExperienceMode = "simple" | "analyst" | "operator"; + +const EXPERIENCE_MODES: { + id: ExperienceMode; + label: string; + body: string; +}[] = [ + { + id: "simple", + label: "かんたん", + body: "初回・スマホ向け", + }, + { + id: "analyst", + label: "分析", + body: "比較と試算を広げる", + }, + { + id: "operator", + label: "データ運用", + body: "取得・根拠・AIまで見る", + }, +]; + const HERO_CARDS = [ - { label: "Holdings", value: "Analyze", desc: "保有・NISA・損益を集計" }, - { label: "Candidates", value: "Screen", desc: "条件一致だけを提示" }, - { label: "Report", value: "Evidence", desc: "計算式と根拠を保存" }, - { label: "Detail", value: "Review", desc: "銘柄・投信を根拠付き確認" }, + { label: "1", value: "データ", desc: "財務・市場区分を準備" }, + { label: "2", value: "保有", desc: "評価額・損益・NISAを確認" }, + { label: "3", value: "候補", desc: "条件に合う対象を比較" }, + { label: "4", value: "出力", desc: "試算と根拠を残す" }, ] as const; +const COMMAND_ACTIONS = [ + { + id: "data", + step: "1", + title: "データ更新", + body: "財務・株価・日足を整える", + }, + { + id: "holdings", + step: "2", + title: "保有入力", + body: "CSV/手入力で現在地を作る", + }, + { + id: "candidates", + step: "3", + title: "候補比較", + body: "条件一致だけを並べる", + }, + { + id: "report", + step: "4", + title: "根拠出力", + body: "計算式と免責を残す", + }, +] satisfies { + id: TabId; + step: string; + title: string; + body: string; +}[]; + +const REPORT_WIZARD_STEPS = [ + { + id: "data", + step: "1", + title: "データ確認", + body: "使う財務データと入力データを確認します。", + }, + { + id: "holdings", + step: "2", + title: "保有確認", + body: "保有データを検証し、分析対象を固めます。", + }, + { + id: "candidates", + step: "3", + title: "候補条件", + body: "候補抽出に使う条件と投信データを確認します。", + }, + { + id: "target", + step: "4", + title: "目標配当", + body: "任意の目標年間配当から必要予算を逆算します。", + }, + { + id: "preview", + step: "5", + title: "プレビュー", + body: "生成、KPI、根拠、公開前監査を確認します。", + }, + { + id: "export", + step: "6", + title: "保存", + body: "履歴、比較、Markdownを扱います。", + }, +] as const; + +type ReportWizardStepId = (typeof REPORT_WIZARD_STEPS)[number]["id"]; + const SUGGESTED_QUESTIONS = [ "選択中の対象銘柄について、配当方針と減配リスクを取得済みIR資料だけで整理して", "選択中の対象銘柄について、株主還元方針と未確認の危険ポイントを分けて", @@ -135,20 +244,20 @@ const SOURCE_PRESETS: SourcePreset[] = [ const AI_GUIDES: GuideCard[] = [ { - title: "1. まず根拠を検索", - body: "RAG DBから関連チャンクを取得します。ハイブリッド検索を使うと語句一致と意味検索を混ぜます。", + title: "1. 根拠を探す", + body: "登録済み資料から関連箇所を探します。", }, { - title: "2. 複数ドラフト", - body: "コスト、リスク、分散など複数観点で下書きを作ります。ドラフト数を増やすほど確認観点が増えます。", + title: "2. 観点を分ける", + body: "配当、リスク、分散を分けて確認します。", }, { - title: "3. レビューと統合", - body: "レビュアーが根拠不足・飛躍・引用漏れを指摘し、統合担当が最終回答へ反映します。", + title: "3. 確認してまとめる", + body: "根拠不足や引用漏れを確認してまとめます。", }, { title: "4. 実APIは任意", - body: "標準はローカル擬似AIです。実Geminiを使うにはバックエンドで許可設定が必要です。", + body: "標準はローカル動作です。Gemini利用は明示設定時だけです。", }, ]; @@ -158,8 +267,8 @@ const SCRAPE_GUIDES: GuideCard[] = [ body: "robots.txt確認、URL安全性確認、レート制限、HTMLテキスト化、保存、RAG登録をまとめて実行します。", }, { - title: "dry-run", - body: "本文を取得せず、robots.txtで取得可能かだけ確認します。新しいURLは最初にdry-runしてください。", + title: "事前確認", + body: "本文取得前に、取得可能かだけ確認します。", }, { title: "手動取込", @@ -198,6 +307,37 @@ const SAMPLE_FUNDS_CSV = "FND999,高コストテーマ型,theme,1.20,distribution,false,user_csv,0.40\n"; const SAMPLE_FINANCIALS_PATH = "examples/financials_sample.csv"; +const DEFAULT_FINANCIALS_PATH = "local_docs/edinet/financials.csv"; +const SAMPLE_FINANCIALS_CSV = + "ticker,name,fiscal_year,operating_cf,equity_ratio,dividend_per_share,payout_policy\n" + + "7203,安定配当ホールディングス,2021,820000,58.2,42,連結配当性向30%目安・累進配当を志向\n" + + "7203,安定配当ホールディングス,2022,910000,59.1,46,連結配当性向30%目安・累進配当を志向\n" + + "7203,安定配当ホールディングス,2023,985000,60.4,52,連結配当性向30%目安・累進配当を志向\n" + + "7203,安定配当ホールディングス,2024,1040000,61.0,58,連結配当性向30%目安・累進配当を志向\n" + + "7203,安定配当ホールディングス,2025,1105000,62.3,64,連結配当性向30%目安・累進配当を志向\n" + + "9999,景気連動マテリアル,2021,310000,38.5,80,業績連動・配当性向40%（下限なし）\n" + + "9999,景気連動マテリアル,2022,420000,41.2,100,業績連動・配当性向40%（下限なし）\n" + + "9999,景気連動マテリアル,2023,150000,36.8,40,業績連動・配当性向40%（下限なし）\n" + + "9999,景気連動マテリアル,2024,260000,39.0,55,業績連動・配当性向40%（下限なし）\n" + + "9999,景気連動マテリアル,2025,180000,37.4,45,業績連動・配当性向40%（下限なし）\n"; + +const SAMPLE_JPX_LISTED_ISSUES_DATA = + "日付,コード,銘柄名,市場・商品区分,33業種区分\n" + + "2026-05-31,7203,トヨタ自動車,プライム（国内株式）,輸送用機器\n" + + "2026-05-31,8306,三菱ＵＦＪフィナンシャル・グループ,プライム（国内株式）,銀行業\n" + + "2026-05-31,9999,サンプルスタンダード,スタンダード（国内株式）,サービス業\n"; + +const SAMPLE_YAHOO_PRICE_CSV = + "Symbol,Regular Market Price,Date,source_ref\n" + + "7203.T,2890,2026-03-24,Yahoo Finance manual check\n" + + "9433.T,4970,2026-03-24,Yahoo Finance manual check\n"; + +const MARKET_SCOPE_OPTIONS = [ + { value: "prime", label: "東証プライム" }, + { value: "nikkei225", label: "日経225" }, + { value: "financials", label: "財務データあり" }, + { value: "all", label: "全件" }, +] as const; const HOLDING_CSV_COLUMNS = [ "asset_type", @@ -226,6 +366,9 @@ const FUND_CSV_COLUMNS = [ "diversification_score", ] as const; +const HOLDING_FILE_ACCEPT = + ".csv,.tsv,.html,.htm,.pdf,text/csv,text/tab-separated-values,text/html,application/pdf"; + const DEFAULT_HOLDING_DRAFT: CsvDraft = { asset_type: "stock", ticker_or_fund_code: "7203", @@ -305,8 +448,21 @@ const DISCLOSURE_AUTO_SOURCES = [ type TabId = (typeof TABS)[number]["id"]; +const NAV_TABS_BY_MODE: Record = { + simple: ["dashboard", "data", "holdings", "candidates", "report"], + analyst: ["dashboard", "data", "holdings", "candidates", "simulate", "detail", "report"], + operator: ["dashboard", "data", "holdings", "candidates", "simulate", "report", "detail", "answer", "evidence"], +}; + export function App() { const [tab, setTab] = useState("dashboard"); + const [experienceMode, setExperienceMode] = useState(() => + readExperienceMode(EXPERIENCE_MODE_STORAGE_KEY, "simple"), + ); + const [financialsCsvPath, setFinancialsCsvPath] = useState(() => + readLocalStorageString(FINANCIALS_CSV_STORAGE_KEY, DEFAULT_FINANCIALS_PATH), + ); + const [financialsRefreshNonce, setFinancialsRefreshNonce] = useState(0); const [detailSeed, setDetailSeed] = useState({ code: "7203", assetType: "stock", @@ -316,20 +472,32 @@ export function App() { setDetailSeed({ code, assetType, nonce: Date.now() }); setTab("detail"); }; + const markFinancialsDataUpdated = () => { + setFinancialsRefreshNonce((value) => value + 1); + }; + useEffect(() => { + writeLocalStorageString(FINANCIALS_CSV_STORAGE_KEY, financialsCsvPath); + }, [financialsCsvPath]); + useEffect(() => { + writeLocalStorageString(EXPERIENCE_MODE_STORAGE_KEY, experienceMode); + }, [experienceMode]); + const visibleTabIds = new Set([...NAV_TABS_BY_MODE[experienceMode], tab]); + const visibleTabs = TABS.filter((item) => visibleTabIds.has(item.id)); return (
-

Investment Research Terminal

-

Investment Assistant

+

投資支援ツール

+

投資アシスタント

- 日本株と投信の保有分析、候補抽出、NISA枠、根拠付きレポートを1画面で進めます。 + 日本株と投信のデータ準備、保有分析、候補比較、試算、レポートを順番に進めます。 + 断定的な推奨や自動売買は行いません。

- 自動売買なし - 売買推奨なし - ローカルRAG + 非助言 + 日本株 + 投信 + EDINET / CSV
@@ -343,28 +511,96 @@ export function App() { ))} -
); @@ -391,6 +627,199 @@ function useAsync() { return { loading, error, data, run }; } +function ExperienceModeSwitch(props: { + value: ExperienceMode; + onChange: (value: ExperienceMode) => void; +}) { + return ( +
+
+

操作モード

+ 画面の密度を選ぶ +
+
+ {EXPERIENCE_MODES.map((mode) => ( + + ))} +
+
+ ); +} + +function CommandCenter(props: { + currentTab: TabId; + experienceMode: ExperienceMode; + financialsCsvPath: string; + onNavigate: (tab: TabId) => void; +}) { + const statusState = useAsync(); + const loadStatus = () => + statusState.run(() => + api("/api/data/status", { + financials_csv: props.financialsCsvPath, + stale_after_days: 7, + }), + ); + + useEffect(() => { + void loadStatus(); + }, [props.financialsCsvPath]); + + const byKey = (statusState.data?.by_key ?? {}) as Json; + const summary = (statusState.data?.summary ?? {}) as Json; + const readyCount = Number(summary.ready_count ?? 0); + const missingCount = Number(summary.missing_count ?? 0); + const signalRows = [ + { key: "financials", label: "財務", target: "data" }, + { key: "market_prices", label: "株価", target: "data" }, + { key: "daily_bars", label: "日足", target: "data" }, + { key: "company_master", label: "会社", target: "data" }, + ] satisfies { key: string; label: string; target: TabId }[]; + + const missingFinancials = datasetStatus(byKey.financials) !== "ready"; + const missingMarket = datasetStatus(byKey.market_prices) !== "ready"; + const missingBars = datasetStatus(byKey.daily_bars) !== "ready"; + const modeText = + props.experienceMode === "simple" + ? "かんたん" + : props.experienceMode === "analyst" + ? "分析" + : "データ運用"; + const commandActions = + props.experienceMode === "simple" + ? COMMAND_ACTIONS + : props.experienceMode === "analyst" + ? [ + ...COMMAND_ACTIONS.slice(0, 3), + { id: "simulate" as const, step: "4", title: "試算", body: "価格と配当を検算する" }, + { id: "report" as const, step: "5", title: "根拠出力", body: "計算式と免責を残す" }, + ] + : [ + ...COMMAND_ACTIONS.slice(0, 1), + { id: "detail" as const, step: "2", title: "詳細確認", body: "銘柄ごとの根拠を見る" }, + { id: "answer" as const, step: "3", title: "AI整理", body: "根拠付きで質問する" }, + { id: "evidence" as const, step: "4", title: "根拠検索", body: "RAGと演算子を確認する" }, + ]; + const next = + missingFinancials + ? { + tab: "data" as const, + title: "まず財務データを整える", + body: "候補抽出、詳細、試算、レポートの共通土台です。取得済みCSVかEDINET更新から始めます。", + cta: "データを更新", + } + : missingMarket || missingBars + ? { + tab: "data" as const, + title: "市場価格と日足をそろえる", + body: "市場価格更新で保有価格へ反映できる状態にします。四本値・出来高は流動性確認にも使います。", + cta: "株価を更新", + } + : props.currentTab === "dashboard" + ? { + tab: "holdings" as const, + title: "保有を入れて現在地を見る", + body: "CSVまたは手入力で、評価額、NISA区分、配当・分配金の見込みを機械的に集計します。", + cta: "保有を入力", + } + : { + tab: "report" as const, + title: "根拠付きレポートにまとめる", + body: "重要KPI、計算式、出典、免責をひとつの流れで確認できます。", + cta: "レポートへ", + }; + + return ( +
+
+

次の一手 / {modeText}

+

{next.title}

+

{next.body}

+
+ + +
+ {statusState.error &&

状態取得エラー: {statusState.error}

} +
+ +
+
+ 準備済み + {readyCount.toLocaleString()} +
+
+ 要確認 + {missingCount.toLocaleString()} +
+ {signalRows.map((row) => { + const item = byKey[row.key] as Json | undefined; + const status = datasetStatus(item); + return ( + + ); + })} +
+ +
+ {commandActions.map((action) => ( + + ))} +
+
+ ); +} + +function datasetStatus(item: unknown): string { + if (!item || typeof item !== "object") return "unknown"; + const status = String((item as Json).status ?? "unknown"); + return status.trim() || "unknown"; +} + +function statusLabel(status: string): string { + if (status === "ready" || status === "fresh") return "利用可"; + if (status === "stale") return "更新推奨"; + if (status === "missing") return "未取得"; + if (status === "invalid" || status === "error") return "要確認"; + return "確認中"; +} + +function statusTone(status: string): string { + if (status === "ready" || status === "fresh") return "ok"; + if (status === "missing" || status === "stale" || status === "invalid" || status === "error") { + return "warn"; + } + return ""; +} + function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } @@ -424,6 +853,109 @@ function Field(props: { label: string; children: ReactNode }) { ); } +function readLocalStorageString(key: string, fallback = ""): string { + if (typeof window === "undefined") return fallback; + try { + const value = window.localStorage.getItem(key); + return value && value.trim() ? value : fallback; + } catch { + return fallback; + } +} + +function readExperienceMode(key: string, fallback: ExperienceMode): ExperienceMode { + const value = readLocalStorageString(key, fallback); + return value === "simple" || value === "analyst" || value === "operator" + ? value + : fallback; +} + +function writeLocalStorageString(key: string, value: string): boolean { + if (typeof window === "undefined") return false; + try { + if (value.trim()) { + window.localStorage.setItem(key, value); + } else { + window.localStorage.removeItem(key); + } + return true; + } catch { + return false; + } +} + +function FinancialsSourceBar(props: { + value: string; + onChange: (value: string) => void; + onOpenData: () => void; + onDataUpdated: () => void; + refreshKey: number; +}) { + const statusState = useAsync(); + const refreshStatus = () => + statusState.run(() => + api("/api/financials/status", { + path: props.value, + stale_after_days: 7, + }), + ); + useEffect(() => { + void refreshStatus(); + }, [props.value, props.refreshKey]); + const status = String(statusState.data?.status ?? "checking"); + const statusLabel = + status === "fresh" + ? "利用可能" + : status === "stale" + ? "更新推奨" + : status === "missing" + ? "未作成" + : status === "invalid" + ? "要確認" + : "確認中"; + const statusBadgeClass = status === "fresh" ? "safe" : status === "checking" ? "" : "warn"; + return ( +
+
+ 現在使う財務データ + 候補抽出、銘柄詳細、試算、レポートで共通利用します。 + {props.value} +
+ {statusLabel} + {statusState.data?.available === true && ( + <> + {Number(statusState.data.company_count ?? 0).toLocaleString()}社 + {Number(statusState.data.point_count ?? 0).toLocaleString()}件 + 更新: {formatDateTime(statusState.data.modified_at)} + + )} + {statusState.data?.available === false && ( + {String(statusState.data.hint ?? "Dataタブで財務データを更新してください。")} + )} +
+
+
+ props.onChange(event.target.value)} + aria-label="財務データパス" + /> + + +
+ 切替 + + +
+
+
+ ); +} + function csvEscape(value: unknown): string { const text = String(value ?? ""); return /[",\r\n]/.test(text) ? `"${text.replace(/"/g, '""')}"` : text; @@ -446,7 +978,8 @@ function appendCsvDraft( } function downloadTextFile(filename: string, text: string, type = "text/csv"): void { - const blob = new Blob([text], { type: `${type};charset=utf-8` }); + const normalizedText = type === "text/csv" && !text.startsWith("\ufeff") ? `\ufeff${text}` : text; + const blob = new Blob([normalizedText], { type: `${type};charset=utf-8` }); const url = URL.createObjectURL(blob); const anchor = document.createElement("a"); anchor.href = url; @@ -457,6 +990,37 @@ function downloadTextFile(filename: string, text: string, type = "text/csv"): vo URL.revokeObjectURL(url); } +function splitTickerInput(value: string): string[] { + return value + .split(/[\s,、，]+/) + .map((item) => item.trim()) + .filter(Boolean); +} + +async function readCsvFileText(file: File): Promise { + const buffer = await file.arrayBuffer(); + for (const encoding of ["utf-8", "shift_jis", "euc-jp"]) { + try { + return new TextDecoder(encoding, { fatal: true }).decode(buffer); + } catch { + // Try the next common Japanese CSV encoding. + } + } + return new TextDecoder("utf-8").decode(buffer); +} + +async function readFileAsBase64(file: File): Promise { + const buffer = await file.arrayBuffer(); + const bytes = new Uint8Array(buffer); + const chunkSize = 0x8000; + let binary = ""; + for (let offset = 0; offset < bytes.length; offset += chunkSize) { + const chunk = bytes.subarray(offset, offset + chunkSize); + binary += String.fromCharCode(...Array.from(chunk)); + } + return btoa(binary); +} + function jsonText(value: unknown): string { return JSON.stringify(value ?? {}, null, 2); } @@ -467,6 +1031,36 @@ function Status(props: { loading: boolean; error: string | null }) { return null; } +function edinetApiKeySourceLabel(value: unknown): string { + switch (String(value ?? "")) { + case "runtime_input": + return "画面入力"; + case "dotenv": + return ".env"; + case "process_env": + return "環境変数"; + case "missing": + return "未設定"; + default: + return "確認中"; + } +} + +function refreshModeLabel(value: unknown): string { + switch (String(value ?? "")) { + case "edinet_api": + return "財務CSV更新"; + case "disclosure_scrape_only": + return "公式ページ取得のみ"; + case "official_disclosure_rag": + return "公式根拠RAG補完"; + case "missing_already_complete": + return "未取得なし"; + default: + return "更新結果"; + } +} + function makeCandidatePresetId(): string { const cryptoValue = globalThis.crypto; if (typeof cryptoValue?.randomUUID === "function") return cryptoValue.randomUUID(); @@ -618,23 +1212,348 @@ function EvidencePanel({ ) : ( 根拠行がありません。入力データまたはprovider設定を確認してください。 )} - - - {note && ( -
-
注記
-
{note}
+ +
+ {note && ( +
+
注記
+
{note}
+
+ )} +
+
免責
+
+ {disclaimerText || + "この表示は比較材料であり、売買推奨や投資助言ではありません。"} +
+
+ + + ); +} + +function formatCompactNumber(value: unknown): string { + if (typeof value === "number") { + return Number.isInteger(value) + ? value.toLocaleString() + : value.toLocaleString(undefined, { maximumFractionDigits: 2 }); + } + return String(value ?? "-"); +} + +function formatRatio(value: unknown): string { + const number = Number(value); + if (!Number.isFinite(number)) return "-"; + return number.toLocaleString(undefined, { maximumFractionDigits: 3 }); +} + +function incomeSourceLabel(value: unknown): string { + const source = String(value ?? ""); + const labels: Record = { + edinet_latest_dividend_per_share: "EDINET最新1株配当", + user_annual_income: "ユーザー入力 年間収入", + user_distribution: "ユーザー入力 分配単価", + not_available: "未入力", + }; + return labels[source] ?? (source || "-"); +} + +function EdinetSummaryPanel({ summary }: { summary?: Json | null }) { + if (!summary || typeof summary !== "object") return null; + const cutYears = Array.isArray(summary.dividend_cut_years) + ? summary.dividend_cut_years.join(", ") + : ""; + return ( +
+
+ EDINET財務 + FY{String(summary.latest_fiscal_year ?? "-")} +
+
+
+
自己資本比率
+
{formatCompactNumber(summary.latest_equity_ratio)}%
+
+
+
1株配当
+
{formatCompactNumber(summary.latest_dividend_per_share)}
+
+
+
営業CF
+
{String(summary.operating_cf_trend_label ?? summary.operating_cf_trend ?? "-")}
+
+
+
減配年度
+
{cutYears || "なし"}
+
+
+ {String(summary.source_ref ?? "")} +
+ ); +} + +function CandidateMetrics({ item }: { item: Json }) { + const metrics = item.metrics ?? {}; + const summary = item.edinet_summary as Json | undefined; + const assetType = String(item.asset_type ?? ""); + const score = typeof item.score === "number" ? item.score : Number(item.score); + return ( +
+ + {assetType === "fund" && ( + + )} +
+ 指標JSON + 
{JSON.stringify(metrics, null, 2)}
+
+
+ ); +} + +function FundScorePanel({ + score, + breakdown, + model, +}: { + score: number | null; + breakdown: Json[]; + model?: Json; +}) { + return ( +
+
+ 投信スコア + {score === null ? "-" : score.toFixed(3)} +
+ {String(model?.note ?? "条件比較のための決定論スコアです。")} + {breakdown.length > 0 && ( + + + + + + + + + + + {breakdown.map((row) => ( + + + + + + + ))} + +
項目重み値寄与
+ {String(row.label ?? row.key)} + {String(row.formula ?? "")} + {formatRatio(row.weight)}{String(row.raw_value ?? "-")}{formatRatio(row.contribution)}
+ )} +
+ ); +} + +function SecuritySearch({ + financialsCsvPath, + onSelect, + onUseSample, + onOpenData, + title = "証券コード検索", +}: { + financialsCsvPath: string; + onSelect: (security: Json) => void; + onUseSample?: () => void; + onOpenData?: () => void; + title?: string; +}) { + const [query, setQuery] = useState(""); + const [limit, setLimit] = useState(10); + const [scope, setScope] = useState("prime"); + const state = useAsync(); + const securities: Json[] = Array.isArray(state.data?.securities) + ? state.data.securities + : []; + const search = () => + state.run(() => + api("/api/market/universe", { + financials_csv: financialsCsvPath, + query, + limit, + scope, + }), + ); + const currentScopeLabel = + MARKET_SCOPE_OPTIONS.find((option) => option.value === scope)?.label ?? scope; + return ( +
+
+
+

{title}

+

+ 東証プライム、日経225、財務データありの範囲を切り替えて、証券コードまたは企業名で検索します。 + 空欄で検索すると選択中の範囲を一覧表示します。 + 選択しても売買操作は行いません。 +

+
+ {currentScopeLabel} +
+
+ + + + + setQuery(event.target.value)} + placeholder="例: 7203 / トヨタ" + /> + + + setLimit(Number(event.target.value))} + /> + + + +
+ + {state.data?.hint &&

{String(state.data.hint)}

} + {state.data?.sources && ( +
+ + 公式ソース + JPX / Nikkei + +
+ +
+
Nikkei 225
+
+ + Nikkei 225 Components + +
+
+
+
+ )} + {state.data?.available === false && ( +
+ 銘柄データがまだありません +

+ {String( + state.data.hint + ?? "DataタブでEDINET取得/手動保存を行うか、サンプルデータに切り替えてください。", + )} +

+
+ {onUseSample && ( + + )} + {onOpenData && ( + + )}
- )} -
-
免責
-
- {disclaimerText || - "この表示は比較材料であり、売買推奨や投資助言ではありません。"} -
- - + )} + {securities.length > 0 && ( + + + + + + + + + + + + + + + + {securities.map((security) => ( + + + + + + + + + + + + ))} + +
コード名称市場日経225財務最新年度自己資本比率1株配当選択
{String(security.ticker)}{String(security.name)}{String(security.market_segment_label ?? security.market_segment ?? "-")}{security.is_nikkei225 ? 日経225 : "-"}{security.has_financials ? あり : "未取得"}{String(security.latest_fiscal_year ?? "-")} + {security.latest_equity_ratio !== undefined + ? `${formatCompactNumber(security.latest_equity_ratio)}%` + : "-"} + + {formatCompactNumber(security.latest_dividend_per_share)} + + +
+ )} + {state.data && securities.length === 0 && ( +

+ {state.data.available === false + ? "財務データまたは市場区分データが見つからないため、銘柄一覧を表示できません。" + : "一致する銘柄がありません。対象範囲、データ取込状況、検索語を確認してください。"} +

+ )} +
); } @@ -741,7 +1660,7 @@ function cleanAssistantAnswer(raw: unknown, skipped?: boolean) { "参照できるローカル文書がまだありません。", "", "1. 結論", - "先にData IntakeでIRページやメモをRAG登録してください。", + "先にデータ画面でIRページやメモをRAG登録してください。", "", "2. 根拠", "このチャットはローカル文書検索結果を根拠に回答する設計です。未登録の情報は根拠化できません。", @@ -785,26 +1704,102 @@ function ResultText({ text, limit = 220 }: { text: unknown; limit?: number }) { ); } -function SearchTab() { - const [query, setQuery] = useState("配当 方針 DOE 配当性向"); +function OperatorCatalog({ data }: { data?: Json | null }) { + const groups: Json[] = Array.isArray(data?.groups) ? data.groups : []; + if (!groups.length) return null; + return ( +
+
+
+

演算子カタログ

+

+ 数値計算、候補抽出、RAG検索がどの式で動くかをレビューできるように固定表示します。 +

+
+ 非助言 / 自動売買なし +
+
+ {groups.map((group) => { + const operators: Json[] = Array.isArray(group.operators) ? group.operators : []; + const weights: Json[] = Array.isArray(group.weights) ? group.weights : []; + return ( +
+

{String(group.label ?? group.key)}

+

{String(group.purpose ?? "")}

+ {group.formula && {String(group.formula)}} + {weights.length > 0 && ( +
+ {weights.map((weight) => ( + + {String(weight.label ?? weight.key)} {Number(weight.weight ?? 0).toFixed(2)} + + ))} +
+ )} +
    + {operators.slice(0, 5).map((operator) => ( +
  • + {String(operator.label ?? operator.key)} + {String(operator.formula ?? "")} +
    • + ))} +
+
+ ); + })} +
+

{String(data?.non_advisory_boundary ?? "")}

+
+ ); +} + +function SearchTabEnhanced() { + const [query, setQuery] = useState("配当方針 DOE 配当性向"); const [dbPath, setDbPath] = useState(DEFAULT_RAG_DB_PATH); const [limit, setLimit] = useState(5); const [hybrid, setHybrid] = useState(true); + const [enhanced, setEnhanced] = useState(true); + const [queryExpansion, setQueryExpansion] = useState(true); + const [maxPerSource, setMaxPerSource] = useState(3); const [alpha, setAlpha] = useState(0.5); const { loading, error, data, run } = useAsync(); + const operators = useAsync(); const search = () => - run(() => api("/api/rag/search", { query, db_path: dbPath, limit, hybrid, alpha })); + run(() => + api("/api/rag/search", { + query, + db_path: dbPath, + limit, + hybrid, + alpha, + enhanced, + query_expansion: queryExpansion, + max_per_source: maxPerSource, + }), + ); + + useEffect(() => { + operators.run(() => api("/api/operators/catalog")); + }, []); const results: Json[] = data?.results ?? []; + const queries: string[] = Array.isArray(data?.queries) ? data.queries.map(String) : []; + const diagnostics = data?.diagnostics ?? null; + const diagnosticOperators: Json[] = Array.isArray(diagnostics?.operators) + ? diagnostics.operators + : []; return (
-

Evidence

-

根拠検索

+

根拠

+

根拠検索と計算式

+

+ 資料から根拠候補を探し、検索方法と計算式を確認できます。 +

- 出典 / 引用 + 出典 / 計算式 / 免責
@@ -817,36 +1812,99 @@ function SearchTab() { setLimit(Number(e.target.value))} /> + + setEnhanced(e.target.checked)} /> + + + setQueryExpansion(e.target.checked)} + disabled={!enhanced} + /> + setHybrid(e.target.checked)} /> - + setAlpha(Number(e.target.value))} disabled={!hybrid} /> + + setMaxPerSource(Number(e.target.value))} + disabled={!enhanced} + /> +
+ {diagnostics && ( +
+
+
+

検索診断

+

+ 方式: {String(diagnostics.mode ?? "")} / 候補:{" "} + {String(diagnostics.candidate_count ?? 0)} +

+
+ RRF k={String(diagnostics.rrf_k ?? "-")} +
+ {queries.length > 0 && ( +
+ {queries.map((item) => ( + {item} + ))} +
+ )} + {diagnosticOperators.length > 0 && ( + + + + + + + + + + {diagnosticOperators.map((operator) => ( + + + + + + ))} + +
演算子式目的
{String(operator.label ?? operator.key)}{String(operator.formula ?? "")}{String(operator.purpose ?? "")}
+ )} +
+ )} {results.length > 0 && ( - + @@ -862,10 +1920,16 @@ function SearchTab() {
# scoresource出典 text
)} + +
); } +function SearchTab() { + return ; +} + // --- Investment-only MVP -------------------------------------------------- function CsvValidationPanel(props: { title: string; data?: Json | null }) { @@ -918,13 +1982,21 @@ function CsvValidationPanel(props: { title: string; data?: Json | null }) { ) : ( -

No blocking CSV issues were found.

+

入力データにブロック要因はありません。

)}
); } -function HoldingsTab() { +function HoldingsTab({ + financialsCsvPath, + onFinancialsCsvPathChange, + onOpenData, +}: { + financialsCsvPath: string; + onFinancialsCsvPathChange: (value: string) => void; + onOpenData: () => void; +}) { const [csv, setCsv] = useState(AUDITABLE_SAMPLE_HOLDINGS_CSV); const importState = useAsync(); const analysisState = useAsync(); @@ -943,30 +2015,30 @@ function HoldingsTab() { const validation = await api("/api/holdings/validate", { csv_text: csv }); setValidationActionMessage( validation.valid === true - ? "Holding CSV validation passed. Analysis can run." - : "Holding CSV validation failed. Analysis is blocked until the listed issues are fixed.", + ? "保有データの検証に通りました。分析できます。" + : "保有データの検証に失敗しました。表示された問題を修正してください。", ); return validation; }); const analyze = () => analysisState.run(async () => { - setValidationActionMessage("Validating holding CSV before analysis."); + setValidationActionMessage("分析前に保有データを検証しています。"); const validation = await validationState.run(() => api("/api/holdings/validate", { csv_text: csv }), ); - if (!validation) throw new Error("Holding CSV validation could not complete."); + if (!validation) throw new Error("保有データの検証を完了できませんでした。"); if (validation.valid !== true) { setValidationActionMessage( - "Analysis stopped: holding CSV validation failed. Review the validation table.", + "保有データの検証に失敗したため、分析を停止しました。検証結果を確認してください。", ); - throw new Error("Fix holding CSV validation errors before analysis."); + throw new Error("分析前に保有データの検証エラーを修正してください。"); } - setValidationActionMessage("Holding CSV validation passed. Running analysis."); + setValidationActionMessage("保有データの検証に通りました。分析を実行しています。"); const result = await api("/api/portfolio/analyze", { csv_text: csv, - financials_csv: SAMPLE_FINANCIALS_PATH, + financials_csv: financialsCsvPath, }); - setValidationActionMessage("Analysis completed after holding CSV validation."); + setValidationActionMessage("保有データ検証後に分析が完了しました。"); return result; }); const loadSampleHoldings = () => setCsv(AUDITABLE_SAMPLE_HOLDINGS_CSV); @@ -975,15 +2047,25 @@ function HoldingsTab() { setHoldingDraft((current) => ({ ...current, [column]: value })); const addHoldingDraftRow = () => { setCsv((current) => appendCsvDraft(current, HOLDING_CSV_COLUMNS, holdingDraft)); - setValidationActionMessage("Manual holding row was added to the CSV. Validate before import."); + setValidationActionMessage("手入力行を保有データに追加しました。取込前に検証してください。"); }; const importHoldingFile = (event: ChangeEvent) => { const file = event.currentTarget.files?.[0]; event.currentTarget.value = ""; if (!file) return; - void file.text().then((text) => { - setCsv(text); - setValidationActionMessage(`Loaded ${file.name}. Validate before import or analysis.`); + void importState.run(async () => { + const converted = await api("/api/holdings/file/convert", { + filename: file.name, + content_type: file.type, + content_base64: await readFileAsBase64(file), + }); + setCsv(String(converted.csv_text ?? "")); + setValidationActionMessage( + converted.valid === true + ? `${file.name} をCSVとして読み取りました。必要に応じて内容を確認して分析できます。` + : `${file.name} を読み取りましたが、保有データとして不足があります。検証結果を確認してください。`, + ); + return converted; }); }; const downloadHoldingCsv = () => downloadTextFile("investment_holdings.csv", csv); @@ -1022,23 +2104,35 @@ function HoldingsTab() {
-

Holdings

-

保有一覧・ポートフォリオ分析

+

保有

+

保有分析

日本株 + 投信

- 保有CSVまたは手入力相当のCSVから、評価額、評価損益、配当/分配金見込み、NISA枠、 + 保有データまたは手入力データから、評価額、評価損益、配当/分配金見込み、NISA枠、 集中度を機械的に集計します。売買推奨や注文連携は行いません。

- +