From d2551a1266aa51998c6adec6956af2af68355332 Mon Sep 17 00:00:00 2001 From: Daniel Bigham Date: Fri, 3 Oct 2025 11:44:16 -0400 Subject: [PATCH 1/5] Add net deposits and total P&L for single accounts --- client/src/App.css | 12 + client/src/App.jsx | 59 ++- client/src/components/SummaryMetrics.jsx | 13 + server/src/index.js | 547 +++++++++++++++++++++++ 4 files changed, 628 insertions(+), 3 deletions(-) diff --git a/client/src/App.css b/client/src/App.css index c6f9f0b..78588d0 100644 --- a/client/src/App.css +++ b/client/src/App.css @@ -743,6 +743,18 @@ textarea { border-bottom: 1px dotted var(--color-border-strong); } +.equity-card__metric-separator { + padding: 8px 0 4px; + text-align: center; + color: var(--color-text-muted); + font-size: 12px; + letter-spacing: 4px; +} + +.equity-card__metric-separator span { + display: inline-block; +} + .equity-card__metric-row[data-interactive='true'] { cursor: pointer; diff --git a/client/src/App.jsx b/client/src/App.jsx index ca72e01..1beb522 100644 --- a/client/src/App.jsx +++ b/client/src/App.jsx @@ -197,6 +197,7 @@ function buildClipboardSummary({ displayTotalEquity, usdToCadRate, pnl, + netDeposits, positions, asOf, currencyOption, @@ -224,6 +225,9 @@ function buildClipboardSummary({ lines.push(`Today's P&L: ${formatSignedMoney(pnl?.dayPnl)}`); lines.push(`Open P&L: ${formatSignedMoney(pnl?.openPnl)}`); lines.push(`Total P&L: ${formatSignedMoney(pnl?.totalPnl)}`); + if (netDeposits !== null && netDeposits !== undefined) { + lines.push(`Net deposits: ${formatSignedMoney(netDeposits)}`); + } lines.push(`Total equity: ${formatMoney(balances?.totalEquity)}`); lines.push(`Market value: ${formatMoney(balances?.marketValue)}`); lines.push(`Cash: ${formatMoney(balances?.cash)}`); @@ -987,6 +991,7 @@ export default function App() { const rawPositions = useMemo(() => data?.positions ?? [], [data?.positions]); const balances = data?.balances || null; const accountBalances = data?.accountBalances ?? EMPTY_OBJECT; + const accountFundingSummaries = data?.accountFundingSummaries ?? EMPTY_OBJECT; const investmentModelEvaluations = data?.investmentModelEvaluations ?? EMPTY_OBJECT; const asOf = data?.asOf || null; @@ -1256,6 +1261,51 @@ export default function App() { }; }, [activeCurrency, balancePnlSummaries, fallbackPnl]); + const selectedAccountFunding = useMemo(() => { + if (!selectedAccountInfo) { + return null; + } + return accountFundingSummaries[selectedAccountInfo.id] || null; + }, [selectedAccountInfo, accountFundingSummaries]); + + const combinedFundingEntry = useMemo(() => { + if (!selectedAccountInfo || !selectedAccountFunding || selectedAccountFunding.status !== 'ok') { + return null; + } + if (!activeCurrency || activeCurrency.scope !== 'combined') { + return null; + } + return pickBalanceEntry(selectedAccountFunding.combined, activeCurrency.currency) || null; + }, [selectedAccountInfo, selectedAccountFunding, activeCurrency]); + + const totalPnlOverride = useMemo(() => { + if (!combinedFundingEntry) { + return null; + } + const value = combinedFundingEntry.totalPnl; + return typeof value === 'number' && Number.isFinite(value) ? value : null; + }, [combinedFundingEntry]); + + const netDepositsValue = useMemo(() => { + if (!combinedFundingEntry) { + return null; + } + const value = combinedFundingEntry.netDeposits; + return typeof value === 'number' && Number.isFinite(value) ? value : null; + }, [combinedFundingEntry]); + + const summaryPnl = useMemo(() => { + const baseTotal = typeof activePnl.totalPnl === 'number' && Number.isFinite(activePnl.totalPnl) + ? activePnl.totalPnl + : null; + const effectiveTotal = totalPnlOverride !== null && totalPnlOverride !== undefined ? totalPnlOverride : baseTotal; + return { + dayPnl: activePnl.dayPnl, + openPnl: activePnl.openPnl, + totalPnl: effectiveTotal, + }; + }, [activePnl, totalPnlOverride]); + const heatmapMarketValue = useMemo(() => { if (activeBalances && typeof activeBalances === 'object') { const balanceTotalEquity = coerceNumber(activeBalances.totalEquity); @@ -1406,7 +1456,8 @@ export default function App() { balances: activeBalances, displayTotalEquity, usdToCadRate, - pnl: activePnl, + pnl: summaryPnl, + netDeposits: netDepositsValue, positions: orderedPositions, asOf, currencyOption: activeCurrency, @@ -1440,7 +1491,8 @@ export default function App() { activeBalances, displayTotalEquity, usdToCadRate, - activePnl, + summaryPnl, + netDepositsValue, orderedPositions, asOf, activeCurrency, @@ -1547,7 +1599,8 @@ export default function App() { currencyOptions={currencyOptions} onCurrencyChange={setCurrencyView} balances={activeBalances} - pnl={activePnl} + pnl={summaryPnl} + netDeposits={netDepositsValue} asOf={asOf} onRefresh={handleRefresh} displayTotalEquity={displayTotalEquity} diff --git a/client/src/components/SummaryMetrics.jsx b/client/src/components/SummaryMetrics.jsx index b831572..a88c2ef 100644 --- a/client/src/components/SummaryMetrics.jsx +++ b/client/src/components/SummaryMetrics.jsx @@ -192,6 +192,7 @@ export default function SummaryMetrics({ onCurrencyChange, balances, pnl, + netDeposits, asOf, onRefresh, displayTotalEquity, @@ -219,6 +220,8 @@ export default function SummaryMetrics({ const formattedToday = formatSignedMoney(pnl?.dayPnl ?? null); const formattedOpen = formatSignedMoney(pnl?.openPnl ?? null); const formattedTotal = formatSignedMoney(pnl?.totalPnl ?? null); + const hasNetDeposits = Number.isFinite(netDeposits); + const formattedNetDeposits = formatSignedMoney(hasNetDeposits ? netDeposits : null); const safeTotalEquity = Number.isFinite(totalEquity) ? totalEquity : null; @@ -356,6 +359,14 @@ export default function SummaryMetrics({ onActivate={onShowPnlBreakdown ? () => onShowPnlBreakdown('open') : null} /> + {hasNetDeposits && ( + <> + + + + )}
@@ -396,6 +407,7 @@ SummaryMetrics.propTypes = { openPnl: PropTypes.number, totalPnl: PropTypes.number, }).isRequired, + netDeposits: PropTypes.number, asOf: PropTypes.string, onRefresh: PropTypes.func, displayTotalEquity: PropTypes.number, @@ -432,4 +444,5 @@ SummaryMetrics.defaultProps = { chatUrl: null, showQqqTemperature: false, qqqSummary: null, + netDeposits: null, }; diff --git a/server/src/index.js b/server/src/index.js index 9f99e8f..b030acb 100644 --- a/server/src/index.js +++ b/server/src/index.js @@ -22,6 +22,21 @@ const ALLOWED_ORIGIN = process.env.CLIENT_ORIGIN || 'http://localhost:5173'; const tokenCache = new NodeCache(); const tokenFilePath = path.join(process.cwd(), 'token-store.json'); +const ENABLE_TOTAL_PNL_DEBUG = (() => { + const raw = process.env.DEBUG_TOTAL_PNL; + if (raw === undefined || raw === null || String(raw).trim() === '') { + return true; + } + const normalized = String(raw).trim().toLowerCase(); + return ['1', 'true', 'yes', 'on'].includes(normalized); +})(); + +const ACTIVITY_WINDOW_DAYS = 30; +const MAX_ACTIVITY_WINDOWS = 540; // Approximately 45 years of monthly windows. +const FRED_SERIES_ID_USD_CAD = 'DEXCAUS'; +const fredRateCache = new Map(); +const fredPendingRequests = new Map(); + function resolveLoginDisplay(login) { if (!login) { return null; @@ -539,6 +554,509 @@ async function fetchBalances(login, accountId) { return data || {}; } +function extractNumeric(value) { + if (typeof value === 'number' && Number.isFinite(value)) { + return value; + } + if (typeof value === 'string') { + const trimmed = value.trim(); + if (!trimmed) { + return null; + } + const numeric = Number(trimmed.replace(/[^0-9.+-]/g, '')); + if (!Number.isNaN(numeric) && Number.isFinite(numeric)) { + return numeric; + } + } + return null; +} + +function resolveActivityCurrency(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + const fields = [ + 'netAmountCurrency', + 'currency', + 'tradeCurrency', + 'symbolCurrency', + 'settlementCurrency', + 'accountCurrency', + ]; + for (const field of fields) { + const value = activity[field]; + if (typeof value === 'string' && value.trim()) { + return value.trim().toUpperCase(); + } + } + return null; +} + +function resolveActivityFxRate(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + const fields = ['fxRate', 'exchangeRate', 'conversionRate', 'rate']; + for (const field of fields) { + const value = extractNumeric(activity[field]); + if (value !== null && value > 0) { + return value; + } + } + return null; +} + +function resolveActivityNetAmount(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + const fields = ['netAmount', 'grossAmount', 'amount', 'cash', 'netCash']; + for (const field of fields) { + const value = extractNumeric(activity[field]); + if (value !== null) { + return value; + } + } + return null; +} + +function parseActivityDate(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + const fields = ['transactionDate', 'tradeDate', 'settlementDate', 'recordDate', 'date']; + for (const field of fields) { + const value = activity[field]; + if (!value) { + continue; + } + const date = new Date(value); + if (!Number.isNaN(date.getTime())) { + return date; + } + } + return null; +} + +function formatDateKey(date) { + if (!(date instanceof Date)) { + return null; + } + const year = date.getUTCFullYear(); + const month = String(date.getUTCMonth() + 1).padStart(2, '0'); + const day = String(date.getUTCDate()).padStart(2, '0'); + return year + '-' + month + '-' + day; +} + +function addDays(date, days) { + const base = date instanceof Date ? new Date(date.getTime()) : new Date(date); + if (Number.isNaN(base.getTime())) { + return null; + } + base.setUTCDate(base.getUTCDate() + days); + return base; +} + +function formatIsoParam(date) { + if (date instanceof Date) { + return date.toISOString(); + } + if (typeof date === 'string') { + const parsed = new Date(date); + if (!Number.isNaN(parsed.getTime())) { + return parsed.toISOString(); + } + } + return null; +} + +async function fetchAccountActivities(login, accountId, startDate, endDate) { + const params = {}; + const start = formatIsoParam(startDate); + const end = formatIsoParam(endDate); + if (start) { + params.startTime = start; + } + if (end) { + params.endTime = end; + } + const data = await questradeRequest(login, '/v1/accounts/' + accountId + '/activities', { params }); + if (!data || !Array.isArray(data.activities)) { + return []; + } + return data.activities; +} + +function determineFundingDirection(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + const inKeywords = [ + 'deposit', + 'transfer in', + 'transfer-in', + 'transferin', + 'journal in', + 'journaled in', + 'journal cash in', + 'journaled cash in', + 'cash in', + 'incoming', + ]; + const outKeywords = [ + 'withdraw', + 'withdrawal', + 'transfer out', + 'transfer-out', + 'transferout', + 'journal out', + 'journaled out', + 'journal cash out', + 'journaled cash out', + 'cash out', + 'outgoing', + ]; + + const candidateFields = [ + activity.type, + activity.action, + activity.transactionSubType, + activity.description, + activity.journalType, + ]; + + const matches = function (keywords) { + return candidateFields.some(function (field) { + if (typeof field !== 'string') { + return false; + } + const normalized = field.toLowerCase(); + return keywords.some(function (keyword) { + return normalized.includes(keyword); + }); + }); + }; + + if (matches(inKeywords)) { + return 'in'; + } + if (matches(outKeywords)) { + return 'out'; + } + + const normalizedType = typeof activity.type === 'string' ? activity.type.toLowerCase() : ''; + if (normalizedType.includes('deposit') || normalizedType.includes('transfer') || normalizedType.includes('withdraw')) { + const netAmount = resolveActivityNetAmount(activity); + if (netAmount !== null) { + if (netAmount > 0) { + return 'in'; + } + if (netAmount < 0) { + return 'out'; + } + } + } + + return null; +} + +function buildActivityKey(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + if (activity.transactionId) { + return 'id:' + activity.transactionId; + } + const components = [activity.type, activity.action, activity.transactionDate, activity.tradeDate, activity.description]; + const serialized = components + .map(function (value) { + return value == null ? '' : String(value); + }) + .join('|'); + const netAmount = resolveActivityNetAmount(activity); + const suffix = Number.isFinite(netAmount) ? ':' + netAmount.toFixed(4) : ''; + return serialized + suffix; +} + +async function fetchFredRateRange(startDate, endDate, debugInfo) { + const apiKey = process.env.FRED_API_KEY; + if (!apiKey) { + return null; + } + const startKey = formatDateKey(startDate); + const endKey = formatDateKey(endDate); + if (!startKey || !endKey) { + return null; + } + const requestKey = startKey + ':' + endKey; + if (fredPendingRequests.has(requestKey)) { + return fredPendingRequests.get(requestKey); + } + const requestPromise = axios + .get('https://api.stlouisfed.org/fred/series/observations', { + params: { + series_id: FRED_SERIES_ID_USD_CAD, + observation_start: startKey, + observation_end: endKey, + api_key: apiKey, + file_type: 'json', + }, + }) + .then(function (response) { + const observations = Array.isArray(response.data && response.data.observations) + ? response.data.observations + : []; + let latest = null; + observations.forEach(function (observation) { + const dateKey = observation && observation.date ? String(observation.date).slice(0, 10) : null; + const numeric = observation ? extractNumeric(observation.value) : null; + if (dateKey && numeric !== null && numeric > 0) { + fredRateCache.set(dateKey, numeric); + latest = { date: dateKey, value: numeric }; + } + }); + if (debugInfo && ENABLE_TOTAL_PNL_DEBUG) { + debugInfo.fxLookups.push({ + request: { start: startKey, end: endKey }, + observations: observations.length, + latest: latest || null, + }); + } + return latest; + }) + .catch(function (error) { + if (ENABLE_TOTAL_PNL_DEBUG && debugInfo) { + debugInfo.errors.push({ + scope: 'fred', + message: error && error.message ? error.message : 'Failed to fetch FRED data', + request: { start: startKey, end: endKey }, + }); + } + return null; + }) + .finally(function () { + fredPendingRequests.delete(requestKey); + }); + fredPendingRequests.set(requestKey, requestPromise); + return requestPromise; +} + +async function resolveUsdToCadRate(date, debugInfo) { + const apiKey = process.env.FRED_API_KEY; + if (!apiKey) { + return null; + } + const normalizedDate = date instanceof Date ? date : new Date(date); + if (Number.isNaN(normalizedDate.getTime())) { + return null; + } + const requestedKey = formatDateKey(normalizedDate); + if (requestedKey && fredRateCache.has(requestedKey)) { + return fredRateCache.get(requestedKey); + } + + const MAX_LOOKBACK_DAYS = 365; + for (let offset = 0; offset <= MAX_LOOKBACK_DAYS; offset += 7) { + const windowEnd = addDays(normalizedDate, -offset); + if (!windowEnd) { + continue; + } + const windowStart = addDays(windowEnd, -7); + if (!windowStart) { + continue; + } + const latest = await fetchFredRateRange(windowStart, windowEnd, debugInfo); + if (latest && latest.value) { + if (requestedKey && !fredRateCache.has(requestedKey)) { + fredRateCache.set(requestedKey, latest.value); + } + return latest.value; + } + } + + return null; +} + +async function loadFundingActivities(login, accountId, debugInfo) { + const now = new Date(); + const collected = []; + const seenKeys = new Set(); + let windowEnd = new Date(now.getTime() + 1000); + let foundFunding = false; + + for (let index = 0; index < MAX_ACTIVITY_WINDOWS; index += 1) { + const windowStart = addDays(windowEnd, -ACTIVITY_WINDOW_DAYS); + if (!windowStart) { + break; + } + const activities = await fetchAccountActivities(login, accountId, windowStart, windowEnd); + const fundingEntries = []; + activities.forEach(function (activity) { + const direction = determineFundingDirection(activity); + if (!direction) { + return; + } + const key = buildActivityKey(activity); + if (key && seenKeys.has(key)) { + return; + } + if (key) { + seenKeys.add(key); + } + fundingEntries.push({ direction, activity }); + }); + + if (ENABLE_TOTAL_PNL_DEBUG && debugInfo) { + debugInfo.windows.push({ + start: formatIsoParam(windowStart), + end: formatIsoParam(windowEnd), + totalActivities: activities.length, + fundingActivities: fundingEntries.length, + }); + } + + if (fundingEntries.length) { + collected.push.apply(collected, fundingEntries); + foundFunding = true; + } else if (foundFunding) { + break; + } + + windowEnd = windowStart; + if (windowEnd.getUTCFullYear() < 1998) { + break; + } + } + + return collected; +} + +async function computeAccountFundingSummary({ login, account, combinedBalances }) { + if (!login || !account) { + return null; + } + + const debugInfo = ENABLE_TOTAL_PNL_DEBUG + ? { accountId: account.id, windows: [], activities: [], fxLookups: [], errors: [] } + : null; + + const fundingEntries = await loadFundingActivities(login, account.number, debugInfo); + if (!fundingEntries.length) { + if (ENABLE_TOTAL_PNL_DEBUG) { + console.log('[total-pnl] No funding activities found for account', account.number); + } + return { status: 'no_data', debug: debugInfo }; + } + + const perCurrency = Object.create(null); + let combinedCad = 0; + + for (const entry of fundingEntries) { + const activity = entry.activity; + const direction = entry.direction === 'out' ? 'out' : 'in'; + const rawAmount = resolveActivityNetAmount(activity); + if (rawAmount === null) { + if (debugInfo) { + debugInfo.errors.push({ + scope: 'activity', + message: 'Missing net amount for funding activity', + activity, + }); + } + continue; + } + const currency = resolveActivityCurrency(activity) || 'CAD'; + const amount = direction === 'out' ? -Math.abs(rawAmount) : Math.abs(rawAmount); + if (!perCurrency[currency]) { + perCurrency[currency] = { currency, netDeposits: 0, entries: [] }; + } + perCurrency[currency].netDeposits += amount; + + let cadEquivalent = null; + let fxRateUsed = null; + if (currency === 'CAD') { + cadEquivalent = amount; + fxRateUsed = 1; + } else if (currency === 'USD') { + fxRateUsed = resolveActivityFxRate(activity); + if (fxRateUsed === null || !(fxRateUsed > 0)) { + const activityDate = parseActivityDate(activity) || new Date(); + fxRateUsed = await resolveUsdToCadRate(activityDate, debugInfo); + } + if (fxRateUsed && fxRateUsed > 0) { + cadEquivalent = amount * fxRateUsed; + } + } + + if (cadEquivalent !== null) { + combinedCad += cadEquivalent; + } else if (debugInfo) { + debugInfo.errors.push({ + scope: 'conversion', + message: 'Unable to convert funding activity to CAD', + activity, + currency, + }); + } + + if (debugInfo) { + debugInfo.activities.push({ + direction, + currency, + amount, + cadEquivalent, + fxRate: fxRateUsed, + date: formatIsoParam(parseActivityDate(activity)), + description: activity && activity.description ? String(activity.description) : null, + type: activity && activity.type ? String(activity.type) : null, + action: activity && activity.action ? String(activity.action) : null, + }); + } + } + + const summary = { + status: 'ok', + perCurrency: {}, + combined: {}, + }; + + Object.keys(perCurrency).forEach(function (currency) { + const bucket = perCurrency[currency]; + summary.perCurrency[currency] = { + currency, + netDeposits: bucket.netDeposits, + }; + }); + + const combinedCadEntry = { currency: 'CAD', netDeposits: combinedCad, totalEquity: null, totalPnl: null }; + if (combinedBalances && typeof combinedBalances === 'object') { + const cadKey = Object.keys(combinedBalances).find(function (key) { + return key && key.toUpperCase() === 'CAD'; + }); + if (cadKey) { + const cadBalance = combinedBalances[cadKey]; + const totalEquity = extractNumeric(cadBalance && cadBalance.totalEquity); + if (totalEquity !== null) { + combinedCadEntry.totalEquity = totalEquity; + if (Number.isFinite(combinedCad)) { + combinedCadEntry.totalPnl = totalEquity - combinedCad; + } + } + } + } + + summary.combined.CAD = combinedCadEntry; + + if (debugInfo) { + summary.debug = debugInfo; + if (ENABLE_TOTAL_PNL_DEBUG) { + console.log('[total-pnl] Funding summary for account', account.number, JSON.stringify(summary, null, 2)); + } + } + + return summary; +} + const BALANCE_NUMERIC_FIELDS = [ 'totalEquity', @@ -1128,6 +1646,7 @@ app.get('/api/summary', async function (req, res) { const decoratedPositions = decoratePositions(flattenedPositions, symbolsMap, accountsMap); const pnl = mergePnL(flattenedPositions); + pnl.totalPnl = null; const balancesSummary = mergeBalances(balancesResults); finalizeBalances(balancesSummary); @@ -1155,6 +1674,33 @@ app.get('/api/summary', async function (req, res) { } } + const accountFundingSummaries = {}; + if (selectedContexts.length === 1) { + const context = selectedContexts[0]; + try { + const combinedBalance = perAccountCombinedBalances[context.account.id] || null; + const fundingSummary = await computeAccountFundingSummary({ + login: context.login, + account: context.account, + combinedBalances: combinedBalance, + }); + if (fundingSummary) { + accountFundingSummaries[context.account.id] = fundingSummary; + if (fundingSummary.status === 'ok' && fundingSummary.combined) { + const cadEntry = fundingSummary.combined.CAD || fundingSummary.combined.Cad || fundingSummary.combined.cad || null; + const totalPnlValue = cadEntry && typeof cadEntry.totalPnl === 'number' ? cadEntry.totalPnl : null; + if (Number.isFinite(totalPnlValue)) { + pnl.totalPnl = totalPnlValue; + } + } + } + } catch (fundingError) { + const message = fundingError && fundingError.message ? fundingError.message : 'Failed to compute funding summary.'; + console.warn('Funding summary error for account ' + context.account.number + ':', message); + accountFundingSummaries[context.account.id] = { status: 'error', message }; + } + } + const responseAccounts = allAccounts.map(function (account) { return { id: account.id, @@ -1192,6 +1738,7 @@ app.get('/api/summary', async function (req, res) { pnl: pnl, balances: balancesSummary, accountBalances: perAccountCombinedBalances, + accountFundingSummaries, investmentModelEvaluations, asOf: new Date().toISOString(), }); From f17f0dbe7eca7b6ce2d70f086a18f20cd8ba18fc Mon Sep 17 00:00:00 2001 From: Daniel Bigham Date: Fri, 3 Oct 2025 12:07:39 -0400 Subject: [PATCH 2/5] Handle in-kind transfers when computing net deposits --- server/src/index.js | 93 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 5 deletions(-) diff --git a/server/src/index.js b/server/src/index.js index b030acb..7045691 100644 --- a/server/src/index.js +++ b/server/src/index.js @@ -610,7 +610,7 @@ function resolveActivityNetAmount(activity) { if (!activity || typeof activity !== 'object') { return null; } - const fields = ['netAmount', 'grossAmount', 'amount', 'cash', 'netCash']; + const fields = ['netAmount', 'grossAmount', 'amount', 'cash', 'netCash', 'bookValue', 'marketValue', 'transferValue', 'transferAmount']; for (const field of fields) { const value = extractNumeric(activity[field]); if (value !== null) { @@ -620,6 +620,76 @@ function resolveActivityNetAmount(activity) { return null; } +function parseBookValueFromDescription(description) { + if (typeof description !== 'string' || !description.trim()) { + return null; + } + const match = description.match(/book value[^0-9+-]*([-+]?[0-9.,]+)/i); + if (!match || !match[1]) { + return null; + } + const numeric = extractNumeric(match[1]); + if (numeric === null) { + return null; + } + return numeric; +} + +function estimateFundingActivityValue(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + + const currency = resolveActivityCurrency(activity); + const normalizedCurrency = currency ? currency.toUpperCase() : null; + + const directFieldCandidates = [ + ['bookValue', 'book_value_field'], + ['marketValue', 'market_value_field'], + ['settlementAmount', 'settlement_amount_field'], + ['transferValue', 'transfer_value_field'], + ['transferAmount', 'transfer_amount_field'], + ]; + + if (!normalizedCurrency || normalizedCurrency === 'CAD') { + directFieldCandidates.push(['bookValueInBase', 'book_value_in_base_field']); + directFieldCandidates.push(['marketValueInBase', 'market_value_in_base_field']); + } + + for (const [field, source] of directFieldCandidates) { + const value = extractNumeric(activity[field]); + if (value !== null && value !== 0) { + return { amount: Math.abs(value), source }; + } + } + + const descriptionValue = parseBookValueFromDescription(activity.description); + if (descriptionValue !== null && descriptionValue !== 0) { + return { amount: Math.abs(descriptionValue), source: 'description_book_value' }; + } + + const quantityFields = ['quantity', 'qty', 'units', 'shares']; + const priceFields = ['price', 'tradePrice', 'bookPrice', 'grossPrice', 'averagePrice']; + let quantity = null; + for (const field of quantityFields) { + const value = extractNumeric(activity[field]); + if (value !== null && value !== 0) { + quantity = value; + break; + } + } + if (quantity !== null) { + for (const field of priceFields) { + const price = extractNumeric(activity[field]); + if (price !== null && price !== 0) { + return { amount: Math.abs(quantity * price), source: 'quantity_price_estimate' }; + } + } + } + + return null; +} + function parseActivityDate(activity) { if (!activity || typeof activity !== 'object') { return null; @@ -954,8 +1024,16 @@ async function computeAccountFundingSummary({ login, account, combinedBalances } for (const entry of fundingEntries) { const activity = entry.activity; const direction = entry.direction === 'out' ? 'out' : 'in'; - const rawAmount = resolveActivityNetAmount(activity); - if (rawAmount === null) { + let rawAmount = resolveActivityNetAmount(activity); + let estimationDetails = null; + if (rawAmount === null || rawAmount === 0) { + const estimation = estimateFundingActivityValue(activity); + if (estimation && typeof estimation.amount === 'number' && estimation.amount !== 0) { + rawAmount = estimation.amount; + estimationDetails = estimation; + } + } + if (rawAmount === null || rawAmount === 0) { if (debugInfo) { debugInfo.errors.push({ scope: 'activity', @@ -1000,7 +1078,7 @@ async function computeAccountFundingSummary({ login, account, combinedBalances } } if (debugInfo) { - debugInfo.activities.push({ + const debugEntry = { direction, currency, amount, @@ -1010,7 +1088,12 @@ async function computeAccountFundingSummary({ login, account, combinedBalances } description: activity && activity.description ? String(activity.description) : null, type: activity && activity.type ? String(activity.type) : null, action: activity && activity.action ? String(activity.action) : null, - }); + rawAmount, + }; + if (estimationDetails) { + debugEntry.estimated = estimationDetails; + } + debugInfo.activities.push(debugEntry); } } From d3fe40e53b154cb8d191627b161ec6b1fcf710d0 Mon Sep 17 00:00:00 2001 From: Daniel Bigham Date: Fri, 3 Oct 2025 12:16:40 -0400 Subject: [PATCH 3/5] Improve funding direction heuristics --- server/src/index.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/server/src/index.js b/server/src/index.js index 7045691..e21f8c5 100644 --- a/server/src/index.js +++ b/server/src/index.js @@ -766,12 +766,14 @@ function determineFundingDirection(activity) { 'transfer in', 'transfer-in', 'transferin', + 'transfer from', 'journal in', 'journaled in', 'journal cash in', 'journaled cash in', 'cash in', 'incoming', + 'from account', ]; const outKeywords = [ 'withdraw', @@ -779,12 +781,14 @@ function determineFundingDirection(activity) { 'transfer out', 'transfer-out', 'transferout', + 'transfer to', 'journal out', 'journaled out', 'journal cash out', 'journaled cash out', 'cash out', 'outgoing', + 'to account', ]; const candidateFields = [ @@ -814,6 +818,22 @@ function determineFundingDirection(activity) { return 'out'; } + const normalizedAction = typeof activity.action === 'string' ? activity.action.trim().toUpperCase() : ''; + if (normalizedAction) { + const actionDirectionMap = { + TF6: 'in', + TFO: 'out', + CON: 'in', + DEP: 'in', + WDL: 'out', + WDR: 'out', + WD: 'out', + }; + if (actionDirectionMap[normalizedAction]) { + return actionDirectionMap[normalizedAction]; + } + } + const normalizedType = typeof activity.type === 'string' ? activity.type.toLowerCase() : ''; if (normalizedType.includes('deposit') || normalizedType.includes('transfer') || normalizedType.includes('withdraw')) { const netAmount = resolveActivityNetAmount(activity); From 101248219d461a1ff12208b8586140206c090726 Mon Sep 17 00:00:00 2001 From: Daniel Bigham Date: Fri, 3 Oct 2025 12:39:31 -0400 Subject: [PATCH 4/5] Improve transfer direction heuristics for net deposits --- client/src/components/SummaryMetrics.jsx | 2 +- server/src/index.js | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/client/src/components/SummaryMetrics.jsx b/client/src/components/SummaryMetrics.jsx index a88c2ef..560539f 100644 --- a/client/src/components/SummaryMetrics.jsx +++ b/client/src/components/SummaryMetrics.jsx @@ -362,7 +362,7 @@ export default function SummaryMetrics({ {hasNetDeposits && ( <> diff --git a/server/src/index.js b/server/src/index.js index e21f8c5..a4a8649 100644 --- a/server/src/index.js +++ b/server/src/index.js @@ -835,7 +835,16 @@ function determineFundingDirection(activity) { } const normalizedType = typeof activity.type === 'string' ? activity.type.toLowerCase() : ''; - if (normalizedType.includes('deposit') || normalizedType.includes('transfer') || normalizedType.includes('withdraw')) { + const normalizedDescription = typeof activity.description === 'string' ? activity.description.toLowerCase() : ''; + const looksLikeFunding = + normalizedType.includes('deposit') || + normalizedType.includes('transfer') || + normalizedType.includes('withdraw') || + normalizedType.includes('journal') || + normalizedDescription.includes('transfer') || + normalizedDescription.includes('journal'); + + if (looksLikeFunding) { const netAmount = resolveActivityNetAmount(activity); if (netAmount !== null) { if (netAmount > 0) { @@ -845,6 +854,13 @@ function determineFundingDirection(activity) { return 'out'; } } + const quantityFields = ['quantity', 'qty', 'units', 'shares']; + for (const field of quantityFields) { + const quantity = extractNumeric(activity[field]); + if (quantity !== null && quantity !== 0) { + return quantity > 0 ? 'in' : 'out'; + } + } } return null; From cc5992953d1aad7e5659183637ec3652202259f7 Mon Sep 17 00:00:00 2001 From: Daniel Bigham Date: Fri, 3 Oct 2025 12:55:47 -0400 Subject: [PATCH 5/5] Pair zero-cash transfers to recover missing inflow amounts --- server/src/index.js | 143 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 125 insertions(+), 18 deletions(-) diff --git a/server/src/index.js b/server/src/index.js index a4a8649..3c9a556 100644 --- a/server/src/index.js +++ b/server/src/index.js @@ -635,6 +635,68 @@ function parseBookValueFromDescription(description) { return numeric; } +function normalizeTransferDescription(description) { + if (typeof description !== 'string') { + return null; + } + let normalized = description.toLowerCase(); + normalized = normalized.replace(/book value[^0-9+-]*[-+]?[0-9.,]+/gi, ' '); + normalized = normalized.replace(/\b(to|from)\s+account\b.*$/gi, ' '); + normalized = normalized.replace(/\s+/g, ' ').trim(); + return normalized || null; +} + +function buildTransferPairKey(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + const dateKey = formatDateKey(parseActivityDate(activity)); + const quantity = resolveActivityQuantity(activity); + const normalizedQuantity = quantity !== null ? Math.abs(quantity).toFixed(6) : ''; + const symbolId = activity.symbolId ? String(activity.symbolId).trim() : ''; + const symbol = typeof activity.symbol === 'string' ? activity.symbol.trim().toUpperCase() : ''; + const descriptionKey = normalizeTransferDescription(activity.description); + const components = []; + if (dateKey) { + components.push('date:' + dateKey); + } + if (symbolId) { + components.push('sid:' + symbolId); + } else if (symbol) { + components.push('sym:' + symbol); + } + if (normalizedQuantity) { + components.push('qty:' + normalizedQuantity); + } + if (descriptionKey) { + components.push('desc:' + descriptionKey); + } + if (activity.transactionId) { + components.push('tx:' + String(activity.transactionId)); + } + if (activity.activityId) { + components.push('act:' + String(activity.activityId)); + } + if (!components.length) { + return null; + } + return components.join('|'); +} + +function resolveActivityQuantity(activity) { + if (!activity || typeof activity !== 'object') { + return null; + } + const quantityFields = ['quantity', 'qty', 'units', 'shares']; + for (const field of quantityFields) { + const value = extractNumeric(activity[field]); + if (value !== null && value !== 0) { + return value; + } + } + return null; +} + function estimateFundingActivityValue(activity) { if (!activity || typeof activity !== 'object') { return null; @@ -668,17 +730,9 @@ function estimateFundingActivityValue(activity) { return { amount: Math.abs(descriptionValue), source: 'description_book_value' }; } - const quantityFields = ['quantity', 'qty', 'units', 'shares']; - const priceFields = ['price', 'tradePrice', 'bookPrice', 'grossPrice', 'averagePrice']; - let quantity = null; - for (const field of quantityFields) { - const value = extractNumeric(activity[field]); - if (value !== null && value !== 0) { - quantity = value; - break; - } - } + const quantity = resolveActivityQuantity(activity); if (quantity !== null) { + const priceFields = ['price', 'tradePrice', 'bookPrice', 'grossPrice', 'averagePrice']; for (const field of priceFields) { const price = extractNumeric(activity[field]); if (price !== null && price !== 0) { @@ -1054,21 +1108,71 @@ async function computeAccountFundingSummary({ login, account, combinedBalances } return { status: 'no_data', debug: debugInfo }; } - const perCurrency = Object.create(null); - let combinedCad = 0; - - for (const entry of fundingEntries) { + const preparedEntries = fundingEntries.map(function (entry) { const activity = entry.activity; const direction = entry.direction === 'out' ? 'out' : 'in'; let rawAmount = resolveActivityNetAmount(activity); + if (rawAmount !== null) { + const normalized = Math.abs(rawAmount); + rawAmount = normalized > 0 ? normalized : null; + } let estimationDetails = null; - if (rawAmount === null || rawAmount === 0) { + if (rawAmount === null) { const estimation = estimateFundingActivityValue(activity); if (estimation && typeof estimation.amount === 'number' && estimation.amount !== 0) { - rawAmount = estimation.amount; + rawAmount = Math.abs(estimation.amount); estimationDetails = estimation; } } + const pairKey = buildTransferPairKey(activity); + return { direction, activity, rawAmount, estimation: estimationDetails, pairKey }; + }); + + const pairingBuckets = new Map(); + preparedEntries.forEach(function (entry) { + if (!entry.pairKey) { + return; + } + if (!pairingBuckets.has(entry.pairKey)) { + pairingBuckets.set(entry.pairKey, { resolved: [], pending: [] }); + } + const bucket = pairingBuckets.get(entry.pairKey); + if (entry.rawAmount && entry.rawAmount > 0) { + bucket.resolved.push(entry); + } else { + bucket.pending.push(entry); + } + }); + + pairingBuckets.forEach(function (bucket) { + if (!bucket.pending.length || !bucket.resolved.length) { + return; + } + const resolvedAmount = bucket.resolved + .map(function (entry) { + return entry.rawAmount; + }) + .reduce(function (sum, value) { + return sum + value; + }, 0) / bucket.resolved.length; + if (!(resolvedAmount > 0)) { + return; + } + bucket.pending.forEach(function (entry) { + if (!entry.rawAmount || entry.rawAmount === 0) { + entry.rawAmount = resolvedAmount; + entry.estimation = { amount: resolvedAmount, source: 'paired_transfer' }; + } + }); + }); + + const perCurrency = Object.create(null); + let combinedCad = 0; + + for (const entry of preparedEntries) { + const activity = entry.activity; + const direction = entry.direction; + const rawAmount = entry.rawAmount; if (rawAmount === null || rawAmount === 0) { if (debugInfo) { debugInfo.errors.push({ @@ -1126,8 +1230,11 @@ async function computeAccountFundingSummary({ login, account, combinedBalances } action: activity && activity.action ? String(activity.action) : null, rawAmount, }; - if (estimationDetails) { - debugEntry.estimated = estimationDetails; + if (entry.estimation) { + debugEntry.estimated = entry.estimation; + } + if (entry.pairKey) { + debugEntry.pairKey = entry.pairKey; } debugInfo.activities.push(debugEntry); }