From 31414697a2369c57fdd2a2a0399439816274fde2 Mon Sep 17 00:00:00 2001 From: Pavel Fadeev Date: Sun, 16 Aug 2026 01:56:40 +0200 Subject: [PATCH 1/2] feat(data): add provenance and missing-data diagnostics --- src/api/schemas.ts | 68 ++++++-- src/api/ta.ts | 204 ++++++++++++++++++----- src/api/validation.ts | 3 + src/tests/dataQuality.test.ts | 180 ++++++++++++++++++++ src/tests/integration/core-tools.test.ts | 4 +- src/tests/lookup.test.ts | 5 +- src/tests/ta.test.ts | 7 +- src/tools/metainfo.ts | 25 ++- src/tools/screen.ts | 119 +++++++++++-- src/tools/search.ts | 23 ++- src/utils/resultMetadata.ts | 96 +++++++++++ 11 files changed, 644 insertions(+), 90 deletions(-) create mode 100644 src/tests/dataQuality.test.ts create mode 100644 src/utils/resultMetadata.ts diff --git a/src/api/schemas.ts b/src/api/schemas.ts index 1cca71f..b21b5dd 100644 --- a/src/api/schemas.ts +++ b/src/api/schemas.ts @@ -35,14 +35,45 @@ const STRING_ARRAY = { items: { type: "string" }, } as const; +const RESULT_METADATA_SCHEMA = { + type: "object", + properties: { + retrieved_at: { type: "string", format: "date-time" }, + source: { type: "string" }, + cache_hit: { type: "boolean" }, + requested_count: { type: "integer", minimum: 0 }, + returned_count: { type: "integer", minimum: 0 }, + missing_symbols: STRING_ARRAY, + unavailable_symbols: STRING_ARRAY, + }, + required: [ + "retrieved_at", + "source", + "cache_hit", + "requested_count", + "returned_count", + "missing_symbols", + ], + additionalProperties: false, +} as const; + +function withMetadata(properties: Record) { + return { + ...properties, + metadata: RESULT_METADATA_SCHEMA, + }; +} + +const METADATA_REQUIRED = ["metadata"] as const; + function collectionSchema(name: string) { return { type: "object", - properties: { + properties: withMetadata({ total_count: { type: "integer" }, [name]: { type: "array", items: OBJECT_ROW }, - }, - required: ["total_count", name], + }), + required: ["total_count", name, ...METADATA_REQUIRED], additionalProperties: true, } as const; } @@ -54,50 +85,51 @@ export const OUTPUT_SCHEMAS = { screen_etf: collectionSchema("etfs"), lookup_symbols: { type: "object", - properties: { + properties: withMetadata({ total_count: { type: "integer" }, symbols: { type: "array", items: OBJECT_ROW }, - }, - required: ["total_count", "symbols"], + }), + required: ["total_count", "symbols", ...METADATA_REQUIRED], additionalProperties: true, }, search_symbols: { type: "object", - properties: { + properties: withMetadata({ query: { type: "string" }, count: { type: "integer" }, symbols: { type: "array", items: OBJECT_ROW }, - }, - required: ["query", "count", "symbols"], + }), + required: ["query", "count", "symbols", ...METADATA_REQUIRED], additionalProperties: true, }, get_market_metainfo: { type: "object", - properties: { + properties: withMetadata({ market: { type: "string" }, requested_fields: STRING_ARRAY, metainfo: OBJECT_ROW, - }, - required: ["market"], + }), + required: ["market", ...METADATA_REQUIRED], additionalProperties: true, }, get_ta_summary: { type: "object", - properties: { + properties: withMetadata({ symbols: { type: "array", items: OBJECT_ROW }, - }, - required: ["symbols"], + }), + required: ["symbols", ...METADATA_REQUIRED], additionalProperties: true, }, rank_by_ta: { type: "object", - properties: { + properties: withMetadata({ requested_symbols: { type: "integer" }, timeframes: STRING_ARRAY, weights: { type: "object", additionalProperties: { type: "number" } }, ranked: { type: "array", items: OBJECT_ROW }, - }, - required: ["requested_symbols", "timeframes", "weights", "ranked"], + excluded_symbols: { type: "array", items: OBJECT_ROW }, + }), + required: ["requested_symbols", "timeframes", "weights", "ranked", "excluded_symbols", ...METADATA_REQUIRED], additionalProperties: true, }, list_fields: { diff --git a/src/api/ta.ts b/src/api/ta.ts index 444488f..a7cf123 100644 --- a/src/api/ta.ts +++ b/src/api/ta.ts @@ -9,6 +9,12 @@ import type { TradingViewClient } from "./client.js"; import type { Cache } from "../utils/cache.js"; import type { RateLimiter } from "../utils/rateLimit.js"; +import { + createResultMetadata, + withCacheHitMetadata, + withResultMetadata, +} from "../utils/resultMetadata.js"; +import type { ResultMetadata } from "../utils/resultMetadata.js"; export type Timeframe = "1" | "3" | "5" | "15" | "30" | "45" | "60" | "120" | "180" | "240" | "1D" | "1W" | "1M"; @@ -26,16 +32,18 @@ export interface TASymbolSummary { symbol: string; timeframes: Record; } export interface TASummaryResponse { symbols: TASymbolSummary[]; + metadata: ResultMetadata; } export interface RanksByTAInput { @@ -51,11 +59,18 @@ export interface RankedSymbol { breakdown: Record; } +export interface ExcludedTASymbol { + symbol: string; + reason: "missing_symbol" | "unavailable_ta"; +} + export interface RankByTAResponse { requested_symbols: number; timeframes: string[]; weights: Record; ranked: RankedSymbol[]; + excluded_symbols: ExcludedTASymbol[]; + metadata: ResultMetadata; } /** @@ -74,6 +89,49 @@ export function scoreToLabel(score: number): string { return "strong_buy"; } +function validateTimeframes(timeframes: Timeframe[]): void { + if (timeframes.length === 0) { + throw new Error("At least one timeframe is required"); + } + + for (const tf of timeframes) { + if (!VALID_TIMEFRAMES.includes(tf)) { + throw new Error(`Invalid timeframe '${tf}'. Valid timeframes: ${VALID_TIMEFRAMES.join(", ")}`); + } + } +} + +export function validateTAWeights( + weights: Record | undefined, + timeframes: Timeframe[], +): Record { + const validated: Record = {}; + const selectedTimeframes = new Set(timeframes); + + for (const [timeframe, weight] of Object.entries(weights ?? {})) { + if (!VALID_TIMEFRAMES.includes(timeframe as Timeframe)) { + throw new Error(`Invalid timeframe '${timeframe}' in weights`); + } + if (!selectedTimeframes.has(timeframe as Timeframe)) { + throw new Error(`weights contains timeframe '${timeframe}' that is not being ranked`); + } + if (typeof weight !== "number" || !Number.isFinite(weight) || weight < 0) { + throw new Error(`weight for timeframe '${timeframe}' must be a finite non-negative number`); + } + validated[timeframe] = weight; + } + + const totalWeight = timeframes.reduce( + (total, timeframe) => total + (validated[timeframe] ?? 1), + 0, + ); + if (totalWeight <= 0) { + throw new Error("weights must assign a positive total weight"); + } + + return validated; +} + /** * Build the columns needed for a TA summary across given timeframes. * @@ -97,14 +155,15 @@ function buildTAColumns(timeframes: Timeframe[]): string[] { } /** - * Parse score: TradingView returns scores in range [-1, 1], - * where -1 = strong_sell, 0 = neutral, 1 = strong_buy. - * Sometimes values are null/undefined — treat as neutral. + * Parse score. Missing, blank, and non-numeric values stay unavailable rather + * than becoming a neutral score. */ -function parseScore(val: any): number { - if (val === null || val === undefined) return 0; +function parseScore(val: unknown): number | null { + if (val === null || val === undefined || (typeof val === "string" && val.trim() === "")) { + return null; + } const num = Number(val); - if (isNaN(num)) return 0; + if (!Number.isFinite(num)) return null; return Math.max(-1, Math.min(1, num)); } @@ -133,24 +192,35 @@ export class TAClient { if (symbols.length > 50) { throw new Error("Maximum 50 symbols allowed"); } + validateTimeframes(timeframes); - // Validate timeframes - for (const tf of timeframes) { - if (!VALID_TIMEFRAMES.includes(tf)) { - throw new Error(`Invalid timeframe '${tf}'. Valid timeframes: ${VALID_TIMEFRAMES.join(", ")}`); - } - } - - // Build cache key + const cacheSymbols = [...symbols].sort(); const cacheKey = JSON.stringify({ type: "ta_summary", - symbols: symbols.sort(), + symbols: cacheSymbols, timeframes, include_components, }); + const source = "https://scanner.tradingview.com/global/scan"; const cached = this.cache.get(cacheKey); - if (cached) return cached; + if (cached) { + const cachedSymbols = Array.isArray(cached.symbols) + ? cached.symbols as TASymbolSummary[] + : []; + const cachedUnavailable = cachedSymbols + .filter((item) => timeframes.every((tf) => item.timeframes[tf]?.available !== true)) + .map((item) => item.symbol); + return withCacheHitMetadata(cached, { + source, + requested_count: symbols.length, + returned_count: cachedSymbols.length, + missing_symbols: symbols.filter((symbol) => + !cachedSymbols.some((item) => item.symbol === symbol) + ), + unavailable_symbols: cachedUnavailable, + }); + } // Build columns const columns = buildTAColumns(timeframes); @@ -176,24 +246,26 @@ export class TAClient { // Parse response const results: TASymbolSummary[] = response.data.map((item) => { - const timeframeData: Record = {}; + const timeframeData: TASymbolSummary["timeframes"] = {}; for (const tf of timeframes) { const allIdx = columns.indexOf(`Recommend.All|${tf}`); const otherIdx = columns.indexOf(`Recommend.Other|${tf}`); const maIdx = columns.indexOf(`Recommend.MA|${tf}`); - const allScore = allIdx >= 0 ? parseScore(item.d[allIdx]) : 0; - const otherScore = otherIdx >= 0 ? parseScore(item.d[otherIdx]) : undefined; - const maScore = maIdx >= 0 ? parseScore(item.d[maIdx]) : undefined; + const allScore = allIdx >= 0 ? parseScore(item.d?.[allIdx]) : null; + const otherScore = otherIdx >= 0 ? parseScore(item.d?.[otherIdx]) : null; + const maScore = maIdx >= 0 ? parseScore(item.d?.[maIdx]) : null; + const available = allScore !== null; timeframeData[tf] = { - summary: scoreToLabel(allScore), + summary: available ? scoreToLabel(allScore) : "unavailable", + available, scores: include_components ? { all: allScore, - ...(otherScore !== undefined ? { oscillators: otherScore } : {}), - ...(maScore !== undefined ? { moving_averages: maScore } : {}), + ...(otherIdx >= 0 ? { oscillators: otherScore } : {}), + ...(maIdx >= 0 ? { moving_averages: maScore } : {}), } : { all: allScore }, }; @@ -204,8 +276,20 @@ export class TAClient { timeframes: timeframeData, }; }); - - const result: TASummaryResponse = { symbols: results }; + const returnedSymbols = results.map((item) => item.symbol); + const unavailableSymbols = results + .filter((item) => timeframes.every((tf) => item.timeframes[tf]?.available !== true)) + .map((item) => item.symbol); + const result: TASummaryResponse = withResultMetadata( + { symbols: results }, + createResultMetadata({ + source, + requested_count: symbols.length, + returned_count: results.length, + missing_symbols: symbols.filter((symbol) => !returnedSymbols.includes(symbol)), + unavailable_symbols: unavailableSymbols, + }), + ); this.cache.set(cacheKey, result); return result; } @@ -221,11 +305,13 @@ export class TAClient { weights, } = input; - // Default weights: equal weight + validateTimeframes(timeframes); + const validatedWeights = validateTAWeights(weights, timeframes); + const tfWeights: Record = {}; let totalWeight = 0; for (const tf of timeframes) { - tfWeights[tf] = weights?.[tf] ?? 1; + tfWeights[tf] = validatedWeights[tf] ?? 1; totalWeight += tfWeights[tf]; } @@ -236,39 +322,69 @@ export class TAClient { include_components: false, }); - // Filter to only requested symbols and compute weighted scores - const requestedSet = new Set(symbols); - const ranked: RankedSymbol[] = summary.symbols - .filter((item) => requestedSet.has(item.symbol)) - .map((item) => { + const summaryBySymbol = new Map(summary.symbols.map((item) => [item.symbol, item])); + const ranked: RankedSymbol[] = []; + const excludedSymbols: ExcludedTASymbol[] = []; + + for (const symbol of symbols) { + const item = summaryBySymbol.get(symbol); + if (!item) { + excludedSymbols.push({ symbol, reason: "missing_symbol" }); + continue; + } + + const unavailable = timeframes.some((tf) => { + const timeframeData = item.timeframes[tf]; + return timeframeData?.available !== true || timeframeData.scores.all === null; + }); + if (unavailable) { + excludedSymbols.push({ symbol, reason: "unavailable_ta" }); + continue; + } + const breakdown: Record = {}; let weightedSum = 0; - for (const tf of timeframes) { - const tfData = item.timeframes[tf]; - const score = tfData?.scores?.all ?? 0; + const score = item.timeframes[tf].scores.all; + if (score === null) { + throw new Error(`TA score became unavailable for ${symbol} on ${tf}`); + } breakdown[tf] = score; weightedSum += score * tfWeights[tf]; } - const finalScore = totalWeight > 0 ? weightedSum / totalWeight : 0; - - return { - symbol: item.symbol, + const finalScore = weightedSum / totalWeight; + ranked.push({ + symbol, score: Math.round(finalScore * 100) / 100, label: scoreToLabel(finalScore), breakdown, - }; - }); + }); + } // Sort by score descending ranked.sort((a, b) => b.score - a.score); + const missingSymbols = excludedSymbols + .filter((item) => item.reason === "missing_symbol") + .map((item) => item.symbol); + const unavailableSymbols = excludedSymbols + .filter((item) => item.reason === "unavailable_ta") + .map((item) => item.symbol); return { requested_symbols: symbols.length, timeframes, weights: tfWeights, ranked, + excluded_symbols: excludedSymbols, + metadata: createResultMetadata({ + source: "https://scanner.tradingview.com/global/scan", + cache_hit: summary.metadata.cache_hit, + requested_count: symbols.length, + returned_count: ranked.length, + missing_symbols: missingSymbols, + unavailable_symbols: unavailableSymbols, + }), }; } } \ No newline at end of file diff --git a/src/api/validation.ts b/src/api/validation.ts index 084d24c..d81ff35 100644 --- a/src/api/validation.ts +++ b/src/api/validation.ts @@ -1,7 +1,9 @@ import type { MetainfoInput } from "./metainfo.js"; import type { SearchSymbolsInput } from "./search.js"; import { + DEFAULT_TIMEFRAMES, VALID_TIMEFRAMES, + validateTAWeights, type RanksByTAInput, type TASummaryInput, type Timeframe, @@ -390,6 +392,7 @@ export function validateRankByTAInput(value: unknown): RanksByTAInput { weights[timeframe] = weight; } } + validateTAWeights(weights, timeframes ?? DEFAULT_TIMEFRAMES); return { symbols, ...(timeframes === undefined ? {} : { timeframes }), diff --git a/src/tests/dataQuality.test.ts b/src/tests/dataQuality.test.ts new file mode 100644 index 0000000..f0842b7 --- /dev/null +++ b/src/tests/dataQuality.test.ts @@ -0,0 +1,180 @@ +import { describe, it, mock } from "node:test"; +import assert from "node:assert/strict"; + +import { MetainfoTool } from "../tools/metainfo.js"; +import { SearchTool } from "../tools/search.js"; +import { ScreenTool } from "../tools/screen.js"; +import { TAClient } from "../api/ta.js"; +import type { MetainfoClient } from "../api/metainfo.js"; +import type { SearchClient } from "../api/search.js"; +import type { TradingViewClient } from "../api/client.js"; +import type { Cache } from "../utils/cache.js"; +import type { RateLimiter } from "../utils/rateLimit.js"; + +function makeCache() { + let value: unknown = null; + return { + cache: { + get: mock.fn(() => value), + set: mock.fn((_key: string, next: unknown) => { + value = next; + }), + } as unknown as Cache, + read: () => value, + }; +} + +function makeRateLimiter(): RateLimiter { + return { acquire: mock.fn(async () => {}) } as unknown as RateLimiter; +} + +function assertIsoTimestamp(value: unknown): asserts value is string { + if (typeof value !== "string") { + throw new TypeError("expected ISO timestamp"); + } + assert.match(value, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); +} + +describe("result provenance and completeness metadata", () => { + it("reports missing lookup symbols and cache hits", async () => { + const cache = makeCache(); + const scanStocks = mock.fn(async () => ({ + totalCount: 1, + data: [{ + s: "NASDAQ:AAPL", + d: ["Apple", 180, 1, 1000, 2_000_000_000_000, 200, 120, 190, 130], + }], + })); + const client = { scanStocks } as unknown as TradingViewClient; + const tool = new ScreenTool(client, cache.cache, makeRateLimiter()); + + const fresh = await tool.lookupSymbols({ symbols: ["NASDAQ:AAPL", "NASDAQ:MSFT"] }); + assert.deepEqual(fresh.metadata.missing_symbols, ["NASDAQ:MSFT"]); + assert.equal(fresh.metadata.requested_count, 2); + assert.equal(fresh.metadata.returned_count, 1); + assert.equal(fresh.metadata.cache_hit, false); + assert.equal(fresh.metadata.source, "https://scanner.tradingview.com/global/scan"); + assertIsoTimestamp(fresh.metadata.retrieved_at); + + const cached = await tool.lookupSymbols({ symbols: ["NASDAQ:AAPL", "NASDAQ:MSFT"] }); + assert.equal(cached.metadata.cache_hit, true); + assert.deepEqual(cached.metadata.missing_symbols, ["NASDAQ:MSFT"]); + assert.equal(scanStocks.mock.calls.length, 1); + }); + + it("adds counts and source metadata to screening, search, and metainfo results", async () => { + const screenCache = makeCache(); + const screenClient = { + scanStocks: mock.fn(async () => ({ + totalCount: 1, + data: [{ s: "NASDAQ:AAPL", d: ["Apple", 180, 1, 2, 3, 4, 5] }], + })), + } as unknown as TradingViewClient; + const screen = new ScreenTool(screenClient, screenCache.cache, makeRateLimiter()); + const screened = await screen.screenStocks({ filters: [], limit: 5 }); + assert.equal(screened.metadata.requested_count, 5); + assert.equal(screened.metadata.returned_count, 1); + assert.equal(screened.metadata.source, "https://scanner.tradingview.com/global/scan"); + + const searchCache = makeCache(); + const searchClient = { + searchSymbols: mock.fn(async () => ({ + query: "apple", + count: 2, + symbols: [{ symbol: "NASDAQ:AAPL" }, { symbol: "NASDAQ:APLE" }], + })), + } as unknown as SearchClient; + const search = new SearchTool(searchClient, searchCache.cache, makeRateLimiter()); + const searched = await search.searchSymbols({ query: "apple", limit: 10 }); + assert.equal(searched.metadata.requested_count, 10); + assert.equal(searched.metadata.returned_count, 2); + assert.equal(searched.metadata.source, "https://symbol-search.tradingview.com/symbol_search/v3"); + + const metainfoCache = makeCache(); + const metainfoClient = { + getMetainfo: mock.fn(async () => ({ + market: "america", + requested_fields: ["close"], + metainfo: { + available: true, + field_count: 1, + fields: [{ name: "close" }], + }, + })), + } as unknown as MetainfoClient; + const metainfo = new MetainfoTool(metainfoClient, metainfoCache.cache, makeRateLimiter()); + const info = await metainfo.getMetainfo({ market: "america", fields: ["close"] }); + assert.equal(info.metadata.requested_count, 1); + assert.equal(info.metadata.returned_count, 1); + assert.equal(info.metadata.source, "https://scanner.tradingview.com/america/metainfo"); + }); +}); + +describe("TA missing-data semantics", () => { + function makeTAClient(response: unknown, cache = makeCache()) { + const client = { + scanStocks: mock.fn(async () => response), + } as unknown as TradingViewClient; + return new TAClient(client, cache.cache, makeRateLimiter()); + } + + it("distinguishes unavailable TA values from a genuine neutral score", async () => { + const taClient = makeTAClient({ + totalCount: 2, + data: [ + { s: "NASDAQ:AAPL", d: ["Apple", null, null, null] }, + { s: "NASDAQ:MSFT", d: ["Microsoft", 0, 0, 0] }, + ], + }); + const result = await taClient.getTASummary({ + symbols: ["NASDAQ:AAPL", "NASDAQ:MSFT"], + timeframes: ["60"], + }); + + const unavailable = result.symbols.find((item) => item.symbol === "NASDAQ:AAPL"); + const neutral = result.symbols.find((item) => item.symbol === "NASDAQ:MSFT"); + assert.ok(unavailable); + assert.ok(neutral); + assert.equal(unavailable.timeframes["60"].summary, "unavailable"); + assert.equal(unavailable.timeframes["60"].available, false); + assert.equal(unavailable.timeframes["60"].scores.all, null); + assert.equal(neutral.timeframes["60"].summary, "neutral"); + assert.equal(neutral.timeframes["60"].available, true); + assert.equal(neutral.timeframes["60"].scores.all, 0); + assert.deepEqual(result.metadata.unavailable_symbols, ["NASDAQ:AAPL"]); + }); + + it("excludes unavailable and missing symbols from ranking with reasons", async () => { + const taClient = makeTAClient({ + totalCount: 1, + data: [{ s: "NASDAQ:MSFT", d: ["Microsoft", 0.4, 0.1, 0.5] }], + }); + const result = await taClient.rankByTA({ + symbols: ["NASDAQ:AAPL", "NASDAQ:MSFT"], + timeframes: ["60"], + }); + const cached = await taClient.rankByTA({ + symbols: ["NASDAQ:AAPL", "NASDAQ:MSFT"], + timeframes: ["60"], + }); + assert.equal(cached.metadata.cache_hit, true); + + assert.deepEqual(result.ranked.map((item) => item.symbol), ["NASDAQ:MSFT"]); + assert.deepEqual(result.excluded_symbols, [ + { symbol: "NASDAQ:AAPL", reason: "missing_symbol" }, + ]); + assert.deepEqual(result.metadata.missing_symbols, ["NASDAQ:AAPL"]); + }); + + it("rejects invalid effective TA weights", async () => { + const taClient = makeTAClient({ totalCount: 0, data: [] }); + await assert.rejects( + () => taClient.rankByTA({ symbols: ["NASDAQ:AAPL"], timeframes: ["60"], weights: { "60": 0 } }), + /weights must assign a positive total weight/ + ); + await assert.rejects( + () => taClient.rankByTA({ symbols: ["NASDAQ:AAPL"], timeframes: ["60"], weights: { "1D": 1 } }), + /weights contains timeframe '1D' that is not being ranked/ + ); + }); +}); diff --git a/src/tests/integration/core-tools.test.ts b/src/tests/integration/core-tools.test.ts index ad15b36..99e768e 100644 --- a/src/tests/integration/core-tools.test.ts +++ b/src/tests/integration/core-tools.test.ts @@ -193,7 +193,9 @@ describe("Integration — Core-4: get_ta_summary", () => { const tf60 = aapl.timeframes["60"]; assert.ok(VALID_LABELS.includes(tf60.summary), `Invalid label: ${tf60.summary}`); - assert.equal(typeof tf60.scores.all, "number"); + if (typeof tf60.scores.all !== "number") { + throw new Error("Expected a numeric TA score"); + } assert.ok(tf60.scores.all >= -1 && tf60.scores.all <= 1, `Score out of range: ${tf60.scores.all}`); }); diff --git a/src/tests/lookup.test.ts b/src/tests/lookup.test.ts index f7d2cb1..755da07 100644 --- a/src/tests/lookup.test.ts +++ b/src/tests/lookup.test.ts @@ -189,8 +189,9 @@ describe("ScreenTool - lookupSymbols Integration", () => { const result = await screenTool.lookupSymbols({ symbols }); - // Should return cached result without calling client - assert.deepStrictEqual(result, cachedResult); + // Preserve the cached collection while adding cache provenance. + assert.deepStrictEqual(result.symbols, cachedResult.symbols); + assert.equal(result.metadata.cache_hit, true); // Verify scanStocks was not called const callCount = (mockClient.scanStocks as any).mock.calls.length; diff --git a/src/tests/ta.test.ts b/src/tests/ta.test.ts index 07758eb..a78ff30 100644 --- a/src/tests/ta.test.ts +++ b/src/tests/ta.test.ts @@ -203,10 +203,11 @@ describe("TAClient - getTASummary", () => { symbols: ["NASDAQ:NULL"], timeframes: ["60"], }); - const symbol = result.symbols[0]; - assert.strictEqual(symbol.timeframes["60"].summary, "neutral"); - assert.strictEqual(symbol.timeframes["60"].scores.all, 0); + + assert.strictEqual(symbol.timeframes["60"].summary, "unavailable"); + assert.strictEqual(symbol.timeframes["60"].available, false); + assert.strictEqual(symbol.timeframes["60"].scores.all, null); }); }); diff --git a/src/tools/metainfo.ts b/src/tools/metainfo.ts index 2474917..e63700d 100644 --- a/src/tools/metainfo.ts +++ b/src/tools/metainfo.ts @@ -5,6 +5,11 @@ import type { MetainfoClient, MetainfoInput } from "../api/metainfo.js"; import type { Cache } from "../utils/cache.js"; import type { RateLimiter } from "../utils/rateLimit.js"; +import { + createResultMetadata, + withCacheHitMetadata, + withResultMetadata, +} from "../utils/resultMetadata.js"; export class MetainfoTool { constructor( @@ -30,7 +35,12 @@ export class MetainfoTool { // Check cache const cached = this.cache.get(cacheKey); if (cached) { - return cached; + const cachedFields = cached.metainfo?.fields; + return withCacheHitMetadata(cached, { + source: `https://scanner.tradingview.com/${input.market}/metainfo`, + requested_count: input.fields?.length ?? 0, + returned_count: Array.isArray(cachedFields) ? cachedFields.length : 0, + }); } // Rate limit @@ -39,9 +49,16 @@ export class MetainfoTool { // Make request const result = await this.client.getMetainfo(input); - // Cache result - this.cache.set(cacheKey, result); + const resultWithMetadata = withResultMetadata( + result, + createResultMetadata({ + source: `https://scanner.tradingview.com/${input.market}/metainfo`, + requested_count: input.fields?.length ?? 0, + returned_count: Array.isArray(result.metainfo?.fields) ? result.metainfo.fields.length : 0, + }), + ); + this.cache.set(cacheKey, resultWithMetadata); - return result; + return resultWithMetadata; } } \ No newline at end of file diff --git a/src/tools/screen.ts b/src/tools/screen.ts index df2ff25..fb73fa6 100644 --- a/src/tools/screen.ts +++ b/src/tools/screen.ts @@ -11,7 +11,24 @@ import { validateScreenFilters } from "../api/validation.js"; import type { TradingViewClient } from "../api/client.js"; import type { Cache } from "../utils/cache.js"; import type { RateLimiter } from "../utils/rateLimit.js"; +import { + createResultMetadata, + withCacheHitMetadata, + withResultMetadata, +} from "../utils/resultMetadata.js"; + export { validateScreenFilters } from "../api/validation.js"; +const STOCK_SOURCE = "https://scanner.tradingview.com/global/scan"; +const FOREX_SOURCE = "https://scanner.tradingview.com/forex/scan"; +const CRYPTO_SOURCE = "https://scanner.tradingview.com/crypto/scan"; + +function cachedCollectionLength(cached: unknown, key: string): number { + if (typeof cached !== "object" || cached === null || Array.isArray(cached)) { + return 0; + } + const collection = (cached as Record)[key]; + return Array.isArray(collection) ? collection.length : 0; +} // Minimal default columns for lean responses const DEFAULT_COLUMNS = [ @@ -103,7 +120,11 @@ export class ScreenTool { // Check cache const cached = this.cache.get(cacheKey); if (cached) { - return cached; + return withCacheHitMetadata(cached, { + source: STOCK_SOURCE, + requested_count: limit, + returned_count: cachedCollectionLength(cached, "stocks"), + }); } // Convert filters to TradingView format @@ -150,11 +171,19 @@ export class ScreenTool { return stock; }), }; + const resultWithMetadata = withResultMetadata( + result, + createResultMetadata({ + source: STOCK_SOURCE, + requested_count: limit, + returned_count: result.stocks.length, + }), + ); // Cache result - this.cache.set(cacheKey, result); + this.cache.set(cacheKey, resultWithMetadata); - return result; + return resultWithMetadata; } async screenForex(input: Omit): Promise { @@ -173,7 +202,13 @@ export class ScreenTool { const cacheKey = JSON.stringify({ type: "forex", filters, sort_by, sort_order, limit, columns: inputColumns }); const cached = this.cache.get(cacheKey); - if (cached) return cached; + if (cached) { + return withCacheHitMetadata(cached, { + source: FOREX_SOURCE, + requested_count: limit, + returned_count: cachedCollectionLength(cached, "pairs"), + }); + } const tvFilters = this.validateAndConvertFilters(filters); @@ -201,9 +236,17 @@ export class ScreenTool { return pair; }), }; + const resultWithMetadata = withResultMetadata( + result, + createResultMetadata({ + source: FOREX_SOURCE, + requested_count: limit, + returned_count: result.pairs.length, + }), + ); - this.cache.set(cacheKey, result); - return result; + this.cache.set(cacheKey, resultWithMetadata); + return resultWithMetadata; } async screenCrypto(input: Omit): Promise { @@ -222,7 +265,13 @@ export class ScreenTool { const cacheKey = JSON.stringify({ type: "crypto", filters, sort_by, sort_order, limit, columns: inputColumns }); const cached = this.cache.get(cacheKey); - if (cached) return cached; + if (cached) { + return withCacheHitMetadata(cached, { + source: CRYPTO_SOURCE, + requested_count: limit, + returned_count: cachedCollectionLength(cached, "cryptocurrencies"), + }); + } const tvFilters = this.validateAndConvertFilters(filters); @@ -250,9 +299,17 @@ export class ScreenTool { return crypto; }), }; + const resultWithMetadata = withResultMetadata( + result, + createResultMetadata({ + source: CRYPTO_SOURCE, + requested_count: limit, + returned_count: result.cryptocurrencies.length, + }), + ); - this.cache.set(cacheKey, result); - return result; + this.cache.set(cacheKey, resultWithMetadata); + return resultWithMetadata; } async screenETF(input: ScreenStocksInput): Promise { @@ -277,7 +334,11 @@ export class ScreenTool { // Check cache const cached = this.cache.get(cacheKey); if (cached) { - return cached; + return withCacheHitMetadata(cached, { + source: STOCK_SOURCE, + requested_count: limit, + returned_count: cachedCollectionLength(cached, "etfs"), + }); } // Convert filters to TradingView format @@ -327,11 +388,19 @@ export class ScreenTool { return etf; }), }; + const resultWithMetadata = withResultMetadata( + result, + createResultMetadata({ + source: STOCK_SOURCE, + requested_count: limit, + returned_count: result.etfs.length, + }), + ); // Cache result - this.cache.set(cacheKey, result); + this.cache.set(cacheKey, resultWithMetadata); - return result; + return resultWithMetadata; } async lookupSymbols(input: { symbols: string[]; columns?: string[] }): Promise { @@ -355,7 +424,17 @@ export class ScreenTool { // Check cache const cached = this.cache.get(cacheKey); if (cached) { - return cached; + const cachedSymbols = Array.isArray(cached.symbols) + ? cached.symbols + .map((item: { symbol?: unknown }) => item.symbol) + .filter((symbol: unknown): symbol is string => typeof symbol === "string") + : []; + return withCacheHitMetadata(cached, { + source: STOCK_SOURCE, + requested_count: symbols.length, + returned_count: cachedSymbols.length, + missing_symbols: symbols.filter((symbol) => !cachedSymbols.includes(symbol)), + }); } // Default columns for symbol lookup @@ -408,10 +487,20 @@ export class ScreenTool { return symbol; }), }; + const returnedSymbols = result.symbols.map((item) => item.symbol); + const resultWithMetadata = withResultMetadata( + result, + createResultMetadata({ + source: STOCK_SOURCE, + requested_count: symbols.length, + returned_count: result.symbols.length, + missing_symbols: symbols.filter((symbol) => !returnedSymbols.includes(symbol)), + }), + ); // Cache result - this.cache.set(cacheKey, result); + this.cache.set(cacheKey, resultWithMetadata); - return result; + return resultWithMetadata; } } diff --git a/src/tools/search.ts b/src/tools/search.ts index fd25f55..b0ee01c 100644 --- a/src/tools/search.ts +++ b/src/tools/search.ts @@ -6,6 +6,11 @@ import { createRequire } from "module"; import type { SearchClient, SearchSymbolsInput } from "../api/search.js"; import type { Cache } from "../utils/cache.js"; import type { RateLimiter } from "../utils/rateLimit.js"; +import { + createResultMetadata, + withCacheHitMetadata, + withResultMetadata, +} from "../utils/resultMetadata.js"; const require = createRequire(import.meta.url); const pkg = require("../../package.json"); @@ -40,7 +45,11 @@ export class SearchTool { // Check cache const cached = this.cache.get(cacheKey); if (cached) { - return cached; + return withCacheHitMetadata(cached, { + source: "https://symbol-search.tradingview.com/symbol_search/v3", + requested_count: input.limit ?? 20, + returned_count: Array.isArray(cached.symbols) ? cached.symbols.length : 0, + }); } // Rate limit @@ -50,8 +59,16 @@ export class SearchTool { const result = await this.client.searchSymbols(input); // Cache result - this.cache.set(cacheKey, result); + const resultWithMetadata = withResultMetadata( + result, + createResultMetadata({ + source: "https://symbol-search.tradingview.com/symbol_search/v3", + requested_count: input.limit ?? 20, + returned_count: Array.isArray(result.symbols) ? result.symbols.length : 0, + }), + ); + this.cache.set(cacheKey, resultWithMetadata); - return result; + return resultWithMetadata; } } \ No newline at end of file diff --git a/src/utils/resultMetadata.ts b/src/utils/resultMetadata.ts new file mode 100644 index 0000000..615a56d --- /dev/null +++ b/src/utils/resultMetadata.ts @@ -0,0 +1,96 @@ +export interface ResultMetadata { + retrieved_at: string; + source: string; + cache_hit: boolean; + requested_count: number; + returned_count: number; + missing_symbols: string[]; + unavailable_symbols?: string[]; +} + +export interface ResultMetadataInput { + source: string; + requested_count: number; + returned_count: number; + missing_symbols?: string[]; + unavailable_symbols?: string[]; + cache_hit?: boolean; +} + +function now(): string { + return new Date().toISOString(); +} + +export function createResultMetadata(input: ResultMetadataInput): ResultMetadata { + return { + retrieved_at: now(), + source: input.source, + cache_hit: input.cache_hit ?? false, + requested_count: input.requested_count, + returned_count: input.returned_count, + missing_symbols: input.missing_symbols ?? [], + ...(input.unavailable_symbols ? { unavailable_symbols: input.unavailable_symbols } : {}), + }; +} + +export function withResultMetadata( + result: T, + metadata: ResultMetadata, +): T & { metadata: ResultMetadata } { + return { ...result, metadata }; +} + +function readStringArray(value: unknown): string[] | undefined { + if (!Array.isArray(value) || !value.every((item) => typeof item === "string")) { + return undefined; + } + return value; +} + +function readCachedMetadata(value: unknown): Partial { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return {}; + } + + const metadata: Partial = {}; + if ("source" in value && typeof value.source === "string") { + metadata.source = value.source; + } + if ("requested_count" in value && typeof value.requested_count === "number") { + metadata.requested_count = value.requested_count; + } + if ("returned_count" in value && typeof value.returned_count === "number") { + metadata.returned_count = value.returned_count; + } + if ("missing_symbols" in value) { + const missingSymbols = readStringArray(value.missing_symbols); + if (missingSymbols) metadata.missing_symbols = missingSymbols; + } + if ("unavailable_symbols" in value) { + const unavailableSymbols = readStringArray(value.unavailable_symbols); + if (unavailableSymbols) metadata.unavailable_symbols = unavailableSymbols; + } + return metadata; +} + +export function withCacheHitMetadata( + cached: T, + fallback: ResultMetadataInput, +): T & { metadata: ResultMetadata } { + const existing = "metadata" in cached ? readCachedMetadata(cached.metadata) : {}; + const metadata: ResultMetadata = { + retrieved_at: now(), + source: existing.source ?? fallback.source, + cache_hit: true, + requested_count: existing.requested_count ?? fallback.requested_count, + returned_count: existing.returned_count ?? fallback.returned_count, + missing_symbols: existing.missing_symbols ?? fallback.missing_symbols ?? [], + ...(existing.unavailable_symbols + ? { unavailable_symbols: existing.unavailable_symbols } + : fallback.unavailable_symbols + ? { unavailable_symbols: fallback.unavailable_symbols } + : {}), + }; + + return { ...cached, metadata }; +} From ac0dfb7581f78d801ed70fe5f9b2ba0d69ed2a3c Mon Sep 17 00:00:00 2001 From: Pavel Fadeev Date: Sun, 16 Aug 2026 02:16:49 +0200 Subject: [PATCH 2/2] fix(data): preserve source timestamps and raw counts --- src/api/ta.ts | 1 + src/tests/dataQuality.test.ts | 41 +++++++++++++++++++++++++++++++++++ src/tools/metainfo.ts | 38 +++++++++++++++++++++++++++----- src/utils/resultMetadata.ts | 8 +++++-- 4 files changed, 80 insertions(+), 8 deletions(-) diff --git a/src/api/ta.ts b/src/api/ta.ts index a7cf123..970270b 100644 --- a/src/api/ta.ts +++ b/src/api/ta.ts @@ -380,6 +380,7 @@ export class TAClient { metadata: createResultMetadata({ source: "https://scanner.tradingview.com/global/scan", cache_hit: summary.metadata.cache_hit, + retrieved_at: summary.metadata.retrieved_at, requested_count: symbols.length, returned_count: ranked.length, missing_symbols: missingSymbols, diff --git a/src/tests/dataQuality.test.ts b/src/tests/dataQuality.test.ts index f0842b7..7623e67 100644 --- a/src/tests/dataQuality.test.ts +++ b/src/tests/dataQuality.test.ts @@ -55,10 +55,12 @@ describe("result provenance and completeness metadata", () => { assert.equal(fresh.metadata.cache_hit, false); assert.equal(fresh.metadata.source, "https://scanner.tradingview.com/global/scan"); assertIsoTimestamp(fresh.metadata.retrieved_at); + const freshRetrievedAt = fresh.metadata.retrieved_at; const cached = await tool.lookupSymbols({ symbols: ["NASDAQ:AAPL", "NASDAQ:MSFT"] }); assert.equal(cached.metadata.cache_hit, true); assert.deepEqual(cached.metadata.missing_symbols, ["NASDAQ:MSFT"]); + assert.equal(cached.metadata.retrieved_at, freshRetrievedAt); assert.equal(scanStocks.mock.calls.length, 1); }); @@ -107,6 +109,44 @@ describe("result provenance and completeness metadata", () => { assert.equal(info.metadata.requested_count, 1); assert.equal(info.metadata.returned_count, 1); assert.equal(info.metadata.source, "https://scanner.tradingview.com/america/metainfo"); + const rawMetainfo = new MetainfoTool( + { + getMetainfo: mock.fn(async () => ({ + market: "america", + raw: { fields: [{ name: "close" }, { name: "name" }] }, + })), + } as unknown as MetainfoClient, + makeCache().cache, + makeRateLimiter(), + ); + const rawInfo = await rawMetainfo.getMetainfo({ market: " america ", mode: "raw" }); + assert.equal(rawInfo.metadata.returned_count, 2); + assert.equal(rawInfo.metadata.source, "https://scanner.tradingview.com/america/metainfo"); + const rootArrayMetainfo = new MetainfoTool( + { + getMetainfo: mock.fn(async () => ({ + market: "america", + raw: [{ name: "close" }, { name: "name" }], + })), + } as unknown as MetainfoClient, + makeCache().cache, + makeRateLimiter(), + ); + const rootArrayInfo = await rootArrayMetainfo.getMetainfo({ market: "america", mode: "raw" }); + assert.equal(rootArrayInfo.metadata.returned_count, 2); + + const fieldMapMetainfo = new MetainfoTool( + { + getMetainfo: mock.fn(async () => ({ + market: "america", + raw: { close: { type: "number" }, name: { type: "string" } }, + })), + } as unknown as MetainfoClient, + makeCache().cache, + makeRateLimiter(), + ); + const fieldMapInfo = await fieldMapMetainfo.getMetainfo({ market: "america", mode: "raw" }); + assert.equal(fieldMapInfo.metadata.returned_count, 2); }); }); @@ -163,6 +203,7 @@ describe("TA missing-data semantics", () => { assert.deepEqual(result.excluded_symbols, [ { symbol: "NASDAQ:AAPL", reason: "missing_symbol" }, ]); + assert.equal(cached.metadata.retrieved_at, result.metadata.retrieved_at); assert.deepEqual(result.metadata.missing_symbols, ["NASDAQ:AAPL"]); }); diff --git a/src/tools/metainfo.ts b/src/tools/metainfo.ts index e63700d..c0c0ab0 100644 --- a/src/tools/metainfo.ts +++ b/src/tools/metainfo.ts @@ -11,6 +11,34 @@ import { withResultMetadata, } from "../utils/resultMetadata.js"; +function countFieldCollection(value: unknown): number { + if (Array.isArray(value)) return value.length; + if (typeof value === "object" && value !== null) return Object.keys(value).length; + return 0; +} + +function countReturnedFields(value: unknown): number { + if (typeof value !== "object" || value === null || Array.isArray(value)) return 0; + if ("metainfo" in value && typeof value.metainfo === "object" && value.metainfo !== null) { + const metainfo = value.metainfo; + if ("fields" in metainfo) return countFieldCollection(metainfo.fields); + if ("columns" in metainfo) return countFieldCollection(metainfo.columns); + } + if ("raw" in value) { + const raw = value.raw; + if (typeof raw === "object" && raw !== null && !Array.isArray(raw)) { + if ("fields" in raw) return countFieldCollection(raw.fields); + if ("columns" in raw) return countFieldCollection(raw.columns); + } + return countFieldCollection(raw); + } + return 0; +} + +function metainfoSource(market: string): string { + return `https://scanner.tradingview.com/${encodeURIComponent(market.trim())}/metainfo`; +} + export class MetainfoTool { constructor( private client: MetainfoClient, @@ -35,11 +63,10 @@ export class MetainfoTool { // Check cache const cached = this.cache.get(cacheKey); if (cached) { - const cachedFields = cached.metainfo?.fields; return withCacheHitMetadata(cached, { - source: `https://scanner.tradingview.com/${input.market}/metainfo`, + source: metainfoSource(input.market), requested_count: input.fields?.length ?? 0, - returned_count: Array.isArray(cachedFields) ? cachedFields.length : 0, + returned_count: countReturnedFields(cached), }); } @@ -48,13 +75,12 @@ export class MetainfoTool { // Make request const result = await this.client.getMetainfo(input); - const resultWithMetadata = withResultMetadata( result, createResultMetadata({ - source: `https://scanner.tradingview.com/${input.market}/metainfo`, + source: metainfoSource(input.market), requested_count: input.fields?.length ?? 0, - returned_count: Array.isArray(result.metainfo?.fields) ? result.metainfo.fields.length : 0, + returned_count: countReturnedFields(result), }), ); this.cache.set(cacheKey, resultWithMetadata); diff --git a/src/utils/resultMetadata.ts b/src/utils/resultMetadata.ts index 615a56d..0168512 100644 --- a/src/utils/resultMetadata.ts +++ b/src/utils/resultMetadata.ts @@ -15,6 +15,7 @@ export interface ResultMetadataInput { missing_symbols?: string[]; unavailable_symbols?: string[]; cache_hit?: boolean; + retrieved_at?: string; } function now(): string { @@ -23,7 +24,7 @@ function now(): string { export function createResultMetadata(input: ResultMetadataInput): ResultMetadata { return { - retrieved_at: now(), + retrieved_at: input.retrieved_at ?? now(), source: input.source, cache_hit: input.cache_hit ?? false, requested_count: input.requested_count, @@ -53,6 +54,9 @@ function readCachedMetadata(value: unknown): Partial { } const metadata: Partial = {}; + if ("retrieved_at" in value && typeof value.retrieved_at === "string") { + metadata.retrieved_at = value.retrieved_at; + } if ("source" in value && typeof value.source === "string") { metadata.source = value.source; } @@ -79,7 +83,7 @@ export function withCacheHitMetadata( ): T & { metadata: ResultMetadata } { const existing = "metadata" in cached ? readCachedMetadata(cached.metadata) : {}; const metadata: ResultMetadata = { - retrieved_at: now(), + retrieved_at: existing.retrieved_at ?? fallback.retrieved_at ?? now(), source: existing.source ?? fallback.source, cache_hit: true, requested_count: existing.requested_count ?? fallback.requested_count,