From be8a708c91fd6f0e92d1ad5e6eb769afa13a4f75 Mon Sep 17 00:00:00 2001 From: Pavel Fadeev Date: Fri, 21 Aug 2026 00:30:34 +0200 Subject: [PATCH 1/2] fix(screen): validate ETF subtype taxonomy --- src/tests/screen.test.ts | 40 +++++++++++++++++++++++++ src/tools/screen.ts | 63 ++++++++++++++++++---------------------- 2 files changed, 68 insertions(+), 35 deletions(-) diff --git a/src/tests/screen.test.ts b/src/tests/screen.test.ts index 96a6179..dee9e13 100644 --- a/src/tests/screen.test.ts +++ b/src/tests/screen.test.ts @@ -536,6 +536,46 @@ describe("ScreenTool - Filter Validation", () => { }); }); }); + describe("ETF taxonomy", () => { + it("filters fund rows to verified ETF subtypes and preserves missing expense ratio as null", async () => { + (mockClient.scanStocks as any).mock.mockImplementation(async (request: any) => { + const rows = [ + ["AMEX:SPY", "SPDR S&P 500 ETF", "etf", null], + ["NYSE:ABC-P", "Preferred Share", "preferred", 0.4], + ["NASDAQ:CORP", "Corporate Instrument", "corporate", null], + ["NYSE:CEF", "Closed End Fund", "closed_end", 1.2], + ["NASDAQ:MUTF", "Mutual Fund", "mutual_fund", 0.8], + ["NYSE:TRUST", "Trust", "trust", null], + ]; + return { + totalCount: rows.length, + data: rows.map(([symbol, name, subtype, expense]) => ({ + s: symbol, + d: request.columns.map((column: string) => ({ name, close: 10, subtype, type: "fund", expense_ratio: expense }[column] ?? null)), + })), + }; + }); + + const result = await screenTool.screenETF({ filters: [], limit: 20 }); + assert.deepEqual(result.etfs.map((item: any) => item.symbol), ["AMEX:SPY"]); + assert.equal(result.etfs[0].type, "fund"); + assert.equal(result.etfs[0].subtype, "etf"); + assert.equal(result.etfs[0].expense_ratio, null); + assert.equal(result.etfs[0].etfClassification, "verified"); + }); + + it("adds an ETF subtype discriminator to the upstream request", async () => { + (mockClient.scanStocks as any).mock.mockImplementation(async (request: any) => { + assert.deepEqual(request.filter.slice(-2), [ + { left: "type", operation: "equal", right: "fund" }, + { left: "subtype", operation: "equal", right: "etf" }, + ]); + return { totalCount: 0, data: [] }; + }); + + await screenTool.screenETF({ filters: [] }); + }); + }); describe("Valid filter conversion", () => { it("should convert valid filters to TradingView format", async () => { diff --git a/src/tools/screen.ts b/src/tools/screen.ts index fb73fa6..990036e 100644 --- a/src/tools/screen.ts +++ b/src/tools/screen.ts @@ -313,7 +313,9 @@ export class ScreenTool { } async screenETF(input: ScreenStocksInput): Promise { - // ETFs/Funds screening - similar to stocks but with type filter + // TradingView's fund type includes preferred shares, corporate instruments, + // trusts, and other fund-like rows. The subtype discriminator is the + // authoritative ETF taxonomy boundary. const { filters = [], markets = ["america"], @@ -323,15 +325,11 @@ export class ScreenTool { columns: inputColumns, } = input; - // Validate limit if (limit < 1 || limit > 200) { throw new Error("Limit must be between 1 and 200"); } - // Build cache key const cacheKey = JSON.stringify({ type: "etf", filters, markets, sort_by, sort_order, limit, columns: inputColumns }); - - // Check cache const cached = this.cache.get(cacheKey); if (cached) { return withCacheHitMetadata(cached, { @@ -341,65 +339,60 @@ export class ScreenTool { }); } - // Convert filters to TradingView format const tvFilters = this.validateAndConvertFilters(filters); - - // Extract unique fields from filters for columns const filterFields = filters.map((f) => f.field); - const baseColumns = inputColumns || ["name", "close", "volume", "change", "change_from_open"]; + const baseColumns = inputColumns || [ + "name", + "close", + "volume", + "change", + "change_from_open", + "type", + "subtype", + "expense_ratio", + ]; const columns = [...new Set([...baseColumns, ...filterFields])]; - // Build request with fund type filter const request: ScreenerRequest = { filter: [ ...tvFilters, - { left: "type", operation: "equal", right: "fund" }, // Filter for ETFs/Funds + { left: "type", operation: "equal", right: "fund" }, + { left: "subtype", operation: "equal", right: "etf" }, ], columns, - sort: { - sortBy: sort_by, - sortOrder: sort_order, - }, + sort: { sortBy: sort_by, sortOrder: sort_order }, range: [0, limit], options: { lang: "en" }, - symbols: { - query: { types: [] }, - tickers: [], - }, + symbols: { query: { types: [] }, tickers: [] }, markets, }; - // Rate limit await this.rateLimiter.acquire(); - - // Make request const response = await this.client.scanStocks(request); - - // Format response - const result = { - total_count: response.totalCount, - etfs: response.data.map((item) => { + const etfs = response.data + .map((item) => { const etf: Record = { symbol: item.s }; - columns.forEach((col, idx) => { - etf[col] = item.d[idx]; + // A null expense_ratio is intentional: TradingView has no value for + // that instrument, rather than the field being omitted or guessed. + etf[col] = item.d[idx] ?? null; }); - return etf; - }), - }; + }) + .filter((etf) => String(etf.subtype ?? "").toLowerCase() === "etf") + .map((etf) => ({ ...etf, etfClassification: "verified" as const })); + + const result = { total_count: response.totalCount, etfs }; const resultWithMetadata = withResultMetadata( result, createResultMetadata({ source: STOCK_SOURCE, requested_count: limit, - returned_count: result.etfs.length, + returned_count: etfs.length, }), ); - // Cache result this.cache.set(cacheKey, resultWithMetadata); - return resultWithMetadata; } From 01baeeaec4e738cb9c7e0b03d7d17c8950af7a8e Mon Sep 17 00:00:00 2001 From: Pavel Fadeev Date: Fri, 21 Aug 2026 00:30:36 +0200 Subject: [PATCH 2/2] test(screen): add gated ETF taxonomy integration fixture --- docs/API_REFERENCE.md | 7 ++-- src/tests/integration/etf-taxonomy.test.ts | 40 ++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 src/tests/integration/etf-taxonomy.test.ts diff --git a/docs/API_REFERENCE.md b/docs/API_REFERENCE.md index 69a3dce..c114c5e 100644 --- a/docs/API_REFERENCE.md +++ b/docs/API_REFERENCE.md @@ -141,10 +141,13 @@ Screen ETFs (Exchange-Traded Funds) based on performance and technical criteria. | `sort_by` | `string` | No | `"market_cap_basic"` | Field to sort results by (market_cap_basic approximates AUM for ETFs) | | `sort_order` | `"asc" \| "desc"` | No | `"desc"` | Sort direction | | `limit` | `number` | No | `20` | Number of results (1–200) | -| `columns` | `string[]` | No | `["name","close","volume","change","change_from_open"]` | Columns to include | +| `columns` | `string[]` | No | `["name","close","volume","change","change_from_open","type","subtype","expense_ratio"]` | Columns to include; `expense_ratio: null` explicitly means TradingView did not publish a value | -Note: Internally applies a `type = fund` filter in addition to user-supplied filters. +The server applies both `type = fund` and `subtype = etf` filters. Returned rows include `type`, `subtype`, and `etfClassification: "verified"`; preferred shares, corporate instruments, closed-end funds, mutual funds, and trusts are not presented as ETFs. Retrieval provenance remains in `metadata` (`retrieved_at`, `source`, and `cache_hit`). +When an upstream response lacks a usable subtype, do not infer ETF status from naming or `expense_ratio`. Use `lookup_symbols` for the ticker and corroborate the issuer's instrument classification before treating it as an ETF. + +Note: Internally applies `type = fund` and `subtype = etf` filters in addition to user-supplied filters. **Example** ```json diff --git a/src/tests/integration/etf-taxonomy.test.ts b/src/tests/integration/etf-taxonomy.test.ts new file mode 100644 index 0000000..a7d32da --- /dev/null +++ b/src/tests/integration/etf-taxonomy.test.ts @@ -0,0 +1,40 @@ +/** + * ETF taxonomy integration regression. Run only with TV_INTEGRATION=1. + */ +import { describe, it } from "node:test"; +import assert from "node:assert/strict"; + +if (process.env.TV_INTEGRATION !== "1") { + console.log("Skipping ETF taxonomy integration test (set TV_INTEGRATION=1 to enable)"); + process.exit(0); +} + +import { TradingViewClient } from "../../api/client.js"; +import { Cache } from "../../utils/cache.js"; +import { RateLimiter } from "../../utils/rateLimit.js"; +import { ScreenTool } from "../../tools/screen.js"; + +describe("Integration — screen_etf taxonomy", () => { + it("does not return known non-ETF fund contamination", { timeout: 30_000 }, async () => { + const tool = new ScreenTool( + new TradingViewClient(), + new Cache(60), + new RateLimiter(10), + ); + const result = await tool.screenETF({ + filters: [], + markets: ["america"], + sort_by: "market_cap_basic", + sort_order: "desc", + limit: 50, + }); + + assert.ok(Array.isArray(result.etfs)); + for (const item of result.etfs) { + assert.equal(String(item.type).toLowerCase(), "fund"); + assert.equal(String(item.subtype).toLowerCase(), "etf"); + assert.equal(item.etfClassification, "verified"); + assert.notEqual(String(item.symbol), "NYSE:ABC-P"); + } + }); +});