fix(screen): validate ETF subtype taxonomy

docs/API_REFERENCE.md

@@ -141,10 +141,13 @@ Screen ETFs (Exchange-Traded Funds) based on performance and technical criteria.
 | `sort_by` | `string` | No | `"market_cap_basic"` | Field to sort results by (market_cap_basic approximates AUM for ETFs) |
 | `sort_order` | `"asc" \| "desc"` | No | `"desc"` | Sort direction |
 | `limit` | `number` | No | `20` | Number of results (1–200) |
-| `columns` | `string[]` | No | `["name","close","volume","change","change_from_open"]` | Columns to include |
+| `columns` | `string[]` | No | `["name","close","volume","change","change_from_open","type","subtype","expense_ratio"]` | Columns to include; `expense_ratio: null` explicitly means TradingView did not publish a value |
 
-Note: Internally applies a `type = fund` filter in addition to user-supplied filters.
+The server applies both `type = fund` and `subtype = etf` filters. Returned rows include `type`, `subtype`, and `etfClassification: "verified"`; preferred shares, corporate instruments, closed-end funds, mutual funds, and trusts are not presented as ETFs. Retrieval provenance remains in `metadata` (`retrieved_at`, `source`, and `cache_hit`).
 
+When an upstream response lacks a usable subtype, do not infer ETF status from naming or `expense_ratio`. Use `lookup_symbols` for the ticker and corroborate the issuer's instrument classification before treating it as an ETF.
+
+Note: Internally applies `type = fund` and `subtype = etf` filters in addition to user-supplied filters.
 **Example**
 
 ```json

src/tests/integration/etf-taxonomy.test.ts

@@ -0,0 +1,40 @@
+/**
+ * ETF taxonomy integration regression. Run only with TV_INTEGRATION=1.
+ */
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+
+if (process.env.TV_INTEGRATION !== "1") {
+  console.log("Skipping ETF taxonomy integration test (set TV_INTEGRATION=1 to enable)");
+  process.exit(0);
+}
+
+import { TradingViewClient } from "../../api/client.js";
+import { Cache } from "../../utils/cache.js";
+import { RateLimiter } from "../../utils/rateLimit.js";
+import { ScreenTool } from "../../tools/screen.js";
+
+describe("Integration — screen_etf taxonomy", () => {
+  it("does not return known non-ETF fund contamination", { timeout: 30_000 }, async () => {
+    const tool = new ScreenTool(
+      new TradingViewClient(),
+      new Cache(60),
+      new RateLimiter(10),
+    );
+    const result = await tool.screenETF({
+      filters: [],
+      markets: ["america"],
+      sort_by: "market_cap_basic",
+      sort_order: "desc",
+      limit: 50,
+    });
+
+    assert.ok(Array.isArray(result.etfs));
+    for (const item of result.etfs) {
+      assert.equal(String(item.type).toLowerCase(), "fund");
+      assert.equal(String(item.subtype).toLowerCase(), "etf");
+      assert.equal(item.etfClassification, "verified");
+      assert.notEqual(String(item.symbol), "NYSE:ABC-P");
+    }
+  });
+});

src/tests/screen.test.ts

@@ -536,6 +536,46 @@ describe("ScreenTool - Filter Validation", () => {
       });
     });
   });
+  describe("ETF taxonomy", () => {
+    it("filters fund rows to verified ETF subtypes and preserves missing expense ratio as null", async () => {
+      (mockClient.scanStocks as any).mock.mockImplementation(async (request: any) => {
+        const rows = [
+          ["AMEX:SPY", "SPDR S&P 500 ETF", "etf", null],
+          ["NYSE:ABC-P", "Preferred Share", "preferred", 0.4],
+          ["NASDAQ:CORP", "Corporate Instrument", "corporate", null],
+          ["NYSE:CEF", "Closed End Fund", "closed_end", 1.2],
+          ["NASDAQ:MUTF", "Mutual Fund", "mutual_fund", 0.8],
+          ["NYSE:TRUST", "Trust", "trust", null],
+        ];
+        return {
+          totalCount: rows.length,
+          data: rows.map(([symbol, name, subtype, expense]) => ({
+            s: symbol,
+            d: request.columns.map((column: string) => ({ name, close: 10, subtype, type: "fund", expense_ratio: expense }[column] ?? null)),
+          })),
+        };
+      });
+
+      const result = await screenTool.screenETF({ filters: [], limit: 20 });
+      assert.deepEqual(result.etfs.map((item: any) => item.symbol), ["AMEX:SPY"]);
+      assert.equal(result.etfs[0].type, "fund");
+      assert.equal(result.etfs[0].subtype, "etf");
+      assert.equal(result.etfs[0].expense_ratio, null);
+      assert.equal(result.etfs[0].etfClassification, "verified");
+    });
+
+    it("adds an ETF subtype discriminator to the upstream request", async () => {
+      (mockClient.scanStocks as any).mock.mockImplementation(async (request: any) => {
+        assert.deepEqual(request.filter.slice(-2), [
+          { left: "type", operation: "equal", right: "fund" },
+          { left: "subtype", operation: "equal", right: "etf" },
+        ]);
+        return { totalCount: 0, data: [] };
+      });
+
+      await screenTool.screenETF({ filters: [] });
+    });
+  });
 
   describe("Valid filter conversion", () => {
     it("should convert valid filters to TradingView format", async () => {

src/tools/screen.ts

@@ -313,7 +313,9 @@ export class ScreenTool {
   }
 
   async screenETF(input: ScreenStocksInput): Promise<any> {
-    // ETFs/Funds screening - similar to stocks but with type filter
+    // TradingView's fund type includes preferred shares, corporate instruments,
+    // trusts, and other fund-like rows. The subtype discriminator is the
+    // authoritative ETF taxonomy boundary.
     const {
       filters = [],
       markets = ["america"],
@@ -323,15 +325,11 @@ export class ScreenTool {
       columns: inputColumns,
     } = input;
 
-    // Validate limit
     if (limit < 1 || limit > 200) {
       throw new Error("Limit must be between 1 and 200");
     }
 
-    // Build cache key
     const cacheKey = JSON.stringify({ type: "etf", filters, markets, sort_by, sort_order, limit, columns: inputColumns });
-
-    // Check cache
     const cached = this.cache.get(cacheKey);
     if (cached) {
       return withCacheHitMetadata(cached, {
@@ -341,65 +339,60 @@ export class ScreenTool {
       });
     }
 
-    // Convert filters to TradingView format
     const tvFilters = this.validateAndConvertFilters(filters);
-
-    // Extract unique fields from filters for columns
     const filterFields = filters.map((f) => f.field);
-    const baseColumns = inputColumns || ["name", "close", "volume", "change", "change_from_open"];
+    const baseColumns = inputColumns || [
+      "name",
+      "close",
+      "volume",
+      "change",
+      "change_from_open",
+      "type",
+      "subtype",
+      "expense_ratio",
+    ];
     const columns = [...new Set([...baseColumns, ...filterFields])];
 
-    // Build request with fund type filter
     const request: ScreenerRequest = {
       filter: [
         ...tvFilters,
-        { left: "type", operation: "equal", right: "fund" }, // Filter for ETFs/Funds
+        { left: "type", operation: "equal", right: "fund" },
+        { left: "subtype", operation: "equal", right: "etf" },
       ],
       columns,
-      sort: {
-        sortBy: sort_by,
-        sortOrder: sort_order,
-      },
+      sort: { sortBy: sort_by, sortOrder: sort_order },
       range: [0, limit],
       options: { lang: "en" },
-      symbols: {
-        query: { types: [] },
-        tickers: [],
-      },
+      symbols: { query: { types: [] }, tickers: [] },
       markets,
     };
 
-    // Rate limit
     await this.rateLimiter.acquire();
-
-    // Make request
     const response = await this.client.scanStocks(request);
-
-    // Format response
-    const result = {
-      total_count: response.totalCount,
-      etfs: response.data.map((item) => {
+    const etfs = response.data
+      .map((item) => {
         const etf: Record<string, any> = { symbol: item.s };
-
         columns.forEach((col, idx) => {
-          etf[col] = item.d[idx];
+          // A null expense_ratio is intentional: TradingView has no value for
+          // that instrument, rather than the field being omitted or guessed.
+          etf[col] = item.d[idx] ?? null;
         });
-
         return etf;
-      }),
-    };
+      })
+      .filter((etf) => String(etf.subtype ?? "").toLowerCase() === "etf")
+      .map((etf) => ({ ...etf, etfClassification: "verified" as const }));
+
+    const result = { total_count: response.totalCount, etfs };
     const resultWithMetadata = withResultMetadata(
       result,
       createResultMetadata({
         source: STOCK_SOURCE,
         requested_count: limit,
-        returned_count: result.etfs.length,
+        returned_count: etfs.length,
       }),
     );
 
-    // Cache result
     this.cache.set(cacheKey, resultWithMetadata);
-
     return resultWithMetadata;
   }