From 882a1051a7a7b525cab7135d166b4e657b3ebe56 Mon Sep 17 00:00:00 2001 From: paragmore Date: Thu, 19 Feb 2026 15:57:06 +0530 Subject: [PATCH 1/4] Add new miners list endpoint --- config/common.json.example | 3 +- tests/unit/handlers/miners.handlers.test.js | 312 ++++++++++++++++++ tests/unit/lib/queryUtils.test.js | 277 ++++++++++++++++ tests/unit/routes/miners.routes.test.js | 65 ++++ workers/lib/constants.js | 4 +- .../lib/server/handlers/miners.handlers.js | 226 +++++++++++++ workers/lib/server/index.js | 4 +- workers/lib/server/lib/queryUtils.js | 183 ++++++++++ workers/lib/server/routes/miners.routes.js | 46 +++ 9 files changed, 1117 insertions(+), 3 deletions(-) create mode 100644 tests/unit/handlers/miners.handlers.test.js create mode 100644 tests/unit/lib/queryUtils.test.js create mode 100644 tests/unit/routes/miners.routes.test.js create mode 100644 workers/lib/server/handlers/miners.handlers.js create mode 100644 workers/lib/server/lib/queryUtils.js create mode 100644 workers/lib/server/routes/miners.routes.js diff --git a/config/common.json.example b/config/common.json.example index 832e1b1..da9c9ea 100644 --- a/config/common.json.example +++ b/config/common.json.example @@ -14,7 +14,8 @@ "/auth/actions/:type": "30s", "/auth/actions/:type/:id": "30s", "/auth/global/data": "30s", - "/auth/site/status/live": "15s" + "/auth/site/status/live": "15s", + "/auth/miners": "15s" }, "featureConfig": { "comments": true, diff --git a/tests/unit/handlers/miners.handlers.test.js b/tests/unit/handlers/miners.handlers.test.js new file mode 100644 index 0000000..7a37fed --- /dev/null +++ b/tests/unit/handlers/miners.handlers.test.js @@ -0,0 +1,312 @@ +'use strict' + +const test = require('brittle') +const { + listMiners, + formatMiner, + extractPoolWorkers, + MINER_FIELD_MAP +} = require('../../../workers/lib/server/handlers/miners.handlers') + +function createMockMiner (overrides = {}) { + return { + id: 't-miner-antminer-192-168-1-101', + type: 'antminer-s19xp', + code: 'A101', + tags: ['t-miner'], + rack: 'rack-0', + info: { + container: 'bitdeer-4b', + pos: 'R3-S12', + serialNum: 'SN12345', + macAddress: 'AA:BB:CC:DD:EE:FF' + }, + opts: { address: '192.168.1.101' }, + last: { + snap: { + model: 'S19XP', + stats: { + status: 'mining', + hashrate_mhs: 140000000, + power_w: 3010, + temperature_c: 72, + efficiency_w_ths: 21.5 + }, + config: { + power_mode: 'normal', + firmware_ver: '2024.01.15', + led_status: 'normal', + pool_config: { url: 'stratum+tcp://pool.example.com', worker: 'worker1' } + } + }, + alerts: { critical: 0, high: 0, medium: 1 }, + uptime: 1209600000, + ts: 1709266500000 + }, + comments: [], + ...overrides + } +} + +function createMockCtx (miners, opts = {}) { + return { + conf: { + orks: [{ rpcPublicKey: 'key1' }], + featureConfig: opts.featureConfig || {} + }, + net_r0: { + jRequest: async (key, method, payload) => { + if (method === 'listThings') return miners + if (method === 'getWrkExtData') return opts.poolData || [] + return {} + } + } + } +} + +// --- formatMiner --- + +test('formatMiner - transforms raw miner to clean format', (t) => { + const raw = createMockMiner() + const result = formatMiner(raw, null) + + t.is(result.id, 't-miner-antminer-192-168-1-101') + t.is(result.type, 'antminer-s19xp') + t.is(result.model, 'S19XP') + t.is(result.code, 'A101') + t.is(result.ip, '192.168.1.101') + t.is(result.container, 'bitdeer-4b') + t.is(result.rack, 'rack-0') + t.is(result.position, 'R3-S12') + t.is(result.status, 'mining') + t.is(result.hashrate, 140000000) + t.is(result.power, 3010) + t.is(result.temperature, 72) + t.is(result.efficiency, 21.5) + t.is(result.firmware, '2024.01.15') + t.is(result.powerMode, 'normal') + t.is(result.ledStatus, 'normal') + t.is(result.serialNum, 'SN12345') + t.is(result.lastSeen, 1709266500000) + t.pass() +}) + +test('formatMiner - handles missing nested fields', (t) => { + const raw = { id: 'test', type: 'miner' } + const result = formatMiner(raw, null) + + t.is(result.id, 'test') + t.is(result.hashrate, 0) + t.is(result.power, 0) + t.is(result.efficiency, 0) + t.is(result.status, undefined) + t.is(result.ip, undefined) + t.pass() +}) + +test('formatMiner - enriches with pool hashrate', (t) => { + const raw = createMockMiner() + const poolWorkers = { + 't-miner-antminer-192-168-1-101': { hashrate: 139500000 } + } + const result = formatMiner(raw, poolWorkers) + + t.is(result.poolHashrate, 139500000) + t.pass() +}) + +test('formatMiner - enriches by code fallback', (t) => { + const raw = createMockMiner() + const poolWorkers = { + A101: { hashrate: 139500000 } + } + const result = formatMiner(raw, poolWorkers) + + t.is(result.poolHashrate, 139500000) + t.pass() +}) + +test('formatMiner - no poolHashrate when no match', (t) => { + const raw = createMockMiner() + const poolWorkers = { 'other-id': { hashrate: 100 } } + const result = formatMiner(raw, poolWorkers) + + t.is(result.poolHashrate, undefined) + t.pass() +}) + +// --- extractPoolWorkers --- + +test('extractPoolWorkers - builds worker lookup', (t) => { + const poolData = [ + [ + { + stats: { hashrate: 100 }, + workers: { + 'miner-1': { hashrate: 50 }, + 'miner-2': { hashrate: 50 } + } + } + ] + ] + const result = extractPoolWorkers(poolData) + + t.is(result['miner-1'].hashrate, 50) + t.is(result['miner-2'].hashrate, 50) + t.pass() +}) + +test('extractPoolWorkers - handles empty data', (t) => { + const result = extractPoolWorkers([]) + t.is(Object.keys(result).length, 0) + t.pass() +}) + +test('extractPoolWorkers - handles pools without workers', (t) => { + const poolData = [[{ stats: { hashrate: 100 } }]] + const result = extractPoolWorkers(poolData) + t.is(Object.keys(result).length, 0) + t.pass() +}) + +// --- listMiners --- + +test('listMiners - returns paginated response with formatted miners', async (t) => { + const miners = [createMockMiner(), createMockMiner({ id: 'miner-2', code: 'A102' })] + const ctx = createMockCtx(miners) + const req = { query: {} } + + const result = await listMiners(ctx, req) + + t.ok(result.data) + t.is(result.data.length, 2) + t.is(result.totalCount, 2) + t.is(result.offset, 0) + t.is(result.limit, 50) + t.is(result.hasMore, false) + t.is(result.data[0].id, 't-miner-antminer-192-168-1-101') + t.is(result.data[0].model, 'S19XP') + t.pass() +}) + +test('listMiners - applies pagination', async (t) => { + const miners = Array.from({ length: 10 }, (_, i) => + createMockMiner({ id: `miner-${i}`, code: `A${i}` }) + ) + const ctx = createMockCtx(miners) + const req = { query: { offset: 2, limit: 3 } } + + const result = await listMiners(ctx, req) + + t.is(result.data.length, 3) + t.is(result.totalCount, 10) + t.is(result.offset, 2) + t.is(result.limit, 3) + t.is(result.hasMore, true) + t.pass() +}) + +test('listMiners - enforces max limit of 200', async (t) => { + const ctx = createMockCtx([]) + const req = { query: { limit: 500 } } + + const result = await listMiners(ctx, req) + + t.is(result.limit, 200) + t.pass() +}) + +test('listMiners - parses filter JSON', async (t) => { + let capturedPayload = null + const ctx = { + conf: { + orks: [{ rpcPublicKey: 'key1' }], + featureConfig: {} + }, + net_r0: { + jRequest: async (key, method, payload) => { + capturedPayload = payload + return [] + } + } + } + const req = { query: { filter: '{"status":"error"}' } } + + await listMiners(ctx, req) + + t.ok(capturedPayload.query.$and) + t.is(capturedPayload.query.$and[0].tags.$in[0], 't-miner') + t.is(capturedPayload.query.$and[1]['last.snap.stats.status'], 'error') + t.pass() +}) + +test('listMiners - builds search query', async (t) => { + let capturedPayload = null + const ctx = { + conf: { + orks: [{ rpcPublicKey: 'key1' }], + featureConfig: {} + }, + net_r0: { + jRequest: async (key, method, payload) => { + capturedPayload = payload + return [] + } + } + } + const req = { query: { search: '192.168' } } + + await listMiners(ctx, req) + + const lastCondition = capturedPayload.query.$and[capturedPayload.query.$and.length - 1] + t.ok(lastCondition.$or) + t.ok(lastCondition.$or.some(c => c.id?.$regex === '192.168')) + t.ok(lastCondition.$or.some(c => c['opts.address']?.$regex === '192.168')) + t.pass() +}) + +test('listMiners - handles empty ork results', async (t) => { + const ctx = createMockCtx([]) + const req = { query: {} } + + const result = await listMiners(ctx, req) + + t.is(result.data.length, 0) + t.is(result.totalCount, 0) + t.pass() +}) + +test('listMiners - throws on invalid filter JSON', async (t) => { + const ctx = createMockCtx([]) + const req = { query: { filter: 'not-json' } } + + try { + await listMiners(ctx, req) + t.fail('should have thrown') + } catch (err) { + t.is(err.message, 'ERR_FILTER_INVALID_JSON') + } + t.pass() +}) + +test('listMiners - throws on invalid sort JSON', async (t) => { + const ctx = createMockCtx([]) + const req = { query: { sort: '{invalid' } } + + try { + await listMiners(ctx, req) + t.fail('should have thrown') + } catch (err) { + t.is(err.message, 'ERR_SORT_INVALID_JSON') + } + t.pass() +}) + +test('MINER_FIELD_MAP - has expected field mappings', (t) => { + t.is(MINER_FIELD_MAP.status, 'last.snap.stats.status') + t.is(MINER_FIELD_MAP.hashrate, 'last.snap.stats.hashrate_mhs') + t.is(MINER_FIELD_MAP.ip, 'opts.address') + t.is(MINER_FIELD_MAP.container, 'info.container') + t.is(MINER_FIELD_MAP.model, 'last.snap.model') + t.pass() +}) diff --git a/tests/unit/lib/queryUtils.test.js b/tests/unit/lib/queryUtils.test.js new file mode 100644 index 0000000..5231fed --- /dev/null +++ b/tests/unit/lib/queryUtils.test.js @@ -0,0 +1,277 @@ +'use strict' + +const test = require('brittle') +const { + getNestedValue, + mapFilterFields, + mapSortFields, + buildSearchQuery, + flattenOrkResults, + sortItems, + paginateResults +} = require('../../../workers/lib/server/lib/queryUtils') + +const FIELD_MAP = { + status: 'last.snap.stats.status', + hashrate: 'last.snap.stats.hashrate_mhs', + container: 'info.container', + ip: 'opts.address' +} + +// --- getNestedValue --- + +test('getNestedValue - gets simple key', (t) => { + t.is(getNestedValue({ a: 1 }, 'a'), 1) + t.pass() +}) + +test('getNestedValue - gets nested key', (t) => { + t.is(getNestedValue({ a: { b: { c: 42 } } }, 'a.b.c'), 42) + t.pass() +}) + +test('getNestedValue - returns undefined for missing path', (t) => { + t.is(getNestedValue({ a: 1 }, 'a.b.c'), undefined) + t.pass() +}) + +test('getNestedValue - handles null object', (t) => { + t.is(getNestedValue(null, 'a'), undefined) + t.pass() +}) + +// --- mapFilterFields --- + +test('mapFilterFields - maps simple equality', (t) => { + const result = mapFilterFields({ status: 'error' }, FIELD_MAP) + t.is(result['last.snap.stats.status'], 'error') + t.pass() +}) + +test('mapFilterFields - maps range operators', (t) => { + const result = mapFilterFields({ hashrate: { $gt: 0 } }, FIELD_MAP) + t.ok(result['last.snap.stats.hashrate_mhs']) + t.is(result['last.snap.stats.hashrate_mhs'].$gt, 0) + t.pass() +}) + +test('mapFilterFields - maps $and arrays', (t) => { + const result = mapFilterFields({ + $and: [ + { status: 'error' }, + { hashrate: { $gt: 0 } } + ] + }, FIELD_MAP) + t.ok(Array.isArray(result.$and)) + t.is(result.$and.length, 2) + t.ok(result.$and[0]['last.snap.stats.status']) + t.ok(result.$and[1]['last.snap.stats.hashrate_mhs']) + t.pass() +}) + +test('mapFilterFields - maps $or arrays', (t) => { + const result = mapFilterFields({ + $or: [{ status: 'error' }, { status: 'offline' }] + }, FIELD_MAP) + t.ok(Array.isArray(result.$or)) + t.is(result.$or[0]['last.snap.stats.status'], 'error') + t.is(result.$or[1]['last.snap.stats.status'], 'offline') + t.pass() +}) + +test('mapFilterFields - passes through unknown keys', (t) => { + const result = mapFilterFields({ 'last.snap.model': 'S19XP' }, FIELD_MAP) + t.is(result['last.snap.model'], 'S19XP') + t.pass() +}) + +test('mapFilterFields - handles null/undefined filter', (t) => { + t.is(mapFilterFields(null, FIELD_MAP), null) + t.is(mapFilterFields(undefined, FIELD_MAP), undefined) + t.pass() +}) + +test('mapFilterFields - handles $in operator in value', (t) => { + const result = mapFilterFields({ status: { $in: ['error', 'offline'] } }, FIELD_MAP) + t.ok(result['last.snap.stats.status'].$in) + t.is(result['last.snap.stats.status'].$in.length, 2) + t.pass() +}) + +test('mapFilterFields - handles combined AND/OR', (t) => { + const result = mapFilterFields({ + container: 'bitdeer-4b', + $or: [{ status: 'error' }, { status: 'offline' }] + }, FIELD_MAP) + t.is(result['info.container'], 'bitdeer-4b') + t.ok(Array.isArray(result.$or)) + t.pass() +}) + +// --- mapSortFields --- + +test('mapSortFields - maps sort keys', (t) => { + const result = mapSortFields({ hashrate: -1, status: 1 }, FIELD_MAP) + t.is(result['last.snap.stats.hashrate_mhs'], -1) + t.is(result['last.snap.stats.status'], 1) + t.pass() +}) + +test('mapSortFields - passes through unknown keys', (t) => { + const result = mapSortFields({ 'info.pos': 1 }, FIELD_MAP) + t.is(result['info.pos'], 1) + t.pass() +}) + +test('mapSortFields - handles null', (t) => { + t.is(mapSortFields(null, FIELD_MAP), null) + t.pass() +}) + +// --- buildSearchQuery --- + +test('buildSearchQuery - builds multi-field OR regex', (t) => { + const result = buildSearchQuery('192.168', ['id', 'opts.address', 'code']) + t.ok(result.$or) + t.is(result.$or.length, 3) + t.is(result.$or[0].id.$regex, '192.168') + t.is(result.$or[0].id.$options, 'i') + t.is(result.$or[1]['opts.address'].$regex, '192.168') + t.pass() +}) + +// --- flattenOrkResults --- + +test('flattenOrkResults - flattens multiple ork arrays', (t) => { + const results = [[{ id: 'a' }, { id: 'b' }], [{ id: 'c' }]] + const flat = flattenOrkResults(results) + t.is(flat.length, 3) + t.is(flat[0].id, 'a') + t.is(flat[2].id, 'c') + t.pass() +}) + +test('flattenOrkResults - handles empty arrays', (t) => { + t.is(flattenOrkResults([]).length, 0) + t.is(flattenOrkResults([[], []]).length, 0) + t.pass() +}) + +test('flattenOrkResults - handles non-array ork results', (t) => { + const results = [{ error: 'timeout' }, [{ id: 'a' }]] + const flat = flattenOrkResults(results) + t.is(flat.length, 1) + t.is(flat[0].id, 'a') + t.pass() +}) + +// --- sortItems --- + +test('sortItems - sorts ascending', (t) => { + const items = [{ v: 3 }, { v: 1 }, { v: 2 }] + sortItems(items, { v: 1 }) + t.is(items[0].v, 1) + t.is(items[1].v, 2) + t.is(items[2].v, 3) + t.pass() +}) + +test('sortItems - sorts descending', (t) => { + const items = [{ v: 1 }, { v: 3 }, { v: 2 }] + sortItems(items, { v: -1 }) + t.is(items[0].v, 3) + t.is(items[1].v, 2) + t.is(items[2].v, 1) + t.pass() +}) + +test('sortItems - sorts by nested path', (t) => { + const items = [ + { a: { b: 3 } }, + { a: { b: 1 } }, + { a: { b: 2 } } + ] + sortItems(items, { 'a.b': 1 }) + t.is(items[0].a.b, 1) + t.is(items[2].a.b, 3) + t.pass() +}) + +test('sortItems - handles null sort', (t) => { + const items = [{ v: 2 }, { v: 1 }] + sortItems(items, null) + t.is(items[0].v, 2) + t.pass() +}) + +test('sortItems - null values sort last', (t) => { + const items = [{ v: null }, { v: 2 }, { v: 1 }] + sortItems(items, { v: 1 }) + t.is(items[0].v, 1) + t.is(items[1].v, 2) + t.is(items[2].v, null) + t.pass() +}) + +test('sortItems - multi-key sort', (t) => { + const items = [ + { a: 1, b: 2 }, + { a: 1, b: 1 }, + { a: 2, b: 1 } + ] + sortItems(items, { a: 1, b: 1 }) + t.is(items[0].b, 1) + t.is(items[1].b, 2) + t.is(items[2].a, 2) + t.pass() +}) + +// --- paginateResults --- + +test('paginateResults - first page', (t) => { + const items = Array.from({ length: 100 }, (_, i) => ({ id: i })) + const result = paginateResults(items, 0, 10) + t.is(result.data.length, 10) + t.is(result.totalCount, 100) + t.is(result.offset, 0) + t.is(result.limit, 10) + t.is(result.hasMore, true) + t.is(result.data[0].id, 0) + t.pass() +}) + +test('paginateResults - middle page', (t) => { + const items = Array.from({ length: 100 }, (_, i) => ({ id: i })) + const result = paginateResults(items, 20, 10) + t.is(result.data.length, 10) + t.is(result.data[0].id, 20) + t.is(result.offset, 20) + t.is(result.hasMore, true) + t.pass() +}) + +test('paginateResults - last page', (t) => { + const items = Array.from({ length: 25 }, (_, i) => ({ id: i })) + const result = paginateResults(items, 20, 10) + t.is(result.data.length, 5) + t.is(result.totalCount, 25) + t.is(result.hasMore, false) + t.pass() +}) + +test('paginateResults - empty results', (t) => { + const result = paginateResults([], 0, 10) + t.is(result.data.length, 0) + t.is(result.totalCount, 0) + t.is(result.hasMore, false) + t.pass() +}) + +test('paginateResults - offset beyond total', (t) => { + const items = [{ id: 1 }] + const result = paginateResults(items, 50, 10) + t.is(result.data.length, 0) + t.is(result.totalCount, 1) + t.is(result.hasMore, false) + t.pass() +}) diff --git a/tests/unit/routes/miners.routes.test.js b/tests/unit/routes/miners.routes.test.js new file mode 100644 index 0000000..a887183 --- /dev/null +++ b/tests/unit/routes/miners.routes.test.js @@ -0,0 +1,65 @@ +'use strict' + +const test = require('brittle') +const { testModuleStructure, testHandlerFunctions } = require('../helpers/routeTestHelpers') +const { createRoutesForTest } = require('../helpers/mockHelpers') + +const ROUTES_PATH = '../../../workers/lib/server/routes/miners.routes.js' + +test('miners routes - module structure', (t) => { + testModuleStructure(t, ROUTES_PATH, 'miners') + t.pass() +}) + +test('miners routes - route definitions', (t) => { + const routes = createRoutesForTest(ROUTES_PATH) + + const routeUrls = routes.map(route => route.url) + t.ok(routeUrls.includes('/auth/miners'), 'should have miners route') + + t.pass() +}) + +test('miners routes - HTTP methods', (t) => { + const routes = createRoutesForTest(ROUTES_PATH) + + const minersRoute = routes.find(r => r.url === '/auth/miners') + t.is(minersRoute.method, 'GET', 'miners route should be GET') + + t.pass() +}) + +test('miners routes - schema validation', (t) => { + const routes = createRoutesForTest(ROUTES_PATH) + + const minersRoute = routes.find(r => r.url === '/auth/miners') + t.ok(minersRoute.schema, 'should have schema') + t.ok(minersRoute.schema.querystring, 'should have querystring schema') + + const props = minersRoute.schema.querystring.properties + t.is(props.filter.type, 'string', 'filter should be string') + t.is(props.sort.type, 'string', 'sort should be string') + t.is(props.fields.type, 'string', 'fields should be string') + t.is(props.search.type, 'string', 'search should be string') + t.is(props.offset.type, 'integer', 'offset should be integer') + t.is(props.limit.type, 'integer', 'limit should be integer') + t.is(props.overwriteCache.type, 'boolean', 'overwriteCache should be boolean') + + t.pass() +}) + +test('miners routes - handler functions', (t) => { + const routes = createRoutesForTest(ROUTES_PATH) + testHandlerFunctions(t, routes, 'miners') + t.pass() +}) + +test('miners routes - onRequest functions (auth)', (t) => { + const routes = createRoutesForTest(ROUTES_PATH) + + routes.forEach(route => { + t.ok(typeof route.onRequest === 'function', `miners route ${route.url} should have onRequest function`) + }) + + t.pass() +}) diff --git a/workers/lib/constants.js b/workers/lib/constants.js index a621419..b6767bb 100644 --- a/workers/lib/constants.js +++ b/workers/lib/constants.js @@ -131,7 +131,9 @@ const ENDPOINTS = { POOL_MANAGER_ASSIGN: '/auth/pool-manager/miners/assign', POOL_MANAGER_POWER_MODE: '/auth/pool-manager/miners/power-mode', - SITE_STATUS_LIVE: '/auth/site/status/live' + SITE_STATUS_LIVE: '/auth/site/status/live', + + MINERS: '/auth/miners' } const HTTP_METHODS = { diff --git a/workers/lib/server/handlers/miners.handlers.js b/workers/lib/server/handlers/miners.handlers.js new file mode 100644 index 0000000..52fdc6d --- /dev/null +++ b/workers/lib/server/handlers/miners.handlers.js @@ -0,0 +1,226 @@ +'use strict' + +const { parseJsonQueryParam, requestRpcMapLimit } = require('../../utils') +const { + mapFilterFields, + mapSortFields, + buildSearchQuery, + flattenOrkResults, + sortItems, + paginateResults +} = require('../lib/queryUtils') + +/** + * Maps clean miner field names to internal dot-paths. + * Users can filter/sort using clean names; unknown keys pass through as-is. + */ +const MINER_FIELD_MAP = { + status: 'last.snap.stats.status', + hashrate: 'last.snap.stats.hashrate_mhs', + power: 'last.snap.stats.power_w', + efficiency: 'last.snap.stats.efficiency_w_ths', + temperature: 'last.snap.stats.temperature_c', + powerMode: 'last.snap.config.power_mode', + firmware: 'last.snap.config.firmware_ver', + model: 'last.snap.model', + ip: 'opts.address', + container: 'info.container', + rack: 'rack', + serialNum: 'info.serialNum', + macAddress: 'info.macAddress', + pool: 'last.snap.config.pool_config.url', + led: 'last.snap.config.led_status', + alerts: 'last.alerts' +} + +/** + * Internal fields searched when using the `search` query param. + */ +const MINER_SEARCH_FIELDS = [ + 'id', + 'opts.address', + 'info.serialNum', + 'info.macAddress', + 'info.container', + 'code', + 'type' +] + +/** + * Default field projection when user doesn't specify fields. + * Includes all fields needed for formatMiner. + */ +const MINER_DEFAULT_FIELDS = { + id: 1, + type: 1, + code: 1, + info: 1, + tags: 1, + rack: 1, + comments: 1, + 'last.alerts': 1, + 'last.snap.stats': 1, + 'last.snap.config': 1, + 'last.snap.model': 1, + 'last.uptime': 1, + 'last.ts': 1, + 'opts.address': 1, + ts: 1 +} + +const MAX_LIMIT = 200 +const DEFAULT_LIMIT = 50 + +/** + * Transforms a raw miner thing into the clean response format. + * + * @param {Object} raw - Raw miner object from listThings RPC + * @param {Object|null} poolWorkers - Pool worker lookup { minerId: { hashrate } } + * @returns {Object} Clean miner response object + */ +function formatMiner (raw, poolWorkers) { + const snap = raw.last?.snap || {} + const stats = snap.stats || {} + const config = snap.config || {} + + const miner = { + id: raw.id, + type: raw.type, + model: snap.model || raw.type, + code: raw.code, + ip: raw.opts?.address, + container: raw.info?.container, + rack: raw.rack, + position: raw.info?.pos, + status: stats.status, + hashrate: stats.hashrate_mhs || 0, + power: stats.power_w || 0, + temperature: stats.temperature_c, + efficiency: stats.efficiency_w_ths || 0, + uptime: raw.last?.uptime, + firmware: config.firmware_ver, + powerMode: config.power_mode, + ledStatus: config.led_status, + poolConfig: config.pool_config, + alerts: raw.last?.alerts, + comments: raw.comments, + serialNum: raw.info?.serialNum, + macAddress: raw.info?.macAddress, + lastSeen: raw.last?.ts || raw.ts + } + + if (poolWorkers) { + const poolWorker = poolWorkers[raw.id] || poolWorkers[raw.code] + if (poolWorker) { + miner.poolHashrate = poolWorker.hashrate || 0 + } + } + + return miner +} + +/** + * Extracts a worker hashrate lookup from pool data RPC results. + * Builds a map: workerId → { hashrate } + * + * @param {Array} poolDataResults - Array of ork responses from getWrkExtData + * @returns {Object} Worker lookup map + */ +function extractPoolWorkers (poolDataResults) { + const workers = {} + for (const orkResult of poolDataResults) { + if (!Array.isArray(orkResult)) continue + for (const pool of orkResult) { + if (!pool || !pool.workers) continue + for (const [workerId, workerData] of Object.entries(pool.workers)) { + workers[workerId] = workerData + } + } + } + return workers +} + +/** + * GET /auth/miners + * + * Lists miners with unified filter/sort/search/pagination. + * Auto-injects miner type filter, maps clean field names to internal paths, + * aggregates multi-ork results, and returns a paginated response. + * + * Replaces: GET /auth/list-things?query={"tags":{"$in":["t-miner"]}}&status=1&... + */ +async function listMiners (ctx, req) { + const userFilter = req.query.filter + ? parseJsonQueryParam(req.query.filter, 'ERR_FILTER_INVALID_JSON') + : {} + + const mappedFilter = mapFilterFields(userFilter, MINER_FIELD_MAP) + + const query = { + $and: [ + { tags: { $in: ['t-miner'] } }, + ...(Object.keys(mappedFilter).length ? [mappedFilter] : []), + ...(req.query.search ? [buildSearchQuery(req.query.search, MINER_SEARCH_FIELDS)] : []) + ] + } + + const mappedSort = req.query.sort + ? mapSortFields(parseJsonQueryParam(req.query.sort, 'ERR_SORT_INVALID_JSON'), MINER_FIELD_MAP) + : undefined + + const fields = req.query.fields + ? parseJsonQueryParam(req.query.fields, 'ERR_FIELDS_INVALID_JSON') + : MINER_DEFAULT_FIELDS + + const offset = req.query.offset || 0 + const limit = Math.min(req.query.limit || DEFAULT_LIMIT, MAX_LIMIT) + + const rpcPayload = { + query, + fields, + status: 1 + } + + if (mappedSort) { + rpcPayload.sort = mappedSort + } + + const orkResults = await requestRpcMapLimit(ctx, 'listThings', rpcPayload) + let items = flattenOrkResults(orkResults) + + if (mappedSort) { + items = sortItems(items, mappedSort) + } + + let poolWorkers = null + if (ctx.conf.featureConfig?.poolStats) { + try { + const poolData = await requestRpcMapLimit(ctx, 'getWrkExtData', { + type: 'minerpool', + query: { key: 'stats' } + }) + poolWorkers = extractPoolWorkers(poolData) + } catch { + // Pool enrichment is optional — don't fail the request + } + } + + const paginated = paginateResults(items, offset, limit) + + return { + data: paginated.data.map(miner => formatMiner(miner, poolWorkers)), + totalCount: paginated.totalCount, + offset: paginated.offset, + limit: paginated.limit, + hasMore: paginated.hasMore + } +} + +module.exports = { + listMiners, + formatMiner, + extractPoolWorkers, + MINER_FIELD_MAP, + MINER_SEARCH_FIELDS, + MINER_DEFAULT_FIELDS +} diff --git a/workers/lib/server/index.js b/workers/lib/server/index.js index 840e405..ce28308 100644 --- a/workers/lib/server/index.js +++ b/workers/lib/server/index.js @@ -12,6 +12,7 @@ const financeRoutes = require('./routes/finance.routes') const poolsRoutes = require('./routes/pools.routes') const poolManagerRoutes = require('./routes/poolManager.routes') const siteRoutes = require('./routes/site.routes') +const minersRoutes = require("./routes/miners.routes"); /** * Collect all routes into a flat array for server injection. @@ -30,7 +31,8 @@ function routes (ctx) { ...financeRoutes(ctx), ...poolsRoutes(ctx), ...poolManagerRoutes(ctx), - ...siteRoutes(ctx) + ...siteRoutes(ctx), + ...minersRoutes(ctx), ] } diff --git a/workers/lib/server/lib/queryUtils.js b/workers/lib/server/lib/queryUtils.js new file mode 100644 index 0000000..bc0b9ff --- /dev/null +++ b/workers/lib/server/lib/queryUtils.js @@ -0,0 +1,183 @@ +'use strict' + +/** + * Shared query utilities for all list endpoints. + * Provides MongoDB-style filter field mapping, sort mapping, + * text search, multi-ork result flattening, sorting, and pagination. + */ + +const MONGO_OPERATORS = new Set([ + '$gt', '$gte', '$lt', '$lte', '$eq', '$ne', + '$in', '$nin', '$regex', '$options', '$exists', + '$elemMatch', '$not', '$type', '$size' +]) + +/** + * Gets a nested value from an object using a dot-separated path. + * + * @param {Object} obj - Source object + * @param {string} path - Dot-separated path (e.g. 'last.snap.stats.status') + * @returns {*} The value at the path, or undefined + */ +function getNestedValue (obj, path) { + const parts = path.split('.') + let current = obj + for (const part of parts) { + if (current == null) return undefined + current = current[part] + } + return current +} + +/** + * Recursively maps clean field names to internal dot-paths in a MongoDB-style filter. + * Handles $and, $or, $not, $elemMatch operators by recursing into their values. + * Unknown keys are passed through as-is (allows raw internal paths). + * + * @param {Object} filter - MongoDB-style filter object + * @param {Object} fieldMap - Map of clean name → internal path + * @returns {Object} Filter with mapped field names + * + * @example + * mapFilterFields({ status: 'error' }, { status: 'last.snap.stats.status' }) + * // → { 'last.snap.stats.status': 'error' } + * + * mapFilterFields({ $or: [{ status: 'error' }, { hashrate: { $gt: 0 } }] }, fieldMap) + * // → { $or: [{ 'last.snap.stats.status': 'error' }, { 'last.snap.stats.hashrate_mhs': { $gt: 0 } }] } + */ +function mapFilterFields (filter, fieldMap) { + if (!filter || typeof filter !== 'object') return filter + if (Array.isArray(filter)) return filter.map(f => mapFilterFields(f, fieldMap)) + + const mapped = {} + for (const [key, value] of Object.entries(filter)) { + if (key === '$and' || key === '$or') { + mapped[key] = Array.isArray(value) + ? value.map(f => mapFilterFields(f, fieldMap)) + : value + } else if (key === '$elemMatch' || key === '$not') { + mapped[key] = mapFilterFields(value, fieldMap) + } else if (MONGO_OPERATORS.has(key)) { + mapped[key] = value + } else { + const mappedKey = fieldMap[key] || key + mapped[mappedKey] = value + } + } + return mapped +} + +/** + * Maps clean field names to internal paths in a sort specification. + * + * @param {Object} sort - Sort spec: { field: 1 } (1=asc, -1=desc) + * @param {Object} fieldMap - Map of clean name → internal path + * @returns {Object} Sort with mapped field names + * + * @example + * mapSortFields({ hashrate: -1 }, { hashrate: 'last.snap.stats.hashrate_mhs' }) + * // → { 'last.snap.stats.hashrate_mhs': -1 } + */ +function mapSortFields (sort, fieldMap) { + if (!sort || typeof sort !== 'object') return sort + const mapped = {} + for (const [key, value] of Object.entries(sort)) { + mapped[fieldMap[key] || key] = value + } + return mapped +} + +/** + * Builds a text search query as a multi-field $or with $regex. + * + * @param {string} search - Search term + * @param {Array} searchFields - Internal field paths to search across + * @returns {Object} MongoDB-style $or query + * + * @example + * buildSearchQuery('192.168', ['id', 'opts.address']) + * // → { $or: [{ id: { $regex: '192.168', $options: 'i' } }, { 'opts.address': { $regex: '192.168', $options: 'i' } }] } + */ +function buildSearchQuery (search, searchFields) { + return { + $or: searchFields.map(field => ({ + [field]: { $regex: search, $options: 'i' } + })) + } +} + +/** + * Flattens multi-ork results into a single array. + * Each ork response for listThings is an array of items. + * requestRpcMapLimit returns [orkResult1, orkResult2, ...]. + * + * @param {Array} orkResults - Array of ork responses + * @returns {Array} Flattened array of all items + */ +function flattenOrkResults (orkResults) { + const items = [] + for (const orkResult of orkResults) { + if (Array.isArray(orkResult)) { + items.push(...orkResult) + } + } + return items +} + +/** + * Sorts items by a sort specification using internal dot-path fields. + * Handles multi-key sorting. Null/undefined values sort last. + * + * @param {Array} items - Items to sort + * @param {Object} sort - Sort spec: { 'internal.path': 1 or -1 } + * @returns {Array} Sorted items (mutates the original array) + */ +function sortItems (items, sort) { + if (!sort || typeof sort !== 'object' || Object.keys(sort).length === 0) return items + + const sortEntries = Object.entries(sort) + + return items.sort((a, b) => { + for (const [field, direction] of sortEntries) { + const aVal = getNestedValue(a, field) + const bVal = getNestedValue(b, field) + + if (aVal === bVal) continue + if (aVal == null) return direction + if (bVal == null) return -direction + + if (aVal < bVal) return -direction + if (aVal > bVal) return direction + } + return 0 + }) +} + +/** + * Creates a paginated response from a flat array of items. + * + * @param {Array} items - All matching items (already sorted) + * @param {number} offset - Pagination offset + * @param {number} limit - Page size + * @returns {Object} Paginated response + */ +function paginateResults (items, offset, limit) { + const page = items.slice(offset, offset + limit) + return { + data: page, + totalCount: items.length, + offset, + limit, + hasMore: offset + limit < items.length + } +} + +module.exports = { + getNestedValue, + mapFilterFields, + mapSortFields, + buildSearchQuery, + flattenOrkResults, + sortItems, + paginateResults +} diff --git a/workers/lib/server/routes/miners.routes.js b/workers/lib/server/routes/miners.routes.js new file mode 100644 index 0000000..cb6cc12 --- /dev/null +++ b/workers/lib/server/routes/miners.routes.js @@ -0,0 +1,46 @@ +'use strict' + +const { + ENDPOINTS, + HTTP_METHODS, + AUTH_CAPS, + AUTH_LEVELS +} = require('../../constants') +const { listMiners } = require('../handlers/miners.handlers') +const { createCachedAuthRoute } = require('../lib/routeHelpers') + +module.exports = (ctx) => [ + { + method: HTTP_METHODS.GET, + url: ENDPOINTS.MINERS, + schema: { + querystring: { + type: 'object', + properties: { + filter: { type: 'string' }, + sort: { type: 'string' }, + fields: { type: 'string' }, + search: { type: 'string' }, + offset: { type: 'integer' }, + limit: { type: 'integer' }, + overwriteCache: { type: 'boolean' } + } + } + }, + ...createCachedAuthRoute( + ctx, + (req) => [ + 'miners', + req.query.filter, + req.query.sort, + req.query.fields, + req.query.search, + req.query.offset, + req.query.limit + ], + ENDPOINTS.MINERS, + listMiners, + [`${AUTH_CAPS.m}:${AUTH_LEVELS.READ}`] + ) + } +] From 154746322a460925351e4e0ba17737a04f527dba Mon Sep 17 00:00:00 2001 From: paragmore Date: Thu, 19 Feb 2026 17:28:58 +0530 Subject: [PATCH 2/4] Added integration tests --- tests/integration/api.miners.test.js | 713 +++++++++++++++++++++++++++ 1 file changed, 713 insertions(+) create mode 100644 tests/integration/api.miners.test.js diff --git a/tests/integration/api.miners.test.js b/tests/integration/api.miners.test.js new file mode 100644 index 0000000..f6e0cf5 --- /dev/null +++ b/tests/integration/api.miners.test.js @@ -0,0 +1,713 @@ +'use strict' + +const test = require('brittle') +const fs = require('fs') +const { createWorker } = require('tether-svc-test-helper').worker +const { setTimeout: sleep } = require('timers/promises') +const HttpFacility = require('bfx-facs-http') +const { ENDPOINTS } = require('../../workers/lib/constants') + +test('Miners API', { timeout: 90000 }, async (main) => { + const baseDir = 'tests/integration' + let worker + let httpClient + const appNodePort = 5002 + const ip = '127.0.0.1' + const appNodeBaseUrl = `http://${ip}:${appNodePort}` + const readonlyUser = 'readonly-miners@test' + const siteOperatorUser = 'siteoperator-miners@test' + const encoding = 'json' + const invalidToken = 'invalid-token' + + main.teardown(async () => { + await httpClient.stop() + await worker.stop() + await sleep(2000) + fs.rmSync(`./${baseDir}/store`, { recursive: true, force: true }) + fs.rmSync(`./${baseDir}/status`, { recursive: true, force: true }) + fs.rmSync(`./${baseDir}/config`, { recursive: true, force: true }) + fs.rmSync(`./${baseDir}/db`, { recursive: true, force: true }) + }) + + const mockMiners = [ + { + id: 'miner-001', + type: 'antminer-s19', + code: 'M001', + info: { + container: 'container-A', + serialNum: 'SN-001', + macAddress: 'AA:BB:CC:DD:EE:01', + pos: 'A1' + }, + tags: ['t-miner'], + rack: 'rack-1', + comments: [], + opts: { address: '192.168.1.100' }, + ts: Date.now() - 60000, + last: { + ts: Date.now(), + uptime: 86400, + alerts: [], + snap: { + model: 'Antminer S19 XP', + stats: { + status: 'online', + hashrate_mhs: 140000, + power_w: 3010, + efficiency_w_ths: 21.5, + temperature_c: 65 + }, + config: { + firmware_ver: '2024.01.01', + power_mode: 'normal', + led_status: 'off', + pool_config: { + url: 'stratum+tcp://btc.f2pool.com:3333', + user: 'tether.worker1' + } + } + } + } + }, + { + id: 'miner-002', + type: 'antminer-s19', + code: 'M002', + info: { + container: 'container-A', + serialNum: 'SN-002', + macAddress: 'AA:BB:CC:DD:EE:02', + pos: 'A2' + }, + tags: ['t-miner'], + rack: 'rack-1', + comments: [{ text: 'needs maintenance' }], + opts: { address: '192.168.1.101' }, + ts: Date.now() - 120000, + last: { + ts: Date.now(), + uptime: 172800, + alerts: [{ type: 'high_temp', severity: 'medium' }], + snap: { + model: 'Antminer S19 XP', + stats: { + status: 'online', + hashrate_mhs: 135000, + power_w: 2980, + efficiency_w_ths: 22.1, + temperature_c: 72 + }, + config: { + firmware_ver: '2024.01.01', + power_mode: 'normal', + led_status: 'off', + pool_config: { + url: 'stratum+tcp://btc.f2pool.com:3333', + user: 'tether.worker2' + } + } + } + } + }, + { + id: 'miner-003', + type: 'whatsminer-m50s', + code: 'M003', + info: { + container: 'container-B', + serialNum: 'SN-003', + macAddress: 'AA:BB:CC:DD:EE:03', + pos: 'B1' + }, + tags: ['t-miner'], + rack: 'rack-2', + comments: [], + opts: { address: '192.168.2.100' }, + ts: Date.now() - 180000, + last: { + ts: Date.now() - 300000, + uptime: 3600, + alerts: [], + snap: { + model: 'Whatsminer M50S', + stats: { + status: 'error', + hashrate_mhs: 0, + power_w: 50, + efficiency_w_ths: 0, + temperature_c: 30 + }, + config: { + firmware_ver: '2023.12.15', + power_mode: 'normal', + led_status: 'on', + pool_config: { + url: 'stratum+tcp://ocean.xyz:3333', + user: 'tether.worker3' + } + } + } + } + }, + { + id: 'miner-004', + type: 'antminer-s19', + code: 'M004', + info: { + container: 'container-B', + serialNum: 'SN-004', + macAddress: 'AA:BB:CC:DD:EE:04', + pos: 'B2' + }, + tags: ['t-miner'], + rack: 'rack-2', + comments: [], + opts: { address: '192.168.2.101' }, + ts: Date.now() - 240000, + last: { + ts: Date.now(), + uptime: 43200, + alerts: [], + snap: { + model: 'Antminer S19 XP', + stats: { + status: 'sleep', + hashrate_mhs: 0, + power_w: 10, + efficiency_w_ths: 0, + temperature_c: 25 + }, + config: { + firmware_ver: '2024.01.01', + power_mode: 'sleep', + led_status: 'off', + pool_config: { + url: 'stratum+tcp://btc.f2pool.com:3333', + user: 'tether.worker4' + } + } + } + } + }, + { + id: 'miner-005', + type: 'antminer-s19', + code: 'M005', + info: { + container: 'container-A', + serialNum: 'SN-005', + macAddress: 'AA:BB:CC:DD:EE:05', + pos: 'A3' + }, + tags: ['t-miner'], + rack: 'rack-1', + comments: [], + opts: { address: '192.168.1.102' }, + ts: Date.now() - 300000, + last: { + ts: Date.now(), + uptime: 259200, + alerts: [{ type: 'low_hashrate', severity: 'high' }], + snap: { + model: 'Antminer S19 XP', + stats: { + status: 'online', + hashrate_mhs: 120000, + power_w: 2900, + efficiency_w_ths: 24.2, + temperature_c: 68 + }, + config: { + firmware_ver: '2023.12.15', + power_mode: 'low', + led_status: 'off', + pool_config: { + url: 'stratum+tcp://btc.f2pool.com:3333', + user: 'tether.worker5' + } + } + } + } + } + ] + + const createConfig = () => { + if (!fs.existsSync(`./${baseDir}/config/facs`)) { + if (!fs.existsSync(`./${baseDir}/config`)) fs.mkdirSync(`./${baseDir}/config`) + fs.mkdirSync(`./${baseDir}/config/facs`) + } + if (!fs.existsSync(`./${baseDir}/db`)) fs.mkdirSync(`./${baseDir}/db`) + + const commonConf = { + dir_log: 'logs', + debug: 0, + orks: { 'cluster-1': { rpcPublicKey: '' } }, + cacheTiming: {}, + featureConfig: {} + } + const netConf = { r0: {} } + const httpdConf = { h0: {} } + const httpdOauthConf = { + h0: { + method: 'google', + credentials: { client: { id: 'i', secret: 's' } }, + users: [ + { email: readonlyUser }, + { email: siteOperatorUser, write: true } + ] + } + } + const authConf = require('../../config/facs/auth.config.json') + + fs.writeFileSync(`./${baseDir}/config/common.json`, JSON.stringify(commonConf)) + fs.writeFileSync(`./${baseDir}/config/facs/net.config.json`, JSON.stringify(netConf)) + fs.writeFileSync(`./${baseDir}/config/facs/httpd.config.json`, JSON.stringify(httpdConf)) + fs.writeFileSync(`./${baseDir}/config/facs/httpd-oauth2.config.json`, JSON.stringify(httpdOauthConf)) + fs.writeFileSync(`./${baseDir}/config/facs/auth.config.json`, JSON.stringify(authConf)) + } + + const startWorker = async () => { + worker = createWorker({ + env: 'test', + wtype: 'wrk-node-http-test', + rack: 'test-rack', + tmpdir: baseDir, + storeDir: 'test-store', + serviceRoot: `${process.cwd()}/${baseDir}`, + port: appNodePort + }) + + await worker.start() + worker.worker.net_r0.jRequest = (publicKey, method) => { + if (method === 'listThings') { + return Promise.resolve(mockMiners) + } + if (method === 'getWrkExtData') { + return Promise.resolve([]) + } + return Promise.resolve({}) + } + } + + const createHttpClient = async () => { + httpClient = new HttpFacility({}, { ns: 'c0', timeout: 30000, debug: false }, { env: 'test' }) + await httpClient.start() + } + + const getTestToken = async (email) => { + worker.worker.authLib._auth.addHandlers({ + google: () => { return { email } } + }) + const token = await worker.worker.auth_a0.authCallbackHandler('google', { ip }) + return token + } + + const createAuthHeaders = async (userEmail) => { + const token = await getTestToken(userEmail) + return { Authorization: `Bearer ${token}` } + } + + createConfig() + await startWorker() + await createHttpClient() + await sleep(2000) + + const minersApi = `${appNodeBaseUrl}${ENDPOINTS.MINERS}` + + // --- Auth security tests --- + + await main.test('Api: miners - auth security', async (n) => { + await n.test('should fail for missing auth token', async (t) => { + try { + await httpClient.get(minersApi, { encoding }) + t.fail('Expected error for missing auth token') + } catch (e) { + t.is(e.response.message.includes('ERR_AUTH_FAIL'), true) + } + }) + + await n.test('should fail for invalid auth token', async (t) => { + const headers = { Authorization: `Bearer ${invalidToken}` } + try { + await httpClient.get(minersApi, { headers, encoding }) + t.fail('Expected error for invalid auth token') + } catch (e) { + t.is(e.response.message.includes('ERR_AUTH_FAIL'), true) + } + }) + + await n.test('should fail for readonly user (capCheck requires write)', async (t) => { + const headers = await createAuthHeaders(readonlyUser) + try { + await httpClient.get(minersApi, { headers, encoding }) + t.fail('Expected error for readonly user') + } catch (e) { + t.is(e.response.message.includes('ERR_AUTH_FAIL'), true) + } + }) + + await n.test('should succeed for site operator user (has actions:rw)', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + try { + await httpClient.get(minersApi, { headers, encoding }) + t.pass() + } catch (e) { + t.fail(`Expected success but got: ${e.message || e}`) + } + }) + }) + + // --- Response structure tests --- + + await main.test('Api: miners - response structure', async (n) => { + await n.test('should return paginated response with correct top-level fields', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const { body: data } = await httpClient.get(minersApi, { headers, encoding }) + + t.ok(Array.isArray(data.data), 'data should be an array') + t.ok(typeof data.totalCount === 'number', 'totalCount should be a number') + t.ok(typeof data.offset === 'number', 'offset should be a number') + t.ok(typeof data.limit === 'number', 'limit should be a number') + t.ok(typeof data.hasMore === 'boolean', 'hasMore should be a boolean') + }) + + await n.test('should return all mock miners with default pagination', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const { body: data } = await httpClient.get(minersApi, { headers, encoding }) + + t.is(data.totalCount, 5, 'totalCount should be 5') + t.is(data.data.length, 5, 'data should have 5 items') + t.is(data.offset, 0, 'offset should default to 0') + t.is(data.hasMore, false, 'hasMore should be false when all items fit') + }) + + await n.test('each miner should have clean formatted fields', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const { body: data } = await httpClient.get(minersApi, { headers, encoding }) + const miner = data.data[0] + + t.ok(miner.id, 'should have id') + t.ok(miner.type, 'should have type') + t.ok(miner.model, 'should have model') + t.ok(miner.code, 'should have code') + t.ok(miner.ip, 'should have ip') + t.ok(miner.container, 'should have container') + t.ok(miner.rack, 'should have rack') + t.ok(miner.status !== undefined, 'should have status') + t.ok(typeof miner.hashrate === 'number', 'hashrate should be a number') + t.ok(typeof miner.power === 'number', 'power should be a number') + t.ok(typeof miner.efficiency === 'number', 'efficiency should be a number') + t.ok(miner.temperature !== undefined, 'should have temperature') + t.ok(miner.firmware, 'should have firmware') + t.ok(miner.powerMode, 'should have powerMode') + t.ok(miner.ledStatus !== undefined, 'should have ledStatus') + t.ok(miner.poolConfig, 'should have poolConfig') + t.ok(miner.lastSeen, 'should have lastSeen') + }) + + await n.test('formatted miner should have correct values from raw data', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const { body: data } = await httpClient.get(minersApi, { headers, encoding }) + + const miner = data.data.find(m => m.id === 'miner-001') + t.ok(miner, 'should find miner-001') + t.is(miner.id, 'miner-001') + t.is(miner.type, 'antminer-s19') + t.is(miner.model, 'Antminer S19 XP') + t.is(miner.code, 'M001') + t.is(miner.ip, '192.168.1.100') + t.is(miner.container, 'container-A') + t.is(miner.rack, 'rack-1') + t.is(miner.status, 'online') + t.is(miner.hashrate, 140000) + t.is(miner.power, 3010) + t.is(miner.efficiency, 21.5) + t.is(miner.temperature, 65) + t.is(miner.firmware, '2024.01.01') + t.is(miner.powerMode, 'normal') + }) + }) + + // --- Pagination tests --- + + await main.test('Api: miners - pagination', async (n) => { + await n.test('should respect limit parameter', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?limit=2` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.is(data.data.length, 2, 'should return only 2 items') + t.is(data.totalCount, 5, 'totalCount should still be 5') + t.is(data.limit, 2, 'limit should be 2') + t.is(data.hasMore, true, 'hasMore should be true') + }) + + await n.test('should respect offset parameter', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?offset=3&limit=10` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.is(data.data.length, 2, 'should return remaining 2 items') + t.is(data.totalCount, 5, 'totalCount should still be 5') + t.is(data.offset, 3, 'offset should be 3') + t.is(data.hasMore, false, 'hasMore should be false') + }) + + await n.test('should return empty page when offset exceeds total', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?offset=100` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.is(data.data.length, 0, 'should return 0 items') + t.is(data.totalCount, 5, 'totalCount should still be 5') + t.is(data.hasMore, false, 'hasMore should be false') + }) + + await n.test('should cap limit at MAX_LIMIT (200)', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?limit=500` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.ok(data.limit <= 200, 'limit should be capped at 200') + }) + }) + + // --- Filter tests --- + // Note: Filtering is done on the ork side via mingo. The mock RPC returns + // all miners regardless of query. These tests verify the API accepts filter + // params correctly and returns valid responses. Filter logic is covered + // by unit tests in miners.handlers.test.js and queryUtils.test.js. + + await main.test('Api: miners - filtering', async (n) => { + await n.test('should accept filter param and return valid response', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const filter = JSON.stringify({ status: 'online' }) + const api = `${minersApi}?filter=${encodeURIComponent(filter)}` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.ok(Array.isArray(data.data), 'should return data array') + t.ok(typeof data.totalCount === 'number', 'should have totalCount') + }) + + await n.test('should accept $or filter without error', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const filter = JSON.stringify({ $or: [{ status: 'error' }, { status: 'sleep' }] }) + const api = `${minersApi}?filter=${encodeURIComponent(filter)}` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.ok(Array.isArray(data.data), 'should return data array') + }) + + await n.test('should return error for invalid filter JSON', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?filter=not-valid-json` + try { + await httpClient.get(api, { headers, encoding }) + t.fail('Expected error for invalid JSON') + } catch (e) { + t.ok(e.response.message.includes('ERR_FILTER_INVALID_JSON'), 'should return filter JSON error') + } + }) + }) + + // --- Sort tests --- + + await main.test('Api: miners - sorting', async (n) => { + await n.test('should sort by hashrate descending', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const sort = JSON.stringify({ hashrate: -1 }) + const api = `${minersApi}?sort=${encodeURIComponent(sort)}` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.ok(data.data.length > 1, 'should return multiple items') + for (let i = 1; i < data.data.length; i++) { + t.ok( + data.data[i - 1].hashrate >= data.data[i].hashrate, + `hashrate should be descending: ${data.data[i - 1].hashrate} >= ${data.data[i].hashrate}` + ) + } + }) + + await n.test('should sort by hashrate ascending', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const sort = JSON.stringify({ hashrate: 1 }) + const api = `${minersApi}?sort=${encodeURIComponent(sort)}` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.ok(data.data.length > 1, 'should return multiple items') + for (let i = 1; i < data.data.length; i++) { + t.ok( + data.data[i - 1].hashrate <= data.data[i].hashrate, + `hashrate should be ascending: ${data.data[i - 1].hashrate} <= ${data.data[i].hashrate}` + ) + } + }) + + await n.test('should return error for invalid sort JSON', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?sort=not-valid-json` + try { + await httpClient.get(api, { headers, encoding }) + t.fail('Expected error for invalid JSON') + } catch (e) { + t.ok(e.response.message.includes('ERR_SORT_INVALID_JSON'), 'should return sort JSON error') + } + }) + }) + + // --- Search tests --- + // Note: Search query is built into the RPC payload and executed on the ork. + // The mock returns all miners regardless. These tests verify the API + // accepts search params and the query is correctly built (unit-tested). + + await main.test('Api: miners - search', async (n) => { + await n.test('should accept search param and return valid response', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?search=192.168` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.ok(Array.isArray(data.data), 'should return data array') + t.ok(typeof data.totalCount === 'number', 'should have totalCount') + }) + + await n.test('should accept search combined with other params', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const sort = JSON.stringify({ hashrate: -1 }) + const api = `${minersApi}?search=miner&sort=${encodeURIComponent(sort)}&limit=3` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.ok(Array.isArray(data.data), 'should return data array') + t.ok(data.data.length <= 3, 'should respect limit') + }) + }) + + // --- Combined query param tests --- + + await main.test('Api: miners - combined query params', async (n) => { + await n.test('should accept filter, sort, and pagination together', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const filter = JSON.stringify({ status: 'online' }) + const sort = JSON.stringify({ hashrate: -1 }) + const api = `${minersApi}?filter=${encodeURIComponent(filter)}&sort=${encodeURIComponent(sort)}&limit=2&offset=0` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.is(data.limit, 2, 'limit should be 2') + t.ok(data.data.length <= 2, 'should return at most 2 items') + if (data.data.length > 1) { + t.ok(data.data[0].hashrate >= data.data[1].hashrate, 'should be sorted by hashrate desc') + } + }) + + await n.test('should accept all query params together without error', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const filter = JSON.stringify({ status: 'online' }) + const sort = JSON.stringify({ temperature: 1 }) + const fields = JSON.stringify({ id: 1, 'last.snap.stats': 1, 'opts.address': 1 }) + const api = `${minersApi}?filter=${encodeURIComponent(filter)}&sort=${encodeURIComponent(sort)}&fields=${encodeURIComponent(fields)}&search=192&limit=3&offset=0` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.ok(Array.isArray(data.data), 'should return data array') + t.ok(data.data.length <= 3, 'should respect limit') + t.is(data.offset, 0, 'offset should be 0') + }) + }) + + // --- overwriteCache tests --- + + await main.test('Api: miners - overwriteCache', async (n) => { + await n.test('should accept overwriteCache=true without error', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?overwriteCache=true` + try { + const { body: data } = await httpClient.get(api, { headers, encoding }) + t.ok(data.data, 'should return data') + t.pass() + } catch (e) { + t.fail(`Expected success but got: ${e.message || e}`) + } + }) + }) + + // --- Pool enrichment tests --- + + await main.test('Api: miners - pool enrichment', async (n) => { + await n.test('should include poolHashrate when poolStats feature is enabled', async (t) => { + worker.worker.conf.featureConfig = { poolStats: true } + + const originalJRequest = worker.worker.net_r0.jRequest + worker.worker.net_r0.jRequest = (publicKey, method) => { + if (method === 'listThings') { + return Promise.resolve(mockMiners) + } + if (method === 'getWrkExtData') { + return Promise.resolve([{ + workers: { + 'miner-001': { hashrate: 139500 }, + M002: { hashrate: 134000 } + } + }]) + } + return Promise.resolve({}) + } + + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?overwriteCache=true` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + const miner1 = data.data.find(m => m.id === 'miner-001') + t.ok(miner1, 'should find miner-001') + t.is(miner1.poolHashrate, 139500, 'miner-001 should have poolHashrate from pool data') + + const miner2 = data.data.find(m => m.id === 'miner-002') + t.ok(miner2, 'should find miner-002') + t.is(miner2.poolHashrate, 134000, 'miner-002 should have poolHashrate matched by code') + + worker.worker.conf.featureConfig = {} + worker.worker.net_r0.jRequest = originalJRequest + }) + + await n.test('should not include poolHashrate when poolStats feature is disabled', async (t) => { + worker.worker.conf.featureConfig = {} + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?overwriteCache=true` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + const miner = data.data[0] + t.is(miner.poolHashrate, undefined, 'poolHashrate should not be present') + }) + }) + + // --- Error/edge case handling --- + + await main.test('Api: miners - edge cases', async (n) => { + await n.test('should handle empty RPC response gracefully', async (t) => { + const originalJRequest = worker.worker.net_r0.jRequest + worker.worker.net_r0.jRequest = (publicKey, method) => { + if (method === 'listThings') return Promise.resolve([]) + return Promise.resolve({}) + } + + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?overwriteCache=true` + const { body: data } = await httpClient.get(api, { headers, encoding }) + + t.is(data.data.length, 0, 'should return empty data array') + t.is(data.totalCount, 0, 'totalCount should be 0') + t.is(data.hasMore, false, 'hasMore should be false') + + worker.worker.net_r0.jRequest = originalJRequest + }) + + await n.test('should return error for invalid fields JSON', async (t) => { + const headers = await createAuthHeaders(siteOperatorUser) + const api = `${minersApi}?fields=not-valid-json` + try { + await httpClient.get(api, { headers, encoding }) + t.fail('Expected error for invalid JSON') + } catch (e) { + t.ok(e.response.message.includes('ERR_FIELDS_INVALID_JSON'), 'should return fields JSON error') + } + }) + }) +}) From 2cf880bafc941a959dc4a5e3fbd11754b05e1c16 Mon Sep 17 00:00:00 2001 From: paragmore Date: Thu, 19 Feb 2026 18:29:18 +0530 Subject: [PATCH 3/4] Update tests --- tests/integration/api.miners.test.js | 12 +- tests/unit/handlers/miners.handlers.test.js | 233 +++++++++++++++++++- 2 files changed, 235 insertions(+), 10 deletions(-) diff --git a/tests/integration/api.miners.test.js b/tests/integration/api.miners.test.js index f6e0cf5..5805ace 100644 --- a/tests/integration/api.miners.test.js +++ b/tests/integration/api.miners.test.js @@ -603,13 +603,23 @@ test('Miners API', { timeout: 90000 }, async (main) => { const headers = await createAuthHeaders(siteOperatorUser) const filter = JSON.stringify({ status: 'online' }) const sort = JSON.stringify({ temperature: 1 }) - const fields = JSON.stringify({ id: 1, 'last.snap.stats': 1, 'opts.address': 1 }) + const fields = JSON.stringify({ status: 1, ip: 1, temperature: 1 }) const api = `${minersApi}?filter=${encodeURIComponent(filter)}&sort=${encodeURIComponent(sort)}&fields=${encodeURIComponent(fields)}&search=192&limit=3&offset=0` const { body: data } = await httpClient.get(api, { headers, encoding }) t.ok(Array.isArray(data.data), 'should return data array') t.ok(data.data.length <= 3, 'should respect limit') t.is(data.offset, 0, 'offset should be 0') + // Verify projection: only requested fields + id should be present + if (data.data.length > 0) { + const miner = data.data[0] + t.ok(miner.id, 'should always include id') + t.ok(miner.status !== undefined, 'should include requested field: status') + t.ok(miner.ip !== undefined, 'should include requested field: ip') + t.is(miner.hashrate, undefined, 'should exclude non-requested field: hashrate') + t.is(miner.power, undefined, 'should exclude non-requested field: power') + t.is(miner.firmware, undefined, 'should exclude non-requested field: firmware') + } }) }) diff --git a/tests/unit/handlers/miners.handlers.test.js b/tests/unit/handlers/miners.handlers.test.js index 7a37fed..11adf9b 100644 --- a/tests/unit/handlers/miners.handlers.test.js +++ b/tests/unit/handlers/miners.handlers.test.js @@ -5,8 +5,12 @@ const { listMiners, formatMiner, extractPoolWorkers, - MINER_FIELD_MAP + buildOrkProjection } = require('../../../workers/lib/server/handlers/miners.handlers') +const { + MINER_FIELD_MAP, + MINER_PROJECTION_MAP +} = require('../../../workers/lib/constants') function createMockMiner (overrides = {}) { return { @@ -217,7 +221,7 @@ test('listMiners - enforces max limit of 200', async (t) => { }) test('listMiners - parses filter JSON', async (t) => { - let capturedPayload = null + const capturedPayloads = [] const ctx = { conf: { orks: [{ rpcPublicKey: 'key1' }], @@ -225,7 +229,7 @@ test('listMiners - parses filter JSON', async (t) => { }, net_r0: { jRequest: async (key, method, payload) => { - capturedPayload = payload + capturedPayloads.push(payload) return [] } } @@ -234,14 +238,16 @@ test('listMiners - parses filter JSON', async (t) => { await listMiners(ctx, req) - t.ok(capturedPayload.query.$and) - t.is(capturedPayload.query.$and[0].tags.$in[0], 't-miner') - t.is(capturedPayload.query.$and[1]['last.snap.stats.status'], 'error') + // Both data and count payloads share the same query + const dataPayload = capturedPayloads.find(p => p.limit !== undefined) + t.ok(dataPayload.query.$and) + t.is(dataPayload.query.$and[0].tags.$in[0], 't-miner') + t.is(dataPayload.query.$and[1]['last.snap.stats.status'], 'error') t.pass() }) test('listMiners - builds search query', async (t) => { - let capturedPayload = null + const capturedPayloads = [] const ctx = { conf: { orks: [{ rpcPublicKey: 'key1' }], @@ -249,7 +255,7 @@ test('listMiners - builds search query', async (t) => { }, net_r0: { jRequest: async (key, method, payload) => { - capturedPayload = payload + capturedPayloads.push(payload) return [] } } @@ -258,7 +264,8 @@ test('listMiners - builds search query', async (t) => { await listMiners(ctx, req) - const lastCondition = capturedPayload.query.$and[capturedPayload.query.$and.length - 1] + const dataPayload = capturedPayloads.find(p => p.limit !== undefined) + const lastCondition = dataPayload.query.$and[dataPayload.query.$and.length - 1] t.ok(lastCondition.$or) t.ok(lastCondition.$or.some(c => c.id?.$regex === '192.168')) t.ok(lastCondition.$or.some(c => c['opts.address']?.$regex === '192.168')) @@ -310,3 +317,211 @@ test('MINER_FIELD_MAP - has expected field mappings', (t) => { t.is(MINER_FIELD_MAP.model, 'last.snap.model') t.pass() }) + +test('listMiners - sends limit (offset+limit) to ork data query', async (t) => { + const capturedPayloads = [] + const ctx = { + conf: { + orks: [{ rpcPublicKey: 'key1' }], + featureConfig: {} + }, + net_r0: { + jRequest: async (key, method, payload) => { + capturedPayloads.push(payload) + return [] + } + } + } + const req = { query: { offset: 10, limit: 25 } } + + await listMiners(ctx, req) + + // Data payload should have limit = offset + limit = 35 + const dataPayload = capturedPayloads.find(p => p.limit !== undefined) + t.is(dataPayload.limit, 35) + t.pass() +}) + +test('listMiners - sends lightweight count query with id-only projection', async (t) => { + const capturedPayloads = [] + const ctx = { + conf: { + orks: [{ rpcPublicKey: 'key1' }], + featureConfig: {} + }, + net_r0: { + jRequest: async (key, method, payload) => { + capturedPayloads.push(payload) + return [] + } + } + } + const req = { query: {} } + + await listMiners(ctx, req) + + // Count payload should have fields: { id: 1 } and no limit + const countPayload = capturedPayloads.find(p => p.limit === undefined) + t.ok(countPayload) + t.alike(countPayload.fields, { id: 1 }) + t.is(countPayload.sort, undefined) + t.pass() +}) + +test('listMiners - totalCount reflects all matching items, not just overfetched page', async (t) => { + // Simulate: 50 miners exist, but user requests offset=0, limit=5 → overfetch 5 from ork + // The count query returns all 50, data query returns 5 (mock returns all, but real ork would limit) + const allMiners = Array.from({ length: 50 }, (_, i) => + createMockMiner({ id: `miner-${i}`, code: `A${i}` }) + ) + const ctx = createMockCtx(allMiners) + const req = { query: { offset: 0, limit: 5 } } + + const result = await listMiners(ctx, req) + + // totalCount should be 50 (from count query), data should be 5 (sliced) + t.is(result.totalCount, 50) + t.is(result.data.length, 5) + t.is(result.hasMore, true) + t.pass() +}) + +test('listMiners - makes two listThings RPC calls (data + count)', async (t) => { + let callCount = 0 + const ctx = { + conf: { + orks: [{ rpcPublicKey: 'key1' }], + featureConfig: {} + }, + net_r0: { + jRequest: async (key, method, payload) => { + if (method === 'listThings') callCount++ + return [] + } + } + } + const req = { query: {} } + + await listMiners(ctx, req) + + t.is(callCount, 2) + t.pass() +}) + +// --- Projection: clean field names --- + +test('buildOrkProjection - maps clean names to internal paths', (t) => { + const result = buildOrkProjection({ firmware: 1, ip: 1 }) + + t.is(result.id, 1, 'always includes id') + t.is(result.code, 1, 'always includes code') + t.is(result['last.snap.config.firmware_ver'], 1, 'maps firmware') + t.is(result['opts.address'], 1, 'maps ip') + t.is(result['last.snap.stats.hashrate_mhs'], undefined, 'does not include unrequested fields') + t.pass() +}) + +test('buildOrkProjection - includes sort fields for app-side sorting', (t) => { + const userFields = { status: 1 } + const mappedSort = { 'last.snap.stats.hashrate_mhs': -1 } + const result = buildOrkProjection(userFields, mappedSort) + + t.is(result['last.snap.stats.status'], 1, 'maps requested field') + t.is(result['last.snap.stats.hashrate_mhs'], 1, 'includes sort field') + t.pass() +}) + +test('buildOrkProjection - handles multi-path fields (model needs snap.model + type)', (t) => { + const result = buildOrkProjection({ model: 1 }) + + t.is(result['last.snap.model'], 1, 'includes primary path') + t.is(result.type, 1, 'includes fallback path') + t.pass() +}) + +test('buildOrkProjection - passes through unknown field names as-is', (t) => { + const result = buildOrkProjection({ 'some.custom.path': 1 }) + + t.is(result['some.custom.path'], 1, 'passes through raw path') + t.pass() +}) + +test('MINER_PROJECTION_MAP - covers all response fields', (t) => { + const expectedFields = [ + 'id', 'type', 'model', 'code', 'ip', 'container', 'rack', 'position', + 'status', 'hashrate', 'power', 'temperature', 'efficiency', 'uptime', + 'firmware', 'powerMode', 'ledStatus', 'poolConfig', 'alerts', + 'comments', 'serialNum', 'macAddress', 'lastSeen' + ] + for (const field of expectedFields) { + t.ok(MINER_PROJECTION_MAP[field], `should have mapping for ${field}`) + t.ok(Array.isArray(MINER_PROJECTION_MAP[field]), `${field} should be an array of paths`) + } + t.pass() +}) + +test('formatMiner - only includes requested fields when projection specified', (t) => { + const raw = createMockMiner() + const requestedFields = new Set(['firmware', 'ip', 'status']) + const result = formatMiner(raw, null, requestedFields) + + t.is(result.id, 't-miner-antminer-192-168-1-101', 'always includes id') + t.is(result.firmware, '2024.01.15', 'includes requested firmware') + t.is(result.ip, '192.168.1.101', 'includes requested ip') + t.is(result.status, 'mining', 'includes requested status') + t.is(result.hashrate, undefined, 'excludes unrequested hashrate') + t.is(result.power, undefined, 'excludes unrequested power') + t.is(result.efficiency, undefined, 'excludes unrequested efficiency') + t.is(result.model, undefined, 'excludes unrequested model') + t.is(result.container, undefined, 'excludes unrequested container') + t.pass() +}) + +test('formatMiner - returns all fields when no projection (null)', (t) => { + const raw = createMockMiner() + const result = formatMiner(raw, null, null) + + t.ok(result.hashrate !== undefined, 'includes hashrate') + t.ok(result.power !== undefined, 'includes power') + t.ok(result.efficiency !== undefined, 'includes efficiency') + t.ok(result.firmware !== undefined, 'includes firmware') + t.ok(result.model !== undefined, 'includes model') + t.pass() +}) + +test('listMiners - maps user fields to ork projection and filters response', async (t) => { + const capturedPayloads = [] + const miners = [createMockMiner()] + const ctx = { + conf: { + orks: [{ rpcPublicKey: 'key1' }], + featureConfig: {} + }, + net_r0: { + jRequest: async (key, method, payload) => { + capturedPayloads.push(payload) + return miners + } + } + } + const req = { query: { fields: '{"firmware":1,"ip":1}' } } + + const result = await listMiners(ctx, req) + + // Check ork projection was mapped correctly + const dataPayload = capturedPayloads.find(p => p.limit !== undefined) + t.is(dataPayload.fields['last.snap.config.firmware_ver'], 1, 'maps firmware to ork path') + t.is(dataPayload.fields['opts.address'], 1, 'maps ip to ork path') + t.is(dataPayload.fields.id, 1, 'always includes id') + t.is(dataPayload.fields.code, 1, 'always includes code') + + // Check response only has requested fields + const miner = result.data[0] + t.ok(miner.id, 'always includes id in response') + t.ok(miner.firmware, 'includes requested firmware') + t.ok(miner.ip, 'includes requested ip') + t.is(miner.hashrate, undefined, 'excludes unrequested hashrate') + t.is(miner.power, undefined, 'excludes unrequested power') + t.is(miner.efficiency, undefined, 'excludes unrequested efficiency') + t.pass() +}) From c386647d48a940298cda1fde0b68e6c0f7a69339 Mon Sep 17 00:00:00 2001 From: paragmore Date: Thu, 19 Feb 2026 18:29:38 +0530 Subject: [PATCH 4/4] fix projections and pagination --- workers/lib/constants.js | 84 ++++++- .../lib/server/handlers/miners.handlers.js | 223 +++++++----------- workers/lib/server/index.js | 4 +- workers/lib/utils.js | 38 +++ 4 files changed, 214 insertions(+), 135 deletions(-) diff --git a/workers/lib/constants.js b/workers/lib/constants.js index b6767bb..f2d5ea1 100644 --- a/workers/lib/constants.js +++ b/workers/lib/constants.js @@ -269,6 +269,82 @@ const RPC_TIMEOUT = 15000 const RPC_CONCURRENCY_LIMIT = 2 const RPC_PAGE_LIMIT = 100 +const MINER_FIELD_MAP = { + status: 'last.snap.stats.status', + hashrate: 'last.snap.stats.hashrate_mhs', + power: 'last.snap.stats.power_w', + efficiency: 'last.snap.stats.efficiency_w_ths', + temperature: 'last.snap.stats.temperature_c', + powerMode: 'last.snap.config.power_mode', + firmware: 'last.snap.config.firmware_ver', + model: 'last.snap.model', + ip: 'opts.address', + container: 'info.container', + rack: 'rack', + serialNum: 'info.serialNum', + macAddress: 'info.macAddress', + pool: 'last.snap.config.pool_config.url', + led: 'last.snap.config.led_status', + alerts: 'last.alerts' +} + +const MINER_PROJECTION_MAP = { + id: ['id'], + type: ['type'], + model: ['last.snap.model', 'type'], + code: ['code'], + ip: ['opts.address'], + container: ['info.container'], + rack: ['rack'], + position: ['info.pos'], + status: ['last.snap.stats.status'], + hashrate: ['last.snap.stats.hashrate_mhs'], + power: ['last.snap.stats.power_w'], + temperature: ['last.snap.stats.temperature_c'], + efficiency: ['last.snap.stats.efficiency_w_ths'], + uptime: ['last.uptime'], + firmware: ['last.snap.config.firmware_ver'], + powerMode: ['last.snap.config.power_mode'], + ledStatus: ['last.snap.config.led_status'], + poolConfig: ['last.snap.config.pool_config'], + alerts: ['last.alerts'], + comments: ['comments'], + serialNum: ['info.serialNum'], + macAddress: ['info.macAddress'], + lastSeen: ['last.ts', 'ts'] +} + +const MINER_SEARCH_FIELDS = [ + 'id', + 'opts.address', + 'info.serialNum', + 'info.macAddress', + 'info.container', + 'code', + 'type' +] + +const MINER_DEFAULT_FIELDS = { + id: 1, + type: 1, + code: 1, + info: 1, + tags: 1, + rack: 1, + comments: 1, + 'last.alerts': 1, + 'last.snap.stats': 1, + 'last.snap.config': 1, + 'last.snap.model': 1, + 'last.uptime': 1, + 'last.ts': 1, + 'opts.address': 1, + ts: 1 +} + +const MINER_MAX_LIMIT = 200 +const MINER_DEFAULT_LIMIT = 50 + module.exports = { SUPER_ADMIN_ROLE, GLOBAL_DATA_TYPES, @@ -301,5 +377,11 @@ module.exports = { MINERPOOL_EXT_DATA_KEYS, NON_METRIC_KEYS, BTC_SATS, - RANGE_BUCKETS + RANGE_BUCKETS, + MINER_FIELD_MAP, + MINER_PROJECTION_MAP, + MINER_SEARCH_FIELDS, + MINER_DEFAULT_FIELDS, + MINER_MAX_LIMIT, + MINER_DEFAULT_LIMIT } diff --git a/workers/lib/server/handlers/miners.handlers.js b/workers/lib/server/handlers/miners.handlers.js index 52fdc6d..f655ced 100644 --- a/workers/lib/server/handlers/miners.handlers.js +++ b/workers/lib/server/handlers/miners.handlers.js @@ -1,115 +1,83 @@ 'use strict' -const { parseJsonQueryParam, requestRpcMapLimit } = require('../../utils') +const { parseJsonQueryParam, requestRpcMapLimit, requestRpcMapBatchLimit } = require('../../utils') +const { + RPC_PAGE_LIMIT, + MINER_FIELD_MAP, + MINER_PROJECTION_MAP, + MINER_SEARCH_FIELDS, + MINER_DEFAULT_FIELDS, + MINER_MAX_LIMIT, + MINER_DEFAULT_LIMIT +} = require('../../constants') const { mapFilterFields, mapSortFields, buildSearchQuery, flattenOrkResults, - sortItems, - paginateResults + sortItems } = require('../lib/queryUtils') /** - * Maps clean miner field names to internal dot-paths. - * Users can filter/sort using clean names; unknown keys pass through as-is. + * Builds ork projection from user-requested clean field names. + * Always includes id and code (needed for pool matching). + * Includes sort field paths so app-side sorting works on projected data. */ -const MINER_FIELD_MAP = { - status: 'last.snap.stats.status', - hashrate: 'last.snap.stats.hashrate_mhs', - power: 'last.snap.stats.power_w', - efficiency: 'last.snap.stats.efficiency_w_ths', - temperature: 'last.snap.stats.temperature_c', - powerMode: 'last.snap.config.power_mode', - firmware: 'last.snap.config.firmware_ver', - model: 'last.snap.model', - ip: 'opts.address', - container: 'info.container', - rack: 'rack', - serialNum: 'info.serialNum', - macAddress: 'info.macAddress', - pool: 'last.snap.config.pool_config.url', - led: 'last.snap.config.led_status', - alerts: 'last.alerts' -} +function buildOrkProjection (userFields, mappedSort) { + const projection = { id: 1, code: 1 } + + for (const [field, value] of Object.entries(userFields)) { + if (value !== 1) continue + const paths = MINER_PROJECTION_MAP[field] + if (paths) { + for (const path of paths) { projection[path] = 1 } + } else { + projection[field] = value + } + } -/** - * Internal fields searched when using the `search` query param. - */ -const MINER_SEARCH_FIELDS = [ - 'id', - 'opts.address', - 'info.serialNum', - 'info.macAddress', - 'info.container', - 'code', - 'type' -] + if (mappedSort) { + for (const path of Object.keys(mappedSort)) { + projection[path] = 1 + } + } -/** - * Default field projection when user doesn't specify fields. - * Includes all fields needed for formatMiner. - */ -const MINER_DEFAULT_FIELDS = { - id: 1, - type: 1, - code: 1, - info: 1, - tags: 1, - rack: 1, - comments: 1, - 'last.alerts': 1, - 'last.snap.stats': 1, - 'last.snap.config': 1, - 'last.snap.model': 1, - 'last.uptime': 1, - 'last.ts': 1, - 'opts.address': 1, - ts: 1 + return projection } -const MAX_LIMIT = 200 -const DEFAULT_LIMIT = 50 - -/** - * Transforms a raw miner thing into the clean response format. - * - * @param {Object} raw - Raw miner object from listThings RPC - * @param {Object|null} poolWorkers - Pool worker lookup { minerId: { hashrate } } - * @returns {Object} Clean miner response object - */ -function formatMiner (raw, poolWorkers) { +function formatMiner (raw, poolWorkers, requestedFields) { const snap = raw.last?.snap || {} const stats = snap.stats || {} const config = snap.config || {} - const miner = { - id: raw.id, - type: raw.type, - model: snap.model || raw.type, - code: raw.code, - ip: raw.opts?.address, - container: raw.info?.container, - rack: raw.rack, - position: raw.info?.pos, - status: stats.status, - hashrate: stats.hashrate_mhs || 0, - power: stats.power_w || 0, - temperature: stats.temperature_c, - efficiency: stats.efficiency_w_ths || 0, - uptime: raw.last?.uptime, - firmware: config.firmware_ver, - powerMode: config.power_mode, - ledStatus: config.led_status, - poolConfig: config.pool_config, - alerts: raw.last?.alerts, - comments: raw.comments, - serialNum: raw.info?.serialNum, - macAddress: raw.info?.macAddress, - lastSeen: raw.last?.ts || raw.ts - } - - if (poolWorkers) { + const include = (field) => !requestedFields || requestedFields.has(field) + + const miner = { id: raw.id } + + if (include('type')) miner.type = raw.type + if (include('model')) miner.model = snap.model || raw.type + if (include('code')) miner.code = raw.code + if (include('ip')) miner.ip = raw.opts?.address + if (include('container')) miner.container = raw.info?.container + if (include('rack')) miner.rack = raw.rack + if (include('position')) miner.position = raw.info?.pos + if (include('status')) miner.status = stats.status + if (include('hashrate')) miner.hashrate = stats.hashrate_mhs || 0 + if (include('power')) miner.power = stats.power_w || 0 + if (include('temperature')) miner.temperature = stats.temperature_c + if (include('efficiency')) miner.efficiency = stats.efficiency_w_ths || 0 + if (include('uptime')) miner.uptime = raw.last?.uptime + if (include('firmware')) miner.firmware = config.firmware_ver + if (include('powerMode')) miner.powerMode = config.power_mode + if (include('ledStatus')) miner.ledStatus = config.led_status + if (include('poolConfig')) miner.poolConfig = config.pool_config + if (include('alerts')) miner.alerts = raw.last?.alerts + if (include('comments')) miner.comments = raw.comments + if (include('serialNum')) miner.serialNum = raw.info?.serialNum + if (include('macAddress')) miner.macAddress = raw.info?.macAddress + if (include('lastSeen')) miner.lastSeen = raw.last?.ts || raw.ts + + if (poolWorkers && include('poolHashrate')) { const poolWorker = poolWorkers[raw.id] || poolWorkers[raw.code] if (poolWorker) { miner.poolHashrate = poolWorker.hashrate || 0 @@ -119,13 +87,6 @@ function formatMiner (raw, poolWorkers) { return miner } -/** - * Extracts a worker hashrate lookup from pool data RPC results. - * Builds a map: workerId → { hashrate } - * - * @param {Array} poolDataResults - Array of ork responses from getWrkExtData - * @returns {Object} Worker lookup map - */ function extractPoolWorkers (poolDataResults) { const workers = {} for (const orkResult of poolDataResults) { @@ -140,15 +101,6 @@ function extractPoolWorkers (poolDataResults) { return workers } -/** - * GET /auth/miners - * - * Lists miners with unified filter/sort/search/pagination. - * Auto-injects miner type filter, maps clean field names to internal paths, - * aggregates multi-ork results, and returns a paginated response. - * - * Replaces: GET /auth/list-things?query={"tags":{"$in":["t-miner"]}}&status=1&... - */ async function listMiners (ctx, req) { const userFilter = req.query.filter ? parseJsonQueryParam(req.query.filter, 'ERR_FILTER_INVALID_JSON') @@ -168,25 +120,36 @@ async function listMiners (ctx, req) { ? mapSortFields(parseJsonQueryParam(req.query.sort, 'ERR_SORT_INVALID_JSON'), MINER_FIELD_MAP) : undefined - const fields = req.query.fields + const userFields = req.query.fields ? parseJsonQueryParam(req.query.fields, 'ERR_FIELDS_INVALID_JSON') + : null + + const orkProjection = userFields + ? buildOrkProjection(userFields, mappedSort) : MINER_DEFAULT_FIELDS + const requestedFields = userFields + ? new Set(Object.keys(userFields).filter(k => userFields[k] === 1)) + : null + const offset = req.query.offset || 0 - const limit = Math.min(req.query.limit || DEFAULT_LIMIT, MAX_LIMIT) + const limit = Math.min(req.query.limit || MINER_DEFAULT_LIMIT, MINER_MAX_LIMIT) + const fetchLimit = offset + limit - const rpcPayload = { - query, - fields, - status: 1 - } + const dataPayload = { query, fields: orkProjection, status: 1 } + if (mappedSort) { dataPayload.sort = mappedSort } - if (mappedSort) { - rpcPayload.sort = mappedSort - } + const countPayload = { query, fields: { id: 1 }, status: 1 } + + const [orkResults, countResults] = await Promise.all([ + fetchLimit <= RPC_PAGE_LIMIT + ? requestRpcMapLimit(ctx, 'listThings', { ...dataPayload, limit: fetchLimit }) + : requestRpcMapBatchLimit(ctx, 'listThings', dataPayload, fetchLimit), + requestRpcMapLimit(ctx, 'listThings', countPayload) + ]) - const orkResults = await requestRpcMapLimit(ctx, 'listThings', rpcPayload) let items = flattenOrkResults(orkResults) + const totalCount = flattenOrkResults(countResults).length if (mappedSort) { items = sortItems(items, mappedSort) @@ -200,19 +163,17 @@ async function listMiners (ctx, req) { query: { key: 'stats' } }) poolWorkers = extractPoolWorkers(poolData) - } catch { - // Pool enrichment is optional — don't fail the request - } + } catch { } } - const paginated = paginateResults(items, offset, limit) + const page = items.slice(offset, offset + limit) return { - data: paginated.data.map(miner => formatMiner(miner, poolWorkers)), - totalCount: paginated.totalCount, - offset: paginated.offset, - limit: paginated.limit, - hasMore: paginated.hasMore + data: page.map(miner => formatMiner(miner, poolWorkers, requestedFields)), + totalCount, + offset, + limit, + hasMore: offset + limit < totalCount } } @@ -220,7 +181,5 @@ module.exports = { listMiners, formatMiner, extractPoolWorkers, - MINER_FIELD_MAP, - MINER_SEARCH_FIELDS, - MINER_DEFAULT_FIELDS + buildOrkProjection } diff --git a/workers/lib/server/index.js b/workers/lib/server/index.js index ce28308..9fb9aa8 100644 --- a/workers/lib/server/index.js +++ b/workers/lib/server/index.js @@ -12,7 +12,7 @@ const financeRoutes = require('./routes/finance.routes') const poolsRoutes = require('./routes/pools.routes') const poolManagerRoutes = require('./routes/poolManager.routes') const siteRoutes = require('./routes/site.routes') -const minersRoutes = require("./routes/miners.routes"); +const minersRoutes = require('./routes/miners.routes') /** * Collect all routes into a flat array for server injection. @@ -32,7 +32,7 @@ function routes (ctx) { ...poolsRoutes(ctx), ...poolManagerRoutes(ctx), ...siteRoutes(ctx), - ...minersRoutes(ctx), + ...minersRoutes(ctx) ] } diff --git a/workers/lib/utils.js b/workers/lib/utils.js index cc31980..fc35a81 100644 --- a/workers/lib/utils.js +++ b/workers/lib/utils.js @@ -129,6 +129,43 @@ const requestRpcMapLimit = async (ctx, method, payload) => { }) } +/** + * Fetches up to maxPerOrk items from each ork, batching in pages if needed. + * Use when the total items to fetch may exceed RPC_PAGE_LIMIT per ork. + * + * @param {Object} ctx - Context object + * @param {string} method - RPC method name + * @param {Object} payload - RPC payload (limit/offset will be managed internally) + * @param {number} maxPerOrk - Maximum items to fetch per ork + * @param {number} pageLimit - Items per page (default: RPC_PAGE_LIMIT) + * @returns {Promise} Array of results per ork (up to maxPerOrk each) + */ +const requestRpcMapBatchLimit = async (ctx, method, payload, maxPerOrk, pageLimit = RPC_PAGE_LIMIT) => { + const concurrency = ctx.conf?.rpcConcurrencyLimit || RPC_CONCURRENCY_LIMIT + + return await async.mapLimit(ctx.conf.orks, concurrency, async (store) => { + const allItems = [] + let offset = 0 + + while (allItems.length < maxPerOrk) { + const batchSize = Math.min(pageLimit, maxPerOrk - allItems.length) + const batch = await ctx.net_r0.jRequest( + store.rpcPublicKey, + method, + { ...payload, limit: batchSize, offset }, + { timeout: getRpcTimeout(ctx.conf) } + ) + + if (!Array.isArray(batch) || batch.length === 0) break + allItems.push(...batch) + if (batch.length < batchSize) break + offset += batch.length + } + + return allItems + }) +} + /** * Paginates RPC requests across multiple orks, fetching all pages per ork * @param {Object} ctx - Context object @@ -187,6 +224,7 @@ module.exports = { parseJsonQueryParam, requestRpcEachLimit, requestRpcMapLimit, + requestRpcMapBatchLimit, requestRpcMapAllPages, getStartOfDay, safeDiv,