diff --git a/api/.env.example b/api/.env.example index 09b880e64f..e86f7b785c 100644 --- a/api/.env.example +++ b/api/.env.example @@ -12,6 +12,11 @@ DB_MAX_SOCKETS=512 # Sync tolerance in milliseconds SYNC_TOLERANCE=1000 +# Brotli quality (0-11) for compressed responses, default 6. Higher is smaller but costs CPU +# that is shared with change-request image processing. Only applies when the API compresses +# itself — a reverse proxy that compresses instead is configured on its own side. +COMPRESS_BROTLI_QUALITY=5 + # Maximum `limit` accepted on a POST /query request (enforced for all query identifiers). # Requests above this are rejected with 400. Guards against huge result-set requests. QUERY_MAX_LIMIT=500 diff --git a/api/src/dto/MongoQueryDto.ts b/api/src/dto/MongoQueryDto.ts index fd528db43f..2380521d39 100644 --- a/api/src/dto/MongoQueryDto.ts +++ b/api/src/dto/MongoQueryDto.ts @@ -19,4 +19,9 @@ export class MongoQueryDto { /** Custom field indicating if expired content documents should be included in sync results. * Used during update syncs (APP mode only) so offline clients receive expiry changes on published docs. */ includeExpired?: boolean; + + /** Custom field naming document fields to drop from each returned doc, so a caller that never + * reads a heavy field (e.g. `text`, `fts`) does not pay to download it. Applied server-side + * after the find; the fields the server itself reads are protected by validateQuery. */ + omitFields?: string[]; } diff --git a/api/src/endpoints/query.service.spec.ts b/api/src/endpoints/query.service.spec.ts index 33d0b10318..1dabc354da 100644 --- a/api/src/endpoints/query.service.spec.ts +++ b/api/src/endpoints/query.service.spec.ts @@ -922,6 +922,146 @@ describe("QueryService", () => { expect(res.docs[0]).not.toHaveProperty("fts"); }); + it("omits the caller-nominated fields from returned docs", async () => { + const access = { + [DocType.Post]: ["gp1"], + [DocType.Language]: ["lang-g1"], + } as any; + (permissions.PermissionSystem.accessMapToGroups as jest.Mock).mockReturnValueOnce( + access, + ); + (service as any).languages = [{ _id: "lang-eng", memberOf: ["lang-g1"] }]; + + const query = makeQuery((s) => { + (s as any).type = DocType.Content; + (s as any).parentType = DocType.Post; + }); + (query as any).omitFields = ["fts", "ftsTokenCount", "text", "memberOf"]; + + dbService.executeFindQuery.mockResolvedValueOnce({ + docs: [ + { + _id: "c1", + type: DocType.Content, + status: PublishStatus.Published, + updatedTimeUtc: 5, + memberOf: ["gp1"], + language: "lang-eng", + title: "a title", + text: "

a very long body

", + fts: ["abc:1"], + ftsTokenCount: 3, + }, + ], + }); + + const res = await service.query(query, mockUser); + + expect(res.docs[0]).toHaveProperty("title", "a title"); + expect(res.docs[0]).toHaveProperty("updatedTimeUtc", 5); + expect(res.docs[0]).not.toHaveProperty("text"); + expect(res.docs[0]).not.toHaveProperty("fts"); + expect(res.docs[0]).not.toHaveProperty("ftsTokenCount"); + expect(res.docs[0]).not.toHaveProperty("memberOf"); + }); + + it("omits fields for cms:true responses too", async () => { + const access = { + [DocType.Post]: ["gp1"], + [DocType.Language]: ["lang-g1"], + } as any; + (permissions.PermissionSystem.accessMapToGroups as jest.Mock).mockReturnValueOnce( + access, + ); + (service as any).languages = [{ _id: "lang-eng", memberOf: ["lang-g1"] }]; + + const query = makeQuery((s) => { + (s as any).type = DocType.Content; + (s as any).parentType = DocType.Post; + }); + (query as any).cms = true; + (query as any).omitFields = ["text"]; + + dbService.executeFindQuery.mockResolvedValueOnce({ + docs: [ + { + _id: "c1", + type: DocType.Content, + status: PublishStatus.Draft, + title: "a title", + text: "

a very long body

", + }, + ], + }); + + const res = await service.query(query, mockUser); + + expect(res.docs[0]).toHaveProperty("title", "a title"); + expect(res.docs[0]).not.toHaveProperty("text"); + }); + + it("does not forward omitFields to CouchDB", async () => { + const access = { [DocType.Post]: ["gp1"] } as any; + (permissions.PermissionSystem.accessMapToGroups as jest.Mock).mockReturnValueOnce( + access, + ); + + const query = makeQuery((s) => { + (s as any).type = DocType.Post; + }); + (query as any).omitFields = ["text"]; + + dbService.executeFindQuery.mockResolvedValueOnce({ docs: [] }); + + await service.query(query, mockUser); + + expect(dbService.executeFindQuery.mock.calls[0][0]).not.toHaveProperty("omitFields"); + }); + + it("keeps the expired-content stub intact when omitFields is also set", async () => { + const access = { + [DocType.Post]: ["gp1"], + [DocType.Language]: ["lang-g1"], + } as any; + (permissions.PermissionSystem.accessMapToGroups as jest.Mock).mockReturnValueOnce( + access, + ); + (service as any).languages = [{ _id: "lang-eng", memberOf: ["lang-g1"] }]; + + const query = makeQuery((s) => { + (s as any).type = DocType.Content; + (s as any).parentType = DocType.Post; + }); + (query as any).includeExpired = true; + // memberOf is on the stub's keep-list; the stub wins, so the client can still + // route/prune the doc it is being told to drop. + (query as any).omitFields = ["text", "memberOf"]; + + dbService.executeFindQuery.mockResolvedValueOnce({ + docs: [ + { + _id: "c1", + type: DocType.Content, + status: PublishStatus.Published, + expiryDate: 1, + updatedTimeUtc: 5, + memberOf: ["gp1"], + language: "lang-eng", + title: "secret title", + text: "

secret body

", + }, + ], + }); + + const res = await service.query(query, mockUser); + + expect(res.docs[0]).toEqual( + expect.objectContaining({ _id: "c1", expiryDate: 1, memberOf: ["gp1"] }), + ); + expect(res.docs[0]).not.toHaveProperty("title"); + expect(res.docs[0]).not.toHaveProperty("text"); + }); + it("does NOT strip expired Content for cms:true responses", async () => { const access = { [DocType.Post]: ["gp1"], diff --git a/api/src/endpoints/query.service.ts b/api/src/endpoints/query.service.ts index 73890a17b4..a30e8a0d64 100644 --- a/api/src/endpoints/query.service.ts +++ b/api/src/endpoints/query.service.ts @@ -240,8 +240,11 @@ export class QueryService { viewGroups = userViewGroups[type as DocType] || []; } + const omitFields = Array.isArray(query.omitFields) ? query.omitFields : []; + delete query.cms; delete query.includeExpired; + delete query.omitFields; // For content queries without parentType the per-parentType $or above already injected // memberOf scoping; otherwise apply the single global memberOf filter here. @@ -277,15 +280,33 @@ export class QueryService { // returned purely so the client can prune its stale copy — never to display. Strip the body // so it never crosses the wire. CMS (cms:true / CmsView-validated) responses keep full docs. // See util/stripExpiredContent.ts; the Socket.io base-room emit applies the same projection. - if (!isCms && Array.isArray(result?.docs)) { - result.docs = result.docs.map((doc: any) => - isExpiredContent(doc, now) ? stripExpiredContent(doc) : doc, - ); + // + // The caller's `omitFields` projection runs in the same pass, after executeFindQuery so the + // blockStart/blockEnd cursor is already computed from the full docs. An expired-content stub + // is already minimal, so it needs no further projection. + if ((!isCms || omitFields.length) && Array.isArray(result?.docs)) { + result.docs = result.docs.map((doc: any) => { + if (!isCms && isExpiredContent(doc, now)) return stripExpiredContent(doc); + return omitDocFields(doc, omitFields); + }); } return result; } } +/** + * Drop the caller-nominated fields from a returned doc. Returns the doc untouched when there is + * nothing to omit, so an unprojected query allocates nothing. + */ +function omitDocFields>(doc: T, fields: string[]): T { + if (!fields.length || !doc) return doc; + if (!fields.some((f) => doc[f] !== undefined)) return doc; + + const projected: Record = { ...doc }; + for (const field of fields) delete projected[field]; + return projected as T; +} + /** * Extract memberOf groups from the top-level $and array. * (After expansion, memberOf will always be a condition in the $and array. diff --git a/api/src/main.spec.ts b/api/src/main.spec.ts index 09583b0574..98e041ea64 100644 --- a/api/src/main.spec.ts +++ b/api/src/main.spec.ts @@ -28,6 +28,7 @@ import { PermissionSystem } from "./permissions/permissions.service"; import { upgradeDbSchema } from "./db/db.upgrade"; import { reconcileLanguageTranslationSeeds } from "./db/languageSeedReconciliation"; import { bootstrap } from "./main"; +import { constants } from "zlib"; describe("bootstrap", () => { let mockApp: any; @@ -76,6 +77,31 @@ describe("bootstrap", () => { expect(mockApp.listen).toHaveBeenCalledWith("3000", "0.0.0.0"); }); + it("should register compression with an explicit Brotli quality", async () => { + process.argv = ["node", "main.js"]; + process.env.COMPRESS_BROTLI_QUALITY = "7"; + + await bootstrap(); + + const [, options] = mockApp.register.mock.calls.find( + ([, opts]: [unknown, any]) => opts?.encodings, + ); + expect(options.encodings).toEqual(["br", "gzip", "deflate"]); + expect(options.brotliOptions.params[constants.BROTLI_PARAM_QUALITY]).toBe(7); + }); + + it("should default the Brotli quality above the plugin's own default", async () => { + process.argv = ["node", "main.js"]; + delete process.env.COMPRESS_BROTLI_QUALITY; + + await bootstrap(); + + const [, options] = mockApp.register.mock.calls.find( + ([, opts]: [unknown, any]) => opts?.encodings, + ); + expect(options.brotliOptions.params[constants.BROTLI_PARAM_QUALITY]).toBe(6); + }); + it("should seed and exit when 'seed' argument is provided", async () => { process.argv = ["node", "main.js", "seed"]; // process.exit never returns in reality, so the mock must actually halt bootstrap() here diff --git a/api/src/main.ts b/api/src/main.ts index b5e2ba4386..48266c8ac2 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -7,6 +7,7 @@ import { PermissionSystem } from "./permissions/permissions.service"; import { upgradeDbSchema } from "./db/db.upgrade"; import { ValidationPipe } from "@nestjs/common"; import compress from "@fastify/compress"; +import { constants } from "zlib"; import multipart from "@fastify/multipart"; import { AllExceptionsFilter } from "./exceptions/allExceptions.filter"; import { S3Service } from "./s3/s3.service"; @@ -31,9 +32,18 @@ export async function bootstrap() { }, }); - // Register compression plugin (Brotli/gzip) for the query endpoint + // Register compression plugin (Brotli/gzip) for the query endpoint. The plugin's own Brotli + // default is quality 4; 6 is the knee of the size/CPU curve on realistic /query bodies + // (~7% smaller for ~2ms more, where 11 costs ~200ms). Tune per deployment — the API shares + // CPU with change-request image processing. await app.register(compress, { encodings: ["br", "gzip", "deflate"], + brotliOptions: { + params: { + [constants.BROTLI_PARAM_QUALITY]: + parseInt(process.env.COMPRESS_BROTLI_QUALITY, 10) || 6, + }, + }, }); const dbService = app.get(DbService); diff --git a/api/src/validation/query/validateQuery.spec.ts b/api/src/validation/query/validateQuery.spec.ts index a9ff163a1f..0aba65ed0b 100644 --- a/api/src/validation/query/validateQuery.spec.ts +++ b/api/src/validation/query/validateQuery.spec.ts @@ -154,6 +154,46 @@ describe("validateQuery", () => { }); }); + describe("omitFields (response projection)", () => { + it("accepts a projection of heavy, client-unread fields", () => { + const q: any = validHybridQuery(); + q.omitFields = ["fts", "ftsTokenCount", "text", "memberOf", "_rev"]; + expect(validateQuery(q)).toEqual({ valid: true, error: "" }); + }); + + it("rejects a non-array omitFields", () => { + const q: any = validHybridQuery(); + q.omitFields = "text"; + expect(validateQuery(q).error).toMatch(/'omitFields' must be an array/); + }); + + it("rejects non-string / empty members", () => { + const q1: any = validHybridQuery(); + q1.omitFields = ["text", 7]; + expect(validateQuery(q1).error).toMatch(/'omitFields' must contain non-empty strings/); + const q2: any = validHybridQuery(); + q2.omitFields = [""]; + expect(validateQuery(q2).valid).toBe(false); + }); + + it("rejects omitting a field the server itself reads after the find", () => { + // updatedTimeUtc drives blockStart/blockEnd; the rest drive the expired-content strip. + for (const field of ["_id", "type", "updatedTimeUtc", "status", "expiryDate"]) { + const q: any = validHybridQuery(); + q.omitFields = ["text", field]; + expect(validateQuery(q).error).toMatch( + new RegExp(`'omitFields' may not omit '${field}'`), + ); + } + }); + + it("rejects an implausibly long projection", () => { + const q: any = validHybridQuery(); + q.omitFields = Array.from({ length: 33 }, (_, i) => `field${i}`); + expect(validateQuery(q).error).toMatch(/'omitFields' exceeds maximum length/); + }); + }); + describe("limit cap", () => { it("rejects a limit above the default maximum", () => { const q: any = validHybridQuery(); diff --git a/api/src/validation/query/validateQuery.ts b/api/src/validation/query/validateQuery.ts index 5ff97808c3..70c64530ed 100644 --- a/api/src/validation/query/validateQuery.ts +++ b/api/src/validation/query/validateQuery.ts @@ -89,8 +89,20 @@ const ALLOWED_TOP_LEVEL_KEYS = new Set([ "use_index", "cms", "includeExpired", + "omitFields", ]); +/** + * Fields `omitFields` may never drop, because the server itself reads them after the find: + * `updatedTimeUtc` produces the `blockStart`/`blockEnd` sync cursor (db.service `calcBlockStartEnd`) + * and `_id`/`type`/`status`/`expiryDate` drive the expired-Content strip (util/stripExpiredContent). + * Rejected rather than silently kept, so a client bug surfaces at the boundary. + */ +const OMIT_FIELDS_PROTECTED = new Set(["_id", "type", "updatedTimeUtc", "status", "expiryDate"]); + +/** Bound on `omitFields` length — a projection list far longer than a document is a client bug. */ +const MAX_OMIT_FIELDS = 32; + /** * The `identifier` label is client-supplied and lands in structured logs, so it is * constrained to a known set to bound log cardinality and keep arbitrary client text @@ -113,6 +125,7 @@ export const ALLOWED_IDENTIFIERS = new Set(["sync", "hybridQuery", "ssgDrain", " * - an operator policy (no `$regex` / `$where`; `$elemMatch` only on array fields; * no `null` member in an `$in` / `$nin` / `$all` array — it crashes CouchDB's * `_find` with an unhandled `function_clause`), + * - an `omitFields` check (bounded, never dropping a field the server reads post-find), * - selector depth / clause-count caps, * - a per-request language cap for NON-CMS queries (guards query cost; CMS is exempt as it * syncs all languages). Enforced here, before query.service injects the permission-language @@ -169,6 +182,24 @@ export function validateQuery(query: any, options: ValidateQueryOptions = {}): V return fail("'includeExpired' must be a boolean"); } + // omitFields — optional projection. It can only ever narrow a permission-scoped response, so + // the entries need no allowlist; the checks below only stop it breaking the server's own + // post-find reads. + if (query.omitFields !== undefined) { + if (!Array.isArray(query.omitFields)) return fail("'omitFields' must be an array"); + if (query.omitFields.length > MAX_OMIT_FIELDS) { + return fail(`'omitFields' exceeds maximum length (${MAX_OMIT_FIELDS})`); + } + for (const field of query.omitFields) { + if (typeof field !== "string" || field.length === 0) { + return fail("'omitFields' must contain non-empty strings"); + } + if (OMIT_FIELDS_PROTECTED.has(field)) { + return fail(`'omitFields' may not omit '${field}'`); + } + } + } + // use_index — optional; must be a known Mango index name. if (query.use_index !== undefined) { if (typeof query.use_index !== "string") return fail("'use_index' must be a string"); diff --git a/shared/src/util/HybridQuery/HybridQuery.spec.ts b/shared/src/util/HybridQuery/HybridQuery.spec.ts index 881a7e662f..a9901d2040 100644 --- a/shared/src/util/HybridQuery/HybridQuery.spec.ts +++ b/shared/src/util/HybridQuery/HybridQuery.spec.ts @@ -992,6 +992,27 @@ describe("HybridQuery", () => { }); }); + describe("omitFields forwarding (queryRemote)", () => { + it("forwards omitFields so the API never sends the unread fields", async () => { + postHttpMock.mockResolvedValueOnce({ docs: [] }); + + await queryRemote({ selector: { type: "content" }, omitFields: ["text", "fts"] }); + + const payload = postHttpMock.mock.calls[0]![1] as Record; + expect(payload.omitFields).toEqual(["text", "fts"]); + }); + + it("omits the key when unset or empty, so the API returns full docs", async () => { + postHttpMock.mockResolvedValueOnce({ docs: [] }); + await queryRemote({ selector: { type: "content" } }); + expect("omitFields" in (postHttpMock.mock.calls[0]![1] as object)).toBe(false); + + postHttpMock.mockResolvedValueOnce({ docs: [] }); + await queryRemote({ selector: { type: "content" }, omitFields: [] }); + expect("omitFields" in (postHttpMock.mock.calls[1]![1] as object)).toBe(false); + }); + }); + describe("cutoff threading", () => { // Parameterize across the three content sub-branches so a regression that // hard-codes the cutoff (or reads it at module-load) would fail at least @@ -2667,6 +2688,42 @@ describe("HybridQuery", () => { expect(s1).not.toHaveProperty("text"); }); + it("asks the API to omit the stripped fields instead of downloading them", async () => { + mocks.mangoToDexieMock.mockResolvedValueOnce([]); + postHttpMock.mockResolvedValueOnce({ docs: [] }); + + new HybridQuery(contentQuery, { stripFields: ["text", "fts"] }); + await flush(); + + const payload = postHttpMock.mock.calls[0]![1] as Record; + expect(payload.omitFields).toEqual(["text", "fts"]); + }); + + it("does NOT project when persistOffline needs the full doc for IndexedDB", async () => { + mocks.mangoToDexieMock.mockResolvedValueOnce([]); + postHttpMock.mockResolvedValueOnce({ docs: [] }); + + new HybridQuery(contentQuery, { + stripFields: ["text", "fts"], + persistOffline: true, + }); + await flush(); + + expect("omitFields" in (postHttpMock.mock.calls[0]![1] as object)).toBe(false); + }); + + it("never projects away a field the API reads off the response", async () => { + mocks.mangoToDexieMock.mockResolvedValueOnce([]); + postHttpMock.mockResolvedValueOnce({ docs: [] }); + + // updatedTimeUtc drives the API's blockStart/blockEnd; forwarding it would be rejected. + new HybridQuery(contentQuery, { stripFields: ["text", "updatedTimeUtc", "status"] }); + await flush(); + + const payload = postHttpMock.mock.calls[0]![1] as Record; + expect(payload.omitFields).toEqual(["text"]); + }); + it("default (no stripFields) leaves docs untouched", async () => { mocks.mangoToDexieMock.mockResolvedValueOnce([ { _id: "a", updatedTimeUtc: 5, publishDate: 2000, type: "content", text: "keep" }, diff --git a/shared/src/util/HybridQuery/HybridQuery.ts b/shared/src/util/HybridQuery/HybridQuery.ts index 790bd3e41a..b613f853e8 100644 --- a/shared/src/util/HybridQuery/HybridQuery.ts +++ b/shared/src/util/HybridQuery/HybridQuery.ts @@ -56,6 +56,12 @@ import { OPEN_MIN } from "../../api/sync/utils"; */ export const DEFAULT_REMOTE_QUERY_LIMIT = 500; +/** + * Fields the API reads off each returned doc (sync cursor, expired-content minimization), so it + * rejects a `MangoQuery.omitFields` naming any of them. + */ +const SERVER_REQUIRED_FIELDS = new Set(["_id", "type", "updatedTimeUtc", "status", "expiryDate"]); + let _httpService: HttpReq | undefined; // Test-only registry of live instances, so `_resetHybridQueryForTests()` can dispose @@ -125,6 +131,11 @@ export async function queryRemote(query: MangoQuery): Promise // Opt-in only: without it the API filters expired Content out of the response, hiding // exactly the docs a caller watching for expiry crossings needs to see. if (query.includeExpired === true) payload.includeExpired = true; + // Wire-level projection: fields the caller never reads are dropped server-side rather than + // downloaded and discarded. + if (Array.isArray(query.omitFields) && query.omitFields.length) { + payload.omitFields = query.omitFields; + } const res = await _httpService.post("query", payload as any); return (res?.docs ?? []) as T[]; @@ -240,6 +251,12 @@ export type HybridQueryOptions = { * cache footprint too. {@link cacheStripFields} is the narrower tool: it removes * *additional* fields from the cache **only**, leaving them in `output`. * + * **Also a wire saving.** Unless {@link persistOffline} is on, these fields are additionally + * forwarded to the API as a projection, so they are never downloaded in the first place — for + * a field like a full HTML body that is most of the response. `persistOffline` opts out + * because it writes the response to IndexedDB *before* the strip, and an offline read of a + * projected doc would be missing its body. + * * **Safety.** The merge / sort / dedup / live-update machinery reads only `_id` * and `updatedTimeUtc`, so stripping any other field is correctness-neutral — * but never strip those two. {@link persistOffline} is unaffected: the full, @@ -835,7 +852,9 @@ export class HybridQuery { // allSettled (not all): one query's transient failure must not blank the whole batch — // keep the queries that succeeded. If EVERY query fails we leave `_remote` untouched // (preserving any cache seed, healing on remount), matching the prior behaviour. - const queries = apis.flatMap((api) => planRemoteContentQueries(api)); + const queries = apis + .flatMap((api) => planRemoteContentQueries(api)) + .map((q) => this._withRemoteProjection(q)); const settled = await Promise.allSettled(queries.map((q) => queryRemote(q))); // A rebuild (dep change) or dispose may have superseded this POST while // it was in flight — its result belongs to a dead generation. @@ -995,6 +1014,22 @@ export class HybridQuery { return this._stripFields.length ? docs.map((d) => omitFields(d, this._stripFields)) : docs; } + /** + * Ask the API not to send `_stripFields` at all, since the supplement's docs are stripped at + * ingest anyway. Skipped under `persistOffline`, which writes the response to IndexedDB + * *before* the strip and so needs the full doc (an offline read of a projected doc would be + * missing its body). An explicit `omitFields` on the query wins. + * + * Server-required fields are filtered out rather than forwarded: `stripFields` is a heap + * concern the caller tunes freely, and listing one must not turn into a rejected query. + */ + private _withRemoteProjection(query: MangoQuery): MangoQuery { + if (query.omitFields || this._persistOffline || !this._stripFields.length) return query; + + const omitFields = this._stripFields.filter((f) => !SERVER_REQUIRED_FIELDS.has(f)); + return omitFields.length ? { ...query, omitFields } : query; + } + /** Replace the local contribution (the live emission may have dropped docs). */ private _setLocal(local: T[]): void { local = this._strip(local); diff --git a/shared/src/util/MangoQuery/MangoTypes.ts b/shared/src/util/MangoQuery/MangoTypes.ts index 9fd54462cd..a5ad68c70c 100644 --- a/shared/src/util/MangoQuery/MangoTypes.ts +++ b/shared/src/util/MangoQuery/MangoTypes.ts @@ -45,6 +45,14 @@ export type MangoQuery = { * Not validated server-side against any known set. */ identifier?: string; + /** + * Document fields the API should drop from every returned doc, so a caller that never reads a + * heavy field (`text`, `fts`, …) does not pay to download it. Remote-only — the local Dexie + * read is unaffected. The API rejects the fields it reads itself (`_id`, `type`, + * `updatedTimeUtc`, `status`, `expiryDate`); older API versions ignore the key and return + * full docs. + */ + omitFields?: string[]; }; /** Comparison object { $op: value } */