fix(analytics): pin the matrix reports' date-bucket intent and drop the empty column (#523)

.changeset/matrix-report-date-granularity.md

@@ -0,0 +1,31 @@
+---
+'hotcrm': patch
+---
+
+Record the matrix reports' date-bucket intent and drop the empty `—` column
+(#523). The v9 single-form migration dropped `groupingsAcross[].dateGranularity`
+and never carried it to the dataset dimensions, so every date axis groups by the
+raw timestamp — one column per distinct value. Re-declaring it turns out to be
+blocked on the platform, not on us, and the blockage was measured rather than
+assumed: on the pinned @objectstack 16.1 a bucketed dimension does not bucket the
+axis, it EMPTIES the surface. A granular dimension is refused by
+NativeSQLStrategy, so the query falls to the auto-bridged `executeAggregate`,
+which calls `engine.aggregate()` with no ExecutionContext — sharing then
+composes `id = '__deny_all__'` for every private object (all of ours), and
+`lead_metrics` goes 21 rows → 0. A `Field.datetime()` column additionally buckets
+to a single NULL, because SQLite stores it as epoch millis and 16.1 formats it
+with a bare `strftime()`. Both are fixed in 17.0.0-rc.0 (#3602/#3597 and
+driver-sql's epoch normalisation), so the declarations belong with that upgrade,
+not ahead of it — which is also why #500 could not honour `dateGranularity`.
+
+So this ships what 16.1 can honour and pins the rest: the intended bucket for
+every date dimension (and for each matrix report's axis, including the dedicated
+quarter dimension `pipeline_coverage_by_quarter` needs — `close_date` is shared
+with the revenue trends, which want month) now lives in a new guard,
+`test/dataset-granularity.test.ts`. It fails today if anyone re-declares a bucket
+on 16.x, and flips to demanding every declaration the moment the platform pin
+crosses 17, so the upgrade cannot go green with the migration unfinished.
+`lead_inflow_by_month_source` and `cases_opened_by_day_priority` now exclude
+records with no date, which is what produced the headerless `—` column (the lead
+report loses it: 21 groups → 20, no null bucket). Three report comments that
+described a `dateGranularity` no dataset declares are corrected.

src/datasets/account.dataset.ts

@@ -11,8 +11,8 @@ export const AccountDataset = defineDataset({
   dimensions: [
     { name: 'industry', label: 'Industry', field: 'industry', type: 'string' },
     { name: 'type', label: 'Type', field: 'type', type: 'string' },
-    // No `dateGranularity`: bucketed date dims hit a fail-closed read scope in
-    // the platform's aggregate bridge (see opportunity_metrics.close_date).
+    // No `dateGranularity` (intended: month) — see the note on
+    // opportunity_metrics.close_date for why it cannot be declared on 16.x.
     { name: 'created_at', label: 'Created', field: 'created_at', type: 'date' },
   ],
   measures: [

src/datasets/case.dataset.ts

@@ -17,6 +17,10 @@ export const CaseDataset = defineDataset({
     { name: 'priority', label: 'Priority', field: 'priority', type: 'string' },
     { name: 'origin', label: 'Origin', field: 'origin', type: 'string' },
     { name: 'type', label: 'Type', field: 'type', type: 'string' },
+    // No `dateGranularity` (intended: day, for cases_opened_by_day_priority and
+    // the Service dashboard's inflow trend) — see the note on
+    // opportunity_metrics.close_date. `crm_case.created_date` is a
+    // `Field.datetime()`, so it is blocked by BOTH gaps described there.
     { name: 'created_date', label: 'Created', field: 'created_date', type: 'date' },
   ],
 

src/datasets/lead.dataset.ts

@@ -12,6 +12,10 @@ export const LeadDataset = defineDataset({
     { name: 'status', label: 'Status', field: 'status', type: 'string' },
     { name: 'lead_source', label: 'Source', field: 'lead_source', type: 'string' },
     { name: 'created_at', label: 'Created', field: 'created_at', type: 'date' },
+    // No `dateGranularity` (intended: month, for lead_inflow_by_month_source)
+    // — see the note on opportunity_metrics.close_date for why it cannot be
+    // declared on 16.x. `crm_lead.last_contacted_date` is a `Field.datetime()`,
+    // so it is blocked by BOTH gaps described there.
     { name: 'last_contacted_date', label: 'Last Contacted', field: 'last_contacted_date', type: 'date' },
   ],
   measures: [{ name: 'lead_count', label: 'Leads', aggregate: 'count' }],

src/datasets/opportunity.dataset.ts

@@ -26,11 +26,37 @@ export const OpportunityDataset = defineDataset({
     { name: 'forecast_category', label: 'Forecast Category', field: 'forecast_category', type: 'string' },
     { name: 'type', label: 'Deal Type', field: 'type', type: 'string' },
     { name: 'owner', label: 'Owner', field: 'owner', type: 'lookup' },
-    // NOTE: no `dateGranularity` here — a bucketed date dimension routes the
-    // query through the platform's ObjectQL aggregate bridge, which drops the
-    // caller's ExecutionContext and fail-closes the read scope (id =
-    // '__deny_all__') on @objectstack 16.1, returning zero rows. Trend widgets
-    // group by the raw date until that is fixed upstream.
+    // ─── Why no `dateGranularity` on any date dimension (hotcrm#523) ────
+    // This is the canonical note; the other datasets point here.
+    //
+    // Declaring a bucket on a dataset dimension is the supported mechanism and
+    // IS what these dimensions want (month for the trend widgets, quarter for
+    // pipeline_coverage_by_quarter, day for cases_opened_by_day_priority). It
+    // cannot be declared while the app is pinned to @objectstack 16.x: doing so
+    // makes every affected surface render EMPTY, which is worse than the
+    // un-bucketed columns it would fix. Two independent 16.x gaps, both
+    // verified against the pinned packages, both fixed in 17.0.0-rc.0:
+    //
+    //   1. Read scope. A granular dimension is refused by NativeSQLStrategy
+    //      (service-analytics `canHandle` bails on any timeDimension carrying a
+    //      granularity), so the query falls through to ObjectQLStrategy → the
+    //      auto-bridged `executeAggregate`, which calls `engine.aggregate()`
+    //      WITHOUT the caller's ExecutionContext. The sharing middleware then
+    //      sees an empty principal and composes `{ id: '__deny_all__' }` for
+    //      every `sharingModel: 'private'` object — which is all of ours. Fixed
+    //      upstream by #3602/#3597: the bridge threads `context`, and the
+    //      strategy applies the read scope itself.
+    //   2. Bucketing. A `Field.datetime()` column lands in SQLite as INTEGER
+    //      epoch millis, and 16.x buckets with a bare `strftime('%Y-%m', col)`
+    //      — NULL for every row, i.e. a single '—' column. 17.0's driver-sql
+    //      normalises epoch storage before formatting. `Field.date()` columns
+    //      (close_date) are TEXT and bucket correctly even on 16.x, but they
+    //      still hit gap 1.
+    //
+    // The intended bucket per dimension is not lost — it lives in
+    // `test/dataset-granularity.test.ts`, whose guard flips from "must not
+    // declare" to "must declare" the moment this app moves to @objectstack 17.
+    // Until then the trend widgets and matrix reports group by the raw date.
     { name: 'close_date', label: 'Close Date', field: 'close_date', type: 'date' },
     // Cross-object dimension: account industry via the crm_account relationship.
     { name: 'account_industry', label: 'Account Industry', field: 'crm_account.industry', type: 'string' },

src/reports/case.report.ts

@@ -29,15 +29,25 @@ export const SlaPerformanceReport: ReportInput = {
 };
 
 /**
- * Daily case inflow by priority — matrix with day-level bucketing. Support
- * managers use this to spot priority spikes (e.g. a P1 burst on Tuesday) and
- * staff accordingly. Exercises the finest `dateGranularity: 'day'` bucket and
- * its interaction with a small categorical axis.
+ * Daily case inflow by priority — matrix over `created_date`. Support managers
+ * use this to spot priority spikes (e.g. a P1 burst on Tuesday) and staff
+ * accordingly.
+ *
+ * The across axis is NOT bucketed by day yet: `case_metrics` cannot declare
+ * `dateGranularity` while the app is pinned to @objectstack 16.x (see the note
+ * on `opportunity_metrics.close_date`; hotcrm#523). Seeded cases carry a
+ * midnight timestamp, so the raw columns happen to READ as days — but they are
+ * raw timestamps, and a case created mid-afternoon gets its own column. The
+ * day bucket is pinned as intent in `test/dataset-granularity.test.ts`.
  */
 export const CasesOpenedByDayPriorityReport: ReportInput = {
   name: 'cases_opened_by_day_priority',
   label: 'Cases Opened by Priority × Day',
   description: 'Daily case inflow split by priority',
   dataset: 'case_metrics', rows: ['priority'], columns: ['created_date'], values: ['case_count'],
   type: 'matrix',
+  // `created_date` is stamped by the platform, not required by the schema — a
+  // case that reaches the table without one would group into a headerless '—'
+  // column. "Cases opened" needs an open date; exclude the empty bucket.
+  runtimeFilter: { created_date: { $ne: null } },
 };

src/reports/lead.report.ts

@@ -3,16 +3,26 @@
 import type { ReportInput } from '@objectstack/spec/ui';
 
 /**
- * Lead engagement trend — matrix with monthly bucketing on
- * `last_contacted_date` (a user-managed datetime, so it can be set in seed
- * data and varied across months for a realistic demo). Lets a marketing-ops
- * lead spot which channels we're actively working month-over-month.
- * Exercises the `dateGranularity: 'month'` server-side aggregation path.
+ * Lead engagement trend — matrix over `last_contacted_date` (a user-managed
+ * datetime, so it can be set in seed data and varied across months for a
+ * realistic demo). Lets a marketing-ops lead spot which channels we're
+ * actively working month-over-month.
+ *
+ * The across axis is NOT bucketed by month yet: `lead_metrics` cannot declare
+ * `dateGranularity` while the app is pinned to @objectstack 16.x (both gaps
+ * are spelled out on `opportunity_metrics.close_date`; hotcrm#523), so the
+ * columns are one-per-raw-date until the 17.0 upgrade lands. The month bucket
+ * is pinned as intent in `test/dataset-granularity.test.ts`.
  */
 export const LeadInflowByMonthSourceReport: ReportInput = {
   name: 'lead_inflow_by_month_source',
   label: 'Lead Engagement by Month × Source',
   description: 'Contacted-lead volume per month, broken down by acquisition channel',
   dataset: 'lead_metrics', rows: ['lead_source'], columns: ['last_contacted_date'], values: ['lead_count'],
   type: 'matrix',
+  // A never-contacted lead has no engagement month, and grouping it produced a
+  // headerless '—' column next to the real ones. This report counts CONTACTED
+  // leads (see the label/description), so the empty bucket is excluded at the
+  // source rather than rendered as a mystery column.
+  runtimeFilter: { last_contacted_date: { $ne: null } },
 };

src/reports/opportunity.report.ts

@@ -29,8 +29,16 @@ export const WonOpportunitiesByOwnerReport: ReportInput = {
  * Quarterly pipeline coverage — the matrix view that powers the classic
  * sales-ops "pipeline coverage" conversation: each forecast category against
  * the quarter the deal is expected to close in. `close_date` is the across
- * dimension (`columns`); the dataset's `dateGranularity` buckets it into
- * quarters in a single server-side aggregate query.
+ * dimension (`columns`).
+ *
+ * It is NOT bucketed into quarters yet — the comment here used to claim "the
+ * dataset's dateGranularity buckets it into quarters", but no such declaration
+ * exists (it was dropped in the v9 single-form migration and cannot be
+ * restored on @objectstack 16.x; see the note on
+ * `opportunity_metrics.close_date`, hotcrm#523). Restoring it needs a
+ * DEDICATED quarter dimension rather than a bucket on `close_date` itself:
+ * that dimension is shared with the Sales/CRM/Executive revenue trends, which
+ * want month. Both intents are pinned in `test/dataset-granularity.test.ts`.
  */
 export const PipelineCoverageByQuarterReport: ReportInput = {
   name: 'pipeline_coverage_by_quarter',