Feat/column metadata rework

AGENTS.md

@@ -83,6 +83,22 @@ Use tRPC as the bridge between client components and server logic:
 - **Server components**: call the server-side tRPC caller from `src/services/trpc/server.tsx`
 - **API routes**: use for streaming responses or bulk data that would be impractical over tRPC
 
+## Code Style
+
+- Do not use `!!x` to cast to `boolean`. Use `Boolean(x)`.
+- Functions with more than 2 parameters should use the destructured input pattern,
+  e.g. `const sum = ({ a, b, c }: { a: number, b: number, c: number }) => a + b + c`
+- Constants should either go in a `constants/` directory or a `constants.ts` file.
+- Utils (pure functions) should go in a `utils/` directory or a `utils.ts` file.
+- Functions with side effects, or async functions, should go in a `services/` directory, or somewhere more specific.
+- Complex components that require their own directory should be in a file with the same name as the directory,
+  e.g. `Choropleth/Choropleth.tsx`.
+
+## English Dialect
+
+- Use American spellings for code, British spellings for user-facing text, comments, and documentation.
+- Use British spellings in code if there is precedent (e.g. "visualisation") for consistency.
+
 ## Database
 
 ### Kysely ORM

migrations/1774204667146_data_source_column_visualisations.ts

@@ -0,0 +1,18 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import type { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable("dataSource")
+    .addColumn("columnVisualisations", "jsonb", (col) =>
+      col.notNull().defaultTo("[]"),
+    )
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable("dataSource")
+    .dropColumn("columnVisualisations")
+    .execute();
+}

migrations/1774204667147_data_source_organisation_override.ts

@@ -0,0 +1,28 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import type { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable("columnMetadataOverride")
+    .renameTo("dataSourceOrganisationOverride")
+    .execute();
+
+  await db.schema
+    .alterTable("dataSourceOrganisationOverride")
+    .addColumn("columnVisualisations", "jsonb", (col) =>
+      col.notNull().defaultTo("[]"),
+    )
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable("dataSourceOrganisationOverride")
+    .dropColumn("columnVisualisations")
+    .execute();
+
+  await db.schema
+    .alterTable("dataSourceOrganisationOverride")
+    .renameTo("columnMetadataOverride")
+    .execute();
+}

migrations/1774206346032_inspector_config_boundaries_to_data_sources.ts

@@ -0,0 +1,19 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { sql } from "kysely";
+import type { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`
+    UPDATE map_view
+    SET inspector_config = (inspector_config - 'boundaries') || jsonb_build_object('dataSources', inspector_config->'boundaries')
+    WHERE inspector_config ? 'boundaries'
+  `.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`
+    UPDATE map_view
+    SET inspector_config = (inspector_config - 'dataSources') || jsonb_build_object('boundaries', inspector_config->'dataSources')
+    WHERE inspector_config ? 'dataSources'
+  `.execute(db);
+}

migrations/1774206346033_rename_column_visualisations_to_inspector_columns.ts

@@ -0,0 +1,26 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import type { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable("dataSource")
+    .renameColumn("columnVisualisations", "inspectorColumns")
+    .execute();
+
+  await db.schema
+    .alterTable("dataSourceOrganisationOverride")
+    .renameColumn("columnVisualisations", "inspectorColumns")
+    .execute();
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await db.schema
+    .alterTable("dataSourceOrganisationOverride")
+    .renameColumn("inspectorColumns", "columnVisualisations")
+    .execute();
+
+  await db.schema
+    .alterTable("dataSource")
+    .renameColumn("inspectorColumns", "columnVisualisations")
+    .execute();
+}

migrations/1774222928000_rename_map_view_category_colors.ts

@@ -0,0 +1,19 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { sql } from "kysely";
+import type { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`
+    UPDATE "mapView"
+    SET config = jsonb_set(config - 'categoryColors', '{colorMappings}', config->'categoryColors')
+    WHERE config ? 'categoryColors'
+  `.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`
+    UPDATE "mapView"
+    SET config = jsonb_set(config - 'colorMappings', '{categoryColors}', config->'colorMappings')
+    WHERE config ? 'colorMappings'
+  `.execute(db);
+}

migrations/1774300000000_rename_column_metadata_color_mappings.ts

@@ -0,0 +1,75 @@
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { sql } from "kysely";
+import type { Kysely } from "kysely";
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`
+    UPDATE "dataSource"
+    SET "columnMetadata" = (
+      SELECT jsonb_agg(
+        CASE
+          WHEN col ? 'colorMappings'
+          THEN jsonb_set(col - 'colorMappings', '{valueColors}', col->'colorMappings')
+          ELSE col
+        END
+      )
+      FROM jsonb_array_elements("columnMetadata") AS col
+    )
+    WHERE EXISTS (
+      SELECT 1 FROM jsonb_array_elements("columnMetadata") AS col WHERE col ? 'colorMappings'
+    )
+  `.execute(db);
+
+  await sql`
+    UPDATE "dataSourceOrganisationOverride"
+    SET "columnMetadata" = (
+      SELECT jsonb_agg(
+        CASE
+          WHEN col ? 'colorMappings'
+          THEN jsonb_set(col - 'colorMappings', '{valueColors}', col->'colorMappings')
+          ELSE col
+        END
+      )
+      FROM jsonb_array_elements("columnMetadata") AS col
+    )
+    WHERE EXISTS (
+      SELECT 1 FROM jsonb_array_elements("columnMetadata") AS col WHERE col ? 'colorMappings'
+    )
+  `.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`
+    UPDATE "dataSource"
+    SET "columnMetadata" = (
+      SELECT jsonb_agg(
+        CASE
+          WHEN col ? 'valueColors'
+          THEN jsonb_set(col - 'valueColors', '{colorMappings}', col->'valueColors')
+          ELSE col
+        END
+      )
+      FROM jsonb_array_elements("columnMetadata") AS col
+    )
+    WHERE EXISTS (
+      SELECT 1 FROM jsonb_array_elements("columnMetadata") AS col WHERE col ? 'valueColors'
+    )
+  `.execute(db);
+
+  await sql`
+    UPDATE "dataSourceOrganisationOverride"
+    SET "columnMetadata" = (
+      SELECT jsonb_agg(
+        CASE
+          WHEN col ? 'valueColors'
+          THEN jsonb_set(col - 'valueColors', '{colorMappings}', col->'valueColors')
+          ELSE col
+        END
+      )
+      FROM jsonb_array_elements("columnMetadata") AS col
+    )
+    WHERE EXISTS (
+      SELECT 1 FROM jsonb_array_elements("columnMetadata") AS col WHERE col ? 'valueColors'
+    )
+  `.execute(db);
+}

src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx

@@ -1,6 +1,6 @@
 "use client";
 
-import { useMutation, useQuery } from "@tanstack/react-query";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
 import { useSubscription } from "@trpc/tanstack-react-query";
 import { format, formatDistanceToNow } from "date-fns";
 import {
@@ -34,9 +34,10 @@ import {
 import { Button } from "@/shadcn/ui/button";
 import { Separator } from "@/shadcn/ui/separator";
 import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/shadcn/ui/tabs";
-import ColumnMetadataForm from "./components/ColumnMetadataForm";
+import ColumnMetadataTable from "./components/ColumnMetadataTable";
+import ColumnVisualisationPanel from "./components/ColumnVisualisationPanel";
 import ConfigurationForm from "./components/ConfigurationForm";
-import { DataSourceEnrichmentDashboard } from "./DataSourceEnrichmentDashboard";
+import EnrichmentTable from "./components/EnrichmentTable";
 import type { RouterOutputs } from "@/services/trpc/react";
 
 export function DataSourceDashboard({
@@ -67,6 +68,7 @@ export function DataSourceDashboard({
   );
 
   const trpc = useTRPC();
+  const queryClient = useQueryClient();
 
   // Check webhook status for Google Sheets data sources
   const { data: webhookStatus } = useQuery(
@@ -104,6 +106,11 @@ export function DataSourceDashboard({
           if (dataSourceEvent.event === "ImportComplete") {
             setImporting(false);
             setLastImported(dataSourceEvent.at);
+            queryClient.invalidateQueries({
+              queryKey: trpc.dataSource.byId.queryKey({
+                dataSourceId: dataSource.id,
+              }),
+            });
           }
         },
       },
@@ -150,7 +157,7 @@ export function DataSourceDashboard({
   );
 
   return (
-    <div className="p-4 mx-auto max-w-5xl w-full">
+    <div className="flex flex-col h-full p-4 mx-auto max-w-5xl w-full">
       <div className="flex gap-12">
         <div className="grow">
           <Breadcrumb className="mb-4">
@@ -224,9 +231,11 @@ export function DataSourceDashboard({
         </Alert>
       )}
 
-      <Tabs defaultValue="settings" className="gap-6">
+      <Tabs defaultValue="settings" className="flex-1 min-h-0 gap-6">
         <TabsList>
-          <TabsTrigger value="settings">Settings</TabsTrigger>
+          <TabsTrigger value="settings">Configuration</TabsTrigger>
+          <TabsTrigger value="columns">Column metadata</TabsTrigger>
+          <TabsTrigger value="inspector">Map inspector</TabsTrigger>
           {showEnrichment && (
             <TabsTrigger value="enrichment">Enrichment</TabsTrigger>
           )}
@@ -249,7 +258,6 @@ export function DataSourceDashboard({
                     : mappedInformation
                 }
               />
-              <ColumnMetadataForm dataSource={dataSource} />
             </div>
 
             <div className="flex flex-col gap-6">
@@ -268,9 +276,17 @@ export function DataSourceDashboard({
           </div>
         </TabsContent>
 
+        <TabsContent value="columns">
+          <ColumnMetadataTable dataSource={dataSource} />
+        </TabsContent>
+
+        <TabsContent value="inspector" className="flex flex-col min-h-0">
+          <ColumnVisualisationPanel dataSource={dataSource} />
+        </TabsContent>
+
         {showEnrichment && (
           <TabsContent value="enrichment">
-            <DataSourceEnrichmentDashboard dataSource={dataSource} />
+            <EnrichmentTable dataSource={dataSource} />
           </TabsContent>
         )}
       </Tabs>