From 6dca38b26ed0e37fb0ff9139c2033f6f4670471e Mon Sep 17 00:00:00 2001 From: Joaquim d'Souza Date: Sun, 22 Mar 2026 22:55:01 +0100 Subject: [PATCH 1/6] feat: rework column metadata UI in data source dashboard --- AGENTS.md | 4 + ...67146_data_source_column_visualisations.ts | 18 + ...67147_data_source_organisation_override.ts | 28 ++ ...ector_config_boundaries_to_data_sources.ts | 19 + .../data-sources/[id]/DataSourceDashboard.tsx | 22 +- .../[id]/components/ColumnMetadataForm.tsx | 318 ---------------- .../[id]/components/ColumnMetadataTable.tsx | 355 ++++++++++++++++++ .../components/EnrichmentColumnDialog.tsx | 0 .../EnrichmentTable.tsx} | 9 +- .../[id]/components/ColumnMetadataIcons.tsx | 5 +- .../components/EditColumnMetadataModal.tsx | 21 +- .../(private)/map/[id]/components/Legend.tsx | 4 +- .../map/[id]/components/MapViews.tsx | 2 +- .../VisualisationPanel/VisualisationPanel.tsx | 4 +- .../inspector/BoundaryConfigItem.tsx | 6 +- .../inspector/BoundaryDataPanel.tsx | 4 +- .../inspector/DataSourcePropertiesList.tsx | 10 +- .../inspector/InspectorDataConfig.tsx | 31 +- .../components/inspector/InspectorDataTab.tsx | 10 +- .../map/[id]/hooks/useDisplayAreaStats.ts | 2 +- .../map/[id]/hooks/useInitialMapView.ts | 2 +- src/labels.ts | 7 + src/models/ColumnMetadataOverride.ts | 13 - src/models/DataSource.ts | 33 ++ src/models/DataSourceOrganisationOverride.ts | 14 + src/models/MapView.ts | 18 +- src/server/commands/ensureOrganisationMap.ts | 2 + src/server/jobs/importDataRecords.ts | 7 +- src/server/jobs/importDataSource.ts | 86 ++++- src/server/models/ColumnMetadataOverride.ts | 7 - .../models/DataSourceOrganisationOverride.ts | 9 + src/server/repositories/DataRecord.ts | 22 ++ ...e.ts => DataSourceOrganisationOverride.ts} | 24 +- src/server/services/database/index.ts | 4 +- src/server/services/database/schema.ts | 1 + src/server/trpc/routers/dataSource.ts | 27 +- tests/feature/dataSourcesAPI.test.ts | 3 + tests/feature/enrichDataSource.test.ts | 1 + tests/feature/geojsonAPI.test.ts | 1 + tests/feature/importDataSource.test.ts | 103 +++++ tests/feature/stats.test.ts | 1 + tests/resources/percentages.csv | 5 + .../unit/server/jobs/importDataSource.test.ts | 110 ++++++ .../server/repositories/DataRecord.test.ts | 1 + .../server/trpc/routers/dataRecord.test.ts | 1 + .../server/trpc/routers/dataSource.test.ts | 1 + tests/unit/utils/superjson.test.ts | 1 + 47 files changed, 948 insertions(+), 428 deletions(-) create mode 100644 migrations/1774204667146_data_source_column_visualisations.ts create mode 100644 migrations/1774204667147_data_source_organisation_override.ts create mode 100644 migrations/1774206346032_inspector_config_boundaries_to_data_sources.ts delete mode 100644 src/app/(private)/(dashboards)/data-sources/[id]/components/ColumnMetadataForm.tsx create mode 100644 src/app/(private)/(dashboards)/data-sources/[id]/components/ColumnMetadataTable.tsx rename src/app/(private)/(dashboards)/data-sources/{ => [id]}/components/EnrichmentColumnDialog.tsx (100%) rename src/app/(private)/(dashboards)/data-sources/[id]/{DataSourceEnrichmentDashboard.tsx => components/EnrichmentTable.tsx} (98%) delete mode 100644 src/models/ColumnMetadataOverride.ts create mode 100644 src/models/DataSourceOrganisationOverride.ts delete mode 100644 src/server/models/ColumnMetadataOverride.ts create mode 100644 src/server/models/DataSourceOrganisationOverride.ts rename src/server/repositories/{ColumnMetadataOverride.ts => DataSourceOrganisationOverride.ts} (55%) create mode 100644 tests/resources/percentages.csv create mode 100644 tests/unit/server/jobs/importDataSource.test.ts diff --git a/AGENTS.md b/AGENTS.md index fa26e1e3..a3b4bafa 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,10 @@ Use tRPC as the bridge between client components and server logic: - **Server components**: call the server-side tRPC caller from `src/services/trpc/server.tsx` - **API routes**: use for streaming responses or bulk data that would be impractical over tRPC +## Code Style + +- Do not use `!!x` to cast to `boolean`. Use `Boolean(x)`. + ## Database ### Kysely ORM diff --git a/migrations/1774204667146_data_source_column_visualisations.ts b/migrations/1774204667146_data_source_column_visualisations.ts new file mode 100644 index 00000000..7b2e6674 --- /dev/null +++ b/migrations/1774204667146_data_source_column_visualisations.ts @@ -0,0 +1,18 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable("dataSource") + .addColumn("columnVisualisations", "jsonb", (col) => + col.notNull().defaultTo("[]"), + ) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable("dataSource") + .dropColumn("columnVisualisations") + .execute(); +} diff --git a/migrations/1774204667147_data_source_organisation_override.ts b/migrations/1774204667147_data_source_organisation_override.ts new file mode 100644 index 00000000..dfe1c4e1 --- /dev/null +++ b/migrations/1774204667147_data_source_organisation_override.ts @@ -0,0 +1,28 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable("columnMetadataOverride") + .renameTo("dataSourceOrganisationOverride") + .execute(); + + await db.schema + .alterTable("dataSourceOrganisationOverride") + .addColumn("columnVisualisations", "jsonb", (col) => + col.notNull().defaultTo("[]"), + ) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable("dataSourceOrganisationOverride") + .dropColumn("columnVisualisations") + .execute(); + + await db.schema + .alterTable("dataSourceOrganisationOverride") + .renameTo("columnMetadataOverride") + .execute(); +} diff --git a/migrations/1774206346032_inspector_config_boundaries_to_data_sources.ts b/migrations/1774206346032_inspector_config_boundaries_to_data_sources.ts new file mode 100644 index 00000000..5735c209 --- /dev/null +++ b/migrations/1774206346032_inspector_config_boundaries_to_data_sources.ts @@ -0,0 +1,19 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { sql } from "kysely"; +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await sql` + UPDATE map_view + SET inspector_config = (inspector_config - 'boundaries') || jsonb_build_object('dataSources', inspector_config->'boundaries') + WHERE inspector_config ? 'boundaries' + `.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql` + UPDATE map_view + SET inspector_config = (inspector_config - 'dataSources') || jsonb_build_object('boundaries', inspector_config->'dataSources') + WHERE inspector_config ? 'dataSources' + `.execute(db); +} diff --git a/src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx b/src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx index 3649395a..94b3ae58 100644 --- a/src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx +++ b/src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx @@ -1,6 +1,6 @@ "use client"; -import { useMutation, useQuery } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useSubscription } from "@trpc/tanstack-react-query"; import { format, formatDistanceToNow } from "date-fns"; import { @@ -34,9 +34,9 @@ import { import { Button } from "@/shadcn/ui/button"; import { Separator } from "@/shadcn/ui/separator"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/shadcn/ui/tabs"; -import ColumnMetadataForm from "./components/ColumnMetadataForm"; +import ColumnMetadataTable from "./components/ColumnMetadataTable"; import ConfigurationForm from "./components/ConfigurationForm"; -import { DataSourceEnrichmentDashboard } from "./DataSourceEnrichmentDashboard"; +import EnrichmentTable from "./components/EnrichmentTable"; import type { RouterOutputs } from "@/services/trpc/react"; export function DataSourceDashboard({ @@ -67,6 +67,7 @@ export function DataSourceDashboard({ ); const trpc = useTRPC(); + const queryClient = useQueryClient(); // Check webhook status for Google Sheets data sources const { data: webhookStatus } = useQuery( @@ -104,6 +105,11 @@ export function DataSourceDashboard({ if (dataSourceEvent.event === "ImportComplete") { setImporting(false); setLastImported(dataSourceEvent.at); + queryClient.invalidateQueries({ + queryKey: trpc.dataSource.byId.queryKey({ + dataSourceId: dataSource.id, + }), + }); } }, }, @@ -226,7 +232,8 @@ export function DataSourceDashboard({ - Settings + Configuration + Column metadata {showEnrichment && ( Enrichment )} @@ -249,7 +256,6 @@ export function DataSourceDashboard({ : mappedInformation } /> -
@@ -268,9 +274,13 @@ export function DataSourceDashboard({
+ + + + {showEnrichment && ( - + )}
diff --git a/src/app/(private)/(dashboards)/data-sources/[id]/components/ColumnMetadataForm.tsx b/src/app/(private)/(dashboards)/data-sources/[id]/components/ColumnMetadataForm.tsx deleted file mode 100644 index 15bd8f16..00000000 --- a/src/app/(private)/(dashboards)/data-sources/[id]/components/ColumnMetadataForm.tsx +++ /dev/null @@ -1,318 +0,0 @@ -"use client"; - -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Pencil, Plus, Save } from "lucide-react"; -import { useCallback, useMemo, useState } from "react"; -import { toast } from "sonner"; -import { type ColumnMetadata, ColumnType } from "@/models/DataSource"; -import { useTRPC } from "@/services/trpc/react"; -import { Button } from "@/shadcn/ui/button"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, -} from "@/shadcn/ui/dialog"; -import { Input } from "@/shadcn/ui/input"; -import { ScrollArea } from "@/shadcn/ui/scroll-area"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/shadcn/ui/table"; -import { Textarea } from "@/shadcn/ui/textarea"; -import type { RouterOutputs } from "@/services/trpc/react"; - -type DataSource = NonNullable; - -function hasMetadata(col: ColumnMetadata) { - return col.description !== "" || Object.keys(col.valueLabels).length > 0; -} - -export default function ColumnMetadataForm({ - dataSource, -}: { - dataSource: DataSource; -}) { - const initialMetadata = useMemo(() => { - const existing = new Map( - (dataSource.columnMetadata ?? []).map((m) => [m.name, m]), - ); - return dataSource.columnDefs.map((col) => ({ - name: col.name, - valueLabels: existing.get(col.name)?.valueLabels ?? {}, - description: existing.get(col.name)?.description ?? "", - })); - }, [dataSource.columnDefs, dataSource.columnMetadata]); - - const [metadata, setMetadata] = useState(initialMetadata); - const [loading, setLoading] = useState(false); - const [editingIndex, setEditingIndex] = useState(null); - const [draftDescription, setDraftDescription] = useState(""); - const [draftValueLabels, setDraftValueLabels] = useState< - Record - >({}); - - const client = useQueryClient(); - const trpc = useTRPC(); - const editingCol = editingIndex !== null ? metadata[editingIndex] : null; - const { data: rawColumnValues } = useQuery( - trpc.dataSource.uniqueColumnValues.queryOptions( - { dataSourceId: dataSource.id, column: editingCol?.name ?? "" }, - { enabled: editingCol !== null }, - ), - ); - - const editingColumnType = dataSource.columnDefs.find( - (col) => col.name === editingCol?.name, - )?.type; - - // If nullIsZero is true for a numeric column, combine "", "null" and "undefined" into "0" - const columnValues = useMemo(() => { - if (rawColumnValues == null) return rawColumnValues; - if (!dataSource.nullIsZero || editingColumnType !== ColumnType.Number) { - return rawColumnValues; - } - const blankValues = new Set(["", "null", "undefined"]); - const hasBlankOrZero = - rawColumnValues.some((v) => blankValues.has(v)) || - rawColumnValues.includes("0"); - if (!hasBlankOrZero) return rawColumnValues; - const filtered = rawColumnValues.filter((v) => !blankValues.has(v)); - if (!filtered.includes("0")) { - filtered.push("0"); - } - return filtered; - }, [rawColumnValues, dataSource.nullIsZero, editingColumnType]); - const { mutate: updateDataSourceConfig } = useMutation( - trpc.dataSource.updateConfig.mutationOptions({ - onSuccess: () => { - setLoading(false); - toast.success("Column metadata saved."); - client.invalidateQueries({ - queryKey: trpc.dataSource.listReadable.queryKey(), - }); - }, - onError: (error) => { - setLoading(false); - toast.error(error.message || "Could not save column metadata."); - }, - }), - ); - - const openDialog = useCallback( - (index: number) => { - const col = metadata[index]; - setDraftDescription(col.description); - setDraftValueLabels(col.valueLabels); - setEditingIndex(index); - }, - [metadata], - ); - - const closeDialog = useCallback(() => { - setEditingIndex(null); - }, []); - - const saveDialog = useCallback(() => { - if (editingIndex === null) return; - const updated = metadata.map((m, i) => - i === editingIndex - ? { ...m, description: draftDescription, valueLabels: draftValueLabels } - : m, - ); - setMetadata(updated); - setEditingIndex(null); - setLoading(true); - updateDataSourceConfig({ - dataSourceId: dataSource.id, - columnMetadata: updated, - }); - }, [ - editingIndex, - metadata, - draftDescription, - draftValueLabels, - dataSource.id, - updateDataSourceConfig, - ]); - - if (dataSource.columnDefs.length === 0) { - return ( -

- Import data to configure column metadata. -

- ); - } - - return ( -
-

Column metadata

-

- Add labels and descriptions for each column. These are shown to users - viewing your data. -

-
    - {metadata.map((col, index) => { - const hasMeta = hasMetadata(col); - return ( -
  • - {col.name} - -
  • - ); - })} -
- - { - if (!open) closeDialog(); - }} - > - - - - Metadata for {editingCol?.name} - - - Configure a description and value labels for this column. - - - {editingCol && ( -
-
- -