diff --git a/AGENTS.md b/AGENTS.md index fa26e1e3..4b40e2ea 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,6 +83,22 @@ Use tRPC as the bridge between client components and server logic: - **Server components**: call the server-side tRPC caller from `src/services/trpc/server.tsx` - **API routes**: use for streaming responses or bulk data that would be impractical over tRPC +## Code Style + +- Do not use `!!x` to cast to `boolean`. Use `Boolean(x)`. +- Functions with more than 2 parameters should use the destructured input pattern, + e.g. `const sum = ({ a, b, c }: { a: number, b: number, c: number }) => a + b + c` +- Constants should either go in a `constants/` directory or a `constants.ts` file. +- Utils (pure functions) should go in a `utils/` directory or a `utils.ts` file. +- Functions with side effects, or async functions, should go in a `services/` directory, or somewhere more specific. +- Complex components that require their own directory should be in a file with the same name as the directory, + e.g. `Choropleth/Choropleth.tsx`. + +## English Dialect + +- Use American spellings for code, British spellings for user-facing text, comments, and documentation. +- Use British spellings in code if there is precedent (e.g. "visualisation") for consistency. + ## Database ### Kysely ORM diff --git a/migrations/1774204667146_data_source_column_visualisations.ts b/migrations/1774204667146_data_source_column_visualisations.ts new file mode 100644 index 00000000..7b2e6674 --- /dev/null +++ b/migrations/1774204667146_data_source_column_visualisations.ts @@ -0,0 +1,18 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable("dataSource") + .addColumn("columnVisualisations", "jsonb", (col) => + col.notNull().defaultTo("[]"), + ) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable("dataSource") + .dropColumn("columnVisualisations") + .execute(); +} diff --git a/migrations/1774204667147_data_source_organisation_override.ts b/migrations/1774204667147_data_source_organisation_override.ts new file mode 100644 index 00000000..dfe1c4e1 --- /dev/null +++ b/migrations/1774204667147_data_source_organisation_override.ts @@ -0,0 +1,28 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable("columnMetadataOverride") + .renameTo("dataSourceOrganisationOverride") + .execute(); + + await db.schema + .alterTable("dataSourceOrganisationOverride") + .addColumn("columnVisualisations", "jsonb", (col) => + col.notNull().defaultTo("[]"), + ) + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable("dataSourceOrganisationOverride") + .dropColumn("columnVisualisations") + .execute(); + + await db.schema + .alterTable("dataSourceOrganisationOverride") + .renameTo("columnMetadataOverride") + .execute(); +} diff --git a/migrations/1774206346032_inspector_config_boundaries_to_data_sources.ts b/migrations/1774206346032_inspector_config_boundaries_to_data_sources.ts new file mode 100644 index 00000000..5735c209 --- /dev/null +++ b/migrations/1774206346032_inspector_config_boundaries_to_data_sources.ts @@ -0,0 +1,19 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { sql } from "kysely"; +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await sql` + UPDATE map_view + SET inspector_config = (inspector_config - 'boundaries') || jsonb_build_object('dataSources', inspector_config->'boundaries') + WHERE inspector_config ? 'boundaries' + `.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql` + UPDATE map_view + SET inspector_config = (inspector_config - 'dataSources') || jsonb_build_object('boundaries', inspector_config->'dataSources') + WHERE inspector_config ? 'dataSources' + `.execute(db); +} diff --git a/migrations/1774206346033_rename_column_visualisations_to_inspector_columns.ts b/migrations/1774206346033_rename_column_visualisations_to_inspector_columns.ts new file mode 100644 index 00000000..f59a533e --- /dev/null +++ b/migrations/1774206346033_rename_column_visualisations_to_inspector_columns.ts @@ -0,0 +1,26 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await db.schema + .alterTable("dataSource") + .renameColumn("columnVisualisations", "inspectorColumns") + .execute(); + + await db.schema + .alterTable("dataSourceOrganisationOverride") + .renameColumn("columnVisualisations", "inspectorColumns") + .execute(); +} + +export async function down(db: Kysely): Promise { + await db.schema + .alterTable("dataSourceOrganisationOverride") + .renameColumn("inspectorColumns", "columnVisualisations") + .execute(); + + await db.schema + .alterTable("dataSource") + .renameColumn("inspectorColumns", "columnVisualisations") + .execute(); +} diff --git a/migrations/1774222928000_rename_map_view_category_colors.ts b/migrations/1774222928000_rename_map_view_category_colors.ts new file mode 100644 index 00000000..47a91476 --- /dev/null +++ b/migrations/1774222928000_rename_map_view_category_colors.ts @@ -0,0 +1,19 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { sql } from "kysely"; +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await sql` + UPDATE "map_view" + SET config = jsonb_set(config - 'categoryColors', '{colorMappings}', config->'categoryColors') + WHERE config ? 'categoryColors' + `.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql` + UPDATE "map_view" + SET config = jsonb_set(config - 'colorMappings', '{categoryColors}', config->'colorMappings') + WHERE config ? 'colorMappings' + `.execute(db); +} diff --git a/migrations/1774222928001_rename_column_metadata_color_mappings.ts b/migrations/1774222928001_rename_column_metadata_color_mappings.ts new file mode 100644 index 00000000..3a26b291 --- /dev/null +++ b/migrations/1774222928001_rename_column_metadata_color_mappings.ts @@ -0,0 +1,75 @@ +/* eslint-disable @typescript-eslint/no-explicit-any */ +import { sql } from "kysely"; +import type { Kysely } from "kysely"; + +export async function up(db: Kysely): Promise { + await sql` + UPDATE "data_source" + SET "column_metadata" = ( + SELECT jsonb_agg( + CASE + WHEN col ? 'colorMappings' + THEN jsonb_set(col - 'colorMappings', '{valueColors}', col->'colorMappings') + ELSE col + END + ) + FROM jsonb_array_elements("column_metadata") AS col + ) + WHERE EXISTS ( + SELECT 1 FROM jsonb_array_elements("column_metadata") AS col WHERE col ? 'colorMappings' + ) + `.execute(db); + + await sql` + UPDATE "data_source_organisation_override" + SET "column_metadata" = ( + SELECT jsonb_agg( + CASE + WHEN col ? 'colorMappings' + THEN jsonb_set(col - 'colorMappings', '{valueColors}', col->'colorMappings') + ELSE col + END + ) + FROM jsonb_array_elements("column_metadata") AS col + ) + WHERE EXISTS ( + SELECT 1 FROM jsonb_array_elements("column_metadata") AS col WHERE col ? 'colorMappings' + ) + `.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql` + UPDATE "data_source" + SET "column_metadata" = ( + SELECT jsonb_agg( + CASE + WHEN col ? 'valueColors' + THEN jsonb_set(col - 'valueColors', '{colorMappings}', col->'valueColors') + ELSE col + END + ) + FROM jsonb_array_elements("column_metadata") AS col + ) + WHERE EXISTS ( + SELECT 1 FROM jsonb_array_elements("column_metadata") AS col WHERE col ? 'valueColors' + ) + `.execute(db); + + await sql` + UPDATE "data_source_organisation_override" + SET "column_metadata" = ( + SELECT jsonb_agg( + CASE + WHEN col ? 'valueColors' + THEN jsonb_set(col - 'valueColors', '{colorMappings}', col->'valueColors') + ELSE col + END + ) + FROM jsonb_array_elements("column_metadata") AS col + ) + WHERE EXISTS ( + SELECT 1 FROM jsonb_array_elements("column_metadata") AS col WHERE col ? 'valueColors' + ) + `.execute(db); +} diff --git a/src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx b/src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx index 3649395a..aed123a0 100644 --- a/src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx +++ b/src/app/(private)/(dashboards)/data-sources/[id]/DataSourceDashboard.tsx @@ -1,6 +1,6 @@ "use client"; -import { useMutation, useQuery } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { useSubscription } from "@trpc/tanstack-react-query"; import { format, formatDistanceToNow } from "date-fns"; import { @@ -34,9 +34,10 @@ import { import { Button } from "@/shadcn/ui/button"; import { Separator } from "@/shadcn/ui/separator"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/shadcn/ui/tabs"; -import ColumnMetadataForm from "./components/ColumnMetadataForm"; +import ColumnMetadataTable from "./components/ColumnMetadataTable"; +import ColumnVisualisationPanel from "./components/ColumnVisualisationPanel"; import ConfigurationForm from "./components/ConfigurationForm"; -import { DataSourceEnrichmentDashboard } from "./DataSourceEnrichmentDashboard"; +import EnrichmentTable from "./components/EnrichmentTable"; import type { RouterOutputs } from "@/services/trpc/react"; export function DataSourceDashboard({ @@ -67,6 +68,7 @@ export function DataSourceDashboard({ ); const trpc = useTRPC(); + const queryClient = useQueryClient(); // Check webhook status for Google Sheets data sources const { data: webhookStatus } = useQuery( @@ -104,6 +106,11 @@ export function DataSourceDashboard({ if (dataSourceEvent.event === "ImportComplete") { setImporting(false); setLastImported(dataSourceEvent.at); + queryClient.invalidateQueries({ + queryKey: trpc.dataSource.byId.queryKey({ + dataSourceId: dataSource.id, + }), + }); } }, }, @@ -150,7 +157,7 @@ export function DataSourceDashboard({ ); return ( -
+
@@ -224,9 +231,11 @@ export function DataSourceDashboard({ )} - + - Settings + Configuration + Column metadata + Map inspector {showEnrichment && ( Enrichment )} @@ -249,7 +258,6 @@ export function DataSourceDashboard({ : mappedInformation } /> -
@@ -268,9 +276,17 @@ export function DataSourceDashboard({
+ + + + + + + + {showEnrichment && ( - + )} diff --git a/src/app/(private)/(dashboards)/data-sources/[id]/components/ColumnMetadataForm.tsx b/src/app/(private)/(dashboards)/data-sources/[id]/components/ColumnMetadataForm.tsx deleted file mode 100644 index 15bd8f16..00000000 --- a/src/app/(private)/(dashboards)/data-sources/[id]/components/ColumnMetadataForm.tsx +++ /dev/null @@ -1,318 +0,0 @@ -"use client"; - -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; -import { Pencil, Plus, Save } from "lucide-react"; -import { useCallback, useMemo, useState } from "react"; -import { toast } from "sonner"; -import { type ColumnMetadata, ColumnType } from "@/models/DataSource"; -import { useTRPC } from "@/services/trpc/react"; -import { Button } from "@/shadcn/ui/button"; -import { - Dialog, - DialogContent, - DialogDescription, - DialogFooter, - DialogHeader, - DialogTitle, -} from "@/shadcn/ui/dialog"; -import { Input } from "@/shadcn/ui/input"; -import { ScrollArea } from "@/shadcn/ui/scroll-area"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeader, - TableRow, -} from "@/shadcn/ui/table"; -import { Textarea } from "@/shadcn/ui/textarea"; -import type { RouterOutputs } from "@/services/trpc/react"; - -type DataSource = NonNullable; - -function hasMetadata(col: ColumnMetadata) { - return col.description !== "" || Object.keys(col.valueLabels).length > 0; -} - -export default function ColumnMetadataForm({ - dataSource, -}: { - dataSource: DataSource; -}) { - const initialMetadata = useMemo(() => { - const existing = new Map( - (dataSource.columnMetadata ?? []).map((m) => [m.name, m]), - ); - return dataSource.columnDefs.map((col) => ({ - name: col.name, - valueLabels: existing.get(col.name)?.valueLabels ?? {}, - description: existing.get(col.name)?.description ?? "", - })); - }, [dataSource.columnDefs, dataSource.columnMetadata]); - - const [metadata, setMetadata] = useState(initialMetadata); - const [loading, setLoading] = useState(false); - const [editingIndex, setEditingIndex] = useState(null); - const [draftDescription, setDraftDescription] = useState(""); - const [draftValueLabels, setDraftValueLabels] = useState< - Record - >({}); - - const client = useQueryClient(); - const trpc = useTRPC(); - const editingCol = editingIndex !== null ? metadata[editingIndex] : null; - const { data: rawColumnValues } = useQuery( - trpc.dataSource.uniqueColumnValues.queryOptions( - { dataSourceId: dataSource.id, column: editingCol?.name ?? "" }, - { enabled: editingCol !== null }, - ), - ); - - const editingColumnType = dataSource.columnDefs.find( - (col) => col.name === editingCol?.name, - )?.type; - - // If nullIsZero is true for a numeric column, combine "", "null" and "undefined" into "0" - const columnValues = useMemo(() => { - if (rawColumnValues == null) return rawColumnValues; - if (!dataSource.nullIsZero || editingColumnType !== ColumnType.Number) { - return rawColumnValues; - } - const blankValues = new Set(["", "null", "undefined"]); - const hasBlankOrZero = - rawColumnValues.some((v) => blankValues.has(v)) || - rawColumnValues.includes("0"); - if (!hasBlankOrZero) return rawColumnValues; - const filtered = rawColumnValues.filter((v) => !blankValues.has(v)); - if (!filtered.includes("0")) { - filtered.push("0"); - } - return filtered; - }, [rawColumnValues, dataSource.nullIsZero, editingColumnType]); - const { mutate: updateDataSourceConfig } = useMutation( - trpc.dataSource.updateConfig.mutationOptions({ - onSuccess: () => { - setLoading(false); - toast.success("Column metadata saved."); - client.invalidateQueries({ - queryKey: trpc.dataSource.listReadable.queryKey(), - }); - }, - onError: (error) => { - setLoading(false); - toast.error(error.message || "Could not save column metadata."); - }, - }), - ); - - const openDialog = useCallback( - (index: number) => { - const col = metadata[index]; - setDraftDescription(col.description); - setDraftValueLabels(col.valueLabels); - setEditingIndex(index); - }, - [metadata], - ); - - const closeDialog = useCallback(() => { - setEditingIndex(null); - }, []); - - const saveDialog = useCallback(() => { - if (editingIndex === null) return; - const updated = metadata.map((m, i) => - i === editingIndex - ? { ...m, description: draftDescription, valueLabels: draftValueLabels } - : m, - ); - setMetadata(updated); - setEditingIndex(null); - setLoading(true); - updateDataSourceConfig({ - dataSourceId: dataSource.id, - columnMetadata: updated, - }); - }, [ - editingIndex, - metadata, - draftDescription, - draftValueLabels, - dataSource.id, - updateDataSourceConfig, - ]); - - if (dataSource.columnDefs.length === 0) { - return ( -

- Import data to configure column metadata. -

- ); - } - - return ( -
-

Column metadata

-

- Add labels and descriptions for each column. These are shown to users - viewing your data. -

-
    - {metadata.map((col, index) => { - const hasMeta = hasMetadata(col); - return ( -
  • - {col.name} - -
  • - ); - })} -
- - { - if (!open) closeDialog(); - }} - > - - - - Metadata for {editingCol?.name} - - - Configure a description and value labels for this column. - - - {editingCol && ( -
-
- -