From 01f2513c29abff97b9c4aaa1316bec39a44828be Mon Sep 17 00:00:00 2001 From: Derek Date: Wed, 11 Nov 2026 16:53:42 +1100 Subject: [PATCH 1/3] fix(console): read the column vocabulary from the engine The schema form had local lists of primitives, use cases, attributes and override types. The derived schema picker had another use case list. These lists offered retired use cases and missed map, vector and key_search. - Primitives, use cases, attributes and override types come from GET /api/v1/schemas/vocabulary. - Override inputs use engine-declared parameters instead of switching on the type name. - A failed system defaults fetch shows the error. The client no longer supplies its own retention and header version. - The retention form takes its maximum from the engine: 36500 days instead of the hard-coded 365. --- .../EditSystemDefaultsForm.test.ts | 30 + .../EditSystemDefaultsForm.tsx | 19 +- .../SystemDefaultSettings.test.tsx | 1 + .../SystemDefaultsSettings/index.tsx | 12 +- .../useUpdateDefaults.test.tsx | 1 + .../CreateDerivedSchemaDrawer.mocks.ts | 1 + .../DerivedSchemaFieldPicker.helpers.test.ts | 12 +- .../DerivedSchemaFieldPicker.helpers.ts | 14 +- .../DerivedSchemaFieldPicker.mocks.ts | 1 + .../DerivedSchemaFieldPicker/index.tsx | 36 +- .../Schemas/constants/indexUseCase.test.ts | 82 +-- .../src/Schemas/constants/indexUseCase.ts | 83 ++- .../CreateDerivedSchemaForm.schema.ts | 4 +- .../MetaSchemaForm.mocks.ts | 1 + .../ConfigurationDetailsTabContent.mocks.ts | 1 + .../AddSchemaTable/AddSchemaTable.test.tsx | 49 +- .../TypeOverrideSelect/LazyMountedSelect.tsx | 115 ++-- .../TypeOverrideSelect.helpers.test.tsx | 146 +++-- .../TypeOverrideSelect.helpers.ts | 140 +++-- .../TypeOverrideSelect/index.tsx | 10 +- .../AddSchemaTable/fieldOptions.constants.tsx | 357 ++---------- .../CreateSchemaForm/AddSchemaTable/index.tsx | 526 +++++++++--------- .../InvalidColumnsTable.test.tsx | 17 +- .../api/endpoints/generator/mocks/schemas.ts | 27 + .../api/endpoints/generator/mocks/system.ts | 2 + .../src/core/config/api/endpoints/index.ts | 1 + .../core/config/api/endpoints/queryKeys.tsx | 1 + .../useFetchDefaults.test.tsx | 1 + .../hooks/useFetchSchemaVocabulary/api.ts | 7 + .../hooks/useFetchSchemaVocabulary/index.tsx | 19 + .../hooks/useFetchSchemaVocabulary/types.ts | 12 + .../useFetchSchemaVocabulary.mocks.ts | 6 + .../useFetchSchemaVocabulary.test.tsx | 37 ++ .../vocabulary.fixture.ts | 149 +++++ .../hooks/useFetchSystemDefaults/index.tsx | 27 +- .../useFetchSystemDefaults.mocks.ts | 1 + .../useFetchSystemDefaults.test.tsx | 3 + packages/dfe-engine-types/specs/openapi.json | 305 +++++++++- packages/dfe-engine-types/types/index.d.ts | 185 ++++++ 39 files changed, 1550 insertions(+), 891 deletions(-) create mode 100644 apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.test.ts create mode 100644 apps/dfe-core-ui/src/core/hooks/useFetchSchemaVocabulary/api.ts create mode 100644 apps/dfe-core-ui/src/core/hooks/useFetchSchemaVocabulary/index.tsx create mode 100644 apps/dfe-core-ui/src/core/hooks/useFetchSchemaVocabulary/types.ts create mode 100644 apps/dfe-core-ui/src/core/hooks/useFetchSchemaVocabulary/useFetchSchemaVocabulary.mocks.ts create mode 100644 apps/dfe-core-ui/src/core/hooks/useFetchSchemaVocabulary/useFetchSchemaVocabulary.test.tsx create mode 100644 apps/dfe-core-ui/src/core/hooks/useFetchSchemaVocabulary/vocabulary.fixture.ts diff --git a/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.test.ts b/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.test.ts new file mode 100644 index 00000000..be791c9f --- /dev/null +++ b/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.test.ts @@ -0,0 +1,30 @@ +import { describe, expect, it } from 'vitest'; +import { buildFormSchema } from './EditSystemDefaultsForm'; + +const values = (ttl_days: number) => ({ + ttl_days, + engine: 'MergeTree', + common_header_type: 'common-header/timeseries', + common_header_version: '1.0.3', +}); + +describe('buildFormSchema', () => { + it.each([ + [0, true], + [365, true], + [1000, true], + [36500, true], + [36501, false], + [-1, false], + ])( + 'retention of %i days against a maximum of 36500 is valid: %s', + (days, ok) => { + expect(buildFormSchema(36500).safeParse(values(days)).success).toBe(ok); + }, + ); + + it('takes the bound it is given, not one of its own', () => { + expect(buildFormSchema(30).safeParse(values(31)).success).toBe(false); + expect(buildFormSchema(30).safeParse(values(30)).success).toBe(true); + }); +}); diff --git a/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.tsx b/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.tsx index a7047bdd..5ff86afa 100644 --- a/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.tsx +++ b/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.tsx @@ -10,14 +10,16 @@ import isInteger from 'lodash/isInteger'; import { useCallback, useMemo, useState } from 'react'; import z from 'zod'; -const formSchema = z.object({ - ttl_days: z.number().min(0).max(365), - engine: z.string().min(1), - common_header_version: z.string().min(1), - common_header_type: z.string().min(1), -}); +/** The retention bound is the engine's, served with the defaults. */ +export const buildFormSchema = (maxTtlDays: number) => + z.object({ + ttl_days: z.number().min(0).max(maxTtlDays), + engine: z.string().min(1), + common_header_version: z.string().min(1), + common_header_type: z.string().min(1), + }); -type TFormValues = z.infer; +type TFormValues = z.infer>; const normalizeHeaderType = (value?: string) => (value ?? '').replace(/^common-header\//, ''); @@ -78,10 +80,12 @@ const hasDefaultsChanged = ({ }; export const EditSystemDefaultsForm = ({ initialValues, + maxTtlDays, onSuccess, onClose, }: { initialValues?: TFormValues; + maxTtlDays: number; onSuccess?: ( response: TUpdateSystemDefaultsResponse & { changed: boolean }, ) => void; @@ -116,6 +120,7 @@ export const EditSystemDefaultsForm = ({ }); const [form] = Form.useForm(); + const formSchema = useMemo(() => buildFormSchema(maxTtlDays), [maxTtlDays]); const formValidation = useAntdZodResolver(formSchema); const ttlDays = Form.useWatch('ttl_days', form); const engine = Form.useWatch('engine', form); diff --git a/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/SystemDefaultSettings.test.tsx b/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/SystemDefaultSettings.test.tsx index e738d113..01e406fc 100644 --- a/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/SystemDefaultSettings.test.tsx +++ b/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/SystemDefaultSettings.test.tsx @@ -36,6 +36,7 @@ describe('SystemDefaultsSettings', () => { stored: null, origin: 'deployment', deployment_default: 0, + maximum: 36500, }, common_header_type: { effective: 'common-header/timeseries', diff --git a/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/index.tsx b/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/index.tsx index 1b3504dd..ba7423a4 100644 --- a/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/index.tsx +++ b/apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/index.tsx @@ -22,19 +22,19 @@ export const SystemDefaultsSettings = () => { /> } > - {isEditDefaultsOpen ? ( + {isEditDefaultsOpen && defaults ? ( setIsEditDefaultsOpen(false)} onSuccess={({ changed }) => { setIsEditDefaultsOpen(false); setHasDefaultsChanged(changed); }} + maxTtlDays={defaults.ttl_days.maximum} initialValues={{ - ttl_days: defaults?.ttl_days?.effective ?? 0, - engine: defaults?.engine?.effective ?? '', - common_header_version: - defaults?.common_header_version?.effective ?? '', - common_header_type: `common-header/${defaults?.common_header_type?.effective?.replace('common-header/', '')}`, + ttl_days: defaults.ttl_days.effective, + engine: defaults.engine.effective, + common_header_version: defaults.common_header_version.effective, + common_header_type: `common-header/${defaults.common_header_type.effective.replace('common-header/', '')}`, }} /> ) : ( diff --git a/apps/dfe-core-ui/src/Platform/hooks/system/useUpdateDefaults/useUpdateDefaults.test.tsx b/apps/dfe-core-ui/src/Platform/hooks/system/useUpdateDefaults/useUpdateDefaults.test.tsx index 95345be6..a4e8830f 100644 --- a/apps/dfe-core-ui/src/Platform/hooks/system/useUpdateDefaults/useUpdateDefaults.test.tsx +++ b/apps/dfe-core-ui/src/Platform/hooks/system/useUpdateDefaults/useUpdateDefaults.test.tsx @@ -57,6 +57,7 @@ describe('.useUpdateSystemDefaults', () => { stored: null, origin: 'deployment', deployment_default: 90, + maximum: 36500, }, common_header_type: { effective: 'string', diff --git a/apps/dfe-core-ui/src/Schemas/components/CreateDerivedSchemaDrawer/CreateDerivedSchemaDrawer.mocks.ts b/apps/dfe-core-ui/src/Schemas/components/CreateDerivedSchemaDrawer/CreateDerivedSchemaDrawer.mocks.ts index 27c29deb..af3ca7a1 100644 --- a/apps/dfe-core-ui/src/Schemas/components/CreateDerivedSchemaDrawer/CreateDerivedSchemaDrawer.mocks.ts +++ b/apps/dfe-core-ui/src/Schemas/components/CreateDerivedSchemaDrawer/CreateDerivedSchemaDrawer.mocks.ts @@ -60,6 +60,7 @@ export const server = setupServer( API_CONFIG_MOCKS.schemas.default.get.success({ mockedResponse: SCHEMA_LIST_RESPONSE, }), + API_CONFIG_MOCKS.schemas.vocabulary.get.success(), API_CONFIG_MOCKS.schemas.schemaDetail.get.success({ mockedResponse: BASE_COLUMNS_RESPONSE, schema_path: BASE_SCHEMA_PATH, diff --git a/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.test.ts b/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.test.ts index 59b311bc..6758f4d1 100644 --- a/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.test.ts +++ b/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.test.ts @@ -6,6 +6,7 @@ import { selectEntry, withIndexUseCase, } from '@/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers'; +import { SCHEMA_VOCABULARY_FIXTURE } from '@/core/hooks/useFetchSchemaVocabulary/vocabulary.fixture'; import { describe, expect, it } from 'vitest'; const baseColumns: DerivedBaseColumn[] = [ @@ -17,14 +18,21 @@ const baseColumns: DerivedBaseColumn[] = [ describe('baseColumnFrom', () => { it('reads the base column index through the rename', () => { - expect(baseColumnFrom({ name: 'message', use_case: 'fulltext' })).toEqual({ + expect( + baseColumnFrom( + { name: 'message', use_case: 'fulltext' }, + SCHEMA_VOCABULARY_FIXTURE, + ), + ).toEqual({ name: 'message', baseIndex: 'word_search', }); }); it('gives a column with no index use case none', () => { - expect(baseColumnFrom({ name: 'log_offset' })).toEqual({ + expect( + baseColumnFrom({ name: 'log_offset' }, SCHEMA_VOCABULARY_FIXTURE), + ).toEqual({ name: 'log_offset', baseIndex: 'none', }); diff --git a/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.ts b/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.ts index 9df75909..79fdc0f1 100644 --- a/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.ts +++ b/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.ts @@ -1,3 +1,4 @@ +import type { TSchemaVocabularyResponse } from '@/core/hooks/useFetchSchemaVocabulary/types'; import { IndexUseCase, resolveIndexUseCase, @@ -20,12 +21,15 @@ export type DerivedBaseColumn = { baseIndex: IndexUseCase; }; -export const baseColumnFrom = (column: { - name?: string | null; - use_case?: string | null; -}): DerivedBaseColumn => ({ +export const baseColumnFrom = ( + column: { + name?: string | null; + use_case?: string | null; + }, + vocabulary: TSchemaVocabularyResponse, +): DerivedBaseColumn => ({ name: column.name ?? '', - baseIndex: resolveIndexUseCase(column.use_case), + baseIndex: resolveIndexUseCase(column.use_case, vocabulary), }); /** An omitted `index` keeps the base column's, so only a change is written. */ diff --git a/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.mocks.ts b/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.mocks.ts index 90273dd4..4e22e711 100644 --- a/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.mocks.ts +++ b/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.mocks.ts @@ -49,6 +49,7 @@ export const BASE_COLUMNS_RESPONSE: TMetaSchemaDetailResponse = { }; export const server = setupServer( + API_CONFIG_MOCKS.schemas.vocabulary.get.success(), API_CONFIG_MOCKS.schemas.schemaDetail.get.success({ mockedResponse: BASE_COLUMNS_RESPONSE, schema_path: BASE_SCHEMA_PATH, diff --git a/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/index.tsx b/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/index.tsx index d9c59f16..1388bf12 100644 --- a/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/index.tsx +++ b/apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/index.tsx @@ -1,7 +1,8 @@ import type { FormControlAria } from '@/core/components/Form/formControlAria'; import { Table } from '@/core/components/Table'; import { Tooltip } from '@/core/components/Tooltip'; -import { INDEX_USE_CASE_OPTIONS } from '@/Schemas/constants/indexUseCase'; +import { useFetchSchemaVocabulary } from '@/core/hooks/useFetchSchemaVocabulary'; +import { indexUseCaseOptions } from '@/Schemas/constants/indexUseCase'; import { useFetchInfiniteFilteredSchemaDetailColumns } from '@/Schemas/hooks/useFetchInfiniteSchemaDetailColumns'; import { TMetaSchemaDetailColumnItem } from '@/Schemas/hooks/useFetchInfiniteSchemaDetailColumns/types'; import { IconInfoCircle } from '@repo/dfe-icons'; @@ -64,11 +65,26 @@ export const DerivedSchemaFieldPicker = ({ fetch_all: true, }); + const { + data: vocabulary, + isLoading: isLoadingVocabulary, + error: vocabularyError, + } = useFetchSchemaVocabulary(); + const columnItems = useMemo(() => data?.items ?? [], [data]); + // A base column's index is read through the vocabulary, so none is read before it is in. const baseColumns: DerivedBaseColumn[] = useMemo( - () => columnItems.map(baseColumnFrom), - [columnItems], + () => + vocabulary + ? columnItems.map((column) => baseColumnFrom(column, vocabulary)) + : [], + [columnItems, vocabulary], + ); + + const indexOptions = useMemo( + () => (vocabulary ? indexUseCaseOptions(vocabulary) : []), + [vocabulary], ); const baseColumnByName = useMemo( @@ -118,7 +134,7 @@ export const DerivedSchemaFieldPicker = ({ className="w-full" size="small" aria-label={`Index use case for ${column.name}`} - options={INDEX_USE_CASE_OPTIONS} + options={indexOptions} popupMatchSelectWidth={false} value={chosenIndexFor(value, column)} disabled={disabled || !selected} @@ -166,7 +182,7 @@ export const DerivedSchemaFieldPicker = ({ }, ]; // `value` and `onChange` drive the two interactive cells; the rest are static. - }, [baseColumnByName, value, onChange, disabled]); + }, [baseColumnByName, indexOptions, value, onChange, disabled]); if (!basePath || !baseVersion) { return ( @@ -179,12 +195,16 @@ export const DerivedSchemaFieldPicker = ({ ); } - if (isError) { + if (isError || vocabularyError) { return ( ); } @@ -220,7 +240,7 @@ export const DerivedSchemaFieldPicker = ({ columns={tableColumns} dataSource={visibleItems} rowKey="name" - loading={isLoading || isFetchingNextPage} + loading={isLoading || isLoadingVocabulary || isFetchingNextPage} pagination={false} scroll={{ y: 360, x: 'max-content' }} rowSelection={{ diff --git a/apps/dfe-core-ui/src/Schemas/constants/indexUseCase.test.ts b/apps/dfe-core-ui/src/Schemas/constants/indexUseCase.test.ts index 746f081f..1937dfa3 100644 --- a/apps/dfe-core-ui/src/Schemas/constants/indexUseCase.test.ts +++ b/apps/dfe-core-ui/src/Schemas/constants/indexUseCase.test.ts @@ -1,52 +1,43 @@ +import { SCHEMA_VOCABULARY_FIXTURE } from '@/core/hooks/useFetchSchemaVocabulary/vocabulary.fixture'; import { - INDEX_USE_CASE_LABELS, - INDEX_USE_CASE_OPTIONS, + indexUseCaseOptions, resolveIndexUseCase, } from '@/Schemas/constants/indexUseCase'; import { describe, expect, it } from 'vitest'; +const vocabulary = SCHEMA_VOCABULARY_FIXTURE; + describe('resolveIndexUseCase', () => { - it('keeps a use case that is already current', () => { - expect(resolveIndexUseCase('dimension')).toBe('dimension'); - expect(resolveIndexUseCase('key_search')).toBe('key_search'); + it('keeps a use case the vocabulary has', () => { + expect(resolveIndexUseCase('dimension', vocabulary)).toBe('dimension'); + expect(resolveIndexUseCase('key_search', vocabulary)).toBe('key_search'); }); - it('maps the pre-rename vocabulary a base column may still carry', () => { - expect(resolveIndexUseCase('fulltext')).toBe('word_search'); - expect(resolveIndexUseCase('text_search')).toBe('substring_search'); - expect(resolveIndexUseCase('bloom')).toBe('exact_match'); + it('maps a retired name to the use case that replaced it', () => { + expect(resolveIndexUseCase('fulltext', vocabulary)).toBe('word_search'); + expect(resolveIndexUseCase('text_search', vocabulary)).toBe( + 'substring_search', + ); + expect(resolveIndexUseCase('bloom', vocabulary)).toBe('exact_match'); }); it('treats an empty, missing or unknown value as no index', () => { - expect(resolveIndexUseCase(undefined)).toBe('none'); - expect(resolveIndexUseCase(null)).toBe('none'); - expect(resolveIndexUseCase(' ')).toBe('none'); - expect(resolveIndexUseCase('tokenbf_v1')).toBe('none'); + expect(resolveIndexUseCase(undefined, vocabulary)).toBe('none'); + expect(resolveIndexUseCase(null, vocabulary)).toBe('none'); + expect(resolveIndexUseCase(' ', vocabulary)).toBe('none'); + expect(resolveIndexUseCase('tokenbf_v1', vocabulary)).toBe('none'); }); it('ignores case and surrounding space', () => { - expect(resolveIndexUseCase(' Fulltext ')).toBe('word_search'); + expect(resolveIndexUseCase(' Fulltext ', vocabulary)).toBe('word_search'); }); }); -describe('index use case labels', () => { - it('names the question, never the ClickHouse primitive', () => { - const labels = Object.values(INDEX_USE_CASE_LABELS).join(' ').toLowerCase(); - for (const primitive of [ - 'set(0)', - 'minmax', - 'bloom_filter', - 'tokenbf', - 'ngrambf', - 'tokenizer', - 'granularity', - ]) { - expect(labels).not.toContain(primitive); - } - }); - - it('offers every use case as an option', () => { - expect(INDEX_USE_CASE_OPTIONS.map((option) => option.value)).toEqual([ +describe('indexUseCaseOptions', () => { + it('offers the use cases that take no argument, then no index', () => { + expect( + indexUseCaseOptions(vocabulary).map((option) => option.value), + ).toEqual([ 'dimension', 'exact_match', 'range', @@ -55,9 +46,32 @@ describe('index use case labels', () => { 'key_search', 'none', ]); + }); + + it('labels each with the question it answers, never a ClickHouse primitive', () => { + const options = indexUseCaseOptions(vocabulary); + expect( - INDEX_USE_CASE_OPTIONS.find((option) => option.value === 'word_search') - ?.label, + options.find((option) => option.value === 'word_search')?.label, ).toBe('I search for whole words'); + const labels = options + .map((option) => option.label) + .join(' ') + .toLowerCase(); + for (const primitive of ['set(0)', 'minmax', 'bloom_filter', 'tokenbf']) { + expect(labels).not.toContain(primitive); + } + }); + + it('falls back to the label when a use case has no description', () => { + const bare = { + ...vocabulary, + use_cases: [{ ...vocabulary.use_cases[0], description: '' }], + }; + + expect(indexUseCaseOptions(bare)[0]).toEqual({ + label: 'Dimension', + value: 'dimension', + }); }); }); diff --git a/apps/dfe-core-ui/src/Schemas/constants/indexUseCase.ts b/apps/dfe-core-ui/src/Schemas/constants/indexUseCase.ts index de6e3696..e3116781 100644 --- a/apps/dfe-core-ui/src/Schemas/constants/indexUseCase.ts +++ b/apps/dfe-core-ui/src/Schemas/constants/indexUseCase.ts @@ -1,65 +1,58 @@ +import type { TSchemaVocabularyResponse } from '@/core/hooks/useFetchSchemaVocabulary/types'; + /** * Index use cases: the question the user wants to ask of a column. * - * The engine turns the use case into a ClickHouse index, so nothing here names - * an index type, a tokenizer or a granularity. + * The engine owns the list and serves it with the schema vocabulary. It turns a + * use case into a ClickHouse index, so nothing here names an index type, a + * tokenizer or a granularity. */ -export const INDEX_USE_CASES = [ - 'dimension', - 'exact_match', - 'range', - 'word_search', - 'substring_search', - 'key_search', - 'none', -] as const; - -export type IndexUseCase = (typeof INDEX_USE_CASES)[number]; -/** The question each use case answers, which is what the picker shows. */ -export const INDEX_USE_CASE_LABELS: Record = { - dimension: 'I filter and group by this', - exact_match: 'I look up specific values', - range: 'I query ranges, between, time windows', - word_search: 'I search for whole words', - substring_search: 'I search for fragments inside words', - key_search: 'I search the keys and values of a map column', - none: 'No index', -}; +/** A use case name from the engine's vocabulary, or {@link NO_INDEX}. */ +export type IndexUseCase = string; -export const INDEX_USE_CASE_OPTIONS: { - label: string; - value: IndexUseCase; -}[] = INDEX_USE_CASES.map((useCase) => ({ - label: INDEX_USE_CASE_LABELS[useCase], - value: useCase, -})); +/** Not a use case of the engine's: the choice to carry no index. */ +export const NO_INDEX = 'none'; -/** A base column written before the rename still carries the old name. */ -const RENAMED_INDEX_USE_CASES: Record = { - fulltext: 'word_search', - text_search: 'substring_search', - bloom: 'exact_match', -}; - -const isIndexUseCase = (value: string): value is IndexUseCase => - (INDEX_USE_CASES as readonly string[]).includes(value); +/** + * What the picker offers: the question each use case answers, then no index. + * A use case that needs an argument is left out, because the picker cannot + * take one and the engine refuses the bare name. + */ +export const indexUseCaseOptions = ( + vocabulary: TSchemaVocabularyResponse, +): { label: string; value: IndexUseCase }[] => [ + ...vocabulary.use_cases + .filter(({ parameters }) => parameters.length === 0) + .map(({ name, label, description }) => ({ + label: description || label, + value: name, + })), + { label: 'No index', value: NO_INDEX }, +]; /** * A base column's index use case in the current vocabulary. * - * Returns `none` for a column with no index and for a value this build does not - * know, so an unreadable base never turns into an index nobody asked for. + * A name the engine retired maps to the one that replaced it. Returns `none` + * for a column with no index and for a value the vocabulary does not know, so an + * unreadable base never turns into an index nobody asked for. */ export const resolveIndexUseCase = ( value: string | null | undefined, + vocabulary: TSchemaVocabularyResponse, ): IndexUseCase => { const normalised = (value ?? '').trim().toLowerCase(); if (normalised === '') { - return 'none'; + return NO_INDEX; } - if (isIndexUseCase(normalised)) { - return normalised; + for (const useCase of vocabulary.use_cases) { + if (useCase.name === normalised) { + return useCase.name; + } + if (useCase.retired_names.includes(normalised)) { + return useCase.name; + } } - return RENAMED_INDEX_USE_CASES[normalised] ?? 'none'; + return NO_INDEX; }; diff --git a/apps/dfe-core-ui/src/Schemas/validationSchemas/CreateDerivedSchemaForm.schema.ts b/apps/dfe-core-ui/src/Schemas/validationSchemas/CreateDerivedSchemaForm.schema.ts index f0235db7..20230efb 100644 --- a/apps/dfe-core-ui/src/Schemas/validationSchemas/CreateDerivedSchemaForm.schema.ts +++ b/apps/dfe-core-ui/src/Schemas/validationSchemas/CreateDerivedSchemaForm.schema.ts @@ -3,12 +3,12 @@ import { schemaNameValidator, schemaVersionValidator, } from '@/core/validationSchemas/CreateSchemaForm/utils'; -import { INDEX_USE_CASES } from '@/Schemas/constants/indexUseCase'; import z from 'zod'; const selectEntrySchema = z.object({ name: z.string().min(1, { message: 'Column name is required' }), - index: z.enum(INDEX_USE_CASES).optional(), + // The engine owns the use case names and refuses one it does not know. + index: z.string().min(1).optional(), }); export const formSchema = z.object({ diff --git a/apps/dfe-core-ui/src/Sources/components/CreateUpdateSourceForm/SchemaConfigTabContent/MetaSchemaForm.mocks.ts b/apps/dfe-core-ui/src/Sources/components/CreateUpdateSourceForm/SchemaConfigTabContent/MetaSchemaForm.mocks.ts index 026357d6..53cd718e 100644 --- a/apps/dfe-core-ui/src/Sources/components/CreateUpdateSourceForm/SchemaConfigTabContent/MetaSchemaForm.mocks.ts +++ b/apps/dfe-core-ui/src/Sources/components/CreateUpdateSourceForm/SchemaConfigTabContent/MetaSchemaForm.mocks.ts @@ -31,6 +31,7 @@ export const server = setupServer( stored: null, origin: 'deployment', deployment_default: 90, + maximum: 36500, }, common_header_type: { effective: 'common-header/timeseries', diff --git a/apps/dfe-core-ui/src/Sources/components/ViewSourceTabs/ConfigurationDetailsTabContent/ConfigurationDetailsTabContent.mocks.ts b/apps/dfe-core-ui/src/Sources/components/ViewSourceTabs/ConfigurationDetailsTabContent/ConfigurationDetailsTabContent.mocks.ts index c11d3df9..a2cc532f 100644 --- a/apps/dfe-core-ui/src/Sources/components/ViewSourceTabs/ConfigurationDetailsTabContent/ConfigurationDetailsTabContent.mocks.ts +++ b/apps/dfe-core-ui/src/Sources/components/ViewSourceTabs/ConfigurationDetailsTabContent/ConfigurationDetailsTabContent.mocks.ts @@ -9,6 +9,7 @@ const handlers = [ stored: 90, origin: 'deployment', deployment_default: 90, + maximum: 36500, }, common_header_type: { effective: 'common-header/timeseries', diff --git a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/AddSchemaTable.test.tsx b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/AddSchemaTable.test.tsx index 4503c484..0273a997 100644 --- a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/AddSchemaTable.test.tsx +++ b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/AddSchemaTable.test.tsx @@ -1,13 +1,24 @@ import { SCHEMA_FIELD_TYPES } from '@/core/components/CreateSchemaForm/fieldType.constants'; import { UploadedSchemaRow } from '@/core/components/CreateSchemaForm/types'; import { Form as AntdForm, Form } from '@/core/components/Form'; +import { API_CONFIG_MOCKS } from '@/core/config/api/endpoints/generator'; +import { server } from '@/core/hooks/useFetchSchemaVocabulary/useFetchSchemaVocabulary.mocks'; import { buildTestWrapper } from '@/core/utils/test-utils/buildTestWrapper'; import { rowSchema } from '@/core/validationSchemas/CreateSchemaForm/AddSchemaTable.schema'; import { render, screen, waitFor, within } from '@testing-library/react'; import userEvent from '@testing-library/user-event'; import type { FormInstance, FormRule } from 'antd'; import { useEffect } from 'react'; -import { beforeEach, describe, expect, test, vi } from 'vitest'; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + test, + vi, +} from 'vitest'; import { AddSchemaTable, isBlankSchemaListRow, @@ -19,7 +30,11 @@ const bypassFormValidation = { validator: async () => Promise.resolve(), } as FormRule; -const { wrapper } = buildTestWrapper(); +beforeAll(() => server.listen({ onUnhandledRequest: 'error' })); +afterEach(() => server.resetHandlers()); +afterAll(() => server.close()); + +const { wrapper } = buildTestWrapper().withReactQuery(); const validRow = (overrides: Partial = {}): RowSchema => ({ id: 'row-1', @@ -500,6 +515,36 @@ describe('AddSchemaTable', { timeout: 15_000 }, () => { }); }); + test('offers the primitives and index use cases the engine serves', async () => { + const user = userEvent.setup(); + renderAddSchemaTable({ + initialValues: [validRow()], + visibleColumns: ['name', 'type', 'use_case', '__rowId'], + config: { defaultEditFields: ['type', 'use_case'] }, + }); + + const [typeSelect, useCaseSelect] = await screen.findAllByRole('combobox'); + + await user.click(typeSelect); + expect(await screen.findByText('Map')).toBeInTheDocument(); + expect(screen.getByText('Vector')).toBeInTheDocument(); + + await user.click(useCaseSelect); + expect(await screen.findByText('Key search')).toBeInTheDocument(); + expect(screen.getByText('Word search')).toBeInTheDocument(); + expect(screen.queryByText('Fulltext')).not.toBeInTheDocument(); + expect(screen.queryByText('Bloom')).not.toBeInTheDocument(); + }); + + test('says so when the vocabulary cannot be loaded', async () => { + server.use(API_CONFIG_MOCKS.schemas.vocabulary.get.error()); + renderAddSchemaTable(); + + expect( + await screen.findByText(/The column types could not be loaded/), + ).toBeInTheDocument(); + }); + test('forwards table props to the underlying table', () => { renderAddSchemaTable({ className: 'custom-schema-table' }); diff --git a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/LazyMountedSelect.tsx b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/LazyMountedSelect.tsx index 6dd36873..71ad6b75 100644 --- a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/LazyMountedSelect.tsx +++ b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/LazyMountedSelect.tsx @@ -1,9 +1,10 @@ import { - ADVANCED_OPTIONS, + overrideSelectOptions, TransformFunctionParams, } from '@/core/components/CreateSchemaForm/AddSchemaTable/fieldOptions.constants'; import { Form } from '@/core/components/Form'; import { FormNotification } from '@/core/components/FormNotification'; +import { useFetchSchemaVocabulary } from '@/core/hooks/useFetchSchemaVocabulary'; import { Button, FormInstance, @@ -29,35 +30,49 @@ interface LazyMountedSelectProps extends SelectProps { form: FormInstance; } +const NO_OVERRIDES: never[] = []; + export const LazyMountedSelect = ({ typeDetails, form, onChange, ...props }: LazyMountedSelectProps) => { + const { data: vocabulary } = useFetchSchemaVocabulary(); + const overrides = vocabulary?.ch_overrides ?? NO_OVERRIDES; const [overrideType, setOverrideType] = useState( - getInitialOverrideType(props.value), + getInitialOverrideType(props.value, overrides), ); const [validationError, setValidationError] = useState(null); const [advancedOptions, setAdvancedOptions] = - useState(getInitialAdvancedOptions(props.value)); + useState( + getInitialAdvancedOptions(props.value, overrides), + ); // Only fetch the form values when the select is mounted const baseTypeValue = Form.useWatch(typeDetails.type_form_name, form); const options = useMemo(() => { if (!baseTypeValue) return []; - return ADVANCED_OPTIONS.filter( - (option) => option.primitive === baseTypeValue, - ); - }, [baseTypeValue]); + return overrideSelectOptions(overrides, baseTypeValue); + }, [baseTypeValue, overrides]); + + const parameters = + overrides.find((option) => option.value === overrideType)?.parameters ?? []; + const numberParameters = parameters.filter(({ kind }) => kind === 'integer'); + const textParameters = parameters.filter(({ kind }) => kind === 'text'); + + const setParameter = (name: string, value: number | string | undefined) => { + setValidationError(null); + setAdvancedOptions({ ...advancedOptions, [name]: value }); + }; const handleApplyOverride = () => { - const error = validateOverride(overrideType, advancedOptions); + const error = validateOverride(overrideType, advancedOptions, overrides); if (error) { setValidationError(error); return; } - onChange?.(transformOverride(overrideType, advancedOptions)); + onChange?.(transformOverride(overrideType, advancedOptions, overrides)); }; return ( @@ -74,80 +89,36 @@ export const LazyMountedSelect = ({ value={overrideType} /> - {(overrideType === 'Decimal' || - overrideType === 'Decimal64' || - overrideType === 'DateTime64') && ( + {numberParameters.length > 0 && (
- {overrideType !== 'Decimal64' && ( - - + {numberParameters.map(({ name, label }) => ( + + { - setValidationError(null); - setAdvancedOptions({ - ...advancedOptions, - precision: value || undefined, - }); - }} - placeholder="Input Precision" + value={advancedOptions[name] as number | undefined} + onChange={(value: number | null) => + setParameter(name, value || undefined) + } + placeholder={`Input ${label}`} /> - )} - {overrideType !== 'DateTime64' && ( - - - { - setValidationError(null); - setAdvancedOptions({ - ...advancedOptions, - scale: value || undefined, - }); - }} - /> - - )} + ))}
)} - {overrideType === 'FixedString' && ( - - - { - setValidationError(null); - setAdvancedOptions({ - ...advancedOptions, - length: value || undefined, - }); - }} - /> - - )} - {overrideType === 'Enum16' && ( - - + {textParameters.map(({ name, label }) => ( + + ) => { - setValidationError(null); - setAdvancedOptions({ - ...advancedOptions, - values: e.target.value, - }); - }} + placeholder={`Input ${label.split(' (')[0]}`} + value={advancedOptions[name] as string | undefined} + onChange={(e: React.ChangeEvent) => + setParameter(name, e.target.value) + } /> - )} + ))} {validationError && ( )} diff --git a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/TypeOverrideSelect.helpers.test.tsx b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/TypeOverrideSelect.helpers.test.tsx index 6efae319..1b5f172a 100644 --- a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/TypeOverrideSelect.helpers.test.tsx +++ b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/TypeOverrideSelect.helpers.test.tsx @@ -1,3 +1,4 @@ +import { SCHEMA_VOCABULARY_FIXTURE } from '@/core/hooks/useFetchSchemaVocabulary/vocabulary.fixture'; import { describe, expect, test } from 'vitest'; import { getInitialAdvancedOptions, @@ -6,162 +7,207 @@ import { validateOverride, } from './TypeOverrideSelect.helpers'; +const options = SCHEMA_VOCABULARY_FIXTURE.ch_overrides; + describe('validateOverride', () => { test('returns null when override is null or empty', () => { - expect(validateOverride(null, {})).toBeNull(); - expect(validateOverride('', {})).toBeNull(); + expect(validateOverride(null, {}, options)).toBeNull(); + expect(validateOverride('', {}, options)).toBeNull(); }); test('returns null for overrides that need no params', () => { - expect(validateOverride('Int64', {})).toBeNull(); - expect(validateOverride('IPv4', {})).toBeNull(); + expect(validateOverride('Int64', {}, options)).toBeNull(); + expect(validateOverride('IPv4', {}, options)).toBeNull(); }); test('requires precision and scale for Decimal', () => { - expect(validateOverride('Decimal', {})).toBe( - 'Precision and scale are required', + expect(validateOverride('Decimal', {}, options)).toBe( + 'Precision and scale required', ); - expect(validateOverride('Decimal', { precision: 10 })).toBe( - 'Precision and scale are required', + expect(validateOverride('Decimal', { precision: 10 }, options)).toBe( + 'Scale required', ); - expect(validateOverride('Decimal', { scale: 2 })).toBe( - 'Precision and scale are required', + expect(validateOverride('Decimal', { scale: 2 }, options)).toBe( + 'Precision required', ); - expect(validateOverride('Decimal', { precision: 10, scale: 2 })).toBeNull(); + expect( + validateOverride('Decimal', { precision: 10, scale: 2 }, options), + ).toBeNull(); }); test('requires scale for Decimal64', () => { - expect(validateOverride('Decimal64', {})).toBe('Scale is required'); - expect(validateOverride('Decimal64', { scale: 4 })).toBeNull(); + expect(validateOverride('Decimal64', {}, options)).toBe('Scale required'); + expect(validateOverride('Decimal64', { scale: 4 }, options)).toBeNull(); }); test('requires precision for DateTime64', () => { - expect(validateOverride('DateTime64', {})).toBe('Precision is required'); - expect(validateOverride('DateTime64', { precision: 3 })).toBeNull(); + expect(validateOverride('DateTime64', {}, options)).toBe( + 'Precision required', + ); + expect( + validateOverride('DateTime64', { precision: 3 }, options), + ).toBeNull(); }); test('requires length for FixedString', () => { - expect(validateOverride('FixedString', {})).toBe('Length is required'); - expect(validateOverride('FixedString', { length: 16 })).toBeNull(); + expect(validateOverride('FixedString', {}, options)).toBe( + 'Length required', + ); + expect(validateOverride('FixedString', { length: 16 }, options)).toBeNull(); }); test('requires values for Enum16', () => { - expect(validateOverride('Enum16', {})).toBe('Values are required'); + expect(validateOverride('Enum16', {}, options)).toBe('Values required'); expect( - validateOverride('Enum16', { values: "'a' = 1, 'b' = 2" }), + validateOverride('Enum16', { values: "'a' = 1, 'b' = 2" }, options), ).toBeNull(); }); }); describe('transformOverride', () => { test('returns null when override type is null or empty', () => { - expect(transformOverride(null, {})).toBeNull(); - expect(transformOverride('', {})).toBeNull(); + expect(transformOverride(null, {}, options)).toBeNull(); + expect(transformOverride('', {}, options)).toBeNull(); }); test('returns the override type when no transformFunction exists', () => { - expect(transformOverride('Int64', {})).toBe('Int64'); + expect(transformOverride('Int64', {}, options)).toBe('Int64'); }); test('returns null for unknown override types', () => { - expect(transformOverride('NotARealType', {})).toBeNull(); + expect(transformOverride('NotARealType', {}, options)).toBeNull(); }); test('applies Decimal transform', () => { - expect(transformOverride('Decimal', { precision: 10, scale: 2 })).toBe( - 'Decimal(10,2)', - ); + expect( + transformOverride('Decimal', { precision: 10, scale: 2 }, options), + ).toBe('Decimal(10,2)'); }); test('applies Decimal64 transform', () => { - expect(transformOverride('Decimal64', { scale: 4 })).toBe('Decimal64(4)'); + expect(transformOverride('Decimal64', { scale: 4 }, options)).toBe( + 'Decimal64(4)', + ); }); test('applies FixedString transform', () => { - expect(transformOverride('FixedString', { length: 32 })).toBe( + expect(transformOverride('FixedString', { length: 32 }, options)).toBe( 'FixedString(32)', ); }); test('applies DateTime64 transform', () => { - expect(transformOverride('DateTime64', { precision: 3 })).toBe( + expect(transformOverride('DateTime64', { precision: 3 }, options)).toBe( "DateTime64(3,'UTC')", ); }); test('applies Enum16 transform', () => { - expect(transformOverride('Enum16', { values: "'ok' = 1, 'err' = 2" })).toBe( - "Enum16('ok' = 1, 'err' = 2)", - ); + expect( + transformOverride('Enum16', { values: "'ok' = 1, 'err' = 2" }, options), + ).toBe("Enum16('ok' = 1, 'err' = 2)"); }); }); describe('getInitialOverrideType', () => { test('returns null for null or empty values', () => { - expect(getInitialOverrideType(null)).toBeNull(); - expect(getInitialOverrideType('')).toBeNull(); + expect(getInitialOverrideType(null, options)).toBeNull(); + expect(getInitialOverrideType('', options)).toBeNull(); }); test('returns the bare type when there are no params', () => { - expect(getInitialOverrideType('Int64')).toBe('Int64'); + expect(getInitialOverrideType('Int64', options)).toBe('Int64'); }); test('strips parenthetical params', () => { - expect(getInitialOverrideType('Decimal(10, 2)')).toBe('Decimal'); - expect(getInitialOverrideType('Decimal64(4)')).toBe('Decimal64'); - expect(getInitialOverrideType("DateTime64(3, 'UTC')")).toBe('DateTime64'); - expect(getInitialOverrideType('FixedString(16)')).toBe('FixedString'); + expect(getInitialOverrideType('Decimal(10, 2)', options)).toBe('Decimal'); + expect(getInitialOverrideType('Decimal64(4)', options)).toBe('Decimal64'); + expect(getInitialOverrideType("DateTime64(3, 'UTC')", options)).toBe( + 'DateTime64', + ); + expect(getInitialOverrideType('FixedString(16)', options)).toBe( + 'FixedString', + ); }); test('trims whitespace around the type name', () => { - expect(getInitialOverrideType(' Decimal64 (4)')).toBe('Decimal64'); + expect(getInitialOverrideType(' Decimal64 (4)', options)).toBe( + 'Decimal64', + ); + }); +}); + +describe('an override with arguments and a bare one', () => { + test('a stored value that is an option whole keeps its type', () => { + const withWhole = [ + ...options, + { + value: 'Array(String)', + label: 'Array of String', + primitive: 'json', + parameters: [], + template: null, + hint: '', + }, + ]; + + expect(getInitialOverrideType('Array(String)', withWhole)).toBe( + 'Array(String)', + ); + expect(getInitialOverrideType('Decimal(10, 2)', withWhole)).toBe('Decimal'); }); }); describe('getInitialAdvancedOptions', () => { test('returns empty object for null or empty values', () => { - expect(getInitialAdvancedOptions(null)).toEqual({}); - expect(getInitialAdvancedOptions('')).toEqual({}); + expect(getInitialAdvancedOptions(null, options)).toEqual({}); + expect(getInitialAdvancedOptions('', options)).toEqual({}); }); test('returns empty object for overrides without params', () => { - expect(getInitialAdvancedOptions('Int64')).toEqual({}); + expect(getInitialAdvancedOptions('Int64', options)).toEqual({}); }); test('parses Decimal precision and scale', () => { - expect(getInitialAdvancedOptions('Decimal(18, 4)')).toEqual({ + expect(getInitialAdvancedOptions('Decimal(18, 4)', options)).toEqual({ precision: 18, scale: 4, }); }); test('parses Decimal64 scale', () => { - expect(getInitialAdvancedOptions('Decimal64(6)')).toEqual({ scale: 6 }); + expect(getInitialAdvancedOptions('Decimal64(6)', options)).toEqual({ + scale: 6, + }); }); test('parses DateTime64 precision from the first arg', () => { - expect(getInitialAdvancedOptions("DateTime64(3, 'UTC')")).toEqual({ + expect(getInitialAdvancedOptions("DateTime64(3, 'UTC')", options)).toEqual({ precision: 3, }); }); test('parses FixedString length', () => { - expect(getInitialAdvancedOptions('FixedString(64)')).toEqual({ + expect(getInitialAdvancedOptions('FixedString(64)', options)).toEqual({ length: 64, }); }); test('parses Enum16 values string', () => { - expect(getInitialAdvancedOptions("Enum16('a' = 1, 'b' = 2)")).toEqual({ + expect( + getInitialAdvancedOptions("Enum16('a' = 1, 'b' = 2)", options), + ).toEqual({ values: "'a' = 1, 'b' = 2", }); }); test('defaults missing numeric params to 0', () => { - expect(getInitialAdvancedOptions('Decimal()')).toEqual({ + expect(getInitialAdvancedOptions('Decimal()', options)).toEqual({ precision: 0, scale: 0, }); - expect(getInitialAdvancedOptions('FixedString()')).toEqual({ length: 0 }); + expect(getInitialAdvancedOptions('FixedString()', options)).toEqual({ + length: 0, + }); }); }); diff --git a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/TypeOverrideSelect.helpers.ts b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/TypeOverrideSelect.helpers.ts index 255e39e5..9982d971 100644 --- a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/TypeOverrideSelect.helpers.ts +++ b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/TypeOverrideSelect.helpers.ts @@ -1,99 +1,95 @@ -import { - ADVANCED_OPTIONS_MAP, - TransformFunctionParams, -} from '@/core/components/CreateSchemaForm/AddSchemaTable/fieldOptions.constants'; +import type { TransformFunctionParams } from '@/core/components/CreateSchemaForm/AddSchemaTable/fieldOptions.constants'; +import type { TSchemaVocabularyChOverride } from '@/core/hooks/useFetchSchemaVocabulary/types'; + +type OverrideOptions = TSchemaVocabularyChOverride[]; + +const findOption = ( + override: string | null, + options: OverrideOptions, +): TSchemaVocabularyChOverride | undefined => + options.find((option) => option.value === override); export const validateOverride = ( override: string | null, - options: TransformFunctionParams, + params: TransformFunctionParams, + options: OverrideOptions, ): string | null => { if (!override) return null; - switch (override) { - case 'Decimal': - if (options.precision === undefined || options.scale === undefined) { - return 'Precision and scale are required'; - } - return null; - case 'Decimal64': - if (options.scale === undefined) { - return 'Scale is required'; - } - return null; - case 'DateTime64': - if (options.precision === undefined) { - return 'Precision is required'; - } - return null; - case 'FixedString': - if (options.length === undefined) { - return 'Length is required'; - } - return null; - case 'Enum16': - if (options.values === undefined) { - return 'Values are required'; - } - return null; - default: - return null; - } + const missing = (findOption(override, options)?.parameters ?? []).filter( + ({ name }) => params[name] === undefined, + ); + if (missing.length === 0) return null; + + // A label such as `Values (comma separated)` names its input, so only the noun is used. + const [first, ...rest] = missing.map(({ label }) => label.split(' (')[0]); + const names = [first, ...rest.map((label) => label.toLowerCase())]; + return `${names.join(' and ')} required`; }; export const transformOverride = ( overrideType: string | null, - advancedOptions: TransformFunctionParams, + params: TransformFunctionParams, + options: OverrideOptions, ): string | null => { if (!overrideType) return null; - const option = ADVANCED_OPTIONS_MAP[overrideType]; + const option = findOption(overrideType, options); if (!option) return null; - return option.transformFunction?.(advancedOptions) ?? overrideType; + return ( + option.template?.replace(/\{(\w+)\}/g, (_, name: string) => + String(params[name] ?? ''), + ) ?? option.value + ); }; -export const getInitialOverrideType = (value: string | null) => { +/** + * The override type a stored value belongs to. A value that is an option whole, + * such as `Array(String)`, is that option. Any other has its arguments dropped, + * so `Decimal(10, 2)` is `Decimal`. + */ +export const getInitialOverrideType = ( + value: string | null, + options: OverrideOptions = [], +) => { if (!value) return null; + if (findOption(value, options)) return value; return value.split('(')[0].trim(); }; +const parseNum = (raw: string | undefined) => { + const parsed = parseInt(raw?.trim() ?? '0', 10); + return Number.isNaN(parsed) ? 0 : parsed; +}; + +/** + * The parameter values inside a stored override such as `Decimal(18, 4)`. + * + * Arguments are split on commas, and the last parameter takes whatever is left, + * so a text parameter may hold commas of its own. An integer reads as far as its + * first non-digit, which is how `3` comes out of `3, 'UTC'`. + */ export const getInitialAdvancedOptions = ( value: string | null, + options: OverrideOptions = [], ): TransformFunctionParams => { - const overrideType = getInitialOverrideType(value); - if (!overrideType) return {}; + const option = findOption(getInitialOverrideType(value, options), options); + if (!value || !option || option.parameters.length === 0) return {}; - const advancedOptions = value?.replace(')', '').split('(')[1]; - const parseNum = (raw: string | undefined) => { - const parsed = parseInt(raw?.trim() ?? '0', 10); - return Number.isNaN(parsed) ? 0 : parsed; - }; + const inner = value.slice( + value.indexOf('(') + 1, + value.lastIndexOf(')') === -1 ? undefined : value.lastIndexOf(')'), + ); + const pieces = inner.split(','); + const last = option.parameters.length - 1; - switch (overrideType) { - case 'Decimal': { - const [precision, scale] = (advancedOptions ?? '').split(','); - return { - precision: parseNum(precision), - scale: parseNum(scale), - }; - } - case 'Decimal64': - return { - scale: parseNum(advancedOptions), - }; - case 'DateTime64': - return { - precision: parseNum(advancedOptions?.split(',')[0]), - }; - case 'FixedString': - return { - length: parseNum(advancedOptions?.split(',')[0]), - }; - case 'Enum16': - return { - values: advancedOptions?.trim(), - }; - default: - return {}; - } + return Object.fromEntries( + option.parameters.map(({ name, kind }, index) => { + const raw = ( + index === last ? pieces.slice(index).join(',') : (pieces[index] ?? '') + ).trim(); + return [name, kind === 'integer' ? parseNum(raw) : raw]; + }), + ); }; diff --git a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/index.tsx b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/index.tsx index a8d025f9..0f3e6bd5 100644 --- a/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/index.tsx +++ b/apps/dfe-core-ui/src/core/components/CreateSchemaForm/AddSchemaTable/TypeOverrideSelect/index.tsx @@ -1,4 +1,5 @@ import { Popover } from '@/core/components/Popover'; +import { useFetchSchemaVocabulary } from '@/core/hooks/useFetchSchemaVocabulary'; import { IconDots, IconX } from '@repo/dfe-icons'; import { Button, FormInstance, SelectProps } from 'antd'; import { useState } from 'react'; @@ -19,11 +20,18 @@ export const TypeOverrideSelect = ({ ...props }: TypeOverrideSelectProps) => { const [open, setOpen] = useState(false); + const { data: vocabulary } = useFetchSchemaVocabulary(); + // A free-text argument can be long, so only the type name is shown for it. + const takesText = vocabulary?.ch_overrides.some( + (option) => + value?.startsWith(option.value) && + option.parameters.some(({ kind }) => kind === 'text'), + ); return (
{value && ( - {value.startsWith('Enum16') ? value.split('(')[0] : value} + {takesText ? value.split('(')[0] : value}
- ); + ); + }, }, - }, - { - title: 'Index Type', - dataIndex: 'use_case', - key: 'use_case', - render: (_: unknown, record: SchemaColumnListRow) => { - const { key: _rowKey, name: rowIndex, ...restField } = record; - return ( - - - - ); - }, - }, - { - title: 'Attributes', - dataIndex: 'attribute', - key: 'attribute', - width: 200, - render: (_: unknown, record: SchemaColumnListRow) => { - const { key: _rowKey, name: rowIndex, ...restField } = record; - return ( - - - - ); - }, - }, - { - title: 'Expression (CTE)', - dataIndex: 'expr', - key: 'expr', - render: (_: unknown, record: SchemaColumnListRow) => { - const { key: _rowKey, name: rowIndex, ...restField } = record; - return ( - - - - ); - }, - }, - { - title: 'Comment', - dataIndex: 'comment', - key: 'comment', - render: (_: unknown, record: SchemaColumnListRow) => { - const { key: _rowKey, name: rowIndex, ...restField } = record; - return ( - - - - ); + { + title: 'Attributes', + dataIndex: 'attribute', + key: 'attribute', + width: 200, + render: (_: unknown, record: SchemaColumnListRow) => { + const { key: _rowKey, name: rowIndex, ...restField } = record; + return ( + + + + ); + }, }, - }, - /** Ant Design Form only persists fields registered via Form.Item — `id` / `_field_type` must be stored for list merges and API mapping. */ - { - title: '', - key: '__rowId', - width: 0, - render: (_: unknown, record: SchemaColumnListRow) => { - const { key: _rowKey, name: rowIndex, ...restField } = record; - return ( - <> -