fix(console): read the column vocabulary from the engine

apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from 'vitest';
+import { buildFormSchema } from './EditSystemDefaultsForm';
+
+const values = (ttl_days: number) => ({
+  ttl_days,
+  engine: 'MergeTree',
+  common_header_type: 'common-header/timeseries',
+  common_header_version: '1.0.3',
+});
+
+describe('buildFormSchema', () => {
+  it.each([
+    [0, true],
+    [365, true],
+    [1000, true],
+    [36500, true],
+    [36501, false],
+    [-1, false],
+  ])(
+    'retention of %i days against a maximum of 36500 is valid: %s',
+    (days, ok) => {
+      expect(buildFormSchema(36500).safeParse(values(days)).success).toBe(ok);
+    },
+  );
+
+  it('takes the bound it is given, not one of its own', () => {
+    expect(buildFormSchema(30).safeParse(values(31)).success).toBe(false);
+    expect(buildFormSchema(30).safeParse(values(30)).success).toBe(true);
+  });
+});

apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/EditSystemDefaultsForm.tsx

@@ -10,14 +10,16 @@ import isInteger from 'lodash/isInteger';
 import { useCallback, useMemo, useState } from 'react';
 import z from 'zod';
 
-const formSchema = z.object({
-  ttl_days: z.number().min(0).max(365),
-  engine: z.string().min(1),
-  common_header_version: z.string().min(1),
-  common_header_type: z.string().min(1),
-});
+/** The retention bound is the engine's, served with the defaults. */
+export const buildFormSchema = (maxTtlDays: number) =>
+  z.object({
+    ttl_days: z.number().min(0).max(maxTtlDays),
+    engine: z.string().min(1),
+    common_header_version: z.string().min(1),
+    common_header_type: z.string().min(1),
+  });
 
-type TFormValues = z.infer<typeof formSchema>;
+type TFormValues = z.infer<ReturnType<typeof buildFormSchema>>;
 
 const normalizeHeaderType = (value?: string) =>
   (value ?? '').replace(/^common-header\//, '');
@@ -78,10 +80,12 @@ const hasDefaultsChanged = ({
 };
 export const EditSystemDefaultsForm = ({
   initialValues,
+  maxTtlDays,
   onSuccess,
   onClose,
 }: {
   initialValues?: TFormValues;
+  maxTtlDays: number;
   onSuccess?: (
     response: TUpdateSystemDefaultsResponse & { changed: boolean },
   ) => void;
@@ -116,6 +120,7 @@ export const EditSystemDefaultsForm = ({
   });
 
   const [form] = Form.useForm<TFormValues>();
+  const formSchema = useMemo(() => buildFormSchema(maxTtlDays), [maxTtlDays]);
   const formValidation = useAntdZodResolver(formSchema);
   const ttlDays = Form.useWatch('ttl_days', form);
   const engine = Form.useWatch('engine', form);

apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/SystemDefaultSettings.test.tsx

@@ -1,6 +1,7 @@
 import { API_CONFIG_MOCKS } from '@/core/config/api/endpoints/generator';
 import { buildTestWrapper } from '@/core/utils/test-utils/buildTestWrapper';
-import { render, screen } from '@testing-library/react';
+import { fireEvent, render, screen } from '@testing-library/react';
+import { delay, http, HttpResponse } from 'msw';
 import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest';
 import { SystemDefaultsSettings } from '.';
 import { server } from './SystemDefaultSettings.mocks';
@@ -11,6 +12,27 @@ afterAll(() => server.close());
 
 const { wrapper } = buildTestWrapper().withTheme().withReactQuery();
 
+const stringDefault = (effective: string) => ({
+  effective,
+  stored: null,
+  origin: 'deployment' as const,
+  deployment_default: effective,
+});
+
+const defaultsBody = {
+  ttl_days: {
+    effective: 90,
+    stored: null,
+    origin: 'deployment' as const,
+    deployment_default: 90,
+    maximum: 36500,
+  },
+  common_header_type: stringDefault('timeseries'),
+  common_header_version: stringDefault('1.0.3'),
+  engine: stringDefault('MergeTree'),
+  editable: true,
+};
+
 describe('SystemDefaultsSettings', () => {
   it('shows the environment default, with the editor closed', async () => {
     render(<SystemDefaultsSettings />, { wrapper });
@@ -36,6 +58,7 @@ describe('SystemDefaultsSettings', () => {
             stored: null,
             origin: 'deployment',
             deployment_default: 0,
+            maximum: 36500,
           },
           common_header_type: {
             effective: 'common-header/timeseries',
@@ -66,4 +89,47 @@ describe('SystemDefaultsSettings', () => {
       await screen.findByText('none (kept forever)', {}, { timeout: 15000 }),
     ).toBeInTheDocument();
   });
+
+  it('says the defaults cannot be edited when they failed to load', async () => {
+    server.use(API_CONFIG_MOCKS.system.defaults.get.error());
+    render(<SystemDefaultsSettings />, { wrapper });
+
+    fireEvent.click(
+      await screen.findByRole('button', { name: 'System defaults actions' }),
+    );
+    fireEvent.click(await screen.findByText('Edit Defaults'));
+
+    expect(
+      await screen.findByText(/could not be loaded, so they cannot be edited/),
+    ).toBeInTheDocument();
+    fireEvent.click(screen.getByRole('button', { name: 'Close' }));
+    expect(
+      screen.queryByText(/so they cannot be edited/),
+    ).not.toBeInTheDocument();
+  });
+
+  it('shows the same error notice as the edit form, and no zero values, when the defaults failed', async () => {
+    server.use(API_CONFIG_MOCKS.system.defaults.get.error());
+    render(<SystemDefaultsSettings />, { wrapper });
+
+    expect(
+      await screen.findByText(/The defaults could not be loaded:/),
+    ).toBeInTheDocument();
+    expect(screen.queryByText('none (kept forever)')).not.toBeInTheDocument();
+    expect(screen.queryByText(/@v/)).not.toBeInTheDocument();
+  });
+
+  it('says it is loading, and shows no zero values, until the defaults arrive', async () => {
+    server.use(
+      http.get(API_CONFIG_MOCKS.system.defaults.mockedUrl, async () => {
+        await delay(300);
+        return HttpResponse.json(defaultsBody);
+      }),
+    );
+    render(<SystemDefaultsSettings />, { wrapper });
+
+    expect(await screen.findByText('Loading defaults...')).toBeInTheDocument();
+    expect(screen.queryByText('none (kept forever)')).not.toBeInTheDocument();
+    expect(await screen.findByText('90 days')).toBeInTheDocument();
+  });
 });

apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/ViewSystemDefaults.tsx

@@ -1,4 +1,4 @@
-import { NotificationCard } from '@/core/components/NotificationCard';
+import { FormNotification } from '@/core/components/FormNotification';
 import { RbacProtected } from '@/core/components/RbacProtected';
 import { TSystemDefaultsResponse } from '@/core/hooks/useFetchDefaults/types';
 
@@ -9,27 +9,37 @@ const formatDays = (days: number) =>
 export const ViewSystemDefaults = ({
   defaults,
   error,
+  isLoading = false,
 }: {
   defaults?: TSystemDefaultsResponse;
   error: Error | null;
+  isLoading?: boolean;
 }) => {
   return (
     <RbacProtected action={RbacProtected.rbacActions.system_read}>
       <RbacProtected.Unrestricted>
         <>
-          {error && <NotificationCard type="error" title={error.message} />}
-          <dl className="grid grid-rows-[34px] max-[1452px]:grid-cols-[150px_1fr] grid-cols-[150px_1fr_100px_1fr] gap-x-4 gap-y-1 text-xs items-center">
-            <dt className={dataListTermStyle}>Default Retention</dt>
-            <dd>{formatDays(defaults?.ttl_days?.effective ?? 0)}</dd>
+          {error ? (
+            <FormNotification
+              type="error"
+              text={`The defaults could not be loaded: ${error.message}`}
+            />
+          ) : isLoading || !defaults ? (
+            <p className={dataListTermStyle}>Loading defaults...</p>
+          ) : (
+            <dl className="grid grid-rows-[34px] max-[1452px]:grid-cols-[150px_1fr] grid-cols-[150px_1fr_100px_1fr] gap-x-4 gap-y-1 text-xs items-center">
+              <dt className={dataListTermStyle}>Default Retention</dt>
+              <dd>{formatDays(defaults.ttl_days.effective)}</dd>
 
-            <dt className={dataListTermStyle}>Default Engine</dt>
-            <dd>{defaults?.engine?.effective ?? ''}</dd>
-            <dt className={dataListTermStyle}>Default Header & Version</dt>
-            <dd className="max-[1452px]:col-span-1 col-span-3">
-              {defaults?.common_header_type?.effective ?? ''}@v
-              {defaults?.common_header_version?.effective ?? ''}
-            </dd>
-          </dl>
+              <dt className={dataListTermStyle}>Default Engine</dt>
+              <dd>{defaults.engine.effective}</dd>
+              <dt className={dataListTermStyle}>Default Header & Version</dt>
+              <dd className="max-[1452px]:col-span-1 col-span-3">
+                {defaults.common_header_type.effective}@v
+                {defaults.common_header_version.effective}
+              </dd>
+            </dl>
+          )}
           <p className="mt-2 break-words text-xs text-foreground/50 dark:text-foreground/50">
             Set by DFE_ENGINE_CLICKHOUSE_DEFAULT_TTL_DAYS. A change applies to
             every table that follows the default when the engine restarts.

apps/dfe-core-ui/src/Platform/components/SystemManagement/SystemDefaultsSettings/index.tsx

@@ -1,12 +1,14 @@
+import { FormNotification } from '@/core/components/FormNotification';
 import { SectionCard } from '@/core/components/SectionCard';
 import { useFetchDefaults } from '@/core/hooks/useFetchDefaults';
+import { Button } from 'antd';
 import { useState } from 'react';
 import { EditSystemDefaultsForm } from './EditSystemDefaultsForm';
 import { SystemDefaultsActionMenu } from './SystemDefaultsActionMenu';
 import { ViewSystemDefaults } from './ViewSystemDefaults';
 
 export const SystemDefaultsSettings = () => {
-  const { data: defaults, error } = useFetchDefaults();
+  const { data: defaults, error, isLoading } = useFetchDefaults();
 
   const [isEditDefaultsOpen, setIsEditDefaultsOpen] = useState(false);
   const [hasDefaultsChanged, setHasDefaultsChanged] = useState(false);
@@ -22,23 +24,37 @@ export const SystemDefaultsSettings = () => {
         />
       }
     >
-      {isEditDefaultsOpen ? (
+      {isEditDefaultsOpen && !defaults && error ? (
+        <div className="flex flex-col gap-2">
+          <FormNotification
+            type="error"
+            text={`The defaults could not be loaded, so they cannot be edited: ${error.message}`}
+          />
+          <div className="flex justify-end">
+            <Button onClick={() => setIsEditDefaultsOpen(false)}>Close</Button>
+          </div>
+        </div>
+      ) : isEditDefaultsOpen && defaults ? (
         <EditSystemDefaultsForm
           onClose={() => setIsEditDefaultsOpen(false)}
           onSuccess={({ changed }) => {
             setIsEditDefaultsOpen(false);
             setHasDefaultsChanged(changed);
           }}
+          maxTtlDays={defaults.ttl_days.maximum}
           initialValues={{
-            ttl_days: defaults?.ttl_days?.effective ?? 0,
-            engine: defaults?.engine?.effective ?? '',
-            common_header_version:
-              defaults?.common_header_version?.effective ?? '',
-            common_header_type: `common-header/${defaults?.common_header_type?.effective?.replace('common-header/', '')}`,
+            ttl_days: defaults.ttl_days.effective,
+            engine: defaults.engine.effective,
+            common_header_version: defaults.common_header_version.effective,
+            common_header_type: `common-header/${defaults.common_header_type.effective.replace('common-header/', '')}`,
           }}
         />
       ) : (
-        <ViewSystemDefaults defaults={defaults} error={error} />
+        <ViewSystemDefaults
+          defaults={defaults}
+          error={error}
+          isLoading={isLoading}
+        />
       )}
     </SectionCard>
   );

apps/dfe-core-ui/src/Platform/hooks/system/useUpdateDefaults/useUpdateDefaults.test.tsx

@@ -57,6 +57,7 @@ describe('.useUpdateSystemDefaults', () => {
           stored: null,
           origin: 'deployment',
           deployment_default: 90,
+          maximum: 36500,
         },
         common_header_type: {
           effective: 'string',

apps/dfe-core-ui/src/Schemas/components/CreateDerivedSchemaDrawer/CreateDerivedSchemaDrawer.mocks.ts

@@ -60,6 +60,7 @@ export const server = setupServer(
   API_CONFIG_MOCKS.schemas.default.get.success({
     mockedResponse: SCHEMA_LIST_RESPONSE,
   }),
+  API_CONFIG_MOCKS.schemas.vocabulary.get.success(),
   API_CONFIG_MOCKS.schemas.schemaDetail.get.success({
     mockedResponse: BASE_COLUMNS_RESPONSE,
     schema_path: BASE_SCHEMA_PATH,

apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.test.ts

@@ -6,6 +6,7 @@ import {
   selectEntry,
   withIndexUseCase,
 } from '@/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers';
+import { SCHEMA_VOCABULARY_FIXTURE } from '@/core/hooks/useFetchSchemaVocabulary/vocabulary.fixture';
 import { describe, expect, it } from 'vitest';
 
 const baseColumns: DerivedBaseColumn[] = [
@@ -17,14 +18,21 @@ const baseColumns: DerivedBaseColumn[] = [
 
 describe('baseColumnFrom', () => {
   it('reads the base column index through the rename', () => {
-    expect(baseColumnFrom({ name: 'message', use_case: 'fulltext' })).toEqual({
+    expect(
+      baseColumnFrom(
+        { name: 'message', use_case: 'fulltext' },
+        SCHEMA_VOCABULARY_FIXTURE,
+      ),
+    ).toEqual({
       name: 'message',
       baseIndex: 'word_search',
     });
   });
 
   it('gives a column with no index use case none', () => {
-    expect(baseColumnFrom({ name: 'log_offset' })).toEqual({
+    expect(
+      baseColumnFrom({ name: 'log_offset' }, SCHEMA_VOCABULARY_FIXTURE),
+    ).toEqual({
       name: 'log_offset',
       baseIndex: 'none',
     });

apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.helpers.ts

@@ -1,3 +1,4 @@
+import type { TSchemaVocabularyResponse } from '@/core/hooks/useFetchSchemaVocabulary/types';
 import {
   IndexUseCase,
   resolveIndexUseCase,
@@ -20,12 +21,15 @@ export type DerivedBaseColumn = {
   baseIndex: IndexUseCase;
 };
 
-export const baseColumnFrom = (column: {
-  name?: string | null;
-  use_case?: string | null;
-}): DerivedBaseColumn => ({
+export const baseColumnFrom = (
+  column: {
+    name?: string | null;
+    use_case?: string | null;
+  },
+  vocabulary: TSchemaVocabularyResponse,
+): DerivedBaseColumn => ({
   name: column.name ?? '',
-  baseIndex: resolveIndexUseCase(column.use_case),
+  baseIndex: resolveIndexUseCase(column.use_case, vocabulary),
 });
 
 /** An omitted `index` keeps the base column's, so only a change is written. */

apps/dfe-core-ui/src/Schemas/components/DerivedSchemaFieldPicker/DerivedSchemaFieldPicker.mocks.ts

@@ -49,6 +49,7 @@ export const BASE_COLUMNS_RESPONSE: TMetaSchemaDetailResponse = {
 };
 
 export const server = setupServer(
+  API_CONFIG_MOCKS.schemas.vocabulary.get.success(),
   API_CONFIG_MOCKS.schemas.schemaDetail.get.success({
     mockedResponse: BASE_COLUMNS_RESPONSE,
     schema_path: BASE_SCHEMA_PATH,