feat(eda): log retention Phase 2 UI — paginated logs, DEBUG toggle, clear logs

frontend/eda/interfaces/EdaRulebookActivation.ts

@@ -12,5 +12,6 @@ export type EdaRulebookActivation = Omit<ActivationRead, 'project'> & {
 export type EdaRulebookActivationCreate = ActivationCreate & {
   enable_persistence?: boolean;
   rule_engine_credential_id?: number | null;
+  store_debug_logs?: boolean;
 };
 export type EdaRulebookActivationStatus = StatusEnum;

frontend/eda/rulebook-activations/ActivationInstancePage/ActivationInstanceEvents.tsx

@@ -2,9 +2,8 @@ import { IFilterState, IToolbarFilter } from '@ansible/ansible-ui-framework';
 import { AwxItemsResponse } from '@ansible/awx-ui/common/AwxItemsResponse';
 import { useScrollControls } from '@ansible/awx-ui/views/jobs/JobOutput/useScrollControls';
 import { requestGet } from '@ansible/common-ui/crud/Data';
-import { useGet } from '@ansible/common-ui/crud/useGet';
 import { PageSection } from '@patternfly/react-core';
-import { useEffect, useRef, useState } from 'react';
+import { useCallback, useEffect, useRef, useState } from 'react';
 import { useParams } from 'react-router-dom';
 import styled from 'styled-components';
 import { useVirtualizedList } from '../../..//common/utils/useVirtualized';
@@ -14,6 +13,9 @@ import { edaAPI } from '../../common/eda-utils';
 import { EdaActivationInstanceLog } from '../../interfaces/EdaActivationInstanceLog';
 import { ActivationInstanceOutputRow } from './ActivationInstanceOutputRow';
 
+const INITIAL_PAGE_SIZE = 5000;
+const POLL_INTERVAL_MS = 5000;
+
 const ScrollContainer = styled.div`
   overflow: auto;
   background-color: var(--pf-t--global--background--color--primary--default);
@@ -38,57 +40,122 @@ interface IActivationInstanceEventsProps {
 }
 
 export function ActivationInstanceEvents(props: IActivationInstanceEventsProps) {
-  const [activationInstanceLog, setActivationInstanceLog] =
-    useState<AwxItemsResponse<EdaActivationInstanceLog>>();
+  const [logs, setLogs] = useState<EdaActivationInstanceLog[]>([]);
+  const [hasOlderLogs, setHasOlderLogs] = useState(false);
+  const latestTimestampRef = useRef<number>(0);
 
   const params = useParams<{ instanceId: string }>();
   const { toolbarFilters, filterState, isFollowModeEnabled, setIsFollowModeEnabled, isRunning } =
     props;
 
-  const { data: activationInstanceLogInfo } = useGet<AwxItemsResponse<EdaActivationInstanceLog>>(
-    edaAPI`/activation-instances/${params.instanceId ?? ''}/logs/?page_size=1`
-  );
+  const buildFilterString = useCallback(() => {
+    return getFiltersQueryString(toolbarFilters, filterState);
+  }, [toolbarFilters, filterState]);
 
   useEffect(() => {
-    async function fetchData() {
-      const filterString = getFiltersQueryString(toolbarFilters, filterState);
-      const qsParts = [`page_size=${activationInstanceLogInfo?.count.toString() ?? '10'}`];
+    async function initialLoad() {
+      const filterString = buildFilterString();
+      const qsParts = [`page_size=${INITIAL_PAGE_SIZE}`, 'ordering=-id'];
       if (filterString) {
         qsParts.push(filterString);
       }
-      const activationInstanceLogOutput = await requestGet<
-        AwxItemsResponse<EdaActivationInstanceLog>
-      >(
+      const response = await requestGet<AwxItemsResponse<EdaActivationInstanceLog>>(
         edaAPI`/activation-instances/${params.instanceId ?? ''}/logs/`.concat(
           `?${qsParts.join('&')}`
         )
       );
 
-      setActivationInstanceLog(activationInstanceLogOutput);
+      const results = [...(response.results ?? [])].reverse();
+      setLogs(results);
+      setHasOlderLogs((response.count ?? 0) > INITIAL_PAGE_SIZE);
+
+      if (results.length > 0) {
+        const lastLog = results[results.length - 1];
+        latestTimestampRef.current = lastLog.log_timestamp ?? 0;
+      }
     }
 
-    void fetchData();
-  }, [params.instanceId, activationInstanceLogInfo?.count, toolbarFilters, filterState]);
+    void initialLoad();
+  }, [params.instanceId, buildFilterString]);
+
+  useEffect(() => {
+    if (!isRunning && !isFollowModeEnabled) return;
+
+    const interval = setInterval(async () => {
+      if (latestTimestampRef.current === 0) return;
+
+      const filterString = buildFilterString();
+      const qsParts = [
+        `log_timestamp__gt=${latestTimestampRef.current}`,
+        `page_size=${INITIAL_PAGE_SIZE}`,
+      ];
+      if (filterString) {
+        qsParts.push(filterString);
+      }
+
+      const response = await requestGet<AwxItemsResponse<EdaActivationInstanceLog>>(
+        edaAPI`/activation-instances/${params.instanceId ?? ''}/logs/`.concat(
+          `?${qsParts.join('&')}`
+        )
+      );
 
-  const estimatedMaxLines = (activationInstanceLog?.results.length ?? 0) * 10;
+      const newLogs = response.results ?? [];
+      if (newLogs.length > 0) {
+        setLogs((prev) => [...prev, ...newLogs]);
+        const lastLog = newLogs[newLogs.length - 1];
+        latestTimestampRef.current = lastLog.log_timestamp ?? latestTimestampRef.current;
+      }
+    }, POLL_INTERVAL_MS);
+
+    return () => clearInterval(interval);
+  }, [params.instanceId, isRunning, isFollowModeEnabled, buildFilterString]);
+
+  const loadOlderLogs = useCallback(async () => {
+    if (logs.length === 0 || !hasOlderLogs) return;
+
+    const oldestTimestamp = logs[0].log_timestamp ?? 0;
+    const filterString = buildFilterString();
+    const qsParts = [
+      `log_timestamp__lt=${oldestTimestamp}`,
+      `page_size=${INITIAL_PAGE_SIZE}`,
+      'ordering=-id',
+    ];
+    if (filterString) {
+      qsParts.push(filterString);
+    }
+
+    const response = await requestGet<AwxItemsResponse<EdaActivationInstanceLog>>(
+      edaAPI`/activation-instances/${params.instanceId ?? ''}/logs/`.concat(
+        `?${qsParts.join('&')}`
+      )
+    );
+
+    const olderLogs = [...(response.results ?? [])].reverse();
+    if (olderLogs.length > 0) {
+      setLogs((prev) => [...olderLogs, ...prev]);
+    }
+    setHasOlderLogs((response.count ?? 0) > INITIAL_PAGE_SIZE);
+  }, [logs, hasOlderLogs, params.instanceId, buildFilterString]);
+
+  const estimatedMaxLines = (logs.length ?? 0) * 10;
   const outputLineChars = String(estimatedMaxLines).length;
   const containerRef = useRef<HTMLDivElement>(null);
 
   const { handleScroll, scrollToTop, scrollToBottom, scrollPageDown, scrollPageUp } =
-    useScrollControls(
-      containerRef,
-      isFollowModeEnabled,
-      setIsFollowModeEnabled,
-      activationInstanceLog?.results.length ?? 0,
-      isRunning
-    );
+    useScrollControls(containerRef, isFollowModeEnabled, setIsFollowModeEnabled, logs.length, isRunning);
+
+  const onScroll = useCallback(
+    (el: HTMLElement) => {
+      handleScroll(el);
+      if (el.scrollTop === 0 && hasOlderLogs) {
+        void loadOlderLogs();
+      }
+    },
+    [handleScroll, hasOlderLogs, loadOlderLogs]
+  );
 
   const { beforeRowsHeight, visibleItems, afterRowsHeight, setRowHeight } =
-    useVirtualizedList<EdaActivationInstanceLog>(
-      containerRef,
-      activationInstanceLog?.results ?? [],
-      handleScroll
-    );
+    useVirtualizedList<EdaActivationInstanceLog>(containerRef, logs, onScroll);
 
   return (
     <Section>
@@ -110,7 +177,7 @@ export function ActivationInstanceEvents(props: IActivationInstanceEventsProps)
             {visibleItems?.map((row) => (
               <ActivationInstanceOutputRow
                 key={row.id}
-                index={activationInstanceLog?.results.findIndex((r) => r.id === row.id) ?? 0}
+                index={logs.findIndex((r) => r.id === row.id)}
                 row={row}
                 setHeight={setRowHeight}
               />

frontend/eda/rulebook-activations/RulebookActivationForm.tsx

@@ -197,6 +197,10 @@ export function RulebookActivationInputs() {
     name: 'enable_persistence',
   }) as boolean;
 
+  const logLevel = useWatch<IEdaRulebookActivationInputs>({
+    name: 'log_level',
+  }) as string;
+
   useEffect(() => {
     setValue('source_mappings', jsyaml.dump(sourceMappings));
   }, [setValue, sourceMappings]);
@@ -222,6 +226,12 @@ export function RulebookActivationInputs() {
     }
   }, [enablePersistence, config?.managed_cloud_install, setValue]);
 
+  useEffect(() => {
+    if (logLevel !== 'debug') {
+      setValue('store_debug_logs', false);
+    }
+  }, [logLevel, setValue]);
+
   return (
     <>
       <PageFormTextInput<IEdaRulebookActivationInputs>
@@ -282,6 +292,26 @@ export function RulebookActivationInputs() {
         labelHelp={logLevelHelpBlock}
         labelHelpTitle={t('Log level')}
       />
+      {logLevel === 'debug' && (
+        <>
+          <Alert
+            variant="warning"
+            isInline
+            isPlain
+            title={t(
+              'Debug logging generates significantly more data. By default, debug lines are sent to container stdout but not stored in the database. Enable "Store debug logs in database" below to persist them, but be aware this can increase database storage by ~225x.'
+            )}
+          />
+          <PageFormCheckbox<IEdaRulebookActivationInputs>
+            label={t`Store debug logs in database`}
+            labelHelpTitle={t('Store debug logs in database')}
+            labelHelp={t(
+              'When enabled, DEBUG-level log lines are stored in the database in addition to container stdout. When disabled (default), DEBUG lines are only available in container logs, significantly reducing database storage.'
+            )}
+            name="store_debug_logs"
+          />
+        </>
+      )}
       {config?.deployment_type === 'k8s' && (
         <PageFormTextInput<IEdaRulebookActivationInputs>
           name="k8s_service_name"

frontend/eda/rulebook-activations/RulebookActivationPage/RulebookActivationHistory.tsx

@@ -1,5 +1,14 @@
-import { PageLayout, PageTable } from '@ansible/ansible-ui-framework';
-import { CubesIcon } from '@patternfly/react-icons';
+import {
+  IPageAction,
+  PageActionSelection,
+  PageActionType,
+  PageLayout,
+  PageTable,
+  usePageAlertToaster,
+} from '@ansible/ansible-ui-framework';
+import { postRequest } from '@ansible/common-ui/crud/Data';
+import { CubesIcon, TimesCircleIcon } from '@patternfly/react-icons';
+import { useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useParams } from 'react-router-dom';
 import { edaAPI } from '../../common/eda-utils';
@@ -11,6 +20,7 @@ import { useActivationHistoryFilters } from '../hooks/useActivationHistoryFilter
 export function RulebookActivationHistory() {
   const params = useParams<{ id: string }>();
   const { t } = useTranslation();
+  const alertToaster = usePageAlertToaster();
 
   const toolbarFilters = useActivationHistoryFilters();
 
@@ -20,10 +30,55 @@ export function RulebookActivationHistory() {
     toolbarFilters,
     tableColumns,
   });
+
+  const clearLogs = useCallback(async () => {
+    const activationId = params?.id;
+    if (!activationId) return;
+    if (
+      !confirm(
+        t('Are you sure you want to clear all logs for this activation? This action is irreversible.')
+      )
+    ) {
+      return;
+    }
+    try {
+      const result = await postRequest<{ deleted: number }>(
+        edaAPI`/activations/${activationId}/clear-logs/`,
+        {}
+      );
+      alertToaster.addAlert({
+        variant: 'success',
+        title: t('Cleared {{count}} log records.', { count: result.deleted }),
+        timeout: 5000,
+      });
+    } catch {
+      alertToaster.addAlert({
+        variant: 'danger',
+        title: t('Failed to clear logs'),
+        timeout: 5000,
+      });
+    }
+  }, [params?.id, alertToaster, t]);
+
+  const toolbarActions = useMemo<IPageAction<EdaActivationInstance>[]>(
+    () => [
+      {
+        type: PageActionType.Button,
+        selection: PageActionSelection.None,
+        icon: TimesCircleIcon,
+        label: t('Clear logs'),
+        onClick: () => clearLogs(),
+        isDanger: true,
+      },
+    ],
+    [clearLogs, t]
+  );
+
   return (
     <PageLayout>
       <PageTable
         tableColumns={tableColumns}
+        toolbarActions={toolbarActions}
         toolbarFilters={toolbarFilters}
         errorStateTitle={t('Error loading history')}
         emptyStateTitle={t('No activation history')}

frontend/eda/rulebook-activations/hooks/useRulebookActivationsActions.tsx

@@ -8,7 +8,13 @@ import {
 import { postRequest } from '@ansible/common-ui/crud/Data';
 import { useOptions } from '@ansible/common-ui/crud/useOptions';
 import { AlertProps, ButtonVariant } from '@patternfly/react-core';
-import { MinusCircleIcon, PlusCircleIcon, RedoIcon, TrashIcon } from '@patternfly/react-icons';
+import {
+  MinusCircleIcon,
+  PlusCircleIcon,
+  RedoIcon,
+  TimesCircleIcon,
+  TrashIcon,
+} from '@patternfly/react-icons';
 import { useCallback, useMemo } from 'react';
 import { useTranslation } from 'react-i18next';
 import { edaAPI, hasCopyNamePattern } from '../../common/eda-utils';
@@ -29,6 +35,7 @@ import {
   useDeleteRulebookActivationsWithWarning,
 } from './useDeleteRulebookActivations';
 import { StatusEnum } from '../../interfaces/generated/eda-api';
+import { useEdaActiveUser } from '../../common/useEdaActiveUser';
 
 export function useRulebookActivationsActions(view: IEdaView<EdaRulebookActivation>) {
   const { t } = useTranslation();
@@ -52,6 +59,29 @@ export function useRulebookActivationsActions(view: IEdaView<EdaRulebookActivati
     view.unselectItemsAndRefresh
   );
   const getPageUrl = useGetPageUrl();
+  const { activeEdaUser } = useEdaActiveUser();
+
+  const clearAllLogs = useCallback(async () => {
+    if (!confirm(t('Are you sure you want to clear ALL activation logs? This action is irreversible.'))) {
+      return;
+    }
+    try {
+      const result = await postRequest<{ deleted: number }>(edaAPI`/logs/purge/`, {});
+      alertToaster.addAlert({
+        variant: 'success',
+        title: t('Cleared {{count}} log records.', { count: result.deleted }),
+        timeout: 5000,
+      });
+    } catch (err) {
+      const errorResults = parseError(err as Error);
+      alertToaster.addAlert({
+        variant: 'danger',
+        title: t('Failed to clear all logs'),
+        children: <>{errorResults.parsedErrors.map((errorResult) => errorResult.message)}</>,
+        timeout: 5000,
+      });
+    }
+  }, [alertToaster, parseError, t]);
   const enableRulebookActivation: (activation: EdaRulebookActivation) => Promise<void> =
     useCallback(
       async (activation) => {
@@ -168,6 +198,18 @@ export function useRulebookActivationsActions(view: IEdaView<EdaRulebookActivati
       {
         type: PageActionType.Seperator,
       },
+      ...(activeEdaUser?.is_superuser
+        ? [
+            {
+              type: PageActionType.Button as const,
+              selection: PageActionSelection.None as const,
+              icon: TimesCircleIcon,
+              label: t('Clear all logs'),
+              onClick: () => clearAllLogs(),
+              isDanger: true,
+            },
+          ]
+        : []),
       {
         type: PageActionType.Button,
         selection: PageActionSelection.Multiple,
@@ -186,6 +228,8 @@ export function useRulebookActivationsActions(view: IEdaView<EdaRulebookActivati
     disableRulebookActivations,
     restartRulebookActivations,
     deleteRulebookActivations,
+    clearAllLogs,
+    activeEdaUser?.is_superuser,
     getPageUrl,
   ]);
 }