diff --git a/src/aap_eda/api/filters/activation.py b/src/aap_eda/api/filters/activation.py index 413013e1d..d49abda37 100644 --- a/src/aap_eda/api/filters/activation.py +++ b/src/aap_eda/api/filters/activation.py @@ -68,7 +68,17 @@ class ActivationInstanceLogFilter(django_filters.FilterSet): lookup_expr="icontains", label="Filter by activation instance log.", ) + log_timestamp__gt = django_filters.NumberFilter( + field_name="log_timestamp", + lookup_expr="gt", + label="Filter logs with timestamp greater than value.", + ) + log_timestamp__lt = django_filters.NumberFilter( + field_name="log_timestamp", + lookup_expr="lt", + label="Filter logs with timestamp less than value.", + ) class Meta: model = models.RulebookProcessLog - fields = ["log"] + fields = ["log", "log_timestamp__gt", "log_timestamp__lt"] diff --git a/src/aap_eda/api/serializers/__init__.py b/src/aap_eda/api/serializers/__init__.py index 9c072c486..3c0a069e0 100644 --- a/src/aap_eda/api/serializers/__init__.py +++ b/src/aap_eda/api/serializers/__init__.py @@ -21,6 +21,8 @@ ActivationReadSerializer, ActivationSerializer, ActivationUpdateSerializer, + LogPurgeRequestSerializer, + LogPurgeResponseSerializer, PostActivationSerializer, ) from .auth import JWTTokenSerializer, LoginSerializer, RefreshTokenSerializer diff --git a/src/aap_eda/api/serializers/activation.py b/src/aap_eda/api/serializers/activation.py index 44998b3ca..ae957fd75 100644 --- a/src/aap_eda/api/serializers/activation.py +++ b/src/aap_eda/api/serializers/activation.py @@ -485,6 +485,7 @@ class Meta: "status_message", "awx_token_id", "log_level", + "store_debug_logs", "eda_credentials", "k8s_service_name", "event_streams", @@ -557,6 +558,7 @@ def to_representation(self, activation): "status_message": activation.status_message, "awx_token_id": activation.awx_token_id, "log_level": activation.log_level, + "store_debug_logs": activation.store_debug_logs, "eda_credentials": eda_credentials, "k8s_service_name": activation.k8s_service_name, "event_streams": event_streams, @@ -595,6 +597,7 @@ class Meta: "restart_policy", "awx_token_id", "log_level", + "store_debug_logs", "eda_credentials", "k8s_service_name", "source_mappings", @@ -750,6 +753,7 @@ def copy(self) -> dict: "restart_policy": activation.restart_policy, "awx_token_id": activation.awx_token, "log_level": activation.log_level, + "store_debug_logs": activation.store_debug_logs, "eda_credentials": activation.eda_credentials.all(), "k8s_service_name": activation.k8s_service_name, "source_mappings": activation.source_mappings, @@ -803,6 +807,7 @@ class Meta: "restart_policy", "awx_token_id", "log_level", + "store_debug_logs", "eda_credentials", "k8s_service_name", "source_mappings", @@ -1157,6 +1162,7 @@ class Meta: "awx_token_id", "eda_credentials", "log_level", + "store_debug_logs", "k8s_service_name", "k8s_pod_service_account_name", "k8s_pod_labels", @@ -1299,6 +1305,7 @@ def to_representation(self, activation): "status_message": activation.status_message, "awx_token_id": activation.awx_token_id, "log_level": activation.log_level, + "store_debug_logs": activation.store_debug_logs, "eda_credentials": eda_credentials, "k8s_service_name": activation.k8s_service_name, **_activation_k8s_pod_metadata_payload(activation), @@ -1816,3 +1823,22 @@ def _validate_persistence_credential(data: dict) -> None: f"'{settings.DEFAULT_SYSTEM_RULE_ENGINE_CREDENTIAL_NAME}' " "could not be found. Contact your system administrator." ) + + +class LogPurgeRequestSerializer(serializers.Serializer): + """Serializer for log purge request body.""" + + before_date = serializers.DateTimeField( + required=False, + allow_null=True, + help_text="Delete logs older than this date. " + "If omitted, all logs are deleted.", + ) + + +class LogPurgeResponseSerializer(serializers.Serializer): + """Serializer for log purge response.""" + + deleted = serializers.IntegerField( + help_text="Number of log records deleted.", + ) diff --git a/src/aap_eda/api/urls.py b/src/aap_eda/api/urls.py index 83c673d29..e7b56485c 100644 --- a/src/aap_eda/api/urls.py +++ b/src/aap_eda/api/urls.py @@ -56,6 +56,7 @@ router.register("organizations", views.OrganizationViewSet) router.register("teams", views.TeamViewSet) router.register("event-streams", views.EventStreamViewSet) +router.register("logs", views.LogPurgeViewSet, basename="logs") router.register( "external_event_stream", views.ExternalEventStreamViewSet, diff --git a/src/aap_eda/api/views/__init__.py b/src/aap_eda/api/views/__init__.py index 3b61fdf77..472c47e2e 100644 --- a/src/aap_eda/api/views/__init__.py +++ b/src/aap_eda/api/views/__init__.py @@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from .activation import ActivationInstanceViewSet, ActivationViewSet +from .activation import ( + ActivationInstanceViewSet, + ActivationViewSet, + LogPurgeViewSet, +) from .auth import SessionLoginView, SessionLogoutView, TokenRefreshView from .config import ConfigView from .credential_input_source import CredentialInputSourceViewSet @@ -40,6 +44,8 @@ # activations "ActivationViewSet", "ActivationInstanceViewSet", + # logs + "LogPurgeViewSet", # user "CurrentUserView", "CurrentUserAwxTokenViewSet", diff --git a/src/aap_eda/api/views/activation.py b/src/aap_eda/api/views/activation.py index 95d21d939..9abfe2e67 100644 --- a/src/aap_eda/api/views/activation.py +++ b/src/aap_eda/api/views/activation.py @@ -37,6 +37,11 @@ from aap_eda.core.enums import Action, ActivationStatus, ProcessParentType from aap_eda.core.health import check_dispatcherd_workers_health from aap_eda.core.utils import logging_utils +from aap_eda.core.utils.delete_log_util import ( + delete_all_logs, + delete_logs_for_activation, + delete_logs_older_than, +) from aap_eda.tasks.orchestrator import ( delete_rulebook_process, restart_rulebook_process, @@ -678,6 +683,41 @@ def copy(self, request, pk): status=status.HTTP_201_CREATED, ) + @extend_schema( + request=serializers.LogPurgeRequestSerializer, + responses={ + status.HTTP_200_OK: serializers.LogPurgeResponseSerializer, + status.HTTP_404_NOT_FOUND: OpenApiResponse( + None, description="Activation not found." + ), + }, + ) + @action( + methods=["post"], + detail=True, + rbac_action=Action.DELETE, + url_path="clear-logs", + ) + def clear_logs(self, request, pk): + activation = self.get_object() + request_serializer = serializers.LogPurgeRequestSerializer( + data=request.data, + ) + request_serializer.is_valid(raise_exception=True) + before_date = request_serializer.validated_data.get("before_date") + + if before_date: + deleted = delete_logs_older_than( + before_date, activation_id=activation.id + ) + else: + deleted = delete_logs_for_activation(activation.id) + + return Response( + serializers.LogPurgeResponseSerializer({"deleted": deleted}).data, + status=status.HTTP_200_OK, + ) + def _sync_project_if_needed( self, activation: models.Activation ) -> Response | None: @@ -874,3 +914,37 @@ def logs(self, request, id): results, many=True ) return self.get_paginated_response(serializer.data) + + +class LogPurgeViewSet(viewsets.ViewSet): + """Global log purge endpoint (admin only).""" + + @extend_schema( + request=serializers.LogPurgeRequestSerializer, + responses={ + status.HTTP_200_OK: serializers.LogPurgeResponseSerializer, + }, + ) + @action( + methods=["post"], + detail=False, + url_path="purge", + ) + def purge(self, request): + if not request.user.is_superuser: + raise exceptions.PermissionDenied( + "Only administrators can purge all logs." + ) + + request_serializer = serializers.LogPurgeRequestSerializer( + data=request.data, + ) + request_serializer.is_valid(raise_exception=True) + before_date = request_serializer.validated_data.get("before_date") + + deleted = delete_all_logs(cutoff=before_date) + + return Response( + serializers.LogPurgeResponseSerializer({"deleted": deleted}).data, + status=status.HTTP_200_OK, + ) diff --git a/src/aap_eda/core/migrations/0074_activation_store_debug_logs.py b/src/aap_eda/core/migrations/0074_activation_store_debug_logs.py new file mode 100644 index 000000000..3505a1807 --- /dev/null +++ b/src/aap_eda/core/migrations/0074_activation_store_debug_logs.py @@ -0,0 +1,20 @@ +# Generated by Django 5.2.9 on 2026-08-11 02:17 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0073_activation_k8s_pod_tolerations"), + ] + + operations = [ + migrations.AddField( + model_name="activation", + name="store_debug_logs", + field=models.BooleanField( + default=False, + help_text="When false (default), DEBUG-level log lines are sent to container stdout but not stored in the database.", + ), + ), + ] diff --git a/src/aap_eda/core/models/activation.py b/src/aap_eda/core/models/activation.py index 8eef9d170..9afb894b6 100644 --- a/src/aap_eda/core/models/activation.py +++ b/src/aap_eda/core/models/activation.py @@ -143,6 +143,13 @@ class Meta: choices=RulebookProcessLogLevel.choices(), default=get_default_log_level, ) + store_debug_logs = models.BooleanField( + default=False, + help_text=( + "When false (default), DEBUG-level log lines are sent to " + "container stdout but not stored in the database." + ), + ) eda_credentials = models.ManyToManyField( "EdaCredential", related_name="activations", default=None ) diff --git a/src/aap_eda/core/utils/delete_log_util.py b/src/aap_eda/core/utils/delete_log_util.py index ccd28b7bc..e4df34bb0 100644 --- a/src/aap_eda/core/utils/delete_log_util.py +++ b/src/aap_eda/core/utils/delete_log_util.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from datetime import datetime from django.db.models import Q @@ -19,17 +20,80 @@ from aap_eda.core import models +logger = logging.getLogger(__name__) -def delete_logs_older_than(cutoff: datetime) -> int: +BATCH_SIZE = 10_000 + + +def delete_logs_older_than( + cutoff: datetime, + activation_id: int | None = None, +) -> int: """Delete all RulebookProcessLog records older than the cutoff. + If activation_id is provided, only delete logs for instances + belonging to that activation. + Returns the number of records deleted. """ cutoff_ts = int(cutoff.timestamp()) - deleted, _ = models.RulebookProcessLog.objects.filter( + qs = models.RulebookProcessLog.objects.filter( log_timestamp__lt=cutoff_ts, - ).delete() - return deleted + ) + if activation_id is not None: + instance_ids = models.RulebookProcess.objects.filter( + activation_id=activation_id, + ).values_list("id", flat=True) + qs = qs.filter(activation_instance_id__in=instance_ids) + return _batched_delete(qs) + + +def delete_logs_for_activation(activation_id: int) -> int: + """Delete all logs for a given activation's instances. + + Returns the number of records deleted. + """ + instance_ids = models.RulebookProcess.objects.filter( + activation_id=activation_id, + ).values_list("id", flat=True) + qs = models.RulebookProcessLog.objects.filter( + activation_instance_id__in=instance_ids, + ) + return _batched_delete(qs) + + +def delete_all_logs(cutoff: datetime | None = None) -> int: + """Delete all RulebookProcessLog records. + + If cutoff is provided, only delete logs older than the cutoff. + + Returns the number of records deleted. + """ + qs = models.RulebookProcessLog.objects.all() + if cutoff is not None: + cutoff_ts = int(cutoff.timestamp()) + qs = qs.filter(log_timestamp__lt=cutoff_ts) + return _batched_delete(qs) + + +def _batched_delete(queryset) -> int: + """Delete queryset in batches to avoid long-running queries.""" + total_deleted = 0 + upper_id = queryset.order_by("-id").values_list("id", flat=True).first() + if upper_id is None: + return total_deleted + queryset = queryset.filter(id__lte=upper_id) + + while True: + batch_ids = list(queryset.values_list("id", flat=True)[:BATCH_SIZE]) + if not batch_ids: + break + deleted, _ = models.RulebookProcessLog.objects.filter( + id__in=batch_ids, + ).delete() + total_deleted += deleted + logger.info("Purged %d log records (batch)", deleted) + return total_deleted def create_audit_trail( diff --git a/src/aap_eda/services/activation/activation_manager.py b/src/aap_eda/services/activation/activation_manager.py index 495c2926f..4f58e7052 100644 --- a/src/aap_eda/services/activation/activation_manager.py +++ b/src/aap_eda/services/activation/activation_manager.py @@ -14,6 +14,7 @@ """Module for Activation Manager.""" import contextlib +import functools import logging import typing as tp from datetime import timedelta @@ -76,7 +77,10 @@ def __init__( db_instance.id, self.db_instance_type ) - self.container_logger_class = container_logger_class + self.container_logger_class = functools.partial( + container_logger_class, + store_debug_logs=getattr(db_instance, "store_debug_logs", False), + ) def _update_started_time(self) -> None: """Update latest instance's started_at to now.""" diff --git a/src/aap_eda/services/activation/db_log_handler.py b/src/aap_eda/services/activation/db_log_handler.py index 222f5d34f..6a490fa20 100644 --- a/src/aap_eda/services/activation/db_log_handler.py +++ b/src/aap_eda/services/activation/db_log_handler.py @@ -30,11 +30,17 @@ from aap_eda.utils.log_sanitizer import sanitize_string LOGGER = logging.getLogger(__name__) +LOG_LEVEL_SEARCH_INDEX = 40 class DBLogger(LogHandler): - def __init__(self, activation_instance_id: int): + def __init__( + self, + activation_instance_id: int, + store_debug_logs: bool = False, + ): self.activation_instance_id = activation_instance_id + self.store_debug_logs = store_debug_logs self.line_count = 0 self.activation_instance_log_buffer = [] if str(settings.ANSIBLE_RULEBOOK_FLUSH_AFTER) == "end": @@ -80,6 +86,12 @@ def write( self.flush() def flush(self) -> None: + if not self.store_debug_logs: + self.activation_instance_log_buffer = [ + buf + for buf in self.activation_instance_log_buffer + if "DEBUG" not in buf.log[:LOG_LEVEL_SEARCH_INDEX] + ] try: if self.activation_instance_log_buffer: models.RulebookProcessLog.objects.bulk_create( @@ -92,6 +104,37 @@ def flush(self) -> None: raise ContainerUpdateLogsError(message) self.activation_instance_log_buffer = [] + self._enforce_max_log_lines() + + def _enforce_max_log_lines(self) -> None: + max_lines = int(settings.EDA_MAX_LOG_LINES_PER_INSTANCE) + if max_lines <= 0: + return + + count = self.num_of_log_lines() + if count <= max_lines: + return + + excess = count - max_lines + oldest_ids = list( + models.RulebookProcessLog.objects.filter( + activation_instance_id=self.activation_instance_id, + ) + .order_by("id") + .values_list("id", flat=True)[:excess] + ) + if oldest_ids: + models.RulebookProcessLog.objects.filter( + id__in=oldest_ids, + ).delete() + LOGGER.warning( + "Instance %s: trimmed %d oldest log lines " + "(cap: %d, was: %d)", + self.activation_instance_id, + len(oldest_ids), + max_lines, + count, + ) def get_log_read_at(self) -> Optional[datetime]: try: diff --git a/src/aap_eda/services/activation/tee_system_logger.py b/src/aap_eda/services/activation/tee_system_logger.py index 1a8899e7c..07503af37 100644 --- a/src/aap_eda/services/activation/tee_system_logger.py +++ b/src/aap_eda/services/activation/tee_system_logger.py @@ -15,7 +15,10 @@ import logging from datetime import datetime, timezone -from aap_eda.services.activation.db_log_handler import DBLogger +from aap_eda.services.activation.db_log_handler import ( + LOG_LEVEL_SEARCH_INDEX, + DBLogger, +) EXCEPTIONS_TO_CATCH = ( OverflowError, @@ -28,8 +31,6 @@ LOGGER = logging.getLogger(__name__) -LOG_LEVEL_SEARCH_INDEX = 40 - class TeeSystemLogger(DBLogger): """ @@ -83,6 +84,4 @@ def flush(self): extra=extra, ) finally: - # This will call the DBLoggers flush which will - # write to the Database and clear the log buffer super().flush() diff --git a/src/aap_eda/settings/defaults.py b/src/aap_eda/settings/defaults.py index 754921f8e..a01f814ae 100644 --- a/src/aap_eda/settings/defaults.py +++ b/src/aap_eda/settings/defaults.py @@ -191,6 +191,10 @@ # --------------------------------------------------------- ACTIVATION_DB_LOG_RETENTION_DAYS: int = 0 +# Maximum log lines kept per activation instance (0 = unlimited). +# Checked every 1000 lines; oldest rows are trimmed when exceeded. +EDA_MAX_LOG_LINES_PER_INSTANCE: int = 500_000 + # --------------------------------------------------------- # DJANGO ANSIBLE BASE JWT SETTINGS # --------------------------------------------------------- diff --git a/tests/integration/api/test_activation.py b/tests/integration/api/test_activation.py index 900a7e6e4..d18c21858 100644 --- a/tests/integration/api/test_activation.py +++ b/tests/integration/api/test_activation.py @@ -971,6 +971,7 @@ def assert_activation_base_data( assert data["created_at"] == activation.created_at assert data["modified_at"] <= activation.modified_at assert data["status_message"] + assert data["store_debug_logs"] == activation.store_debug_logs def assert_activation_related_object_fks( diff --git a/tests/integration/api/test_activation_instance.py b/tests/integration/api/test_activation_instance.py index e6cb0223d..b8c7696bb 100644 --- a/tests/integration/api/test_activation_instance.py +++ b/tests/integration/api/test_activation_instance.py @@ -155,6 +155,72 @@ def test_list_activation_instance_logs_filter_non_existent( assert data == [] +@pytest.mark.django_db +def test_list_activation_instance_logs_filter_timestamp_gt( + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + admin_client: APIClient, +): + instance = default_activation_instances[0] + response = admin_client.get( + f"{api_url_v1}/activation-instances/{instance.id}" + f"/logs/?log_timestamp__gt=1000" + ) + assert response.status_code == status.HTTP_200_OK + results = response.data["results"] + assert len(results) == 1 + assert results[0]["log"] == "activation-instance-log-2" + + +@pytest.mark.django_db +def test_list_activation_instance_logs_filter_timestamp_lt( + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + admin_client: APIClient, +): + instance = default_activation_instances[0] + response = admin_client.get( + f"{api_url_v1}/activation-instances/{instance.id}" + f"/logs/?log_timestamp__lt=2000" + ) + assert response.status_code == status.HTTP_200_OK + results = response.data["results"] + assert len(results) == 1 + assert results[0]["log"] == "activation-instance-log-1" + + +@pytest.mark.django_db +def test_list_activation_instance_logs_filter_timestamp_range( + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + admin_client: APIClient, +): + instance = default_activation_instances[0] + response = admin_client.get( + f"{api_url_v1}/activation-instances/{instance.id}" + f"/logs/?log_timestamp__gt=500&log_timestamp__lt=1500" + ) + assert response.status_code == status.HTTP_200_OK + results = response.data["results"] + assert len(results) == 1 + assert results[0]["log"] == "activation-instance-log-1" + + +@pytest.mark.django_db +def test_list_activation_instance_logs_filter_timestamp_no_results( + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + admin_client: APIClient, +): + instance = default_activation_instances[0] + response = admin_client.get( + f"{api_url_v1}/activation-instances/{instance.id}" + f"/logs/?log_timestamp__gt=9999" + ) + assert response.status_code == status.HTTP_200_OK + assert response.data["results"] == [] + + @pytest.mark.django_db def test_logs_page_size_capped_at_max( default_activation_instances: List[models.RulebookProcess], diff --git a/tests/integration/api/test_log_purge.py b/tests/integration/api/test_log_purge.py new file mode 100644 index 000000000..453a50c20 --- /dev/null +++ b/tests/integration/api/test_log_purge.py @@ -0,0 +1,129 @@ +# Copyright 2025 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List + +import pytest +from rest_framework import status +from rest_framework.test import APIClient + +from aap_eda.core import models + +api_url_v1 = "/api/eda/v1" + + +@pytest.mark.django_db +def test_clear_logs_per_activation( + default_activation: models.Activation, + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + admin_client: APIClient, +): + activation_id = default_activation.id + initial_count = models.RulebookProcessLog.objects.filter( + activation_instance__activation_id=activation_id, + ).count() + assert initial_count > 0 + + response = admin_client.post( + f"{api_url_v1}/activations/{activation_id}/clear-logs/" + ) + assert response.status_code == status.HTTP_200_OK + assert response.data["deleted"] == initial_count + + remaining = models.RulebookProcessLog.objects.filter( + activation_instance__activation_id=activation_id, + ).count() + assert remaining == 0 + + +@pytest.mark.django_db +def test_clear_logs_with_before_date( + default_activation: models.Activation, + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + admin_client: APIClient, +): + activation_id = default_activation.id + response = admin_client.post( + f"{api_url_v1}/activations/{activation_id}/clear-logs/", + data={"before_date": "1970-01-01T00:17:00Z"}, + format="json", + ) + assert response.status_code == status.HTTP_200_OK + assert response.data["deleted"] == 1 + + remaining = models.RulebookProcessLog.objects.filter( + activation_instance__activation_id=activation_id, + ).count() + assert remaining == 1 + + +@pytest.mark.django_db +def test_clear_logs_without_date_deletes_all( + default_activation: models.Activation, + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + admin_client: APIClient, +): + activation_id = default_activation.id + response = admin_client.post( + f"{api_url_v1}/activations/{activation_id}/clear-logs/", + format="json", + ) + assert response.status_code == status.HTTP_200_OK + assert response.data["deleted"] == 2 + + +@pytest.mark.django_db +def test_purge_global( + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + superuser_client: APIClient, +): + initial_count = models.RulebookProcessLog.objects.count() + assert initial_count > 0 + + response = superuser_client.post( + f"{api_url_v1}/logs/purge/", + ) + assert response.status_code == status.HTTP_200_OK + assert response.data["deleted"] == initial_count + assert models.RulebookProcessLog.objects.count() == 0 + + +@pytest.mark.django_db +def test_purge_global_requires_superuser( + default_activation_instance_logs: List[models.RulebookProcessLog], + admin_client: APIClient, +): + response = admin_client.post( + f"{api_url_v1}/logs/purge/", + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + + +@pytest.mark.django_db +def test_purge_returns_deleted_count( + default_activation_instances: List[models.RulebookProcess], + default_activation_instance_logs: List[models.RulebookProcessLog], + superuser_client: APIClient, +): + response = superuser_client.post( + f"{api_url_v1}/logs/purge/", + ) + assert response.status_code == status.HTTP_200_OK + assert "deleted" in response.data + assert isinstance(response.data["deleted"], int) + assert response.data["deleted"] >= 0 diff --git a/tests/integration/api/test_root.py b/tests/integration/api/test_root.py index 93b0628b6..86c07979a 100644 --- a/tests/integration/api/test_root.py +++ b/tests/integration/api/test_root.py @@ -46,6 +46,7 @@ "/teams/", "/event-streams/", "/credential-input-sources/", + "/logs/", "/role_definitions/", "/role_user_assignments/", "/role_team_assignments/", @@ -102,6 +103,7 @@ # have migrated away from this endpoint "/feature_flags_state/", "/credential-input-sources/", + "/logs/", ], False, id="no_shared_resource", diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 2f703b284..e05212a0e 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -745,10 +745,12 @@ def default_activation_instance_logs( models.RulebookProcessLog( log="activation-instance-log-1", activation_instance=default_activation_instances[0], + log_timestamp=1000, ), models.RulebookProcessLog( log="activation-instance-log-2", activation_instance=default_activation_instances[0], + log_timestamp=2000, ), ] ) diff --git a/tests/integration/services/activation/test_db_log_handler.py b/tests/integration/services/activation/test_db_log_handler.py new file mode 100644 index 000000000..9b83a10e9 --- /dev/null +++ b/tests/integration/services/activation/test_db_log_handler.py @@ -0,0 +1,108 @@ +# Copyright 2025 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from unittest.mock import patch + +import pytest + +from aap_eda.core.models.rulebook_process import RulebookProcessLog +from aap_eda.services.activation.db_log_handler import DBLogger + + +@pytest.mark.django_db +def test_enforce_max_log_lines_trims_oldest(default_activation_instance): + """Oldest rows are deleted when count exceeds cap.""" + with patch("django.conf.settings.EDA_MAX_LOG_LINES_PER_INSTANCE", 5): + obj = DBLogger(default_activation_instance.id) + for i in range(1000): + obj.write(f"line-{i:04d}") # noqa: E231 + obj.flush() + + logs = list( + RulebookProcessLog.objects.filter( + activation_instance=default_activation_instance, + ) + .order_by("id") + .values_list("log", flat=True) + ) + assert len(logs) == 5 + assert logs[0] == "line-0995" + assert logs[-1] == "line-0999" + + +@pytest.mark.django_db +def test_enforce_max_log_lines_disabled_when_zero( + default_activation_instance, +): + """Setting=0 means no cap; all lines are kept.""" + with patch("django.conf.settings.EDA_MAX_LOG_LINES_PER_INSTANCE", 0): + obj = DBLogger(default_activation_instance.id) + for i in range(1000): + obj.write(f"line-{i}") + obj.flush() + + count = RulebookProcessLog.objects.filter( + activation_instance=default_activation_instance, + ).count() + assert count == 1000 + + +@pytest.mark.django_db +def test_enforce_max_log_lines_check_interval(default_activation_instance): + """Trimming fires on every flush when cap is exceeded.""" + with patch("django.conf.settings.EDA_MAX_LOG_LINES_PER_INSTANCE", 5): + obj = DBLogger(default_activation_instance.id) + for i in range(999): + obj.write(f"line-{i}") + obj.flush() + + count = RulebookProcessLog.objects.filter( + activation_instance=default_activation_instance, + ).count() + assert count == 5 + + +@pytest.mark.django_db +def test_enforce_max_log_lines_fires_across_instances( + default_activation_instance, +): + """Trimming works correctly across separate DBLogger instances.""" + with patch("django.conf.settings.EDA_MAX_LOG_LINES_PER_INSTANCE", 10): + # First poll cycle - write 7 lines + obj1 = DBLogger(default_activation_instance.id) + for i in range(7): + obj1.write(f"poll1-line-{i}") + obj1.flush() + + # Second poll cycle - write 8 more lines (total 15) + obj2 = DBLogger(default_activation_instance.id) + for i in range(8): + obj2.write(f"poll2-line-{i}") + obj2.flush() + + # Should have trimmed to cap of 10 + logs = list( + RulebookProcessLog.objects.filter( + activation_instance=default_activation_instance, + ) + .order_by("id") + .values_list("log", flat=True) + ) + assert len(logs) == 10 + # Oldest 5 from poll1 should be deleted, keeping last 2 from poll1 + assert logs[0] == "poll1-line-5" + assert logs[1] == "poll1-line-6" + # All 8 from poll2 should be kept + assert logs[2] == "poll2-line-0" + assert logs[-1] == "poll2-line-7" diff --git a/tests/integration/services/activation/test_tee_system_logger.py b/tests/integration/services/activation/test_tee_system_logger.py index 256b83fd3..b7e6dc1cc 100644 --- a/tests/integration/services/activation/test_tee_system_logger.py +++ b/tests/integration/services/activation/test_tee_system_logger.py @@ -56,7 +56,9 @@ def test_logging( """Test that TeeSystemLogger writes to DB and log.""" eda_log = caplog_factory(LOGGER, level=logging.DEBUG) - obj = TeeSystemLogger(default_activation_instance.id) + obj = TeeSystemLogger( + default_activation_instance.id, store_debug_logs=True + ) for line in log_lines: obj.write(line) obj.flush() @@ -72,6 +74,77 @@ def test_logging( ).count() == len(expectations) +@pytest.mark.django_db +def test_debug_lines_excluded_from_db_by_default( + caplog_factory, default_activation_instance +): + """With store_debug_logs=False, DEBUG lines go to stdout but not DB.""" + eda_log = caplog_factory(LOGGER, level=logging.DEBUG) + + obj = TeeSystemLogger( + default_activation_instance.id, store_debug_logs=False + ) + obj.write("DEBUG This is a debug message") + obj.write("ERROR This is an error message") + obj.write("INFO This is an info message") + obj.flush() + + assert len(eda_log.records) == 3 + + db_logs = RulebookProcessLog.objects.filter( + activation_instance=default_activation_instance + ) + assert db_logs.count() == 2 + log_texts = [log.log for log in db_logs] + assert any("ERROR" in t for t in log_texts) + assert any("INFO" in t for t in log_texts) + assert not any("DEBUG" in t for t in log_texts) + + +@pytest.mark.django_db +def test_debug_lines_stored_when_opted_in( + caplog_factory, default_activation_instance +): + """With store_debug_logs=True, all lines including DEBUG go to DB.""" + eda_log = caplog_factory(LOGGER, level=logging.DEBUG) + + obj = TeeSystemLogger( + default_activation_instance.id, store_debug_logs=True + ) + obj.write("DEBUG This is a debug message") + obj.write("ERROR This is an error message") + obj.flush() + + assert len(eda_log.records) == 2 + + db_logs = RulebookProcessLog.objects.filter( + activation_instance=default_activation_instance + ) + assert db_logs.count() == 2 + + +@pytest.mark.django_db +def test_non_debug_lines_always_stored( + caplog_factory, default_activation_instance +): + """ERROR/WARNING/INFO always go to DB regardless of toggle.""" + caplog_factory(LOGGER, level=logging.DEBUG) + + obj = TeeSystemLogger( + default_activation_instance.id, store_debug_logs=False + ) + obj.write("ERROR an error") + obj.write("WARN a warning") + obj.write("INFO an info") + obj.write("CRITICAL a critical") + obj.flush() + + db_logs = RulebookProcessLog.objects.filter( + activation_instance=default_activation_instance + ) + assert db_logs.count() == 4 + + @pytest.mark.django_db def test_logging_exception(caplog_factory, default_activation_instance): """Test that TeeSystemLogger writes to DB even if there is exception."""