diff --git a/application/__init__.py b/application/__init__.py index 454ca7c38..68a7c7e52 100644 --- a/application/__init__.py +++ b/application/__init__.py @@ -8,6 +8,7 @@ from flask_caching import Cache from flask_compress import Compress from application.config import config +from application.utils.logging_config import configure_logging import os import random @@ -26,6 +27,7 @@ def create_app(mode: str = "production", conf: any = None) -> Any: + configure_logging() app = Flask(__name__) if not conf: app.config.from_object(config[mode]) diff --git a/application/cmd/cre_main.py b/application/cmd/cre_main.py index ead5a4281..fbbfd2406 100644 --- a/application/cmd/cre_main.py +++ b/application/cmd/cre_main.py @@ -25,10 +25,10 @@ from alive_progress import alive_bar from application.prompt_client import prompt_client as prompt_client from application.utils import gap_analysis +from application.utils.logging_config import configure_logging -logging.basicConfig() +configure_logging() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) app = None diff --git a/application/database/db.py b/application/database/db.py index d4ac9b7e8..a6f19ec86 100644 --- a/application/database/db.py +++ b/application/database/db.py @@ -47,9 +47,7 @@ from .. import sqla # type: ignore -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) BaseModel: DefaultMeta = sqla.Model diff --git a/application/database/inmemory_graph.py b/application/database/inmemory_graph.py index be747326e..51a7c0949 100644 --- a/application/database/inmemory_graph.py +++ b/application/database/inmemory_graph.py @@ -5,9 +5,7 @@ from application.defs import cre_defs as defs -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class CycleDetectedError(Exception): diff --git a/application/defs/osib_defs.py b/application/defs/osib_defs.py index 89c204704..769312b3e 100644 --- a/application/defs/osib_defs.py +++ b/application/defs/osib_defs.py @@ -22,9 +22,7 @@ from_dict, ) -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) # used for serialising and deserialising yaml OSIB documents diff --git a/application/prompt_client/openai_prompt_client.py b/application/prompt_client/openai_prompt_client.py index bda51b896..a4bf5353b 100644 --- a/application/prompt_client/openai_prompt_client.py +++ b/application/prompt_client/openai_prompt_client.py @@ -1,9 +1,7 @@ import openai import logging -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class OpenAIPromptClient: diff --git a/application/prompt_client/prompt_client.py b/application/prompt_client/prompt_client.py index 09546c204..5cd1195d9 100644 --- a/application/prompt_client/prompt_client.py +++ b/application/prompt_client/prompt_client.py @@ -17,9 +17,7 @@ import re import requests -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) SIMILARITY_THRESHOLD = float(os.environ.get("CHATBOT_SIMILARITY_THRESHOLD", "0.7")) diff --git a/application/prompt_client/vertex_prompt_client.py b/application/prompt_client/vertex_prompt_client.py index 9ed8d696b..a65de8ec8 100644 --- a/application/prompt_client/vertex_prompt_client.py +++ b/application/prompt_client/vertex_prompt_client.py @@ -21,9 +21,7 @@ import grpc_status import time -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) MAX_OUTPUT_TOKENS = 1024 diff --git a/application/tests/logging_config_test.py b/application/tests/logging_config_test.py new file mode 100644 index 000000000..6bd5a9fe3 --- /dev/null +++ b/application/tests/logging_config_test.py @@ -0,0 +1,121 @@ +import json +import logging +import unittest +from io import StringIO +from unittest.mock import patch + +from application.utils.logging_config import JSONFormatter, configure_logging + + +class TestJSONFormatter(unittest.TestCase): + def _make_record(self, msg="hello", level=logging.INFO, name="test"): + record = logging.LogRecord( + name=name, + level=level, + pathname="", + lineno=0, + msg=msg, + args=(), + exc_info=None, + ) + return record + + def test_output_is_valid_json(self): + formatter = JSONFormatter() + record = self._make_record() + output = formatter.format(record) + parsed = json.loads(output) + self.assertIsInstance(parsed, dict) + + def test_required_fields_present(self): + formatter = JSONFormatter() + record = self._make_record(msg="test message", name="mylogger") + parsed = json.loads(formatter.format(record)) + self.assertIn("timestamp", parsed) + self.assertIn("level", parsed) + self.assertIn("logger", parsed) + self.assertIn("message", parsed) + + def test_message_and_level_values(self): + formatter = JSONFormatter() + record = self._make_record(msg="check value", level=logging.WARNING, name="mod") + parsed = json.loads(formatter.format(record)) + self.assertEqual(parsed["message"], "check value") + self.assertEqual(parsed["level"], "WARNING") + self.assertEqual(parsed["logger"], "mod") + + def test_exception_included_when_present(self): + formatter = JSONFormatter() + try: + raise ValueError("boom") + except ValueError: + import sys + + exc_info = sys.exc_info() + record = logging.LogRecord( + name="test", + level=logging.ERROR, + pathname="", + lineno=0, + msg="error", + args=(), + exc_info=exc_info, + ) + parsed = json.loads(formatter.format(record)) + self.assertIn("exception", parsed) + self.assertIn("ValueError", parsed["exception"]) + + +class TestConfigureLogging(unittest.TestCase): + def setUp(self): + root = logging.getLogger() + root.handlers.clear() + + def test_default_level_is_info(self): + with patch.dict("os.environ", {}, clear=False): + os.environ.pop("LOG_LEVEL", None) + os.environ.pop("LOG_FORMAT", None) + configure_logging() + self.assertEqual(logging.getLogger().level, logging.INFO) + + def test_log_level_env_respected(self): + with patch.dict("os.environ", {"LOG_LEVEL": "DEBUG"}): + configure_logging() + self.assertEqual(logging.getLogger().level, logging.DEBUG) + + def test_text_format_uses_standard_formatter(self): + with patch.dict("os.environ", {"LOG_FORMAT": "text"}): + configure_logging() + root = logging.getLogger() + self.assertEqual(len(root.handlers), 1) + self.assertNotIsInstance(root.handlers[0].formatter, JSONFormatter) + + def test_json_format_uses_json_formatter(self): + with patch.dict("os.environ", {"LOG_FORMAT": "json"}): + configure_logging() + root = logging.getLogger() + self.assertEqual(len(root.handlers), 1) + self.assertIsInstance(root.handlers[0].formatter, JSONFormatter) + + def test_repeated_calls_do_not_add_handlers(self): + configure_logging() + configure_logging() + self.assertEqual(len(logging.getLogger().handlers), 1) + + def test_json_output_is_parseable(self): + stream = StringIO() + with patch.dict("os.environ", {"LOG_FORMAT": "json"}): + configure_logging() + root = logging.getLogger() + root.handlers[0].stream = stream + logging.getLogger("test.json").info("structured message") + output = stream.getvalue().strip() + parsed = json.loads(output) + self.assertEqual(parsed["message"], "structured message") + self.assertEqual(parsed["level"], "INFO") + + +import os + +if __name__ == "__main__": + unittest.main() diff --git a/application/utils/external_project_parsers/base_parser.py b/application/utils/external_project_parsers/base_parser.py index 5bff50e63..d73986cb2 100644 --- a/application/utils/external_project_parsers/base_parser.py +++ b/application/utils/external_project_parsers/base_parser.py @@ -9,9 +9,7 @@ from application.utils import gap_analysis import os, json -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class BaseParser: diff --git a/application/utils/external_project_parsers/parsers/capec_parser.py b/application/utils/external_project_parsers/parsers/capec_parser.py index 0da07b020..2bccc15fc 100644 --- a/application/utils/external_project_parsers/parsers/capec_parser.py +++ b/application/utils/external_project_parsers/parsers/capec_parser.py @@ -7,9 +7,7 @@ from application.defs import cre_defs as defs import xmltodict -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) from application.utils.external_project_parsers.base_parser_defs import ( ParserInterface, diff --git a/application/utils/external_project_parsers/parsers/ccmv4.py b/application/utils/external_project_parsers/parsers/ccmv4.py index 5aeb20bb8..9937a763c 100644 --- a/application/utils/external_project_parsers/parsers/ccmv4.py +++ b/application/utils/external_project_parsers/parsers/ccmv4.py @@ -5,9 +5,7 @@ import re -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) from application.utils.external_project_parsers.base_parser_defs import ( ParserInterface, diff --git a/application/utils/external_project_parsers/parsers/cloud_native_security_controls.py b/application/utils/external_project_parsers/parsers/cloud_native_security_controls.py index be216c27d..b8191410b 100644 --- a/application/utils/external_project_parsers/parsers/cloud_native_security_controls.py +++ b/application/utils/external_project_parsers/parsers/cloud_native_security_controls.py @@ -10,9 +10,7 @@ ) import requests -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class CloudNativeSecurityControls(ParserInterface): diff --git a/application/utils/external_project_parsers/parsers/cwe.py b/application/utils/external_project_parsers/parsers/cwe.py index b0821aba5..008250769 100644 --- a/application/utils/external_project_parsers/parsers/cwe.py +++ b/application/utils/external_project_parsers/parsers/cwe.py @@ -13,9 +13,7 @@ ParseResult, ) -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class CWE(ParserInterface): diff --git a/application/utils/external_project_parsers/parsers/dsomm.py b/application/utils/external_project_parsers/parsers/dsomm.py index c1341bd93..0714e1095 100644 --- a/application/utils/external_project_parsers/parsers/dsomm.py +++ b/application/utils/external_project_parsers/parsers/dsomm.py @@ -10,9 +10,7 @@ ParseResult, ) -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class DSOMM(ParserInterface): diff --git a/application/utils/external_project_parsers/parsers/iso27001.py b/application/utils/external_project_parsers/parsers/iso27001.py index 65ec6aaeb..4c38bfe29 100644 --- a/application/utils/external_project_parsers/parsers/iso27001.py +++ b/application/utils/external_project_parsers/parsers/iso27001.py @@ -14,9 +14,7 @@ ) from typing import List -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) nist_id_re = re.compile("(?P\w\w\-\d+)") diff --git a/application/utils/external_project_parsers/parsers/juiceshop.py b/application/utils/external_project_parsers/parsers/juiceshop.py index 2f8cd21c6..2eb134dcd 100644 --- a/application/utils/external_project_parsers/parsers/juiceshop.py +++ b/application/utils/external_project_parsers/parsers/juiceshop.py @@ -13,9 +13,7 @@ ) import requests -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class JuiceShop(ParserInterface): diff --git a/application/utils/external_project_parsers/parsers/misc_tools_parser.py b/application/utils/external_project_parsers/parsers/misc_tools_parser.py index 1a750240a..e46efda3e 100644 --- a/application/utils/external_project_parsers/parsers/misc_tools_parser.py +++ b/application/utils/external_project_parsers/parsers/misc_tools_parser.py @@ -15,9 +15,7 @@ ) import requests -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class MiscTools(ParserInterface): diff --git a/application/utils/external_project_parsers/parsers/pci_dss.py b/application/utils/external_project_parsers/parsers/pci_dss.py index 53611577b..b0ec8a07a 100644 --- a/application/utils/external_project_parsers/parsers/pci_dss.py +++ b/application/utils/external_project_parsers/parsers/pci_dss.py @@ -12,9 +12,7 @@ ParseResult, ) -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class PciDss(ParserInterface): diff --git a/application/utils/external_project_parsers/parsers/zap_alerts_parser.py b/application/utils/external_project_parsers/parsers/zap_alerts_parser.py index 85488b457..8710f19ac 100644 --- a/application/utils/external_project_parsers/parsers/zap_alerts_parser.py +++ b/application/utils/external_project_parsers/parsers/zap_alerts_parser.py @@ -9,9 +9,7 @@ from application.defs import cre_defs as defs from application.utils import git -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) from application.prompt_client import prompt_client as prompt_client from application.utils.external_project_parsers.base_parser_defs import ( diff --git a/application/utils/gap_analysis.py b/application/utils/gap_analysis.py index 9e3dab04d..505a1c30f 100644 --- a/application/utils/gap_analysis.py +++ b/application/utils/gap_analysis.py @@ -9,9 +9,7 @@ import json from application.defs import cre_defs as defs -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) PENALTIES = { "RELATED": 2, diff --git a/application/utils/git.py b/application/utils/git.py index 31ef5ccfc..070c3fa91 100644 --- a/application/utils/git.py +++ b/application/utils/git.py @@ -10,8 +10,6 @@ from github import Github logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logging.basicConfig() commit_msg_base = "cre_sync_%s" % (datetime.now().isoformat().replace(":", ".")) diff --git a/application/utils/logging_config.py b/application/utils/logging_config.py new file mode 100644 index 000000000..5b493123f --- /dev/null +++ b/application/utils/logging_config.py @@ -0,0 +1,42 @@ +import json +import logging +import os + + +class JSONFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + log_entry = { + "timestamp": self.formatTime(record, self.datefmt), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + if record.exc_info: + log_entry["exception"] = self.formatException(record.exc_info) + if record.stack_info: + log_entry["stack_info"] = self.formatStack(record.stack_info) + return json.dumps(log_entry) + + +def configure_logging() -> None: + """Configure root logger from environment variables. + + LOG_LEVEL: log level name (default INFO) + LOG_FORMAT: 'json' for structured JSON output, anything else for plain text + """ + level_name = os.environ.get("LOG_LEVEL", "INFO").upper() + level = getattr(logging, level_name, logging.INFO) + log_format = os.environ.get("LOG_FORMAT", "text").lower() + + handler = logging.StreamHandler() + if log_format == "json": + handler.setFormatter(JSONFormatter()) + else: + handler.setFormatter( + logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") + ) + + root = logging.getLogger() + root.setLevel(level) + root.handlers.clear() + root.addHandler(handler) diff --git a/application/utils/redis.py b/application/utils/redis.py index 90b2296bd..acd66489d 100644 --- a/application/utils/redis.py +++ b/application/utils/redis.py @@ -6,9 +6,7 @@ import rq import time -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) def empty_queues(redis: redis.Redis): diff --git a/application/utils/spreadsheet.py b/application/utils/spreadsheet.py index b18b493dd..2585e5dee 100644 --- a/application/utils/spreadsheet.py +++ b/application/utils/spreadsheet.py @@ -12,8 +12,6 @@ from enum import Enum logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logging.basicConfig() class GspreadAuth(Enum): diff --git a/application/utils/spreadsheet_parsers.py b/application/utils/spreadsheet_parsers.py index 4c5397156..f009711d8 100644 --- a/application/utils/spreadsheet_parsers.py +++ b/application/utils/spreadsheet_parsers.py @@ -12,8 +12,6 @@ logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) -logging.basicConfig() # the supported resources from the main CSV diff --git a/application/web/web_main.py b/application/web/web_main.py index 29567470a..1ff1d03ff 100644 --- a/application/web/web_main.py +++ b/application/web/web_main.py @@ -57,9 +57,7 @@ ), ) -logging.basicConfig() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) class SupportedFormats(Enum): diff --git a/application/worker.py b/application/worker.py index a26256622..d54d56c07 100644 --- a/application/worker.py +++ b/application/worker.py @@ -1,10 +1,10 @@ from rq import Worker, Queue import logging from application.utils import redis +from application.utils.logging_config import configure_logging -logging.basicConfig() +configure_logging() logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) listen = ["high", "default", "low"]