Add centralized structured logging, remove per-module basicConfig calls

application/__init__.py

@@ -8,6 +8,7 @@
 from flask_caching import Cache
 from flask_compress import Compress
 from application.config import config
+from application.utils.logging_config import configure_logging
 import os
 import random
 
@@ -26,6 +27,7 @@
 
 
 def create_app(mode: str = "production", conf: any = None) -> Any:
+    configure_logging()
     app = Flask(__name__)
     if not conf:
         app.config.from_object(config[mode])

application/cmd/cre_main.py

@@ -25,10 +25,10 @@
 from alive_progress import alive_bar
 from application.prompt_client import prompt_client as prompt_client
 from application.utils import gap_analysis
+from application.utils.logging_config import configure_logging
 
-logging.basicConfig()
+configure_logging()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 app = None
 

application/database/db.py

@@ -47,9 +47,7 @@
 
 from .. import sqla  # type: ignore
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 BaseModel: DefaultMeta = sqla.Model

application/database/inmemory_graph.py

@@ -5,9 +5,7 @@
 from application.defs import cre_defs as defs
 
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class CycleDetectedError(Exception):

application/defs/osib_defs.py

@@ -22,9 +22,7 @@
     from_dict,
 )
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 # used for serialising and deserialising yaml OSIB documents

application/prompt_client/openai_prompt_client.py

@@ -1,9 +1,7 @@
 import openai
 import logging
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class OpenAIPromptClient:

application/prompt_client/prompt_client.py

@@ -17,9 +17,7 @@
 import re
 import requests
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 SIMILARITY_THRESHOLD = float(os.environ.get("CHATBOT_SIMILARITY_THRESHOLD", "0.7"))
 

application/prompt_client/vertex_prompt_client.py

@@ -21,9 +21,7 @@
 import grpc_status
 import time
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 MAX_OUTPUT_TOKENS = 1024
 

application/tests/logging_config_test.py

@@ -0,0 +1,121 @@
+import json
+import logging
+import unittest
+from io import StringIO
+from unittest.mock import patch
+
+from application.utils.logging_config import JSONFormatter, configure_logging
+
+
+class TestJSONFormatter(unittest.TestCase):
+    def _make_record(self, msg="hello", level=logging.INFO, name="test"):
+        record = logging.LogRecord(
+            name=name,
+            level=level,
+            pathname="",
+            lineno=0,
+            msg=msg,
+            args=(),
+            exc_info=None,
+        )
+        return record
+
+    def test_output_is_valid_json(self):
+        formatter = JSONFormatter()
+        record = self._make_record()
+        output = formatter.format(record)
+        parsed = json.loads(output)
+        self.assertIsInstance(parsed, dict)
+
+    def test_required_fields_present(self):
+        formatter = JSONFormatter()
+        record = self._make_record(msg="test message", name="mylogger")
+        parsed = json.loads(formatter.format(record))
+        self.assertIn("timestamp", parsed)
+        self.assertIn("level", parsed)
+        self.assertIn("logger", parsed)
+        self.assertIn("message", parsed)
+
+    def test_message_and_level_values(self):
+        formatter = JSONFormatter()
+        record = self._make_record(msg="check value", level=logging.WARNING, name="mod")
+        parsed = json.loads(formatter.format(record))
+        self.assertEqual(parsed["message"], "check value")
+        self.assertEqual(parsed["level"], "WARNING")
+        self.assertEqual(parsed["logger"], "mod")
+
+    def test_exception_included_when_present(self):
+        formatter = JSONFormatter()
+        try:
+            raise ValueError("boom")
+        except ValueError:
+            import sys
+
+            exc_info = sys.exc_info()
+        record = logging.LogRecord(
+            name="test",
+            level=logging.ERROR,
+            pathname="",
+            lineno=0,
+            msg="error",
+            args=(),
+            exc_info=exc_info,
+        )
+        parsed = json.loads(formatter.format(record))
+        self.assertIn("exception", parsed)
+        self.assertIn("ValueError", parsed["exception"])
+
+
+class TestConfigureLogging(unittest.TestCase):
+    def setUp(self):
+        root = logging.getLogger()
+        root.handlers.clear()
+
+    def test_default_level_is_info(self):
+        with patch.dict("os.environ", {}, clear=False):
+            os.environ.pop("LOG_LEVEL", None)
+            os.environ.pop("LOG_FORMAT", None)
+            configure_logging()
+        self.assertEqual(logging.getLogger().level, logging.INFO)
+
+    def test_log_level_env_respected(self):
+        with patch.dict("os.environ", {"LOG_LEVEL": "DEBUG"}):
+            configure_logging()
+        self.assertEqual(logging.getLogger().level, logging.DEBUG)
+
+    def test_text_format_uses_standard_formatter(self):
+        with patch.dict("os.environ", {"LOG_FORMAT": "text"}):
+            configure_logging()
+        root = logging.getLogger()
+        self.assertEqual(len(root.handlers), 1)
+        self.assertNotIsInstance(root.handlers[0].formatter, JSONFormatter)
+
+    def test_json_format_uses_json_formatter(self):
+        with patch.dict("os.environ", {"LOG_FORMAT": "json"}):
+            configure_logging()
+        root = logging.getLogger()
+        self.assertEqual(len(root.handlers), 1)
+        self.assertIsInstance(root.handlers[0].formatter, JSONFormatter)
+
+    def test_repeated_calls_do_not_add_handlers(self):
+        configure_logging()
+        configure_logging()
+        self.assertEqual(len(logging.getLogger().handlers), 1)
+
+    def test_json_output_is_parseable(self):
+        stream = StringIO()
+        with patch.dict("os.environ", {"LOG_FORMAT": "json"}):
+            configure_logging()
+        root = logging.getLogger()
+        root.handlers[0].stream = stream
+        logging.getLogger("test.json").info("structured message")
+        output = stream.getvalue().strip()
+        parsed = json.loads(output)
+        self.assertEqual(parsed["message"], "structured message")
+        self.assertEqual(parsed["level"], "INFO")
+
+
+import os
+
+if __name__ == "__main__":
+    unittest.main()

application/utils/external_project_parsers/base_parser.py

@@ -9,9 +9,7 @@
 from application.utils import gap_analysis
 import os, json
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class BaseParser:

application/utils/external_project_parsers/parsers/capec_parser.py

@@ -7,9 +7,7 @@
 from application.defs import cre_defs as defs
 import xmltodict
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 from application.utils.external_project_parsers.base_parser_defs import (
     ParserInterface,

application/utils/external_project_parsers/parsers/ccmv4.py

@@ -5,9 +5,7 @@
 
 import re
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 from application.utils.external_project_parsers.base_parser_defs import (
     ParserInterface,

application/utils/external_project_parsers/parsers/cloud_native_security_controls.py

@@ -10,9 +10,7 @@
 )
 import requests
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class CloudNativeSecurityControls(ParserInterface):

application/utils/external_project_parsers/parsers/cwe.py

@@ -13,9 +13,7 @@
     ParseResult,
 )
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class CWE(ParserInterface):

application/utils/external_project_parsers/parsers/dsomm.py

@@ -10,9 +10,7 @@
     ParseResult,
 )
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class DSOMM(ParserInterface):

application/utils/external_project_parsers/parsers/iso27001.py

@@ -14,9 +14,7 @@
 )
 from typing import List
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 nist_id_re = re.compile("(?P<nist_id>\w\w\-\d+)")
 

application/utils/external_project_parsers/parsers/juiceshop.py

@@ -13,9 +13,7 @@
 )
 import requests
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class JuiceShop(ParserInterface):

application/utils/external_project_parsers/parsers/misc_tools_parser.py

@@ -15,9 +15,7 @@
 )
 import requests
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class MiscTools(ParserInterface):

application/utils/external_project_parsers/parsers/pci_dss.py

@@ -12,9 +12,7 @@
     ParseResult,
 )
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 class PciDss(ParserInterface):

application/utils/external_project_parsers/parsers/zap_alerts_parser.py

@@ -9,9 +9,7 @@
 from application.defs import cre_defs as defs
 from application.utils import git
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 from application.prompt_client import prompt_client as prompt_client
 from application.utils.external_project_parsers.base_parser_defs import (

application/utils/gap_analysis.py

@@ -9,9 +9,7 @@
 import json
 from application.defs import cre_defs as defs
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 PENALTIES = {
     "RELATED": 2,

application/utils/git.py

@@ -10,8 +10,6 @@
 from github import Github
 
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
-logging.basicConfig()
 
 commit_msg_base = "cre_sync_%s" % (datetime.now().isoformat().replace(":", "."))
 

application/utils/logging_config.py

@@ -0,0 +1,42 @@
+import json
+import logging
+import os
+
+
+class JSONFormatter(logging.Formatter):
+    def format(self, record: logging.LogRecord) -> str:
+        log_entry = {
+            "timestamp": self.formatTime(record, self.datefmt),
+            "level": record.levelname,
+            "logger": record.name,
+            "message": record.getMessage(),
+        }
+        if record.exc_info:
+            log_entry["exception"] = self.formatException(record.exc_info)
+        if record.stack_info:
+            log_entry["stack_info"] = self.formatStack(record.stack_info)
+        return json.dumps(log_entry)
+
+
+def configure_logging() -> None:
+    """Configure root logger from environment variables.
+
+    LOG_LEVEL: log level name (default INFO)
+    LOG_FORMAT: 'json' for structured JSON output, anything else for plain text
+    """
+    level_name = os.environ.get("LOG_LEVEL", "INFO").upper()
+    level = getattr(logging, level_name, logging.INFO)
+    log_format = os.environ.get("LOG_FORMAT", "text").lower()
+
+    handler = logging.StreamHandler()
+    if log_format == "json":
+        handler.setFormatter(JSONFormatter())
+    else:
+        handler.setFormatter(
+            logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s")
+        )
+
+    root = logging.getLogger()
+    root.setLevel(level)
+    root.handlers.clear()
+    root.addHandler(handler)

application/utils/redis.py

@@ -6,9 +6,7 @@
 import rq
 import time
 
-logging.basicConfig()
 logger = logging.getLogger(__name__)
-logger.setLevel(logging.INFO)
 
 
 def empty_queues(redis: redis.Redis):