🐛(smtp) fix opportunistic TLS against MXes with mismatched certs

docs/env.md

@@ -93,7 +93,7 @@ The application uses a new environment file structure with `.defaults` and `.loc
 | `MTA_OUT_RELAY_PASSWORD` | `pass` | Outbound SMTP password for relay mode | Optional |
 | `MTA_OUT_DIRECT_PROXIES` | `[]` | List of SOCKS proxy URLs (randomly chosen when non-empty; used in direct mode) | Optional |
 | `MTA_OUT_DIRECT_PORT` | `25` | TCP port for direct mode on remote MX servers | Optional |
-| `MTA_OUT_SMTP_TLS_SECURITY_LEVEL` | `may` | SMTP TLS security level ("none", "may") | Optional |
+| `MTA_OUT_SMTP_TLS_SECURITY_LEVEL` | `may` | SMTP TLS security level: `none`, `may` (opportunistic, no cert check, matches Postfix), or `secure` (mandatory TLS + CA chain + hostname check). Applied to both direct and relay modes — set to `secure` when running against a controlled relay with a valid cert. | Optional |
 | `MDA_API_SECRET` | `my-shared-secret-mda` | Shared secret for MDA API | Required |
 | `MDA_API_BASE_URL` | `http://backend-dev:8000/api/v1.0/` | Base URL for MDA API | Dev |
 

src/backend/core/mda/outbound.py

@@ -754,6 +754,7 @@ def send_outbound_email(
             message_content=mime_data,
             smtp_username=mta_out_smtp_username,
             smtp_password=mta_out_smtp_password,
+            smtp_tls_security_level=settings.MTA_OUT_SMTP_TLS_SECURITY_LEVEL,
         )
         return statuses
 

src/backend/core/mda/outbound_direct.py

@@ -14,7 +14,7 @@
 
 import dns.resolver
 
-from core.mda.smtp import send_smtp_mail
+from core.mda.smtp import SmtpProxy, send_smtp_mail
 
 logger = logging.getLogger(__name__)
 
@@ -85,21 +85,19 @@ def group_recipients_by_mx(recipients: List[str]) -> Dict[str, Dict[str, Any]]:
     return domain_map
 
 
-def select_smtp_proxy() -> Dict[str, Any]:
-    """
-    Select an SMTP proxy to use for sending messages.
-    """
+def select_smtp_proxy() -> Optional[SmtpProxy]:
+    """Pick a SOCKS5 proxy at random from MTA_OUT_DIRECT_PROXIES, if any."""
     if len(settings.MTA_OUT_DIRECT_PROXIES) > 0:
-        proxy = random.choice(settings.MTA_OUT_DIRECT_PROXIES)  # noqa: S311
-        parsed = urlparse(proxy)
-        return {
-            "proxy_host": parsed.hostname,
-            "proxy_port": parsed.port,
-            "proxy_username": parsed.username,
-            "proxy_password": parsed.password,
-            "sender_hostname": parsed.hostname,
-        }
-    return {}
+        url = random.choice(settings.MTA_OUT_DIRECT_PROXIES)  # noqa: S311
+        parsed = urlparse(url)
+        return SmtpProxy(
+            host=parsed.hostname,
+            port=parsed.port,
+            username=parsed.username,
+            password=parsed.password,
+            sender_hostname=parsed.hostname,
+        )
+    return None
 
 
 def send_message_via_mx(envelope_from, recipient_emails, mime_data) -> Dict[str, Any]:
@@ -151,8 +149,6 @@ def send_message_via_mx(envelope_from, recipient_emails, mime_data) -> Dict[str,
                 remaining_recipients,
             )
 
-            proxy_settings = select_smtp_proxy()
-
             # Use direct SMTP, no auth
             smtp_statuses = send_smtp_mail(
                 smtp_host=mx_hostname,
@@ -161,7 +157,8 @@ def send_message_via_mx(envelope_from, recipient_emails, mime_data) -> Dict[str,
                 envelope_from=envelope_from,
                 recipient_emails=remaining_recipients.copy(),
                 message_content=mime_data,
-                **proxy_settings,
+                smtp_tls_security_level=settings.MTA_OUT_SMTP_TLS_SECURITY_LEVEL,
+                proxy=select_smtp_proxy(),
             )
 
             # Process results and update remaining recipients

src/backend/core/mda/smtp.py

@@ -3,10 +3,23 @@
 import logging
 import smtplib
 import ssl
+from dataclasses import dataclass
 from typing import Any, Dict, Optional
 
 import socks
 
+
+@dataclass(frozen=True)
+class SmtpProxy:
+    """SOCKS5 proxy + EHLO identity to use when sending."""
+
+    host: str
+    port: int
+    username: Optional[str] = None
+    password: Optional[str] = None
+    sender_hostname: Optional[str] = None
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -78,6 +91,58 @@ def _get_socket(self, host, port, timeout):
         )
 
 
+def _build_tls_context(level: str) -> ssl.SSLContext:
+    """Build an SSL context matching Postfix's smtp_tls_security_level semantics.
+
+    "secure" performs full PKI verification (CA chain + hostname). "may" creates
+    an unverified context: many public MXes serve mismatched or self-signed
+    certs, and rejecting them would just push delivery to cleartext anyway.
+    """
+    ctx = ssl.create_default_context()
+    if level != "secure":
+        ctx.check_hostname = False
+        ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
+def _starttls_upgrade(
+    client: "ProxySMTP", level: str, sender_hostname: Optional[str]
+) -> Optional[str]:
+    """Run the STARTTLS dance, with policy-aware fallback.
+
+    Returns None on success (or when policy allows continuing in cleartext),
+    the sentinel ``"fallback"`` to ask the caller to retry the whole session
+    without TLS (``may`` only), or an error string to defer delivery.
+    """
+    if level == "none":
+        return None
+    if not client.has_extn("starttls"):
+        if level == "secure":
+            return (
+                "STARTTLS not advertised by server "
+                "(required by smtp_tls_security_level=secure)"
+            )
+        return None
+    try:
+        code, msg = client.starttls(context=_build_tls_context(level))
+        logger.debug("SMTP: STARTTLS response: %s %s", code, msg)
+        if not 200 <= code <= 299:
+            raise RuntimeError(f"STARTTLS rejected: {code} {msg}")
+        code, msg = client.ehlo(sender_hostname)
+        logger.debug("SMTP: EHLO2 response: %s %s", code, msg)
+        if not 200 <= code <= 299:
+            raise RuntimeError(f"EHLO after STARTTLS failed: {code} {msg}")
+        return None
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        if level == "may":
+            logger.warning(
+                "SMTP: STARTTLS failed: %s, falling back to unencrypted socket", e
+            )
+            return "fallback"
+        logger.error("SMTP: Failed to send email with TLS: %s", e, exc_info=True)
+        return "Failed to send email with TLS"
+
+
 # pylint: disable=too-many-arguments
 def send_smtp_mail(
     smtp_host: str,
@@ -88,11 +153,7 @@ def send_smtp_mail(
     smtp_username: Optional[str] = None,
     smtp_password: Optional[str] = None,
     timeout: int = 60,
-    proxy_host: Optional[str] = None,
-    proxy_port: Optional[int] = None,
-    proxy_username: Optional[str] = None,
-    proxy_password: Optional[str] = None,
-    sender_hostname: Optional[str] = None,
+    proxy: Optional[SmtpProxy] = None,
     smtp_ip: Optional[str] = None,
     smtp_tls_security_level: Optional[str] = "may",
 ) -> Dict[str, Any]:
@@ -109,12 +170,8 @@ def send_smtp_mail(
         smtp_username: SMTP username (optional)
         smtp_password: SMTP password (optional)
         timeout: Connection timeout in seconds
-        proxy_host: SOCKS5 proxy hostname
-        proxy_port: SOCKS5 proxy port
-        proxy_username: SOCKS5 proxy username
-        proxy_password: SOCKS5 proxy password
-        sender_hostname: Local hostname to use for SMTP EHLO/HELO
-        smtp_tls_security_level: SMTP TLS security level ("none", "may")
+        proxy: Optional SOCKS5 proxy and local hostname to present in EHLO/HELO
+        smtp_tls_security_level: SMTP TLS security level ("none", "may", "secure")
 
     Returns:
         Dict mapping recipient emails to delivery status with retry flag:
@@ -127,6 +184,8 @@ def send_smtp_mail(
         }
     """
     statuses = {}
+    sender_hostname = proxy.sender_hostname if proxy else None
+    proxy_host = proxy.host if proxy else None
 
     def error_for_all_recipients(error: str, retry: bool) -> Dict[str, Any]:
         return {
@@ -145,9 +204,9 @@ def error_for_all_recipients(error: str, retry: bool) -> Dict[str, Any]:
         port=None,
         timeout=timeout,
         proxy_host=proxy_host,
-        proxy_port=proxy_port,
-        proxy_username=proxy_username,
-        proxy_password=proxy_password,
+        proxy_port=proxy.port if proxy else None,
+        proxy_username=proxy.username if proxy else None,
+        proxy_password=proxy.password if proxy else None,
         local_hostname=sender_hostname,
     )
 
@@ -180,61 +239,25 @@ def _quit():
                 _quit()
                 return error_for_all_recipients(f"HELO failed: {code} {msg}", True)
 
-        if client.has_extn("starttls") and smtp_tls_security_level != "none":
-            try:
-                # smtplib.SMTP.starttls() doesn't validate certificates by default!
-                # https://github.com/python/cpython/issues/91826
-                ssl_context = ssl.create_default_context()
-                ssl_context.check_hostname = True
-                ssl_context.verify_mode = ssl.CERT_REQUIRED
-                (code, msg) = client.starttls(context=ssl_context)
-                logger.debug("SMTP: STARTTLS response: %s %s", code, msg)
-                if not 200 <= code <= 299:
-                    _quit()
-                    if smtp_tls_security_level == "may":
-                        raise Exception(f"STARTTLS failed : {code} {msg}")  # pylint: disable=broad-exception-raised
-                    return error_for_all_recipients(
-                        f"STARTTLS failed: {code} {msg}", True
-                    )
-
-                # Restart the SMTP session now that we're in TLS mode
-                (code, msg) = client.ehlo(sender_hostname)
-                logger.debug("SMTP: EHLO2 response: %s %s", code, msg)
-                if not 200 <= code <= 299:
-                    _quit()
-                    if smtp_tls_security_level == "may":
-                        raise Exception(f"STARTTLS failed : {code} {msg}")  # pylint: disable=broad-exception-raised
-                    return error_for_all_recipients(
-                        f"EHLO after STARTTLS failed: {code} {msg}", True
-                    )
-            except Exception as e:  # pylint: disable=broad-exception-caught
-                if smtp_tls_security_level == "may":
-                    logger.warning(
-                        "SMTP: STARTTLS failed: %s, falling back to unencrypted socket",
-                        e,
-                    )
-                    return send_smtp_mail(
-                        smtp_host=smtp_host,
-                        smtp_ip=smtp_ip,
-                        smtp_port=smtp_port,
-                        envelope_from=envelope_from,
-                        recipient_emails=recipient_emails,
-                        message_content=message_content,
-                        smtp_username=smtp_username,
-                        smtp_password=smtp_password,
-                        timeout=timeout,
-                        proxy_host=proxy_host,
-                        proxy_port=proxy_port,
-                        proxy_username=proxy_username,
-                        proxy_password=proxy_password,
-                        sender_hostname=sender_hostname,
-                        smtp_tls_security_level="none",
-                    )
-                logger.error(
-                    "SMTP: Failed to send email with TLS: %s", e, exc_info=True
-                )
-                _quit()
-                return error_for_all_recipients("Failed to send email with TLS", True)
+        tls_result = _starttls_upgrade(client, smtp_tls_security_level, sender_hostname)
+        if tls_result == "fallback":
+            _quit()
+            return send_smtp_mail(
+                smtp_host=smtp_host,
+                smtp_ip=smtp_ip,
+                smtp_port=smtp_port,
+                envelope_from=envelope_from,
+                recipient_emails=recipient_emails,
+                message_content=message_content,
+                smtp_username=smtp_username,
+                smtp_password=smtp_password,
+                timeout=timeout,
+                proxy=proxy,
+                smtp_tls_security_level="none",
+            )
+        if tls_result is not None:
+            _quit()
+            return error_for_all_recipients(tls_result, True)
 
         if smtp_username and smtp_password:
             try:

src/backend/core/tests/mda/test_outbound.py

@@ -16,6 +16,7 @@
 from core import enums, factories, models
 from core.mda import outbound
 from core.mda.signing import generate_dkim_key, sign_message_dkim
+from core.mda.smtp import SmtpProxy
 
 SCHEMA_CUSTOM_ATTRIBUTES = {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
@@ -205,6 +206,7 @@ def test_outbound_send_relay(self, mock_smtp_send, draft_message):
             message_content=draft_message.blob.get_content(),
             smtp_username="smtp_user",
             smtp_password="smtp_pass",
+            smtp_tls_security_level="may",
         )
 
         # Check message object updated
@@ -347,6 +349,14 @@ def resolve_return_value(domain, record_type, **kwargs):
 
         sorted_calls = sorted(mock_smtp_send.mock_calls, key=lambda x: x[2]["smtp_ip"])
 
+        expected_proxy = SmtpProxy(
+            host="smtp.proxy",
+            port=1080,
+            username="proxyuser",
+            password="proxyuser",
+            sender_hostname="smtp.proxy",
+        )
+
         # Check first call - to@example.com, cc@example.com, cc2@example.com to mx1.example.com
         assert sorted_calls[0] == call(
             smtp_host="mx1.example.com",
@@ -355,11 +365,8 @@ def resolve_return_value(domain, record_type, **kwargs):
             envelope_from=draft_message.sender.email,
             recipient_emails={"to@example.com", "cc@example.com", "cc2@example.com"},
             message_content=draft_message.blob.get_content(),
-            proxy_host="smtp.proxy",
-            proxy_port=1080,
-            proxy_username="proxyuser",
-            proxy_password="proxyuser",
-            sender_hostname="smtp.proxy",
+            smtp_tls_security_level="may",
+            proxy=expected_proxy,
         )
 
         # Check second call - cc@example.com, to@example.com retry to mx2.example.com
@@ -370,11 +377,8 @@ def resolve_return_value(domain, record_type, **kwargs):
             envelope_from=draft_message.sender.email,
             recipient_emails={"cc@example.com", "to@example.com"},
             message_content=draft_message.blob.get_content(),
-            proxy_host="smtp.proxy",
-            proxy_port=1080,
-            proxy_username="proxyuser",
-            proxy_password="proxyuser",
-            sender_hostname="smtp.proxy",
+            smtp_tls_security_level="may",
+            proxy=expected_proxy,
         )
 
         # Check third call - bcc@example2.com to mx1.example2.com
@@ -385,11 +389,8 @@ def resolve_return_value(domain, record_type, **kwargs):
             envelope_from=draft_message.sender.email,
             recipient_emails={"bcc@example2.com"},
             message_content=draft_message.blob.get_content(),
-            proxy_host="smtp.proxy",
-            proxy_port=1080,
-            proxy_username="proxyuser",
-            proxy_password="proxyuser",
-            sender_hostname="smtp.proxy",
+            smtp_tls_security_level="may",
+            proxy=expected_proxy,
         )
 
     @patch("core.mda.outbound_direct.dns.resolver.resolve")
@@ -431,6 +432,8 @@ def smtp_return_value(*args, **kwargs):
             envelope_from=draft_message.sender.email,
             recipient_emails={"bcc@example2.com"},
             message_content=draft_message.blob.get_content(),
+            smtp_tls_security_level="may",
+            proxy=None,
         )
 
         # Check message object updated