From 769ccac44732901d165822789eb855fde93f8971 Mon Sep 17 00:00:00 2001 From: Yosuke Otosu Date: Tue, 16 Dec 2025 16:31:45 +0900 Subject: [PATCH 1/7] Implement Microsoft OAuth2 authentication for SMTP --- ENV_LIST.md | 66 ++++-- app/config.py | 16 ++ app/model/mail/mail.py | 46 +++- app/model/mail/token_provider.py | 114 ++++++++++ tests/app/model/mail/test_mail_unit.py | 121 ++++++++++ tests/app/model/mail/test_token_provider.py | 239 ++++++++++++++++++++ 6 files changed, 580 insertions(+), 22 deletions(-) create mode 100644 app/model/mail/token_provider.py create mode 100644 tests/app/model/mail/test_mail_unit.py create mode 100644 tests/app/model/mail/test_token_provider.py diff --git a/ENV_LIST.md b/ENV_LIST.md index 852d19f48..2bf5bc88e 100644 --- a/ENV_LIST.md +++ b/ENV_LIST.md @@ -3,9 +3,9 @@ The list of environment variables that can be set for this system is as follows. ## Basic Settings -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |-------------------------|----------|------------------------------------------|-------------------------------------------------|-----------------------------------------------------------| -| APP_ENV | False | Running environment | local / dev / live | local | +| APP_ENV | False | Running environment | local / dev / live | local | | NETWORK | False | Running network | IBET / IBETFIN | IBET | | WEB3_CHAINID | False | Blockchain network ID | 1010032 | IBET: 1500002, IBETFIN: 1010032 | | WEB3_HTTP_PROVIDER | False | Web3 provider | http://localhost:8545 | http://localhost:8545 | @@ -18,7 +18,7 @@ The list of environment variables that can be set for this system is as follows. | APP_LOGFILE | False | Output location for application logs | /some/directory | /dev/stdout (standard output) | | ACCESS_LOGFILE | False | Output location for access logs | /some/directory | /dev/stdout (standard output) | | TZ | False | Time Zone | Europe/Berlin | Asia/Tokyo | -| DEFAULT_CURRENCY | False | Default currency code | EUR | JPY | +| DEFAULT_CURRENCY | False | Default currency code | EUR | JPY | ## API Server Settings @@ -27,9 +27,9 @@ The following parameters can be set as environment variables as startup paramete See [Gunicorn's official documentation](https://docs.gunicorn.org/en/stable/run.html#commonly-used-arguments) for details. -| Variable Name | Required | Details | Default | +| Variable Name | Required | Details | Default | |----------------------------|----------|--------------------------------------------------------------------------|---------| -| WORKER_COUNT | False | The number of worker processes. | 2 | +| WORKER_COUNT | False | The number of worker processes. | 2 | | WORKER_TIMEOUT | False | Workers silent for more than this many seconds are killed and restarted. | 60 | | WORKER_MAX_REQUESTS | False | The maximum number of requests a worker will process before restarting. | 0 | | WORKER_MAX_REQUESTS_JITTER | False | The maximum jitter to add to the max_requests setting. | 0 | @@ -38,7 +38,7 @@ See [Gunicorn's official documentation](https://docs.gunicorn.org/en/stable/run. ## Settings for each use case ### Token -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |--------------------------------|----------|--------------------------------------------------------------------|--------------------------------------------|---------| | BOND_TOKEN_ENABLED | False | Using ibet Bond token (security token) | 0 (not using) / 1 (using) | 0 | | SHARE_TOKEN_ENABLED | False | Using ibet Share token (security token) | 0 (not using) / 1 (using) | 0 | @@ -52,19 +52,19 @@ See [Gunicorn's official documentation](https://docs.gunicorn.org/en/stable/run. | TOKEN_SHORT_TERM_CACHE_TTL | False | Token attribute data cache (Short-Term) expiration time (seconds) | 60 | 40 | ### Token Escrow -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |---------------------------------------------|----------|---------------------------------------------|--------------------------------------------|---------| | IBET_ESCROW_CONTRACT_ADDRESS | False | Ibet Escrow contract address | 0x0000000000000000000000000000000000000000 | -- | | IBET_SECURITY_TOKEN_ESCROW_CONTRACT_ADDRESS | False | Ibet Security Token Escrow contract address | 0x0000000000000000000000000000000000000000 | -- | ### Token DVP -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |---------------------------------------------|----------|---------------------------------------------|--------------------------------------------|---------| | IBET_SECURITY_TOKEN_DVP_CONTRACT_ADDRESS | False | Ibet Security Token DVP contract address | 0x0000000000000000000000000000000000000000 | -- | ### On-chain Exchange (Only for utility tokens) -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |---------------------------------------------|----------|--------------------------------------------------------------------------|--------------------------------------------|---------| | PAYMENT_GATEWAY_CONTRACT_ADDRESS | False | PaymentGateway contract address | 0x0000000000000000000000000000000000000000 | -- | | IBET_MEMBERSHIP_EXCHANGE_CONTRACT_ADDRESS | False | IbetExchange contract address for Membership tokens | 0x0000000000000000000000000000000000000000 | -- | @@ -72,22 +72,23 @@ See [Gunicorn's official documentation](https://docs.gunicorn.org/en/stable/run. | EXCHANGE_NOTIFICATION_ENABLED | True* | Use of exchange-related notification (*Set only if you use IbetExchange) | 0 (not using) / 1 (using) | -- | ### Blockchain Explorer -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |---------------------|----------|-----------------------------------------------------|---------------------------|---------| | BC_EXPLORER_ENABLED | False | Parameter for starting the Blockchain Explorer | 0 (not using) / 1 (using) | 0 | ### Email Common -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |---------------------------------|----------|--------------------------------------------|---------------------------------|---------| | SMTP_METHOD | False | Email sending method | 0:SMTP server, 1:Amazon SES | 0 | +| SMTP_AUTH_METHOD | False | Authentication method | 0:PASSWORD, 1:XOAUTH2 | 0 | | SMTP_SENDER_NAME | False | Sender name | | -- | | SMTP_SENDER_EMAIL | False | Sender email address | example@example.com | -- | SMTP server -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |-------------------------------|----------|-------------------------------------------|---------------------------------|---------| | SMTP_SERVER_HOST | False | SMTP server name | smtp.office365.com | -- | | SMTP_SERVER_PORT | False | SMTP server port | 587 | -- | @@ -96,13 +97,22 @@ SMTP server Amazon SES -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |---------------------|----------|-----------------|-----------|---------| | AWS_SES_REGION_NAME | False | AWS region name | us-east-1 | -- | +SMTP XOAUTH2 (Microsoft) + +| Variable Name | Required | Details | Example | Default | +|-----------------------|----------|-------------------------|---------|---------| +| SMTP_MS_TENANT_ID | True | Microsoft Entra ID (Azure AD) Tenant ID | | -- | +| SMTP_MS_CLIENT_ID | True | Microsoft Entra ID (Azure AD) Client ID | | -- | +| SMTP_MS_CLIENT_SECRET | True | Microsoft Entra ID (Azure AD) Client Secret | | -- | +| SMTP_MS_REFRESH_TOKEN | True | OAuth2 Refresh Token (offline_access, SMTP.Send) | | -- | + Send settings -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |--------------------------------------------|----------|---------------------------------------------------------------------------------|-------------------------------|---------| | ALLOWED_EMAIL_DESTINATION_DOMAIN_LIST | False | Domains allowed to send email. Not set if all domains are allowed. | example.com,example.net | -- | | DISALLOWED_DESTINATION_EMAIL_ADDRESS_REGEX | False | Regular expression for destination email addresses that are not allowed to send | ^[a-zA-Z0-9_.+-]+@example.com | -- | @@ -111,6 +121,32 @@ Send settings ### Chat Webhook -| Variable Name | Required | Details | Example | Default | +| Variable Name | Required | Details | Example | Default | |-------------------|----------|------------------|--------------------------------------------------------------------------------|---------| | CHAT_WEBHOOK_URL | False | Chat webhook url | https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX | -- | + + +## Appendix + +### Email Configuration Matrix + +| Variable / Pattern | SMTP Password | SMTP XOAUTH2 | Amazon SES | Details | +| :--- | :---: | :---: | :---: | :--- | +| **BASE SETTINGS** | | | | | +| `SMTP_METHOD` | `0` | `0` | `1` | 0:SMTP, 1:SES | +| `SMTP_SENDER_EMAIL` | Required | Required | Required | Sender definition | +| `SMTP_SENDER_NAME` | Optional | Optional | Optional | Sender display name | +| **SMTP AUTH** | | | | | +| `SMTP_AUTH_METHOD` | `0` | `1` | - | 0:Password, 1:XOAUTH2 | +| **SMTP SERVER** | | | | | +| `SMTP_SERVER_HOST` | Required | Required | - | e.g. smtp.office365.com | +| `SMTP_SERVER_PORT` | Required | Required | - | e.g. 587 | +| `SMTP_SERVER_ENCRYPTION_METHOD`| Optional | Optional | - | 0:STARTTLS (Default), 1:SSL, 2:None | +| `SMTP_SENDER_PASSWORD` | Required | - | - | For SMTP Auth | +| **MICROSOFT OAUTH** | | | | | +| `SMTP_MS_TENANT_ID` | - | Required | - | | +| `SMTP_MS_CLIENT_ID` | - | Required | - | | +| `SMTP_MS_CLIENT_SECRET` | - | Required | - | | +| `SMTP_MS_REFRESH_TOKEN` | - | Required | - | | +| **AMAZON SES** | | | | | +| `AWS_SES_REGION_NAME` | - | - | Required | | diff --git a/app/config.py b/app/config.py index b88530f91..48190961a 100644 --- a/app/config.py +++ b/app/config.py @@ -20,6 +20,7 @@ import configparser import os from email.policy import HTTP, SMTP, SMTPUTF8 +from typing import Literal from dotenv import load_dotenv @@ -358,6 +359,21 @@ # Amazon SES AWS_SES_REGION_NAME = os.environ.get("AWS_SES_REGION_NAME") +# SMTP Auth Method (0:PASSWORD, 1:XOAUTH2) +SMTP_AUTH_METHOD: int = int(os.environ.get("SMTP_AUTH_METHOD", 0)) +# Token Provider (microsoft fixed) +SMTP_PROVIDER: Literal["microsoft"] = "microsoft" + +# Microsoft OAuth settings +# - Tenant ID: Microsoft Entra ID (Azure AD) Tenant ID +SMTP_MS_TENANT_ID: str | None = os.environ.get("SMTP_MS_TENANT_ID") +# - Client ID: Microsoft Entra ID (Azure AD) Client ID +SMTP_MS_CLIENT_ID: str | None = os.environ.get("SMTP_MS_CLIENT_ID") +# - Client Secret: Microsoft Entra ID (Azure AD) Client Secret +SMTP_MS_CLIENT_SECRET: str | None = os.environ.get("SMTP_MS_CLIENT_SECRET") +# - Refresh Token: OAuth2 Refresh Token (acquired with offline_access and SMTP.Send scopes) +SMTP_MS_REFRESH_TOKEN: str | None = os.environ.get("SMTP_MS_REFRESH_TOKEN") + # Send settings ALLOWED_EMAIL_DESTINATION_DOMAIN_LIST = ( os.environ.get("AUTHORIZED_EMAIL_DESTINATION_DOMAIN_LIST").split(",") diff --git a/app/model/mail/mail.py b/app/model/mail/mail.py index 698586a59..92972bea5 100644 --- a/app/model/mail/mail.py +++ b/app/model/mail/mail.py @@ -30,6 +30,7 @@ from app.config import ( AWS_SES_REGION_NAME, + SMTP_AUTH_METHOD, SMTP_METHOD, SMTP_POLICY, SMTP_SENDER_EMAIL, @@ -59,13 +60,20 @@ def __init__( html_content: str, file: File | None, ): + if SMTP_SENDER_EMAIL is None: + raise RuntimeError("SMTP sender email is not set.") self.sender_email = SMTP_SENDER_EMAIL self.to_email = to_email if SMTP_METHOD == 0: # SMTP server + if SMTP_SERVER_HOST is None or SMTP_SERVER_PORT is None: + raise RuntimeError("SMTP server host or port is not set.") self.server_host = SMTP_SERVER_HOST self.server_port = SMTP_SERVER_PORT - self.sender_password = SMTP_SENDER_PASSWORD + if SMTP_AUTH_METHOD == 0: # PASSWORD + self.sender_password = SMTP_SENDER_PASSWORD + elif SMTP_AUTH_METHOD == 1: # XOAUTH2 + pass elif SMTP_METHOD == 1: # Amazon SES self.aws_region_name = AWS_SES_REGION_NAME self.msg = MIMEMultipart("alternative", policy=SMTP_POLICY) @@ -94,22 +102,46 @@ def send_mail(self): if SMTP_METHOD == 0: # SMTP server # Initialize a new smtp client if SMTP_SERVER_ENCRYPTION_METHOD == 0: # STARTTLS - smtp_client = smtplib.SMTP(host=self.server_host, port=self.server_port) + smtp_client = smtplib.SMTP( + host=self.server_host, port=int(self.server_port) + ) smtp_client.ehlo() smtp_client.starttls() smtp_client.ehlo() elif SMTP_SERVER_ENCRYPTION_METHOD == 1: # SSL smtp_client = smtplib.SMTP_SSL( host=self.server_host, - port=self.server_port, + port=int(self.server_port), context=ssl.create_default_context(), ) else: # NO-ENCRYPT - smtp_client = smtplib.SMTP(host=self.server_host, port=self.server_port) - + smtp_client = smtplib.SMTP( + host=self.server_host, port=int(self.server_port) + ) # LOGIN - if self.sender_password is not None: - smtp_client.login(self.sender_email, self.sender_password) + if SMTP_AUTH_METHOD == 0: # PASSWORD + if self.sender_password is not None: + smtp_client.login(self.sender_email, self.sender_password) + elif SMTP_AUTH_METHOD == 1: # XOAUTH2 + # Get Access Token + from app.config import SMTP_PROVIDER + from app.model.mail.token_provider import MicrosoftTokenProvider + + match SMTP_PROVIDER: + case "microsoft": + token_provider = MicrosoftTokenProvider() + case _: + raise ValueError(f"Unknown SMTP_PROVIDER: {SMTP_PROVIDER}") + access_token = token_provider.get_access_token() + + # Auth + import base64 + + auth_str = ( + f"user={self.sender_email}\x01auth=Bearer {access_token}\x01\x01" + ) + auth_b64 = base64.b64encode(auth_str.encode("utf-8")).decode("utf-8") + smtp_client.docmd("AUTH", "XOAUTH2 " + auth_b64) # Send mail try: diff --git a/app/model/mail/token_provider.py b/app/model/mail/token_provider.py new file mode 100644 index 000000000..3b248f092 --- /dev/null +++ b/app/model/mail/token_provider.py @@ -0,0 +1,114 @@ +""" +Copyright BOOSTRY Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" + +import abc + +import requests +from pydantic import BaseModel, ValidationError +from requests.adapters import HTTPAdapter +from urllib3 import Retry + +from app.config import ( + SMTP_MS_CLIENT_ID, + SMTP_MS_CLIENT_SECRET, + SMTP_MS_REFRESH_TOKEN, + SMTP_MS_TENANT_ID, +) + + +class TokenProvider(metaclass=abc.ABCMeta): + @abc.abstractmethod + def get_access_token(self) -> str: + """ + Get access token for authentication + """ + pass + + +class MicrosoftTokenResponse(BaseModel): + """ + Schema for Microsoft Identity Platform token response + """ + + access_token: str + expires_in: int + token_type: str + + +class MicrosoftTokenProvider(TokenProvider): + """ + Provider that fetches access token from Microsoft Identity Platform + using a Refresh Token. + """ + + _access_token: str | None = None + _token_expiry: float = 0.0 + + def get_access_token(self) -> str: + import time + + # Return cached token if valid (with 60 seconds safety buffer) + if self._access_token and time.time() < self._token_expiry - 60: + return self._access_token + + tenant_id = SMTP_MS_TENANT_ID + client_id = SMTP_MS_CLIENT_ID + client_secret = SMTP_MS_CLIENT_SECRET + refresh_token = SMTP_MS_REFRESH_TOKEN + + if not all([tenant_id, client_id, client_secret, refresh_token]): + raise ValueError("Missing Microsoft OAuth configuration") + + token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" + + data = { + "grant_type": "refresh_token", + "client_id": client_id, + "client_secret": client_secret, + "refresh_token": refresh_token, + "scope": "https://outlook.office365.com/.default", + } + + try: + with requests.Session() as session: + retries = Retry( + total=3, + backoff_factor=1, + status_forcelist=[500, 502, 503, 504], + allowed_methods=["POST"], + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount("https://", adapter) + + response = session.post(token_url, data=data, timeout=10) + response.raise_for_status() + + # Validate response schema + token_data = MicrosoftTokenResponse.model_validate(response.json()) + + # Update cache + self.__class__._access_token = token_data.access_token + # Set expiry time relative to now + self.__class__._token_expiry = time.time() + token_data.expires_in + + return token_data.access_token + + except (requests.exceptions.RequestException, ValidationError) as e: + # You might want to log this error in a real app + raise RuntimeError(f"Failed to refresh access token: {str(e)}") from e diff --git a/tests/app/model/mail/test_mail_unit.py b/tests/app/model/mail/test_mail_unit.py new file mode 100644 index 000000000..2b3c31748 --- /dev/null +++ b/tests/app/model/mail/test_mail_unit.py @@ -0,0 +1,121 @@ +""" +Copyright BOOSTRY Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" + +from unittest.mock import MagicMock, patch + +from app.model.mail import Mail + + +class TestMailUnit: + """ + Unit tests for app.model.mail.Mail + Mocks smtplib to verify command interactions + """ + + def test_send_mail_password_auth(self): + """ + Verify that PASSWORD auth (default) calls login() + """ + # Arrange + with ( + patch("app.model.mail.mail.SMTP_METHOD", 0), + patch("app.model.mail.mail.SMTP_AUTH_METHOD", 0), + patch("app.model.mail.mail.SMTP_SENDER_NAME", "Sender Name"), + patch("app.model.mail.mail.SMTP_SENDER_EMAIL", "sender@example.com"), + patch("smtplib.SMTP") as mock_smtp_cls, + ): + mock_smtp = MagicMock() + mock_smtp_cls.return_value = mock_smtp + + mail = Mail( + to_email="test@example.com", + subject="Test Subject", + text_content="Body", + html_content="

Body

", + file=None, + ) + mail.sender_password = "password" + + # Act + mail.send_mail() + + # Assert + mock_smtp.login.assert_called_once() + mock_smtp.docmd.assert_not_called() + mock_smtp.sendmail.assert_called_once() + mock_smtp.quit.assert_called_once() + + def test_send_mail_xoauth2_auth(self): + """ + Verify that XOAUTH2 auth calls docmd("AUTH", ...) + """ + # Arrange + with ( + patch("app.model.mail.mail.SMTP_METHOD", 0), + patch("app.model.mail.mail.SMTP_AUTH_METHOD", 1), + patch("app.model.mail.mail.SMTP_SENDER_NAME", "Sender Name"), + patch("app.model.mail.mail.SMTP_SENDER_EMAIL", "sender@example.com"), + patch("app.config.SMTP_PROVIDER", "microsoft"), + patch( + "app.model.mail.token_provider.MicrosoftTokenProvider" + ) as MockTokenProvider, + patch("smtplib.SMTP") as mock_smtp_cls, + ): + mock_smtp = MagicMock() + mock_smtp_cls.return_value = mock_smtp + + mock_token_provider = MagicMock() + MockTokenProvider.return_value = mock_token_provider + mock_token_provider.get_access_token.return_value = "test_token" + + mail = Mail( + to_email="test@example.com", + subject="Test Subject", + text_content="Body", + html_content="

Body

", + file=None, + ) + + # Act + mail.send_mail() + + # Assert + mock_smtp.login.assert_not_called() + + # Verify AUTH XOAUTH2 command + # Expected auth string: user=\x01auth=Bearer \x01\x01 + # Note: config.SMTP_SENDER_EMAIL is mocked/defaulted during imports, + # ideally we should patch it if we want to be strict about the content. + # Here we just check if docmd was called with AUTH + mock_smtp.docmd.assert_called_once() + args, _ = mock_smtp.docmd.call_args + assert args[0] == "AUTH" + assert args[1].startswith("XOAUTH2 ") + + # Verify basic structure of the base64 encoded part + import base64 + + sent_b64 = args[1].split(" ")[1] + decoded_auth = base64.b64decode(sent_b64).decode("utf-8") + assert "user=sender@example.com" in decoded_auth + assert "auth=Bearer test_token" in decoded_auth + assert decoded_auth.endswith("\x01\x01") + + mock_smtp.sendmail.assert_called_once() + mock_smtp.quit.assert_called_once() diff --git a/tests/app/model/mail/test_token_provider.py b/tests/app/model/mail/test_token_provider.py new file mode 100644 index 000000000..bf5eabcd5 --- /dev/null +++ b/tests/app/model/mail/test_token_provider.py @@ -0,0 +1,239 @@ +""" +Copyright BOOSTRY Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" + +from unittest.mock import MagicMock, patch + +import pytest +import requests +from requests.adapters import HTTPAdapter + +from app.model.mail.token_provider import MicrosoftTokenProvider + + +class TestMicrosoftTokenProvider: + """ + Unit tests for app.model.mail.token_provider.MicrosoftTokenProvider + """ + + def test_get_access_token_success(self): + """ + Verify that get_access_token correctly parses successful response + """ + # Arrange + with ( + patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), + patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), + patch("requests.Session.post") as mock_post, + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "token_type": "Bearer", + "scope": "https://outlook.office365.com/.default", + "expires_in": 3599, + "ext_expires_in": 3599, + "access_token": "valid_access_token", + "refresh_token": "new_refresh_token", + } + mock_post.return_value = mock_response + + provider = MicrosoftTokenProvider() + + # Act + token = provider.get_access_token() + + # Assert + assert token == "valid_access_token" + + def test_get_access_token_missing_config(self): + """ + Verify that missing config raises ValueError + """ + # Ensure cache is empty + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 + + with patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", None): + provider = MicrosoftTokenProvider() + with pytest.raises( + ValueError, match="Missing Microsoft OAuth configuration" + ): + provider.get_access_token() + + def test_get_access_token_api_error(self): + """ + Verify that API error raises RuntimeError + """ + # Ensure cache is empty + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 + + # Arrange + with ( + patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), + patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), + patch("requests.Session.post") as mock_post, + ): + mock_response = MagicMock() + mock_response.status_code = 400 + mock_response.raise_for_status.side_effect = requests.exceptions.HTTPError( + "400 Bad Request" + ) + mock_post.return_value = mock_response + + provider = MicrosoftTokenProvider() + + # Act & Assert + with pytest.raises(RuntimeError, match="Failed to refresh access token"): + provider.get_access_token() + + def test_get_access_token_validation_error(self): + """ + Verify that invalid response schema raises RuntimeError (wrapping ValidationError) + """ + # Ensure cache is empty + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 + + # Arrange + with ( + patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), + patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), + patch("requests.Session.post") as mock_post, + ): + mock_response = MagicMock() + mock_response.status_code = 200 + # Missing expires_in (and token_type) will cause validation error + mock_response.json.return_value = {"access_token": "valid_access_token"} + mock_post.return_value = mock_response + + provider = MicrosoftTokenProvider() + + # Act & Assert + with pytest.raises(RuntimeError, match="Failed to refresh access token"): + provider.get_access_token() + + def test_get_access_token_retry_logic(self): + """ + Verify that retries are configured + """ + # Ensure cache is empty + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 + + # Arrange + with ( + patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), + patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), + patch("requests.Session.post") as mock_post, + patch("requests.Session.mount") as mock_mount, + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "access_token": "retry_token", + "expires_in": 3599, + "token_type": "Bearer", + } + mock_post.return_value = mock_response + + provider = MicrosoftTokenProvider() + provider.get_access_token() + + # Verify adapter mount + mock_mount.assert_called() + args, _ = mock_mount.call_args + adapter = args[1] + assert isinstance(adapter, HTTPAdapter) + assert adapter.max_retries.total == 3 + assert adapter.max_retries.status_forcelist == [500, 502, 503, 504] + + def test_get_access_token_cache_hit(self): + """ + Verify that cached token is returned if valid + """ + import time + + # Manually set cache + future_time = time.time() + 3600 + MicrosoftTokenProvider._access_token = "cached_token" + MicrosoftTokenProvider._token_expiry = future_time + + provider = MicrosoftTokenProvider() + + with patch("requests.Session.post") as mock_post: + token = provider.get_access_token() + + assert token == "cached_token" + mock_post.assert_not_called() + + # Cleanup + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 + + def test_get_access_token_cache_renew(self): + """ + Verify that token is renewed if cache is expired + """ + import time + from unittest.mock import MagicMock + + # Manually set expired cache + past_time = time.time() - 3600 + MicrosoftTokenProvider._access_token = "old_token" + MicrosoftTokenProvider._token_expiry = past_time + + provider = MicrosoftTokenProvider() + + with ( + patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), + patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), + patch("requests.Session.post") as mock_post, + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "access_token": "new_token", + "expires_in": 3599, + "token_type": "Bearer", + } + mock_post.return_value = mock_response + + token = provider.get_access_token() + + assert token == "new_token" + mock_post.assert_called_once() + + # Verify cache updated + assert MicrosoftTokenProvider._access_token == "new_token" + assert MicrosoftTokenProvider._token_expiry > time.time() + 3500 + + # Cleanup + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 From 19143479c3de9975182a2d187d0570e588541554 Mon Sep 17 00:00:00 2001 From: Yosuke Otosu Date: Wed, 17 Dec 2025 09:51:18 +0900 Subject: [PATCH 2/7] Fix file_content type to bytes and update Mail class tests with SMTP configuration mocks --- app/model/db/messaging.py | 2 +- app/model/mail/mail.py | 8 +++----- tests/batch/processor_Send_Mail_test.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/app/model/db/messaging.py b/app/model/db/messaging.py index e69c08225..47a4d09c3 100644 --- a/app/model/db/messaging.py +++ b/app/model/db/messaging.py @@ -44,7 +44,7 @@ class Mail(Base): # file name file_name: Mapped[str | None] = mapped_column(String(255), nullable=True) # file content - file_content: Mapped[str | None] = mapped_column( + file_content: Mapped[bytes | None] = mapped_column( LargeBinary().with_variant(LONGBLOB, "mysql"), nullable=True ) diff --git a/app/model/mail/mail.py b/app/model/mail/mail.py index 92972bea5..f514db301 100644 --- a/app/model/mail/mail.py +++ b/app/model/mail/mail.py @@ -17,6 +17,7 @@ SPDX-License-Identifier: Apache-2.0 """ +import base64 import mimetypes import smtplib import ssl @@ -33,6 +34,7 @@ SMTP_AUTH_METHOD, SMTP_METHOD, SMTP_POLICY, + SMTP_PROVIDER, SMTP_SENDER_EMAIL, SMTP_SENDER_NAME, SMTP_SENDER_PASSWORD, @@ -40,6 +42,7 @@ SMTP_SERVER_HOST, SMTP_SERVER_PORT, ) +from app.model.mail.token_provider import MicrosoftTokenProvider class File: @@ -124,9 +127,6 @@ def send_mail(self): smtp_client.login(self.sender_email, self.sender_password) elif SMTP_AUTH_METHOD == 1: # XOAUTH2 # Get Access Token - from app.config import SMTP_PROVIDER - from app.model.mail.token_provider import MicrosoftTokenProvider - match SMTP_PROVIDER: case "microsoft": token_provider = MicrosoftTokenProvider() @@ -135,8 +135,6 @@ def send_mail(self): access_token = token_provider.get_access_token() # Auth - import base64 - auth_str = ( f"user={self.sender_email}\x01auth=Bearer {access_token}\x01\x01" ) diff --git a/tests/batch/processor_Send_Mail_test.py b/tests/batch/processor_Send_Mail_test.py index 4842fc889..f98ad5033 100644 --- a/tests/batch/processor_Send_Mail_test.py +++ b/tests/batch/processor_Send_Mail_test.py @@ -47,6 +47,18 @@ def caplog(caplog: pytest.LogCaptureFixture): LOG.setLevel(default_log_level) +@mock.patch( + "app.model.mail.mail.SMTP_SENDER_EMAIL", + "test@example.com", +) +@mock.patch( + "app.model.mail.mail.SMTP_SERVER_HOST", + "example.com", +) +@mock.patch( + "app.model.mail.mail.SMTP_SERVER_PORT", + 587, +) class TestProcessorSendMail: # Normal_1 # No unsent email exists From 8143f4e77fcb47d33f719265e1dd6d8350059e8f Mon Sep 17 00:00:00 2001 From: Yosuke Otosu Date: Wed, 17 Dec 2025 09:57:57 +0900 Subject: [PATCH 3/7] Rename SMTP_PROVIDER to SMTP_AUTH_PROVIDER for clarity and update related references --- app/config.py | 2 +- app/model/mail/mail.py | 8 +++++--- tests/app/model/mail/test_mail_unit.py | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/config.py b/app/config.py index 48190961a..f320eb526 100644 --- a/app/config.py +++ b/app/config.py @@ -362,7 +362,7 @@ # SMTP Auth Method (0:PASSWORD, 1:XOAUTH2) SMTP_AUTH_METHOD: int = int(os.environ.get("SMTP_AUTH_METHOD", 0)) # Token Provider (microsoft fixed) -SMTP_PROVIDER: Literal["microsoft"] = "microsoft" +SMTP_AUTH_PROVIDER: Literal["microsoft"] = "microsoft" # Microsoft OAuth settings # - Tenant ID: Microsoft Entra ID (Azure AD) Tenant ID diff --git a/app/model/mail/mail.py b/app/model/mail/mail.py index f514db301..438435bc6 100644 --- a/app/model/mail/mail.py +++ b/app/model/mail/mail.py @@ -32,9 +32,9 @@ from app.config import ( AWS_SES_REGION_NAME, SMTP_AUTH_METHOD, + SMTP_AUTH_PROVIDER, SMTP_METHOD, SMTP_POLICY, - SMTP_PROVIDER, SMTP_SENDER_EMAIL, SMTP_SENDER_NAME, SMTP_SENDER_PASSWORD, @@ -127,11 +127,13 @@ def send_mail(self): smtp_client.login(self.sender_email, self.sender_password) elif SMTP_AUTH_METHOD == 1: # XOAUTH2 # Get Access Token - match SMTP_PROVIDER: + match SMTP_AUTH_PROVIDER: case "microsoft": token_provider = MicrosoftTokenProvider() case _: - raise ValueError(f"Unknown SMTP_PROVIDER: {SMTP_PROVIDER}") + raise ValueError( + f"Unknown SMTP_AUTH_PROVIDER: {SMTP_AUTH_PROVIDER}" + ) access_token = token_provider.get_access_token() # Auth diff --git a/tests/app/model/mail/test_mail_unit.py b/tests/app/model/mail/test_mail_unit.py index 2b3c31748..cdf6c76ff 100644 --- a/tests/app/model/mail/test_mail_unit.py +++ b/tests/app/model/mail/test_mail_unit.py @@ -71,7 +71,7 @@ def test_send_mail_xoauth2_auth(self): patch("app.model.mail.mail.SMTP_AUTH_METHOD", 1), patch("app.model.mail.mail.SMTP_SENDER_NAME", "Sender Name"), patch("app.model.mail.mail.SMTP_SENDER_EMAIL", "sender@example.com"), - patch("app.config.SMTP_PROVIDER", "microsoft"), + patch("app.config.SMTP_AUTH_PROVIDER", "microsoft"), patch( "app.model.mail.token_provider.MicrosoftTokenProvider" ) as MockTokenProvider, From 0ca1e9724b12f422ab7a208112f92617af702d2b Mon Sep 17 00:00:00 2001 From: Yosuke Otosu Date: Wed, 17 Dec 2025 10:28:30 +0900 Subject: [PATCH 4/7] fix tests --- app/model/mail/token_provider.py | 3 +-- tests/app/model/mail/test_mail_unit.py | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/app/model/mail/token_provider.py b/app/model/mail/token_provider.py index 3b248f092..18a7140ec 100644 --- a/app/model/mail/token_provider.py +++ b/app/model/mail/token_provider.py @@ -18,6 +18,7 @@ """ import abc +import time import requests from pydantic import BaseModel, ValidationError @@ -61,8 +62,6 @@ class MicrosoftTokenProvider(TokenProvider): _token_expiry: float = 0.0 def get_access_token(self) -> str: - import time - # Return cached token if valid (with 60 seconds safety buffer) if self._access_token and time.time() < self._token_expiry - 60: return self._access_token diff --git a/tests/app/model/mail/test_mail_unit.py b/tests/app/model/mail/test_mail_unit.py index cdf6c76ff..7fade04ac 100644 --- a/tests/app/model/mail/test_mail_unit.py +++ b/tests/app/model/mail/test_mail_unit.py @@ -22,6 +22,18 @@ from app.model.mail import Mail +@patch( + "app.model.mail.mail.SMTP_SENDER_EMAIL", + "test@example.com", +) +@patch( + "app.model.mail.mail.SMTP_SERVER_HOST", + "example.com", +) +@patch( + "app.model.mail.mail.SMTP_SERVER_PORT", + 587, +) class TestMailUnit: """ Unit tests for app.model.mail.Mail @@ -72,9 +84,7 @@ def test_send_mail_xoauth2_auth(self): patch("app.model.mail.mail.SMTP_SENDER_NAME", "Sender Name"), patch("app.model.mail.mail.SMTP_SENDER_EMAIL", "sender@example.com"), patch("app.config.SMTP_AUTH_PROVIDER", "microsoft"), - patch( - "app.model.mail.token_provider.MicrosoftTokenProvider" - ) as MockTokenProvider, + patch("app.model.mail.mail.MicrosoftTokenProvider") as MockTokenProvider, patch("smtplib.SMTP") as mock_smtp_cls, ): mock_smtp = MagicMock() From 419b14b12cd4d702228ed8fdc22e341a912de54a Mon Sep 17 00:00:00 2001 From: Yosuke Otosu Date: Wed, 17 Dec 2025 13:12:26 +0900 Subject: [PATCH 5/7] fix and add client credentials flow --- ENV_LIST.md | 8 +- app/config.py | 6 +- app/model/mail/token_provider.py | 101 ++++++++- pyproject.toml | 1 + tests/app/model/mail/test_token_provider.py | 13 +- .../model/mail/test_token_provider_cert.py | 197 ++++++++++++++++++ uv.lock | 2 + 7 files changed, 310 insertions(+), 18 deletions(-) create mode 100644 tests/app/model/mail/test_token_provider_cert.py diff --git a/ENV_LIST.md b/ENV_LIST.md index 2bf5bc88e..7066c521d 100644 --- a/ENV_LIST.md +++ b/ENV_LIST.md @@ -107,8 +107,8 @@ SMTP XOAUTH2 (Microsoft) |-----------------------|----------|-------------------------|---------|---------| | SMTP_MS_TENANT_ID | True | Microsoft Entra ID (Azure AD) Tenant ID | | -- | | SMTP_MS_CLIENT_ID | True | Microsoft Entra ID (Azure AD) Client ID | | -- | -| SMTP_MS_CLIENT_SECRET | True | Microsoft Entra ID (Azure AD) Client Secret | | -- | -| SMTP_MS_REFRESH_TOKEN | True | OAuth2 Refresh Token (offline_access, SMTP.Send) | | -- | +| SMTP_MS_CLIENT_SECRET | True* | Microsoft Entra ID (Azure AD) Client Secret (*Required if Client Certificate is unused) | | -- | +| SMTP_MS_CLIENT_CERT_PATH | False* | Path to Client Certificate Private Key (PEM) (*Alternative to Client Secret) | | data/smtp_ms_client_certificate.pem | Send settings @@ -146,7 +146,7 @@ Send settings | **MICROSOFT OAUTH** | | | | | | `SMTP_MS_TENANT_ID` | - | Required | - | | | `SMTP_MS_CLIENT_ID` | - | Required | - | | -| `SMTP_MS_CLIENT_SECRET` | - | Required | - | | -| `SMTP_MS_REFRESH_TOKEN` | - | Required | - | | +| `SMTP_MS_CLIENT_SECRET` | - | Required* | - | *Required if Client Certificate is unused | +| `SMTP_MS_CLIENT_CERT_PATH` | - | Optional* | - | Default: `data/smtp_ms_client_certificate.pem` (*Alternative to Client Secret) | | **AMAZON SES** | | | | | | `AWS_SES_REGION_NAME` | - | - | Required | | diff --git a/app/config.py b/app/config.py index f320eb526..14e42bde0 100644 --- a/app/config.py +++ b/app/config.py @@ -371,8 +371,10 @@ SMTP_MS_CLIENT_ID: str | None = os.environ.get("SMTP_MS_CLIENT_ID") # - Client Secret: Microsoft Entra ID (Azure AD) Client Secret SMTP_MS_CLIENT_SECRET: str | None = os.environ.get("SMTP_MS_CLIENT_SECRET") -# - Refresh Token: OAuth2 Refresh Token (acquired with offline_access and SMTP.Send scopes) -SMTP_MS_REFRESH_TOKEN: str | None = os.environ.get("SMTP_MS_REFRESH_TOKEN") +# - Client Certificate Path: Microsoft Entra ID (Azure AD) Client Certificate Path +SMTP_MS_CLIENT_CERT_PATH = os.environ.get( + "SMTP_MS_CLIENT_CERT_PATH", "data/smtp_ms_client_certificate.pem" +) # Send settings ALLOWED_EMAIL_DESTINATION_DOMAIN_LIST = ( diff --git a/app/model/mail/token_provider.py b/app/model/mail/token_provider.py index 18a7140ec..ed53de081 100644 --- a/app/model/mail/token_provider.py +++ b/app/model/mail/token_provider.py @@ -18,17 +18,23 @@ """ import abc +import base64 +import json +import os import time +import uuid import requests +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import padding, rsa from pydantic import BaseModel, ValidationError from requests.adapters import HTTPAdapter from urllib3 import Retry from app.config import ( + SMTP_MS_CLIENT_CERT_PATH, SMTP_MS_CLIENT_ID, SMTP_MS_CLIENT_SECRET, - SMTP_MS_REFRESH_TOKEN, SMTP_MS_TENANT_ID, ) @@ -61,6 +67,66 @@ class MicrosoftTokenProvider(TokenProvider): _access_token: str | None = None _token_expiry: float = 0.0 + @staticmethod + def _generate_client_assertion( + client_id: str, tenant_id: str, private_key_path: str + ) -> str: + """ + Generate JWT Client Assertion signed with the certificate's private key. + """ + try: + with open(private_key_path, "rb") as key_file: + private_key = serialization.load_pem_private_key( + key_file.read(), password=None + ) + except Exception as e: + raise ValueError(f"Failed to load private key from {private_key_path}: {e}") + + if not isinstance(private_key, rsa.RSAPrivateKey): + raise ValueError("Private key must be an RSA key") + + # JWT Claims + now = time.time() + # Header + header = { + "alg": "RS256", + "typ": "JWT", + "x5t": None, # x5t (Thumbprint) is optional but recommended if available. + } + # Payload + payload = { + "iss": client_id, + "sub": client_id, + "aud": f"https://login.microsoftonline.com/{tenant_id}/v2.0/token", + "jti": str(uuid.uuid4()), + "nbf": int(now), + "exp": int(now) + 300, # 5 minutes expiration + } + + # NOTE: Since pyjwt is not guaranteed to be in the environment, + # and installing new dependencies might not be desired, + # we construct the JWT manually using cryptography for signing. + # This is compliant with RFC 7515. + + def b64url_encode(data: bytes) -> str: + return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8") + + # 1. Create Signing Input + encoded_header = b64url_encode(json.dumps(header).encode("utf-8")) + encoded_payload = b64url_encode(json.dumps(payload).encode("utf-8")) + signing_input = f"{encoded_header}.{encoded_payload}".encode("utf-8") + + # 2. Sign + signature = private_key.sign( + signing_input, + padding.PKCS1v15(), + hashes.SHA256(), + ) + + # 3. Concatenate + encoded_signature = b64url_encode(signature) + return f"{encoded_header}.{encoded_payload}.{encoded_signature}" + def get_access_token(self) -> str: # Return cached token if valid (with 60 seconds safety buffer) if self._access_token and time.time() < self._token_expiry - 60: @@ -69,21 +135,44 @@ def get_access_token(self) -> str: tenant_id = SMTP_MS_TENANT_ID client_id = SMTP_MS_CLIENT_ID client_secret = SMTP_MS_CLIENT_SECRET - refresh_token = SMTP_MS_REFRESH_TOKEN + cert_path = SMTP_MS_CLIENT_CERT_PATH - if not all([tenant_id, client_id, client_secret, refresh_token]): + if tenant_id is None or client_id is None: raise ValueError("Missing Microsoft OAuth configuration") token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" data = { - "grant_type": "refresh_token", + "grant_type": "client_credentials", "client_id": client_id, - "client_secret": client_secret, - "refresh_token": refresh_token, "scope": "https://outlook.office365.com/.default", } + # Determine authentication method: Certificate (Client Assertion) or Secret + use_cert_auth = False + if cert_path and os.path.exists(cert_path): + use_cert_auth = True + elif not client_secret: + raise ValueError( + "Missing Microsoft OAuth configuration: Neither Client Secret nor Certificate is available." + ) + + if use_cert_auth: + try: + client_assertion = self._generate_client_assertion( + client_id, tenant_id, cert_path + ) + data["client_assertion_type"] = ( + "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ) + data["client_assertion"] = client_assertion + except Exception as e: + raise RuntimeError( + f"Failed to generate client assertion from certificate: {e}" + ) from e + else: + data["client_secret"] = client_secret + try: with requests.Session() as session: retries = Retry( diff --git a/pyproject.toml b/pyproject.toml index 13ec616a9..dc5ce561f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ "opentelemetry-exporter-otlp-proto-grpc>=1.33.1,<2.0.0", "requests>=2.32.3", "python-dotenv~=1.1.0", + "cryptography>=45.0.3", ] [tool.uv] diff --git a/tests/app/model/mail/test_token_provider.py b/tests/app/model/mail/test_token_provider.py index bf5eabcd5..c4ddd08b5 100644 --- a/tests/app/model/mail/test_token_provider.py +++ b/tests/app/model/mail/test_token_provider.py @@ -40,7 +40,6 @@ def test_get_access_token_success(self): patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), - patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), patch("requests.Session.post") as mock_post, ): mock_response = MagicMock() @@ -51,7 +50,6 @@ def test_get_access_token_success(self): "expires_in": 3599, "ext_expires_in": 3599, "access_token": "valid_access_token", - "refresh_token": "new_refresh_token", } mock_post.return_value = mock_response @@ -63,6 +61,13 @@ def test_get_access_token_success(self): # Assert assert token == "valid_access_token" + # Verify request data + mock_post.assert_called_once() + args, kwargs = mock_post.call_args + data = kwargs["data"] + assert data["grant_type"] == "client_credentials" + assert "refresh_token" not in data + def test_get_access_token_missing_config(self): """ Verify that missing config raises ValueError @@ -91,7 +96,6 @@ def test_get_access_token_api_error(self): patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), - patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), patch("requests.Session.post") as mock_post, ): mock_response = MagicMock() @@ -120,7 +124,6 @@ def test_get_access_token_validation_error(self): patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), - patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), patch("requests.Session.post") as mock_post, ): mock_response = MagicMock() @@ -148,7 +151,6 @@ def test_get_access_token_retry_logic(self): patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), - patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), patch("requests.Session.post") as mock_post, patch("requests.Session.mount") as mock_mount, ): @@ -213,7 +215,6 @@ def test_get_access_token_cache_renew(self): patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), - patch("app.model.mail.token_provider.SMTP_MS_REFRESH_TOKEN", "refresh-1"), patch("requests.Session.post") as mock_post, ): mock_response = MagicMock() diff --git a/tests/app/model/mail/test_token_provider_cert.py b/tests/app/model/mail/test_token_provider_cert.py new file mode 100644 index 000000000..d06b5dd92 --- /dev/null +++ b/tests/app/model/mail/test_token_provider_cert.py @@ -0,0 +1,197 @@ +""" +Copyright BOOSTRY Co., Ltd. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and +limitations under the License. + +SPDX-License-Identifier: Apache-2.0 +""" + +import base64 +import json +from unittest.mock import MagicMock, patch + +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from app.model.mail.token_provider import MicrosoftTokenProvider + + +class TestMicrosoftTokenProviderCert: + """ + Unit tests for app.model.mail.token_provider.MicrosoftTokenProvider + Focusing on Certificate Authentication (Client Assertion) + """ + + @pytest.fixture + def mock_private_key(self): + # Generate a temporary RSA private key for testing + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + pem = key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + return pem + + def test_get_access_token_with_certificate(self, mock_private_key, tmp_path): + """ + Verify that get_access_token uses Client Assertion when certificate is available + """ + # Ensure cache is empty + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 + + # Create dummy certificate file + cert_file = tmp_path / "test_cert.pem" + cert_file.write_bytes(mock_private_key) + cert_path_str = str(cert_file) + + # Arrange + with ( + patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), + # Client Secret is mocked to None to ensure fallback logic passes if using cert + # But the code shouldn't use it anyway if cert is present. + # Let's provide a dummy secret to ensure it's NOT used. + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), + patch( + "app.model.mail.token_provider.SMTP_MS_CLIENT_CERT_PATH", cert_path_str + ), + patch("requests.Session.post") as mock_post, + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "access_token": "valid_access_token_via_cert", + "expires_in": 3599, + "token_type": "Bearer", + } + mock_post.return_value = mock_response + + provider = MicrosoftTokenProvider() + + # Act + token = provider.get_access_token() + + # Assert + assert token == "valid_access_token_via_cert" + + # Verify request data + mock_post.assert_called_once() + args, kwargs = mock_post.call_args + data = kwargs["data"] + + # Key checks + assert ( + data["client_assertion_type"] + == "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ) + assert "client_assertion" in data + assert "client_secret" not in data # MUST NOT send secret if using cert + assert data["grant_type"] == "client_credentials" # Updated check + + # Verify JWT structure + client_assertion = data["client_assertion"] + parts = client_assertion.split(".") + assert len(parts) == 3 + + # Verify Header + header = json.loads( + base64.urlsafe_b64decode(parts[0] + "==").decode("utf-8") + ) + assert header["alg"] == "RS256" + assert header["typ"] == "JWT" + + # Verify Payload + payload = json.loads( + base64.urlsafe_b64decode(parts[1] + "==").decode("utf-8") + ) + assert payload["iss"] == "client-1" + assert payload["sub"] == "client-1" + assert ( + payload["aud"] + == "https://login.microsoftonline.com/tenant-1/v2.0/token" + ) + assert "jti" in payload + assert "exp" in payload + + def test_get_access_token_fallback_to_secret(self): + """ + Verify that get_access_token falls back to Client Secret if certificate file does not exist + """ + # Ensure cache is empty + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 + + # Arrange + with ( + patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), + patch( + "app.model.mail.token_provider.SMTP_MS_CLIENT_CERT_PATH", + "/non/existent/path.pem", + ), + patch("requests.Session.post") as mock_post, + ): + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = { + "access_token": "valid_access_token_via_secret", + "expires_in": 3599, + "token_type": "Bearer", + } + mock_post.return_value = mock_response + + provider = MicrosoftTokenProvider() + + # Act + token = provider.get_access_token() + + # Assert + assert token == "valid_access_token_via_secret" + + mock_post.assert_called_once() + args, kwargs = mock_post.call_args + data = kwargs["data"] + + assert "client_secret" in data + assert data["client_secret"] == "secret-1" + assert "client_assertion" not in data + assert data["grant_type"] == "client_credentials" + + def test_get_access_token_no_auth_available(self): + """ + Verify that ValueError is raised if neither secret nor certificate is available + """ + # Ensure cache is empty + MicrosoftTokenProvider._access_token = None + MicrosoftTokenProvider._token_expiry = 0.0 + + with ( + patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), + patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", None), + patch( + "app.model.mail.token_provider.SMTP_MS_CLIENT_CERT_PATH", + "/non/existent/path.pem", + ), + ): + provider = MicrosoftTokenProvider() + + with pytest.raises( + ValueError, match="Neither Client Secret nor Certificate is available" + ): + provider.get_access_token() diff --git a/uv.lock b/uv.lock index c95c741b3..a38bf1f49 100644 --- a/uv.lock +++ b/uv.lock @@ -809,6 +809,7 @@ dependencies = [ { name = "asyncpg" }, { name = "boto3" }, { name = "coincurve" }, + { name = "cryptography" }, { name = "eth-keyfile" }, { name = "eth-utils" }, { name = "fastapi" }, @@ -876,6 +877,7 @@ requires-dist = [ { name = "asyncpg", specifier = "~=0.30.0" }, { name = "boto3", specifier = "~=1.37.0" }, { name = "coincurve", specifier = "~=21.0.0" }, + { name = "cryptography", specifier = ">=45.0.3" }, { name = "eth-keyfile", specifier = "~=0.9.1" }, { name = "eth-utils", specifier = "~=5.2.0" }, { name = "fastapi", specifier = "~=0.116.1" }, From 6aa354e776474230385aa03cdf256740399f6925 Mon Sep 17 00:00:00 2001 From: Yosuke Otosu Date: Wed, 17 Dec 2025 16:15:53 +0900 Subject: [PATCH 6/7] Remove Client Certificate Path from configuration and update Microsoft OAuth handling --- ENV_LIST.md | 4 +- app/config.py | 4 - app/model/mail/token_provider.py | 31 +-- .../model/mail/test_token_provider_cert.py | 197 ------------------ 4 files changed, 3 insertions(+), 233 deletions(-) delete mode 100644 tests/app/model/mail/test_token_provider_cert.py diff --git a/ENV_LIST.md b/ENV_LIST.md index 7066c521d..651c3b04b 100644 --- a/ENV_LIST.md +++ b/ENV_LIST.md @@ -108,7 +108,6 @@ SMTP XOAUTH2 (Microsoft) | SMTP_MS_TENANT_ID | True | Microsoft Entra ID (Azure AD) Tenant ID | | -- | | SMTP_MS_CLIENT_ID | True | Microsoft Entra ID (Azure AD) Client ID | | -- | | SMTP_MS_CLIENT_SECRET | True* | Microsoft Entra ID (Azure AD) Client Secret (*Required if Client Certificate is unused) | | -- | -| SMTP_MS_CLIENT_CERT_PATH | False* | Path to Client Certificate Private Key (PEM) (*Alternative to Client Secret) | | data/smtp_ms_client_certificate.pem | Send settings @@ -146,7 +145,6 @@ Send settings | **MICROSOFT OAUTH** | | | | | | `SMTP_MS_TENANT_ID` | - | Required | - | | | `SMTP_MS_CLIENT_ID` | - | Required | - | | -| `SMTP_MS_CLIENT_SECRET` | - | Required* | - | *Required if Client Certificate is unused | -| `SMTP_MS_CLIENT_CERT_PATH` | - | Optional* | - | Default: `data/smtp_ms_client_certificate.pem` (*Alternative to Client Secret) | +| `SMTP_MS_CLIENT_SECRET` | - | Required | - | | | **AMAZON SES** | | | | | | `AWS_SES_REGION_NAME` | - | - | Required | | diff --git a/app/config.py b/app/config.py index 14e42bde0..098167517 100644 --- a/app/config.py +++ b/app/config.py @@ -371,10 +371,6 @@ SMTP_MS_CLIENT_ID: str | None = os.environ.get("SMTP_MS_CLIENT_ID") # - Client Secret: Microsoft Entra ID (Azure AD) Client Secret SMTP_MS_CLIENT_SECRET: str | None = os.environ.get("SMTP_MS_CLIENT_SECRET") -# - Client Certificate Path: Microsoft Entra ID (Azure AD) Client Certificate Path -SMTP_MS_CLIENT_CERT_PATH = os.environ.get( - "SMTP_MS_CLIENT_CERT_PATH", "data/smtp_ms_client_certificate.pem" -) # Send settings ALLOWED_EMAIL_DESTINATION_DOMAIN_LIST = ( diff --git a/app/model/mail/token_provider.py b/app/model/mail/token_provider.py index ed53de081..3e25f6f2a 100644 --- a/app/model/mail/token_provider.py +++ b/app/model/mail/token_provider.py @@ -20,7 +20,6 @@ import abc import base64 import json -import os import time import uuid @@ -32,7 +31,6 @@ from urllib3 import Retry from app.config import ( - SMTP_MS_CLIENT_CERT_PATH, SMTP_MS_CLIENT_ID, SMTP_MS_CLIENT_SECRET, SMTP_MS_TENANT_ID, @@ -135,9 +133,8 @@ def get_access_token(self) -> str: tenant_id = SMTP_MS_TENANT_ID client_id = SMTP_MS_CLIENT_ID client_secret = SMTP_MS_CLIENT_SECRET - cert_path = SMTP_MS_CLIENT_CERT_PATH - if tenant_id is None or client_id is None: + if tenant_id is None or client_id is None or (client_secret is None): raise ValueError("Missing Microsoft OAuth configuration") token_url = f"https://login.microsoftonline.com/{tenant_id}/oauth2/v2.0/token" @@ -145,34 +142,10 @@ def get_access_token(self) -> str: data = { "grant_type": "client_credentials", "client_id": client_id, + "client_secret": client_secret, "scope": "https://outlook.office365.com/.default", } - # Determine authentication method: Certificate (Client Assertion) or Secret - use_cert_auth = False - if cert_path and os.path.exists(cert_path): - use_cert_auth = True - elif not client_secret: - raise ValueError( - "Missing Microsoft OAuth configuration: Neither Client Secret nor Certificate is available." - ) - - if use_cert_auth: - try: - client_assertion = self._generate_client_assertion( - client_id, tenant_id, cert_path - ) - data["client_assertion_type"] = ( - "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" - ) - data["client_assertion"] = client_assertion - except Exception as e: - raise RuntimeError( - f"Failed to generate client assertion from certificate: {e}" - ) from e - else: - data["client_secret"] = client_secret - try: with requests.Session() as session: retries = Retry( diff --git a/tests/app/model/mail/test_token_provider_cert.py b/tests/app/model/mail/test_token_provider_cert.py deleted file mode 100644 index d06b5dd92..000000000 --- a/tests/app/model/mail/test_token_provider_cert.py +++ /dev/null @@ -1,197 +0,0 @@ -""" -Copyright BOOSTRY Co., Ltd. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. - -You may obtain a copy of the License at -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - -See the License for the specific language governing permissions and -limitations under the License. - -SPDX-License-Identifier: Apache-2.0 -""" - -import base64 -import json -from unittest.mock import MagicMock, patch - -import pytest -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa - -from app.model.mail.token_provider import MicrosoftTokenProvider - - -class TestMicrosoftTokenProviderCert: - """ - Unit tests for app.model.mail.token_provider.MicrosoftTokenProvider - Focusing on Certificate Authentication (Client Assertion) - """ - - @pytest.fixture - def mock_private_key(self): - # Generate a temporary RSA private key for testing - key = rsa.generate_private_key(public_exponent=65537, key_size=2048) - pem = key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.PKCS8, - encryption_algorithm=serialization.NoEncryption(), - ) - return pem - - def test_get_access_token_with_certificate(self, mock_private_key, tmp_path): - """ - Verify that get_access_token uses Client Assertion when certificate is available - """ - # Ensure cache is empty - MicrosoftTokenProvider._access_token = None - MicrosoftTokenProvider._token_expiry = 0.0 - - # Create dummy certificate file - cert_file = tmp_path / "test_cert.pem" - cert_file.write_bytes(mock_private_key) - cert_path_str = str(cert_file) - - # Arrange - with ( - patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), - patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), - # Client Secret is mocked to None to ensure fallback logic passes if using cert - # But the code shouldn't use it anyway if cert is present. - # Let's provide a dummy secret to ensure it's NOT used. - patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), - patch( - "app.model.mail.token_provider.SMTP_MS_CLIENT_CERT_PATH", cert_path_str - ), - patch("requests.Session.post") as mock_post, - ): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "access_token": "valid_access_token_via_cert", - "expires_in": 3599, - "token_type": "Bearer", - } - mock_post.return_value = mock_response - - provider = MicrosoftTokenProvider() - - # Act - token = provider.get_access_token() - - # Assert - assert token == "valid_access_token_via_cert" - - # Verify request data - mock_post.assert_called_once() - args, kwargs = mock_post.call_args - data = kwargs["data"] - - # Key checks - assert ( - data["client_assertion_type"] - == "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" - ) - assert "client_assertion" in data - assert "client_secret" not in data # MUST NOT send secret if using cert - assert data["grant_type"] == "client_credentials" # Updated check - - # Verify JWT structure - client_assertion = data["client_assertion"] - parts = client_assertion.split(".") - assert len(parts) == 3 - - # Verify Header - header = json.loads( - base64.urlsafe_b64decode(parts[0] + "==").decode("utf-8") - ) - assert header["alg"] == "RS256" - assert header["typ"] == "JWT" - - # Verify Payload - payload = json.loads( - base64.urlsafe_b64decode(parts[1] + "==").decode("utf-8") - ) - assert payload["iss"] == "client-1" - assert payload["sub"] == "client-1" - assert ( - payload["aud"] - == "https://login.microsoftonline.com/tenant-1/v2.0/token" - ) - assert "jti" in payload - assert "exp" in payload - - def test_get_access_token_fallback_to_secret(self): - """ - Verify that get_access_token falls back to Client Secret if certificate file does not exist - """ - # Ensure cache is empty - MicrosoftTokenProvider._access_token = None - MicrosoftTokenProvider._token_expiry = 0.0 - - # Arrange - with ( - patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), - patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), - patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", "secret-1"), - patch( - "app.model.mail.token_provider.SMTP_MS_CLIENT_CERT_PATH", - "/non/existent/path.pem", - ), - patch("requests.Session.post") as mock_post, - ): - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "access_token": "valid_access_token_via_secret", - "expires_in": 3599, - "token_type": "Bearer", - } - mock_post.return_value = mock_response - - provider = MicrosoftTokenProvider() - - # Act - token = provider.get_access_token() - - # Assert - assert token == "valid_access_token_via_secret" - - mock_post.assert_called_once() - args, kwargs = mock_post.call_args - data = kwargs["data"] - - assert "client_secret" in data - assert data["client_secret"] == "secret-1" - assert "client_assertion" not in data - assert data["grant_type"] == "client_credentials" - - def test_get_access_token_no_auth_available(self): - """ - Verify that ValueError is raised if neither secret nor certificate is available - """ - # Ensure cache is empty - MicrosoftTokenProvider._access_token = None - MicrosoftTokenProvider._token_expiry = 0.0 - - with ( - patch("app.model.mail.token_provider.SMTP_MS_TENANT_ID", "tenant-1"), - patch("app.model.mail.token_provider.SMTP_MS_CLIENT_ID", "client-1"), - patch("app.model.mail.token_provider.SMTP_MS_CLIENT_SECRET", None), - patch( - "app.model.mail.token_provider.SMTP_MS_CLIENT_CERT_PATH", - "/non/existent/path.pem", - ), - ): - provider = MicrosoftTokenProvider() - - with pytest.raises( - ValueError, match="Neither Client Secret nor Certificate is available" - ): - provider.get_access_token() From e2f544c9d9903b7b60f0ab4408e8601e9d1e9ce4 Mon Sep 17 00:00:00 2001 From: Yosuke Otosu Date: Wed, 17 Dec 2025 16:19:55 +0900 Subject: [PATCH 7/7] [no ci] remove unused method --- app/model/mail/token_provider.py | 60 -------------------------------- 1 file changed, 60 deletions(-) diff --git a/app/model/mail/token_provider.py b/app/model/mail/token_provider.py index 3e25f6f2a..35ba916b5 100644 --- a/app/model/mail/token_provider.py +++ b/app/model/mail/token_provider.py @@ -65,66 +65,6 @@ class MicrosoftTokenProvider(TokenProvider): _access_token: str | None = None _token_expiry: float = 0.0 - @staticmethod - def _generate_client_assertion( - client_id: str, tenant_id: str, private_key_path: str - ) -> str: - """ - Generate JWT Client Assertion signed with the certificate's private key. - """ - try: - with open(private_key_path, "rb") as key_file: - private_key = serialization.load_pem_private_key( - key_file.read(), password=None - ) - except Exception as e: - raise ValueError(f"Failed to load private key from {private_key_path}: {e}") - - if not isinstance(private_key, rsa.RSAPrivateKey): - raise ValueError("Private key must be an RSA key") - - # JWT Claims - now = time.time() - # Header - header = { - "alg": "RS256", - "typ": "JWT", - "x5t": None, # x5t (Thumbprint) is optional but recommended if available. - } - # Payload - payload = { - "iss": client_id, - "sub": client_id, - "aud": f"https://login.microsoftonline.com/{tenant_id}/v2.0/token", - "jti": str(uuid.uuid4()), - "nbf": int(now), - "exp": int(now) + 300, # 5 minutes expiration - } - - # NOTE: Since pyjwt is not guaranteed to be in the environment, - # and installing new dependencies might not be desired, - # we construct the JWT manually using cryptography for signing. - # This is compliant with RFC 7515. - - def b64url_encode(data: bytes) -> str: - return base64.urlsafe_b64encode(data).rstrip(b"=").decode("utf-8") - - # 1. Create Signing Input - encoded_header = b64url_encode(json.dumps(header).encode("utf-8")) - encoded_payload = b64url_encode(json.dumps(payload).encode("utf-8")) - signing_input = f"{encoded_header}.{encoded_payload}".encode("utf-8") - - # 2. Sign - signature = private_key.sign( - signing_input, - padding.PKCS1v15(), - hashes.SHA256(), - ) - - # 3. Concatenate - encoded_signature = b64url_encode(signature) - return f"{encoded_header}.{encoded_payload}.{encoded_signature}" - def get_access_token(self) -> str: # Return cached token if valid (with 60 seconds safety buffer) if self._access_token and time.time() < self._token_expiry - 60: