feat(warehouse): add Shipmail source

frontend/src/queries/schema/schema-general.ts

@@ -7141,6 +7141,7 @@ export const externalDataSources = [
     'Ebay',
     'Commercetools',
     'LightspeedRetail',
+    'Shipmail',
     'ShipStation',
     'ConstantContact',
     'Mailgun',

posthog/schema_enums.py

@@ -1363,6 +1363,7 @@ class ExternalDataSourceType(StrEnum):
     EBAY = "Ebay"
     COMMERCETOOLS = "Commercetools"
     LIGHTSPEED_RETAIL = "LightspeedRetail"
+    SHIPMAIL = "Shipmail"
     SHIP_STATION = "ShipStation"
     CONSTANT_CONTACT = "ConstantContact"
     MAILGUN = "Mailgun"

products/warehouse_sources/backend/migrations/max_migration.txt

@@ -1 +1 @@
-0115_scaffold_four_requested_sources
+0116_shipmail_source

products/warehouse_sources/backend/temporal/data_imports/sources/SOURCES.md

@@ -536,6 +536,7 @@ the row lists both.
 | sentinelone                      | HTTP                        | requests                                                        | ✅                          |
 | sentry                           | HTTP                        | requests + `rest_source.RESTClient`                             | ✅                          |
 | servicenow                       | HTTP                        | requests                                                        | ✅                          |
+| shipmail                         | HTTP                        | requests + `rest_source.RESTClient`                             | ✅                          |
 | shippo                           | HTTP                        | requests                                                        | ✅                          |
 | shipstation                      | HTTP                        | requests                                                        | ✅                          |
 | shopify                          | HTTP                        | requests                                                        | ✅                          |

products/warehouse_sources/backend/temporal/data_imports/sources/_load_all.py

@@ -1022,6 +1022,7 @@
 from .sftp.source import SFTPSource
 from .sharepoint.source import SharePointSource
 from .sharetribe.source import SharetribeSource
+from .shipmail.source import ShipmailSource
 from .shippo.source import ShippoSource
 from .shipstation.source import ShipStationSource
 from .shopify.source import ShopifySource

products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/shipmail.py

@@ -0,0 +1,9 @@
+# This file is automatically generated from `SourceRegistry.get_all_sources()`
+# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+
+
+@config.config
+class ShipmailSourceConfig(config.Config):
+    api_key: str

products/warehouse_sources/backend/temporal/data_imports/sources/shipmail/canonical_descriptions.py

@@ -0,0 +1,79 @@
+"""Canonical descriptions sourced from the Shipmail API documentation."""
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
+    CanonicalDescriptions,
+)
+
+CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
+    "messages": {
+        "description": "Analytics-safe message records in stable update order, without message content or sensitive headers.",
+        "docs_url": "https://shipmail.to/docs/api/messages",
+        "columns": {
+            "object": "The API object type. Always `message_analytics`.",
+            "id": "Unique identifier for the message.",
+            "mailbox_id": "Identifier of the mailbox associated with the message.",
+            "thread_id": "Identifier of the conversation thread, when available.",
+            "client_reference": "Caller-provided reference used to associate the message with an external record.",
+            "direction": "Whether the message was inbound or outbound.",
+            "contact_addresses": "Email addresses of contacts involved in the message.",
+            "recipient_count": "Number of recipients on the message.",
+            "attachment_count": "Number of attachments on the message.",
+            "source": "System or integration that created the message.",
+            "mode": "Whether the message belongs to live or sandbox data.",
+            "status": "Current message status.",
+            "scheduled_at": "Time at which the message was scheduled to be sent, when applicable.",
+            "created_at": "Time at which the message was created.",
+            "updated_at": "Time at which the message was last updated.",
+        },
+    },
+    "mailboxes": {
+        "description": "Mailboxes belonging to the authenticated Shipmail organization.",
+        "docs_url": "https://shipmail.to/docs/api/mailboxes",
+        "columns": {
+            "object": "The API object type. Always `mailbox`.",
+            "id": "Unique identifier for the mailbox.",
+            "domain_id": "Identifier of the domain that hosts the mailbox.",
+            "address": "Email address of the mailbox.",
+            "display_name": "Display name used for the mailbox.",
+            "suspended_at": "Time at which the mailbox was suspended, when applicable.",
+            "suspension_reasons": "Reasons the mailbox is suspended.",
+            "spam_filter_threshold": "Spam score threshold applied to inbound messages.",
+            "auto_reply": "Automatic reply configuration for the mailbox.",
+            "created_at": "Time at which the mailbox was created.",
+            "updated_at": "Time at which the mailbox was last updated.",
+        },
+    },
+    "domains": {
+        "description": "Sending and receiving domains configured in the authenticated Shipmail organization.",
+        "docs_url": "https://shipmail.to/docs/api/domains",
+        "columns": {
+            "object": "The API object type. Always `domain`.",
+            "id": "Unique identifier for the domain.",
+            "name": "Domain name.",
+            "status": "Current domain status.",
+            "managed_by": "Whether DNS is managed externally or by Shipmail.",
+            "dns_provider": "Detected DNS provider, when available.",
+            "mx_verified": "Whether the MX record is verified.",
+            "spf_verified": "Whether the SPF record is verified.",
+            "dkim_verified": "Whether the DKIM record is verified.",
+            "dmarc_verified": "Whether the DMARC record is verified.",
+            "dmarc_managed_externally": "Whether DMARC is managed outside Shipmail.",
+            "outbound_verified": "Whether outbound sending is verified.",
+            "catch_all_mailbox_id": "Identifier of the catch-all mailbox, when configured.",
+            "verified_at": "Time at which the domain completed verification.",
+            "created_at": "Time at which the domain was created.",
+            "updated_at": "Time at which the domain was last updated.",
+            "registration": "Registration details for domains managed by Shipmail, when available.",
+        },
+    },
+    "suppressions": {
+        "description": "Email addresses suppressed from outbound delivery.",
+        "docs_url": "https://shipmail.to/docs/api/suppressions",
+        "columns": {
+            "object": "The API object type. Always `suppression`.",
+            "email_address": "Suppressed email address.",
+            "reason": "Reason the address was suppressed.",
+            "created_at": "Time at which the suppression was created.",
+        },
+    },
+}

products/warehouse_sources/backend/temporal/data_imports/sources/shipmail/settings.py

@@ -0,0 +1,53 @@
+from dataclasses import dataclass, field
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
+from products.warehouse_sources.backend.types import IncrementalField
+
+
+@dataclass(frozen=True)
+class ShipmailEndpointConfig:
+    name: str
+    path: str
+    primary_keys: list[str]
+    required_scope: str
+    partition_key: str
+    incremental_fields: list[IncrementalField] = field(default_factory=list)
+
+
+SHIPMAIL_ENDPOINTS: dict[str, ShipmailEndpointConfig] = {
+    "messages": ShipmailEndpointConfig(
+        name="messages",
+        path="/messages/analytics",
+        primary_keys=["id"],
+        required_scope="messages:read",
+        partition_key="created_at",
+        incremental_fields=[incremental_field("updated_at")],
+    ),
+    "mailboxes": ShipmailEndpointConfig(
+        name="mailboxes",
+        path="/mailboxes",
+        primary_keys=["id"],
+        required_scope="mailboxes:read",
+        partition_key="created_at",
+    ),
+    "domains": ShipmailEndpointConfig(
+        name="domains",
+        path="/domains",
+        primary_keys=["id"],
+        required_scope="domains:read",
+        partition_key="created_at",
+    ),
+    "suppressions": ShipmailEndpointConfig(
+        name="suppressions",
+        path="/suppressions",
+        primary_keys=["email_address"],
+        required_scope="suppressions:read",
+        partition_key="created_at",
+    ),
+}
+
+ENDPOINTS = tuple(SHIPMAIL_ENDPOINTS.keys())
+
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+    name: config.incremental_fields for name, config in SHIPMAIL_ENDPOINTS.items() if config.incremental_fields
+}

products/warehouse_sources/backend/temporal/data_imports/sources/shipmail/shipmail.py

@@ -0,0 +1,130 @@
+import dataclasses
+from datetime import datetime
+from typing import Any, Optional
+
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
+    RESTAPIConfig,
+    rest_api_resource,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+    JSONResponseCursorPaginator,
+)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
+from products.warehouse_sources.backend.temporal.data_imports.sources.shipmail.settings import SHIPMAIL_ENDPOINTS
+
+SHIPMAIL_BASE_URL = "https://shipmail.to/api/v1"
+
+
+@dataclasses.dataclass
+class ShipmailResumeConfig:
+    cursor: str | None = None
+
+
+def _incremental_value(value: Any) -> str:
+    if isinstance(value, datetime):
+        return value.isoformat()
+    return str(value)
+
+
+def _initial_paginator_state(
+    resumable_source_manager: ResumableSourceManager[ShipmailResumeConfig],
+) -> dict[str, str] | None:
+    if not resumable_source_manager.can_resume():
+        return None
+    resume = resumable_source_manager.load_state()
+    if resume is None or resume.cursor is None:
+        return None
+    return {"cursor": resume.cursor}
+
+
+def shipmail_source(
+    api_key: str,
+    endpoint: str,
+    team_id: int,
+    job_id: str,
+    resumable_source_manager: ResumableSourceManager[ShipmailResumeConfig],
+    should_use_incremental_field: bool = False,
+    db_incremental_field_last_value: Optional[Any] = None,
+    incremental_field: Optional[str] = None,
+) -> SourceResponse:
+    endpoint_config = SHIPMAIL_ENDPOINTS[endpoint]
+    params: dict[str, Any] = {"limit": 100}
+
+    if (
+        endpoint == "messages"
+        and should_use_incremental_field
+        and incremental_field == "updated_at"
+        and db_incremental_field_last_value is not None
+    ):
+        params["updated_after"] = _incremental_value(db_incremental_field_last_value)
+
+    rest_config: RESTAPIConfig = {
+        "client": {
+            "base_url": SHIPMAIL_BASE_URL,
+            "headers": {"Accept": "application/json"},
+            "auth": {"type": "bearer", "token": api_key},
+            "paginator": JSONResponseCursorPaginator(
+                cursor_path="pagination.next_cursor",
+                cursor_param="cursor",
+            ),
+            "session": make_tracked_session(redact_values=(api_key,), capture=False),
+        },
+        "resource_defaults": {},
+        "resources": [
+            {
+                "name": endpoint,
+                "endpoint": {
+                    "path": endpoint_config.path,
+                    "params": params,
+                    "data_selector": "data",
+                    "data_selector_required": True,
+                },
+            }
+        ],
+    }
+
+    def save_checkpoint(state: dict[str, Any] | None) -> None:
+        if state and state.get("cursor"):
+            resumable_source_manager.save_state(ShipmailResumeConfig(cursor=str(state["cursor"])))
+
+    resource = rest_api_resource(
+        rest_config,
+        team_id,
+        job_id,
+        db_incremental_field_last_value,
+        resume_hook=save_checkpoint,
+        initial_paginator_state=_initial_paginator_state(resumable_source_manager),
+    )
+
+    return SourceResponse(
+        name=endpoint,
+        items=lambda: resource,
+        primary_keys=endpoint_config.primary_keys,
+        partition_count=1,
+        partition_size=1,
+        partition_mode="datetime",
+        partition_format="month",
+        partition_keys=[endpoint_config.partition_key],
+        sort_mode="asc" if endpoint == "messages" else "desc",
+    )
+
+
+def get_capabilities(api_key: str) -> tuple[int | None, set[str]]:
+    try:
+        session = make_tracked_session(redact_values=(api_key,), capture=False)
+        response = session.get(
+            f"{SHIPMAIL_BASE_URL}/capabilities",
+            headers={"Authorization": f"Bearer {api_key}", "Accept": "application/json"},
+            timeout=10,
+        )
+        if response.status_code != 200:
+            return response.status_code, set()
+        body = response.json()
+        scopes = body.get("scopes")
+        if not isinstance(scopes, list) or not all(isinstance(scope, str) for scope in scopes):
+            return response.status_code, set()
+        return response.status_code, {scope for scope in scopes if isinstance(scope, str)}
+    except Exception:  # noqa: BLE001 - credential probes must return a validation result, never raise
+        return None, set()