diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f8b920b..ee60a2d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -117,10 +117,17 @@ jobs: # untouched `pip install ruff` below, which isn't flagged only because it # isn't part of this PR's diff). run: cd sdk/python && pip install -e ".[dev,connectors]" # NOSONAR python:S4830 python:S5445 + - name: Install connector/pack deps + # google-auth is only needed by the Google Sheet connector, so it lives in + # that connector's requirements rather than the SDK's extras. + run: pip install -r connectors/google_sheet/requirements.txt # NOSONAR python:S4830 python:S5445 - name: Lint - run: cd sdk/python && pip install ruff && ruff check . - - name: Test + # Same ephemeral-CI-runner rationale as the installs above. + run: pip install ruff && ruff check sdk/python connectors packs # NOSONAR python:S4830 python:S5445 + - name: Test SDK run: cd sdk/python && pytest tests/ -v + - name: Test connectors & packs + run: pytest connectors packs -q typescript: name: TypeScript SDK Tests diff --git a/CLAUDE.md b/CLAUDE.md index 5f880ca..ef94c21 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -49,11 +49,21 @@ magic-claw/ │ ├── knowledge/ # Shared knowledge hub + semantic search │ ├── webhook/ # Event-driven webhook delivery (at-least-once) │ └── audit/ # Audit log -├── sdk/python/ # Python SDK +├── sdk/python/ # Python SDK (+ magic_ai_sdk/connectors/ = Connector Framework) ├── sdk/go/ # Go SDK +├── connectors/ # Platform integrations, run as external workers +│ ├── zalo/ # Zalo OA — send/receive messages +│ └── google_sheet/ # Google Sheets — orders & customers +├── packs/ # Vertical business workflows +│ └── ecomops/ # Customer service for VN online shops └── examples/ # Example workers ``` +**connectors vs packs:** a connector answers "which platform" (API calls + +Common Schema mapping, no business logic); a pack answers "which business +process" (workflow logic, no hard-coded platform). Both run as external Python +workers — neither lives in the Go core. + ### Module Tiers | Tier | Modules | Purpose | @@ -112,6 +122,11 @@ cd core && MAGIC_POSTGRES_URL="postgres://user:pass@localhost/magic" ./magic ser # Python SDK cd sdk/python && pip install -e ".[dev]" cd sdk/python && pytest + +# Connectors + packs (run from repo root — root pytest.ini supplies asyncio_mode) +pip install -e 'sdk/python[dev,connectors]' -r connectors/google_sheet/requirements.txt +pytest connectors packs +ruff check sdk/python connectors packs ``` ## Key Design Decisions diff --git a/connectors/google_sheet/connector.py b/connectors/google_sheet/connector.py index 1c9c031..8876cfd 100644 --- a/connectors/google_sheet/connector.py +++ b/connectors/google_sheet/connector.py @@ -23,7 +23,6 @@ from google.auth.transport import Request as GoogleAuthRequestBase from google.auth.transport import Response as GoogleAuthResponseBase from google.oauth2 import service_account - from magic_ai_sdk.connectors.base import BaseConnector from magic_ai_sdk.connectors.errors import AuthError, PermanentError, RateLimitError, TransientError, with_retry diff --git a/connectors/google_sheet/tests/test_connector.py b/connectors/google_sheet/tests/test_connector.py index 878ef69..0c23dc5 100644 --- a/connectors/google_sheet/tests/test_connector.py +++ b/connectors/google_sheet/tests/test_connector.py @@ -6,9 +6,9 @@ import pytest import respx from google.auth.exceptions import RefreshError +from magic_ai_sdk.connectors.errors import AuthError, PermanentError, RateLimitError, TransientError from google_sheet.connector import API_BASE, GoogleSheetConnector -from magic_ai_sdk.connectors.errors import AuthError, PermanentError, RateLimitError, TransientError CONFIG = {"spreadsheet_id": "sheet-123", "service_account_info": {"type": "service_account"}} diff --git a/connectors/zalo/connector.py b/connectors/zalo/connector.py index b96ae2c..26488ab 100644 --- a/connectors/zalo/connector.py +++ b/connectors/zalo/connector.py @@ -25,7 +25,6 @@ from typing import Any, Callable, Mapping import httpx - from magic_ai_sdk.connectors.base import BaseConnector from magic_ai_sdk.connectors.errors import AuthError, PermanentError, RateLimitError, TransientError, with_retry diff --git a/connectors/zalo/tests/test_connector.py b/connectors/zalo/tests/test_connector.py index 07c2f52..e479287 100644 --- a/connectors/zalo/tests/test_connector.py +++ b/connectors/zalo/tests/test_connector.py @@ -7,8 +7,8 @@ import httpx import pytest import respx - from magic_ai_sdk.connectors.errors import AuthError, PermanentError, RateLimitError, TransientError + from zalo.connector import REFRESH_TOKEN_URL, SEND_MESSAGE_URL, ZaloConnector BASE_CONFIG = { diff --git a/packs/README.md b/packs/README.md new file mode 100644 index 0000000..04a1610 --- /dev/null +++ b/packs/README.md @@ -0,0 +1,26 @@ +# MagiC Solution Packs + +Pack = workflow nghiệp vụ cho một ngành cụ thể, xây trên Connector Framework và +core MagiC. + +Khác biệt với `connectors/`: + +- **Connector** trả lời câu hỏi *"kết nối tới nền tảng nào"* (Zalo, Google Sheet, + Nhanh.vn) — chỉ gọi API và map dữ liệu về Common Schema. +- **Pack** trả lời câu hỏi *"làm nghiệp vụ gì"* (trả lời khách, tra đơn, kiểm tra + guardrail) — chứa logic nghiệp vụ, và **không** được hard-code nền tảng nào. + +``` +packs/ +└── ecomops/ # Chăm sóc khách hàng cho shop bán hàng online +``` + +## Nguyên tắc + +1. Pack nhận mọi phụ thuộc bên ngoài qua **dependency injection** (classifier, + tra đơn, model soạn tin), không tự gọi connector cụ thể — nhờ vậy cùng một + workflow chạy được trên Zalo, Facebook, hay test harness. +2. Dữ liệu ra/vào dùng **Common Data Schema** (`sdk/python/magic_ai_sdk/connectors/schema.py`). +3. Mọi câu trả lời gửi tới khách phải đi qua **guardrail** trước. +4. Gọi LLM qua **LLM gateway của MagiC** (`POST /api/v1/llm/chat`) thay vì SDK của + nhà cung cấp, để chi phí token được Cost Controller ghi nhận cùng mọi workload khác. diff --git a/packs/conftest.py b/packs/conftest.py new file mode 100644 index 0000000..d9a2649 --- /dev/null +++ b/packs/conftest.py @@ -0,0 +1,8 @@ +"""Makes each pack importable (e.g. `import ecomops`) during tests without a +full pip install — mirrors connectors/conftest.py. +""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) diff --git a/packs/ecomops/README.md b/packs/ecomops/README.md new file mode 100644 index 0000000..83814e6 --- /dev/null +++ b/packs/ecomops/README.md @@ -0,0 +1,124 @@ +# EcomOps Pack + +Workflow chăm sóc khách hàng cho shop bán hàng online Việt Nam: + +``` +Nhận tin → Phân loại ý định → Trích mã đơn/SĐT → Tra đơn hàng + → Soạn câu trả lời → Kiểm tra Guardrail → Gửi hoặc chuyển nhân viên +``` + +Pack này chạy như **worker độc lập**, không nằm trong core Go. Mọi phụ thuộc bên +ngoài (classifier, tra đơn, model soạn tin) đều được **inject vào**, nên workflow +tự nó không gọi mạng và không biết gì về Zalo hay Google Sheet — cùng một workflow +chạy được trên bất kỳ kênh nào. + +## 1. Cài đặt + +Chạy từ thư mục gốc repo (`magic/`): + +```bash +pip install -e 'sdk/python[connectors]' +pip install -r packs/ecomops/requirements.txt +pytest packs/ecomops/tests +``` + +## 2. Các thành phần + +| Module | Vai trò | +|--------|---------| +| `text.py` | Chuẩn hoá tiếng Việt (bỏ dấu) để so khớp không phụ thuộc cách gõ | +| `intents.py` | Phân loại ý định: `RuleBasedIntentClassifier`, `LLMIntentClassifier`, `HybridIntentClassifier` | +| `extraction.py` | Trích mã đơn hàng & số điện thoại từ tin nhắn | +| `prompts.py` | Thư viện prompt tiếng Việt theo từng ý định | +| `guardrails.py` | Chặn câu trả lời rủi ro trước khi gửi cho khách | +| `workflow.py` | `EcomOpsWorkflow` — pipeline nối tất cả lại | + +### Ý định được hỗ trợ + +`order_status` (tình trạng đơn) · `shipping_fee` (phí ship) · `return_exchange` +(đổi trả) · `complaint` (khiếu nại) · `product_info` (thông tin sản phẩm) · +`greeting` · `other` + +Classifier **không phụ thuộc dấu tiếng Việt** — "đơn hàng của em đâu rồi" và +"don hang cua em dau roi" cho cùng kết quả, vì khách hay gõ không dấu. Nó cũng +cho phép chèn 1 từ đệm giữa các từ khoá, nên "khi nào **em** nhận được hàng" vẫn +khớp từ khoá "khi nao nhan". + +`HybridIntentClassifier` chạy rule trước, **chỉ gọi LLM khi rule không chắc** — +tiết kiệm chi phí vì phần lớn tin nhắn của khách rất ngắn và lặp lại. + +### Guardrail (quan trọng nhất) + +Câu trả lời do LLM soạn **không mặc nhiên an toàn** để gửi từ tài khoản chính +thức của shop. Các rule mặc định chặn: + +| Rule | Chặn điều gì | +|------|--------------| +| `empty_draft` | Câu trả lời rỗng | +| `leaked_contact` | SĐT/email không phải của khách hoặc của shop (lộ thông tin khách khác) | +| `delivery_promise` | Hứa chắc chắn ngày giao ("chắc chắn giao ngày mai") | +| `compensation_promise` | Tự hứa hoàn tiền/voucher/giảm giá khi chưa được duyệt | +| `internal_leak` | Lộ system prompt, API key, ghi chú nội bộ | + +Guardrail cũng chạy trên text đã bỏ dấu, nên **không thể lách bằng cách gõ không dấu**. + +Bất kỳ vi phạm mức `BLOCK` nào → workflow **không gửi draft đó**, mà trả về câu +trả lời an toàn và đánh dấu `action="handoff"` để nhân viên tiếp nhận. + +## 3. Dùng nhanh + +```python +import asyncio +from ecomops import EcomOpsWorkflow, RuleBasedIntentClassifier + +async def my_order_lookup(order_code, phone): + return {"id": order_code, "status": "shipping", "total_amount": 250000} + +async def my_drafter(messages): + ... # gọi LLM, hoặc dùng ecomops.workflow.LLMDrafter + +async def main(): + wf = EcomOpsWorkflow( + classifier=RuleBasedIntentClassifier(), + drafter=my_drafter, + order_lookup=my_order_lookup, + ) + result = await wf.handle("đơn #DH12345 của em tới đâu rồi ạ") + print(result.action, result.intent, result.reply) + +asyncio.run(main()) +``` + +`WorkflowResult`: + +- `action`: `"send"` (bot xử lý xong) hoặc `"handoff"` (cần nhân viên tiếp nhận) +- `reply`: **luôn an toàn để gửi** — nếu draft bị guardrail chặn, đây là câu trả lời dự phòng +- `intent`, `confidence`, `extracted`, `order`, `violations`, `handoff_reason`, `trace` + +Khi nào chuyển nhân viên: khiếu nại (`complaint`), độ tin cậy thấp, guardrail chặn, +hoặc soạn tin lỗi. + +## 4. Demo end-to-end (Zalo + Google Sheet) + +`example_worker.py` nối: webhook Zalo → tra đơn trong Google Sheet → trả lời có +guardrail → gửi lại qua Zalo. + +```bash +# cần connectors/zalo/config.yaml và connectors/google_sheet/config.yaml +export MAGIC_URL=http://localhost:8080 +export MAGIC_API_KEY=your-key +cd packs && python -m ecomops.example_worker +``` + +Sheet cần tab `Orders` với dòng header khớp Common Schema +(`id`, `customer_id`, `status`, `total_amount`, `shipping_fee`, `created_at`) — +xem `connectors/google_sheet/README.md`. + +## 5. Giới hạn hiện tại + +- Từ khoá phân loại là **danh sách cố định trong code**, chưa cấu hình được qua file. +- Chưa có Case Management (tạo case khi khiếu nại/đổi trả) — thuộc Tuần 3–4 của roadmap. +- `example_worker.py` quét toàn bộ dải `Orders!A1:H500` mỗi lần tra đơn; với sheet + lớn nên thay bằng Nhanh.vn connector hoặc thêm cache. +- Guardrail dựa trên cụm từ, không phải mô hình — bắt được các mẫu hứa hẹn phổ biến, + nhưng không thay thế được người duyệt với các đơn giá trị cao. diff --git a/packs/ecomops/__init__.py b/packs/ecomops/__init__.py new file mode 100644 index 0000000..224325a --- /dev/null +++ b/packs/ecomops/__init__.py @@ -0,0 +1,31 @@ +"""EcomOps Pack — customer-service workflow for Vietnamese online shops. + +Open-source reference implementation of the EcomOps workflow: classify an +inbound message, look the order up, draft a reply, and guardrail it before it +reaches the customer. +""" + +from .guardrails import GuardrailContext, GuardrailEngine, GuardrailViolation, Severity +from .intents import ( + HybridIntentClassifier, + Intent, + IntentResult, + LLMIntentClassifier, + RuleBasedIntentClassifier, +) +from .workflow import EcomOpsWorkflow, LLMDrafter, WorkflowResult + +__all__ = [ + "Intent", + "IntentResult", + "RuleBasedIntentClassifier", + "LLMIntentClassifier", + "HybridIntentClassifier", + "GuardrailEngine", + "GuardrailContext", + "GuardrailViolation", + "Severity", + "EcomOpsWorkflow", + "LLMDrafter", + "WorkflowResult", +] diff --git a/packs/ecomops/example_worker.py b/packs/ecomops/example_worker.py new file mode 100644 index 0000000..bf5263d --- /dev/null +++ b/packs/ecomops/example_worker.py @@ -0,0 +1,115 @@ +"""EcomOps end-to-end demo: tin nhắn Zalo → tra đơn trong Google Sheet → trả lời có guardrail. + +Đây là deliverable Tuần 2 của roadmap: "AI trả lời được câu hỏi trạng thái đơn +hàng qua Zalo, dùng dữ liệu từ Google Sheet". + +Chạy: + python -m ecomops.example_worker + +Cần 2 file config (xem connectors/*/config.example.yaml): + connectors/zalo/config.yaml + connectors/google_sheet/config.yaml +""" + +import asyncio +import logging +import os +import sys +import threading +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "connectors")) + +from google_sheet.connector import GoogleSheetConnector # noqa: E402 +from zalo.connector import ZaloConnector # noqa: E402 +from zalo.webhook import serve # noqa: E402 + +from ecomops.intents import HybridIntentClassifier, LLMIntentClassifier # noqa: E402 +from ecomops.workflow import EcomOpsWorkflow, LLMDrafter # noqa: E402 + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("ecomops.demo") + +MAGIC_URL = os.getenv("MAGIC_URL", "http://localhost:8080") +MAGIC_API_KEY = os.getenv("MAGIC_API_KEY", "") +ORDERS_RANGE = os.getenv("ECOMOPS_ORDERS_RANGE", "Orders!A1:H500") + + +def make_order_lookup(sheet: GoogleSheetConnector): + """Find an order in the sheet by order code, falling back to phone.""" + + async def lookup(order_code: str | None, phone: str | None) -> dict | None: + data = await sheet.execute("read_range", {"range": ORDERS_RANGE}) + rows = data.get("values") or [] + if len(rows) < 2: + return None + + header, *body = rows + for row in body: + order = sheet.map_to_common_schema({"header": header, "row": row}, "order") + if order_code and order.get("id") == order_code: + return order + if phone and phone in row: + return order + return None + + return lookup + + +async def handle_message(workflow: EcomOpsWorkflow, zalo: ZaloConnector, record: dict) -> None: + user_id = record.get("sender_id") + text = record.get("text") or "" + if not (user_id and text): + return + + result = await workflow.handle(text, customer={"phone": None}) + logger.info("intent=%s action=%s trace=%s", result.intent.value, result.action, result.trace) + + await zalo.execute("send_text_message", {"user_id": user_id, "text": result.reply}) + + if result.action == "handoff": + # Nơi để nối vào Human Inbox / hệ thống case sau này. + logger.warning( + "HANDOFF user=%s reason=%s violations=%s", + user_id, result.handoff_reason, [v.rule for v in result.violations], + ) + + +def main() -> None: + zalo_cfg = yaml.safe_load((REPO_ROOT / "connectors/zalo/config.yaml").read_text()) + sheet_cfg = yaml.safe_load((REPO_ROOT / "connectors/google_sheet/config.yaml").read_text()) + + zalo = ZaloConnector(zalo_cfg) + sheet = GoogleSheetConnector(sheet_cfg) + + workflow = EcomOpsWorkflow( + classifier=HybridIntentClassifier(llm=LLMIntentClassifier(MAGIC_URL, MAGIC_API_KEY)), + drafter=LLMDrafter(MAGIC_URL, MAGIC_API_KEY), + order_lookup=make_order_lookup(sheet), + allowed_contacts=zalo_cfg.get("shop_hotlines", []), + ) + + # Zalo's webhook server is threaded and synchronous, so the async pipeline + # runs on its own event loop in a background thread; each webhook thread + # hands work to that loop instead of spinning up a fresh one per message + # (which would also re-open the HTTP connection pools every time). + loop = asyncio.new_event_loop() + threading.Thread(target=loop.run_forever, daemon=True).start() + + asyncio.run_coroutine_threadsafe(zalo.connect(), loop).result() + asyncio.run_coroutine_threadsafe(sheet.connect(), loop).result() + + def on_event(records: list[dict]) -> None: + for record in records: + asyncio.run_coroutine_threadsafe(handle_message(workflow, zalo, record), loop) + + port = int(zalo_cfg.get("webhook", {}).get("port", 9100)) + logger.info("EcomOps demo listening for Zalo webhooks on :%d", port) + serve(zalo, on_event, port=port) + + +if __name__ == "__main__": + main() diff --git a/packs/ecomops/extraction.py b/packs/ecomops/extraction.py new file mode 100644 index 0000000..a3e5046 --- /dev/null +++ b/packs/ecomops/extraction.py @@ -0,0 +1,73 @@ +"""Pull the order code / phone number out of a free-text customer message. + +Cheap regex extraction on purpose: an order lookup only needs the identifier, +and asking an LLM to copy a number out of a sentence is slow, costs money, and +occasionally hallucinates a digit. +""" + +import re +from dataclasses import dataclass + +# Vietnamese mobile numbers: 0XXXXXXXXX (10 digits) or +84/84 + 9 digits. +PHONE_PATTERN = re.compile(r"(?:(? str | None: + """Fold any spelling of a Vietnamese mobile number to local 0XXXXXXXXX form. + + Handles "+84 987 654 321", "84987654321", "0987-654-321" and "987654321". + Returns None if it isn't a plausible VN mobile number, so callers can tell + "not a phone" apart from "a phone we normalized". + """ + digits = re.sub(r"\D", "", value) + if digits.startswith("84"): + digits = "0" + digits[2:] + elif len(digits) == 9: + digits = "0" + digits + return digits if len(digits) == 10 and digits.startswith("0") else None + + +def extract_phone(message: str) -> str | None: + """Return the phone in local 0XXXXXXXXX form, whatever prefix was typed.""" + match = PHONE_PATTERN.search(message) + return f"0{match.group(1)}" if match else None + + +def extract_order_code(message: str, known_phone: str | None = None) -> str | None: + """Find an order code, ignoring digits that are actually the phone number. + + Without this, "đơn của em sđt 0901234567" would happily return the phone as + an order code via the bare-digits pattern and the lookup would miss. + """ + haystack = message + phone = known_phone if known_phone is not None else extract_phone(message) + if phone: + # Blank out every spelling of that phone (0..., 84..., +84...) so the + # numeric fallback pattern can't pick it up. + local = phone.lstrip("0") + haystack = re.sub(rf"(?:\+?84|0)?{re.escape(local)}", " ", haystack) + + for pattern in ORDER_CODE_PATTERNS: + match = pattern.search(haystack) + if match: + return match.group(1) + return None + + +def extract(message: str) -> ExtractedInfo: + phone = extract_phone(message) + return ExtractedInfo(order_code=extract_order_code(message, known_phone=phone), phone=phone) diff --git a/packs/ecomops/guardrails.py b/packs/ecomops/guardrails.py new file mode 100644 index 0000000..6d9bfa2 --- /dev/null +++ b/packs/ecomops/guardrails.py @@ -0,0 +1,201 @@ +"""Guardrail engine — inspects a draft reply BEFORE it reaches the customer. + +An LLM draft is not automatically safe to send from a shop's official account: +it can promise a delivery date nobody can honour, offer a refund the shop never +approved, or echo another customer's phone number out of the context window. +Each rule here catches one of those, and anything BLOCK-severity routes the +conversation to a human instead of being sent. + +Rules operate on diacritics-normalized text (see `text.normalize`) so they can't +be sidestepped by typing without dấu. +""" + +import re +from dataclasses import dataclass, field +from enum import Enum +from typing import Protocol + +from .extraction import PHONE_PATTERN, canonical_phone +from .text import normalize + +EMAIL_PATTERN = re.compile(r"[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,}") + + +class Severity(str, Enum): + BLOCK = "block" + WARN = "warn" + + +@dataclass +class GuardrailViolation: + rule: str + severity: Severity + message: str + evidence: str = "" + + +@dataclass +class GuardrailContext: + """What the rules need to judge a draft in context.""" + + customer_phone: str | None = None + customer_email: str | None = None + # The shop's own hotline/support address — legitimately quotable in a reply. + allowed_contacts: list[str] = field(default_factory=list) + # Set once a human has actually approved compensation for this conversation. + compensation_approved: bool = False + + +class GuardrailRule(Protocol): + name: str + + def check(self, draft: str, context: GuardrailContext) -> list[GuardrailViolation]: ... + + +def _phrase_hits(normalized_draft: str, phrases: list[str]) -> list[str]: + return [p for p in phrases if p in normalized_draft] + + +class EmptyDraftRule: + """A blank reply is worse than no reply — never send one.""" + + name = "empty_draft" + + def check(self, draft: str, context: GuardrailContext) -> list[GuardrailViolation]: + if draft and draft.strip(): + return [] + return [GuardrailViolation(self.name, Severity.BLOCK, "Draft trả lời rỗng.")] + + +class LeakedContactRule: + """Block phone numbers / emails that belong to neither the customer nor the shop.""" + + name = "leaked_contact" + + def check(self, draft: str, context: GuardrailContext) -> list[GuardrailViolation]: + # Canonicalize both sides: a shop that configures its hotline as + # "+84987654321" must still match the "0987654321" the model wrote, or + # the guardrail blocks the shop's own number. + allowed_phones = {canonical_phone(c) for c in [*context.allowed_contacts, context.customer_phone or ""]} + allowed_phones.discard(None) + + allowed_emails = {c.lower() for c in context.allowed_contacts} + if context.customer_email: + allowed_emails.add(context.customer_email.lower()) + + violations = [] + for match in PHONE_PATTERN.finditer(draft): + phone = f"0{match.group(1)}" + if phone not in allowed_phones: + violations.append( + GuardrailViolation( + self.name, Severity.BLOCK, + "Draft chứa số điện thoại không phải của khách hoặc của shop.", + evidence=phone, + ) + ) + for match in EMAIL_PATTERN.finditer(draft): + if match.group(0).lower() not in allowed_emails: + violations.append( + GuardrailViolation( + self.name, Severity.BLOCK, + "Draft chứa email không phải của khách hoặc của shop.", + evidence=match.group(0), + ) + ) + return violations + + +class DeliveryPromiseRule: + """Block hard delivery guarantees — the shop doesn't control the carrier.""" + + name = "delivery_promise" + + PHRASES = [ + "chac chan giao", "chac chan nhan", "chac chan den", "cam ket giao", + "dam bao giao", "dam bao nhan", "chac chan ngay mai", "100% giao", + "giao dung ngay", "nhat dinh se giao", + ] + + def check(self, draft: str, context: GuardrailContext) -> list[GuardrailViolation]: + hits = _phrase_hits(normalize(draft), self.PHRASES) + return [ + GuardrailViolation( + self.name, Severity.BLOCK, + "Draft hứa chắc chắn về thời gian giao hàng.", + evidence=hit, + ) + for hit in hits + ] + + +class CompensationPromiseRule: + """Block refunds/vouchers/free shipping the shop hasn't approved.""" + + name = "compensation_promise" + + PHRASES = [ + "hoan tien 100", "hoan tien ngay", "hoan lai toan bo", "tang voucher", + "tang ma giam gia", "boi thuong", "den bu", "mien phi ship cho anh", + "mien phi ship cho chi", "giam gia cho anh", "giam gia cho chi", + ] + + def check(self, draft: str, context: GuardrailContext) -> list[GuardrailViolation]: + if context.compensation_approved: + return [] + hits = _phrase_hits(normalize(draft), self.PHRASES) + return [ + GuardrailViolation( + self.name, Severity.BLOCK, + "Draft tự hứa đền bù/hoàn tiền/giảm giá khi chưa được duyệt.", + evidence=hit, + ) + for hit in hits + ] + + +class InternalLeakRule: + """Block prompt/system/credential text that leaked into the draft.""" + + name = "internal_leak" + + PHRASES = [ + "system prompt", "system:", "internal note", "[internal]", "nguyen tac bat buoc", + "ban la nhan vien cham soc", "api key", "bearer ", "sk-", "todo:", + ] + + def check(self, draft: str, context: GuardrailContext) -> list[GuardrailViolation]: + hits = _phrase_hits(normalize(draft), self.PHRASES) + return [ + GuardrailViolation( + self.name, Severity.BLOCK, + "Draft lộ nội dung nội bộ / prompt hệ thống.", + evidence=hit, + ) + for hit in hits + ] + + +DEFAULT_RULES: list[GuardrailRule] = [ + EmptyDraftRule(), + LeakedContactRule(), + DeliveryPromiseRule(), + CompensationPromiseRule(), + InternalLeakRule(), +] + + +class GuardrailEngine: + def __init__(self, rules: list[GuardrailRule] | None = None): + self.rules = rules if rules is not None else list(DEFAULT_RULES) + + def check(self, draft: str, context: GuardrailContext | None = None) -> list[GuardrailViolation]: + ctx = context or GuardrailContext() + violations: list[GuardrailViolation] = [] + for rule in self.rules: + violations.extend(rule.check(draft, ctx)) + return violations + + @staticmethod + def is_blocked(violations: list[GuardrailViolation]) -> bool: + return any(v.severity is Severity.BLOCK for v in violations) diff --git a/packs/ecomops/intents.py b/packs/ecomops/intents.py new file mode 100644 index 0000000..59c7a01 --- /dev/null +++ b/packs/ecomops/intents.py @@ -0,0 +1,217 @@ +"""Intent classification for EcomOps customer messages (Vietnamese). + +Two classifiers, meant to be layered: + +- `RuleBasedIntentClassifier` — keyword matching over diacritics-normalized text. + Free, instant, and handles the bulk of real shop traffic, which is short and + formulaic ("đơn hàng của em tới đâu rồi ạ"). +- `LLMIntentClassifier` — asks MagiC's own LLM gateway (`POST /api/v1/llm/chat`), + so the pack stays provider-agnostic and every call is cost-tracked by core. + +`HybridIntentClassifier` runs the rules first and only pays for an LLM call when +the rules are unsure — quality where it matters, no spend where it doesn't. +""" + +import re +from dataclasses import dataclass, field +from enum import Enum + +import httpx + +from .text import normalize + + +class Intent(str, Enum): + ORDER_STATUS = "order_status" + SHIPPING_FEE = "shipping_fee" + RETURN_EXCHANGE = "return_exchange" + COMPLAINT = "complaint" + PRODUCT_INFO = "product_info" + GREETING = "greeting" + OTHER = "other" + + +@dataclass +class IntentResult: + intent: Intent + confidence: float + matched: list[str] = field(default_factory=list) + source: str = "rules" + + +# Keywords are written in normalized form (lowercase, no diacritics) because the +# incoming message is normalized the same way — that makes each entry match both +# "đơn hàng" and "don hang" without duplicating the list. +# +# Deliberately multi-word where a single word would collide: bare "hong" would +# match "màu hồng" (pink) as a damage complaint, "vo" would match "vợ"/"vô", and +# "bao nhieu" alone spans both price and shipping questions. +INTENT_KEYWORDS: dict[Intent, list[str]] = { + Intent.ORDER_STATUS: [ + "don hang", "kien hang", "van don", "ma don", "tinh trang don", "check don", + "tra cuu don", "khi nao nhan", "khi nao giao", "bao gio nhan", "bao gio giao", + "giao chua", "den chua", "toi dau", "dang o dau", "shipper", "da gui chua", + ], + Intent.SHIPPING_FEE: [ + "phi ship", "phi van chuyen", "phi giao hang", "tien ship", "cuoc van chuyen", + "ship bao nhieu", "ship het bao nhieu", "freeship", "mien phi ship", "mien ship", + ], + Intent.RETURN_EXCHANGE: [ + "doi tra", "doi hang", "tra hang", "tra lai", "hoan tra", "doi size", "doi mau", + "chinh sach doi", "bao hanh", "hoan tien", + ], + Intent.COMPLAINT: [ + "khieu nai", "phan anh", "that vong", "te qua", "lua dao", "boc phot", + "kem chat luong", "hang loi", "bi hong", "bi vo", "bi rach", "loi san pham", + "giao sai", "giao thieu", "thieu hang", "khong dung mau", "khong dung size", + "buc xuc", "cham qua", "qua lau", + ], + Intent.PRODUCT_INFO: [ + "con hang", "con size", "con mau", "co san", "gia bao nhieu", "gia the nao", + "bao nhieu tien", "chat lieu", "thong tin san pham", "size nao", "mau nao", + ], + Intent.GREETING: [ + "xin chao", "chao shop", "chao ban", "chao em", "alo", "hello", "hi shop", + ], +} + + +# Vietnamese slots pronouns and particles inside otherwise fixed phrases — +# "khi nào **em** nhận được hàng" should still match the keyword "khi nao nhan". +# So keyword tokens are matched in order with up to this many filler words +# between them, rather than as one literal substring. Kept at 1: it absorbs the +# common single-pronoun insertion without letting a keyword match words scattered +# across a whole sentence. +_MAX_GAP_WORDS = 1 + + +def _compile_keyword(keyword: str) -> re.Pattern[str]: + gap = rf"(?:\s+\S+){{0,{_MAX_GAP_WORDS}}}\s+" + body = gap.join(re.escape(token) for token in keyword.split()) + return re.compile(rf"(? IntentResult: + norm = normalize(message) + + hits: dict[Intent, list[str]] = {} + for intent, compiled in self._compiled.items(): + found = [kw for kw, pattern in compiled if pattern.search(norm)] + if found: + hits[intent] = found + + # A greeting is almost always a prefix to the real question ("chào shop, + # đơn hàng của em đâu rồi") — it should only win when it's all there is. + if len(hits) > 1: + hits.pop(Intent.GREETING, None) + + if not hits: + return IntentResult(Intent.OTHER, 0.0, []) + + # Rank by total matched length, so a specific phrase outranks a short one. + best = max(hits, key=lambda i: (sum(len(kw) for kw in hits[i]), len(hits[i]))) + return IntentResult(best, _confidence(len(hits[best])), sorted(hits[best])) + + +def _confidence(n_hits: int) -> float: + """Heuristic: one keyword is a decent signal, several is a strong one. + + Capped below 1.0 — keyword matching should never claim certainty, so a + hybrid/LLM layer above can still override it if it wants to. + """ + return min(0.95, 0.5 + 0.15 * n_hits) + + +_LLM_SYSTEM_PROMPT = """Bạn là bộ phân loại ý định cho chatbot chăm sóc khách hàng của shop bán hàng online Việt Nam. +Phân loại tin nhắn của khách vào ĐÚNG MỘT nhãn trong danh sách sau: + +- order_status: hỏi tình trạng/vị trí đơn hàng, khi nào nhận được hàng +- shipping_fee: hỏi phí ship, phí vận chuyển +- return_exchange: muốn đổi hàng, trả hàng, bảo hành, hoàn tiền +- complaint: khiếu nại, phàn nàn về hàng lỗi/giao sai/giao chậm/thái độ +- product_info: hỏi thông tin sản phẩm, giá, size, màu, còn hàng không +- greeting: chỉ chào hỏi, chưa có yêu cầu cụ thể +- other: không thuộc các nhóm trên + +Chỉ trả lời bằng đúng một nhãn, viết thường, không giải thích, không thêm dấu câu.""" + +_VALID_LABELS = {i.value for i in Intent} + + +class LLMIntentClassifier: + """Classifies via MagiC's LLM gateway (`POST /api/v1/llm/chat`). + + Uses the `cheapest` routing strategy — labelling a short message doesn't + need a frontier model, and this runs on every inbound customer message. + """ + + def __init__( + self, + magic_url: str, + api_key: str = "", + model: str = "", + timeout: float = 15.0, + strategy: str = "cheapest", + ): + self.magic_url = magic_url.rstrip("/") + self.api_key = api_key + self.model = model + self.timeout = timeout + self.strategy = strategy + + async def classify(self, message: str) -> IntentResult: + headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {} + payload: dict = { + "messages": [ + {"role": "system", "content": _LLM_SYSTEM_PROMPT}, + {"role": "user", "content": message}, + ], + "strategy": self.strategy, + "max_tokens": 16, + } + if self.model: + payload["model"] = self.model + + async with httpx.AsyncClient(timeout=self.timeout) as client: + resp = await client.post(f"{self.magic_url}/api/v1/llm/chat", headers=headers, json=payload) + resp.raise_for_status() + content = (resp.json().get("content") or "").strip().lower() + + # Models like to add punctuation or a stray word even when told not to; + # take the first token that is a label we actually know. + for token in content.replace(",", " ").replace(".", " ").split(): + if token in _VALID_LABELS: + return IntentResult(Intent(token), 0.9, [], source="llm") + return IntentResult(Intent.OTHER, 0.0, [], source="llm") + + +class HybridIntentClassifier: + """Rules first; fall back to the LLM only when the rules aren't confident.""" + + def __init__(self, llm: LLMIntentClassifier | None = None, threshold: float = 0.65): + self.rules = RuleBasedIntentClassifier() + self.llm = llm + self.threshold = threshold + + async def classify(self, message: str) -> IntentResult: + result = self.rules.classify(message) + if result.confidence >= self.threshold or self.llm is None: + return result + try: + llm_result = await self.llm.classify(message) + except (httpx.HTTPError, ValueError): + # LLM unavailable is not a reason to drop the message — keep the + # rule-based guess and let the workflow's low-confidence handoff + # rule decide whether a human should look at it. + return result + return llm_result if llm_result.intent != Intent.OTHER else result diff --git a/packs/ecomops/prompts.py b/packs/ecomops/prompts.py new file mode 100644 index 0000000..dce4a22 --- /dev/null +++ b/packs/ecomops/prompts.py @@ -0,0 +1,102 @@ +"""Vietnamese prompt library for EcomOps draft replies. + +Kept as data (not f-strings scattered through the workflow) so a shop can tune +tone and policy wording without touching pipeline code — and so the guardrail +tests can assert against the exact instructions the model was given. +""" + +from .intents import Intent + +SYSTEM_PROMPT = """Bạn là nhân viên chăm sóc khách hàng của một shop bán hàng online tại Việt Nam. + +Nguyên tắc bắt buộc: +- Trả lời ngắn gọn, lịch sự, xưng "shop" và gọi khách là "anh/chị". +- CHỈ dùng thông tin được cung cấp trong phần "Dữ liệu". Không được bịa thông tin đơn hàng, + ngày giao, giá, hay chính sách. +- KHÔNG hứa chắc chắn về ngày giao hàng. Chỉ nói theo trạng thái thực tế của đơn. +- KHÔNG tự ý hứa hoàn tiền, giảm giá, tặng voucher hay miễn phí vận chuyển. +- KHÔNG nhắc tới số điện thoại, email hay thông tin của khách hàng khác. +- Nếu không đủ thông tin để trả lời, hãy nói sẽ chuyển cho nhân viên hỗ trợ kiểm tra giúp. + +Trả lời bằng tiếng Việt, tối đa 4 câu.""" + + +INTENT_INSTRUCTIONS: dict[Intent, str] = { + Intent.ORDER_STATUS: ( + "Khách đang hỏi tình trạng đơn hàng. Dựa vào dữ liệu đơn hàng bên dưới, " + "cho khách biết trạng thái hiện tại. Nếu không có dữ liệu đơn, hãy hỏi khách " + "mã đơn hàng hoặc số điện thoại đặt hàng." + ), + Intent.SHIPPING_FEE: ( + "Khách đang hỏi phí vận chuyển. Trả lời theo đúng biểu phí trong dữ liệu. " + "Nếu không có dữ liệu về phí ship, hãy nói sẽ nhờ nhân viên báo giá chính xác." + ), + Intent.RETURN_EXCHANGE: ( + "Khách muốn đổi/trả hàng. Nêu đúng điều kiện đổi trả có trong dữ liệu chính sách. " + "Không tự cam kết chấp nhận đổi trả khi chưa kiểm tra đơn." + ), + Intent.PRODUCT_INFO: ( + "Khách hỏi thông tin sản phẩm. Chỉ trả lời theo dữ liệu sản phẩm được cung cấp. " + "Nếu thiếu thông tin, hãy nói sẽ kiểm tra và phản hồi lại." + ), + Intent.GREETING: ( + "Khách mới chào hỏi. Chào lại ngắn gọn và hỏi shop có thể hỗ trợ gì cho khách." + ), + Intent.COMPLAINT: ( + "Khách đang khiếu nại. Xin lỗi chân thành, ghi nhận vấn đề, và cho biết sẽ chuyển " + "nhân viên phụ trách xử lý. Không tự đưa ra phương án đền bù." + ), + Intent.OTHER: ( + "Chưa xác định được yêu cầu của khách. Hỏi lại khách một cách lịch sự để làm rõ." + ), +} + + +def build_messages( + intent: Intent, + customer_message: str, + order: dict | None = None, + knowledge: list[str] | None = None, +) -> list[dict[str, str]]: + """Assemble the chat messages for a draft reply. + + The order/knowledge blocks are rendered explicitly under a "Dữ liệu" heading + that the system prompt refers to, so "only use the provided data" is an + instruction the model can actually follow. + """ + sections = [INTENT_INSTRUCTIONS.get(intent, INTENT_INSTRUCTIONS[Intent.OTHER])] + + data_parts = [] + if order: + data_parts.append("Đơn hàng:\n" + _render_order(order)) + if knowledge: + data_parts.append("Chính sách / thông tin shop:\n" + "\n".join(f"- {k}" for k in knowledge)) + sections.append("Dữ liệu:\n" + ("\n\n".join(data_parts) if data_parts else "(không có dữ liệu)")) + + sections.append(f"Tin nhắn của khách:\n{customer_message}") + + return [ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": "\n\n".join(sections)}, + ] + + +_ORDER_LABELS = { + "id": "Mã đơn", + "status": "Trạng thái", + "total_amount": "Tổng tiền", + "shipping_fee": "Phí ship", + "created_at": "Ngày đặt", +} + + +def _render_order(order: dict) -> str: + lines = [f"- {label}: {order[key]}" for key, label in _ORDER_LABELS.items() if order.get(key)] + items = order.get("items") or [] + if items: + rendered = ", ".join( + f"{it.get('name', '?')} x{it.get('quantity', 1)}" for it in items if isinstance(it, dict) + ) + if rendered: + lines.append(f"- Sản phẩm: {rendered}") + return "\n".join(lines) if lines else "(không có thông tin)" diff --git a/packs/ecomops/pytest.ini b/packs/ecomops/pytest.ini new file mode 100644 index 0000000..2f4c80e --- /dev/null +++ b/packs/ecomops/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +asyncio_mode = auto diff --git a/packs/ecomops/requirements.txt b/packs/ecomops/requirements.txt new file mode 100644 index 0000000..a51ee1d --- /dev/null +++ b/packs/ecomops/requirements.txt @@ -0,0 +1,2 @@ +# Install the framework first: pip install -e ../../sdk/python[connectors] +httpx>=0.27 diff --git a/packs/ecomops/tests/__init__.py b/packs/ecomops/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packs/ecomops/tests/test_extraction.py b/packs/ecomops/tests/test_extraction.py new file mode 100644 index 0000000..1fe2871 --- /dev/null +++ b/packs/ecomops/tests/test_extraction.py @@ -0,0 +1,78 @@ +"""Order-code / phone extraction tests.""" + +import pytest + +from ecomops.extraction import canonical_phone, extract, extract_order_code, extract_phone + + +@pytest.mark.parametrize( + "message,expected", + [ + ("sdt cua em la 0901234567", "0901234567"), + ("lien he +84901234567 nhe", "0901234567"), + ("so 84901234567", "0901234567"), + ("khong co so nao", None), + ], +) +def test_extract_phone(message, expected): + assert extract_phone(message) == expected + + +@pytest.mark.parametrize( + "message,expected", + [ + ("cho hoi don #DH12345", "DH12345"), + ("ma don DH-556677 giao chua", "DH-556677"), + ("don hang ORD_998877", "ORD_998877"), + ("ma 987654321 toi dau", "987654321"), + ("khong co ma nao ca", None), + ], +) +def test_extract_order_code(message, expected): + assert extract_order_code(message) == expected + + +def test_phone_is_not_mistaken_for_an_order_code(): + """A bare 10-digit phone must not be returned as the order id, or the + order lookup silently misses.""" + assert extract_order_code("don cua em sdt 0901234567 giao chua") is None + + +def test_phone_and_order_code_coexist(): + info = extract("em dat hang, sdt 0901234567, ma don DH-556677") + assert info.phone == "0901234567" + assert info.order_code == "DH-556677" + + +def test_international_phone_does_not_leak_into_order_code(): + info = extract("sdt +84901234567 kiem tra giup em") + assert info.phone == "0901234567" + assert info.order_code is None + + +def test_prefixed_code_wins_over_bare_digits(): + """'#'/'DH' prefixes are a much stronger signal than a loose number.""" + assert extract_order_code("don #DH12345 ngay 20260721") == "DH12345" + + +def test_empty_message(): + info = extract("") + assert info.order_code is None and info.phone is None + + +@pytest.mark.parametrize( + "value,expected", + [ + ("0987654321", "0987654321"), + ("+84987654321", "0987654321"), + ("84987654321", "0987654321"), + ("987654321", "0987654321"), + ("0987 654 321", "0987654321"), + ("+84 987-654-321", "0987654321"), + ("0847123456", "0847123456"), # local number that merely starts with 084 + ("khong phai so", None), + ("12345", None), + ], +) +def test_canonical_phone(value, expected): + assert canonical_phone(value) == expected diff --git a/packs/ecomops/tests/test_guardrails.py b/packs/ecomops/tests/test_guardrails.py new file mode 100644 index 0000000..a16acc0 --- /dev/null +++ b/packs/ecomops/tests/test_guardrails.py @@ -0,0 +1,145 @@ +"""Guardrail tests — every rule, plus the diacritics-free evasion path.""" + +import pytest + +from ecomops.guardrails import ( + CompensationPromiseRule, + DeliveryPromiseRule, + GuardrailContext, + GuardrailEngine, + InternalLeakRule, + LeakedContactRule, + Severity, +) + + +@pytest.fixture +def engine(): + return GuardrailEngine() + + +def test_clean_draft_passes(engine): + draft = "Dạ đơn hàng của anh/chị đang được vận chuyển ạ." + assert engine.check(draft, GuardrailContext()) == [] + + +def test_empty_draft_is_blocked(engine): + violations = engine.check(" ", GuardrailContext()) + assert [v.rule for v in violations] == ["empty_draft"] + assert GuardrailEngine.is_blocked(violations) + + +# ---- leaked contact ---- + + +def test_other_customers_phone_is_blocked(): + draft = "Đơn của chị Lan số 0912345678 cũng đang giao ạ." + violations = LeakedContactRule().check(draft, GuardrailContext(customer_phone="0901234567")) + assert [v.rule for v in violations] == ["leaked_contact"] + assert violations[0].evidence == "0912345678" + + +def test_customers_own_phone_is_allowed(): + draft = "Shop xác nhận đơn giao tới số 0901234567 của anh/chị ạ." + assert LeakedContactRule().check(draft, GuardrailContext(customer_phone="0901234567")) == [] + + +@pytest.mark.parametrize( + "configured_hotline", + ["0987654321", "+84987654321", "84987654321", "0987 654 321", "+84 987-654-321"], +) +def test_shop_hotline_is_allowed_however_it_is_configured(configured_hotline): + """A shop writing its hotline as +84… must not get its own number blocked.""" + draft = "Anh/chị gọi hotline 0987654321 giúp shop nhé." + ctx = GuardrailContext(customer_phone="0901234567", allowed_contacts=[configured_hotline]) + assert LeakedContactRule().check(draft, ctx) == [] + + +def test_allowlisting_the_hotline_does_not_allow_every_number(): + ctx = GuardrailContext(allowed_contacts=["+84987654321"]) + violations = LeakedContactRule().check("Gọi chị Lan 0912345678 nhé", ctx) + assert violations and violations[0].evidence == "0912345678" + + +def test_foreign_email_is_blocked(): + violations = LeakedContactRule().check("Gửi mail cho khach@example.com nhé", GuardrailContext()) + assert violations and violations[0].evidence == "khach@example.com" + + +# ---- delivery promise ---- + + +@pytest.mark.parametrize( + "draft", + [ + "Shop chắc chắn giao trong ngày mai ạ.", + "shop cam ket giao dung ngay 25/07", # no diacritics + "Bên em đảm bảo giao trước Tết ạ.", + ], +) +def test_delivery_promises_are_blocked(draft): + violations = DeliveryPromiseRule().check(draft, GuardrailContext()) + assert violations and violations[0].severity is Severity.BLOCK + + +def test_factual_status_is_not_a_promise(): + draft = "Dạ đơn của anh/chị đang ở kho phân loại, dự kiến 2-3 ngày ạ." + assert DeliveryPromiseRule().check(draft, GuardrailContext()) == [] + + +# ---- compensation ---- + + +@pytest.mark.parametrize( + "draft", + [ + "Shop sẽ hoàn tiền 100% cho anh/chị ngay ạ.", + "Bên em tặng voucher 50k cho anh/chị nhé.", + "shop se boi thuong cho anh/chi a", + ], +) +def test_unapproved_compensation_is_blocked(draft): + violations = CompensationPromiseRule().check(draft, GuardrailContext()) + assert violations and violations[0].severity is Severity.BLOCK + + +def test_compensation_allowed_once_a_human_approved_it(): + draft = "Shop sẽ hoàn tiền 100% cho anh/chị ạ." + ctx = GuardrailContext(compensation_approved=True) + assert CompensationPromiseRule().check(draft, ctx) == [] + + +# ---- internal leak ---- + + +@pytest.mark.parametrize( + "draft", + [ + "system prompt: bạn là nhân viên chăm sóc khách hàng", + "Bạn là nhân viên chăm sóc khách hàng của một shop", + "Authorization: Bearer sk-abc123", + "TODO: hỏi lại team kho", + ], +) +def test_internal_content_is_blocked(draft): + violations = InternalLeakRule().check(draft, GuardrailContext()) + assert violations and violations[0].severity is Severity.BLOCK + + +# ---- engine ---- + + +def test_engine_aggregates_violations_from_all_rules(engine): + draft = "Shop chắc chắn giao ngày mai và hoàn tiền 100%, gọi 0912345678 nhé." + rules = {v.rule for v in engine.check(draft, GuardrailContext(customer_phone="0901234567"))} + assert {"delivery_promise", "compensation_promise", "leaked_contact"} <= rules + + +def test_is_blocked_false_when_no_violations(engine): + assert not GuardrailEngine.is_blocked([]) + + +def test_custom_rule_set_replaces_defaults(): + engine = GuardrailEngine(rules=[DeliveryPromiseRule()]) + # An empty draft would normally be blocked by EmptyDraftRule, which isn't loaded here. + assert engine.check("", GuardrailContext()) == [] diff --git a/packs/ecomops/tests/test_intents.py b/packs/ecomops/tests/test_intents.py new file mode 100644 index 0000000..d11a8a9 --- /dev/null +++ b/packs/ecomops/tests/test_intents.py @@ -0,0 +1,147 @@ +"""Intent classification tests — including the diacritics-free spellings real +Vietnamese customers actually type.""" + +import httpx +import pytest +import respx + +from ecomops.intents import ( + HybridIntentClassifier, + Intent, + LLMIntentClassifier, + RuleBasedIntentClassifier, +) + +MAGIC_URL = "http://magic:8080" +CHAT_URL = f"{MAGIC_URL}/api/v1/llm/chat" + + +@pytest.fixture +def rules(): + return RuleBasedIntentClassifier() + + +@pytest.mark.parametrize( + "message,expected", + [ + ("Đơn hàng của em tới đâu rồi ạ?", Intent.ORDER_STATUS), + ("don hang cua em toi dau roi a", Intent.ORDER_STATUS), # no diacritics + ("Khi nào em nhận được hàng vậy shop", Intent.ORDER_STATUS), + ("Phí ship về Đà Nẵng bao nhiêu ạ?", Intent.SHIPPING_FEE), + ("phi van chuyen la bao nhieu", Intent.SHIPPING_FEE), + ("Em muốn đổi size áo này được không", Intent.RETURN_EXCHANGE), + ("shop giao sai màu rồi, bực quá", Intent.COMPLAINT), + ("hang loi, em muon khieu nai", Intent.COMPLAINT), + ("Áo này còn size M không shop?", Intent.PRODUCT_INFO), + ("Xin chào shop", Intent.GREETING), + ("ừm", Intent.OTHER), + ], +) +def test_rule_based_classification(rules, message, expected): + assert rules.classify(message).intent is expected + + +def test_diacritic_and_plain_spelling_agree(rules): + with_marks = rules.classify("Đơn hàng của tôi đâu rồi?") + without_marks = rules.classify("Don hang cua toi dau roi?") + assert with_marks.intent is without_marks.intent is Intent.ORDER_STATUS + assert with_marks.confidence == without_marks.confidence + + +def test_greeting_loses_to_the_real_question(rules): + """'chào shop, đơn hàng của em đâu' is an order question, not a greeting.""" + result = rules.classify("Chào shop, đơn hàng của em đâu rồi ạ") + assert result.intent is Intent.ORDER_STATUS + + +def test_greeting_wins_when_alone(rules): + assert rules.classify("chao shop").intent is Intent.GREETING + + +def test_confidence_grows_with_more_matches_but_never_reaches_one(rules): + one_hit = rules.classify("đơn hàng") + many_hits = rules.classify("đơn hàng của em khi nào giao, giao chưa shop, mã đơn đâu") + assert one_hit.confidence < many_hits.confidence <= 0.95 + + +def test_unknown_message_has_zero_confidence(rules): + result = rules.classify("abcxyz") + assert result.intent is Intent.OTHER + assert result.confidence == 0.0 + + +# ---- LLM classifier ---- + + +@respx.mock +async def test_llm_classifier_parses_label(): + respx.post(CHAT_URL).mock(return_value=httpx.Response(200, json={"content": "shipping_fee"})) + result = await LLMIntentClassifier(MAGIC_URL, api_key="k").classify("cho hỏi ship") + assert result.intent is Intent.SHIPPING_FEE + assert result.source == "llm" + + +@respx.mock +async def test_llm_classifier_tolerates_chatty_output(): + """Models add punctuation/extra words even when told not to.""" + respx.post(CHAT_URL).mock(return_value=httpx.Response(200, json={"content": " order_status.\n"})) + result = await LLMIntentClassifier(MAGIC_URL).classify("đơn đâu") + assert result.intent is Intent.ORDER_STATUS + + +@respx.mock +async def test_llm_classifier_unknown_label_is_other(): + respx.post(CHAT_URL).mock(return_value=httpx.Response(200, json={"content": "khong_biet"})) + assert (await LLMIntentClassifier(MAGIC_URL).classify("???")).intent is Intent.OTHER + + +@respx.mock +async def test_llm_classifier_sends_auth_and_cheap_strategy(): + route = respx.post(CHAT_URL).mock(return_value=httpx.Response(200, json={"content": "greeting"})) + await LLMIntentClassifier(MAGIC_URL, api_key="secret").classify("hi") + + import json as jsonlib + + body = jsonlib.loads(route.calls.last.request.content) + assert route.calls.last.request.headers["Authorization"] == "Bearer secret" + assert body["strategy"] == "cheapest" + + +# ---- Hybrid ---- + + +@respx.mock +async def test_hybrid_skips_llm_when_rules_are_confident(): + route = respx.post(CHAT_URL).mock(return_value=httpx.Response(200, json={"content": "other"})) + hybrid = HybridIntentClassifier(llm=LLMIntentClassifier(MAGIC_URL)) + + result = await hybrid.classify("đơn hàng của em khi nào giao, giao chưa shop") + + assert result.intent is Intent.ORDER_STATUS + assert route.call_count == 0 # no spend when the rules already know + + +@respx.mock +async def test_hybrid_falls_back_to_llm_when_rules_are_unsure(): + respx.post(CHAT_URL).mock(return_value=httpx.Response(200, json={"content": "product_info"})) + hybrid = HybridIntentClassifier(llm=LLMIntentClassifier(MAGIC_URL)) + + result = await hybrid.classify("cái này xài sao vậy shop") + + assert result.intent is Intent.PRODUCT_INFO + assert result.source == "llm" + + +@respx.mock +async def test_hybrid_keeps_rule_guess_when_llm_is_down(): + respx.post(CHAT_URL).mock(side_effect=httpx.ConnectError("down")) + hybrid = HybridIntentClassifier(llm=LLMIntentClassifier(MAGIC_URL)) + + result = await hybrid.classify("cái này xài sao vậy shop") + + assert result.source == "rules" # degraded, not crashed + + +async def test_hybrid_without_llm_is_just_rules(): + result = await HybridIntentClassifier(llm=None).classify("phí ship bao nhiêu") + assert result.intent is Intent.SHIPPING_FEE diff --git a/packs/ecomops/tests/test_workflow.py b/packs/ecomops/tests/test_workflow.py new file mode 100644 index 0000000..bc45adc --- /dev/null +++ b/packs/ecomops/tests/test_workflow.py @@ -0,0 +1,278 @@ +"""End-to-end tests for the EcomOps pipeline (no network — deps are injected).""" + +import httpx +import pytest +import respx + +from ecomops.guardrails import GuardrailEngine +from ecomops.intents import Intent, IntentResult, RuleBasedIntentClassifier +from ecomops.workflow import ( + COMPLAINT_ACK_REPLY, + HANDOFF_REPLY, + EcomOpsWorkflow, + LLMDrafter, +) + +ORDER = {"id": "DH12345", "status": "shipping", "total_amount": 250000} + + +class drafter_returning: + """Async drafter stub that records the prompt it was handed.""" + + def __init__(self, text: str): + self.text = text + self.messages: list[dict[str, str]] | None = None + + async def __call__(self, messages): + self.messages = messages + return self.text + + +class lookup_returning: + """Async order-lookup stub that records the identifiers it was called with.""" + + def __init__(self, order): + self.order = order + self.called_with: tuple | None = None + + async def __call__(self, order_code, phone): + self.called_with = (order_code, phone) + return self.order + + +class StubClassifier: + """Sync classifier stub — also proves the workflow accepts non-async ones.""" + + def __init__(self, intent: Intent, confidence: float = 0.9): + self.result = IntentResult(intent, confidence) + + def classify(self, message): + return self.result + + +class AsyncStubClassifier: + def __init__(self, intent: Intent, confidence: float = 0.9): + self.result = IntentResult(intent, confidence) + + async def classify(self, message): + return self.result + + +# ---- happy path ---- + + +async def test_confident_intent_with_clean_draft_is_sent(): + drafter = drafter_returning("Dạ đơn của anh/chị đang được giao ạ.") + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.ORDER_STATUS), + drafter=drafter, + order_lookup=lookup_returning(ORDER), + ) + + result = await wf.handle("đơn #DH12345 tới đâu rồi") + + assert result.action == "send" + assert result.reply == "Dạ đơn của anh/chị đang được giao ạ." + assert result.order == ORDER + assert result.violations == [] + + +async def test_async_classifier_is_also_supported(): + wf = EcomOpsWorkflow( + classifier=AsyncStubClassifier(Intent.SHIPPING_FEE), + drafter=drafter_returning("Dạ phí ship 30k ạ."), + ) + assert (await wf.handle("phí ship bao nhiêu")).action == "send" + + +async def test_real_rule_classifier_end_to_end(): + wf = EcomOpsWorkflow( + classifier=RuleBasedIntentClassifier(), + drafter=drafter_returning("Dạ phí ship về Đà Nẵng là 30.000đ ạ."), + ) + result = await wf.handle("phí ship về Đà Nẵng bao nhiêu shop, phí vận chuyển ấy") + assert result.intent is Intent.SHIPPING_FEE + assert result.action == "send" + + +# ---- order lookup ---- + + +async def test_lookup_receives_extracted_identifiers(): + lookup = lookup_returning(ORDER) + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.ORDER_STATUS), + drafter=drafter_returning("ok ạ"), + order_lookup=lookup, + ) + + await wf.handle("đơn DH-556677 của em, sdt 0901234567") + + assert lookup.called_with == ("DH-556677", "0901234567") + + +async def test_lookup_is_skipped_without_identifiers(): + lookup = lookup_returning(ORDER) + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.ORDER_STATUS), + drafter=drafter_returning("ok ạ"), + order_lookup=lookup, + ) + + await wf.handle("đơn của em sao rồi shop") + + assert lookup.called_with is None + + +async def test_lookup_failure_does_not_drop_the_message(): + async def failing_lookup(order_code, phone): + raise RuntimeError("sheet unavailable") + + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.ORDER_STATUS), + drafter=drafter_returning("Dạ shop kiểm tra giúp anh/chị ạ."), + order_lookup=failing_lookup, + ) + + result = await wf.handle("đơn #DH12345 đâu rồi") + + assert result.action == "send" # degraded to a data-less draft, not a crash + assert result.order is None + assert "lookup:error" in result.trace + + +async def test_order_data_reaches_the_prompt(): + drafter = drafter_returning("ok ạ") + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.ORDER_STATUS), + drafter=drafter, + order_lookup=lookup_returning(ORDER), + ) + + await wf.handle("đơn #DH12345 đâu") + + assert "DH12345" in drafter.messages[-1]["content"] + + +async def test_knowledge_reaches_the_prompt(): + async def knowledge(intent): + return ["Đổi trả trong 7 ngày kể từ khi nhận hàng."] + + drafter = drafter_returning("ok ạ") + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.RETURN_EXCHANGE), + drafter=drafter, + knowledge_lookup=knowledge, + ) + + await wf.handle("em muốn đổi size") + + assert "Đổi trả trong 7 ngày" in drafter.messages[-1]["content"] + + +# ---- handoff paths ---- + + +async def test_complaint_always_goes_to_a_human(): + drafter = drafter_returning("shop hoàn tiền ngay cho anh/chị") + wf = EcomOpsWorkflow(classifier=StubClassifier(Intent.COMPLAINT), drafter=drafter) + + result = await wf.handle("hàng lỗi, tôi muốn khiếu nại") + + assert result.action == "handoff" + assert result.reply == COMPLAINT_ACK_REPLY + assert drafter.messages is None # never asked the model to improvise here + + +async def test_low_confidence_goes_to_a_human(): + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.OTHER, confidence=0.0), + drafter=drafter_returning("..."), + ) + + result = await wf.handle("???") + + assert result.action == "handoff" + assert result.handoff_reason == "low_confidence" + assert result.reply == HANDOFF_REPLY + + +async def test_missing_drafter_goes_to_a_human(): + wf = EcomOpsWorkflow(classifier=StubClassifier(Intent.ORDER_STATUS), drafter=None) + result = await wf.handle("đơn #DH12345 đâu") + assert result.action == "handoff" + assert result.handoff_reason == "no_drafter" + + +async def test_drafting_error_goes_to_a_human(): + async def failing_drafter(messages): + raise RuntimeError("llm down") + + wf = EcomOpsWorkflow(classifier=StubClassifier(Intent.ORDER_STATUS), drafter=failing_drafter) + + result = await wf.handle("đơn #DH12345 đâu") + + assert result.action == "handoff" + assert result.handoff_reason == "draft_error" + + +async def test_guardrail_blocked_draft_is_never_sent_to_the_customer(): + bad_draft = "Shop chắc chắn giao ngày mai và hoàn tiền 100% cho anh/chị ạ." + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.ORDER_STATUS), + drafter=drafter_returning(bad_draft), + guardrails=GuardrailEngine(), + ) + + result = await wf.handle("đơn #DH12345 bao giờ giao") + + assert result.action == "handoff" + assert result.handoff_reason == "guardrail_blocked" + assert result.reply == HANDOFF_REPLY + assert bad_draft not in result.reply + assert {"delivery_promise", "compensation_promise"} <= {v.rule for v in result.violations} + + +async def test_customer_own_phone_in_draft_is_not_treated_as_a_leak(): + draft = "Dạ shop giao tới số 0901234567 của anh/chị ạ." + wf = EcomOpsWorkflow(classifier=StubClassifier(Intent.ORDER_STATUS), drafter=drafter_returning(draft)) + + result = await wf.handle("đơn của em sdt 0901234567", customer={"phone": "0901234567"}) + + assert result.action == "send" + + +async def test_trace_records_the_pipeline_steps(): + wf = EcomOpsWorkflow( + classifier=StubClassifier(Intent.ORDER_STATUS), + drafter=drafter_returning("ok ạ"), + order_lookup=lookup_returning(ORDER), + ) + + result = await wf.handle("đơn #DH12345 đâu") + + assert any(t.startswith("classify:order_status") for t in result.trace) + assert "lookup:hit" in result.trace + assert "send" in result.trace + + +# ---- LLMDrafter ---- + + +@respx.mock +async def test_llm_drafter_calls_magic_gateway(): + route = respx.post("http://magic:8080/api/v1/llm/chat").mock( + return_value=httpx.Response(200, json={"content": " Dạ shop đã nhận đơn ạ. "}) + ) + + text = await LLMDrafter("http://magic:8080", api_key="k")([{"role": "user", "content": "hi"}]) + + assert text == "Dạ shop đã nhận đơn ạ." + assert route.calls.last.request.headers["Authorization"] == "Bearer k" + + +@respx.mock +async def test_llm_drafter_raises_on_gateway_error(): + respx.post("http://magic:8080/api/v1/llm/chat").mock(return_value=httpx.Response(502, json={})) + with pytest.raises(httpx.HTTPStatusError): + await LLMDrafter("http://magic:8080")([{"role": "user", "content": "hi"}]) diff --git a/packs/ecomops/text.py b/packs/ecomops/text.py new file mode 100644 index 0000000..612ce98 --- /dev/null +++ b/packs/ecomops/text.py @@ -0,0 +1,23 @@ +"""Vietnamese text normalization shared by the intent and guardrail engines. + +Vietnamese customers very often type without diacritics ("don hang cua toi +dau roi" instead of "đơn hàng của tôi đâu rồi"). Normalizing both the incoming +message and our keyword lists to a diacritics-free lowercase form lets a single +keyword list match either spelling, instead of maintaining two of everything. +""" + +import unicodedata + + +def strip_diacritics(text: str) -> str: + """Remove Vietnamese tone/vowel marks: 'đơn hàng' -> 'don hang'.""" + # đ/Đ are single codepoints (U+0111/U+0110) that do NOT decompose under NFD, + # so the combining-mark filter below can't reach them — map them by hand first. + text = text.replace("đ", "d").replace("Đ", "D") + decomposed = unicodedata.normalize("NFD", text) + return "".join(ch for ch in decomposed if unicodedata.category(ch) != "Mn") + + +def normalize(text: str) -> str: + """Lowercase + diacritics-free + whitespace-collapsed form used for matching.""" + return " ".join(strip_diacritics(text).lower().split()) diff --git a/packs/ecomops/workflow.py b/packs/ecomops/workflow.py new file mode 100644 index 0000000..946dab3 --- /dev/null +++ b/packs/ecomops/workflow.py @@ -0,0 +1,198 @@ +"""EcomOps workflow — the Tuần 2 pipeline from the Month-1 roadmap. + + Nhận tin → Classify Intent → Extract Order Info → Fetch Order Status + → Draft Response → Guardrail Check → Send hoặc Handoff + +Every external dependency (classifier, order lookup, drafting model, knowledge +lookup) is injected, so the pipeline itself has no network calls and no +platform knowledge — the same workflow runs over Zalo, Facebook, or a test +harness depending on what the caller wires in. +""" + +import inspect +import logging +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable + +import httpx + +from .extraction import ExtractedInfo, extract +from .guardrails import GuardrailContext, GuardrailEngine, GuardrailViolation +from .intents import Intent +from .prompts import build_messages + +logger = logging.getLogger("magic_packs.ecomops") + +# Intents a bot should never close out on its own. +DEFAULT_HANDOFF_INTENTS = frozenset({Intent.COMPLAINT}) + +# Safe, non-committal fallbacks used when we refuse to send a model draft. +HANDOFF_REPLY = ( + "Dạ shop đã ghi nhận thông tin của anh/chị. " + "Shop sẽ nhờ nhân viên kiểm tra và phản hồi lại anh/chị trong thời gian sớm nhất ạ." +) +COMPLAINT_ACK_REPLY = ( + "Dạ shop rất xin lỗi anh/chị về trải nghiệm vừa rồi. " + "Shop đã ghi nhận và sẽ chuyển nhân viên phụ trách kiểm tra, phản hồi lại anh/chị sớm nhất ạ." +) + +Drafter = Callable[[list[dict[str, str]]], Awaitable[str]] +OrderLookup = Callable[[str | None, str | None], Awaitable[dict | None]] +KnowledgeLookup = Callable[[Intent], Awaitable[list[str]]] + + +@dataclass +class WorkflowResult: + """Outcome of handling one customer message. + + `reply` is always safe to send as-is. `action` says whether a human still + needs to pick the conversation up afterwards ("handoff") or whether the bot + fully handled it ("send"). + """ + + action: str + intent: Intent + confidence: float + reply: str + extracted: ExtractedInfo = field(default_factory=ExtractedInfo) + order: dict | None = None + violations: list[GuardrailViolation] = field(default_factory=list) + handoff_reason: str | None = None + trace: list[str] = field(default_factory=list) + + +class EcomOpsWorkflow: + def __init__( + self, + classifier: Any, + drafter: Drafter | None = None, + order_lookup: OrderLookup | None = None, + knowledge_lookup: KnowledgeLookup | None = None, + guardrails: GuardrailEngine | None = None, + handoff_intents: frozenset[Intent] = DEFAULT_HANDOFF_INTENTS, + min_confidence: float = 0.5, + allowed_contacts: list[str] | None = None, + ): + self.classifier = classifier + self.drafter = drafter + self.order_lookup = order_lookup + self.knowledge_lookup = knowledge_lookup + self.guardrails = guardrails or GuardrailEngine() + self.handoff_intents = handoff_intents + self.min_confidence = min_confidence + self.allowed_contacts = allowed_contacts or [] + + async def handle(self, message: str, customer: dict | None = None) -> WorkflowResult: + customer = customer or {} + trace: list[str] = [] + + # 1. Classify — accept both sync (rule-based) and async (LLM/hybrid) classifiers. + intent_result = await _maybe_await(self.classifier.classify(message)) + trace.append(f"classify:{intent_result.intent.value}@{intent_result.confidence:.2f}") + + # 2. Extract order identifiers from the raw message. + extracted = extract(message) + if extracted.order_code or extracted.phone: + trace.append(f"extract:code={extracted.order_code},phone={bool(extracted.phone)}") + + # 3. Look the order up when we have something to look it up by. + order = None + if self.order_lookup and (extracted.order_code or extracted.phone): + try: + order = await self.order_lookup(extracted.order_code, extracted.phone) + trace.append("lookup:hit" if order else "lookup:miss") + except Exception as e: + # A flaky sheet/ERP shouldn't drop the customer's message — fall + # through to handoff with whatever we already know. + logger.warning("order lookup failed: %s", e) + trace.append("lookup:error") + + def _result(action: str, reply: str, reason: str | None, violations=None) -> WorkflowResult: + return WorkflowResult( + action=action, intent=intent_result.intent, confidence=intent_result.confidence, + reply=reply, extracted=extracted, order=order, + violations=violations or [], handoff_reason=reason, trace=trace, + ) + + # Intents we never let the bot close out (complaints) short-circuit here: + # acknowledge, then hand to a human. No model draft is involved, so + # there's nothing for it to over-promise. + if intent_result.intent in self.handoff_intents: + trace.append("handoff:intent") + return _result("handoff", COMPLAINT_ACK_REPLY, f"intent={intent_result.intent.value}") + + if intent_result.confidence < self.min_confidence: + trace.append("handoff:low_confidence") + return _result("handoff", HANDOFF_REPLY, "low_confidence") + + if self.drafter is None: + trace.append("handoff:no_drafter") + return _result("handoff", HANDOFF_REPLY, "no_drafter") + + # 4. Draft. + knowledge = None + if self.knowledge_lookup: + try: + knowledge = await self.knowledge_lookup(intent_result.intent) + except Exception as e: + logger.warning("knowledge lookup failed: %s", e) + + try: + draft = await self.drafter(build_messages(intent_result.intent, message, order, knowledge)) + except Exception as e: + logger.warning("drafting failed: %s", e) + trace.append("handoff:draft_error") + return _result("handoff", HANDOFF_REPLY, "draft_error") + + # 5. Guardrail the draft before it can reach the customer. + ctx = GuardrailContext( + customer_phone=customer.get("phone") or extracted.phone, + customer_email=customer.get("email"), + allowed_contacts=self.allowed_contacts, + ) + violations = self.guardrails.check(draft, ctx) + if GuardrailEngine.is_blocked(violations): + trace.append(f"handoff:guardrail({','.join(v.rule for v in violations)})") + return _result("handoff", HANDOFF_REPLY, "guardrail_blocked", violations) + + trace.append("send") + return _result("send", draft.strip(), None, violations) + + +async def _maybe_await(value): + return await value if inspect.isawaitable(value) else value + + +class LLMDrafter: + """Drafts replies through MagiC's LLM gateway (`POST /api/v1/llm/chat`). + + Routed through core rather than a provider SDK so token spend lands in the + Cost Controller alongside every other MagiC workload. + """ + + def __init__( + self, + magic_url: str, + api_key: str = "", + model: str = "", + timeout: float = 30.0, + max_tokens: int = 300, + strategy: str = "best", + ): + self.magic_url = magic_url.rstrip("/") + self.api_key = api_key + self.model = model + self.timeout = timeout + self.max_tokens = max_tokens + self.strategy = strategy + + async def __call__(self, messages: list[dict[str, str]]) -> str: + headers = {"Authorization": f"Bearer {self.api_key}"} if self.api_key else {} + payload: dict = {"messages": messages, "strategy": self.strategy, "max_tokens": self.max_tokens} + if self.model: + payload["model"] = self.model + + async with httpx.AsyncClient(timeout=self.timeout) as client: + resp = await client.post(f"{self.magic_url}/api/v1/llm/chat", headers=headers, json=payload) + resp.raise_for_status() + return (resp.json().get("content") or "").strip() diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..1da93b7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +# Root pytest config so `pytest connectors packs` works from the repo root +# (CI runs it that way). Each connector/pack also ships its own pytest.ini for +# running its suite standalone from inside its directory. +[pytest] +asyncio_mode = auto diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..c8c5116 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,9 @@ +# Lint settings for the Python code outside sdk/python (connectors/, packs/). +# sdk/python keeps its own [tool.ruff] in pyproject.toml — that config is closer +# to those files, so it wins there. Kept in sync with it so every Python file in +# the repo is held to the same rules. +target-version = "py311" +line-length = 135 + +[lint] +select = ["E", "F", "I"]