diff --git a/.vscode/settings.json b/.vscode/settings.json index 572cae7..069ac59 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,19 @@ { - "python-envs.defaultEnvManager": "ms-python.python:conda", - "python-envs.defaultPackageManager": "ms-python.python:conda", - "python-envs.pythonProjects": [ - { - "path": ".", - "envManager": "ms-python.python:conda", - "packageManager": "ms-python.python:conda" - } - ] + "python.analysis.extraPaths": [ + "apps/src", + "../../SAGE/src" + ], + "python.autoComplete.extraPaths": [ + "apps/src", + "../../SAGE/src" + ], + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [ + { + "path": ".", + "envManager": "ms-python.python:conda", + "packageManager": "ms-python.python:conda" + } + ] } diff --git a/README.md b/README.md index 457be8c..112536c 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ python examples/run_video_intelligence.py ## 📁 Repository Structure -``` +```text sage-examples/ ├── examples/ # 🎯 Production application examples │ ├── run_video_intelligence.py @@ -81,7 +81,21 @@ Complete, runnable applications demonstrating real-world use cases: | 🎫 **Ticket Triage** | Stateful customer support ticket classification, urgency scoring, and routing | `examples/run_ticket_triage.py` | | 🌐 **Ticket Triage API** | FastAPI service for triage ingestion, queue queries, and ticket lookup | `examples/run_ticket_triage_api.py` | -See `examples/README.md` for details. +The repository also includes the full 53-91 domain app suite across research, education, healthcare, +enterprise operations, public service, sustainability, and infrastructure monitoring. + +Representative newer entry points include: + +- `examples/run_patent_competition_monitor.py` +- `examples/run_campus_aid_gap_alert.py` +- `examples/run_lab_turnaround_alert.py` +- `examples/run_policy_search_helper.py` +- `examples/run_community_hotspot_drift.py` +- `examples/run_budget_variance_alert.py` +- `examples/run_carbon_collection.py` +- `examples/run_data_center_watch.py` + +See `examples/README.md` for the complete runnable script index. ## 📦 Installation @@ -107,7 +121,7 @@ pip install -e .[dev] SAGE uses a strict 6-layer architecture with unidirectional dependencies: -``` +```text ┌─────────────────────────────────────────────┐ │ L6: Interface │ CLI, Web UI, Tools ├─────────────────────────────────────────────┤ @@ -180,7 +194,7 @@ See `docs/DEVELOPMENT.md` for complete development guide. - **Development Guide**: `docs/DEVELOPMENT.md` - Contributing - **SAGE Tutorials**: [SAGE/tutorials](https://github.com/intellistream/SAGE/tree/main/tutorials) - Learn SAGE -- **SAGE Docs**: https://intellistream.github.io/SAGE +- **SAGE Docs**: [intellistream.github.io/SAGE](https://intellistream.github.io/SAGE) ## 🤝 Contributing @@ -188,7 +202,8 @@ We welcome contributions! Please see: 1. **Development Guide**: `docs/DEVELOPMENT.md` 1. **Code of Conduct**: Follow respectful collaboration -1. **Issue Tracker**: https://github.com/intellistream/sage-examples/issues +1. **Issue Tracker**: + [github.com/intellistream/sage-examples/issues](https://github.com/intellistream/sage-examples/issues) ### Adding Examples @@ -201,9 +216,10 @@ We welcome contributions! Please see: ## 🔗 Related Repositories -- **SAGE Main**: https://github.com/intellistream/SAGE -- **SAGE Benchmark**: https://github.com/intellistream/sage-benchmark -- **PyPI Packages**: https://pypi.org/search/?q=isage +- **SAGE Main**: [github.com/intellistream/SAGE](https://github.com/intellistream/SAGE) +- **SAGE Benchmark**: + [github.com/intellistream/sage-benchmark](https://github.com/intellistream/sage-benchmark) +- **PyPI Packages**: [pypi.org/search/?q=isage](https://pypi.org/search/?q=isage) ## 📄 License @@ -221,4 +237,4 @@ If you find this project helpful, please consider giving it a ⭐️! ______________________________________________________________________ -**Made with ❤️ by the IntelliStream Team** +Made with ❤️ by the IntelliStream Team. diff --git a/apps/src/sage/apps/_batch.py b/apps/src/sage/apps/_batch.py new file mode 100644 index 0000000..a4cebaa --- /dev/null +++ b/apps/src/sage/apps/_batch.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from typing import Any + +from sage.foundation import BatchFunction + + +class ListBatchSource(BatchFunction): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + self._batch_items: list[Any] | None = None + self._batch_index = 0 + + def load_items(self) -> list[Any]: + raise NotImplementedError + + def execute(self) -> Any | None: + if self._batch_items is None: + self._batch_items = list(self.load_items()) + self._batch_index = 0 + + if self._batch_index >= len(self._batch_items): + return None + + item = self._batch_items[self._batch_index] + self._batch_index += 1 + return item diff --git a/apps/src/sage/apps/academic_metadata/README.md b/apps/src/sage/apps/academic_metadata/README.md new file mode 100644 index 0000000..6528460 --- /dev/null +++ b/apps/src/sage/apps/academic_metadata/README.md @@ -0,0 +1,18 @@ +# Academic Metadata + +学术文献元数据抽取应用。 + +## 功能 + +- 从目录、单文件或 CSV 读取文献文本 +- 提取标题、作者、年份、DOI、邮箱与摘要 +- 标准化作者名称 +- 输出 JSON 结果 + +## 用法 + +```bash +python examples/run_academic_metadata.py \ + --input-path sample_papers \ + --output metadata.json +``` diff --git a/apps/src/sage/apps/academic_metadata/__init__.py b/apps/src/sage/apps/academic_metadata/__init__.py new file mode 100644 index 0000000..406d7ca --- /dev/null +++ b/apps/src/sage/apps/academic_metadata/__init__.py @@ -0,0 +1,5 @@ +"""Academic metadata extraction application.""" + +from .pipeline import run_academic_metadata_pipeline + +__all__ = ["run_academic_metadata_pipeline"] diff --git a/apps/src/sage/apps/academic_metadata/operators.py b/apps/src/sage/apps/academic_metadata/operators.py new file mode 100644 index 0000000..bf1fe26 --- /dev/null +++ b/apps/src/sage/apps/academic_metadata/operators.py @@ -0,0 +1,137 @@ +"""Operators for academic metadata extraction.""" + +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import CustomLogger, MapFunction, SinkFunction + + +class PdfSource(ListBatchSource): + def __init__(self, input_path: str, **kwargs): + super().__init__(**kwargs) + self.input_path = Path(input_path) + self._logger = CustomLogger("PdfSource") + + def load_items(self) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + if self.input_path.is_dir(): + for file_path in sorted(self.input_path.iterdir()): + if file_path.is_file() and file_path.suffix.lower() in {".txt", ".md"}: + items.append( + { + "source_file": str(file_path), + "text": file_path.read_text(encoding="utf-8"), + } + ) + elif self.input_path.suffix.lower() == ".csv": + with self.input_path.open("r", encoding="utf-8", newline="") as handle: + for row in csv.DictReader(handle): + items.append( + {"source_file": row.get("source_file", ""), "text": row.get("text", "")} + ) + else: + items.append( + { + "source_file": str(self.input_path), + "text": self.input_path.read_text(encoding="utf-8"), + } + ) + self.logger.info(f"Loaded {len(items)} academic documents") + return items + + +class TextExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + result = dict(item) + result["text"] = str(item.get("text", "")) + return result + + +class MetadataExtractor(MapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._logger = CustomLogger("MetadataExtractor") + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = item.get("text", "") + lines = [line.strip() for line in text.splitlines() if line.strip()] + title = lines[0] if lines else "" + doi_match = re.search(r"10\.\d{4,9}/[-._;()/:A-Z0-9]+", text, flags=re.IGNORECASE) + year_match = re.search(r"\b(19|20)\d{2}\b", text) + email_matches = re.findall(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", text) + author_line = "" + for line in lines[1:6]: + if "," in line or " and " in line.lower() or len(line.split()) <= 12: + author_line = line + break + authors = [part.strip() for part in re.split(r",| and |;", author_line) if part.strip()] + result = dict(item) + result.update( + { + "title": title, + "authors": authors, + "year": year_match.group(0) if year_match else "", + "doi": doi_match.group(0) if doi_match else "", + "emails": email_matches, + "abstract": self._extract_abstract(text), + } + ) + return result + + def _extract_abstract(self, text: str) -> str: + lowered = text.lower() + marker = lowered.find("abstract") + if marker == -1: + return "" + + snippet = text[marker:] + lines = [line.strip() for line in snippet.splitlines()] + abstract_lines: list[str] = [] + started = False + for line in lines: + if not started: + if line.lower().startswith("abstract"): + remainder = line.split(":", 1)[1].strip() if ":" in line else "" + if remainder: + abstract_lines.append(remainder) + started = True + continue + + lowered_line = line.lower() + if not line or lowered_line.startswith("keywords"): + break + abstract_lines.append(line) + + return " ".join(abstract_lines).strip()[:1200] + + +class AuthorNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + normalized = [] + for author in item.get("authors", []): + cleaned = " ".join(author.replace("*", " ").split()) + if cleaned: + normalized.append(cleaned.title()) + item["authors"] = normalized + item["has_complete_metadata"] = bool(item.get("title") and item.get("authors")) + return item + + +class MetadataSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + with open(self.output_file, "w", encoding="utf-8") as handle: + json.dump(self.items, handle, ensure_ascii=False, indent=2) diff --git a/apps/src/sage/apps/academic_metadata/pipeline.py b/apps/src/sage/apps/academic_metadata/pipeline.py new file mode 100644 index 0000000..d125393 --- /dev/null +++ b/apps/src/sage/apps/academic_metadata/pipeline.py @@ -0,0 +1,28 @@ +"""Academic metadata pipeline.""" + +from __future__ import annotations + +from sage.foundation import CustomLogger +from sage.runtime import LocalEnvironment + +from .operators import AuthorNormalizer, MetadataExtractor, MetadataSink, PdfSource, TextExtractor + + +def run_academic_metadata_pipeline( + input_path: str, + output_file: str, + verbose: bool = False, +) -> None: + logger = CustomLogger("AcademicMetadataPipeline") + if verbose: + logger.info(f"Starting academic metadata extraction: {input_path}") + + env = LocalEnvironment("academic_metadata") + ( + env.from_batch(PdfSource, input_path=input_path) + .map(TextExtractor) + .map(MetadataExtractor) + .map(AuthorNormalizer) + .sink(MetadataSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/api_log_analytics/README.md b/apps/src/sage/apps/api_log_analytics/README.md new file mode 100644 index 0000000..7a17b41 --- /dev/null +++ b/apps/src/sage/apps/api_log_analytics/README.md @@ -0,0 +1,5 @@ +# API Log Analytics + +读取 API 日志，解析状态码和时延并识别异常请求。 + +输入：CSV 或 JSON API 日志。 输出：包含指标提取和异常判断的 JSON 文件。 diff --git a/apps/src/sage/apps/api_log_analytics/__init__.py b/apps/src/sage/apps/api_log_analytics/__init__.py new file mode 100644 index 0000000..81b3654 --- /dev/null +++ b/apps/src/sage/apps/api_log_analytics/__init__.py @@ -0,0 +1,5 @@ +"""API log analytics application.""" + +from .pipeline import run_api_log_analytics_pipeline + +__all__ = ["run_api_log_analytics_pipeline"] diff --git a/apps/src/sage/apps/api_log_analytics/operators.py b/apps/src/sage/apps/api_log_analytics/operators.py new file mode 100644 index 0000000..70cd96e --- /dev/null +++ b/apps/src/sage/apps/api_log_analytics/operators.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class ApiLogSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class ApiLogParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + raw = str(item.get("log") or item.get("message") or "") + status_match = re.search(r"\b(\d{3})\b", raw) + latency_match = re.search(r"(\d+(?:\.\d+)?)ms", raw.lower()) + item["status_code"] = ( + int(status_match.group(1)) if status_match else int(item.get("status_code") or 200) + ) + item["latency_ms"] = ( + float(latency_match.group(1)) if latency_match else float(item.get("latency_ms") or 0) + ) + item["endpoint"] = item.get("endpoint") or item.get("path") or "/unknown" + return item + + +class ApiMetricExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + latency = float(item.get("latency_ms") or 0) + status_code = int(item.get("status_code") or 200) + item["api_metric"] = { + "endpoint": item.get("endpoint"), + "latency_bucket": "slow" if latency >= 800 else "normal", + "is_error": status_code >= 500, + } + return item + + +class ApiAnomalyDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + metric = item.get("api_metric", {}) + item["api_anomaly"] = bool(metric.get("is_error") or metric.get("latency_bucket") == "slow") + item["api_anomaly_reason"] = ( + "server_error" + if metric.get("is_error") + else "slow_request" + if metric.get("latency_bucket") == "slow" + else "normal" + ) + return item + + +class ApiAnalyticsSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/api_log_analytics/pipeline.py b/apps/src/sage/apps/api_log_analytics/pipeline.py new file mode 100644 index 0000000..637c10a --- /dev/null +++ b/apps/src/sage/apps/api_log_analytics/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ApiAnalyticsSink, + ApiAnomalyDetector, + ApiLogParser, + ApiLogSource, + ApiMetricExtractor, +) + + +def run_api_log_analytics_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("api_log_analytics") + ( + env.from_batch(ApiLogSource, input_file=input_file) + .map(ApiLogParser) + .map(ApiMetricExtractor) + .map(ApiAnomalyDetector) + .sink(ApiAnalyticsSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/arbitrage_detector/README.md b/apps/src/sage/apps/arbitrage_detector/README.md new file mode 100644 index 0000000..b9bbf2c --- /dev/null +++ b/apps/src/sage/apps/arbitrage_detector/README.md @@ -0,0 +1,3 @@ +# Arbitrage Detector + +实时汇率转换与套利检测应用。 diff --git a/apps/src/sage/apps/arbitrage_detector/__init__.py b/apps/src/sage/apps/arbitrage_detector/__init__.py new file mode 100644 index 0000000..5ab54e4 --- /dev/null +++ b/apps/src/sage/apps/arbitrage_detector/__init__.py @@ -0,0 +1,5 @@ +"""Arbitrage detector application.""" + +from .pipeline import run_arbitrage_detector_pipeline + +__all__ = ["run_arbitrage_detector_pipeline"] diff --git a/apps/src/sage/apps/arbitrage_detector/operators.py b/apps/src/sage/apps/arbitrage_detector/operators.py new file mode 100644 index 0000000..d80f59f --- /dev/null +++ b/apps/src/sage/apps/arbitrage_detector/operators.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class OrderSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class ExchangeRateFetcher(MapFunction): + def __init__(self, static_rates: dict[str, float] | None = None, **kwargs): + super().__init__(**kwargs) + self.static_rates = static_rates or {"USD/CNY": 7.2, "EUR/CNY": 7.8, "EUR/USD": 1.09} + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + pair = f"{item.get('base_currency', 'USD')}/{item.get('quote_currency', 'CNY')}" + item["rate"] = float(self.static_rates.get(pair, 1.0)) + return item + + +class ConversionCalculator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + amount = float(item.get("amount") or 0) + item["converted_amount"] = round(amount * float(item.get("rate", 1.0)), 4) + return item + + +class ArbitrageMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + market_rate = float(item.get("market_rate") or item.get("rate") or 1.0) + delta = abs(float(item.get("rate", 1.0)) - market_rate) + item["arbitrage_gap"] = round(delta, 6) + item["has_opportunity"] = delta >= 0.02 + return item + + +class ArbitrageSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/arbitrage_detector/pipeline.py b/apps/src/sage/apps/arbitrage_detector/pipeline.py new file mode 100644 index 0000000..63039b2 --- /dev/null +++ b/apps/src/sage/apps/arbitrage_detector/pipeline.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ArbitrageMatcher, + ArbitrageSink, + ConversionCalculator, + ExchangeRateFetcher, + OrderSource, +) + + +def run_arbitrage_detector_pipeline( + input_file: str, output_file: str, static_rates: dict[str, float] | None = None +) -> None: + env = LocalEnvironment("arbitrage_detector") + ( + env.from_batch(OrderSource, input_file=input_file) + .map(ExchangeRateFetcher, static_rates=static_rates) + .map(ConversionCalculator) + .map(ArbitrageMatcher) + .sink(ArbitrageSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/assignment_feedback/README.md b/apps/src/sage/apps/assignment_feedback/README.md new file mode 100644 index 0000000..06a10c2 --- /dev/null +++ b/apps/src/sage/apps/assignment_feedback/README.md @@ -0,0 +1,6 @@ +# 作业初稿反馈系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_assignment_feedback_pipeline` +- Entry script: `examples/run_assignment_feedback.py` diff --git a/apps/src/sage/apps/assignment_feedback/__init__.py b/apps/src/sage/apps/assignment_feedback/__init__.py new file mode 100644 index 0000000..1fe3b88 --- /dev/null +++ b/apps/src/sage/apps/assignment_feedback/__init__.py @@ -0,0 +1,5 @@ +"""作业初稿反馈系统 application.""" + +from .pipeline import run_assignment_feedback_pipeline + +__all__ = ["run_assignment_feedback_pipeline"] diff --git a/apps/src/sage/apps/assignment_feedback/operators.py b/apps/src/sage/apps/assignment_feedback/operators.py new file mode 100644 index 0000000..9e13ca3 --- /dev/null +++ b/apps/src/sage/apps/assignment_feedback/operators.py @@ -0,0 +1,130 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _tokenize(text: str) -> set[str]: + return set(re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", text.lower())) + + +class AssignmentDraftSource(ListBatchSource): + def __init__(self, draft_file: str, rubric_file: str, **kwargs): + super().__init__(**kwargs) + self.draft_file = draft_file + self.rubric_file = rubric_file + + def load_items(self) -> list[dict[str, Any]]: + drafts = _load_records(self.draft_file) + rubric = _load_records(self.rubric_file) + for item in drafts: + item["rubric"] = rubric + return drafts + + +class AssignmentSectionParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + draft_text = str( + payload.get("draft") or payload.get("content") or payload.get("text") or "" + ) + sections = [ + section.strip() for section in re.split(r"[\n]{2,}", draft_text) if section.strip() + ] + payload["section_count"] = len(sections) or 1 + payload["word_count"] = len(re.findall(r"\w+", draft_text)) + payload["draft_terms"] = sorted(_tokenize(draft_text)) + return payload + + +class RubricMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + draft_terms = set(payload.get("draft_terms") or []) + matched: list[dict[str, Any]] = [] + for criterion in payload.get("rubric") or []: + criterion_text = str(criterion.get("criterion") or criterion.get("text") or "") + overlap = sorted(draft_terms & _tokenize(criterion_text)) + matched.append( + { + "criterion": criterion_text, + "matched_terms": overlap, + "covered": bool(overlap), + } + ) + payload["rubric_matches"] = matched + payload["coverage_ratio"] = ( + sum(1 for item in matched if item["covered"]) / len(matched) if matched else 0.0 + ) + return payload + + +class FeedbackComposer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + suggestions: list[str] = [] + if payload.get("word_count", 0) < 120: + suggestions.append("补充论证细节，正文偏短。") + if payload.get("section_count", 0) < 3: + suggestions.append("建议增加更清晰的分段结构。") + uncovered = [ + entry["criterion"] + for entry in payload.get("rubric_matches", []) + if not entry["covered"] + ] + if uncovered: + suggestions.append(f"未明显覆盖评分点：{'; '.join(uncovered[:3])}。") + payload["feedback_level"] = "good" if payload.get("coverage_ratio", 0) >= 0.6 else "revise" + payload["feedback_comments"] = suggestions or ["结构完整，可进入教师复核。"] + return payload + + +class AssignmentFeedbackSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("rubric", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/assignment_feedback/pipeline.py b/apps/src/sage/apps/assignment_feedback/pipeline.py new file mode 100644 index 0000000..05da518 --- /dev/null +++ b/apps/src/sage/apps/assignment_feedback/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + AssignmentDraftSource, + AssignmentFeedbackSink, + AssignmentSectionParser, + FeedbackComposer, + RubricMatcher, +) + + +def run_assignment_feedback_pipeline(draft_file: str, rubric_file: str, output_file: str) -> None: + env = LocalEnvironment("assignment_feedback") + ( + env.from_batch(AssignmentDraftSource, draft_file=draft_file, rubric_file=rubric_file) + .map(AssignmentSectionParser) + .map(RubricMatcher) + .map(FeedbackComposer) + .sink(AssignmentFeedbackSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/attendance_alert/README.md b/apps/src/sage/apps/attendance_alert/README.md new file mode 100644 index 0000000..9f4a668 --- /dev/null +++ b/apps/src/sage/apps/attendance_alert/README.md @@ -0,0 +1,3 @@ +# Attendance Alert + +员工出勤异常预警应用。 diff --git a/apps/src/sage/apps/attendance_alert/__init__.py b/apps/src/sage/apps/attendance_alert/__init__.py new file mode 100644 index 0000000..510cbf8 --- /dev/null +++ b/apps/src/sage/apps/attendance_alert/__init__.py @@ -0,0 +1,5 @@ +"""Attendance alert application.""" + +from .pipeline import run_attendance_alert_pipeline + +__all__ = ["run_attendance_alert_pipeline"] diff --git a/apps/src/sage/apps/attendance_alert/operators.py b/apps/src/sage/apps/attendance_alert/operators.py new file mode 100644 index 0000000..9568a25 --- /dev/null +++ b/apps/src/sage/apps/attendance_alert/operators.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class AttendanceSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class AttendanceNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["late_minutes"] = int(float(item.get("late_minutes") or 0)) + item["absent_days"] = int(float(item.get("absent_days") or 0)) + return item + + +class ScheduleMatcher(MapFunction): + def __init__(self, schedule_file: str | None = None, **kwargs): + super().__init__(**kwargs) + self.schedule_file = schedule_file + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["scheduled_shift"] = str( + item.get("scheduled_shift") or item.get("shift") or "general" + ).lower() + item["actual_shift"] = str( + item.get("actual_shift") or item.get("clock_shift") or item["scheduled_shift"] + ).lower() + return AttendanceNormalizer().execute(item) + + +class AttendanceScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = item["late_minutes"] // 10 + item["absent_days"] * 2 + item["risk_score"] = score + item["alert_level"] = "high" if score >= 4 else "medium" if score >= 2 else "low" + return item + + +class AttendanceAnomalyDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + scored = AttendanceScorer().execute(item) + if scored.get("scheduled_shift") != scored.get("actual_shift"): + scored["risk_score"] += 1 + scored["alert_level"] = "high" if scored["risk_score"] >= 4 else "medium" + return scored + + +class AttendanceSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + +class AttendanceAlertSink(AttendanceSink): + pass diff --git a/apps/src/sage/apps/attendance_alert/pipeline.py b/apps/src/sage/apps/attendance_alert/pipeline.py new file mode 100644 index 0000000..9461cfa --- /dev/null +++ b/apps/src/sage/apps/attendance_alert/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + AttendanceAlertSink, + AttendanceAnomalyDetector, + AttendanceSource, + ScheduleMatcher, +) + + +def run_attendance_alert_pipeline( + clock_file: str, output_file: str, schedule_file: str | None = None +) -> None: + env = LocalEnvironment("attendance_alert") + ( + env.from_batch(AttendanceSource, input_file=clock_file) + .map(ScheduleMatcher, schedule_file=schedule_file) + .map(AttendanceAnomalyDetector) + .sink(AttendanceAlertSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/backup_sync/README.md b/apps/src/sage/apps/backup_sync/README.md new file mode 100644 index 0000000..f5d4b27 --- /dev/null +++ b/apps/src/sage/apps/backup_sync/README.md @@ -0,0 +1,5 @@ +# Backup Sync + +读取备份状态，识别增量变化并输出同步一致性报告。 + +输入：CSV 或 JSON 备份记录。 输出：包含同步计划和一致性状态的 JSON 文件。 diff --git a/apps/src/sage/apps/backup_sync/__init__.py b/apps/src/sage/apps/backup_sync/__init__.py new file mode 100644 index 0000000..fe56a69 --- /dev/null +++ b/apps/src/sage/apps/backup_sync/__init__.py @@ -0,0 +1,5 @@ +"""Backup sync application.""" + +from .pipeline import run_backup_sync_pipeline + +__all__ = ["run_backup_sync_pipeline"] diff --git a/apps/src/sage/apps/backup_sync/operators.py b/apps/src/sage/apps/backup_sync/operators.py new file mode 100644 index 0000000..5ab481a --- /dev/null +++ b/apps/src/sage/apps/backup_sync/operators.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class BackupSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class IncrementDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + current_version = int(item.get("current_version") or item.get("version") or 0) + last_synced_version = int(item.get("last_synced_version") or 0) + item["has_increment"] = current_version > last_synced_version + item["increment_size"] = max(current_version - last_synced_version, 0) + return item + + +class BackupDispatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + target = item.get("target") or item.get("backup_target") or "secondary-storage" + item["dispatch_plan"] = { + "target": target, + "mode": "incremental" if item.get("has_increment") else "skip", + } + return item + + +class ConsistencyChecker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + checksum = str(item.get("checksum") or "") + remote_checksum = str(item.get("remote_checksum") or checksum) + item["consistency_status"] = "consistent" if checksum == remote_checksum else "mismatch" + return item + + +class BackupReportSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/backup_sync/pipeline.py b/apps/src/sage/apps/backup_sync/pipeline.py new file mode 100644 index 0000000..566599b --- /dev/null +++ b/apps/src/sage/apps/backup_sync/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + BackupDispatcher, + BackupReportSink, + BackupSource, + ConsistencyChecker, + IncrementDetector, +) + + +def run_backup_sync_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("backup_sync") + ( + env.from_batch(BackupSource, input_file=input_file) + .map(IncrementDetector) + .map(BackupDispatcher) + .map(ConsistencyChecker) + .sink(BackupReportSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/benchmark_watch/README.md b/apps/src/sage/apps/benchmark_watch/README.md new file mode 100644 index 0000000..c43a319 --- /dev/null +++ b/apps/src/sage/apps/benchmark_watch/README.md @@ -0,0 +1,6 @@ +# 模型评测榜单波动监控系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_benchmark_watch_pipeline` +- Entry script: `examples/run_benchmark_watch.py` diff --git a/apps/src/sage/apps/benchmark_watch/__init__.py b/apps/src/sage/apps/benchmark_watch/__init__.py new file mode 100644 index 0000000..e492c76 --- /dev/null +++ b/apps/src/sage/apps/benchmark_watch/__init__.py @@ -0,0 +1,5 @@ +"""模型评测榜单波动监控系统 application.""" + +from .pipeline import run_benchmark_watch_pipeline + +__all__ = ["run_benchmark_watch_pipeline"] diff --git a/apps/src/sage/apps/benchmark_watch/operators.py b/apps/src/sage/apps/benchmark_watch/operators.py new file mode 100644 index 0000000..da62c28 --- /dev/null +++ b/apps/src/sage/apps/benchmark_watch/operators.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _parse_float(value: Any) -> float: + try: + return float(value) + except (TypeError, ValueError): + return 0.0 + + +class BenchmarkSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_file) + for item in items: + item.setdefault("source_path", self.input_file) + return items + + +class BenchmarkParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["benchmark_name"] = str( + payload.get("benchmark_name") or payload.get("suite") or "unknown" + ) + payload["model_name"] = str(payload.get("model_name") or payload.get("model") or "unknown") + payload["current_score"] = _parse_float( + payload.get("current_score") or payload.get("score") + ) + payload["previous_score"] = _parse_float( + payload.get("previous_score") or payload.get("last_score") + ) + payload["rank"] = int(_parse_float(payload.get("rank") or 0)) + return payload + + +class BenchmarkDiffDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + delta = round( + float(payload.get("current_score") or 0) - float(payload.get("previous_score") or 0), 4 + ) + payload["score_delta"] = delta + if delta >= 2: + change = "surge" + elif delta <= -2: + change = "drop" + else: + change = "stable" + payload["change_type"] = change + return payload + + +class BenchmarkTrendTagger(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + delta = float(payload.get("score_delta") or 0) + rank = int(payload.get("rank") or 0) + if delta <= -2 or rank >= 10: + alert = "needs_attention" + elif delta >= 2 and rank <= 3: + alert = "leaderboard_gain" + else: + alert = "monitor" + payload["trend_tag"] = alert + payload["brief"] = ( + f"{payload.get('model_name')} 在 {payload.get('benchmark_name')} 上分数变化 {delta:+.2f}，" + f"当前排名 {rank or 'N/A'}。" + ) + return payload + + +class BenchmarkWatchSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/benchmark_watch/pipeline.py b/apps/src/sage/apps/benchmark_watch/pipeline.py new file mode 100644 index 0000000..4f76c44 --- /dev/null +++ b/apps/src/sage/apps/benchmark_watch/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + BenchmarkDiffDetector, + BenchmarkParser, + BenchmarkSource, + BenchmarkTrendTagger, + BenchmarkWatchSink, +) + + +def run_benchmark_watch_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("benchmark_watch") + ( + env.from_batch(BenchmarkSource, input_file=input_file) + .map(BenchmarkParser) + .map(BenchmarkDiffDetector) + .map(BenchmarkTrendTagger) + .sink(BenchmarkWatchSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/brand_compliance_review/README.md b/apps/src/sage/apps/brand_compliance_review/README.md new file mode 100644 index 0000000..ffca052 --- /dev/null +++ b/apps/src/sage/apps/brand_compliance_review/README.md @@ -0,0 +1,6 @@ +# 品牌物料合规审核系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_brand_compliance_review_pipeline` +- Entry script: `examples/run_brand_compliance_review.py` diff --git a/apps/src/sage/apps/brand_compliance_review/__init__.py b/apps/src/sage/apps/brand_compliance_review/__init__.py new file mode 100644 index 0000000..46a743a --- /dev/null +++ b/apps/src/sage/apps/brand_compliance_review/__init__.py @@ -0,0 +1,5 @@ +"""品牌物料合规审核系统 application.""" + +from .pipeline import run_brand_compliance_review_pipeline + +__all__ = ["run_brand_compliance_review_pipeline"] diff --git a/apps/src/sage/apps/brand_compliance_review/operators.py b/apps/src/sage/apps/brand_compliance_review/operators.py new file mode 100644 index 0000000..1fd1f04 --- /dev/null +++ b/apps/src/sage/apps/brand_compliance_review/operators.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, str]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, str]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": str(item)} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +class BrandAssetSource(ListBatchSource): + def __init__(self, asset_dir: str, **kwargs): + super().__init__(**kwargs) + self.asset_dir = asset_dir + + def load_items(self) -> list[dict[str, str]]: + items = _load_records(self.asset_dir) + for item in items: + item.setdefault("app_slug", "brand_compliance_review") + item.setdefault("source_path", self.asset_dir) + return items + + +class BrandAssetParser(MapFunction): + def execute(self, item: dict[str, str]) -> dict[str, str]: + payload = dict(item) + payload["title"] = str(payload.get("title") or "") + payload["body"] = str(payload.get("body") or payload.get("text") or "") + payload["asset_version"] = str(payload.get("asset_version") or payload.get("version") or "") + payload["channel"] = str(payload.get("channel") or "generic") + return payload + + +class BrandRuleMatcher(MapFunction): + REQUIRED_TERMS = {"免责声明", "仅供参考", "official"} + + def execute(self, item: dict[str, str]) -> dict[str, str | list[str]]: + payload = dict(item) + text = f"{payload.get('title', '')} {payload.get('body', '')}".lower() + issues: list[str] = [] + if "limited time" in text and "terms apply" not in text: + issues.append("missing_campaign_disclaimer") + if payload.get("asset_version", "") == "": + issues.append("missing_asset_version") + if re.search(r"\bfree\b", text) and "条件" not in payload.get("body", ""): + issues.append("absolute_claim_without_condition") + if "brandx" not in text: + issues.append("missing_brand_name") + payload["compliance_issues"] = issues + payload["required_terms_present"] = [ + term for term in self.REQUIRED_TERMS if term.lower() in text + ] + return payload + + +class BrandRiskScorer(MapFunction): + def execute(self, item: dict[str, str | list[str]]) -> dict[str, str | list[str] | int]: + payload = dict(item) + issue_count = len(payload.get("compliance_issues") or []) + risk_level = "pass" + if issue_count >= 3: + risk_level = "high" + elif issue_count >= 1: + risk_level = "medium" + payload["review_priority"] = risk_level + payload["risk_score"] = issue_count * 25 + payload["review_summary"] = ( + f"Channel {payload.get('channel')} has {issue_count} compliance issues, " + f"review priority {risk_level}." + ) + return payload + + +class BrandReviewSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, object]] = [] + + def execute(self, item: dict[str, object]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/brand_compliance_review/pipeline.py b/apps/src/sage/apps/brand_compliance_review/pipeline.py new file mode 100644 index 0000000..334d6b4 --- /dev/null +++ b/apps/src/sage/apps/brand_compliance_review/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + BrandAssetParser, + BrandAssetSource, + BrandReviewSink, + BrandRiskScorer, + BrandRuleMatcher, +) + + +def run_brand_compliance_review_pipeline(asset_dir: str, output_file: str) -> None: + env = LocalEnvironment("brand_compliance_review") + ( + env.from_batch(BrandAssetSource, asset_dir=asset_dir) + .map(BrandAssetParser) + .map(BrandRuleMatcher) + .map(BrandRiskScorer) + .sink(BrandReviewSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/budget_variance_alert/README.md b/apps/src/sage/apps/budget_variance_alert/README.md new file mode 100644 index 0000000..f9372bd --- /dev/null +++ b/apps/src/sage/apps/budget_variance_alert/README.md @@ -0,0 +1,6 @@ +# 预算执行偏差预警系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_budget_variance_alert_pipeline` +- Entry script: `examples/run_budget_variance_alert.py` diff --git a/apps/src/sage/apps/budget_variance_alert/__init__.py b/apps/src/sage/apps/budget_variance_alert/__init__.py new file mode 100644 index 0000000..569bb42 --- /dev/null +++ b/apps/src/sage/apps/budget_variance_alert/__init__.py @@ -0,0 +1,5 @@ +"""预算执行偏差预警系统 application.""" + +from .pipeline import run_budget_variance_alert_pipeline + +__all__ = ["run_budget_variance_alert_pipeline"] diff --git a/apps/src/sage/apps/budget_variance_alert/operators.py b/apps/src/sage/apps/budget_variance_alert/operators.py new file mode 100644 index 0000000..e923d4c --- /dev/null +++ b/apps/src/sage/apps/budget_variance_alert/operators.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class BudgetPlanSource(ListBatchSource): + def __init__(self, plan_file: str, actual_file: str, **kwargs): + super().__init__(**kwargs) + self.plan_file = plan_file + self.actual_file = actual_file + + def load_items(self) -> list[dict[str, Any]]: + plans = _load_records(self.plan_file) + actuals = _load_records(self.actual_file) + actual_index = { + str(item.get("cost_center") or item.get("category") or ""): item for item in actuals + } + for item in plans: + key = str(item.get("cost_center") or item.get("category") or "") + item.setdefault("app_slug", "budget_variance_alert") + item["actual_record"] = actual_index.get(key, {}) + item.setdefault("source_path", self.plan_file) + return plans + + +class BudgetCategoryMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + actual = payload.get("actual_record") or {} + payload["cost_center"] = str( + payload.get("cost_center") or payload.get("category") or "unknown" + ) + payload["planned_amount"] = _to_float( + payload.get("planned_amount") or payload.get("budget") + ) + payload["actual_amount"] = _to_float(actual.get("actual_amount") or actual.get("spent")) + payload["progress_pct"] = _to_float( + actual.get("progress_pct") or payload.get("progress_pct") + ) + return payload + + +class BudgetVarianceDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + planned = payload.get("planned_amount", 0.0) + actual = payload.get("actual_amount", 0.0) + variance = actual - planned + ratio = round((actual / planned), 2) if planned else 0.0 + alerts: list[str] = [] + if variance > 0: + alerts.append("overspend") + if payload.get("progress_pct", 100.0) < 50 and ratio > 0.7: + alerts.append("spend_ahead_of_progress") + payload["variance_amount"] = round(variance, 2) + payload["variance_ratio"] = ratio + payload["alert_level"] = "critical" if ratio >= 1.2 else "watch" if alerts else "normal" + payload["variance_summary"] = ( + f"Cost center {payload.get('cost_center')} variance {payload.get('variance_amount')}, " + f"alert {payload.get('alert_level')} ({', '.join(alerts) or 'none'})." + ) + return payload + + +class BudgetAlertSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("actual_record", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/budget_variance_alert/pipeline.py b/apps/src/sage/apps/budget_variance_alert/pipeline.py new file mode 100644 index 0000000..416fc35 --- /dev/null +++ b/apps/src/sage/apps/budget_variance_alert/pipeline.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + BudgetAlertSink, + BudgetCategoryMapper, + BudgetPlanSource, + BudgetVarianceDetector, +) + + +def run_budget_variance_alert_pipeline(plan_file: str, actual_file: str, output_file: str) -> None: + env = LocalEnvironment("budget_variance_alert") + ( + env.from_batch(BudgetPlanSource, plan_file=plan_file, actual_file=actual_file) + .map(BudgetCategoryMapper) + .map(BudgetVarianceDetector) + .sink(BudgetAlertSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/campus_aid_gap_alert/README.md b/apps/src/sage/apps/campus_aid_gap_alert/README.md new file mode 100644 index 0000000..bd03324 --- /dev/null +++ b/apps/src/sage/apps/campus_aid_gap_alert/README.md @@ -0,0 +1,6 @@ +# 校园奖助申请缺口预警系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_campus_aid_gap_alert_pipeline` +- Entry script: `examples/run_campus_aid_gap_alert.py` diff --git a/apps/src/sage/apps/campus_aid_gap_alert/__init__.py b/apps/src/sage/apps/campus_aid_gap_alert/__init__.py new file mode 100644 index 0000000..b8ad5c1 --- /dev/null +++ b/apps/src/sage/apps/campus_aid_gap_alert/__init__.py @@ -0,0 +1,5 @@ +"""校园奖助申请缺口预警系统 application.""" + +from .pipeline import run_campus_aid_gap_alert_pipeline + +__all__ = ["run_campus_aid_gap_alert_pipeline"] diff --git a/apps/src/sage/apps/campus_aid_gap_alert/operators.py b/apps/src/sage/apps/campus_aid_gap_alert/operators.py new file mode 100644 index 0000000..72b9c98 --- /dev/null +++ b/apps/src/sage/apps/campus_aid_gap_alert/operators.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _as_list(value: Any) -> list[str]: + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return [part.strip() for part in re.split(r"[,;|]", str(value or "")) if part.strip()] + + +def _parse_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class AidApplicationSource(ListBatchSource): + def __init__(self, application_file: str, profile_file: str | None = None, **kwargs): + super().__init__(**kwargs) + self.application_file = application_file + self.profile_file = profile_file or "" + + def load_items(self) -> list[dict[str, Any]]: + applications = _load_records(self.application_file) + profiles = _load_records(self.profile_file) if self.profile_file else [] + profile_index = { + str( + profile.get("student_id") or profile.get("id") or profile.get("student") or "" + ): profile + for profile in profiles + } + for item in applications: + student_id = str(item.get("student_id") or item.get("id") or item.get("student") or "") + item.setdefault("app_slug", "campus_aid_gap_alert") + item["student_profile"] = profile_index.get(student_id, {}) + item.setdefault("source_path", self.application_file) + return applications + + +class AidRuleExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["required_docs"] = _as_list( + payload.get("required_docs") or payload.get("documents_required") + ) + payload["submitted_docs"] = _as_list( + payload.get("submitted_docs") or payload.get("submitted_materials") + ) + payload["min_gpa"] = _parse_float(payload.get("min_gpa"), 0.0) + payload["deadline_days"] = int(_parse_float(payload.get("deadline_days"), 999)) + payload["hardship_required"] = str(payload.get("aid_type") or "").lower() in { + "grant", + "subsidy", + } + return payload + + +class StudentEligibilityMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + profile = payload.get("student_profile") or {} + gpa = _parse_float(profile.get("gpa") or payload.get("gpa"), 0.0) + household_income = _parse_float(profile.get("household_income") or 0, 0.0) + discipline_flag = str(profile.get("disciplinary_action") or "").strip().lower() in { + "true", + "yes", + "1", + } + missing_docs = sorted( + set(payload.get("required_docs") or []) - set(payload.get("submitted_docs") or []) + ) + issues: list[str] = [] + if gpa < payload.get("min_gpa", 0.0): + issues.append("gpa_below_requirement") + if payload.get("hardship_required") and household_income > 40000: + issues.append("income_above_hardship_threshold") + if discipline_flag: + issues.append("disciplinary_record_present") + payload["student_id"] = str( + profile.get("student_id") or payload.get("student_id") or "unknown" + ) + payload["student_gpa"] = gpa + payload["missing_docs"] = missing_docs + payload["eligibility_issues"] = issues + return payload + + +class AidGapDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + score = ( + len(payload.get("missing_docs") or []) * 2 + + len(payload.get("eligibility_issues") or []) * 3 + ) + if int(payload.get("deadline_days") or 999) <= 7: + score += 3 + elif int(payload.get("deadline_days") or 999) <= 14: + score += 1 + alert_level = "watch" + if score >= 8: + alert_level = "critical" + elif score >= 4: + alert_level = "action_required" + payload["gap_score"] = score + payload["alert_level"] = alert_level + payload["alert_summary"] = ( + f"学生 {payload.get('student_id')} 缺少 {len(payload.get('missing_docs') or [])} 份材料，" + f"资格问题 {len(payload.get('eligibility_issues') or [])} 项，预警级别 {alert_level}。" + ) + return payload + + +class AidAlertSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("student_profile", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/campus_aid_gap_alert/pipeline.py b/apps/src/sage/apps/campus_aid_gap_alert/pipeline.py new file mode 100644 index 0000000..7f059a3 --- /dev/null +++ b/apps/src/sage/apps/campus_aid_gap_alert/pipeline.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + AidAlertSink, + AidApplicationSource, + AidGapDetector, + AidRuleExtractor, + StudentEligibilityMatcher, +) + + +def run_campus_aid_gap_alert_pipeline( + application_file: str, profile_file: str, output_file: str +) -> None: + env = LocalEnvironment("campus_aid_gap_alert") + ( + env.from_batch( + AidApplicationSource, application_file=application_file, profile_file=profile_file + ) + .map(AidRuleExtractor) + .map(StudentEligibilityMatcher) + .map(AidGapDetector) + .sink(AidAlertSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/campus_emission_report/README.md b/apps/src/sage/apps/campus_emission_report/README.md new file mode 100644 index 0000000..3312784 --- /dev/null +++ b/apps/src/sage/apps/campus_emission_report/README.md @@ -0,0 +1,6 @@ +# 校园碳排报告系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_campus_emission_report_pipeline` +- Entry script: `examples/run_campus_emission_report.py` diff --git a/apps/src/sage/apps/campus_emission_report/__init__.py b/apps/src/sage/apps/campus_emission_report/__init__.py new file mode 100644 index 0000000..940a1fc --- /dev/null +++ b/apps/src/sage/apps/campus_emission_report/__init__.py @@ -0,0 +1,5 @@ +"""校园碳排报告系统 application.""" + +from .pipeline import run_campus_emission_report_pipeline + +__all__ = ["run_campus_emission_report_pipeline"] diff --git a/apps/src/sage/apps/campus_emission_report/operators.py b/apps/src/sage/apps/campus_emission_report/operators.py new file mode 100644 index 0000000..2ca5b94 --- /dev/null +++ b/apps/src/sage/apps/campus_emission_report/operators.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class CampusEmissionSource(ListBatchSource): + def __init__(self, input_dir: str, **kwargs): + super().__init__(**kwargs) + self.input_dir = input_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_dir) + for item in items: + item.setdefault("app_slug", "campus_emission_report") + item.setdefault("source_path", self.input_dir) + return items + + +class EmissionFactorMapper(MapFunction): + FACTORS = {"electricity_kwh": 0.0007, "diesel_l": 0.0027, "bus_km": 0.00015} + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["campus"] = str(payload.get("campus") or "main") + payload["source_type"] = str(payload.get("source_type") or "electricity_kwh") + payload["amount"] = _to_float(payload.get("amount")) + factor = self.FACTORS.get(payload["source_type"], 0.001) + payload["emissions_tco2e"] = round(payload["amount"] * factor, 4) + return payload + + +class CampusEmissionAggregator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["missing_fields"] = [ + field + for field in ("campus", "source_type", "amount") + if payload.get(field) in {None, "", 0.0} + ] + payload["report_ready"] = not payload.get("missing_fields") + return payload + + +class CampusReportFormatter(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["report_summary"] = ( + f"Campus {payload.get('campus')} source {payload.get('source_type')} emissions {payload.get('emissions_tco2e')} tCO2e, " + f"ready {payload.get('report_ready')}." + ) + return payload + + +class CampusReportSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/campus_emission_report/pipeline.py b/apps/src/sage/apps/campus_emission_report/pipeline.py new file mode 100644 index 0000000..ffbbbb6 --- /dev/null +++ b/apps/src/sage/apps/campus_emission_report/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CampusEmissionAggregator, + CampusEmissionSource, + CampusReportFormatter, + CampusReportSink, + EmissionFactorMapper, +) + + +def run_campus_emission_report_pipeline(input_dir: str, output_file: str) -> None: + env = LocalEnvironment("campus_emission_report") + ( + env.from_batch(CampusEmissionSource, input_dir=input_dir) + .map(EmissionFactorMapper) + .map(CampusEmissionAggregator) + .map(CampusReportFormatter) + .sink(CampusReportSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/carbon_collection/README.md b/apps/src/sage/apps/carbon_collection/README.md new file mode 100644 index 0000000..45f5116 --- /dev/null +++ b/apps/src/sage/apps/carbon_collection/README.md @@ -0,0 +1,6 @@ +# 碳排数据采集归集系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_carbon_collection_pipeline` +- Entry script: `examples/run_carbon_collection.py` diff --git a/apps/src/sage/apps/carbon_collection/__init__.py b/apps/src/sage/apps/carbon_collection/__init__.py new file mode 100644 index 0000000..1255c1b --- /dev/null +++ b/apps/src/sage/apps/carbon_collection/__init__.py @@ -0,0 +1,5 @@ +"""碳排数据采集归集系统 application.""" + +from .pipeline import run_carbon_collection_pipeline + +__all__ = ["run_carbon_collection_pipeline"] diff --git a/apps/src/sage/apps/carbon_collection/operators.py b/apps/src/sage/apps/carbon_collection/operators.py new file mode 100644 index 0000000..2e1103b --- /dev/null +++ b/apps/src/sage/apps/carbon_collection/operators.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class CarbonDataSource(ListBatchSource): + def __init__(self, input_dir: str, **kwargs): + super().__init__(**kwargs) + self.input_dir = input_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_dir) + for item in items: + item.setdefault("app_slug", "carbon_collection") + item.setdefault("source_path", self.input_dir) + return items + + +class CarbonFieldExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["activity_type"] = str( + payload.get("activity_type") or payload.get("source_type") or "energy" + ) + payload["amount"] = _to_float(payload.get("amount") or payload.get("value")) + payload["unit"] = str(payload.get("unit") or "kwh") + return payload + + +class CarbonUnitNormalizer(MapFunction): + FACTORS = {"kwh": 0.0007, "km": 0.0002, "kg": 0.001} + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + factor = self.FACTORS.get(str(payload.get("unit") or "").lower(), 0.001) + payload["emissions_tco2e"] = round(payload.get("amount", 0.0) * factor, 4) + return payload + + +class CarbonLedgerBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["ledger_status"] = "missing_data" if payload.get("amount", 0.0) == 0 else "ready" + payload["ledger_summary"] = ( + f"Activity {payload.get('activity_type')} emitted {payload.get('emissions_tco2e')} tCO2e, status {payload.get('ledger_status')}." + ) + return payload + + +class CarbonCollectionSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/carbon_collection/pipeline.py b/apps/src/sage/apps/carbon_collection/pipeline.py new file mode 100644 index 0000000..19d68af --- /dev/null +++ b/apps/src/sage/apps/carbon_collection/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CarbonCollectionSink, + CarbonDataSource, + CarbonFieldExtractor, + CarbonLedgerBuilder, + CarbonUnitNormalizer, +) + + +def run_carbon_collection_pipeline(input_dir: str, output_file: str) -> None: + env = LocalEnvironment("carbon_collection") + ( + env.from_batch(CarbonDataSource, input_dir=input_dir) + .map(CarbonFieldExtractor) + .map(CarbonUnitNormalizer) + .map(CarbonLedgerBuilder) + .sink(CarbonCollectionSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/cashflow_watch/README.md b/apps/src/sage/apps/cashflow_watch/README.md new file mode 100644 index 0000000..02d9644 --- /dev/null +++ b/apps/src/sage/apps/cashflow_watch/README.md @@ -0,0 +1,6 @@ +# 企业现金流预测系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_cashflow_watch_pipeline` +- Entry script: `examples/run_cashflow_watch.py` diff --git a/apps/src/sage/apps/cashflow_watch/__init__.py b/apps/src/sage/apps/cashflow_watch/__init__.py new file mode 100644 index 0000000..4abb4ee --- /dev/null +++ b/apps/src/sage/apps/cashflow_watch/__init__.py @@ -0,0 +1,5 @@ +"""企业现金流预测系统 application.""" + +from .pipeline import run_cashflow_watch_pipeline + +__all__ = ["run_cashflow_watch_pipeline"] diff --git a/apps/src/sage/apps/cashflow_watch/operators.py b/apps/src/sage/apps/cashflow_watch/operators.py new file mode 100644 index 0000000..87b8aea --- /dev/null +++ b/apps/src/sage/apps/cashflow_watch/operators.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class CashflowSource(ListBatchSource): + def __init__(self, input_dir: str, **kwargs): + super().__init__(**kwargs) + self.input_dir = input_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_dir) + for item in items: + item.setdefault("app_slug", "cashflow_watch") + item.setdefault("source_path", self.input_dir) + return items + + +class CashflowFeatureBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["cash_in"] = _to_float(payload.get("cash_in") or payload.get("receivable")) + payload["cash_out"] = _to_float(payload.get("cash_out") or payload.get("payable")) + payload["opening_balance"] = _to_float(payload.get("opening_balance"), 0.0) + return payload + + +class CashflowForecaster(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + projected = ( + payload.get("opening_balance", 0.0) + + payload.get("cash_in", 0.0) + - payload.get("cash_out", 0.0) + ) + payload["projected_balance"] = round(projected, 2) + payload["net_flow"] = round(payload.get("cash_in", 0.0) - payload.get("cash_out", 0.0), 2) + return payload + + +class CashflowRiskMarker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + level = "normal" + if payload.get("projected_balance", 0.0) < 0: + level = "critical" + elif payload.get("projected_balance", 0.0) < 10000: + level = "watch" + payload["risk_level"] = level + payload["risk_summary"] = ( + f"Projected balance {payload.get('projected_balance')}, net flow {payload.get('net_flow')}, risk {level}." + ) + return payload + + +class CashflowSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/cashflow_watch/pipeline.py b/apps/src/sage/apps/cashflow_watch/pipeline.py new file mode 100644 index 0000000..5b18c3d --- /dev/null +++ b/apps/src/sage/apps/cashflow_watch/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CashflowFeatureBuilder, + CashflowForecaster, + CashflowRiskMarker, + CashflowSink, + CashflowSource, +) + + +def run_cashflow_watch_pipeline(input_dir: str, output_file: str) -> None: + env = LocalEnvironment("cashflow_watch") + ( + env.from_batch(CashflowSource, input_dir=input_dir) + .map(CashflowFeatureBuilder) + .map(CashflowForecaster) + .map(CashflowRiskMarker) + .sink(CashflowSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/cold_chain_watch/README.md b/apps/src/sage/apps/cold_chain_watch/README.md new file mode 100644 index 0000000..37a925f --- /dev/null +++ b/apps/src/sage/apps/cold_chain_watch/README.md @@ -0,0 +1,6 @@ +# 冷链运输越界监控系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_cold_chain_watch_pipeline` +- Entry script: `examples/run_cold_chain_watch.py` diff --git a/apps/src/sage/apps/cold_chain_watch/__init__.py b/apps/src/sage/apps/cold_chain_watch/__init__.py new file mode 100644 index 0000000..da2bb0c --- /dev/null +++ b/apps/src/sage/apps/cold_chain_watch/__init__.py @@ -0,0 +1,5 @@ +"""冷链运输越界监控系统 application.""" + +from .pipeline import run_cold_chain_watch_pipeline + +__all__ = ["run_cold_chain_watch_pipeline"] diff --git a/apps/src/sage/apps/cold_chain_watch/operators.py b/apps/src/sage/apps/cold_chain_watch/operators.py new file mode 100644 index 0000000..c7eeae8 --- /dev/null +++ b/apps/src/sage/apps/cold_chain_watch/operators.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class ColdChainRecordSource(ListBatchSource): + def __init__(self, record_file: str, **kwargs): + super().__init__(**kwargs) + self.record_file = record_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.record_file) + for item in items: + item.setdefault("app_slug", "cold_chain_watch") + item.setdefault("source_path", self.record_file) + return items + + +class ColdChainBatchMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["batch_id"] = str(payload.get("batch_id") or payload.get("lot") or "unknown") + payload["vehicle_id"] = str(payload.get("vehicle_id") or payload.get("truck") or "unknown") + payload["temperature_c"] = _to_float(payload.get("temperature_c")) + payload["min_temp_c"] = _to_float(payload.get("min_temp_c"), 2.0) + payload["max_temp_c"] = _to_float(payload.get("max_temp_c"), 8.0) + payload["minutes_out_of_range"] = _to_float(payload.get("minutes_out_of_range")) + return payload + + +class TemperatureExcursionDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + excursion_flags: list[str] = [] + if payload.get("temperature_c", 0.0) < payload.get("min_temp_c", 2.0): + excursion_flags.append("below_range") + if payload.get("temperature_c", 0.0) > payload.get("max_temp_c", 8.0): + excursion_flags.append("above_range") + if payload.get("minutes_out_of_range", 0.0) > 30: + excursion_flags.append("prolonged_excursion") + payload["excursion_flags"] = excursion_flags + payload["excursion_detected"] = bool(excursion_flags) + return payload + + +class ColdChainRiskScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + score = len(payload.get("excursion_flags") or []) * 3 + if payload.get("minutes_out_of_range", 0.0) > 60: + score += 2 + risk_level = "normal" + if score >= 6: + risk_level = "critical" + elif score >= 3: + risk_level = "watch" + payload["risk_level"] = risk_level + payload["risk_score"] = score + payload["risk_summary"] = ( + f"Batch {payload.get('batch_id')} on vehicle {payload.get('vehicle_id')} risk {risk_level}, " + f"flags {', '.join(payload.get('excursion_flags') or []) or 'none'}." + ) + return payload + + +class ColdChainSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/cold_chain_watch/pipeline.py b/apps/src/sage/apps/cold_chain_watch/pipeline.py new file mode 100644 index 0000000..8fde47c --- /dev/null +++ b/apps/src/sage/apps/cold_chain_watch/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ColdChainBatchMatcher, + ColdChainRecordSource, + ColdChainRiskScorer, + ColdChainSink, + TemperatureExcursionDetector, +) + + +def run_cold_chain_watch_pipeline(record_file: str, output_file: str) -> None: + env = LocalEnvironment("cold_chain_watch") + ( + env.from_batch(ColdChainRecordSource, record_file=record_file) + .map(ColdChainBatchMatcher) + .map(TemperatureExcursionDetector) + .map(ColdChainRiskScorer) + .sink(ColdChainSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/community_hotspot_drift/README.md b/apps/src/sage/apps/community_hotspot_drift/README.md new file mode 100644 index 0000000..b797990 --- /dev/null +++ b/apps/src/sage/apps/community_hotspot_drift/README.md @@ -0,0 +1,6 @@ +# 社区民生热点漂移监测系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_community_hotspot_drift_pipeline` +- Entry script: `examples/run_community_hotspot_drift.py` diff --git a/apps/src/sage/apps/community_hotspot_drift/__init__.py b/apps/src/sage/apps/community_hotspot_drift/__init__.py new file mode 100644 index 0000000..1425403 --- /dev/null +++ b/apps/src/sage/apps/community_hotspot_drift/__init__.py @@ -0,0 +1,5 @@ +"""社区民生热点漂移监测系统 application.""" + +from .pipeline import run_community_hotspot_drift_pipeline + +__all__ = ["run_community_hotspot_drift_pipeline"] diff --git a/apps/src/sage/apps/community_hotspot_drift/operators.py b/apps/src/sage/apps/community_hotspot_drift/operators.py new file mode 100644 index 0000000..d5c9c32 --- /dev/null +++ b/apps/src/sage/apps/community_hotspot_drift/operators.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +class CommunityEventSource(ListBatchSource): + def __init__(self, event_file: str, **kwargs): + super().__init__(**kwargs) + self.event_file = event_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.event_file) + for item in items: + item.setdefault("app_slug", "community_hotspot_drift") + item.setdefault("source_path", self.event_file) + return items + + +class CommunityZoneMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["zone"] = str(payload.get("zone") or payload.get("district") or "unknown") + payload["current_count"] = int( + float(payload.get("current_count") or payload.get("count_current") or 0) + ) + payload["previous_count"] = int( + float(payload.get("previous_count") or payload.get("count_previous") or 0) + ) + payload["issue_type"] = str(payload.get("issue_type") or payload.get("topic") or "general") + return payload + + +class CommunityTopicAggregator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + current_count = int(payload.get("current_count") or 0) + previous_count = int(payload.get("previous_count") or 0) + delta = current_count - previous_count + payload["topic_delta"] = delta + payload["drift_ratio"] = ( + round((current_count / max(previous_count, 1)), 2) if current_count else 0.0 + ) + return payload + + +class HotspotDriftDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + delta = int(payload.get("topic_delta") or 0) + ratio = float(payload.get("drift_ratio") or 0.0) + drift_status = "stable" + if delta >= 10 or ratio >= 2.0: + drift_status = "significant_shift" + elif delta >= 5 or ratio >= 1.3: + drift_status = "emerging_shift" + payload["drift_status"] = drift_status + payload["governance_priority"] = ( + "high" + if drift_status == "significant_shift" + else "medium" + if drift_status == "emerging_shift" + else "normal" + ) + payload["insight_summary"] = ( + f"Zone {payload.get('zone')} issue {payload.get('issue_type')} drift {drift_status}, " + f"priority {payload.get('governance_priority')}." + ) + return payload + + +class CommunityInsightSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/community_hotspot_drift/pipeline.py b/apps/src/sage/apps/community_hotspot_drift/pipeline.py new file mode 100644 index 0000000..6532f64 --- /dev/null +++ b/apps/src/sage/apps/community_hotspot_drift/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CommunityEventSource, + CommunityInsightSink, + CommunityTopicAggregator, + CommunityZoneMapper, + HotspotDriftDetector, +) + + +def run_community_hotspot_drift_pipeline(event_file: str, output_file: str) -> None: + env = LocalEnvironment("community_hotspot_drift") + ( + env.from_batch(CommunityEventSource, event_file=event_file) + .map(CommunityZoneMapper) + .map(CommunityTopicAggregator) + .map(HotspotDriftDetector) + .sink(CommunityInsightSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/company_credit/README.md b/apps/src/sage/apps/company_credit/README.md new file mode 100644 index 0000000..45fc299 --- /dev/null +++ b/apps/src/sage/apps/company_credit/README.md @@ -0,0 +1,3 @@ +# Company Credit + +读取企业名单，抽取风险因子并输出信用评分报告。 diff --git a/apps/src/sage/apps/company_credit/__init__.py b/apps/src/sage/apps/company_credit/__init__.py new file mode 100644 index 0000000..4668a02 --- /dev/null +++ b/apps/src/sage/apps/company_credit/__init__.py @@ -0,0 +1,5 @@ +"""Company credit application.""" + +from .pipeline import run_company_credit_pipeline + +__all__ = ["run_company_credit_pipeline"] diff --git a/apps/src/sage/apps/company_credit/operators.py b/apps/src/sage/apps/company_credit/operators.py new file mode 100644 index 0000000..dec20ec --- /dev/null +++ b/apps/src/sage/apps/company_credit/operators.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class CompanySource(ListBatchSource): + def __init__(self, company_file: str, **kwargs): + super().__init__(**kwargs) + self.company_file = company_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.company_file, encoding="utf-8", newline="") as handle: + if self.company_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class CompanyInfoFetcher(MapFunction): + def __init__(self, api_config: str | None = None, **kwargs): + super().__init__(**kwargs) + self.api_config = api_config + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["registered_years"] = int(float(item.get("registered_years") or 0)) + item["lawsuit_count"] = int(float(item.get("lawsuit_count") or 0)) + item["tax_score"] = float(item.get("tax_score") or 0) + item["api_config"] = self.api_config or "local_rules" + return item + + +class RiskFactorExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["risk_factor_count"] = int(item.get("lawsuit_count", 0)) + ( + 1 if float(item.get("tax_score", 0)) < 60 else 0 + ) + return item + + +class CreditScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = ( + min(int(item.get("registered_years", 0)), 10) + + float(item.get("tax_score", 0)) / 10 + - int(item.get("risk_factor_count", 0)) * 2 + ) + item["credit_score"] = round(score, 2) + item["credit_level"] = "A" if score >= 12 else "B" if score >= 8 else "C" + return item + + +class CreditReportSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/company_credit/pipeline.py b/apps/src/sage/apps/company_credit/pipeline.py new file mode 100644 index 0000000..c25de61 --- /dev/null +++ b/apps/src/sage/apps/company_credit/pipeline.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CompanyInfoFetcher, + CompanySource, + CreditReportSink, + CreditScorer, + RiskFactorExtractor, +) + + +def run_company_credit_pipeline( + company_file: str, output_file: str, api_config: str | None = None +) -> None: + env = LocalEnvironment("company_credit") + ( + env.from_batch(CompanySource, company_file=company_file) + .map(CompanyInfoFetcher, api_config=api_config) + .map(RiskFactorExtractor) + .map(CreditScorer) + .sink(CreditReportSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/compliance_doc_manager/README.md b/apps/src/sage/apps/compliance_doc_manager/README.md new file mode 100644 index 0000000..fb86009 --- /dev/null +++ b/apps/src/sage/apps/compliance_doc_manager/README.md @@ -0,0 +1,5 @@ +# Compliance Doc Manager + +读取合规文档，分类文档类型并检查复核截止时间。 + +输入：CSV 或 JSON 合规文档记录。 输出：包含文档分类和复核提醒状态的 JSON 文件。 diff --git a/apps/src/sage/apps/compliance_doc_manager/__init__.py b/apps/src/sage/apps/compliance_doc_manager/__init__.py new file mode 100644 index 0000000..f8d5d12 --- /dev/null +++ b/apps/src/sage/apps/compliance_doc_manager/__init__.py @@ -0,0 +1,5 @@ +"""Compliance doc manager application.""" + +from .pipeline import run_compliance_doc_manager_pipeline + +__all__ = ["run_compliance_doc_manager_pipeline"] diff --git a/apps/src/sage/apps/compliance_doc_manager/operators.py b/apps/src/sage/apps/compliance_doc_manager/operators.py new file mode 100644 index 0000000..0adcd81 --- /dev/null +++ b/apps/src/sage/apps/compliance_doc_manager/operators.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import csv +import json +from datetime import date, datetime +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +def _parse_date(value: str | None) -> date | None: + if not value: + return None + for fmt in ("%Y-%m-%d", "%Y/%m/%d"): + try: + return datetime.strptime(value, fmt).date() + except ValueError: + continue + return None + + +class ComplianceDocSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class ComplianceDocClassifier(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = " ".join( + str(item.get(field, "")) for field in ("title", "content", "doc_type") + ).lower() + if "audit" in text or "control" in text: + label = "audit" + elif "policy" in text or "procedure" in text: + label = "policy" + else: + label = "general" + item["compliance_category"] = label + return item + + +class ReviewDeadlineChecker(MapFunction): + def __init__(self, reference_date: str | None = None, **kwargs): + super().__init__(**kwargs) + self.reference_date = _parse_date(reference_date) if reference_date else date.today() + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + deadline = _parse_date(str(item.get("review_deadline") or item.get("deadline") or "")) + days_left = (deadline - self.reference_date).days if deadline else None + item["days_to_review"] = days_left + item["review_status"] = ( + "overdue" + if days_left is not None and days_left < 0 + else "due_soon" + if days_left is not None and days_left <= 7 + else "scheduled" + ) + return item + + +class ComplianceReminderSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/compliance_doc_manager/pipeline.py b/apps/src/sage/apps/compliance_doc_manager/pipeline.py new file mode 100644 index 0000000..8fbc615 --- /dev/null +++ b/apps/src/sage/apps/compliance_doc_manager/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ComplianceDocClassifier, + ComplianceDocSource, + ComplianceReminderSink, + ReviewDeadlineChecker, +) + + +def run_compliance_doc_manager_pipeline( + input_file: str, output_file: str, reference_date: str | None = None +) -> None: + env = LocalEnvironment("compliance_doc_manager") + ( + env.from_batch(ComplianceDocSource, input_file=input_file) + .map(ComplianceDocClassifier) + .map(ReviewDeadlineChecker, reference_date=reference_date) + .sink(ComplianceReminderSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/content_moderation/README.md b/apps/src/sage/apps/content_moderation/README.md new file mode 100644 index 0000000..d958abf --- /dev/null +++ b/apps/src/sage/apps/content_moderation/README.md @@ -0,0 +1,10 @@ +# Content Moderation + +社交媒体内容审核应用。 + +## 功能 + +- 读取文本内容 +- 识别敏感词 +- 计算违规分数 +- 输出允许、复核或拦截建议 diff --git a/apps/src/sage/apps/content_moderation/__init__.py b/apps/src/sage/apps/content_moderation/__init__.py new file mode 100644 index 0000000..dded472 --- /dev/null +++ b/apps/src/sage/apps/content_moderation/__init__.py @@ -0,0 +1,5 @@ +"""Content moderation application.""" + +from .pipeline import run_content_moderation_pipeline + +__all__ = ["run_content_moderation_pipeline"] diff --git a/apps/src/sage/apps/content_moderation/operators.py b/apps/src/sage/apps/content_moderation/operators.py new file mode 100644 index 0000000..442a0eb --- /dev/null +++ b/apps/src/sage/apps/content_moderation/operators.py @@ -0,0 +1,93 @@ +"""Operators for text moderation.""" + +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +class ContentSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + if self.input_file.lower().endswith(".csv"): + return list(csv.DictReader(handle)) + return [ + {"content_id": str(index + 1), "text": line.strip()} + for index, line in enumerate(handle) + if line.strip() + ] + + +class TextExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + cleaned = re.sub(r"\s+", " ", str(item.get("text", ""))).strip() + item["clean_text"] = cleaned + return item + + +class Tokenizer(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + enriched = dict(item) + enriched["tokens"] = re.findall(r"[\w\u4e00-\u9fff]+", item.get("clean_text", "").lower()) + return [enriched] if enriched["tokens"] else [] + + +class SensitiveFilter(MapFunction): + def __init__(self, sensitive_words: list[str] | None = None, **kwargs): + super().__init__(**kwargs) + self.sensitive_words = sensitive_words or [ + "scam", + "fraud", + "hate", + "violence", + "诈骗", + "仇恨", + "暴力", + ] + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = item.get("clean_text", "").lower() + token_set = set(item.get("tokens", [])) + matches = [ + word + for word in self.sensitive_words + if word.lower() in text or word.lower() in token_set + ] + item["matched_terms"] = matches + return item + + +class ViolationScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + matches = item.get("matched_terms", []) + score = len(matches) + item["violation_score"] = score + item["action"] = "block" if score >= 2 else "review" if score == 1 else "allow" + return item + + +class ModerationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/content_moderation/pipeline.py b/apps/src/sage/apps/content_moderation/pipeline.py new file mode 100644 index 0000000..b2ebd0f --- /dev/null +++ b/apps/src/sage/apps/content_moderation/pipeline.py @@ -0,0 +1,29 @@ +"""Content moderation pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ContentSource, + ModerationSink, + SensitiveFilter, + TextExtractor, + Tokenizer, + ViolationScorer, +) + + +def run_content_moderation_pipeline( + input_file: str, output_file: str, sensitive_words: list[str] | None = None +) -> None: + env = LocalEnvironment("content_moderation") + ( + env.from_batch(ContentSource, input_file=input_file) + .map(TextExtractor) + .flatmap(Tokenizer) + .map(SensitiveFilter, sensitive_words=sensitive_words) + .map(ViolationScorer) + .sink(ModerationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/content_scheduler/README.md b/apps/src/sage/apps/content_scheduler/README.md new file mode 100644 index 0000000..f02c824 --- /dev/null +++ b/apps/src/sage/apps/content_scheduler/README.md @@ -0,0 +1,6 @@ +# 多渠道内容排期系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_content_scheduler_pipeline` +- Entry script: `examples/run_content_scheduler.py` diff --git a/apps/src/sage/apps/content_scheduler/__init__.py b/apps/src/sage/apps/content_scheduler/__init__.py new file mode 100644 index 0000000..be056ed --- /dev/null +++ b/apps/src/sage/apps/content_scheduler/__init__.py @@ -0,0 +1,5 @@ +"""多渠道内容排期系统 application.""" + +from .pipeline import run_content_scheduler_pipeline + +__all__ = ["run_content_scheduler_pipeline"] diff --git a/apps/src/sage/apps/content_scheduler/operators.py b/apps/src/sage/apps/content_scheduler/operators.py new file mode 100644 index 0000000..5caf17e --- /dev/null +++ b/apps/src/sage/apps/content_scheduler/operators.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +class CampaignPlanSource(ListBatchSource): + def __init__(self, plan_file: str, channel_file: str, **kwargs): + super().__init__(**kwargs) + self.plan_file = plan_file + self.channel_file = channel_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.plan_file) + channels = _load_records(self.channel_file) + channel_index = {str(item.get("channel") or ""): item for item in channels} + for item in items: + channel = str(item.get("channel") or "") + item.setdefault("app_slug", "content_scheduler") + item["channel_rules"] = channel_index.get(channel, {}) + item.setdefault("source_path", self.plan_file) + return items + + +class ChannelRuleMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + rules = payload.get("channel_rules") or {} + payload["channel"] = str(payload.get("channel") or rules.get("channel") or "general") + payload["max_posts_per_week"] = int(rules.get("max_posts_per_week") or 7) + payload["blackout_dates"] = str(rules.get("blackout_dates") or "") + payload["preferred_format"] = str(rules.get("preferred_format") or "post") + return payload + + +class TopicAllocator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + theme = str(payload.get("theme") or payload.get("campaign_theme") or "general") + publish_date = str(payload.get("publish_date") or payload.get("date") or "") + audience = str(payload.get("audience") or "general") + payload["scheduled_topic"] = f"{theme}-{audience}".strip("-") + payload["suggested_asset_type"] = payload.get("preferred_format") or "post" + payload["schedule_slot"] = f"{payload.get('channel')}@{publish_date or 'tbd'}" + return payload + + +class ScheduleConflictDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + issues: list[str] = [] + topic = str(payload.get("scheduled_topic") or "") + publish_date = str(payload.get("publish_date") or payload.get("date") or "") + blackout = str(payload.get("blackout_dates") or "") + if publish_date and publish_date in blackout: + issues.append("blackout_date_conflict") + if topic.count("launch") > 1 or topic.count("sale") > 1: + issues.append("duplicate_campaign_theme") + if int(payload.get("planned_posts_this_week") or 1) > int( + payload.get("max_posts_per_week") or 7 + ): + issues.append("channel_capacity_exceeded") + payload["schedule_conflicts"] = issues + payload["schedule_status"] = "needs_review" if issues else "ready" + payload["schedule_summary"] = ( + f"Slot {payload.get('schedule_slot')} topic {topic}, conflicts {', '.join(issues) or 'none'}." + ) + return payload + + +class ContentScheduleSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("channel_rules", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/content_scheduler/pipeline.py b/apps/src/sage/apps/content_scheduler/pipeline.py new file mode 100644 index 0000000..fe3808d --- /dev/null +++ b/apps/src/sage/apps/content_scheduler/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CampaignPlanSource, + ChannelRuleMapper, + ContentScheduleSink, + ScheduleConflictDetector, + TopicAllocator, +) + + +def run_content_scheduler_pipeline(plan_file: str, channel_file: str, output_file: str) -> None: + env = LocalEnvironment("content_scheduler") + ( + env.from_batch(CampaignPlanSource, plan_file=plan_file, channel_file=channel_file) + .map(ChannelRuleMapper) + .map(TopicAllocator) + .map(ScheduleConflictDetector) + .sink(ContentScheduleSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/content_tagger/README.md b/apps/src/sage/apps/content_tagger/README.md new file mode 100644 index 0000000..1388a79 --- /dev/null +++ b/apps/src/sage/apps/content_tagger/README.md @@ -0,0 +1,5 @@ +# Content Tagger + +读取内容文本，清洗文本并生成推荐标签。 + +输入：CSV 或 JSON 内容记录。 输出：包含候选标签和最终标签的 JSON 文件。 diff --git a/apps/src/sage/apps/content_tagger/__init__.py b/apps/src/sage/apps/content_tagger/__init__.py new file mode 100644 index 0000000..e7c5613 --- /dev/null +++ b/apps/src/sage/apps/content_tagger/__init__.py @@ -0,0 +1,5 @@ +"""Content tagger application.""" + +from .pipeline import run_content_tagger_pipeline + +__all__ = ["run_content_tagger_pipeline"] diff --git a/apps/src/sage/apps/content_tagger/operators.py b/apps/src/sage/apps/content_tagger/operators.py new file mode 100644 index 0000000..6b2c213 --- /dev/null +++ b/apps/src/sage/apps/content_tagger/operators.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class ContentSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class ContentCleaner(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = " ".join(str(item.get(field, "")) for field in ("title", "content", "summary")) + item["cleaned_text"] = re.sub(r"\s+", " ", text).strip().lower() + return item + + +class TagCandidateExtractor(MapFunction): + def __init__(self, top_k: int = 8, **kwargs): + super().__init__(**kwargs) + self.top_k = top_k + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + tokens = re.findall(r"[a-z]{4,}", item.get("cleaned_text", "")) + counts: dict[str, int] = {} + for token in tokens: + counts[token] = counts.get(token, 0) + 1 + item["tag_candidates"] = [ + token + for token, _ in sorted(counts.items(), key=lambda pair: (-pair[1], pair[0]))[ + : self.top_k + ] + ] + return item + + +class TagSelector(MapFunction): + RULE_TAGS = { + "finance": {"invoice", "budget", "cash", "profit", "revenue"}, + "hr": {"employee", "training", "attendance", "recruitment"}, + "compliance": {"audit", "policy", "regulation", "control"}, + "technology": {"api", "cloud", "model", "platform", "system"}, + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + candidates = set(item.get("tag_candidates", [])) + selected_tags = [tag for tag, words in self.RULE_TAGS.items() if candidates & words] + item["selected_tags"] = selected_tags or item.get("tag_candidates", [])[:3] + return item + + +class TagSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/content_tagger/pipeline.py b/apps/src/sage/apps/content_tagger/pipeline.py new file mode 100644 index 0000000..8089e8e --- /dev/null +++ b/apps/src/sage/apps/content_tagger/pipeline.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ContentCleaner, ContentSource, TagCandidateExtractor, TagSelector, TagSink + + +def run_content_tagger_pipeline(input_file: str, output_file: str, top_k: int = 8) -> None: + env = LocalEnvironment("content_tagger") + ( + env.from_batch(ContentSource, input_file=input_file) + .map(ContentCleaner) + .map(TagCandidateExtractor, top_k=top_k) + .map(TagSelector) + .sink(TagSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/contract_matcher/README.md b/apps/src/sage/apps/contract_matcher/README.md new file mode 100644 index 0000000..fe9ad4c --- /dev/null +++ b/apps/src/sage/apps/contract_matcher/README.md @@ -0,0 +1,10 @@ +# Contract Matcher + +法律文案模板匹配应用。 + +## 功能 + +- 读取需求描述 +- 提取关键词 +- 与模板库计算相似度 +- 输出 Top-K 匹配模板 diff --git a/apps/src/sage/apps/contract_matcher/__init__.py b/apps/src/sage/apps/contract_matcher/__init__.py new file mode 100644 index 0000000..36093b4 --- /dev/null +++ b/apps/src/sage/apps/contract_matcher/__init__.py @@ -0,0 +1,5 @@ +"""Contract matcher application.""" + +from .pipeline import run_contract_matcher_pipeline + +__all__ = ["run_contract_matcher_pipeline"] diff --git a/apps/src/sage/apps/contract_matcher/operators.py b/apps/src/sage/apps/contract_matcher/operators.py new file mode 100644 index 0000000..caf474e --- /dev/null +++ b/apps/src/sage/apps/contract_matcher/operators.py @@ -0,0 +1,111 @@ +"""Operators for matching contract templates.""" + +from __future__ import annotations + +import json +import math +import re +from collections import Counter +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class RequirementSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = Path(input_file) + + def load_items(self) -> list[dict[str, Any]]: + text = self.input_file.read_text(encoding="utf-8") + if self.input_file.suffix.lower() == ".json": + payload = json.loads(text) + if isinstance(payload, list): + return payload + return [ + {"requirement_id": "1", "text": line.strip()} + for line in text.splitlines() + if line.strip() + ] + + +class KeywordExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + tokens = re.findall(r"[\w\u4e00-\u9fff]+", item.get("text", "").lower()) + item["keywords"] = [token for token in tokens if len(token) > 1] + return item + + +class TemplateMatcher(MapFunction): + def __init__(self, template_file: str | None = None, top_k: int = 3, **kwargs): + super().__init__(**kwargs) + self.template_file = template_file + self.top_k = top_k + self.templates = self._load_templates(template_file) + + def _load_templates(self, template_file: str | None) -> list[dict[str, Any]]: + if template_file: + path = Path(template_file) + if path.exists(): + return json.loads(path.read_text(encoding="utf-8")) + return [ + { + "template_id": "employment", + "name": "Employment Contract", + "text": "labor employment confidentiality compensation termination", + }, + { + "template_id": "nda", + "name": "Non Disclosure Agreement", + "text": "confidential information disclosure secrecy obligation remedy", + }, + { + "template_id": "service", + "name": "Service Agreement", + "text": "service scope payment delivery acceptance liability support", + }, + ] + + def _score(self, req_tokens: list[str], template_text: str) -> float: + req_counts = Counter(req_tokens) + tpl_counts = Counter(re.findall(r"[\w\u4e00-\u9fff]+", template_text.lower())) + common = set(req_counts) & set(tpl_counts) + numerator = sum(req_counts[token] * tpl_counts[token] for token in common) + left = math.sqrt(sum(value * value for value in req_counts.values())) + right = math.sqrt(sum(value * value for value in tpl_counts.values())) + if not left or not right: + return 0.0 + return round(numerator / (left * right), 4) + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + matches = [] + req_tokens = item.get("keywords", []) + for template in self.templates: + matches.append( + { + "template_id": template["template_id"], + "name": template["name"], + "score": self._score(req_tokens, template.get("text", "")), + } + ) + item["matches"] = sorted(matches, key=lambda value: value["score"], reverse=True)[ + : self.top_k + ] + return item + + +class MatchSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/contract_matcher/pipeline.py b/apps/src/sage/apps/contract_matcher/pipeline.py new file mode 100644 index 0000000..015f0fc --- /dev/null +++ b/apps/src/sage/apps/contract_matcher/pipeline.py @@ -0,0 +1,23 @@ +"""Contract matcher pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import KeywordExtractor, MatchSink, RequirementSource, TemplateMatcher + + +def run_contract_matcher_pipeline( + input_file: str, + output_file: str, + template_file: str | None = None, + top_k: int = 3, +) -> None: + env = LocalEnvironment("contract_matcher") + ( + env.from_batch(RequirementSource, input_file=input_file) + .map(KeywordExtractor) + .map(TemplateMatcher, template_file=template_file, top_k=top_k) + .sink(MatchSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/contract_risk/README.md b/apps/src/sage/apps/contract_risk/README.md new file mode 100644 index 0000000..71b26d7 --- /dev/null +++ b/apps/src/sage/apps/contract_risk/README.md @@ -0,0 +1,10 @@ +# Contract Risk + +合同条款风险识别应用。 + +## 功能 + +- 读取合同文本 +- 按条款拆分 +- 按规则打风险分 +- 输出条款级风险报告 diff --git a/apps/src/sage/apps/contract_risk/__init__.py b/apps/src/sage/apps/contract_risk/__init__.py new file mode 100644 index 0000000..1ee681a --- /dev/null +++ b/apps/src/sage/apps/contract_risk/__init__.py @@ -0,0 +1,5 @@ +"""Contract risk application.""" + +from .pipeline import run_contract_risk_pipeline + +__all__ = ["run_contract_risk_pipeline"] diff --git a/apps/src/sage/apps/contract_risk/operators.py b/apps/src/sage/apps/contract_risk/operators.py new file mode 100644 index 0000000..510682c --- /dev/null +++ b/apps/src/sage/apps/contract_risk/operators.py @@ -0,0 +1,85 @@ +"""Operators for clause-level contract risk detection.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +class ContractSource(ListBatchSource): + def __init__(self, input_path: str, **kwargs): + super().__init__(**kwargs) + self.input_path = Path(input_path) + + def load_items(self) -> list[dict[str, Any]]: + if self.input_path.is_dir(): + files = [path for path in sorted(self.input_path.iterdir()) if path.is_file()] + else: + files = [self.input_path] + return [ + {"contract_id": path.stem, "text": path.read_text(encoding="utf-8")} for path in files + ] + + +class TextExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["normalized_text"] = re.sub(r"\s+", " ", item.get("text", "")).strip() + return item + + +class ClauseSegmenter(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + segments = re.split(r"(?:\n+|;|。)", item.get("normalized_text", "")) + return [ + { + "contract_id": item.get("contract_id"), + "clause_index": index + 1, + "clause_text": clause.strip(), + } + for index, clause in enumerate(segments) + if clause.strip() + ] + + +class RiskScorer(MapFunction): + def __init__(self, rules: dict[str, int] | None = None, **kwargs): + super().__init__(**kwargs) + self.rules = rules or { + "unlimited liability": 3, + "automatic renewal": 2, + "exclusive jurisdiction": 2, + "indemnify": 2, + "无限责任": 3, + "自动续约": 2, + "单方解除": 2, + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = item.get("clause_text", "").lower() + hits = [keyword for keyword in self.rules if keyword.lower() in text] + item["risk_terms"] = hits + item["risk_score"] = sum(self.rules[keyword] for keyword in hits) + item["risk_level"] = ( + "high" if item["risk_score"] >= 3 else "medium" if item["risk_score"] >= 1 else "low" + ) + return item + + +class RiskReportSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/contract_risk/pipeline.py b/apps/src/sage/apps/contract_risk/pipeline.py new file mode 100644 index 0000000..f9c78bf --- /dev/null +++ b/apps/src/sage/apps/contract_risk/pipeline.py @@ -0,0 +1,21 @@ +"""Contract risk pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ClauseSegmenter, ContractSource, RiskReportSink, RiskScorer, TextExtractor + + +def run_contract_risk_pipeline( + input_path: str, output_file: str, rules: dict[str, int] | None = None +) -> None: + env = LocalEnvironment("contract_risk") + ( + env.from_batch(ContractSource, input_path=input_path) + .map(TextExtractor) + .flatmap(ClauseSegmenter) + .map(RiskScorer, rules=rules) + .sink(RiskReportSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/contract_versioning/README.md b/apps/src/sage/apps/contract_versioning/README.md new file mode 100644 index 0000000..a15de86 --- /dev/null +++ b/apps/src/sage/apps/contract_versioning/README.md @@ -0,0 +1,5 @@ +# Contract Versioning + +读取合同模板版本，解析版本号并分析文本差异。 + +输入：CSV 或 JSON 模板版本记录。 输出：包含版本解析和差异摘要的 JSON 文件。 diff --git a/apps/src/sage/apps/contract_versioning/__init__.py b/apps/src/sage/apps/contract_versioning/__init__.py new file mode 100644 index 0000000..9685500 --- /dev/null +++ b/apps/src/sage/apps/contract_versioning/__init__.py @@ -0,0 +1,5 @@ +"""Contract versioning application.""" + +from .pipeline import run_contract_versioning_pipeline + +__all__ = ["run_contract_versioning_pipeline"] diff --git a/apps/src/sage/apps/contract_versioning/operators.py b/apps/src/sage/apps/contract_versioning/operators.py new file mode 100644 index 0000000..d3a668f --- /dev/null +++ b/apps/src/sage/apps/contract_versioning/operators.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class ContractTemplateSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class VersionParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + version_text = str(item.get("version") or item.get("template_version") or "v1.0") + match = re.search(r"(\d+)(?:\.(\d+))?", version_text) + major = int(match.group(1)) if match else 1 + minor = int(match.group(2) or 0) if match and match.group(2) else 0 + item["parsed_version"] = {"major": major, "minor": minor} + return item + + +class TemplateDiffAnalyzer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + previous_text = str(item.get("previous_template") or "") + current_text = str(item.get("current_template") or item.get("template") or "") + previous_tokens = set(re.findall(r"[a-zA-Z_]{4,}", previous_text.lower())) + current_tokens = set(re.findall(r"[a-zA-Z_]{4,}", current_text.lower())) + item["added_terms"] = sorted(current_tokens - previous_tokens) + item["removed_terms"] = sorted(previous_tokens - current_tokens) + item["change_level"] = ( + "major" if len(item["added_terms"]) + len(item["removed_terms"]) >= 8 else "minor" + ) + return item + + +class VersionRegistrySink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/contract_versioning/pipeline.py b/apps/src/sage/apps/contract_versioning/pipeline.py new file mode 100644 index 0000000..a7e1d70 --- /dev/null +++ b/apps/src/sage/apps/contract_versioning/pipeline.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ContractTemplateSource, + TemplateDiffAnalyzer, + VersionParser, + VersionRegistrySink, +) + + +def run_contract_versioning_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("contract_versioning") + ( + env.from_batch(ContractTemplateSource, input_file=input_file) + .map(VersionParser) + .map(TemplateDiffAnalyzer) + .sink(VersionRegistrySink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/course_qa_helper/README.md b/apps/src/sage/apps/course_qa_helper/README.md new file mode 100644 index 0000000..5cb220f --- /dev/null +++ b/apps/src/sage/apps/course_qa_helper/README.md @@ -0,0 +1,6 @@ +# 课程资料问答助手 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_course_qa_helper_pipeline` +- Entry script: `examples/run_course_qa_helper.py` diff --git a/apps/src/sage/apps/course_qa_helper/__init__.py b/apps/src/sage/apps/course_qa_helper/__init__.py new file mode 100644 index 0000000..05dcadd --- /dev/null +++ b/apps/src/sage/apps/course_qa_helper/__init__.py @@ -0,0 +1,5 @@ +"""课程资料问答助手 application.""" + +from .pipeline import run_course_qa_helper_pipeline + +__all__ = ["run_course_qa_helper_pipeline"] diff --git a/apps/src/sage/apps/course_qa_helper/operators.py b/apps/src/sage/apps/course_qa_helper/operators.py new file mode 100644 index 0000000..146f39e --- /dev/null +++ b/apps/src/sage/apps/course_qa_helper/operators.py @@ -0,0 +1,122 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _tokenize(text: str) -> set[str]: + return set(re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", text.lower())) + + +class CourseDocSource(ListBatchSource): + def __init__(self, doc_dir: str, question_file: str, **kwargs): + super().__init__(**kwargs) + self.doc_dir = doc_dir + self.question_file = question_file + + def load_items(self) -> list[dict[str, Any]]: + docs = _load_records(self.doc_dir) + questions = _load_records(self.question_file) + for item in docs: + item["questions"] = questions + return docs + + +class CourseChunker(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + payload = dict(item) + text = str(payload.get("content") or payload.get("text") or payload.get("body") or "") + parts = [part.strip() for part in re.split(r"[\n]+", text) if part.strip()] + if not parts: + parts = [text] + results: list[dict[str, Any]] = [] + for index, part in enumerate(parts[:6]): + child = dict(payload) + child["chunk_id"] = index + 1 + child["chunk_text"] = part + child["chunk_terms"] = sorted(_tokenize(part)) + results.append(child) + return results + + +class CourseQuestionMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + chunk_terms = set(payload.get("chunk_terms") or []) + best_question = None + best_overlap: list[str] = [] + for question in payload.get("questions") or []: + question_text = str(question.get("question") or question.get("text") or "") + overlap = sorted(chunk_terms & _tokenize(question_text)) + if len(overlap) > len(best_overlap): + best_overlap = overlap + best_question = question_text + payload["matched_question"] = best_question or "" + payload["match_terms"] = best_overlap + payload["match_score"] = len(best_overlap) + return payload + + +class CourseAnswerFormatter(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + chunk_text = str(payload.get("chunk_text") or "") + answer = chunk_text[:180].strip() + if len(chunk_text) > 180: + answer += "..." + payload["answer_excerpt"] = answer + payload["answer_status"] = ( + "answered" if payload.get("match_score") else "needs_manual_review" + ) + return payload + + +class CourseAnswerSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("questions", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/course_qa_helper/pipeline.py b/apps/src/sage/apps/course_qa_helper/pipeline.py new file mode 100644 index 0000000..a06f1fc --- /dev/null +++ b/apps/src/sage/apps/course_qa_helper/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CourseAnswerFormatter, + CourseAnswerSink, + CourseChunker, + CourseDocSource, + CourseQuestionMatcher, +) + + +def run_course_qa_helper_pipeline(doc_dir: str, question_file: str, output_file: str) -> None: + env = LocalEnvironment("course_qa_helper") + ( + env.from_batch(CourseDocSource, doc_dir=doc_dir, question_file=question_file) + .flatmap(CourseChunker) + .map(CourseQuestionMatcher) + .map(CourseAnswerFormatter) + .sink(CourseAnswerSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/customer_deduplication/README.md b/apps/src/sage/apps/customer_deduplication/README.md new file mode 100644 index 0000000..9e78a10 --- /dev/null +++ b/apps/src/sage/apps/customer_deduplication/README.md @@ -0,0 +1,18 @@ +# Customer Deduplication + +客户数据去重应用。 + +## 功能 + +- 读取 CSV 客户数据 +- 生成标准化指纹 +- 基于相似度识别重复客户 +- 输出带去重标记的 JSON + +## 用法 + +```bash +python examples/run_customer_deduplication.py \ + --input-file customers.csv \ + --output deduplicated.json +``` diff --git a/apps/src/sage/apps/customer_deduplication/__init__.py b/apps/src/sage/apps/customer_deduplication/__init__.py new file mode 100644 index 0000000..95217bd --- /dev/null +++ b/apps/src/sage/apps/customer_deduplication/__init__.py @@ -0,0 +1,5 @@ +"""Customer deduplication application.""" + +from .pipeline import run_customer_deduplication_pipeline + +__all__ = ["run_customer_deduplication_pipeline"] diff --git a/apps/src/sage/apps/customer_deduplication/operators.py b/apps/src/sage/apps/customer_deduplication/operators.py new file mode 100644 index 0000000..7e0e46a --- /dev/null +++ b/apps/src/sage/apps/customer_deduplication/operators.py @@ -0,0 +1,66 @@ +"""Operators for customer deduplication.""" + +from __future__ import annotations + +import csv +import json +from difflib import SequenceMatcher +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class CustomerSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + return list(csv.DictReader(handle)) + + +class SimilarityCalculator(MapFunction): + def execute(self, row: dict[str, Any]) -> dict[str, Any]: + name = (row.get("name") or "").strip().lower() + email = (row.get("email") or "").strip().lower() + phone = "".join(char for char in (row.get("phone") or "") if char.isdigit()) + row["fingerprint"] = "|".join([name, email, phone]) + return row + + +class DuplicateDetector(MapFunction): + def __init__(self, threshold: float = 0.9, **kwargs): + super().__init__(**kwargs) + self.threshold = threshold + self.seen: list[dict[str, Any]] = [] + + def execute(self, row: dict[str, Any]) -> dict[str, Any]: + row["is_duplicate"] = False + row["duplicate_of"] = "" + row["similarity"] = 0.0 + fingerprint = row.get("fingerprint", "") + for existing in self.seen: + score = SequenceMatcher(None, fingerprint, existing.get("fingerprint", "")).ratio() + if score >= self.threshold: + row["is_duplicate"] = True + row["duplicate_of"] = existing.get("customer_id") or existing.get("id") or "" + row["similarity"] = round(score, 4) + break + self.seen.append(dict(row)) + return row + + +class DeduplicationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.rows: list[dict[str, Any]] = [] + + def execute(self, row: dict[str, Any]) -> None: + self.rows.append(row) + + def teardown(self, context: Any) -> None: + with open(self.output_file, "w", encoding="utf-8") as handle: + json.dump(self.rows, handle, ensure_ascii=False, indent=2) diff --git a/apps/src/sage/apps/customer_deduplication/pipeline.py b/apps/src/sage/apps/customer_deduplication/pipeline.py new file mode 100644 index 0000000..8a3f815 --- /dev/null +++ b/apps/src/sage/apps/customer_deduplication/pipeline.py @@ -0,0 +1,22 @@ +"""Customer deduplication pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import CustomerSource, DeduplicationSink, DuplicateDetector, SimilarityCalculator + + +def run_customer_deduplication_pipeline( + input_file: str, + output_file: str, + threshold: float = 0.9, +) -> None: + env = LocalEnvironment("customer_deduplication") + ( + env.from_batch(CustomerSource, input_file=input_file) + .map(SimilarityCalculator) + .map(DuplicateDetector, threshold=threshold) + .sink(DeduplicationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/data_center_watch/README.md b/apps/src/sage/apps/data_center_watch/README.md new file mode 100644 index 0000000..57b3e01 --- /dev/null +++ b/apps/src/sage/apps/data_center_watch/README.md @@ -0,0 +1,6 @@ +# 数据中心容量与冷却监测系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_data_center_watch_pipeline` +- Entry script: `examples/run_data_center_watch.py` diff --git a/apps/src/sage/apps/data_center_watch/__init__.py b/apps/src/sage/apps/data_center_watch/__init__.py new file mode 100644 index 0000000..7e50c6b --- /dev/null +++ b/apps/src/sage/apps/data_center_watch/__init__.py @@ -0,0 +1,5 @@ +"""数据中心容量与冷却监测系统 application.""" + +from .pipeline import run_data_center_watch_pipeline + +__all__ = ["run_data_center_watch_pipeline"] diff --git a/apps/src/sage/apps/data_center_watch/operators.py b/apps/src/sage/apps/data_center_watch/operators.py new file mode 100644 index 0000000..d9d9afe --- /dev/null +++ b/apps/src/sage/apps/data_center_watch/operators.py @@ -0,0 +1,116 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class DataCenterMetricSource(ListBatchSource): + def __init__(self, metric_file: str, alert_file: str, **kwargs): + super().__init__(**kwargs) + self.metric_file = metric_file + self.alert_file = alert_file + + def load_items(self) -> list[dict[str, Any]]: + metrics = _load_records(self.metric_file) + alerts = _load_records(self.alert_file) + alert_index = {str(item.get("rack_id") or item.get("rack") or ""): item for item in alerts} + for item in metrics: + rack_id = str(item.get("rack_id") or item.get("rack") or "") + item.setdefault("app_slug", "data_center_watch") + item["alert_ref"] = alert_index.get(rack_id, {}) + item.setdefault("source_path", self.metric_file) + return metrics + + +class RackMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["rack_id"] = str(payload.get("rack_id") or payload.get("rack") or "unknown") + payload["capacity_pct"] = _to_float(payload.get("capacity_pct")) + payload["temperature_c"] = _to_float(payload.get("temperature_c")) + payload["power_kw"] = _to_float(payload.get("power_kw")) + return payload + + +class CapacityCoolingScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + issues: list[str] = [] + if payload.get("capacity_pct", 0.0) > 85: + issues.append("high_capacity") + if payload.get("temperature_c", 0.0) > 30: + issues.append("cooling_risk") + if str((payload.get("alert_ref") or {}).get("alert_type") or "").lower() in { + "fan_failure", + "power_alarm", + }: + issues.append("active_hardware_alert") + payload["dc_flags"] = issues + return payload + + +class DataCenterRiskMarker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + score = len(payload.get("dc_flags") or []) * 3 + level = "normal" + if score >= 6: + level = "critical" + elif score >= 3: + level = "watch" + payload["risk_level"] = level + payload["watch_summary"] = ( + f"Rack {payload.get('rack_id')} risk {level}, flags {', '.join(payload.get('dc_flags') or []) or 'none'}." + ) + return payload + + +class DataCenterWatchSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("alert_ref", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/data_center_watch/pipeline.py b/apps/src/sage/apps/data_center_watch/pipeline.py new file mode 100644 index 0000000..5ed11ad --- /dev/null +++ b/apps/src/sage/apps/data_center_watch/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CapacityCoolingScorer, + DataCenterMetricSource, + DataCenterRiskMarker, + DataCenterWatchSink, + RackMapper, +) + + +def run_data_center_watch_pipeline(metric_file: str, alert_file: str, output_file: str) -> None: + env = LocalEnvironment("data_center_watch") + ( + env.from_batch(DataCenterMetricSource, metric_file=metric_file, alert_file=alert_file) + .map(RackMapper) + .map(CapacityCoolingScorer) + .map(DataCenterRiskMarker) + .sink(DataCenterWatchSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/data_cleaner/README.md b/apps/src/sage/apps/data_cleaner/README.md new file mode 100644 index 0000000..f1fb386 --- /dev/null +++ b/apps/src/sage/apps/data_cleaner/README.md @@ -0,0 +1,29 @@ +# Data Cleaner + +CSV 数据清洗与标准化示例应用。 + +## 功能 + +- 读取 CSV 输入 +- 按字段规则做类型转换 +- 处理缺失值 +- 标记数值异常和重复记录 +- 输出 CSV 或 JSON 结果 + +## 用法 + +```bash +python examples/run_data_cleaner.py \ + --input raw.csv \ + --output cleaned.csv +``` + +```bash +python examples/run_data_cleaner.py \ + --input raw.csv \ + --output cleaned.json \ + --type-rules age:int,salary:float,active:bool \ + --numeric-fields age,salary \ + --key-fields email,phone \ + --output-format json +``` diff --git a/apps/src/sage/apps/data_cleaner/__init__.py b/apps/src/sage/apps/data_cleaner/__init__.py new file mode 100644 index 0000000..7143360 --- /dev/null +++ b/apps/src/sage/apps/data_cleaner/__init__.py @@ -0,0 +1,5 @@ +"""Data cleaner application.""" + +from .pipeline import run_data_cleaner_pipeline + +__all__ = ["run_data_cleaner_pipeline"] diff --git a/apps/src/sage/apps/data_cleaner/operators.py b/apps/src/sage/apps/data_cleaner/operators.py new file mode 100644 index 0000000..126a989 --- /dev/null +++ b/apps/src/sage/apps/data_cleaner/operators.py @@ -0,0 +1,387 @@ +""" +Data Cleaner Operators + +Custom operators for CSV/Excel data cleaning and standardization. +""" + +from __future__ import annotations + +import csv +import json +from datetime import datetime +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import CustomLogger, MapFunction, SinkFunction + + +class CsvSource(ListBatchSource): + """Read CSV/Excel file and return rows as batch.""" + + def __init__(self, input_file: str, delimiter: str = ",", **kwargs): + """Initialize CSV source. + + Args: + input_file: Path to CSV file + delimiter: CSV delimiter (default: ",") + """ + super().__init__(**kwargs) + self.input_file = input_file + self.delimiter = delimiter + self._logger = CustomLogger("CsvSource") + + def load_items(self) -> list[dict[str, str]]: + """Read and return all CSV rows as dictionaries.""" + try: + rows = [] + with open(self.input_file, encoding="utf-8") as f: + reader = csv.DictReader(f, delimiter=self.delimiter) + if reader.fieldnames is None: + self.logger.error(f"Empty CSV file: {self.input_file}") + return [] + + for row in reader: + rows.append(row) + + self.logger.info(f"Read {len(rows)} rows from {self.input_file}") + return rows + except FileNotFoundError: + self.logger.error(f"CSV file not found: {self.input_file}") + return [] + except Exception as e: + self.logger.error(f"Error reading CSV: {e}") + return [] + + +class TypeConverter(MapFunction): + """Convert field types according to rules.""" + + def __init__(self, type_rules: dict[str, str] | None = None, **kwargs): + """Initialize type converter. + + Args: + type_rules: Dict mapping field names to types (int, float, bool, date) + Example: {"age": "int", "salary": "float", "active": "bool"} + """ + super().__init__(**kwargs) + self.type_rules = type_rules or {} + self._logger = CustomLogger("TypeConverter") + + def _convert_value(self, value: str, target_type: str) -> Any: + """Convert a value to target type. + + Args: + value: Value string to convert + target_type: Target type (int, float, bool, date) + + Returns: + Converted value or original if conversion fails + """ + if not value or value.strip() == "": + return None + + value = value.strip() + + try: + if target_type.lower() == "int": + return int(float(value)) # Handle decimal strings like "5.0" + elif target_type.lower() == "float": + return float(value) + elif target_type.lower() == "bool": + return value.lower() in ["true", "yes", "1", "on"] + elif target_type.lower() == "date": + # Try common date formats + for fmt in ["%Y-%m-%d", "%Y/%m/%d", "%d-%m-%Y", "%d/%m/%Y", "%m-%d-%Y"]: + try: + return datetime.strptime(value, fmt).isoformat() + except ValueError: + continue + return value # Return as-is if no format matches + else: + return value + except (ValueError, TypeError): + return value # Return original value if conversion fails + + def execute(self, row: dict[str, str]) -> dict[str, Any]: + """Convert field types in a row. + + Args: + row: Data row + + Returns: + Row with converted types + """ + if not row: + return row + + result = dict(row) # Copy original + + for field, target_type in self.type_rules.items(): + if field in result: + result[field] = self._convert_value(result[field], target_type) + + return result + + +class MissingValueFiller(MapFunction): + """Fill missing values according to strategy.""" + + def __init__(self, fill_strategy: dict[str, str] | str | None = None, **kwargs): + """Initialize missing value filler. + + Args: + fill_strategy: Strategy per field or global strategy + Field-level: {"age": "0", "name": "Unknown"} + Global: "drop" (drop rows), "mean" (use mean), "forward" (forward fill) + """ + super().__init__(**kwargs) + self.fill_strategy = fill_strategy or "drop" + self._logger = CustomLogger("MissingValueFiller") + self.row_count = 0 + self.dropped_count = 0 + + def execute(self, row: dict[str, Any]) -> dict[str, Any] | None: + """Fill missing values in a row. + + Args: + row: Data row + + Returns: + Row with filled values or None if dropped + """ + if not row: + return None + + self.row_count += 1 + + # Check for missing values + missing_fields = [k for k, v in row.items() if v is None or v == ""] + + if not missing_fields: + return row + + # Apply fill strategy + if isinstance(self.fill_strategy, dict): + # Field-specific strategies + for field in missing_fields: + if field in self.fill_strategy: + row[field] = self.fill_strategy[field] + else: + row[field] = None + return row + + elif self.fill_strategy == "drop": + # Drop rows with missing values + self.dropped_count += 1 + return None + + elif self.fill_strategy == "forward": + # Fill with empty string (forward fill would require state) + for field in missing_fields: + row[field] = "" + return row + + else: + # Default: return as-is + return row + + def teardown(self, context: Any) -> None: + """Log statistics.""" + if self.dropped_count > 0: + self.logger.info( + f"Dropped {self.dropped_count}/{self.row_count} rows with missing values" + ) + + +class AnomalyDetector(MapFunction): + """Detect anomalies in numeric fields.""" + + def __init__(self, numeric_fields: list[str] | None = None, **kwargs): + """Initialize anomaly detector. + + Args: + numeric_fields: List of numeric field names to check + """ + super().__init__(**kwargs) + self.numeric_fields = numeric_fields or [] + self._logger = CustomLogger("AnomalyDetector") + + def execute(self, row: dict[str, Any]) -> dict[str, Any]: + """Detect anomalies in a row. + + Adds: + - has_anomaly: boolean flag + - anomaly_fields: list of fields with anomalies + - anomalies: dict of detected anomalies + + Args: + row: Data row + + Returns: + Row with anomaly markers + """ + if not row: + return row + + row["has_anomaly"] = False + row["anomaly_fields"] = [] + row["anomalies"] = {} + + for field in self.numeric_fields: + if field not in row: + continue + + value = row[field] + + # Skip non-numeric values + if not isinstance(value, (int, float)): + continue + + # Check for negative values where not expected + if value < 0 and field.lower() not in ["change", "delta", "difference"]: + row["has_anomaly"] = True + row["anomaly_fields"].append(field) + row["anomalies"][field] = f"negative_value: {value}" + + # Check for extremely large values + if value > 1e9: + row["has_anomaly"] = True + row["anomaly_fields"].append(field) + row["anomalies"][field] = f"extremely_large: {value}" + + return row + + +class DuplicateMarker(MapFunction): + """Mark potential duplicate rows based on key fields.""" + + def __init__(self, key_fields: list[str] | None = None, **kwargs): + """Initialize duplicate marker. + + Args: + key_fields: Fields to use for duplicate detection + """ + super().__init__(**kwargs) + self.key_fields = key_fields or [] + self.seen_keys = {} + self._logger = CustomLogger("DuplicateMarker") + + def execute(self, row: dict[str, Any]) -> dict[str, Any]: + """Mark duplicates based on key fields. + + Args: + row: Data row + + Returns: + Row with duplicate markers + """ + if not row: + return row + + # Build key from key fields + if self.key_fields: + key_values = tuple(row.get(f, "") for f in self.key_fields) + is_duplicate = key_values in self.seen_keys + row["is_duplicate"] = is_duplicate + row["duplicate_index"] = self.seen_keys.get(key_values, 0) + + if not is_duplicate: + self.seen_keys[key_values] = 1 + else: + self.seen_keys[key_values] += 1 + else: + row["is_duplicate"] = False + row["duplicate_index"] = 0 + + return row + + +class CleanedDataSink(SinkFunction): + """Output cleaned data to CSV file.""" + + def __init__(self, output_file: str, **kwargs): + """Initialize cleaned data sink. + + Args: + output_file: Path to output CSV file + """ + super().__init__(**kwargs) + self.output_file = output_file + self._logger = CustomLogger("CleanedDataSink") + self.count = 0 + self.fieldnames = None + self.writer = None + self.file = None + + def setup(self, context: Any) -> None: + """Setup - open output file.""" + self.file = open(self.output_file, "w", newline="", encoding="utf-8") + + def execute(self, row: dict[str, Any]) -> None: + """Write cleaned row to CSV. + + Args: + row: Cleaned data row + """ + if not row: + return + + if self.file is None: + self.file = open(self.output_file, "w", newline="", encoding="utf-8") + + # Initialize writer on first row + if self.writer is None: + self.fieldnames = list(row.keys()) + self.writer = csv.DictWriter(self.file, fieldnames=self.fieldnames) + self.writer.writeheader() + + self.writer.writerow({k: row.get(k, "") for k in self.fieldnames}) + self.file.flush() + self.count += 1 + + def teardown(self, context: Any) -> None: + """Cleanup - close output file.""" + if self.file: + self.file.close() + self.logger.info(f"Written {self.count} cleaned rows to {self.output_file}") + + +class JsonSink(SinkFunction): + """Output cleaned data to JSON file.""" + + def __init__(self, output_file: str, **kwargs): + """Initialize JSON sink. + + Args: + output_file: Path to output JSON file + """ + super().__init__(**kwargs) + self.output_file = output_file + self._logger = CustomLogger("JsonSink") + self.count = 0 + self.items: list[dict[str, Any]] = [] + + def setup(self, context: Any) -> None: + """Setup - prepare output file.""" + with open(self.output_file, "w", encoding="utf-8") as f: + f.write("[\n") + + def execute(self, row: dict[str, Any]) -> None: + """Append row to JSON file. + + Args: + row: Data row + """ + if not row: + return + + self.items.append(row) + with open(self.output_file, "w", encoding="utf-8") as f: + json.dump(self.items, f, ensure_ascii=False, default=str, indent=2) + self.count += 1 + + def teardown(self, context: Any) -> None: + """Cleanup - close JSON array.""" + with open(self.output_file, "a", encoding="utf-8") as f: + f.write("\n]") + self.logger.info(f"Written {self.count} rows to {self.output_file}") diff --git a/apps/src/sage/apps/data_cleaner/pipeline.py b/apps/src/sage/apps/data_cleaner/pipeline.py new file mode 100644 index 0000000..75f78c5 --- /dev/null +++ b/apps/src/sage/apps/data_cleaner/pipeline.py @@ -0,0 +1,99 @@ +""" +Data Cleaner Pipeline + +Main pipeline implementation using SAGE operators for CSV/Excel data cleaning. +""" + +from __future__ import annotations + +from sage.foundation import CustomLogger +from sage.runtime import LocalEnvironment + +from .operators import ( + AnomalyDetector, + CleanedDataSink, + CsvSource, + DuplicateMarker, + JsonSink, + MissingValueFiller, + TypeConverter, +) + + +def run_data_cleaner_pipeline( + input_file: str, + output_file: str, + type_rules: dict[str, str] | None = None, + fill_strategy: dict[str, str] | str | None = None, + numeric_fields: list[str] | None = None, + key_fields: list[str] | None = None, + output_format: str = "csv", + verbose: bool = False, +) -> None: + """ + Run the data cleaner pipeline using SAGE framework. + + Args: + input_file: Path to input CSV file + output_file: Path to output CSV or JSON file + type_rules: Dict mapping field names to types (int, float, bool, date) + fill_strategy: Strategy for missing values ("drop", "forward", or dict) + numeric_fields: List of numeric fields for anomaly detection + key_fields: List of fields for duplicate detection + output_format: Output format ("csv" or "json") + verbose: Enable verbose logging + + Example: + >>> run_data_cleaner_pipeline( + ... input_file="raw_data.csv", + ... output_file="cleaned_data.csv", + ... type_rules={"age": "int", "salary": "float"}, + ... fill_strategy={"age": "0", "salary": "0"}, + ... numeric_fields=["age", "salary"], + ... verbose=True + ... ) + """ + logger = CustomLogger("DataCleanerPipeline") + + if verbose: + logger.info("Starting data cleaner pipeline") + logger.info(f" Input file: {input_file}") + logger.info(f" Output file: {output_file}") + logger.info(f" Type rules: {type_rules}") + logger.info(f" Fill strategy: {fill_strategy}") + + # Create environment + env = LocalEnvironment("data_cleaner") + + try: + # Build pipeline + pipeline = ( + env.from_batch(CsvSource, input_file=input_file) + .map(TypeConverter, type_rules=type_rules or {}) + .map(MissingValueFiller, fill_strategy=fill_strategy or "drop") + .filter(lambda row: row is not None) + ) + + # Add optional anomaly detection + if numeric_fields: + pipeline = pipeline.map(AnomalyDetector, numeric_fields=numeric_fields) + + # Add optional duplicate detection + if key_fields: + pipeline = pipeline.map(DuplicateMarker, key_fields=key_fields) + + # Add sink based on output format + if output_format.lower() == "json": + pipeline = pipeline.sink(JsonSink, output_file=output_file) + else: + pipeline = pipeline.sink(CleanedDataSink, output_file=output_file) + + # Submit and run + env.submit(autostop=True) + + if verbose: + logger.info("Data cleaner pipeline completed successfully") + + except Exception as e: + logger.error(f"Error in data cleaner pipeline: {e}") + raise diff --git a/apps/src/sage/apps/doc_classifier/README.md b/apps/src/sage/apps/doc_classifier/README.md new file mode 100644 index 0000000..054ff3c --- /dev/null +++ b/apps/src/sage/apps/doc_classifier/README.md @@ -0,0 +1,18 @@ +# Doc Classifier + +文档分类应用。 + +## 功能 + +- 读取文本、CSV 或 JSON 文档 +- 清洗文本并抽取高频词 +- 使用规则对文档进行分类 +- 输出分类后的 JSON 结果 + +## 用法 + +```bash +python examples/run_doc_classifier.py \ + --input-file docs.csv \ + --output classified_docs.json +``` diff --git a/apps/src/sage/apps/doc_classifier/__init__.py b/apps/src/sage/apps/doc_classifier/__init__.py new file mode 100644 index 0000000..bd75b3f --- /dev/null +++ b/apps/src/sage/apps/doc_classifier/__init__.py @@ -0,0 +1,5 @@ +"""Document classifier application.""" + +from .pipeline import run_doc_classifier_pipeline + +__all__ = ["run_doc_classifier_pipeline"] diff --git a/apps/src/sage/apps/doc_classifier/operators.py b/apps/src/sage/apps/doc_classifier/operators.py new file mode 100644 index 0000000..2e52c89 --- /dev/null +++ b/apps/src/sage/apps/doc_classifier/operators.py @@ -0,0 +1,98 @@ +"""Operators for rule-based document classification.""" + +from __future__ import annotations + +import csv +import json +import re +from collections import Counter +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +class DocSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".csv"): + rows = list(csv.DictReader(handle)) + return [ + {"doc_id": row.get("doc_id", ""), "text": row.get("text", "")} for row in rows + ] + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return [ + {"doc_id": str(index + 1), "text": line.strip()} + for index, line in enumerate(handle) + if line.strip() + ] + + +class TextExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["clean_text"] = " ".join(str(item.get("text", "")).split()) + return item + + +class Tokenizer(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + tokens = re.findall(r"[\w\u4e00-\u9fff]+", item.get("clean_text", "").lower()) + enriched = dict(item) + enriched["tokens"] = tokens + return [enriched] if tokens else [] + + +class FeatureExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + tokens = item.get("tokens") or re.findall( + r"[\w\u4e00-\u9fff]+", item.get("clean_text", "").lower() + ) + counts = Counter(tokens) + item["token_count"] = len(tokens) + item["top_terms"] = [term for term, _ in counts.most_common(5)] + total = max(len(tokens), 1) + item["tfidf_features"] = {term: round(count / total, 4) for term, count in counts.items()} + return item + + +class Classifier(MapFunction): + def __init__(self, label_rules: dict[str, list[str]] | None = None, **kwargs): + super().__init__(**kwargs) + self.label_rules = label_rules or { + "contract": ["contract", "agreement", "clause", "terms"], + "invoice": ["invoice", "vat", "payment", "tax"], + "resume": ["resume", "experience", "education", "skills"], + "report": ["report", "summary", "analysis", "findings"], + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = item.get("clean_text", "").lower() + best_label = "other" + best_score = 0 + for label, keywords in self.label_rules.items(): + score = sum(1 for keyword in keywords if keyword.lower() in text) + if score > best_score: + best_score = score + best_label = label + item["label"] = best_label + item["label_score"] = best_score + return item + + +class DocSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + with open(self.output_file, "w", encoding="utf-8") as handle: + json.dump(self.items, handle, ensure_ascii=False, indent=2) diff --git a/apps/src/sage/apps/doc_classifier/pipeline.py b/apps/src/sage/apps/doc_classifier/pipeline.py new file mode 100644 index 0000000..ce21fd1 --- /dev/null +++ b/apps/src/sage/apps/doc_classifier/pipeline.py @@ -0,0 +1,24 @@ +"""Document classifier pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import Classifier, DocSink, DocSource, FeatureExtractor, TextExtractor, Tokenizer + + +def run_doc_classifier_pipeline( + input_file: str, + output_file: str, + label_rules: dict[str, list[str]] | None = None, +) -> None: + env = LocalEnvironment("doc_classifier") + ( + env.from_batch(DocSource, input_file=input_file) + .map(TextExtractor) + .flatmap(Tokenizer) + .map(FeatureExtractor) + .map(Classifier, label_rules=label_rules) + .sink(DocSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/dorm_energy_optimizer/README.md b/apps/src/sage/apps/dorm_energy_optimizer/README.md new file mode 100644 index 0000000..76a2f43 --- /dev/null +++ b/apps/src/sage/apps/dorm_energy_optimizer/README.md @@ -0,0 +1,3 @@ +# Dorm Energy Optimizer + +宿舍能耗监测优化应用。 diff --git a/apps/src/sage/apps/dorm_energy_optimizer/__init__.py b/apps/src/sage/apps/dorm_energy_optimizer/__init__.py new file mode 100644 index 0000000..aef1e39 --- /dev/null +++ b/apps/src/sage/apps/dorm_energy_optimizer/__init__.py @@ -0,0 +1,5 @@ +"""Dorm energy optimizer application.""" + +from .pipeline import run_dorm_energy_optimizer_pipeline + +__all__ = ["run_dorm_energy_optimizer_pipeline"] diff --git a/apps/src/sage/apps/dorm_energy_optimizer/operators.py b/apps/src/sage/apps/dorm_energy_optimizer/operators.py new file mode 100644 index 0000000..5e40a88 --- /dev/null +++ b/apps/src/sage/apps/dorm_energy_optimizer/operators.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class EnergySource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class MeterSource(EnergySource): + pass + + +class UsageAnalyzer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + consumption = float(item.get("kwh") or 0) + baseline = float(item.get("baseline_kwh") or 1) + item["usage_ratio"] = round(consumption / baseline, 2) if baseline else consumption + return item + + +class DormMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["dorm_id"] = str(item.get("dorm_id") or item.get("room_no") or "unknown") + return UsageAnalyzer().execute(item) + + +class EnergyBaselineComparer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["baseline_delta"] = round(float(item.get("usage_ratio", 0)) - 1.0, 2) + return item + + +class RecommendationBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + ratio = float(item.get("usage_ratio", 0)) + item["recommendation"] = ( + "Reduce peak-hour usage" if ratio > 1.2 else "Maintain current plan" + ) + return item + + +class EnergyAnomalyDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + compared = EnergyBaselineComparer().execute(item) + compared["anomaly_level"] = ( + "high" + if float(compared.get("usage_ratio", 0)) >= 1.5 + else "medium" + if float(compared.get("usage_ratio", 0)) >= 1.2 + else "low" + ) + return RecommendationBuilder().execute(compared) + + +class EnergySink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + +class EnergyAdviceSink(EnergySink): + pass diff --git a/apps/src/sage/apps/dorm_energy_optimizer/pipeline.py b/apps/src/sage/apps/dorm_energy_optimizer/pipeline.py new file mode 100644 index 0000000..b858178 --- /dev/null +++ b/apps/src/sage/apps/dorm_energy_optimizer/pipeline.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import DormMapper, EnergyAdviceSink, EnergyAnomalyDetector, MeterSource + + +def run_dorm_energy_optimizer_pipeline(meter_file: str, output_file: str) -> None: + env = LocalEnvironment("dorm_energy_optimizer") + ( + env.from_batch(MeterSource, input_file=meter_file) + .map(DormMapper) + .map(EnergyAnomalyDetector) + .sink(EnergyAdviceSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/drug_leaflet_extractor/README.md b/apps/src/sage/apps/drug_leaflet_extractor/README.md new file mode 100644 index 0000000..9920abb --- /dev/null +++ b/apps/src/sage/apps/drug_leaflet_extractor/README.md @@ -0,0 +1,6 @@ +# 药品说明书结构化抽取系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_drug_leaflet_extractor_pipeline` +- Entry script: `examples/run_drug_leaflet_extractor.py` diff --git a/apps/src/sage/apps/drug_leaflet_extractor/__init__.py b/apps/src/sage/apps/drug_leaflet_extractor/__init__.py new file mode 100644 index 0000000..7715f1e --- /dev/null +++ b/apps/src/sage/apps/drug_leaflet_extractor/__init__.py @@ -0,0 +1,5 @@ +"""药品说明书结构化抽取系统 application.""" + +from .pipeline import run_drug_leaflet_extractor_pipeline + +__all__ = ["run_drug_leaflet_extractor_pipeline"] diff --git a/apps/src/sage/apps/drug_leaflet_extractor/operators.py b/apps/src/sage/apps/drug_leaflet_extractor/operators.py new file mode 100644 index 0000000..db252f3 --- /dev/null +++ b/apps/src/sage/apps/drug_leaflet_extractor/operators.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _extract_section(text: str, title: str) -> str: + pattern = rf"{title}\s*[:：]\s*(.+?)(?:\n[A-Z][^\n]*[:：]|\Z)" + match = re.search(pattern, text, re.I | re.S) + return match.group(1).strip() if match else "" + + +class DrugLeafletSource(ListBatchSource): + def __init__(self, input_dir: str, **kwargs): + super().__init__(**kwargs) + self.input_dir = input_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_dir) + for item in items: + item.setdefault("app_slug", "drug_leaflet_extractor") + item.setdefault("source_path", self.input_dir) + return items + + +class DrugLeafletTextExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + text = str(payload.get("content") or payload.get("text") or "") + payload["leaflet_text"] = re.sub(r"\s+", " ", text).strip() + return payload + + +class DrugFieldExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + text = payload.get("leaflet_text", "") + dosage_match = re.search(r"(\d+(?:\.\d+)?)\s*(mg|g|ml)", text, re.I) + payload["drug_name"] = str(payload.get("drug_name") or payload.get("name") or "unknown") + payload["dosage_value"] = float(dosage_match.group(1)) if dosage_match else None + payload["dosage_unit"] = dosage_match.group(2).lower() if dosage_match else "" + payload["contraindications"] = _extract_section( + text, "Contraindications" + ) or _extract_section(text, "禁忌") + payload["adverse_reactions"] = _extract_section( + text, "Adverse Reactions" + ) or _extract_section(text, "不良反应") + payload["warnings"] = _extract_section(text, "Warnings") or _extract_section(text, "警示") + return payload + + +class DrugUnitNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + unit = str(payload.get("dosage_unit") or "").lower() + value = payload.get("dosage_value") + normalized_dose = None + if value is not None: + if unit == "g": + normalized_dose = f"{value * 1000:.0f} mg" + elif unit == "ml": + normalized_dose = f"{value:.0f} ml" + else: + normalized_dose = f"{value:.0f} mg" + payload["normalized_dose"] = normalized_dose or "unknown" + payload["risk_flags"] = [ + flag + for flag, section in ( + ("contraindication_present", payload.get("contraindications")), + ("adverse_reaction_present", payload.get("adverse_reactions")), + ("warning_present", payload.get("warnings")), + ) + if section + ] + return payload + + +class DrugLeafletSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/drug_leaflet_extractor/pipeline.py b/apps/src/sage/apps/drug_leaflet_extractor/pipeline.py new file mode 100644 index 0000000..e001a6a --- /dev/null +++ b/apps/src/sage/apps/drug_leaflet_extractor/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + DrugFieldExtractor, + DrugLeafletSink, + DrugLeafletSource, + DrugLeafletTextExtractor, + DrugUnitNormalizer, +) + + +def run_drug_leaflet_extractor_pipeline(input_dir: str, output_file: str) -> None: + env = LocalEnvironment("drug_leaflet_extractor") + ( + env.from_batch(DrugLeafletSource, input_dir=input_dir) + .map(DrugLeafletTextExtractor) + .map(DrugFieldExtractor) + .map(DrugUnitNormalizer) + .sink(DrugLeafletSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/exhibition_heatmap/README.md b/apps/src/sage/apps/exhibition_heatmap/README.md new file mode 100644 index 0000000..7c41ab0 --- /dev/null +++ b/apps/src/sage/apps/exhibition_heatmap/README.md @@ -0,0 +1,3 @@ +# Exhibition Heatmap + +读取客流数据，完成区域映射、热度计算和拥堵判定。 diff --git a/apps/src/sage/apps/exhibition_heatmap/__init__.py b/apps/src/sage/apps/exhibition_heatmap/__init__.py new file mode 100644 index 0000000..402b446 --- /dev/null +++ b/apps/src/sage/apps/exhibition_heatmap/__init__.py @@ -0,0 +1,5 @@ +"""Exhibition heatmap application.""" + +from .pipeline import run_exhibition_heatmap_pipeline + +__all__ = ["run_exhibition_heatmap_pipeline"] diff --git a/apps/src/sage/apps/exhibition_heatmap/operators.py b/apps/src/sage/apps/exhibition_heatmap/operators.py new file mode 100644 index 0000000..13cbe04 --- /dev/null +++ b/apps/src/sage/apps/exhibition_heatmap/operators.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class VisitorFlowSource(ListBatchSource): + def __init__(self, flow_file: str, **kwargs): + super().__init__(**kwargs) + self.flow_file = flow_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.flow_file, encoding="utf-8", newline="") as handle: + if self.flow_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class ZoneMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["zone"] = str(item.get("zone") or item.get("hall") or "unknown").strip().lower() + return item + + +class HeatScoreCalculator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + visitors = int(float(item.get("visitors") or 0)) + dwell_minutes = float(item.get("dwell_minutes") or 0) + item["heat_score"] = round(visitors * 0.6 + dwell_minutes * 1.5, 2) + return item + + +class CongestionDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = float(item.get("heat_score", 0)) + item["congestion_level"] = "high" if score >= 80 else "medium" if score >= 40 else "low" + return item + + +class HeatmapSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/exhibition_heatmap/pipeline.py b/apps/src/sage/apps/exhibition_heatmap/pipeline.py new file mode 100644 index 0000000..00e476a --- /dev/null +++ b/apps/src/sage/apps/exhibition_heatmap/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CongestionDetector, + HeatmapSink, + HeatScoreCalculator, + VisitorFlowSource, + ZoneMapper, +) + + +def run_exhibition_heatmap_pipeline(flow_file: str, output_file: str) -> None: + env = LocalEnvironment("exhibition_heatmap") + ( + env.from_batch(VisitorFlowSource, flow_file=flow_file) + .map(ZoneMapper) + .map(HeatScoreCalculator) + .map(CongestionDetector) + .sink(HeatmapSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/experiment_review/README.md b/apps/src/sage/apps/experiment_review/README.md new file mode 100644 index 0000000..fcf8542 --- /dev/null +++ b/apps/src/sage/apps/experiment_review/README.md @@ -0,0 +1,6 @@ +# 实验记录异常回顾系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_experiment_review_pipeline` +- Entry script: `examples/run_experiment_review.py` diff --git a/apps/src/sage/apps/experiment_review/__init__.py b/apps/src/sage/apps/experiment_review/__init__.py new file mode 100644 index 0000000..3ee1c9b --- /dev/null +++ b/apps/src/sage/apps/experiment_review/__init__.py @@ -0,0 +1,5 @@ +"""实验记录异常回顾系统 application.""" + +from .pipeline import run_experiment_review_pipeline + +__all__ = ["run_experiment_review_pipeline"] diff --git a/apps/src/sage/apps/experiment_review/operators.py b/apps/src/sage/apps/experiment_review/operators.py new file mode 100644 index 0000000..8ee0ef1 --- /dev/null +++ b/apps/src/sage/apps/experiment_review/operators.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _parse_float(value: Any) -> float | None: + match = re.search(r"-?\d+(?:\.\d+)?", str(value or "")) + return float(match.group(0)) if match else None + + +class ExperimentLogSource(ListBatchSource): + def __init__(self, log_file: str, **kwargs): + super().__init__(**kwargs) + self.log_file = log_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.log_file) + for item in items: + item.setdefault("source_path", self.log_file) + return items + + +class ExperimentStepSplitter(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + payload = dict(item) + raw_text = str(payload.get("log_text") or payload.get("steps") or payload.get("text") or "") + parts = [part.strip() for part in re.split(r"[\n;|]+", raw_text) if part.strip()] + results: list[dict[str, Any]] = [] + for index, part in enumerate(parts or [raw_text]): + child = dict(payload) + child["step_index"] = index + 1 + child["step_text"] = part.strip() + results.append(child) + return results + + +class ExperimentParameterExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + step_text = str(payload.get("step_text") or "") + temperature = ( + _parse_float( + re.search(r"temp(?:erature)?[^\d-]*(-?\d+(?:\.\d+)?)", step_text, re.I).group(1) + ) + if re.search(r"temp(?:erature)?[^\d-]*(-?\d+(?:\.\d+)?)", step_text, re.I) + else None + ) + ph_value = ( + _parse_float(re.search(r"pH[^\d-]*(-?\d+(?:\.\d+)?)", step_text, re.I).group(1)) + if re.search(r"pH[^\d-]*(-?\d+(?:\.\d+)?)", step_text, re.I) + else None + ) + yield_value = ( + _parse_float(re.search(r"yield[^\d-]*(-?\d+(?:\.\d+)?)", step_text, re.I).group(1)) + if re.search(r"yield[^\d-]*(-?\d+(?:\.\d+)?)", step_text, re.I) + else None + ) + payload["temperature_c"] = temperature + payload["ph_value"] = ph_value + payload["yield_percent"] = yield_value + payload["step_tags"] = [ + tag + for tag, marker in ( + ("incubation", "incubat"), + ("wash", "wash"), + ("centrifuge", "centrif"), + ("mix", "mix"), + ) + if marker in step_text.lower() + ] + return payload + + +class ExperimentAnomalyMarker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + reasons: list[str] = [] + temperature = payload.get("temperature_c") + ph_value = payload.get("ph_value") + yield_percent = payload.get("yield_percent") + step_text = str(payload.get("step_text") or "").lower() + if temperature is not None and not 2 <= float(temperature) <= 40: + reasons.append("temperature_out_of_range") + if ph_value is not None and not 5 <= float(ph_value) <= 9: + reasons.append("ph_out_of_range") + if yield_percent is not None and float(yield_percent) < 60: + reasons.append("yield_drop") + if any( + marker in step_text for marker in ("contamination", "failed", "repeat", "unexpected") + ): + reasons.append("manual_flag") + payload["anomaly_reasons"] = reasons + payload["review_status"] = "needs_review" if reasons else "normal" + return payload + + +class ExperimentReviewSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/experiment_review/pipeline.py b/apps/src/sage/apps/experiment_review/pipeline.py new file mode 100644 index 0000000..4094a97 --- /dev/null +++ b/apps/src/sage/apps/experiment_review/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ExperimentAnomalyMarker, + ExperimentLogSource, + ExperimentParameterExtractor, + ExperimentReviewSink, + ExperimentStepSplitter, +) + + +def run_experiment_review_pipeline(log_file: str, output_file: str) -> None: + env = LocalEnvironment("experiment_review") + ( + env.from_batch(ExperimentLogSource, log_file=log_file) + .flatmap(ExperimentStepSplitter) + .map(ExperimentParameterExtractor) + .map(ExperimentAnomalyMarker) + .sink(ExperimentReviewSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/export_transformer/README.md b/apps/src/sage/apps/export_transformer/README.md new file mode 100644 index 0000000..5cf8b6d --- /dev/null +++ b/apps/src/sage/apps/export_transformer/README.md @@ -0,0 +1,5 @@ +# Export Transformer + +读取导出请求，统一字段并转换成目标导出格式。 + +输入：CSV 或 JSON 导出记录。 输出：包含转换后载荷的 JSON 文件。 diff --git a/apps/src/sage/apps/export_transformer/__init__.py b/apps/src/sage/apps/export_transformer/__init__.py new file mode 100644 index 0000000..eb98ef3 --- /dev/null +++ b/apps/src/sage/apps/export_transformer/__init__.py @@ -0,0 +1,5 @@ +"""Export transformer application.""" + +from .pipeline import run_export_transformer_pipeline + +__all__ = ["run_export_transformer_pipeline"] diff --git a/apps/src/sage/apps/export_transformer/operators.py b/apps/src/sage/apps/export_transformer/operators.py new file mode 100644 index 0000000..f2d09f3 --- /dev/null +++ b/apps/src/sage/apps/export_transformer/operators.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class ExportQuerySource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class ExportFieldMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + mapped = { + "record_id": item.get("record_id") + or item.get("id") + or item.get("query_id") + or "unknown", + "name": item.get("name") or item.get("title") or item.get("label") or "", + "value": item.get("value") or item.get("amount") or item.get("score") or "", + } + item["export_record"] = mapped + return item + + +class FormatTransformer(MapFunction): + def __init__(self, output_format: str = "json", **kwargs): + super().__init__(**kwargs) + self.output_format = output_format.lower() + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + record = item.get("export_record", {}) + if self.output_format == "csv": + item["transformed_payload"] = ",".join( + str(record.get(field, "")) for field in ("record_id", "name", "value") + ) + else: + item["transformed_payload"] = json.dumps(record, ensure_ascii=False) + item["output_format"] = self.output_format + return item + + +class ExportSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/export_transformer/pipeline.py b/apps/src/sage/apps/export_transformer/pipeline.py new file mode 100644 index 0000000..e5d4893 --- /dev/null +++ b/apps/src/sage/apps/export_transformer/pipeline.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ExportFieldMapper, ExportQuerySource, ExportSink, FormatTransformer + + +def run_export_transformer_pipeline( + input_file: str, output_file: str, output_format: str = "json" +) -> None: + env = LocalEnvironment("export_transformer") + ( + env.from_batch(ExportQuerySource, input_file=input_file) + .map(ExportFieldMapper) + .map(FormatTransformer, output_format=output_format) + .sink(ExportSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/factory_watch/README.md b/apps/src/sage/apps/factory_watch/README.md new file mode 100644 index 0000000..929a93c --- /dev/null +++ b/apps/src/sage/apps/factory_watch/README.md @@ -0,0 +1,6 @@ +# 产线传感器异常看护系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_factory_watch_pipeline` +- Entry script: `examples/run_factory_watch.py` diff --git a/apps/src/sage/apps/factory_watch/__init__.py b/apps/src/sage/apps/factory_watch/__init__.py new file mode 100644 index 0000000..50a58ed --- /dev/null +++ b/apps/src/sage/apps/factory_watch/__init__.py @@ -0,0 +1,5 @@ +"""产线传感器异常看护系统 application.""" + +from .pipeline import run_factory_watch_pipeline + +__all__ = ["run_factory_watch_pipeline"] diff --git a/apps/src/sage/apps/factory_watch/operators.py b/apps/src/sage/apps/factory_watch/operators.py new file mode 100644 index 0000000..47f4502 --- /dev/null +++ b/apps/src/sage/apps/factory_watch/operators.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class SensorSource(ListBatchSource): + def __init__(self, sensor_file: str, **kwargs): + super().__init__(**kwargs) + self.sensor_file = sensor_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.sensor_file) + for item in items: + item.setdefault("app_slug", "factory_watch") + item.setdefault("source_path", self.sensor_file) + return items + + +class SensorDeviceMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["device_id"] = str(payload.get("device_id") or payload.get("machine") or "unknown") + payload["station"] = str(payload.get("station") or payload.get("line") or "line-a") + payload["temperature_c"] = _to_float(payload.get("temperature_c")) + payload["pressure_kpa"] = _to_float(payload.get("pressure_kpa")) + payload["vibration_mm_s"] = _to_float(payload.get("vibration_mm_s")) + return payload + + +class SensorAnomalyScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + issues: list[str] = [] + score = 0.0 + if payload.get("temperature_c", 0.0) > 85: + issues.append("high_temperature") + score += 4 + if payload.get("pressure_kpa", 0.0) > 220: + issues.append("high_pressure") + score += 3 + if payload.get("vibration_mm_s", 0.0) > 12: + issues.append("high_vibration") + score += 4 + payload["anomaly_flags"] = issues + payload["anomaly_score"] = score + return payload + + +class SensorAlertLeveler(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + level = "normal" + if payload.get("anomaly_score", 0.0) >= 7: + level = "critical" + elif payload.get("anomaly_score", 0.0) >= 3: + level = "watch" + payload["alert_level"] = level + payload["inspection_hint"] = ( + "check cooling and bearing health" + if level != "normal" + else "continue routine monitoring" + ) + payload["alert_summary"] = ( + f"Device {payload.get('device_id')} at {payload.get('station')} level {level}, " + f"flags {', '.join(payload.get('anomaly_flags') or []) or 'none'}." + ) + return payload + + +class SensorWatchSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/factory_watch/pipeline.py b/apps/src/sage/apps/factory_watch/pipeline.py new file mode 100644 index 0000000..140c202 --- /dev/null +++ b/apps/src/sage/apps/factory_watch/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + SensorAlertLeveler, + SensorAnomalyScorer, + SensorDeviceMapper, + SensorSource, + SensorWatchSink, +) + + +def run_factory_watch_pipeline(sensor_file: str, output_file: str) -> None: + env = LocalEnvironment("factory_watch") + ( + env.from_batch(SensorSource, sensor_file=sensor_file) + .map(SensorDeviceMapper) + .map(SensorAnomalyScorer) + .map(SensorAlertLeveler) + .sink(SensorWatchSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/feedback_analyzer/README.md b/apps/src/sage/apps/feedback_analyzer/README.md new file mode 100644 index 0000000..030a62d --- /dev/null +++ b/apps/src/sage/apps/feedback_analyzer/README.md @@ -0,0 +1,26 @@ +# Feedback Analyzer + +客户反馈关键词提取示例应用。 + +## 功能 + +- 读取文本或 CSV 反馈 +- 清洗输入文本 +- 分词并统计高频关键词 +- 输出 JSON 统计结果 + +## 用法 + +```bash +python examples/run_feedback_analyzer.py \ + --feedback-file feedback.txt \ + --output keywords.json +``` + +```bash +python examples/run_feedback_analyzer.py \ + --feedback-file feedback.csv \ + --output keywords.json \ + --top-n 100 \ + --verbose +``` diff --git a/apps/src/sage/apps/feedback_analyzer/__init__.py b/apps/src/sage/apps/feedback_analyzer/__init__.py new file mode 100644 index 0000000..aaa35da --- /dev/null +++ b/apps/src/sage/apps/feedback_analyzer/__init__.py @@ -0,0 +1,5 @@ +"""Feedback analyzer application.""" + +from .pipeline import run_feedback_analyzer_pipeline + +__all__ = ["run_feedback_analyzer_pipeline"] diff --git a/apps/src/sage/apps/feedback_analyzer/operators.py b/apps/src/sage/apps/feedback_analyzer/operators.py new file mode 100644 index 0000000..cdfb7b3 --- /dev/null +++ b/apps/src/sage/apps/feedback_analyzer/operators.py @@ -0,0 +1,323 @@ +""" +Feedback Analyzer Operators + +Custom operators for customer feedback analysis and keyword extraction. +""" + +from __future__ import annotations + +import json +import re +from collections import Counter +from datetime import datetime +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import CustomLogger, FlatMapFunction, MapFunction, SinkFunction + + +class FeedbackSource(ListBatchSource): + """Read feedback data from text file or CSV.""" + + def __init__(self, feedback_file: str, delimiter: str = "\t", **kwargs): + """Initialize feedback source. + + Args: + feedback_file: Path to feedback file + delimiter: Delimiter for CSV format (default: tab) + """ + super().__init__(**kwargs) + self.feedback_file = feedback_file + self.delimiter = delimiter + self._logger = CustomLogger("FeedbackSource") + + def load_items(self) -> list[dict[str, str]]: + """Read and return all feedback entries.""" + import csv + + feedbacks = [] + try: + with open(self.feedback_file, encoding="utf-8") as f: + # Try CSV format first + reader = csv.DictReader(f, delimiter=self.delimiter) + if reader.fieldnames and len(reader.fieldnames) > 1: + for row in reader: + feedbacks.append(row) + else: + # Fall back to line-by-line + f.seek(0) + for i, line in enumerate(f): + feedbacks.append( + { + "id": str(i), + "text": line.strip(), + } + ) + + self.logger.info(f"Read {len(feedbacks)} feedback entries") + return feedbacks + except Exception as e: + self.logger.error(f"Error reading feedback file: {e}") + return [] + + +class TextCleaner(MapFunction): + """Clean and preprocess feedback text.""" + + def __init__(self, **kwargs): + """Initialize text cleaner.""" + super().__init__(**kwargs) + + def execute(self, feedback: dict[str, str]) -> dict[str, str]: + """Clean feedback text. + + Args: + feedback: Feedback entry + + Returns: + Feedback with cleaned text + """ + if not feedback or "text" not in feedback: + return feedback + + text = feedback.get("text", "") + + # Remove URLs + text = re.sub( + r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+", + "", + text, + ) + + # Remove email addresses + text = re.sub(r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}", "", text) + + # Remove special characters but keep spaces + text = re.sub(r"[^\w\s\u4e00-\u9fff]", " ", text) + + # Remove extra whitespace + text = " ".join(text.split()) + + feedback["cleaned_text"] = text + return feedback + + +class SimpleTokenizer(FlatMapFunction): + """Tokenize text into words (simplified tokenizer).""" + + def __init__(self, min_length: int = 2, **kwargs): + """Initialize tokenizer. + + Args: + min_length: Minimum token length + """ + super().__init__(**kwargs) + self.min_length = min_length + + def execute(self, feedback: dict[str, str]) -> list[dict[str, Any]]: + """Tokenize feedback text. + + Args: + feedback: Feedback entry + + Returns: + List of token entries + """ + if not feedback or "cleaned_text" not in feedback: + return [] + + text = feedback.get("cleaned_text", "").lower() + feedback_id = feedback.get("id", "") + + # Simple space-based tokenization + # For Chinese, this is a simplification; production would use jieba or similar + tokens = [t for t in text.split() if len(t) >= self.min_length] + + # Also extract common Chinese bigrams if present + if any("\u4e00" <= c <= "\u9fff" for c in text): + # Simple 2-character extraction + for i in range(len(text) - 1): + if all("\u4e00" <= c <= "\u9fff" for c in text[i : i + 2]): + tokens.append(text[i : i + 2]) + + return [ + { + "feedback_id": feedback_id, + "token": token, + "length": len(token), + } + for token in tokens + ] + + +class KeywordScorer(MapFunction): + """Score keywords based on frequency and relevance.""" + + def __init__(self, common_words: list[str] | None = None, **kwargs): + """Initialize keyword scorer. + + Args: + common_words: List of common words to exclude + """ + super().__init__(**kwargs) + self.common_words = set( + common_words + or [ + "the", + "a", + "an", + "is", + "are", + "was", + "were", + "be", + "have", + "has", + "this", + "that", + "with", + "from", + "your", + "their", + "our", + "for", + "and", + "but", + ] + ) + self.token_count = {} + self.feedback_count = {} + + def execute(self, token_entry: dict[str, Any]) -> dict[str, Any]: + """Score a token entry. + + Args: + token_entry: Token information + + Returns: + Token with score markers + """ + if not token_entry: + return token_entry + + token = token_entry.get("token", "").lower() + + # Skip common words + if token in self.common_words: + token_entry["is_stopword"] = True + return token_entry + + # Count frequency + self.token_count[token] = self.token_count.get(token, 0) + 1 + + # Mark non-stopword + token_entry["is_stopword"] = False + + return token_entry + + +class KeywordExtractor(MapFunction): + """Extract top keywords from aggregated data.""" + + def __init__(self, top_n: int = 20, **kwargs): + """Initialize keyword extractor. + + Args: + top_n: Number of top keywords to extract + """ + super().__init__(**kwargs) + self.top_n = top_n + self._logger = CustomLogger("KeywordExtractor") + self.keyword_stats = {} + + def execute(self, token_entry: dict[str, Any]) -> dict[str, Any] | None: + """Extract keyword information. + + Args: + token_entry: Token entry + + Returns: + Enhanced token or None + """ + if not token_entry or token_entry.get("is_stopword"): + return None + + token = token_entry.get("token", "") + if token not in self.keyword_stats: + self.keyword_stats[token] = { + "count": 0, + "feedback_count": 1, + "first_seen": datetime.now().isoformat(), + } + + self.keyword_stats[token]["count"] += 1 + + token_entry["keyword_score"] = self.keyword_stats[token]["count"] + + return token_entry + + def teardown(self, context: Any) -> None: + """Log top keywords.""" + if self.keyword_stats: + top_keywords = sorted( + self.keyword_stats.items(), key=lambda x: x[1]["count"], reverse=True + )[: self.top_n] + + self.logger.info(f"Top {len(top_keywords)} keywords extracted:") + for keyword, stats in top_keywords: + self.logger.info(f" {keyword}: {stats['count']} occurrences") + + +class StatisticsSink(SinkFunction): + """Output keyword statistics to JSON file.""" + + def __init__(self, output_file: str, **kwargs): + """Initialize statistics sink. + + Args: + output_file: Path to output JSON file + """ + super().__init__(**kwargs) + self.output_file = output_file + self._logger = CustomLogger("StatisticsSink") + self.keywords = Counter() + self.total_tokens = 0 + + def execute(self, token_entry: dict[str, Any]) -> None: + """Process token entry. + + Args: + token_entry: Token with score + """ + if not token_entry: + return + + token = token_entry.get("token", "") + score = token_entry.get("keyword_score", 1) + + self.keywords[token] += score + self.total_tokens += 1 + + def teardown(self, context: Any) -> None: + """Write statistics to file.""" + stats = { + "total_tokens": self.total_tokens, + "unique_keywords": len(self.keywords), + "generated_at": datetime.now().isoformat(), + "top_keywords": [ + { + "keyword": keyword, + "count": count, + "percentage": round(count / self.total_tokens * 100, 2), + } + for keyword, count in self.keywords.most_common(50) + ], + } + + with open(self.output_file, "w", encoding="utf-8") as f: + json.dump(stats, f, ensure_ascii=False, indent=2) + + self.logger.info(f"Statistics written to {self.output_file}") + self.logger.info( + f"Processed {self.total_tokens} tokens, {len(self.keywords)} unique keywords" + ) diff --git a/apps/src/sage/apps/feedback_analyzer/pipeline.py b/apps/src/sage/apps/feedback_analyzer/pipeline.py new file mode 100644 index 0000000..d317402 --- /dev/null +++ b/apps/src/sage/apps/feedback_analyzer/pipeline.py @@ -0,0 +1,77 @@ +""" +Feedback Analyzer Pipeline + +Main pipeline implementation using SAGE operators for customer feedback analysis. +""" + +from __future__ import annotations + +from sage.foundation import CustomLogger +from sage.runtime import LocalEnvironment + +from .operators import ( + FeedbackSource, + KeywordExtractor, + KeywordScorer, + SimpleTokenizer, + StatisticsSink, + TextCleaner, +) + + +def run_feedback_analyzer_pipeline( + feedback_file: str, + output_file: str = "feedback_keywords.json", + top_n: int = 50, + verbose: bool = False, +) -> None: + """ + Run the feedback analyzer pipeline using SAGE framework. + + Args: + feedback_file: Path to feedback file + output_file: Path to output JSON statistics file + top_n: Number of top keywords to include in output + verbose: Enable verbose logging + + Example: + >>> run_feedback_analyzer_pipeline( + ... feedback_file="feedback.txt", + ... output_file="keywords.json", + ... top_n=50, + ... verbose=True + ... ) + """ + logger = CustomLogger("FeedbackAnalyzerPipeline") + + if verbose: + logger.info("Starting feedback analyzer pipeline") + logger.info(f" Feedback file: {feedback_file}") + logger.info(f" Output file: {output_file}") + logger.info(f" Top keywords: {top_n}") + + # Create environment + env = LocalEnvironment("feedback_analyzer") + + try: + # Build pipeline + ( + env.from_batch(FeedbackSource, feedback_file=feedback_file) + .map(TextCleaner) + .flatmap(SimpleTokenizer, min_length=2) + .map(KeywordScorer) + .filter(lambda token: token is not None) + .map(KeywordExtractor, top_n=top_n) + .filter(lambda token: token is not None) + .sink(StatisticsSink, output_file=output_file) + ) + + # Submit and run + env.submit(autostop=True) + + if verbose: + logger.info("Feedback analyzer pipeline completed successfully") + + except Exception as e: + logger.error(f"Error in feedback analyzer pipeline: {e}") + raise diff --git a/apps/src/sage/apps/geo_recommendation/README.md b/apps/src/sage/apps/geo_recommendation/README.md new file mode 100644 index 0000000..f93b334 --- /dev/null +++ b/apps/src/sage/apps/geo_recommendation/README.md @@ -0,0 +1,3 @@ +# Geo Recommendation + +地理位置智能推荐应用。 diff --git a/apps/src/sage/apps/geo_recommendation/__init__.py b/apps/src/sage/apps/geo_recommendation/__init__.py new file mode 100644 index 0000000..c505809 --- /dev/null +++ b/apps/src/sage/apps/geo_recommendation/__init__.py @@ -0,0 +1,5 @@ +"""Geo recommendation application.""" + +from .pipeline import run_geo_recommendation_pipeline + +__all__ = ["run_geo_recommendation_pipeline"] diff --git a/apps/src/sage/apps/geo_recommendation/operators.py b/apps/src/sage/apps/geo_recommendation/operators.py new file mode 100644 index 0000000..e27c450 --- /dev/null +++ b/apps/src/sage/apps/geo_recommendation/operators.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class LocationSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class DistanceScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + distance = float(item.get("distance_km") or 0) + rating = float(item.get("rating") or 0) + item["recommendation_score"] = round(max(0.0, 10 - distance) + rating * 2, 2) + return item + + +class RecommendationFormatter(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = float(item.get("recommendation_score", 0)) + item["tier"] = "primary" if score >= 12 else "secondary" if score >= 8 else "candidate" + return item + + +class RecommendationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + ordered = sorted( + self.items, key=lambda value: value.get("recommendation_score", 0), reverse=True + ) + Path(self.output_file).write_text( + json.dumps(ordered, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/geo_recommendation/pipeline.py b/apps/src/sage/apps/geo_recommendation/pipeline.py new file mode 100644 index 0000000..e20f253 --- /dev/null +++ b/apps/src/sage/apps/geo_recommendation/pipeline.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import DistanceScorer, LocationSource, RecommendationFormatter, RecommendationSink + + +def run_geo_recommendation_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("geo_recommendation") + ( + env.from_batch(LocationSource, input_file=input_file) + .map(DistanceScorer) + .map(RecommendationFormatter) + .sink(RecommendationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/grant_subscription/README.md b/apps/src/sage/apps/grant_subscription/README.md new file mode 100644 index 0000000..fd9cbb4 --- /dev/null +++ b/apps/src/sage/apps/grant_subscription/README.md @@ -0,0 +1,6 @@ +# 科研资助机会订阅系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_grant_subscription_pipeline` +- Entry script: `examples/run_grant_subscription.py` diff --git a/apps/src/sage/apps/grant_subscription/__init__.py b/apps/src/sage/apps/grant_subscription/__init__.py new file mode 100644 index 0000000..ac84a73 --- /dev/null +++ b/apps/src/sage/apps/grant_subscription/__init__.py @@ -0,0 +1,5 @@ +"""科研资助机会订阅系统 application.""" + +from .pipeline import run_grant_subscription_pipeline + +__all__ = ["run_grant_subscription_pipeline"] diff --git a/apps/src/sage/apps/grant_subscription/operators.py b/apps/src/sage/apps/grant_subscription/operators.py new file mode 100644 index 0000000..a91a523 --- /dev/null +++ b/apps/src/sage/apps/grant_subscription/operators.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _tokenize(text: str) -> set[str]: + return set(re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", text.lower())) + + +def _parse_int(value: Any, default: int = 0) -> int: + try: + return int(float(value)) + except (TypeError, ValueError): + return default + + +class GrantAnnouncementSource(ListBatchSource): + def __init__(self, announcement_file: str, profile_file: str, **kwargs): + super().__init__(**kwargs) + self.announcement_file = announcement_file + self.profile_file = profile_file + + def load_items(self) -> list[dict[str, Any]]: + announcements = _load_records(self.announcement_file) + profiles = _load_records(self.profile_file) + for item in announcements: + item["team_profiles"] = profiles + return announcements + + +class GrantRuleExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + call_text = " ".join( + str(payload.get(field, "")) + for field in ("title", "summary", "focus_area", "eligibility", "keywords") + ) + tokens = _tokenize(call_text) + payload["grant_id"] = str( + payload.get("grant_id") or payload.get("id") or payload.get("title") or "grant" + ) + payload["matched_topics"] = sorted(tokens)[:15] + payload["deadline_days"] = _parse_int( + payload.get("deadline_days") or payload.get("days_left"), 999 + ) + payload["budget_amount"] = _parse_int( + payload.get("budget_amount") or payload.get("budget") or 0, 0 + ) + payload["eligibility_flags"] = { + "industry_only": "industry" in tokens, + "phd_required": "phd" in tokens, + "international": "international" in tokens, + } + return payload + + +class TeamProfileMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + matched_topics = set(payload.get("matched_topics") or []) + ranked_matches: list[dict[str, Any]] = [] + for profile in payload.get("team_profiles") or []: + interests = _tokenize( + " ".join( + str(profile.get(field, "")) + for field in ("research_focus", "keywords", "strengths") + ) + ) + overlap = sorted(matched_topics & interests) + if not overlap: + continue + team_size = _parse_int(profile.get("team_size") or 0) + score = len(overlap) * 3 + min(team_size, 8) + ranked_matches.append( + { + "team": profile.get("team") + or profile.get("lab") + or profile.get("name") + or "unknown", + "overlap_terms": overlap, + "match_score": score, + } + ) + ranked_matches.sort(key=lambda entry: (-entry["match_score"], entry["team"])) + payload["recommended_teams"] = ranked_matches[:3] + return payload + + +class GrantPriorityScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + team_score = ( + payload.get("recommended_teams", [{}])[0].get("match_score", 0) + if payload.get("recommended_teams") + else 0 + ) + urgency_bonus = 4 if int(payload.get("deadline_days") or 999) <= 14 else 1 + budget_bonus = 3 if int(payload.get("budget_amount") or 0) >= 500000 else 1 + priority_score = int(team_score) + urgency_bonus + budget_bonus + priority = "watch" + if priority_score >= 12: + priority = "high" + elif priority_score >= 7: + priority = "medium" + payload["priority_score"] = priority_score + payload["priority_level"] = priority + payload["dispatch_summary"] = ( + f"{payload.get('grant_id')} 推荐 {len(payload.get('recommended_teams') or [])} 个团队，" + f"优先级 {priority}。" + ) + return payload + + +class GrantAlertSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("team_profiles", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/grant_subscription/pipeline.py b/apps/src/sage/apps/grant_subscription/pipeline.py new file mode 100644 index 0000000..fc89ce1 --- /dev/null +++ b/apps/src/sage/apps/grant_subscription/pipeline.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + GrantAlertSink, + GrantAnnouncementSource, + GrantPriorityScorer, + GrantRuleExtractor, + TeamProfileMatcher, +) + + +def run_grant_subscription_pipeline( + announcement_file: str, profile_file: str, output_file: str +) -> None: + env = LocalEnvironment("grant_subscription") + ( + env.from_batch( + GrantAnnouncementSource, announcement_file=announcement_file, profile_file=profile_file + ) + .map(GrantRuleExtractor) + .map(TeamProfileMatcher) + .map(GrantPriorityScorer) + .sink(GrantAlertSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/greenhouse_assistant/README.md b/apps/src/sage/apps/greenhouse_assistant/README.md new file mode 100644 index 0000000..d63c792 --- /dev/null +++ b/apps/src/sage/apps/greenhouse_assistant/README.md @@ -0,0 +1,6 @@ +# 温室种植协同助手 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_greenhouse_assistant_pipeline` +- Entry script: `examples/run_greenhouse_assistant.py` diff --git a/apps/src/sage/apps/greenhouse_assistant/__init__.py b/apps/src/sage/apps/greenhouse_assistant/__init__.py new file mode 100644 index 0000000..1d789ce --- /dev/null +++ b/apps/src/sage/apps/greenhouse_assistant/__init__.py @@ -0,0 +1,5 @@ +"""温室种植协同助手 application.""" + +from .pipeline import run_greenhouse_assistant_pipeline + +__all__ = ["run_greenhouse_assistant_pipeline"] diff --git a/apps/src/sage/apps/greenhouse_assistant/operators.py b/apps/src/sage/apps/greenhouse_assistant/operators.py new file mode 100644 index 0000000..2831f90 --- /dev/null +++ b/apps/src/sage/apps/greenhouse_assistant/operators.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class GreenhouseSensorSource(ListBatchSource): + def __init__(self, sensor_file: str, task_file: str, **kwargs): + super().__init__(**kwargs) + self.sensor_file = sensor_file + self.task_file = task_file + + def load_items(self) -> list[dict[str, Any]]: + sensors = _load_records(self.sensor_file) + tasks = _load_records(self.task_file) + task_index = {str(item.get("zone") or item.get("bed") or ""): item for item in tasks} + for item in sensors: + zone = str(item.get("zone") or item.get("bed") or "") + item.setdefault("app_slug", "greenhouse_assistant") + item["zone_task"] = task_index.get(zone, {}) + item.setdefault("source_path", self.sensor_file) + return sensors + + +class GreenhouseZoneMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["zone"] = str(payload.get("zone") or payload.get("bed") or "unknown") + payload["temperature_c"] = _to_float(payload.get("temperature_c")) + payload["humidity_pct"] = _to_float(payload.get("humidity_pct")) + payload["soil_moisture_pct"] = _to_float(payload.get("soil_moisture_pct")) + payload["crop"] = str(payload.get("crop") or payload.get("variety") or "general") + return payload + + +class ClimateAnomalyDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + issues: list[str] = [] + if payload.get("temperature_c", 0.0) > 32: + issues.append("high_temperature") + if payload.get("humidity_pct", 0.0) > 85: + issues.append("high_humidity") + if payload.get("soil_moisture_pct", 100.0) < 25: + issues.append("low_soil_moisture") + payload["climate_flags"] = issues + payload["coordination_level"] = ( + "urgent" if len(issues) >= 2 else "routine" if issues else "stable" + ) + return payload + + +class GreenhouseAdviceBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + zone_task = payload.get("zone_task") or {} + actions: list[str] = [] + if "low_soil_moisture" in (payload.get("climate_flags") or []): + actions.append("start_irrigation_cycle") + if "high_humidity" in (payload.get("climate_flags") or []): + actions.append("increase_ventilation") + if zone_task.get("inspection_required"): + actions.append("send_manual_inspection") + payload["recommended_actions"] = actions or ["continue_monitoring"] + payload["advice_summary"] = ( + f"Zone {payload.get('zone')} crop {payload.get('crop')} level {payload.get('coordination_level')}, " + f"actions {', '.join(payload.get('recommended_actions') or [])}." + ) + return payload + + +class GreenhouseSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("zone_task", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/greenhouse_assistant/pipeline.py b/apps/src/sage/apps/greenhouse_assistant/pipeline.py new file mode 100644 index 0000000..296fef6 --- /dev/null +++ b/apps/src/sage/apps/greenhouse_assistant/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ClimateAnomalyDetector, + GreenhouseAdviceBuilder, + GreenhouseSensorSource, + GreenhouseSink, + GreenhouseZoneMapper, +) + + +def run_greenhouse_assistant_pipeline(sensor_file: str, task_file: str, output_file: str) -> None: + env = LocalEnvironment("greenhouse_assistant") + ( + env.from_batch(GreenhouseSensorSource, sensor_file=sensor_file, task_file=task_file) + .map(GreenhouseZoneMapper) + .map(ClimateAnomalyDetector) + .map(GreenhouseAdviceBuilder) + .sink(GreenhouseSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/interview_coach/README.md b/apps/src/sage/apps/interview_coach/README.md new file mode 100644 index 0000000..416516a --- /dev/null +++ b/apps/src/sage/apps/interview_coach/README.md @@ -0,0 +1,6 @@ +# 岗位面试模拟教练系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_interview_coach_pipeline` +- Entry script: `examples/run_interview_coach.py` diff --git a/apps/src/sage/apps/interview_coach/__init__.py b/apps/src/sage/apps/interview_coach/__init__.py new file mode 100644 index 0000000..f3a7965 --- /dev/null +++ b/apps/src/sage/apps/interview_coach/__init__.py @@ -0,0 +1,5 @@ +"""岗位面试模拟教练系统 application.""" + +from .pipeline import run_interview_coach_pipeline + +__all__ = ["run_interview_coach_pipeline"] diff --git a/apps/src/sage/apps/interview_coach/operators.py b/apps/src/sage/apps/interview_coach/operators.py new file mode 100644 index 0000000..35e287f --- /dev/null +++ b/apps/src/sage/apps/interview_coach/operators.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _tokenize(text: str) -> set[str]: + return set(re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", text.lower())) + + +class InterviewAnswerSource(ListBatchSource): + def __init__(self, answer_file: str, rubric_file: str, **kwargs): + super().__init__(**kwargs) + self.answer_file = answer_file + self.rubric_file = rubric_file + + def load_items(self) -> list[dict[str, Any]]: + answers = _load_records(self.answer_file) + rubric = _load_records(self.rubric_file) + for item in answers: + item["rubric"] = rubric + return answers + + +class InterviewQuestionMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + question = str(payload.get("question") or payload.get("prompt") or "") + answer = str(payload.get("answer") or payload.get("text") or "") + payload["question_terms"] = sorted(_tokenize(question)) + payload["answer_terms"] = sorted(_tokenize(answer)) + payload["uses_star_structure"] = all( + marker in answer.lower() for marker in ("situation", "task", "action", "result") + ) + return payload + + +class InterviewScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + score = 0 + if payload.get("uses_star_structure"): + score += 4 + if len(payload.get("answer_terms") or []) >= 12: + score += 3 + if any(char.isdigit() for char in str(payload.get("answer") or "")): + score += 2 + rubric_hits = 0 + answer_terms = set(payload.get("answer_terms") or []) + for criterion in payload.get("rubric") or []: + if answer_terms & _tokenize( + str(criterion.get("criterion") or criterion.get("text") or "") + ): + rubric_hits += 1 + payload["rubric_hits"] = rubric_hits + payload["interview_score"] = score + rubric_hits + payload["performance_level"] = "strong" if payload["interview_score"] >= 8 else "developing" + return payload + + +class InterviewAdviceBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + advice: list[str] = [] + if not payload.get("uses_star_structure"): + advice.append("补齐 Situation/Task/Action/Result 结构。") + if payload.get("rubric_hits", 0) < 2: + advice.append("多覆盖岗位 rubric 中的关键能力点。") + if not any(char.isdigit() for char in str(payload.get("answer") or "")): + advice.append("补充量化结果，增强说服力。") + payload["coaching_advice"] = advice or ["回答结构较完整，可继续打磨表达精炼度。"] + return payload + + +class InterviewReportSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("rubric", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/interview_coach/pipeline.py b/apps/src/sage/apps/interview_coach/pipeline.py new file mode 100644 index 0000000..6d78140 --- /dev/null +++ b/apps/src/sage/apps/interview_coach/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + InterviewAdviceBuilder, + InterviewAnswerSource, + InterviewQuestionMapper, + InterviewReportSink, + InterviewScorer, +) + + +def run_interview_coach_pipeline(answer_file: str, rubric_file: str, output_file: str) -> None: + env = LocalEnvironment("interview_coach") + ( + env.from_batch(InterviewAnswerSource, answer_file=answer_file, rubric_file=rubric_file) + .map(InterviewQuestionMapper) + .map(InterviewScorer) + .map(InterviewAdviceBuilder) + .sink(InterviewReportSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/inventory_alert/README.md b/apps/src/sage/apps/inventory_alert/README.md new file mode 100644 index 0000000..df2a370 --- /dev/null +++ b/apps/src/sage/apps/inventory_alert/README.md @@ -0,0 +1,3 @@ +# Inventory Alert + +库存异常告警应用。 diff --git a/apps/src/sage/apps/inventory_alert/__init__.py b/apps/src/sage/apps/inventory_alert/__init__.py new file mode 100644 index 0000000..38a3dc1 --- /dev/null +++ b/apps/src/sage/apps/inventory_alert/__init__.py @@ -0,0 +1,5 @@ +"""Inventory alert application.""" + +from .pipeline import run_inventory_alert_pipeline + +__all__ = ["run_inventory_alert_pipeline"] diff --git a/apps/src/sage/apps/inventory_alert/operators.py b/apps/src/sage/apps/inventory_alert/operators.py new file mode 100644 index 0000000..1945603 --- /dev/null +++ b/apps/src/sage/apps/inventory_alert/operators.py @@ -0,0 +1,91 @@ +"""Operators for inventory alerting.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class InventorySource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class InventoryComparator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + current_stock = float(item.get("current_stock") or 0) + reorder_point = float(item.get("reorder_point") or 0) + max_stock = float(item.get("max_stock") or reorder_point * 3 or 100) + item["current_stock"] = current_stock + item["reorder_point"] = reorder_point + item["max_stock"] = max_stock + item["status"] = ( + "low" + if current_stock < reorder_point + else "high" + if current_stock > max_stock + else "normal" + ) + return item + + +class InventoryFeatureBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + enriched = InventoryComparator().execute(item) + gap = enriched["current_stock"] - enriched["reorder_point"] + enriched["inventory_gap"] = round(gap, 2) + return enriched + + +class AlertGenerator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + status = item.get("status", "normal") + if status == "low": + item["alert_message"] = "Reorder required" + elif status == "high": + item["alert_message"] = "Excess inventory detected" + else: + item["alert_message"] = "Inventory healthy" + return item + + +class InventoryAnomalyScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + status = item.get("status", "normal") + item["anomaly_score"] = 2 if status == "low" else 1 if status == "high" else 0 + return item + + +class AlertLevelMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item = AlertGenerator().execute(item) + score = int(item.get("anomaly_score", 0)) + item["alert_level"] = "high" if score >= 2 else "medium" if score == 1 else "low" + return item + + +class AlertSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/inventory_alert/pipeline.py b/apps/src/sage/apps/inventory_alert/pipeline.py new file mode 100644 index 0000000..f27cc7b --- /dev/null +++ b/apps/src/sage/apps/inventory_alert/pipeline.py @@ -0,0 +1,27 @@ +"""Inventory alert pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + AlertLevelMapper, + AlertSink, + InventoryAnomalyScorer, + InventoryFeatureBuilder, + InventorySource, +) + + +def run_inventory_alert_pipeline( + input_file: str, output_file: str, config_path: str | None = None +) -> None: + env = LocalEnvironment("inventory_alert") + ( + env.from_batch(InventorySource, input_file=input_file) + .map(InventoryFeatureBuilder) + .map(InventoryAnomalyScorer) + .map(AlertLevelMapper) + .sink(AlertSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/invoice_reconciliation/README.md b/apps/src/sage/apps/invoice_reconciliation/README.md new file mode 100644 index 0000000..beb04da --- /dev/null +++ b/apps/src/sage/apps/invoice_reconciliation/README.md @@ -0,0 +1,5 @@ +# Invoice Reconciliation + +读取发票和订单数据，统一字段后做金额对账。 + +输入：发票文件，可选订单文件。 输出：包含匹配状态和差额的 JSON 文件。 diff --git a/apps/src/sage/apps/invoice_reconciliation/__init__.py b/apps/src/sage/apps/invoice_reconciliation/__init__.py new file mode 100644 index 0000000..48d1aa7 --- /dev/null +++ b/apps/src/sage/apps/invoice_reconciliation/__init__.py @@ -0,0 +1,5 @@ +"""Invoice reconciliation application.""" + +from .pipeline import run_invoice_reconciliation_pipeline + +__all__ = ["run_invoice_reconciliation_pipeline"] diff --git a/apps/src/sage/apps/invoice_reconciliation/operators.py b/apps/src/sage/apps/invoice_reconciliation/operators.py new file mode 100644 index 0000000..ae7f87e --- /dev/null +++ b/apps/src/sage/apps/invoice_reconciliation/operators.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class InvoiceSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class OrderSource(MapFunction): + def __init__(self, order_file: str | None = None, **kwargs): + super().__init__(**kwargs) + self.order_file = order_file + self._orders = _load_records(order_file) if order_file else [] + self._order_index = { + str( + order.get("order_id") or order.get("id") or order.get("invoice_order_id") or "" + ).strip(): order + for order in self._orders + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + order_id = str( + item.get("order_id") + or item.get("invoice_order_id") + or item.get("reference_order_id") + or "" + ).strip() + item["linked_order"] = self._order_index.get(order_id, {}) + return item + + +class ReconciliationFieldNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["invoice_id"] = item.get("invoice_id") or item.get("id") or "unknown" + item["order_id"] = str(item.get("order_id") or item.get("invoice_order_id") or "").strip() + item["invoice_amount"] = round( + float(item.get("invoice_amount") or item.get("amount") or 0), 2 + ) + linked_order = item.get("linked_order") or {} + item["order_amount"] = round( + float(linked_order.get("order_amount") or linked_order.get("amount") or 0), 2 + ) + return item + + +class InvoiceMatcher(MapFunction): + def __init__(self, tolerance: float = 1.0, **kwargs): + super().__init__(**kwargs) + self.tolerance = tolerance + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + delta = round(abs(item.get("invoice_amount", 0) - item.get("order_amount", 0)), 2) + item["amount_delta"] = delta + item["reconciliation_status"] = ( + "matched" if delta <= self.tolerance and item.get("linked_order") else "unmatched" + ) + return item + + +class ReconciliationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/invoice_reconciliation/pipeline.py b/apps/src/sage/apps/invoice_reconciliation/pipeline.py new file mode 100644 index 0000000..28c4c7c --- /dev/null +++ b/apps/src/sage/apps/invoice_reconciliation/pipeline.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + InvoiceMatcher, + InvoiceSource, + OrderSource, + ReconciliationFieldNormalizer, + ReconciliationSink, +) + + +def run_invoice_reconciliation_pipeline( + input_file: str, output_file: str, order_file: str | None = None, tolerance: float = 1.0 +) -> None: + env = LocalEnvironment("invoice_reconciliation") + ( + env.from_batch(InvoiceSource, input_file=input_file) + .map(OrderSource, order_file=order_file) + .map(ReconciliationFieldNormalizer) + .map(InvoiceMatcher, tolerance=tolerance) + .sink(ReconciliationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/knowledge_cleanup/README.md b/apps/src/sage/apps/knowledge_cleanup/README.md new file mode 100644 index 0000000..f10aa10 --- /dev/null +++ b/apps/src/sage/apps/knowledge_cleanup/README.md @@ -0,0 +1,6 @@ +# 内部知识库去重整理系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_knowledge_cleanup_pipeline` +- Entry script: `examples/run_knowledge_cleanup.py` diff --git a/apps/src/sage/apps/knowledge_cleanup/__init__.py b/apps/src/sage/apps/knowledge_cleanup/__init__.py new file mode 100644 index 0000000..7cfa7d0 --- /dev/null +++ b/apps/src/sage/apps/knowledge_cleanup/__init__.py @@ -0,0 +1,5 @@ +"""内部知识库去重整理系统 application.""" + +from .pipeline import run_knowledge_cleanup_pipeline + +__all__ = ["run_knowledge_cleanup_pipeline"] diff --git a/apps/src/sage/apps/knowledge_cleanup/operators.py b/apps/src/sage/apps/knowledge_cleanup/operators.py new file mode 100644 index 0000000..ca04fd1 --- /dev/null +++ b/apps/src/sage/apps/knowledge_cleanup/operators.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import csv +import json +import re +from hashlib import sha1 +from pathlib import Path + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, str]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, str]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": str(item)} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _tokenize(text: str) -> list[str]: + return re.findall(r"[\w\-]{3,}", text.lower()) + + +class KnowledgeArticleSource(ListBatchSource): + def __init__(self, article_dir: str, **kwargs): + super().__init__(**kwargs) + self.article_dir = article_dir + + def load_items(self) -> list[dict[str, str]]: + items = _load_records(self.article_dir) + for item in items: + item.setdefault("app_slug", "knowledge_cleanup") + item.setdefault("source_path", self.article_dir) + return items + + +class KnowledgeFingerprintBuilder(MapFunction): + def execute(self, item: dict[str, str]) -> dict[str, str | list[str]]: + payload = dict(item) + text = str(payload.get("text") or payload.get("content") or payload.get("body") or "") + tokens = _tokenize(text) + fingerprint_basis = " ".join(sorted(set(tokens[:40]))) + payload["fingerprint"] = sha1(fingerprint_basis.encode("utf-8")).hexdigest()[:12] + payload["top_terms"] = sorted(set(tokens[:8])) + payload["text_length"] = len(text) + return payload + + +class KnowledgeDuplicateDetector(MapFunction): + def execute(self, item: dict[str, str | list[str]]) -> dict[str, str | list[str] | int]: + payload = dict(item) + top_terms = payload.get("top_terms") or [] + source = Path(str(payload.get("source_path") or "")).name.lower() + duplicate_signals = 0 + if len(top_terms) < 4: + duplicate_signals += 1 + if "copy" in source or "duplicate" in source: + duplicate_signals += 2 + payload["duplicate_risk_score"] = duplicate_signals + payload["duplicate_status"] = ( + "likely_duplicate" if duplicate_signals >= 2 else "unique_or_needs_review" + ) + return payload + + +class KnowledgeFreshnessScorer(MapFunction): + def execute(self, item: dict[str, str | list[str] | int]) -> dict[str, str | list[str] | int]: + payload = dict(item) + updated_at = str(payload.get("updated_at") or payload.get("last_updated") or "") + freshness_flags: list[str] = [] + if not updated_at: + freshness_flags.append("missing_update_date") + if int(payload.get("text_length") or 0) < 80: + freshness_flags.append("thin_content") + if str(payload.get("owner") or "").strip() == "": + freshness_flags.append("missing_owner") + payload["freshness_flags"] = freshness_flags + payload["cleanup_priority"] = ( + "high" + if freshness_flags or int(payload.get("duplicate_risk_score") or 0) >= 2 + else "normal" + ) + payload["cleanup_summary"] = ( + f"Fingerprint {payload.get('fingerprint')}, duplicate status {payload.get('duplicate_status')}, " + f"cleanup priority {payload.get('cleanup_priority')}." + ) + return payload + + +class KnowledgeCleanupSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, object]] = [] + + def execute(self, item: dict[str, object]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/knowledge_cleanup/pipeline.py b/apps/src/sage/apps/knowledge_cleanup/pipeline.py new file mode 100644 index 0000000..d945d9a --- /dev/null +++ b/apps/src/sage/apps/knowledge_cleanup/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + KnowledgeArticleSource, + KnowledgeCleanupSink, + KnowledgeDuplicateDetector, + KnowledgeFingerprintBuilder, + KnowledgeFreshnessScorer, +) + + +def run_knowledge_cleanup_pipeline(article_dir: str, output_file: str) -> None: + env = LocalEnvironment("knowledge_cleanup") + ( + env.from_batch(KnowledgeArticleSource, article_dir=article_dir) + .map(KnowledgeFingerprintBuilder) + .map(KnowledgeDuplicateDetector) + .map(KnowledgeFreshnessScorer) + .sink(KnowledgeCleanupSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/lab_turnaround_alert/README.md b/apps/src/sage/apps/lab_turnaround_alert/README.md new file mode 100644 index 0000000..377f17f --- /dev/null +++ b/apps/src/sage/apps/lab_turnaround_alert/README.md @@ -0,0 +1,6 @@ +# 检验样本周转异常预警系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_lab_turnaround_alert_pipeline` +- Entry script: `examples/run_lab_turnaround_alert.py` diff --git a/apps/src/sage/apps/lab_turnaround_alert/__init__.py b/apps/src/sage/apps/lab_turnaround_alert/__init__.py new file mode 100644 index 0000000..180ace0 --- /dev/null +++ b/apps/src/sage/apps/lab_turnaround_alert/__init__.py @@ -0,0 +1,5 @@ +"""检验样本周转异常预警系统 application.""" + +from .pipeline import run_lab_turnaround_alert_pipeline + +__all__ = ["run_lab_turnaround_alert_pipeline"] diff --git a/apps/src/sage/apps/lab_turnaround_alert/operators.py b/apps/src/sage/apps/lab_turnaround_alert/operators.py new file mode 100644 index 0000000..c721302 --- /dev/null +++ b/apps/src/sage/apps/lab_turnaround_alert/operators.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import csv +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _parse_dt(value: Any) -> datetime | None: + text = str(value or "").strip().replace("Z", "+00:00") + if not text: + return None + try: + return datetime.fromisoformat(text) + except ValueError: + return None + + +class LabRecordSource(ListBatchSource): + def __init__(self, record_file: str, **kwargs): + super().__init__(**kwargs) + self.record_file = record_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.record_file) + for item in items: + item.setdefault("app_slug", "lab_turnaround_alert") + item.setdefault("source_path", self.record_file) + return items + + +class LabStageMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["collected_at"] = _parse_dt(payload.get("collected_at")) + payload["received_at"] = _parse_dt(payload.get("received_at")) + payload["reported_at"] = _parse_dt(payload.get("reported_at")) + payload["test_type"] = str(payload.get("test_type") or "routine") + return payload + + +class TurnaroundTimeBuilder(MapFunction): + TARGET_HOURS = {"routine": 8, "chemistry": 6, "pathology": 24, "urgent": 2} + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + collected = payload.get("collected_at") + received = payload.get("received_at") + reported = payload.get("reported_at") + collect_to_receive = ( + ((received - collected).total_seconds() / 3600) if collected and received else None + ) + receive_to_report = ( + ((reported - received).total_seconds() / 3600) if received and reported else None + ) + total = ((reported - collected).total_seconds() / 3600) if collected and reported else None + payload["collect_to_receive_hours"] = ( + round(collect_to_receive, 2) if collect_to_receive is not None else None + ) + payload["receive_to_report_hours"] = ( + round(receive_to_report, 2) if receive_to_report is not None else None + ) + payload["total_turnaround_hours"] = round(total, 2) if total is not None else None + payload["target_hours"] = self.TARGET_HOURS.get(payload.get("test_type"), 8) + return payload + + +class TurnaroundAnomalyDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + blockers: list[str] = [] + if ( + payload.get("collect_to_receive_hours") is not None + and float(payload["collect_to_receive_hours"]) > 2 + ): + blockers.append("transport_delay") + if ( + payload.get("receive_to_report_hours") is not None + and float(payload["receive_to_report_hours"]) + > float(payload.get("target_hours") or 8) * 0.75 + ): + blockers.append("lab_processing_delay") + if payload.get("total_turnaround_hours") is not None and float( + payload["total_turnaround_hours"] + ) > float(payload.get("target_hours") or 8): + blockers.append("sla_breach") + payload["turnaround_alert_level"] = ( + "critical" if "sla_breach" in blockers else "watch" if blockers else "normal" + ) + payload["bottleneck_stage"] = blockers[0] if blockers else "none" + payload["alert_summary"] = ( + f"样本 {payload.get('sample_id', 'unknown')} 周转 {payload.get('total_turnaround_hours')} 小时，" + f"状态 {payload.get('turnaround_alert_level')}。" + ) + return payload + + +class LabAlertSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + for field in ("collected_at", "received_at", "reported_at"): + if payload.get(field) is not None: + payload[field] = payload[field].isoformat() + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/lab_turnaround_alert/pipeline.py b/apps/src/sage/apps/lab_turnaround_alert/pipeline.py new file mode 100644 index 0000000..b73abef --- /dev/null +++ b/apps/src/sage/apps/lab_turnaround_alert/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + LabAlertSink, + LabRecordSource, + LabStageMapper, + TurnaroundAnomalyDetector, + TurnaroundTimeBuilder, +) + + +def run_lab_turnaround_alert_pipeline(record_file: str, output_file: str) -> None: + env = LocalEnvironment("lab_turnaround_alert") + ( + env.from_batch(LabRecordSource, record_file=record_file) + .map(LabStageMapper) + .map(TurnaroundTimeBuilder) + .map(TurnaroundAnomalyDetector) + .sink(LabAlertSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/lead_scoring/README.md b/apps/src/sage/apps/lead_scoring/README.md new file mode 100644 index 0000000..df0ec75 --- /dev/null +++ b/apps/src/sage/apps/lead_scoring/README.md @@ -0,0 +1,3 @@ +# Lead Scoring + +销售机会评分应用。 diff --git a/apps/src/sage/apps/lead_scoring/__init__.py b/apps/src/sage/apps/lead_scoring/__init__.py new file mode 100644 index 0000000..9153c11 --- /dev/null +++ b/apps/src/sage/apps/lead_scoring/__init__.py @@ -0,0 +1,5 @@ +"""Lead scoring application.""" + +from .pipeline import run_lead_scoring_pipeline + +__all__ = ["run_lead_scoring_pipeline"] diff --git a/apps/src/sage/apps/lead_scoring/operators.py b/apps/src/sage/apps/lead_scoring/operators.py new file mode 100644 index 0000000..2835343 --- /dev/null +++ b/apps/src/sage/apps/lead_scoring/operators.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class LeadSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class FeatureScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + company_size = int(float(item.get("company_size") or 0)) + budget = float(item.get("budget") or 0) + interaction_count = int(float(item.get("interaction_count") or 0)) + score = ( + (2 if company_size >= 500 else 1 if company_size >= 50 else 0) + + (2 if budget >= 100000 else 1 if budget >= 10000 else 0) + + min(interaction_count, 3) + ) + item["lead_score"] = score + return item + + +class PriorityRanker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = int(item.get("lead_score", 0)) + item["priority"] = "hot" if score >= 6 else "warm" if score >= 3 else "cold" + return item + + +class LeadSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/lead_scoring/pipeline.py b/apps/src/sage/apps/lead_scoring/pipeline.py new file mode 100644 index 0000000..9f85955 --- /dev/null +++ b/apps/src/sage/apps/lead_scoring/pipeline.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import FeatureScorer, LeadSink, LeadSource, PriorityRanker + + +def run_lead_scoring_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("lead_scoring") + ( + env.from_batch(LeadSource, input_file=input_file) + .map(FeatureScorer) + .map(PriorityRanker) + .sink(LeadSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/learning_record_hub/README.md b/apps/src/sage/apps/learning_record_hub/README.md new file mode 100644 index 0000000..1e62aea --- /dev/null +++ b/apps/src/sage/apps/learning_record_hub/README.md @@ -0,0 +1,5 @@ +# Learning Record Hub + +读取学习记录，映射员工信息并识别认证缺口。 + +输入：CSV 或 JSON 学习记录。 输出：包含课程标准化信息和缺口小时数的 JSON 文件。 diff --git a/apps/src/sage/apps/learning_record_hub/__init__.py b/apps/src/sage/apps/learning_record_hub/__init__.py new file mode 100644 index 0000000..600735b --- /dev/null +++ b/apps/src/sage/apps/learning_record_hub/__init__.py @@ -0,0 +1,5 @@ +"""Learning record hub application.""" + +from .pipeline import run_learning_record_hub_pipeline + +__all__ = ["run_learning_record_hub_pipeline"] diff --git a/apps/src/sage/apps/learning_record_hub/operators.py b/apps/src/sage/apps/learning_record_hub/operators.py new file mode 100644 index 0000000..d931f26 --- /dev/null +++ b/apps/src/sage/apps/learning_record_hub/operators.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class LearningRecordSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class EmployeeMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["employee_id"] = ( + item.get("employee_id") or item.get("staff_id") or item.get("id") or "unknown" + ) + item["employee_name"] = item.get("employee_name") or item.get("name") or "unknown" + return item + + +class CourseNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + course_name = str( + item.get("course_name") or item.get("course") or "general training" + ).strip() + item["course_name"] = course_name.title() + item["completed_hours"] = float(item.get("completed_hours") or item.get("hours") or 0) + item["required_hours"] = float(item.get("required_hours") or item.get("target_hours") or 8) + return item + + +class CertificationGapDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + gap = max(item.get("required_hours", 0) - item.get("completed_hours", 0), 0) + item["certification_gap_hours"] = gap + item["learning_status"] = "complete" if gap == 0 else "pending" + return item + + +class LearningProfileSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/learning_record_hub/pipeline.py b/apps/src/sage/apps/learning_record_hub/pipeline.py new file mode 100644 index 0000000..b9c0841 --- /dev/null +++ b/apps/src/sage/apps/learning_record_hub/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CertificationGapDetector, + CourseNormalizer, + EmployeeMapper, + LearningProfileSink, + LearningRecordSource, +) + + +def run_learning_record_hub_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("learning_record_hub") + ( + env.from_batch(LearningRecordSource, input_file=input_file) + .map(EmployeeMapper) + .map(CourseNormalizer) + .map(CertificationGapDetector) + .sink(LearningProfileSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/lesson_scheduler/README.md b/apps/src/sage/apps/lesson_scheduler/README.md new file mode 100644 index 0000000..dd0771f --- /dev/null +++ b/apps/src/sage/apps/lesson_scheduler/README.md @@ -0,0 +1,6 @@ +# 周课时计划排布系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_lesson_scheduler_pipeline` +- Entry script: `examples/run_lesson_scheduler.py` diff --git a/apps/src/sage/apps/lesson_scheduler/__init__.py b/apps/src/sage/apps/lesson_scheduler/__init__.py new file mode 100644 index 0000000..121af5a --- /dev/null +++ b/apps/src/sage/apps/lesson_scheduler/__init__.py @@ -0,0 +1,5 @@ +"""周课时计划排布系统 application.""" + +from .pipeline import run_lesson_scheduler_pipeline + +__all__ = ["run_lesson_scheduler_pipeline"] diff --git a/apps/src/sage/apps/lesson_scheduler/operators.py b/apps/src/sage/apps/lesson_scheduler/operators.py new file mode 100644 index 0000000..b8c4456 --- /dev/null +++ b/apps/src/sage/apps/lesson_scheduler/operators.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _parse_int(value: Any, default: int = 0) -> int: + try: + return int(float(value)) + except (TypeError, ValueError): + return default + + +class TeachingRequirementSource(ListBatchSource): + def __init__(self, plan_file: str, resource_file: str, **kwargs): + super().__init__(**kwargs) + self.plan_file = plan_file + self.resource_file = resource_file + + def load_items(self) -> list[dict[str, Any]]: + plans = _load_records(self.plan_file) + resources = _load_records(self.resource_file) + for item in plans: + item["resources"] = resources + return plans + + +class TeachingConstraintParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["required_hours"] = _parse_int( + payload.get("required_hours") or payload.get("hours"), 1 + ) + payload["class_size"] = _parse_int(payload.get("class_size") or payload.get("students"), 0) + preferred_days = str(payload.get("preferred_days") or payload.get("preferred_day") or "") + payload["preferred_days"] = [ + part.strip() for part in preferred_days.split(",") if part.strip() + ] + return payload + + +class LessonPlanScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + best_room = None + best_score = -1 + for resource in payload.get("resources") or []: + capacity = _parse_int(resource.get("capacity"), 0) + available_hours = _parse_int(resource.get("available_hours"), 0) + slot_day = str(resource.get("day") or "") + score = 0 + if capacity >= payload.get("class_size", 0): + score += 4 + if available_hours >= payload.get("required_hours", 0): + score += 3 + if slot_day in payload.get("preferred_days", []): + score += 2 + if score > best_score: + best_score = score + best_room = { + "room": resource.get("room") or resource.get("resource") or "unassigned", + "day": slot_day, + "capacity": capacity, + "available_hours": available_hours, + } + payload["recommended_slot"] = best_room or { + "room": "unassigned", + "day": "", + "capacity": 0, + "available_hours": 0, + } + payload["schedule_score"] = best_score if best_score >= 0 else 0 + return payload + + +class LessonConflictChecker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + slot = payload.get("recommended_slot") or {} + conflicts: list[str] = [] + if slot.get("available_hours", 0) < payload.get("required_hours", 0): + conflicts.append("insufficient_hours") + if slot.get("capacity", 0) < payload.get("class_size", 0): + conflicts.append("capacity_shortage") + payload["conflicts"] = conflicts + payload["schedule_status"] = "blocked" if conflicts else "scheduled" + return payload + + +class LessonScheduleSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("resources", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/lesson_scheduler/pipeline.py b/apps/src/sage/apps/lesson_scheduler/pipeline.py new file mode 100644 index 0000000..e2613f1 --- /dev/null +++ b/apps/src/sage/apps/lesson_scheduler/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + LessonConflictChecker, + LessonPlanScorer, + LessonScheduleSink, + TeachingConstraintParser, + TeachingRequirementSource, +) + + +def run_lesson_scheduler_pipeline(plan_file: str, resource_file: str, output_file: str) -> None: + env = LocalEnvironment("lesson_scheduler") + ( + env.from_batch(TeachingRequirementSource, plan_file=plan_file, resource_file=resource_file) + .map(TeachingConstraintParser) + .map(LessonPlanScorer) + .map(LessonConflictChecker) + .sink(LessonScheduleSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/log_parser/README.md b/apps/src/sage/apps/log_parser/README.md new file mode 100644 index 0000000..3c8b3e2 --- /dev/null +++ b/apps/src/sage/apps/log_parser/README.md @@ -0,0 +1,28 @@ +# Log Parser + +日志解析与结构化示例应用。 + +## 功能 + +- 读取文本日志或 JSON 日志 +- 解析常见日志格式 +- 按级别过滤关键日志 +- 提取错误码等辅助字段 +- 输出 JSON 结果 + +## 用法 + +```bash +python examples/run_log_parser.py \ + --log-file app.log \ + --output structured.json +``` + +```bash +python examples/run_log_parser.py \ + --log-file app.log \ + --output structured.json \ + --error-levels ERROR,CRITICAL \ + --console \ + --verbose +``` diff --git a/apps/src/sage/apps/log_parser/__init__.py b/apps/src/sage/apps/log_parser/__init__.py new file mode 100644 index 0000000..745719e --- /dev/null +++ b/apps/src/sage/apps/log_parser/__init__.py @@ -0,0 +1,5 @@ +"""Log parser application.""" + +from .pipeline import run_log_parser_pipeline + +__all__ = ["run_log_parser_pipeline"] diff --git a/apps/src/sage/apps/log_parser/operators.py b/apps/src/sage/apps/log_parser/operators.py new file mode 100644 index 0000000..581038f --- /dev/null +++ b/apps/src/sage/apps/log_parser/operators.py @@ -0,0 +1,254 @@ +""" +Log Parser Operators + +Custom operators for log parsing and structuring. +""" + +from __future__ import annotations + +import json +import re +from datetime import datetime +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import CustomLogger, MapFunction, SinkFunction + + +class LogSource(ListBatchSource): + """Read log file line by line as a batch.""" + + def __init__(self, log_file: str, **kwargs): + """Initialize log source. + + Args: + log_file: Path to the log file to read + """ + super().__init__(**kwargs) + self.log_file = log_file + self._logger = CustomLogger("LogSource") + + def load_items(self) -> list[str]: + """Read and return all log lines.""" + try: + with open(self.log_file, encoding="utf-8") as f: + lines = [line.rstrip("\n") for line in f if line.strip()] + self.logger.info(f"Read {len(lines)} log lines from {self.log_file}") + return lines + except FileNotFoundError: + self.logger.error(f"Log file not found: {self.log_file}") + return [] + + +class LogParser(MapFunction): + """Parse log lines and extract structured fields. + + Supports multiple log formats: + - Apache/Nginx: [timestamp] level message + - Standard: timestamp level [component] message + - JSON: JSON-formatted log lines + """ + + def __init__(self, **kwargs): + """Initialize log parser.""" + super().__init__(**kwargs) + self._logger = CustomLogger("LogParser") + + # Common log patterns + self.patterns = { + "apache": re.compile(r"\[(?P[^\]]+)\]\s+(?P\w+)\s+(?P.+)"), + "standard": re.compile( + r"(?P\S+)\s+(?P\w+)\s+\[(?P[^\]]+)\]\s+(?P.+)" + ), + "json": re.compile(r"^\{.*\}$"), + } + + def execute(self, line: str) -> dict[str, Any] | None: + """Parse a single log line. + + Args: + line: A log line to parse + + Returns: + Parsed log dict or None if unparseable + """ + if not line: + return None + + # Try JSON format first + if self.patterns["json"].match(line): + try: + return json.loads(line) + except json.JSONDecodeError: + pass + + # Try standard format + match = self.patterns["standard"].match(line) + if match: + return match.groupdict() + + # Try Apache format + match = self.patterns["apache"].match(line) + if match: + return match.groupdict() + + # Fallback: return raw line with default structure + return { + "timestamp": datetime.now().isoformat(), + "level": "UNKNOWN", + "message": line, + } + + +class ErrorFilter(MapFunction): + """Filter logs by error level. + + Keeps only ERROR, CRITICAL, and WARN logs by default. + """ + + def __init__(self, error_levels: list[str] | None = None, **kwargs): + """Initialize error filter. + + Args: + error_levels: List of log levels to keep (e.g., ['ERROR', 'CRITICAL']) + """ + super().__init__(**kwargs) + self.error_levels = error_levels or ["ERROR", "CRITICAL", "WARN"] + self._logger = CustomLogger("ErrorFilter") + + def execute(self, log_entry: dict[str, Any]) -> dict[str, Any] | None: + """Filter log by error level. + + Args: + log_entry: Parsed log entry + + Returns: + Log entry if matches error level, None otherwise + """ + if not log_entry: + return None + + level = log_entry.get("level", "").upper() + if level in self.error_levels: + return log_entry + + return None + + +class LogEnricher(MapFunction): + """Enrich log entries with additional computed fields.""" + + def __init__(self, **kwargs): + """Initialize log enricher.""" + super().__init__(**kwargs) + + def execute(self, log_entry: dict[str, Any]) -> dict[str, Any]: + """Enrich a log entry. + + Adds: + - is_critical: boolean flag for critical logs + - message_length: length of message + - has_error_code: whether message contains error codes + - error_code: extracted error code if present + + Args: + log_entry: Parsed log entry + + Returns: + Enriched log entry + """ + if not log_entry: + return log_entry + + # Mark critical logs + level = log_entry.get("level", "").upper() + log_entry["is_critical"] = level in ["ERROR", "CRITICAL"] + + # Message length + message = log_entry.get("message", "") + log_entry["message_length"] = len(message) + + # Extract error codes (e.g., ERR001, ERROR_CODE_500) + error_code_match = re.search(r"(ERR\d+|ERROR_\w+|ERROR\s*#\d+|\b[45]\d{2}\b)", message) + if error_code_match: + log_entry["error_code"] = error_code_match.group(1) + log_entry["has_error_code"] = True + else: + log_entry["has_error_code"] = False + log_entry["error_code"] = None + + return log_entry + + +class JsonSink(SinkFunction): + """Output structured logs to JSON file.""" + + def __init__(self, output_file: str, **kwargs): + """Initialize JSON sink. + + Args: + output_file: Path to output JSON file + """ + super().__init__(**kwargs) + self.output_file = output_file + self._logger = CustomLogger("JsonSink") + self.count = 0 + + def setup(self, context: Any) -> None: + """Setup - prepare output file.""" + # Clear existing file + with open(self.output_file, "w", encoding="utf-8") as f: + f.write("[\n") + + def execute(self, log_entry: dict[str, Any]) -> None: + """Append log entry to JSON file. + + Args: + log_entry: Parsed and enriched log entry + """ + if not log_entry: + return + + with open(self.output_file, "a", encoding="utf-8") as f: + if self.count > 0: + f.write(",\n") + json.dump(log_entry, f, ensure_ascii=False, indent=2) + self.count += 1 + + def teardown(self, context: Any) -> None: + """Cleanup - close JSON array.""" + with open(self.output_file, "a", encoding="utf-8") as f: + f.write("\n]") + self.logger.info(f"Written {self.count} log entries to {self.output_file}") + + +class ConsoleSink(SinkFunction): + """Output logs to console with formatting.""" + + def __init__(self, **kwargs): + """Initialize console sink.""" + super().__init__(**kwargs) + self._logger = CustomLogger("ConsoleSink") + self.count = 0 + + def execute(self, log_entry: dict[str, Any]) -> None: + """Print formatted log entry to console. + + Args: + log_entry: Log entry to print + """ + if not log_entry: + return + + timestamp = log_entry.get("timestamp", "N/A") + level = log_entry.get("level", "UNKNOWN") + message = log_entry.get("message", "") + error_code = log_entry.get("error_code", "") + + error_code_str = f" [{error_code}]" if error_code else "" + print(f"{timestamp} {level}{error_code_str}: {message}") + self.count += 1 + + def teardown(self, context: Any) -> None: + """Cleanup.""" + self.logger.info(f"Displayed {self.count} log entries") diff --git a/apps/src/sage/apps/log_parser/pipeline.py b/apps/src/sage/apps/log_parser/pipeline.py new file mode 100644 index 0000000..cc8ed8a --- /dev/null +++ b/apps/src/sage/apps/log_parser/pipeline.py @@ -0,0 +1,105 @@ +""" +Log Parser Pipeline + +Main pipeline implementation using SAGE operators for enterprise log parsing. +""" + +from __future__ import annotations + +from sage.foundation import CustomLogger +from sage.runtime import LocalEnvironment + +from .operators import ( + ConsoleSink, + ErrorFilter, + JsonSink, + LogEnricher, + LogParser, + LogSource, +) + + +def run_log_parser_pipeline( + log_file: str, + output_file: str | None = None, + error_levels: list[str] | None = None, + verbose: bool = False, + console_output: bool = False, +) -> None: + """ + Run the log parser pipeline using SAGE framework. + + Args: + log_file: Path to the log file to parse + output_file: Path to output JSON file (optional, if None only console output) + error_levels: List of error levels to filter (default: ERROR, CRITICAL, WARN) + verbose: Enable verbose logging + console_output: Also output to console (in addition to file) + + Example: + >>> run_log_parser_pipeline( + ... log_file="app.log", + ... output_file="structured_logs.json", + ... error_levels=["ERROR", "CRITICAL"], + ... verbose=True + ... ) + """ + logger = CustomLogger("LogParserPipeline") + + # Default parameters + if error_levels is None: + error_levels = ["ERROR", "CRITICAL", "WARN"] + + if verbose: + logger.info("Starting log parser pipeline") + logger.info(f" Log file: {log_file}") + logger.info(f" Output file: {output_file}") + logger.info(f" Error levels: {error_levels}") + + # Create environment + env = LocalEnvironment("log_parser") + + try: + # Build pipeline + pipeline = ( + env.from_batch(LogSource, log_file=log_file) + .map(LogParser) + .map(lambda log: log if log else None) + .filter(lambda log: log is not None) + .map(ErrorFilter, error_levels=error_levels) + .filter(lambda log: log is not None) + .map(LogEnricher) + ) + + # Add sinks + if output_file: + pipeline = pipeline.sink(JsonSink, output_file=output_file) + + if console_output: + pipeline = pipeline.sink(ConsoleSink) + + # Submit and run + env.submit(autostop=True) + + if verbose: + logger.info("Log parser pipeline completed successfully") + + except Exception as e: + logger.error(f"Error in log parser pipeline: {e}") + raise + + +if __name__ == "__main__": + # Example usage + import sys + + if len(sys.argv) < 2: + print("Usage: python -m sage.apps.log_parser [output_file]") + sys.exit(1) + + log_file = sys.argv[1] + output_file = sys.argv[2] if len(sys.argv) > 2 else None + + run_log_parser_pipeline( + log_file=log_file, output_file=output_file, verbose=True, console_output=True + ) diff --git a/apps/src/sage/apps/logistics_cost_optimizer/README.md b/apps/src/sage/apps/logistics_cost_optimizer/README.md new file mode 100644 index 0000000..c948da4 --- /dev/null +++ b/apps/src/sage/apps/logistics_cost_optimizer/README.md @@ -0,0 +1,3 @@ +# Logistics Cost Optimizer + +物流成本优化应用。 diff --git a/apps/src/sage/apps/logistics_cost_optimizer/__init__.py b/apps/src/sage/apps/logistics_cost_optimizer/__init__.py new file mode 100644 index 0000000..0eaefc5 --- /dev/null +++ b/apps/src/sage/apps/logistics_cost_optimizer/__init__.py @@ -0,0 +1,5 @@ +"""Logistics cost optimizer application.""" + +from .pipeline import run_logistics_cost_optimizer_pipeline + +__all__ = ["run_logistics_cost_optimizer_pipeline"] diff --git a/apps/src/sage/apps/logistics_cost_optimizer/operators.py b/apps/src/sage/apps/logistics_cost_optimizer/operators.py new file mode 100644 index 0000000..69bd8fd --- /dev/null +++ b/apps/src/sage/apps/logistics_cost_optimizer/operators.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class ShipmentSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class CostCalculator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + weight = float(item.get("weight_kg") or 0) + distance = float(item.get("distance_km") or 0) + item["estimated_cost"] = round(weight * 0.8 + distance * 0.2, 2) + return item + + +class OptionSelector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + cost = float(item.get("estimated_cost", 0)) + item["recommended_mode"] = ( + "express" if cost <= 50 else "standard" if cost <= 150 else "bulk" + ) + return item + + +class LogisticsSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/logistics_cost_optimizer/pipeline.py b/apps/src/sage/apps/logistics_cost_optimizer/pipeline.py new file mode 100644 index 0000000..15d087f --- /dev/null +++ b/apps/src/sage/apps/logistics_cost_optimizer/pipeline.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import CostCalculator, LogisticsSink, OptionSelector, ShipmentSource + + +def run_logistics_cost_optimizer_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("logistics_cost_optimizer") + ( + env.from_batch(ShipmentSource, input_file=input_file) + .map(CostCalculator) + .map(OptionSelector) + .sink(LogisticsSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/mail_classifier/README.md b/apps/src/sage/apps/mail_classifier/README.md new file mode 100644 index 0000000..4e88eeb --- /dev/null +++ b/apps/src/sage/apps/mail_classifier/README.md @@ -0,0 +1,5 @@ +# Mail Classifier + +读取邮件记录，解析正文并输出类别与优先级。 + +输入：CSV 或 JSON 邮件数据。 输出：包含邮件分类和优先级的 JSON 文件。 diff --git a/apps/src/sage/apps/mail_classifier/__init__.py b/apps/src/sage/apps/mail_classifier/__init__.py new file mode 100644 index 0000000..f0d4a56 --- /dev/null +++ b/apps/src/sage/apps/mail_classifier/__init__.py @@ -0,0 +1,5 @@ +"""Mail classifier application.""" + +from .pipeline import run_mail_classifier_pipeline + +__all__ = ["run_mail_classifier_pipeline"] diff --git a/apps/src/sage/apps/mail_classifier/operators.py b/apps/src/sage/apps/mail_classifier/operators.py new file mode 100644 index 0000000..adb7369 --- /dev/null +++ b/apps/src/sage/apps/mail_classifier/operators.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class MailSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class MailParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + subject = str(item.get("subject") or "") + body = str(item.get("body") or item.get("content") or "") + item["mail_text"] = f"{subject} {body}".strip().lower() + item["has_attachment"] = ( + str(item.get("attachment") or item.get("attachments") or "").strip() != "" + ) + return item + + +class MailCategoryClassifier(MapFunction): + RULES = { + "support": {"ticket", "issue", "help", "error"}, + "finance": {"invoice", "payment", "budget", "reimbursement"}, + "hr": {"leave", "candidate", "training", "benefit"}, + "sales": {"proposal", "quote", "renewal", "deal"}, + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = item.get("mail_text", "") + scores = { + label: sum(1 for term in terms if re.search(rf"\b{term}\b", text)) + for label, terms in self.RULES.items() + } + category, score = max(scores.items(), key=lambda pair: pair[1]) + item["mail_category"] = category if score > 0 else "general" + return item + + +class MailPriorityScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = item.get("mail_text", "") + score = 1 if item.get("has_attachment") else 0 + score += sum(1 for term in ("urgent", "asap", "immediately", "critical") if term in text) + item["mail_priority"] = "high" if score >= 2 else "medium" if score == 1 else "low" + return item + + +class MailSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/mail_classifier/pipeline.py b/apps/src/sage/apps/mail_classifier/pipeline.py new file mode 100644 index 0000000..711e319 --- /dev/null +++ b/apps/src/sage/apps/mail_classifier/pipeline.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import MailCategoryClassifier, MailParser, MailPriorityScorer, MailSink, MailSource + + +def run_mail_classifier_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("mail_classifier") + ( + env.from_batch(MailSource, input_file=input_file) + .map(MailParser) + .map(MailCategoryClassifier) + .map(MailPriorityScorer) + .sink(MailSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/media_archive_search/README.md b/apps/src/sage/apps/media_archive_search/README.md new file mode 100644 index 0000000..5e3c1c0 --- /dev/null +++ b/apps/src/sage/apps/media_archive_search/README.md @@ -0,0 +1,6 @@ +# 媒体资料归档检索系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_media_archive_search_pipeline` +- Entry script: `examples/run_media_archive_search.py` diff --git a/apps/src/sage/apps/media_archive_search/__init__.py b/apps/src/sage/apps/media_archive_search/__init__.py new file mode 100644 index 0000000..52d5a8d --- /dev/null +++ b/apps/src/sage/apps/media_archive_search/__init__.py @@ -0,0 +1,5 @@ +"""媒体资料归档检索系统 application.""" + +from .pipeline import run_media_archive_search_pipeline + +__all__ = ["run_media_archive_search_pipeline"] diff --git a/apps/src/sage/apps/media_archive_search/operators.py b/apps/src/sage/apps/media_archive_search/operators.py new file mode 100644 index 0000000..b12c58f --- /dev/null +++ b/apps/src/sage/apps/media_archive_search/operators.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import csv +import json +from hashlib import sha1 +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +class MediaAssetSource(ListBatchSource): + def __init__(self, asset_dir: str, **kwargs): + super().__init__(**kwargs) + self.asset_dir = asset_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.asset_dir) + for item in items: + item.setdefault("app_slug", "media_archive_search") + item.setdefault("source_path", self.asset_dir) + return items + + +class MediaMetadataExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["asset_title"] = str(payload.get("title") or payload.get("filename") or "untitled") + payload["asset_type"] = str(payload.get("asset_type") or payload.get("type") or "document") + payload["people"] = str(payload.get("people") or payload.get("speaker") or "") + payload["event_date"] = str(payload.get("event_date") or payload.get("date") or "") + text = str( + payload.get("text") or payload.get("description") or payload.get("summary") or "" + ) + payload["metadata_fingerprint"] = sha1(text.lower().encode("utf-8")).hexdigest()[:12] + payload["text_length"] = len(text) + return payload + + +class MediaTagger(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + text = str(payload.get("text") or payload.get("description") or "").lower() + tags = [] + for term in ("launch", "interview", "product", "conference", "customer"): + if term in text: + tags.append(term) + if not tags: + tags.append(str(payload.get("asset_type") or "general")) + payload["media_tags"] = tags + payload["search_key"] = f"{payload.get('asset_title')}|{'/'.join(tags)}" + return payload + + +class MediaDuplicateDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + duplicate_signals: list[str] = [] + source_name = Path(str(payload.get("source_path") or "")).name.lower() + if "copy" in source_name or "final_final" in source_name: + duplicate_signals.append("filename_duplicate_pattern") + if int(payload.get("text_length") or 0) < 40: + duplicate_signals.append("insufficient_metadata") + payload["duplicate_signals"] = duplicate_signals + payload["archive_status"] = "review_duplicate" if duplicate_signals else "index_ready" + payload["archive_summary"] = ( + f"Asset {payload.get('asset_title')} status {payload.get('archive_status')}, " + f"tags {', '.join(payload.get('media_tags') or [])}." + ) + return payload + + +class MediaArchiveSink(SinkFunction): + def __init__(self, output_dir: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_dir + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.mkdir(parents=True, exist_ok=True) + (target / "results.json").write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/media_archive_search/pipeline.py b/apps/src/sage/apps/media_archive_search/pipeline.py new file mode 100644 index 0000000..33377cc --- /dev/null +++ b/apps/src/sage/apps/media_archive_search/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + MediaArchiveSink, + MediaAssetSource, + MediaDuplicateDetector, + MediaMetadataExtractor, + MediaTagger, +) + + +def run_media_archive_search_pipeline(asset_dir: str, output_dir: str) -> None: + env = LocalEnvironment("media_archive_search") + ( + env.from_batch(MediaAssetSource, asset_dir=asset_dir) + .map(MediaMetadataExtractor) + .map(MediaTagger) + .map(MediaDuplicateDetector) + .sink(MediaArchiveSink, output_dir=output_dir) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/medical_registration_optimizer/README.md b/apps/src/sage/apps/medical_registration_optimizer/README.md new file mode 100644 index 0000000..19fed9e --- /dev/null +++ b/apps/src/sage/apps/medical_registration_optimizer/README.md @@ -0,0 +1,3 @@ +# Medical Registration Optimizer + +医疗挂号优化应用。 diff --git a/apps/src/sage/apps/medical_registration_optimizer/__init__.py b/apps/src/sage/apps/medical_registration_optimizer/__init__.py new file mode 100644 index 0000000..8c92490 --- /dev/null +++ b/apps/src/sage/apps/medical_registration_optimizer/__init__.py @@ -0,0 +1,5 @@ +"""Medical registration optimizer application.""" + +from .pipeline import run_medical_registration_optimizer_pipeline + +__all__ = ["run_medical_registration_optimizer_pipeline"] diff --git a/apps/src/sage/apps/medical_registration_optimizer/operators.py b/apps/src/sage/apps/medical_registration_optimizer/operators.py new file mode 100644 index 0000000..9d457ca --- /dev/null +++ b/apps/src/sage/apps/medical_registration_optimizer/operators.py @@ -0,0 +1,88 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class RegistrationSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class RegistrationRequestSource(RegistrationSource): + pass + + +class DemandScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + severity = int(float(item.get("severity") or 1)) + wait_days = int(float(item.get("wait_days") or 0)) + item["priority_score"] = severity * 2 + wait_days + return item + + +class DoctorSlotFetcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["doctor_slots"] = int( + float(item.get("doctor_slots") or item.get("available_slots") or 0) + ) + item["distance_km"] = float(item.get("distance_km") or 0) + return item + + +class PatientDoctorMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + scored = DemandScorer().execute(item) + scored["match_score"] = round( + scored["priority_score"] + + max(0.0, 5 - float(scored.get("distance_km", 0))) + + min(int(scored.get("doctor_slots", 0)), 5), + 2, + ) + return scored + + +class SlotAllocator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = int(item.get("priority_score", 0)) + item["recommended_slot"] = ( + "priority_clinic" if score >= 8 else "specialist" if score >= 4 else "general" + ) + return item + + +class RegistrationPlanBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + matched = dict(item) + score = float(matched.get("match_score", matched.get("priority_score", 0))) + matched["recommended_slot"] = ( + "priority_clinic" if score >= 10 else "specialist" if score >= 6 else "general" + ) + return matched + + +class RegistrationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/medical_registration_optimizer/pipeline.py b/apps/src/sage/apps/medical_registration_optimizer/pipeline.py new file mode 100644 index 0000000..f1fe43b --- /dev/null +++ b/apps/src/sage/apps/medical_registration_optimizer/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + DoctorSlotFetcher, + PatientDoctorMatcher, + RegistrationPlanBuilder, + RegistrationRequestSource, + RegistrationSink, +) + + +def run_medical_registration_optimizer_pipeline(request_file: str, output_file: str) -> None: + env = LocalEnvironment("medical_registration_optimizer") + ( + env.from_batch(RegistrationRequestSource, input_file=request_file) + .map(DoctorSlotFetcher) + .map(PatientDoctorMatcher) + .map(RegistrationPlanBuilder) + .sink(RegistrationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/meeting_minutes/README.md b/apps/src/sage/apps/meeting_minutes/README.md new file mode 100644 index 0000000..118053a --- /dev/null +++ b/apps/src/sage/apps/meeting_minutes/README.md @@ -0,0 +1,5 @@ +# Meeting Minutes + +读取会议转录文本，切分议程片段并提取行动项。 + +输入：CSV 或 JSON 会议纪要原始文本。 输出：包含片段摘要和行动项的 JSON 文件。 diff --git a/apps/src/sage/apps/meeting_minutes/__init__.py b/apps/src/sage/apps/meeting_minutes/__init__.py new file mode 100644 index 0000000..f413746 --- /dev/null +++ b/apps/src/sage/apps/meeting_minutes/__init__.py @@ -0,0 +1,5 @@ +"""Meeting minutes application.""" + +from .pipeline import run_meeting_minutes_pipeline + +__all__ = ["run_meeting_minutes_pipeline"] diff --git a/apps/src/sage/apps/meeting_minutes/operators.py b/apps/src/sage/apps/meeting_minutes/operators.py new file mode 100644 index 0000000..c3c4ae7 --- /dev/null +++ b/apps/src/sage/apps/meeting_minutes/operators.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class TranscriptSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class AgendaSegmenter(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + transcript = str(item.get("transcript") or item.get("content") or "") + segments = [ + segment.strip() + for segment in re.split(r"\n\s*\n|(?:^|\n)- ", transcript) + if segment.strip() + ] + return [ + { + **item, + "segment_index": index, + "segment_text": segment, + } + for index, segment in enumerate(segments, start=1) + ] + + +class ActionItemExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + lines = re.split(r"[.;\n]", item.get("segment_text", "")) + actions = [] + for line in lines: + text = line.strip() + lowered = text.lower() + if any(term in lowered for term in ("will", "action", "todo", "follow up", "owner")): + actions.append(text) + item["action_items"] = actions + return item + + +class MinutesFormatter(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["minutes_entry"] = { + "segment": item.get("segment_index"), + "summary": item.get("segment_text", "")[:160], + "actions": item.get("action_items", []), + } + return item + + +class MinutesSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item.get("minutes_entry", item)) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/meeting_minutes/pipeline.py b/apps/src/sage/apps/meeting_minutes/pipeline.py new file mode 100644 index 0000000..f1ecfed --- /dev/null +++ b/apps/src/sage/apps/meeting_minutes/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ActionItemExtractor, + AgendaSegmenter, + MinutesFormatter, + MinutesSink, + TranscriptSource, +) + + +def run_meeting_minutes_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("meeting_minutes") + ( + env.from_batch(TranscriptSource, input_file=input_file) + .flatmap(AgendaSegmenter) + .map(ActionItemExtractor) + .map(MinutesFormatter) + .sink(MinutesSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/movie_scheduling_optimizer/README.md b/apps/src/sage/apps/movie_scheduling_optimizer/README.md new file mode 100644 index 0000000..d84fe3e --- /dev/null +++ b/apps/src/sage/apps/movie_scheduling_optimizer/README.md @@ -0,0 +1,3 @@ +# Movie Scheduling Optimizer + +电影库存与排片优化应用。 diff --git a/apps/src/sage/apps/movie_scheduling_optimizer/__init__.py b/apps/src/sage/apps/movie_scheduling_optimizer/__init__.py new file mode 100644 index 0000000..6e89885 --- /dev/null +++ b/apps/src/sage/apps/movie_scheduling_optimizer/__init__.py @@ -0,0 +1,5 @@ +"""Movie scheduling optimizer application.""" + +from .pipeline import run_movie_scheduling_optimizer_pipeline + +__all__ = ["run_movie_scheduling_optimizer_pipeline"] diff --git a/apps/src/sage/apps/movie_scheduling_optimizer/operators.py b/apps/src/sage/apps/movie_scheduling_optimizer/operators.py new file mode 100644 index 0000000..8167f7c --- /dev/null +++ b/apps/src/sage/apps/movie_scheduling_optimizer/operators.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class ScreeningSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class HeatScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + popularity = float(item.get("popularity") or 0) + presale = float(item.get("presale") or 0) + item["heat_score"] = round(popularity * 0.7 + presale * 0.3, 2) + return item + + +class SlotOptimizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = float(item.get("heat_score", 0)) + item["recommended_slot"] = ( + "prime_time" if score >= 80 else "evening" if score >= 50 else "off_peak" + ) + return item + + +class ScheduleSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/movie_scheduling_optimizer/pipeline.py b/apps/src/sage/apps/movie_scheduling_optimizer/pipeline.py new file mode 100644 index 0000000..8a81b4d --- /dev/null +++ b/apps/src/sage/apps/movie_scheduling_optimizer/pipeline.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import HeatScorer, ScheduleSink, ScreeningSource, SlotOptimizer + + +def run_movie_scheduling_optimizer_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("movie_scheduling_optimizer") + ( + env.from_batch(ScreeningSource, input_file=input_file) + .map(HeatScorer) + .map(SlotOptimizer) + .sink(ScheduleSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/multi_factor_credit_score/README.md b/apps/src/sage/apps/multi_factor_credit_score/README.md new file mode 100644 index 0000000..dd5f232 --- /dev/null +++ b/apps/src/sage/apps/multi_factor_credit_score/README.md @@ -0,0 +1,3 @@ +# Multi Factor Credit Score + +多维用户信用评分应用。 diff --git a/apps/src/sage/apps/multi_factor_credit_score/__init__.py b/apps/src/sage/apps/multi_factor_credit_score/__init__.py new file mode 100644 index 0000000..36dfb9e --- /dev/null +++ b/apps/src/sage/apps/multi_factor_credit_score/__init__.py @@ -0,0 +1,5 @@ +"""Multi factor credit score application.""" + +from .pipeline import run_multi_factor_credit_score_pipeline + +__all__ = ["run_multi_factor_credit_score_pipeline"] diff --git a/apps/src/sage/apps/multi_factor_credit_score/operators.py b/apps/src/sage/apps/multi_factor_credit_score/operators.py new file mode 100644 index 0000000..9ebf10e --- /dev/null +++ b/apps/src/sage/apps/multi_factor_credit_score/operators.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class UserCreditSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class FactorCollector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["income_score"] = float(item.get("income_score") or 0) + item["repayment_score"] = float(item.get("repayment_score") or 0) + item["asset_score"] = float(item.get("asset_score") or 0) + return item + + +class CreditAggregator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = ( + item["income_score"] * 0.3 + item["repayment_score"] * 0.5 + item["asset_score"] * 0.2 + ) + item["composite_credit_score"] = round(score, 2) + item["segment"] = "excellent" if score >= 85 else "good" if score >= 65 else "watch" + return item + + +class CreditResultSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/multi_factor_credit_score/pipeline.py b/apps/src/sage/apps/multi_factor_credit_score/pipeline.py new file mode 100644 index 0000000..5e1fdc7 --- /dev/null +++ b/apps/src/sage/apps/multi_factor_credit_score/pipeline.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import CreditAggregator, CreditResultSink, FactorCollector, UserCreditSource + + +def run_multi_factor_credit_score_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("multi_factor_credit_score") + ( + env.from_batch(UserCreditSource, input_file=input_file) + .map(FactorCollector) + .map(CreditAggregator) + .sink(CreditResultSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/municipal_search/README.md b/apps/src/sage/apps/municipal_search/README.md new file mode 100644 index 0000000..bac0340 --- /dev/null +++ b/apps/src/sage/apps/municipal_search/README.md @@ -0,0 +1,6 @@ +# 市政协同文档检索系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_municipal_search_pipeline` +- Entry script: `examples/run_municipal_search.py` diff --git a/apps/src/sage/apps/municipal_search/__init__.py b/apps/src/sage/apps/municipal_search/__init__.py new file mode 100644 index 0000000..fe44cbc --- /dev/null +++ b/apps/src/sage/apps/municipal_search/__init__.py @@ -0,0 +1,5 @@ +"""市政协同文档检索系统 application.""" + +from .pipeline import run_municipal_search_pipeline + +__all__ = ["run_municipal_search_pipeline"] diff --git a/apps/src/sage/apps/municipal_search/operators.py b/apps/src/sage/apps/municipal_search/operators.py new file mode 100644 index 0000000..8df5291 --- /dev/null +++ b/apps/src/sage/apps/municipal_search/operators.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _tokenize(text: str) -> set[str]: + return {token for token in re.findall(r"[\w\-]{2,}", text.lower()) if token} + + +class MunicipalDocSource(ListBatchSource): + def __init__(self, doc_dir: str, question_file: str, **kwargs): + super().__init__(**kwargs) + self.doc_dir = doc_dir + self.question_file = question_file + + def load_items(self) -> list[dict[str, Any]]: + questions = _load_records(self.question_file) + question_text = "\n".join( + str(item.get("question") or item.get("text") or "") for item in questions + ) + items = _load_records(self.doc_dir) + for item in items: + item.setdefault("app_slug", "municipal_search") + item["question_text"] = question_text + item.setdefault("source_path", self.doc_dir) + return items + + +class MunicipalChunker(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + text = str(item.get("text") or item.get("content") or item.get("body") or "") + parts = [ + part.strip() for part in re.split(r"\n\s*\n|(?<=。)|(?<=\.)\s+", text) if part.strip() + ] + if not parts: + parts = [json.dumps(item, ensure_ascii=False)] + results: list[dict[str, Any]] = [] + for index, part in enumerate(parts[:8]): + child = dict(item) + child["municipal_chunk"] = part + child["chunk_index"] = index + results.append(child) + return results + + +class MunicipalQuestionMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + overlap = _tokenize(str(payload.get("question_text") or "")) & _tokenize( + str(payload.get("municipal_chunk") or "") + ) + payload["matched_terms"] = sorted(overlap) + payload["match_score"] = len(overlap) + return payload + + +class MunicipalAnswerFormatter(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + source_name = Path(str(payload.get("source_path") or "")).name or "document" + payload["answer_excerpt"] = str(payload.get("municipal_chunk") or "")[:220] + payload["citation"] = f"{source_name}#chunk-{payload.get('chunk_index', 0)}" + payload["answer_summary"] = ( + f"Municipal clause from {source_name}, citation {payload.get('citation')}, " + f"matched terms {', '.join(payload.get('matched_terms') or []) or 'none'}." + ) + return payload + + +class MunicipalSearchSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/municipal_search/pipeline.py b/apps/src/sage/apps/municipal_search/pipeline.py new file mode 100644 index 0000000..ebcab13 --- /dev/null +++ b/apps/src/sage/apps/municipal_search/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + MunicipalAnswerFormatter, + MunicipalChunker, + MunicipalDocSource, + MunicipalQuestionMatcher, + MunicipalSearchSink, +) + + +def run_municipal_search_pipeline(doc_dir: str, question_file: str, output_file: str) -> None: + env = LocalEnvironment("municipal_search") + ( + env.from_batch(MunicipalDocSource, doc_dir=doc_dir, question_file=question_file) + .flatmap(MunicipalChunker) + .map(MunicipalQuestionMatcher) + .map(MunicipalAnswerFormatter) + .sink(MunicipalSearchSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/news_aggregator/README.md b/apps/src/sage/apps/news_aggregator/README.md new file mode 100644 index 0000000..df761d4 --- /dev/null +++ b/apps/src/sage/apps/news_aggregator/README.md @@ -0,0 +1,10 @@ +# News Aggregator + +新闻聚合与去重应用。 + +## 功能 + +- 读取 RSS 源列表 +- 解析标题、摘要和链接 +- 计算内容指纹 +- 输出去重后的新闻列表 diff --git a/apps/src/sage/apps/news_aggregator/__init__.py b/apps/src/sage/apps/news_aggregator/__init__.py new file mode 100644 index 0000000..d20a234 --- /dev/null +++ b/apps/src/sage/apps/news_aggregator/__init__.py @@ -0,0 +1,5 @@ +"""News aggregator application.""" + +from .pipeline import run_news_aggregator_pipeline + +__all__ = ["run_news_aggregator_pipeline"] diff --git a/apps/src/sage/apps/news_aggregator/operators.py b/apps/src/sage/apps/news_aggregator/operators.py new file mode 100644 index 0000000..e01182e --- /dev/null +++ b/apps/src/sage/apps/news_aggregator/operators.py @@ -0,0 +1,102 @@ +"""Operators for RSS aggregation and deduplication.""" + +from __future__ import annotations + +import csv +import hashlib +import json +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import Any +from urllib.request import urlopen + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class RssSource(ListBatchSource): + def __init__(self, source_file: str, **kwargs): + super().__init__(**kwargs) + self.source_file = source_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.source_file, encoding="utf-8", newline="") as handle: + if self.source_file.lower().endswith(".csv"): + return [ + {"feed": row.get("feed", "")} + for row in csv.DictReader(handle) + if row.get("feed") + ] + return [{"feed": line.strip()} for line in handle if line.strip()] + + +class NewsExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + feed = item.get("feed", "") + if feed.startswith("http"): + payload = urlopen(feed, timeout=10).read().decode("utf-8", errors="ignore") + else: + payload = Path(feed).read_text(encoding="utf-8") + root = ET.fromstring(payload) + entries = [] + for entry in ( + root.findall(".//item")[:20] + + root.findall(".//{http://www.w3.org/2005/Atom}entry")[:20] + ): + title = self._text(entry, "title") + link = self._text(entry, "link") or ( + entry.find("link").attrib.get("href", "") if entry.find("link") is not None else "" + ) + summary = self._text(entry, "description") or self._text(entry, "summary") + if title: + entries.append({"title": title, "link": link, "summary": summary}) + item["entries"] = entries + return item + + def _text(self, entry: ET.Element, name: str) -> str: + node = entry.find(name) + if node is None: + node = entry.find(f"{{http://www.w3.org/2005/Atom}}{name}") + return (node.text or "").strip() if node is not None and node.text else "" + + +class FingerprintCalculator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + normalized = [] + for entry in item.get("entries", []): + text = f"{entry.get('title', '')}|{entry.get('summary', '')}".lower().strip() + entry["fingerprint"] = hashlib.md5(text.encode("utf-8")).hexdigest() + normalized.append(entry) + item["entries"] = normalized + return item + + +class DeduplicationFilter(MapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.seen: set[str] = set() + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + unique_entries = [] + for entry in item.get("entries", []): + fingerprint = entry.get("fingerprint", "") + if fingerprint and fingerprint not in self.seen: + self.seen.add(fingerprint) + unique_entries.append(entry) + item["entries"] = unique_entries + return item + + +class NewsSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.extend(item.get("entries", [])) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/news_aggregator/pipeline.py b/apps/src/sage/apps/news_aggregator/pipeline.py new file mode 100644 index 0000000..6869e43 --- /dev/null +++ b/apps/src/sage/apps/news_aggregator/pipeline.py @@ -0,0 +1,25 @@ +"""News aggregator pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + DeduplicationFilter, + FingerprintCalculator, + NewsExtractor, + NewsSink, + RssSource, +) + + +def run_news_aggregator_pipeline(source_file: str, output_file: str) -> None: + env = LocalEnvironment("news_aggregator") + ( + env.from_batch(RssSource, source_file=source_file) + .map(NewsExtractor) + .map(FingerprintCalculator) + .map(DeduplicationFilter) + .sink(NewsSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/order_anomaly_detector/README.md b/apps/src/sage/apps/order_anomaly_detector/README.md new file mode 100644 index 0000000..d57bdd0 --- /dev/null +++ b/apps/src/sage/apps/order_anomaly_detector/README.md @@ -0,0 +1,3 @@ +# Order Anomaly Detector + +订单异常检测应用。 diff --git a/apps/src/sage/apps/order_anomaly_detector/__init__.py b/apps/src/sage/apps/order_anomaly_detector/__init__.py new file mode 100644 index 0000000..ca96380 --- /dev/null +++ b/apps/src/sage/apps/order_anomaly_detector/__init__.py @@ -0,0 +1,5 @@ +"""Order anomaly detector application.""" + +from .pipeline import run_order_anomaly_detector_pipeline + +__all__ = ["run_order_anomaly_detector_pipeline"] diff --git a/apps/src/sage/apps/order_anomaly_detector/operators.py b/apps/src/sage/apps/order_anomaly_detector/operators.py new file mode 100644 index 0000000..1766463 --- /dev/null +++ b/apps/src/sage/apps/order_anomaly_detector/operators.py @@ -0,0 +1,62 @@ +"""Operators for order anomaly detection.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class OrderSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class FeatureCalculator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + amount = float(item.get("amount") or 0) + quantity = float(item.get("quantity") or 1) + item["amount"] = amount + item["quantity"] = quantity + item["unit_price"] = round(amount / quantity, 2) if quantity else amount + return item + + +class RuleScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = 0 + if float(item.get("amount", 0)) > 10000: + score += 2 + if float(item.get("quantity", 0)) > 50: + score += 1 + if float(item.get("unit_price", 0)) <= 0: + score += 2 + item["anomaly_score"] = score + item["is_anomaly"] = score >= 2 + return item + + +class AnomalySink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/order_anomaly_detector/pipeline.py b/apps/src/sage/apps/order_anomaly_detector/pipeline.py new file mode 100644 index 0000000..58d2ce3 --- /dev/null +++ b/apps/src/sage/apps/order_anomaly_detector/pipeline.py @@ -0,0 +1,18 @@ +"""Order anomaly detector pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import AnomalySink, FeatureCalculator, OrderSource, RuleScorer + + +def run_order_anomaly_detector_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("order_anomaly_detector") + ( + env.from_batch(OrderSource, input_file=input_file) + .map(FeatureCalculator) + .map(RuleScorer) + .sink(AnomalySink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/paper_classifier/README.md b/apps/src/sage/apps/paper_classifier/README.md new file mode 100644 index 0000000..11df3f1 --- /dev/null +++ b/apps/src/sage/apps/paper_classifier/README.md @@ -0,0 +1,5 @@ +# Paper Classifier + +读取论文标题、摘要或关键词，提取关键词并输出主题分类结果。 + +输入：CSV 或 JSON 论文记录。 输出：包含关键词、主题和分类标签的 JSON 文件。 diff --git a/apps/src/sage/apps/paper_classifier/__init__.py b/apps/src/sage/apps/paper_classifier/__init__.py new file mode 100644 index 0000000..d915404 --- /dev/null +++ b/apps/src/sage/apps/paper_classifier/__init__.py @@ -0,0 +1,5 @@ +"""Paper classifier application.""" + +from .pipeline import run_paper_classifier_pipeline + +__all__ = ["run_paper_classifier_pipeline"] diff --git a/apps/src/sage/apps/paper_classifier/operators.py b/apps/src/sage/apps/paper_classifier/operators.py new file mode 100644 index 0000000..4c51b33 --- /dev/null +++ b/apps/src/sage/apps/paper_classifier/operators.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class PaperSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class PaperKeywordExtractor(MapFunction): + def __init__(self, top_k: int = 5, **kwargs): + super().__init__(**kwargs) + self.top_k = top_k + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = " ".join( + str(item.get(field, "")) for field in ("title", "abstract", "keywords", "content") + ).lower() + tokens = re.findall(r"[a-z]{4,}", text) + counts: dict[str, int] = {} + for token in tokens: + counts[token] = counts.get(token, 0) + 1 + item["paper_keywords"] = [ + token + for token, _ in sorted(counts.items(), key=lambda pair: (-pair[1], pair[0]))[ + : self.top_k + ] + ] + return item + + +class PaperTopicClassifier(MapFunction): + TOPIC_RULES = { + "biology": {"cell", "gene", "biomarker", "protein", "clinical", "medical"}, + "computer_science": {"model", "algorithm", "dataset", "learning", "neural", "system"}, + "finance": {"market", "trading", "risk", "asset", "price", "credit"}, + "policy": {"policy", "regulation", "governance", "compliance", "public"}, + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + keywords = {keyword.lower() for keyword in item.get("paper_keywords", [])} + text = " ".join(str(item.get(field, "")) for field in ("title", "abstract")).lower() + scores = { + topic: sum(1 for term in terms if term in keywords or term in text) + for topic, terms in self.TOPIC_RULES.items() + } + topic, score = max(scores.items(), key=lambda pair: pair[1]) + item["paper_topic"] = topic if score > 0 else "general" + item["paper_label"] = f"{item['paper_topic']}_paper" + return item + + +class PaperClassificationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/paper_classifier/pipeline.py b/apps/src/sage/apps/paper_classifier/pipeline.py new file mode 100644 index 0000000..e514fae --- /dev/null +++ b/apps/src/sage/apps/paper_classifier/pipeline.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + PaperClassificationSink, + PaperKeywordExtractor, + PaperSource, + PaperTopicClassifier, +) + + +def run_paper_classifier_pipeline(input_file: str, output_file: str, top_k: int = 5) -> None: + env = LocalEnvironment("paper_classifier") + ( + env.from_batch(PaperSource, input_file=input_file) + .map(PaperKeywordExtractor, top_k=top_k) + .map(PaperTopicClassifier) + .sink(PaperClassificationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/partner_profile_hub/README.md b/apps/src/sage/apps/partner_profile_hub/README.md new file mode 100644 index 0000000..056d0f3 --- /dev/null +++ b/apps/src/sage/apps/partner_profile_hub/README.md @@ -0,0 +1,5 @@ +# Partner Profile Hub + +读取合作伙伴资料，统一字段、检测重复并生成档案视图。 + +输入：CSV 或 JSON 合作伙伴记录。 输出：包含去重状态和结构化档案的 JSON 文件。 diff --git a/apps/src/sage/apps/partner_profile_hub/__init__.py b/apps/src/sage/apps/partner_profile_hub/__init__.py new file mode 100644 index 0000000..7c4227e --- /dev/null +++ b/apps/src/sage/apps/partner_profile_hub/__init__.py @@ -0,0 +1,5 @@ +"""Partner profile hub application.""" + +from .pipeline import run_partner_profile_hub_pipeline + +__all__ = ["run_partner_profile_hub_pipeline"] diff --git a/apps/src/sage/apps/partner_profile_hub/operators.py b/apps/src/sage/apps/partner_profile_hub/operators.py new file mode 100644 index 0000000..3b5a834 --- /dev/null +++ b/apps/src/sage/apps/partner_profile_hub/operators.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class PartnerSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class PartnerFieldMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["partner_id"] = ( + item.get("partner_id") or item.get("id") or item.get("crm_id") or "unknown" + ) + item["partner_name"] = ( + item.get("partner_name") or item.get("name") or item.get("company") or "unknown" + ) + item["partner_email"] = str(item.get("partner_email") or item.get("email") or "").lower() + item["partner_phone"] = str(item.get("partner_phone") or item.get("phone") or "") + item["partner_region"] = ( + item.get("partner_region") or item.get("region") or item.get("country") or "unknown" + ) + return item + + +class PartnerDeduplicator(MapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._seen_keys: set[str] = set() + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + dedup_key = "|".join( + [ + str(item.get("partner_name", "")).strip().lower(), + str(item.get("partner_email", "")).strip().lower(), + str(item.get("partner_phone", "")).strip(), + ] + ) + item["partner_dedup_key"] = dedup_key + item["is_duplicate"] = dedup_key in self._seen_keys + self._seen_keys.add(dedup_key) + return item + + +class PartnerProfileBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["partner_profile"] = { + "partner_id": item.get("partner_id"), + "partner_name": item.get("partner_name"), + "region": item.get("partner_region"), + "contact": { + "email": item.get("partner_email"), + "phone": item.get("partner_phone"), + }, + "duplicate_status": "duplicate" if item.get("is_duplicate") else "primary", + } + return item + + +class PartnerSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/partner_profile_hub/pipeline.py b/apps/src/sage/apps/partner_profile_hub/pipeline.py new file mode 100644 index 0000000..3d52e22 --- /dev/null +++ b/apps/src/sage/apps/partner_profile_hub/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + PartnerDeduplicator, + PartnerFieldMapper, + PartnerProfileBuilder, + PartnerSink, + PartnerSource, +) + + +def run_partner_profile_hub_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("partner_profile_hub") + ( + env.from_batch(PartnerSource, input_file=input_file) + .map(PartnerFieldMapper) + .map(PartnerDeduplicator) + .map(PartnerProfileBuilder) + .sink(PartnerSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/patent_competition_monitor/README.md b/apps/src/sage/apps/patent_competition_monitor/README.md new file mode 100644 index 0000000..5b89c03 --- /dev/null +++ b/apps/src/sage/apps/patent_competition_monitor/README.md @@ -0,0 +1,6 @@ +# 专利竞争格局监测系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_patent_competition_monitor_pipeline` +- Entry script: `examples/run_patent_competition_monitor.py` diff --git a/apps/src/sage/apps/patent_competition_monitor/__init__.py b/apps/src/sage/apps/patent_competition_monitor/__init__.py new file mode 100644 index 0000000..6f4dad2 --- /dev/null +++ b/apps/src/sage/apps/patent_competition_monitor/__init__.py @@ -0,0 +1,5 @@ +"""专利竞争格局监测系统 application.""" + +from .pipeline import run_patent_competition_monitor_pipeline + +__all__ = ["run_patent_competition_monitor_pipeline"] diff --git a/apps/src/sage/apps/patent_competition_monitor/operators.py b/apps/src/sage/apps/patent_competition_monitor/operators.py new file mode 100644 index 0000000..e97b038 --- /dev/null +++ b/apps/src/sage/apps/patent_competition_monitor/operators.py @@ -0,0 +1,173 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _tokenize(text: str) -> set[str]: + return set(re.findall(r"[a-zA-Z][a-zA-Z0-9_-]{2,}", text.lower())) + + +def _as_list(value: Any) -> list[str]: + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return [part.strip() for part in re.split(r"[,;|]", str(value or "")) if part.strip()] + + +class PatentSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_file) + for item in items: + item.setdefault("source_path", self.input_file) + return items + + +class PatentFieldExtractor(MapFunction): + TECH_FIELDS = { + "semiconductor": {"chip", "semiconductor", "wafer", "packaging"}, + "biotech": {"protein", "gene", "antibody", "biomarker"}, + "battery": {"battery", "cathode", "electrolyte", "anode"}, + "ai": {"model", "neural", "inference", "training", "llm"}, + "network": {"router", "wireless", "5g", "network", "antenna"}, + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + title = str( + payload.get("title") or payload.get("patent_title") or payload.get("text") or "" + ) + abstract = str(payload.get("abstract") or payload.get("summary") or "") + claims = str(payload.get("claims") or payload.get("claim_summary") or "") + combined = " ".join(part for part in (title, abstract, claims) if part) + tokens = _tokenize(combined) + field_scores = { + field: sum(1 for keyword in keywords if keyword in tokens) + for field, keywords in self.TECH_FIELDS.items() + } + best_field, best_score = max( + field_scores.items(), key=lambda pair: pair[1], default=("general", 0) + ) + payload["patent_title"] = title + payload["patent_field"] = best_field if best_score > 0 else "general" + payload["assignee"] = str(payload.get("assignee") or payload.get("applicant") or "unknown") + payload["monitored_company"] = str( + payload.get("monitored_company") or payload.get("target_company") or "" + ) + payload["key_terms"] = sorted(tokens)[:12] + return payload + + +class PatentTopicClassifier(MapFunction): + RISK_TERMS = { + "high": {"exclusive", "infringement", "blocking", "cease", "litigation"}, + "medium": {"claim", "patent", "priority", "licensing", "portfolio"}, + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + title = str(payload.get("patent_title") or "") + tokens = set(payload.get("key_terms") or []) | _tokenize(title) + monitored_terms = _tokenize( + str(payload.get("monitored_terms") or payload.get("watch_terms") or "") + ) + overlap_terms = sorted(tokens & monitored_terms) + cited_competitors = _as_list( + payload.get("cited_competitors") or payload.get("competitors") or "" + ) + monitored_company = str(payload.get("monitored_company") or "").strip().lower() + competitor_hit = any(name.lower() != monitored_company for name in cited_competitors) + severity = "low" + if overlap_terms or any(term in tokens for term in self.RISK_TERMS["high"]): + severity = "high" + elif competitor_hit or any(term in tokens for term in self.RISK_TERMS["medium"]): + severity = "medium" + payload["watch_term_overlap"] = overlap_terms + payload["competitor_hit"] = competitor_hit + payload["risk_severity"] = severity + payload["topic_label"] = f"{payload.get('patent_field', 'general')}_{severity}" + return payload + + +class CompetitorAggregator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + overlap_count = len(payload.get("watch_term_overlap") or []) + competitor_bonus = 3 if payload.get("competitor_hit") else 0 + claim_width = min( + len(_tokenize(str(payload.get("claims") or payload.get("claim_summary") or ""))), 8 + ) + score = overlap_count * 2 + competitor_bonus + claim_width + level = "watch" + if score >= 10: + level = "critical" + elif score >= 6: + level = "elevated" + payload["infringement_risk_score"] = score + payload["alert_level"] = level + payload["alert_summary"] = ( + f"{payload.get('assignee', 'unknown')} 在 {payload.get('patent_field', 'general')} 领域出现 {level} 风险，" + f"命中 {overlap_count} 个关注术语。" + ) + return payload + + +class PatentDigestSink(SinkFunction): + def __init__(self, output_dir: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_dir + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + if target.suffix: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) + else: + target.mkdir(parents=True, exist_ok=True) + (target / "results.json").write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), + encoding="utf-8", + ) diff --git a/apps/src/sage/apps/patent_competition_monitor/pipeline.py b/apps/src/sage/apps/patent_competition_monitor/pipeline.py new file mode 100644 index 0000000..da06d4c --- /dev/null +++ b/apps/src/sage/apps/patent_competition_monitor/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + CompetitorAggregator, + PatentDigestSink, + PatentFieldExtractor, + PatentSource, + PatentTopicClassifier, +) + + +def run_patent_competition_monitor_pipeline(input_file: str, output_dir: str) -> None: + env = LocalEnvironment("patent_competition_monitor") + ( + env.from_batch(PatentSource, input_file=input_file) + .map(PatentFieldExtractor) + .map(PatentTopicClassifier) + .map(CompetitorAggregator) + .sink(PatentDigestSink, output_dir=output_dir) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/permission_audit/README.md b/apps/src/sage/apps/permission_audit/README.md new file mode 100644 index 0000000..d1c5b95 --- /dev/null +++ b/apps/src/sage/apps/permission_audit/README.md @@ -0,0 +1,3 @@ +# Permission Audit + +用户权限变更审计应用。 diff --git a/apps/src/sage/apps/permission_audit/__init__.py b/apps/src/sage/apps/permission_audit/__init__.py new file mode 100644 index 0000000..2c7b01d --- /dev/null +++ b/apps/src/sage/apps/permission_audit/__init__.py @@ -0,0 +1,5 @@ +"""Permission audit application.""" + +from .pipeline import run_permission_audit_pipeline + +__all__ = ["run_permission_audit_pipeline"] diff --git a/apps/src/sage/apps/permission_audit/operators.py b/apps/src/sage/apps/permission_audit/operators.py new file mode 100644 index 0000000..1ea4790 --- /dev/null +++ b/apps/src/sage/apps/permission_audit/operators.py @@ -0,0 +1,79 @@ +"""Operators for permission audit.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class AuditLogSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class AuditParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["user_id"] = str(item.get("user_id", "unknown")).strip() + item["permission"] = str(item.get("permission", "")).strip().lower() + item["action"] = str(item.get("action", "grant")).strip().lower() + return item + + +class AuditLogParser(AuditParser): + pass + + +class SensitivePermissionDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + permission = item.get("permission", "") + item["is_sensitive"] = any( + keyword in permission for keyword in ["admin", "delete", "export", "grant", "root"] + ) + return item + + +class SensitiveActionDetector(SensitivePermissionDetector): + pass + + +class RiskScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = 1 + if item.get("is_sensitive"): + score += 2 + if item.get("action") in {"grant", "elevate"}: + score += 1 + item["risk_score"] = score + item["risk_level"] = "high" if score >= 4 else "medium" if score >= 2 else "low" + return item + + +class AuditRiskScorer(RiskScorer): + pass + + +class AuditSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/permission_audit/pipeline.py b/apps/src/sage/apps/permission_audit/pipeline.py new file mode 100644 index 0000000..d7d4521 --- /dev/null +++ b/apps/src/sage/apps/permission_audit/pipeline.py @@ -0,0 +1,25 @@ +"""Permission audit pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + AuditLogParser, + AuditLogSource, + AuditRiskScorer, + AuditSink, + SensitiveActionDetector, +) + + +def run_permission_audit_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("permission_audit") + ( + env.from_batch(AuditLogSource, input_file=input_file) + .map(AuditLogParser) + .map(SensitiveActionDetector) + .map(AuditRiskScorer) + .sink(AuditSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/permit_material_review/README.md b/apps/src/sage/apps/permit_material_review/README.md new file mode 100644 index 0000000..19f9f1b --- /dev/null +++ b/apps/src/sage/apps/permit_material_review/README.md @@ -0,0 +1,6 @@ +# 许可申报材料审查系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_permit_material_review_pipeline` +- Entry script: `examples/run_permit_material_review.py` diff --git a/apps/src/sage/apps/permit_material_review/__init__.py b/apps/src/sage/apps/permit_material_review/__init__.py new file mode 100644 index 0000000..c60e919 --- /dev/null +++ b/apps/src/sage/apps/permit_material_review/__init__.py @@ -0,0 +1,5 @@ +"""许可申报材料审查系统 application.""" + +from .pipeline import run_permit_material_review_pipeline + +__all__ = ["run_permit_material_review_pipeline"] diff --git a/apps/src/sage/apps/permit_material_review/operators.py b/apps/src/sage/apps/permit_material_review/operators.py new file mode 100644 index 0000000..76abb8c --- /dev/null +++ b/apps/src/sage/apps/permit_material_review/operators.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +class PermitMaterialSource(ListBatchSource): + def __init__(self, input_dir: str, **kwargs): + super().__init__(**kwargs) + self.input_dir = input_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_dir) + for item in items: + item.setdefault("app_slug", "permit_material_review") + item.setdefault("source_path", self.input_dir) + return items + + +class PermitDocumentClassifier(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["application_type"] = str( + payload.get("application_type") or payload.get("permit_type") or "general" + ) + payload["document_type"] = str( + payload.get("document_type") or payload.get("filename") or "unknown" + ) + payload["submitted_docs"] = [ + part.strip() + for part in str(payload.get("submitted_docs") or "").split(",") + if part.strip() + ] + return payload + + +class PermitChecklistChecker(MapFunction): + CHECKLISTS = { + "construction": ["application_form", "site_plan", "safety_commitment"], + "food": ["application_form", "health_certificate", "floor_plan"], + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + required = self.CHECKLISTS.get( + str(payload.get("application_type") or ""), ["application_form", "id_copy"] + ) + submitted = set(payload.get("submitted_docs") or []) + missing = [doc for doc in required if doc not in submitted] + issues = list(missing) + if str(payload.get("has_stamp") or "false").lower() not in {"true", "1", "yes"}: + issues.append("missing_stamp") + if int(float(payload.get("page_count") or 0)) == 0: + issues.append("missing_pages") + payload["required_docs"] = required + payload["missing_items"] = missing + payload["review_issues"] = issues + return payload + + +class PermitReviewFormatter(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + status = "return_for_fix" if payload.get("review_issues") else "accepted" + payload["review_status"] = status + payload["review_summary"] = ( + f"Application {payload.get('application_type')} status {status}, " + f"issues {', '.join(payload.get('review_issues') or []) or 'none'}." + ) + return payload + + +class PermitReviewSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/permit_material_review/pipeline.py b/apps/src/sage/apps/permit_material_review/pipeline.py new file mode 100644 index 0000000..f4c4f92 --- /dev/null +++ b/apps/src/sage/apps/permit_material_review/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + PermitChecklistChecker, + PermitDocumentClassifier, + PermitMaterialSource, + PermitReviewFormatter, + PermitReviewSink, +) + + +def run_permit_material_review_pipeline(input_dir: str, output_file: str) -> None: + env = LocalEnvironment("permit_material_review") + ( + env.from_batch(PermitMaterialSource, input_dir=input_dir) + .map(PermitDocumentClassifier) + .map(PermitChecklistChecker) + .map(PermitReviewFormatter) + .sink(PermitReviewSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/podcast_highlight/README.md b/apps/src/sage/apps/podcast_highlight/README.md new file mode 100644 index 0000000..2dbffd2 --- /dev/null +++ b/apps/src/sage/apps/podcast_highlight/README.md @@ -0,0 +1,6 @@ +# 播客高光切片系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_podcast_highlight_pipeline` +- Entry script: `examples/run_podcast_highlight.py` diff --git a/apps/src/sage/apps/podcast_highlight/__init__.py b/apps/src/sage/apps/podcast_highlight/__init__.py new file mode 100644 index 0000000..6db788c --- /dev/null +++ b/apps/src/sage/apps/podcast_highlight/__init__.py @@ -0,0 +1,5 @@ +"""播客高光切片系统 application.""" + +from .pipeline import run_podcast_highlight_pipeline + +__all__ = ["run_podcast_highlight_pipeline"] diff --git a/apps/src/sage/apps/podcast_highlight/operators.py b/apps/src/sage/apps/podcast_highlight/operators.py new file mode 100644 index 0000000..0f01bf8 --- /dev/null +++ b/apps/src/sage/apps/podcast_highlight/operators.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, str]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, str]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": str(item)} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +class PodcastTranscriptSource(ListBatchSource): + def __init__(self, transcript_file: str, **kwargs): + super().__init__(**kwargs) + self.transcript_file = transcript_file + + def load_items(self) -> list[dict[str, str]]: + items = _load_records(self.transcript_file) + for item in items: + item.setdefault("app_slug", "podcast_highlight") + item.setdefault("source_path", self.transcript_file) + return items + + +class PodcastSegmenter(FlatMapFunction): + def execute(self, item: dict[str, str]) -> list[dict[str, str | int]]: + text = str(item.get("transcript") or item.get("text") or "") + parts = [part.strip() for part in re.split(r"\n\s*\n|\|", text) if part.strip()] + if not parts: + parts = [json.dumps(item, ensure_ascii=False)] + results: list[dict[str, str | int]] = [] + for index, part in enumerate(parts[:8]): + child = dict(item) + child["segment_text"] = part + child["segment_index"] = index + time_match = re.search(r"\[(\d{2}:\d{2})-(\d{2}:\d{2})\]", part) + child["segment_range"] = time_match.group(0) if time_match else f"segment-{index}" + results.append(child) + return results + + +class HighlightScorer(MapFunction): + HIGH_SIGNAL_TERMS = { + "surprising", + "lesson", + "mistake", + "growth", + "turning point", + "breakthrough", + } + + def execute(self, item: dict[str, str | int]) -> dict[str, str | int | float]: + payload = dict(item) + segment = str(payload.get("segment_text") or "") + score = len(segment.split()) / 8 + lowered = segment.lower() + for term in self.HIGH_SIGNAL_TERMS: + if term in lowered: + score += 3 + if "?" in segment: + score += 1 + payload["highlight_score"] = round(score, 2) + payload["highlight_tag"] = "viral_candidate" if score >= 6 else "usable_clip" + return payload + + +class HighlightTitleBuilder(MapFunction): + def execute(self, item: dict[str, str | int | float]) -> dict[str, str | int | float]: + payload = dict(item) + segment = re.sub(r"\[[^\]]+\]\s*", "", str(payload.get("segment_text") or "")).strip() + words = segment.split() + title = " ".join(words[:8]) if words else "Podcast Highlight" + payload["highlight_title"] = title[:80] + payload["distribution_note"] = ( + f"Segment {payload.get('segment_range')} scored {payload.get('highlight_score')}, " + f"recommended tag {payload.get('highlight_tag')}." + ) + return payload + + +class HighlightSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, object]] = [] + + def execute(self, item: dict[str, object]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/podcast_highlight/pipeline.py b/apps/src/sage/apps/podcast_highlight/pipeline.py new file mode 100644 index 0000000..4052738 --- /dev/null +++ b/apps/src/sage/apps/podcast_highlight/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + HighlightScorer, + HighlightSink, + HighlightTitleBuilder, + PodcastSegmenter, + PodcastTranscriptSource, +) + + +def run_podcast_highlight_pipeline(transcript_file: str, output_file: str) -> None: + env = LocalEnvironment("podcast_highlight") + ( + env.from_batch(PodcastTranscriptSource, transcript_file=transcript_file) + .flatmap(PodcastSegmenter) + .map(HighlightScorer) + .map(HighlightTitleBuilder) + .sink(HighlightSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/policy_search_helper/README.md b/apps/src/sage/apps/policy_search_helper/README.md new file mode 100644 index 0000000..5c274f3 --- /dev/null +++ b/apps/src/sage/apps/policy_search_helper/README.md @@ -0,0 +1,6 @@ +# 企业制度检索助手 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_policy_search_helper_pipeline` +- Entry script: `examples/run_policy_search_helper.py` diff --git a/apps/src/sage/apps/policy_search_helper/__init__.py b/apps/src/sage/apps/policy_search_helper/__init__.py new file mode 100644 index 0000000..f6185ff --- /dev/null +++ b/apps/src/sage/apps/policy_search_helper/__init__.py @@ -0,0 +1,5 @@ +"""企业制度检索助手 application.""" + +from .pipeline import run_policy_search_helper_pipeline + +__all__ = ["run_policy_search_helper_pipeline"] diff --git a/apps/src/sage/apps/policy_search_helper/operators.py b/apps/src/sage/apps/policy_search_helper/operators.py new file mode 100644 index 0000000..1fbdd50 --- /dev/null +++ b/apps/src/sage/apps/policy_search_helper/operators.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _tokenize(text: str) -> set[str]: + return {token for token in re.findall(r"[\w\-]{2,}", text.lower()) if token} + + +class PolicyDocSource(ListBatchSource): + def __init__(self, doc_dir: str, question_file: str, **kwargs): + super().__init__(**kwargs) + self.doc_dir = doc_dir + self.question_file = question_file + + def load_items(self) -> list[dict[str, Any]]: + questions = _load_records(self.question_file) + question_text = "\n".join( + str(item.get("question") or item.get("text") or "") for item in questions + ) + items = _load_records(self.doc_dir) + for item in items: + item.setdefault("app_slug", "policy_search_helper") + item["question_text"] = question_text + item["question_file"] = self.question_file + return items + + +class PolicyChunker(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + text = str(item.get("text") or item.get("content") or item.get("body") or "") + parts = [ + part.strip() for part in re.split(r"\n\s*\n|(?<=。)|(?<=\.)\s+", text) if part.strip() + ] + if not parts: + parts = [json.dumps(item, ensure_ascii=False)] + results: list[dict[str, Any]] = [] + for index, part in enumerate(parts[:8]): + child = dict(item) + child["policy_chunk"] = part + child["chunk_index"] = index + results.append(child) + return results + + +class PolicyQuestionMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + question_text = str(payload.get("question_text") or "") + chunk = str(payload.get("policy_chunk") or "") + overlap = _tokenize(question_text) & _tokenize(chunk) + payload["matched_terms"] = sorted(overlap) + payload["match_score"] = len(overlap) + payload["matched_clause"] = chunk + return payload + + +class PolicyAnswerComposer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + source = Path(str(payload.get("source_path") or "")).name + clause = str(payload.get("matched_clause") or "") + payload["answer_excerpt"] = clause[:220] + payload["citation"] = f"{source}#chunk-{payload.get('chunk_index', 0)}" + payload["answer_quality"] = "direct" if payload.get("match_score", 0) >= 2 else "weak" + payload["answer_summary"] = ( + f"Matched clause from {source}, citation {payload.get('citation')}, " + f"matched terms {', '.join(payload.get('matched_terms') or []) or 'none'}." + ) + return payload + + +class PolicyAnswerSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/policy_search_helper/pipeline.py b/apps/src/sage/apps/policy_search_helper/pipeline.py new file mode 100644 index 0000000..067d87f --- /dev/null +++ b/apps/src/sage/apps/policy_search_helper/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + PolicyAnswerComposer, + PolicyAnswerSink, + PolicyChunker, + PolicyDocSource, + PolicyQuestionMatcher, +) + + +def run_policy_search_helper_pipeline(doc_dir: str, question_file: str, output_file: str) -> None: + env = LocalEnvironment("policy_search_helper") + ( + env.from_batch(PolicyDocSource, doc_dir=doc_dir, question_file=question_file) + .flatmap(PolicyChunker) + .map(PolicyQuestionMatcher) + .map(PolicyAnswerComposer) + .sink(PolicyAnswerSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/policy_update_notifier/README.md b/apps/src/sage/apps/policy_update_notifier/README.md new file mode 100644 index 0000000..5386117 --- /dev/null +++ b/apps/src/sage/apps/policy_update_notifier/README.md @@ -0,0 +1,5 @@ +# Policy Update Notifier + +读取制度版本内容，抽取差异并输出影响提醒。 + +输入：CSV 或 JSON 制度版本记录。 输出：包含变更词项和影响级别的 JSON 文件。 diff --git a/apps/src/sage/apps/policy_update_notifier/__init__.py b/apps/src/sage/apps/policy_update_notifier/__init__.py new file mode 100644 index 0000000..c09c862 --- /dev/null +++ b/apps/src/sage/apps/policy_update_notifier/__init__.py @@ -0,0 +1,5 @@ +"""Policy update notifier application.""" + +from .pipeline import run_policy_update_notifier_pipeline + +__all__ = ["run_policy_update_notifier_pipeline"] diff --git a/apps/src/sage/apps/policy_update_notifier/operators.py b/apps/src/sage/apps/policy_update_notifier/operators.py new file mode 100644 index 0000000..2b37f2a --- /dev/null +++ b/apps/src/sage/apps/policy_update_notifier/operators.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class PolicyVersionSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class PolicyDiffExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + previous_text = str(item.get("previous_policy") or item.get("before") or "") + current_text = str( + item.get("current_policy") or item.get("after") or item.get("policy") or "" + ) + before_tokens = set(re.findall(r"[a-zA-Z_]{4,}", previous_text.lower())) + after_tokens = set(re.findall(r"[a-zA-Z_]{4,}", current_text.lower())) + item["added_policy_terms"] = sorted(after_tokens - before_tokens) + item["removed_policy_terms"] = sorted(before_tokens - after_tokens) + return item + + +class PolicyImpactAnalyzer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + impact_keywords = {"security", "privacy", "approval", "access", "audit", "compliance"} + changed = set(item.get("added_policy_terms", [])) | set( + item.get("removed_policy_terms", []) + ) + impact_hits = sorted(changed & impact_keywords) + item["policy_impact_areas"] = impact_hits + item["policy_notice_level"] = ( + "high" if len(impact_hits) >= 2 else "medium" if impact_hits else "low" + ) + return item + + +class PolicyNoticeSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/policy_update_notifier/pipeline.py b/apps/src/sage/apps/policy_update_notifier/pipeline.py new file mode 100644 index 0000000..ca3a4e8 --- /dev/null +++ b/apps/src/sage/apps/policy_update_notifier/pipeline.py @@ -0,0 +1,21 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + PolicyDiffExtractor, + PolicyImpactAnalyzer, + PolicyNoticeSink, + PolicyVersionSource, +) + + +def run_policy_update_notifier_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("policy_update_notifier") + ( + env.from_batch(PolicyVersionSource, input_file=input_file) + .map(PolicyDiffExtractor) + .map(PolicyImpactAnalyzer) + .sink(PolicyNoticeSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/product_sync/README.md b/apps/src/sage/apps/product_sync/README.md new file mode 100644 index 0000000..a7b1d9d --- /dev/null +++ b/apps/src/sage/apps/product_sync/README.md @@ -0,0 +1,18 @@ +# Product Sync + +商品信息同步应用。 + +## 功能 + +- 读取 CSV 或 JSON 商品数据 +- 映射到统一平台字段 +- 校验 SKU、标题、价格、库存 +- 输出同步前的标准化结果 + +## 用法 + +```bash +python examples/run_product_sync.py \ + --input-file products.csv \ + --output platform_products.json +``` diff --git a/apps/src/sage/apps/product_sync/__init__.py b/apps/src/sage/apps/product_sync/__init__.py new file mode 100644 index 0000000..acfdac5 --- /dev/null +++ b/apps/src/sage/apps/product_sync/__init__.py @@ -0,0 +1,5 @@ +"""Product sync application.""" + +from .pipeline import run_product_sync_pipeline + +__all__ = ["run_product_sync_pipeline"] diff --git a/apps/src/sage/apps/product_sync/operators.py b/apps/src/sage/apps/product_sync/operators.py new file mode 100644 index 0000000..8b29d03 --- /dev/null +++ b/apps/src/sage/apps/product_sync/operators.py @@ -0,0 +1,77 @@ +"""Operators for product synchronization.""" + +from __future__ import annotations + +import csv +import json +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class ProductSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class FieldMapper(MapFunction): + def __init__(self, field_map: dict[str, str] | None = None, **kwargs): + super().__init__(**kwargs) + self.field_map = field_map or { + "sku": "sku", + "name": "title", + "price": "price", + "inventory": "stock", + "category": "category", + } + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + mapped = {} + for source_key, target_key in self.field_map.items(): + mapped[target_key] = item.get(source_key, "") + mapped["raw"] = item + return mapped + + +class DataValidator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + errors = [] + if not item.get("sku"): + errors.append("missing_sku") + if not item.get("title"): + errors.append("missing_title") + try: + item["price"] = float(item.get("price") or 0) + except (TypeError, ValueError): + errors.append("invalid_price") + item["price"] = 0.0 + try: + item["stock"] = int(float(item.get("stock") or 0)) + except (TypeError, ValueError): + errors.append("invalid_stock") + item["stock"] = 0 + item["is_valid"] = not errors + item["validation_errors"] = errors + return item + + +class PlatformSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + with open(self.output_file, "w", encoding="utf-8") as handle: + json.dump(self.items, handle, ensure_ascii=False, indent=2) diff --git a/apps/src/sage/apps/product_sync/pipeline.py b/apps/src/sage/apps/product_sync/pipeline.py new file mode 100644 index 0000000..c7eee98 --- /dev/null +++ b/apps/src/sage/apps/product_sync/pipeline.py @@ -0,0 +1,22 @@ +"""Product sync pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import DataValidator, FieldMapper, PlatformSink, ProductSource + + +def run_product_sync_pipeline( + input_file: str, + output_file: str, + field_map: dict[str, str] | None = None, +) -> None: + env = LocalEnvironment("product_sync") + ( + env.from_batch(ProductSource, input_file=input_file) + .map(FieldMapper, field_map=field_map) + .map(DataValidator) + .sink(PlatformSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/project_risk_monitor/README.md b/apps/src/sage/apps/project_risk_monitor/README.md new file mode 100644 index 0000000..5ea0219 --- /dev/null +++ b/apps/src/sage/apps/project_risk_monitor/README.md @@ -0,0 +1,5 @@ +# Project Risk Monitor + +读取项目日志，抽取风险关键词并输出风险等级。 + +输入：CSV 或 JSON 项目日志。 输出：包含风险词、分数和等级的 JSON 文件。 diff --git a/apps/src/sage/apps/project_risk_monitor/__init__.py b/apps/src/sage/apps/project_risk_monitor/__init__.py new file mode 100644 index 0000000..80cadc0 --- /dev/null +++ b/apps/src/sage/apps/project_risk_monitor/__init__.py @@ -0,0 +1,5 @@ +"""Project risk monitor application.""" + +from .pipeline import run_project_risk_monitor_pipeline + +__all__ = ["run_project_risk_monitor_pipeline"] diff --git a/apps/src/sage/apps/project_risk_monitor/operators.py b/apps/src/sage/apps/project_risk_monitor/operators.py new file mode 100644 index 0000000..9f5bc4a --- /dev/null +++ b/apps/src/sage/apps/project_risk_monitor/operators.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class ProjectLogSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class RiskKeywordExtractor(MapFunction): + RISK_TERMS = ("delay", "blocker", "escalation", "overrun", "issue", "risk") + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = " ".join( + str(item.get(field, "")) for field in ("message", "summary", "detail") + ).lower() + item["risk_keywords"] = [ + term for term in self.RISK_TERMS if re.search(rf"\b{term}\b", text) + ] + return item + + +class ProjectRiskScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = len(item.get("risk_keywords", [])) + if str(item.get("priority") or "").lower() in {"high", "critical"}: + score += 1 + item["project_risk_score"] = score + item["project_risk_level"] = "high" if score >= 3 else "medium" if score >= 1 else "low" + return item + + +class ProjectRiskSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/project_risk_monitor/pipeline.py b/apps/src/sage/apps/project_risk_monitor/pipeline.py new file mode 100644 index 0000000..99e12c9 --- /dev/null +++ b/apps/src/sage/apps/project_risk_monitor/pipeline.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ProjectLogSource, ProjectRiskScorer, ProjectRiskSink, RiskKeywordExtractor + + +def run_project_risk_monitor_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("project_risk_monitor") + ( + env.from_batch(ProjectLogSource, input_file=input_file) + .map(RiskKeywordExtractor) + .map(ProjectRiskScorer) + .sink(ProjectRiskSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/quality_defect_filter/README.md b/apps/src/sage/apps/quality_defect_filter/README.md new file mode 100644 index 0000000..f9c4c41 --- /dev/null +++ b/apps/src/sage/apps/quality_defect_filter/README.md @@ -0,0 +1,3 @@ +# Quality Defect Filter + +生产质量缺陷过滤与标准化应用。 diff --git a/apps/src/sage/apps/quality_defect_filter/__init__.py b/apps/src/sage/apps/quality_defect_filter/__init__.py new file mode 100644 index 0000000..4e79a81 --- /dev/null +++ b/apps/src/sage/apps/quality_defect_filter/__init__.py @@ -0,0 +1,5 @@ +"""Quality defect filter application.""" + +from .pipeline import run_quality_defect_filter_pipeline + +__all__ = ["run_quality_defect_filter_pipeline"] diff --git a/apps/src/sage/apps/quality_defect_filter/operators.py b/apps/src/sage/apps/quality_defect_filter/operators.py new file mode 100644 index 0000000..b289084 --- /dev/null +++ b/apps/src/sage/apps/quality_defect_filter/operators.py @@ -0,0 +1,85 @@ +"""Operators for quality defect normalization.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +class QualityReportSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class DefectTextExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + defects = str(item.get("defects") or item.get("description") or "") + item["defect_items"] = [ + part.strip() for part in defects.replace("|", ";").split(";") if part.strip() + ] + return item + + +class DefectSplitter(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + return [ + { + "report_id": item.get("report_id") or item.get("id") or "", + "defect_text": defect, + "severity": item.get("severity", "medium"), + } + for defect in item.get("defect_items", []) + ] + + +class DefectStandardizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = item.get("defect_text", "").lower() + if any(keyword in text for keyword in ["scratch", "dent", "鍒掔棔", "鍑归櫡"]): + item["category"] = "surface" + elif any(keyword in text for keyword in ["leak", "crack", "娓楁紡", "瑁傜汗"]): + item["category"] = "structural" + else: + item["category"] = "other" + return item + + +class DefectSeverityScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + severity = str(item.get("severity", "medium")).lower() + text = item.get("defect_text", "").lower() + score = 2 if severity == "high" else 1 if severity == "medium" else 0 + if any(keyword in text for keyword in ["critical", "鐖嗚", "婕忕數", "safety"]): + score += 2 + elif any(keyword in text for keyword in ["crack", "leak", "瑁傜汗", "娓楁紡"]): + score += 1 + item["severity_score"] = score + item["severity_level"] = "high" if score >= 3 else "medium" if score >= 1 else "low" + return item + + +class DefectSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/quality_defect_filter/pipeline.py b/apps/src/sage/apps/quality_defect_filter/pipeline.py new file mode 100644 index 0000000..9374ea3 --- /dev/null +++ b/apps/src/sage/apps/quality_defect_filter/pipeline.py @@ -0,0 +1,27 @@ +"""Quality defect filter pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + DefectSeverityScorer, + DefectSink, + DefectSplitter, + DefectStandardizer, + DefectTextExtractor, + QualityReportSource, +) + + +def run_quality_defect_filter_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("quality_defect_filter") + ( + env.from_batch(QualityReportSource, input_file=input_file) + .map(DefectTextExtractor) + .flatmap(DefectSplitter) + .map(DefectStandardizer) + .map(DefectSeverityScorer) + .sink(DefectSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/quote_compare/README.md b/apps/src/sage/apps/quote_compare/README.md new file mode 100644 index 0000000..3449fff --- /dev/null +++ b/apps/src/sage/apps/quote_compare/README.md @@ -0,0 +1,6 @@ +# 供应商报价对比系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_quote_compare_pipeline` +- Entry script: `examples/run_quote_compare.py` diff --git a/apps/src/sage/apps/quote_compare/__init__.py b/apps/src/sage/apps/quote_compare/__init__.py new file mode 100644 index 0000000..d3c307a --- /dev/null +++ b/apps/src/sage/apps/quote_compare/__init__.py @@ -0,0 +1,5 @@ +"""供应商报价对比系统 application.""" + +from .pipeline import run_quote_compare_pipeline + +__all__ = ["run_quote_compare_pipeline"] diff --git a/apps/src/sage/apps/quote_compare/operators.py b/apps/src/sage/apps/quote_compare/operators.py new file mode 100644 index 0000000..c67b5a4 --- /dev/null +++ b/apps/src/sage/apps/quote_compare/operators.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class QuoteSource(ListBatchSource): + def __init__(self, input_dir: str, **kwargs): + super().__init__(**kwargs) + self.input_dir = input_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_dir) + for item in items: + item.setdefault("app_slug", "quote_compare") + item.setdefault("source_path", self.input_dir) + return items + + +class QuoteNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + unit_price = _to_float( + payload.get("unit_price") or payload.get("price") or payload.get("quoted_price") + ) + quantity = _to_float(payload.get("quantity") or 1, 1.0) + lead_days = _to_float(payload.get("lead_days") or payload.get("delivery_days"), 999) + payload["vendor"] = str(payload.get("vendor") or payload.get("supplier") or "unknown") + payload["currency"] = str(payload.get("currency") or "CNY") + payload["unit_price"] = unit_price + payload["quantity"] = quantity + payload["lead_days"] = lead_days + payload["total_cost"] = round(unit_price * quantity, 2) + return payload + + +class QuoteConditionComparer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + warranty_months = _to_float(payload.get("warranty_months"), 0) + risk_flags: list[str] = [] + if payload.get("lead_days", 999) > 14: + risk_flags.append("long_lead_time") + if warranty_months < 12: + risk_flags.append("short_warranty") + if str(payload.get("payment_terms") or "").lower() in {"100% prepay", "full prepayment"}: + risk_flags.append("aggressive_payment_terms") + payload["warranty_months"] = warranty_months + payload["risk_flags"] = risk_flags + return payload + + +class QuoteScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + score = 100.0 + score -= payload.get("total_cost", 0.0) / 1000 + score -= max(payload.get("lead_days", 0.0) - 7, 0) * 1.5 + score -= len(payload.get("risk_flags") or []) * 8 + recommendation = "preferred" + if score < 70: + recommendation = "review_required" + if score < 50: + recommendation = "high_risk" + payload["quote_score"] = round(score, 2) + payload["recommendation"] = recommendation + payload["comparison_summary"] = ( + f"Vendor {payload.get('vendor')} total cost {payload.get('total_cost')} {payload.get('currency')}, " + f"lead time {payload.get('lead_days')} days, recommendation {recommendation}." + ) + return payload + + +class QuoteCompareSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/quote_compare/pipeline.py b/apps/src/sage/apps/quote_compare/pipeline.py new file mode 100644 index 0000000..dc4655b --- /dev/null +++ b/apps/src/sage/apps/quote_compare/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + QuoteCompareSink, + QuoteConditionComparer, + QuoteNormalizer, + QuoteScorer, + QuoteSource, +) + + +def run_quote_compare_pipeline(input_dir: str, output_file: str) -> None: + env = LocalEnvironment("quote_compare") + ( + env.from_batch(QuoteSource, input_dir=input_dir) + .map(QuoteNormalizer) + .map(QuoteConditionComparer) + .map(QuoteScorer) + .sink(QuoteCompareSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/radiology_followup_loop/README.md b/apps/src/sage/apps/radiology_followup_loop/README.md new file mode 100644 index 0000000..cdda538 --- /dev/null +++ b/apps/src/sage/apps/radiology_followup_loop/README.md @@ -0,0 +1,6 @@ +# 影像报告随访闭环系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_radiology_followup_loop_pipeline` +- Entry script: `examples/run_radiology_followup_loop.py` diff --git a/apps/src/sage/apps/radiology_followup_loop/__init__.py b/apps/src/sage/apps/radiology_followup_loop/__init__.py new file mode 100644 index 0000000..6b67767 --- /dev/null +++ b/apps/src/sage/apps/radiology_followup_loop/__init__.py @@ -0,0 +1,5 @@ +"""影像报告随访闭环系统 application.""" + +from .pipeline import run_radiology_followup_loop_pipeline + +__all__ = ["run_radiology_followup_loop_pipeline"] diff --git a/apps/src/sage/apps/radiology_followup_loop/operators.py b/apps/src/sage/apps/radiology_followup_loop/operators.py new file mode 100644 index 0000000..a79f8b6 --- /dev/null +++ b/apps/src/sage/apps/radiology_followup_loop/operators.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +class RadiologyReportSource(ListBatchSource): + def __init__(self, report_file: str, patient_file: str, **kwargs): + super().__init__(**kwargs) + self.report_file = report_file + self.patient_file = patient_file + + def load_items(self) -> list[dict[str, Any]]: + reports = _load_records(self.report_file) + patients = _load_records(self.patient_file) + patient_index = { + str(patient.get("patient_id") or patient.get("id") or patient.get("mrn") or ""): patient + for patient in patients + } + for item in reports: + patient_id = str(item.get("patient_id") or item.get("id") or item.get("mrn") or "") + item["patient_profile"] = patient_index.get(patient_id, {}) + item.setdefault("source_path", self.report_file) + return reports + + +class FollowupExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + report_text = str( + payload.get("report_text") or payload.get("impression") or payload.get("text") or "" + ) + match = re.search( + r"follow[- ]?up in\s+(\d+)\s+(day|days|week|weeks|month|months)", report_text, re.I + ) + days = 0 + if match: + amount = int(match.group(1)) + unit = match.group(2).lower() + if "week" in unit: + days = amount * 7 + elif "month" in unit: + days = amount * 30 + else: + days = amount + payload["followup_required"] = days > 0 or "follow-up" in report_text.lower() + payload["followup_due_days"] = days or 30 + payload["recommended_modality"] = ( + "CT" + if "ct" in report_text.lower() + else "MRI" + if "mri" in report_text.lower() + else "ultrasound" + ) + return payload + + +class PatientMatcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + profile = payload.get("patient_profile") or {} + payload["patient_id"] = str( + profile.get("patient_id") or payload.get("patient_id") or "unknown" + ) + payload["patient_name"] = str( + profile.get("name") or payload.get("patient_name") or "unknown" + ) + payload["contact_channel"] = str(profile.get("contact_channel") or "phone") + payload["days_since_report"] = int(float(payload.get("days_since_report") or 0)) + return payload + + +class FollowupDeadlineChecker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + due_days = int(payload.get("followup_due_days") or 30) + elapsed = int(payload.get("days_since_report") or 0) + if not payload.get("followup_required"): + status = "no_followup_needed" + elif elapsed > due_days: + status = "overdue" + elif elapsed >= max(due_days - 7, 0): + status = "due_soon" + else: + status = "scheduled_window" + payload["followup_status"] = status + payload["followup_summary"] = ( + f"患者 {payload.get('patient_name')} 建议 {payload.get('recommended_modality')} 复查，当前状态 {status}。" + ) + return payload + + +class FollowupSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("patient_profile", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/radiology_followup_loop/pipeline.py b/apps/src/sage/apps/radiology_followup_loop/pipeline.py new file mode 100644 index 0000000..83fd890 --- /dev/null +++ b/apps/src/sage/apps/radiology_followup_loop/pipeline.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + FollowupDeadlineChecker, + FollowupExtractor, + FollowupSink, + PatientMatcher, + RadiologyReportSource, +) + + +def run_radiology_followup_loop_pipeline( + report_file: str, patient_file: str, output_file: str +) -> None: + env = LocalEnvironment("radiology_followup_loop") + ( + env.from_batch(RadiologyReportSource, report_file=report_file, patient_file=patient_file) + .map(FollowupExtractor) + .map(PatientMatcher) + .map(FollowupDeadlineChecker) + .sink(FollowupSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/real_estate_valuation/README.md b/apps/src/sage/apps/real_estate_valuation/README.md new file mode 100644 index 0000000..2747709 --- /dev/null +++ b/apps/src/sage/apps/real_estate_valuation/README.md @@ -0,0 +1,3 @@ +# Real Estate Valuation + +房产智能估价应用。 diff --git a/apps/src/sage/apps/real_estate_valuation/__init__.py b/apps/src/sage/apps/real_estate_valuation/__init__.py new file mode 100644 index 0000000..a29a2e1 --- /dev/null +++ b/apps/src/sage/apps/real_estate_valuation/__init__.py @@ -0,0 +1,5 @@ +"""Real estate valuation application.""" + +from .pipeline import run_real_estate_valuation_pipeline + +__all__ = ["run_real_estate_valuation_pipeline"] diff --git a/apps/src/sage/apps/real_estate_valuation/operators.py b/apps/src/sage/apps/real_estate_valuation/operators.py new file mode 100644 index 0000000..fd95de1 --- /dev/null +++ b/apps/src/sage/apps/real_estate_valuation/operators.py @@ -0,0 +1,76 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class PropertySource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class NearbyListingFetcher(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["area_sqm"] = float(item.get("area_sqm") or 0) + item["price_per_sqm"] = float( + item.get("price_per_sqm") or item.get("nearby_avg_price") or 0 + ) + item["location_factor"] = float(item.get("location_factor") or 1) + item["listing_count"] = int( + float(item.get("listing_count") or item.get("comps_count") or 0) + ) + return item + + +class ValueFeatureExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["area_sqm"] = float(item.get("area_sqm") or 0) + item["price_per_sqm"] = float(item.get("price_per_sqm") or 0) + item["location_factor"] = float(item.get("location_factor") or 1) + return item + + +class ValuationFeatureBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + enriched = NearbyListingFetcher().execute(item) + enriched["market_confidence"] = min(1.0, 0.5 + enriched["listing_count"] * 0.05) + return enriched + + +class ValueEstimator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["estimated_value"] = round( + item["area_sqm"] * item["price_per_sqm"] * item["location_factor"], 2 + ) + return item + + +class ValuationCalculator(ValueEstimator): + pass + + +class ValuationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/real_estate_valuation/pipeline.py b/apps/src/sage/apps/real_estate_valuation/pipeline.py new file mode 100644 index 0000000..03a253e --- /dev/null +++ b/apps/src/sage/apps/real_estate_valuation/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + NearbyListingFetcher, + PropertySource, + ValuationCalculator, + ValuationFeatureBuilder, + ValuationSink, +) + + +def run_real_estate_valuation_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("real_estate_valuation") + ( + env.from_batch(PropertySource, input_file=input_file) + .map(NearbyListingFetcher) + .map(ValuationFeatureBuilder) + .map(ValuationCalculator) + .sink(ValuationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/repro_audit/README.md b/apps/src/sage/apps/repro_audit/README.md new file mode 100644 index 0000000..6ec9ae6 --- /dev/null +++ b/apps/src/sage/apps/repro_audit/README.md @@ -0,0 +1,6 @@ +# 科研交付复现审计系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_repro_audit_pipeline` +- Entry script: `examples/run_repro_audit.py` diff --git a/apps/src/sage/apps/repro_audit/__init__.py b/apps/src/sage/apps/repro_audit/__init__.py new file mode 100644 index 0000000..ec78f21 --- /dev/null +++ b/apps/src/sage/apps/repro_audit/__init__.py @@ -0,0 +1,5 @@ +"""科研交付复现审计系统 application.""" + +from .pipeline import run_repro_audit_pipeline + +__all__ = ["run_repro_audit_pipeline"] diff --git a/apps/src/sage/apps/repro_audit/operators.py b/apps/src/sage/apps/repro_audit/operators.py new file mode 100644 index 0000000..02b5330 --- /dev/null +++ b/apps/src/sage/apps/repro_audit/operators.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _as_bool(value: Any) -> bool: + return str(value).strip().lower() in {"1", "true", "yes", "y", "present"} + + +class ReproManifestSource(ListBatchSource): + def __init__(self, manifest_file: str, **kwargs): + super().__init__(**kwargs) + self.manifest_file = manifest_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.manifest_file) + for item in items: + item.setdefault("source_path", self.manifest_file) + return items + + +class ReproMetadataExtractor(MapFunction): + REQUIRED_FIELDS = ("dataset_path", "script_path", "environment", "seed", "metric_name") + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["artifact_presence"] = { + field: bool(payload.get(field)) for field in self.REQUIRED_FIELDS + } + payload["run_id"] = str(payload.get("run_id") or payload.get("experiment_id") or "unknown") + payload["result_reproduced"] = _as_bool(payload.get("result_reproduced")) + payload["config_locked"] = _as_bool(payload.get("config_locked")) + return payload + + +class ReproConsistencyChecker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + missing = [ + field for field, present in payload.get("artifact_presence", {}).items() if not present + ] + issues = list(missing) + if not payload.get("result_reproduced"): + issues.append("result_not_reproduced") + if not payload.get("config_locked"): + issues.append("config_not_locked") + payload["missing_artifacts"] = missing + payload["consistency_issues"] = issues + return payload + + +class ReproRiskScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + score = len(payload.get("missing_artifacts") or []) * 3 + len( + payload.get("consistency_issues") or [] + ) + grade = "pass" + if score >= 8: + grade = "critical" + elif score >= 4: + grade = "warning" + payload["repro_risk_score"] = score + payload["audit_grade"] = grade + payload["audit_summary"] = ( + f"运行 {payload.get('run_id')} 缺失 {len(payload.get('missing_artifacts') or [])} 项资产，" + f"审计结果 {grade}。" + ) + return payload + + +class ReproAuditSink(SinkFunction): + def __init__(self, output_dir: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_dir + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + if target.suffix: + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) + else: + target.mkdir(parents=True, exist_ok=True) + (target / "results.json").write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), + encoding="utf-8", + ) diff --git a/apps/src/sage/apps/repro_audit/pipeline.py b/apps/src/sage/apps/repro_audit/pipeline.py new file mode 100644 index 0000000..114f354 --- /dev/null +++ b/apps/src/sage/apps/repro_audit/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ReproAuditSink, + ReproConsistencyChecker, + ReproManifestSource, + ReproMetadataExtractor, + ReproRiskScorer, +) + + +def run_repro_audit_pipeline(manifest_file: str, output_dir: str) -> None: + env = LocalEnvironment("repro_audit") + ( + env.from_batch(ReproManifestSource, manifest_file=manifest_file) + .map(ReproMetadataExtractor) + .map(ReproConsistencyChecker) + .map(ReproRiskScorer) + .sink(ReproAuditSink, output_dir=output_dir) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/restaurant_sales_analysis/README.md b/apps/src/sage/apps/restaurant_sales_analysis/README.md new file mode 100644 index 0000000..598c41f --- /dev/null +++ b/apps/src/sage/apps/restaurant_sales_analysis/README.md @@ -0,0 +1,3 @@ +# Restaurant Sales Analysis + +餐厅菜品销售分析应用。 diff --git a/apps/src/sage/apps/restaurant_sales_analysis/__init__.py b/apps/src/sage/apps/restaurant_sales_analysis/__init__.py new file mode 100644 index 0000000..e21117d --- /dev/null +++ b/apps/src/sage/apps/restaurant_sales_analysis/__init__.py @@ -0,0 +1,5 @@ +"""Restaurant sales analysis application.""" + +from .pipeline import run_restaurant_sales_analysis_pipeline + +__all__ = ["run_restaurant_sales_analysis_pipeline"] diff --git a/apps/src/sage/apps/restaurant_sales_analysis/operators.py b/apps/src/sage/apps/restaurant_sales_analysis/operators.py new file mode 100644 index 0000000..de25871 --- /dev/null +++ b/apps/src/sage/apps/restaurant_sales_analysis/operators.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import FlatMapFunction, MapFunction, SinkFunction + + +class RestaurantOrderSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class DishSplitter(FlatMapFunction): + def execute(self, item: dict[str, Any]) -> list[dict[str, Any]]: + dish_blob = str(item.get("dishes") or item.get("items") or "") + if not dish_blob: + return [dict(item)] + + dishes: list[dict[str, Any]] = [] + for raw_part in dish_blob.split("|"): + part = raw_part.strip() + if not part: + continue + pieces = [segment.strip() for segment in part.split(":")] + dish_name = pieces[0] + quantity = int(float(pieces[1])) if len(pieces) > 1 and pieces[1] else 1 + revenue = ( + float(pieces[2]) + if len(pieces) > 2 and pieces[2] + else float(item.get("revenue") or 0) + ) + cost = ( + float(pieces[3]) if len(pieces) > 3 and pieces[3] else float(item.get("cost") or 0) + ) + dishes.append( + { + "order_id": item.get("order_id") or item.get("id") or "", + "dish_name": dish_name, + "quantity": quantity, + "revenue": revenue, + "cost": cost, + "inventory_qty": item.get("inventory_qty") or item.get("stock_qty") or 0, + "waste_rate": item.get("waste_rate") or 0, + "sales_count": item.get("sales_count") or quantity, + } + ) + return dishes + + +class InventoryJoiner(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["inventory_qty"] = int(float(item.get("inventory_qty") or 0)) + item["waste_rate"] = float(item.get("waste_rate") or 0) + return item + + +class MenuProfitScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + revenue = float(item.get("revenue") or 0) + cost = float(item.get("cost") or 0) + waste_penalty = float(item.get("waste_rate") or 0) * cost + item["profit"] = round(revenue - cost - waste_penalty, 2) + sales = int(float(item.get("sales_count") or item.get("quantity") or 0)) + inventory_qty = int(item.get("inventory_qty", 0)) + profit = float(item.get("profit", 0)) + if inventory_qty < 5 and sales >= 10: + recommendation = "restock" + elif profit > 30 and sales > 20: + recommendation = "promote" + elif profit <= 0: + recommendation = "remove_or_reprice" + else: + recommendation = "keep" + item["recommendation"] = recommendation + return item + + +class MenuAdviceSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/restaurant_sales_analysis/pipeline.py b/apps/src/sage/apps/restaurant_sales_analysis/pipeline.py new file mode 100644 index 0000000..d3fb017 --- /dev/null +++ b/apps/src/sage/apps/restaurant_sales_analysis/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + DishSplitter, + InventoryJoiner, + MenuAdviceSink, + MenuProfitScorer, + RestaurantOrderSource, +) + + +def run_restaurant_sales_analysis_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("restaurant_sales_analysis") + ( + env.from_batch(RestaurantOrderSource, input_file=input_file) + .flatmap(DishSplitter) + .map(InventoryJoiner) + .map(MenuProfitScorer) + .sink(MenuAdviceSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/resume_parser/README.md b/apps/src/sage/apps/resume_parser/README.md new file mode 100644 index 0000000..f105523 --- /dev/null +++ b/apps/src/sage/apps/resume_parser/README.md @@ -0,0 +1,26 @@ +# Resume Parser + +简历解析与标准化示例应用。 + +## 功能 + +- 读取文本简历 +- 提取姓名、邮箱、电话、教育和技能信息 +- 标准化字段格式 +- 校验简历完整度 +- 输出 JSON 结果 + +## 用法 + +```bash +python examples/run_resume_parser.py \ + --resume-dir ./resumes \ + --output parsed.json +``` + +```bash +python examples/run_resume_parser.py \ + --resume-files resume1.txt resume2.txt \ + --output parsed.json \ + --verbose +``` diff --git a/apps/src/sage/apps/resume_parser/__init__.py b/apps/src/sage/apps/resume_parser/__init__.py new file mode 100644 index 0000000..223e082 --- /dev/null +++ b/apps/src/sage/apps/resume_parser/__init__.py @@ -0,0 +1,5 @@ +"""Resume parser application.""" + +from .pipeline import run_resume_parser_pipeline + +__all__ = ["run_resume_parser_pipeline"] diff --git a/apps/src/sage/apps/resume_parser/operators.py b/apps/src/sage/apps/resume_parser/operators.py new file mode 100644 index 0000000..21d5596 --- /dev/null +++ b/apps/src/sage/apps/resume_parser/operators.py @@ -0,0 +1,293 @@ +""" +Resume Parser Operators + +Custom operators for resume parsing and standardization. +""" + +from __future__ import annotations + +import json +import re +from datetime import datetime +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import CustomLogger, MapFunction, SinkFunction + + +class ResumeSource(ListBatchSource): + """Read resume text files and return them as batch.""" + + def __init__( + self, resume_dir: str | None = None, resume_files: list[str] | None = None, **kwargs + ): + """Initialize resume source. + + Args: + resume_dir: Directory containing resume files + resume_files: List of resume file paths + """ + super().__init__(**kwargs) + self.resume_dir = resume_dir + self.resume_files = resume_files or [] + self._logger = CustomLogger("ResumeSource") + + def load_items(self) -> list[dict[str, str]]: + """Read and return all resume files.""" + import os + + resumes = [] + files_to_process = [] + + if self.resume_dir and os.path.isdir(self.resume_dir): + files_to_process = [ + os.path.join(self.resume_dir, f) + for f in os.listdir(self.resume_dir) + if f.endswith((".txt", ".md")) + ] + + files_to_process.extend(self.resume_files) + + for file_path in files_to_process: + try: + with open(file_path, encoding="utf-8") as f: + content = f.read() + resumes.append( + { + "filename": file_path, + "content": content, + } + ) + except Exception as e: + self.logger.error(f"Error reading {file_path}: {e}") + + self.logger.info(f"Read {len(resumes)} resume files") + return resumes + + +class TextExtractor(MapFunction): + def execute(self, resume: dict[str, str]) -> dict[str, Any]: + enriched = dict(resume) + enriched["text"] = resume.get("content", "") + return enriched + + +class ResumeInfoExtractor(MapFunction): + """Extract structured information from resume text.""" + + def __init__(self, **kwargs): + """Initialize resume info extractor.""" + super().__init__(**kwargs) + self._logger = CustomLogger("ResumeInfoExtractor") + + def execute(self, resume: dict[str, str]) -> dict[str, Any]: + """Extract information from resume. + + Args: + resume: Resume dict with 'content' key + + Returns: + Dict with extracted fields + """ + if not resume or "content" not in resume: + return resume + + content = resume["content"] + result = {"filename": resume.get("filename", "")} + + # Extract name (usually at the beginning) + lines = content.split("\n") + name_candidates = [line.strip() for line in lines[:5] if line.strip()] + result["name"] = name_candidates[0] if name_candidates else "Unknown" + + # Extract email + email_pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + emails = re.findall(email_pattern, content) + result["email"] = emails[0] if emails else "" + + # Extract phone (Chinese phone: 11 digits or +86) + phone_pattern = r"(\+?86)?1[3-9]\d{9}" + phones = re.findall(phone_pattern, content) + result["phone"] = phones[0] if phones else "" + + # Extract education + edu_pattern = r"(瀛﹀＋|纭曞＋|鍗氬＋|Bachelor|Master|PhD|鏈|涓撶)" + educations = re.findall(edu_pattern, content) + result["education_level"] = educations[0] if educations else "" + + # Extract years of experience (look for patterns like "N骞村伐浣滅粡楠?) + exp_pattern = r"(\d+)\s*(?:years?|yrs?)\s*(?:of\s+)?(?:experience|work)" + exp_matches = re.findall(exp_pattern, content) + if exp_matches: + result["years_experience"] = int(exp_matches[0][0]) + else: + result["years_experience"] = 0 + + # Extract skills + skills = [] + for line in lines: + if any(keyword in line for keyword in ["Skills", "Competencies", "Skill"]): + # Extract subsequent lines as skills + idx = lines.index(line) + for skill_line in lines[idx + 1 : idx + 5]: + if skill_line.strip(): + skills.extend([s.strip() for s in skill_line.split(",") if s.strip()]) + result["skills"] = skills[:10] # Top 10 skills + + # Extract companies/positions + positions = [] + company_pattern = r"(鍏徃|Company|Inc\.|Corp\.)" + for i, line in enumerate(lines): + if re.search(company_pattern, line) and i + 1 < len(lines): + positions.append( + { + "company": line.strip(), + "title": lines[i + 1].strip() if i + 1 < len(lines) else "", + } + ) + result["positions"] = positions[:5] # Top 5 positions + + return result + + +class InfoExtractor(ResumeInfoExtractor): + pass + + +class ResumeNormalizer(MapFunction): + """Normalize resume information.""" + + def __init__(self, **kwargs): + """Initialize resume normalizer.""" + super().__init__(**kwargs) + + def execute(self, resume: dict[str, Any]) -> dict[str, Any]: + """Normalize resume data. + + Args: + resume: Extracted resume data + + Returns: + Normalized resume data + """ + if not resume: + return resume + + # Normalize name (title case) + if "name" in resume: + resume["name"] = resume["name"].title() + + # Normalize email (lowercase) + if "email" in resume: + resume["email"] = resume["email"].lower() + + # Normalize education level + education_mapping = { + "瀛﹀＋": "Bachelor", + "纭曞＋": "Master", + "鍗氬＋": "PhD", + "鏈": "Bachelor", + "涓撶": "Associate", + "楂樹腑": "High School", + } + edu = resume.get("education_level", "") + resume["education_level"] = education_mapping.get(edu, edu) + + # Ensure years_experience is int + if "years_experience" in resume: + resume["years_experience"] = int(resume["years_experience"]) + + # Normalize skills (lowercase, deduplicate) + if "skills" in resume: + resume["skills"] = list({skill.lower() for skill in resume["skills"]}) + + # Add processing timestamp + resume["processed_at"] = datetime.now().isoformat() + + return resume + + +class DateNormalizer(ResumeNormalizer): + def execute(self, resume: dict[str, Any]) -> dict[str, Any]: + normalized = super().execute(resume) + for position in normalized.get("positions", []): + if "date" in position and isinstance(position["date"], str): + position["date"] = position["date"].replace("/", "-") + return normalized + + +class ResumeValidator(MapFunction): + """Validate resume completeness.""" + + def __init__(self, required_fields: list[str] | None = None, **kwargs): + """Initialize resume validator. + + Args: + required_fields: List of required fields + """ + super().__init__(**kwargs) + self.required_fields = required_fields or ["name", "email", "phone"] + self._logger = CustomLogger("ResumeValidator") + + def execute(self, resume: dict[str, Any]) -> dict[str, Any]: + """Validate resume. + + Args: + resume: Resume data + + Returns: + Resume with validation markers + """ + if not resume: + return resume + + # Check required fields + missing_fields = [f for f in self.required_fields if not resume.get(f)] + resume["is_complete"] = len(missing_fields) == 0 + resume["missing_fields"] = missing_fields + resume["completeness_score"] = ( + (len(self.required_fields) - len(missing_fields)) / len(self.required_fields) * 100 + ) + + return resume + + +class ResumeSink(SinkFunction): + """Output parsed resumes to JSON file.""" + + def __init__(self, output_file: str, **kwargs): + """Initialize resume sink. + + Args: + output_file: Path to output JSON file + """ + super().__init__(**kwargs) + self.output_file = output_file + self._logger = CustomLogger("ResumeSink") + self.count = 0 + + def setup(self, context: Any) -> None: + """Setup - prepare output file.""" + with open(self.output_file, "w", encoding="utf-8") as f: + f.write("[\n") + + def execute(self, resume: dict[str, Any]) -> None: + """Append resume to JSON file. + + Args: + resume: Parsed resume + """ + if not resume: + return + + with open(self.output_file, "a", encoding="utf-8") as f: + if self.count > 0: + f.write(",\n") + json.dump(resume, f, ensure_ascii=False, default=str, indent=2) + self.count += 1 + + def teardown(self, context: Any) -> None: + """Cleanup - close JSON array.""" + with open(self.output_file, "a", encoding="utf-8") as f: + f.write("\n]") + self.logger.info(f"Written {self.count} parsed resumes to {self.output_file}") diff --git a/apps/src/sage/apps/resume_parser/pipeline.py b/apps/src/sage/apps/resume_parser/pipeline.py new file mode 100644 index 0000000..364fabf --- /dev/null +++ b/apps/src/sage/apps/resume_parser/pipeline.py @@ -0,0 +1,71 @@ +""" +Resume Parser Pipeline + +Main pipeline implementation using SAGE operators for resume parsing. +""" + +from __future__ import annotations + +from sage.foundation import CustomLogger +from sage.runtime import LocalEnvironment + +from .operators import ( + DateNormalizer, + InfoExtractor, + ResumeSink, + ResumeSource, + TextExtractor, +) + + +def run_resume_parser_pipeline( + resume_dir: str | None = None, + resume_files: list[str] | None = None, + output_file: str = "parsed_resumes.json", + verbose: bool = False, +) -> None: + """ + Run the resume parser pipeline using SAGE framework. + + Args: + resume_dir: Directory containing resume files + resume_files: List of resume file paths + output_file: Path to output JSON file + verbose: Enable verbose logging + + Example: + >>> run_resume_parser_pipeline( + ... resume_dir="/path/to/resumes", + ... output_file="parsed_resumes.json", + ... verbose=True + ... ) + """ + logger = CustomLogger("ResumeParserPipeline") + + if verbose: + logger.info("Starting resume parser pipeline") + logger.info(f" Resume dir: {resume_dir}") + logger.info(f" Output file: {output_file}") + + # Create environment + env = LocalEnvironment("resume_parser") + + try: + # Build pipeline + ( + env.from_batch(ResumeSource, resume_dir=resume_dir, resume_files=resume_files) + .map(TextExtractor) + .map(InfoExtractor) + .map(DateNormalizer) + .sink(ResumeSink, output_file=output_file) + ) + + # Submit and run + env.submit(autostop=True) + + if verbose: + logger.info("Resume parser pipeline completed successfully") + + except Exception as e: + logger.error(f"Error in resume parser pipeline: {e}") + raise diff --git a/apps/src/sage/apps/return_reason_mining/README.md b/apps/src/sage/apps/return_reason_mining/README.md new file mode 100644 index 0000000..19fa0af --- /dev/null +++ b/apps/src/sage/apps/return_reason_mining/README.md @@ -0,0 +1,6 @@ +# 电商退货原因挖掘系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_return_reason_mining_pipeline` +- Entry script: `examples/run_return_reason_mining.py` diff --git a/apps/src/sage/apps/return_reason_mining/__init__.py b/apps/src/sage/apps/return_reason_mining/__init__.py new file mode 100644 index 0000000..7685646 --- /dev/null +++ b/apps/src/sage/apps/return_reason_mining/__init__.py @@ -0,0 +1,5 @@ +"""电商退货原因挖掘系统 application.""" + +from .pipeline import run_return_reason_mining_pipeline + +__all__ = ["run_return_reason_mining_pipeline"] diff --git a/apps/src/sage/apps/return_reason_mining/operators.py b/apps/src/sage/apps/return_reason_mining/operators.py new file mode 100644 index 0000000..9ced0e5 --- /dev/null +++ b/apps/src/sage/apps/return_reason_mining/operators.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +class ReturnRecordSource(ListBatchSource): + def __init__(self, return_file: str, **kwargs): + super().__init__(**kwargs) + self.return_file = return_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.return_file) + for item in items: + item.setdefault("app_slug", "return_reason_mining") + item.setdefault("source_path", self.return_file) + return items + + +class ReturnFeatureFusion(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["return_text"] = str( + payload.get("reason_text") or payload.get("customer_note") or "" + ) + payload["product_category"] = str(payload.get("product_category") or "general") + payload["service_notes"] = str(payload.get("service_notes") or "") + return payload + + +class ReturnReasonClusterer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + text = f"{payload.get('return_text', '')} {payload.get('service_notes', '')}".lower() + cluster = "other" + if "size" in text or "fit" in text: + cluster = "size_issue" + elif "broken" in text or "damaged" in text: + cluster = "quality_issue" + elif "late" in text or "delay" in text: + cluster = "delivery_issue" + payload["reason_cluster"] = cluster + return payload + + +class ReturnImprovementBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + suggestion = "review ops workflow" + if payload.get("reason_cluster") == "size_issue": + suggestion = "improve size chart and fit guidance" + elif payload.get("reason_cluster") == "quality_issue": + suggestion = "inspect supplier quality and packaging" + elif payload.get("reason_cluster") == "delivery_issue": + suggestion = "review carrier SLA and warehouse dispatch" + payload["improvement_suggestion"] = suggestion + payload["mining_summary"] = ( + f"Category {payload.get('product_category')} cluster {payload.get('reason_cluster')}, suggestion {suggestion}." + ) + return payload + + +class ReturnMiningSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/return_reason_mining/pipeline.py b/apps/src/sage/apps/return_reason_mining/pipeline.py new file mode 100644 index 0000000..20499c9 --- /dev/null +++ b/apps/src/sage/apps/return_reason_mining/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ReturnFeatureFusion, + ReturnImprovementBuilder, + ReturnMiningSink, + ReturnReasonClusterer, + ReturnRecordSource, +) + + +def run_return_reason_mining_pipeline(return_file: str, output_file: str) -> None: + env = LocalEnvironment("return_reason_mining") + ( + env.from_batch(ReturnRecordSource, return_file=return_file) + .map(ReturnFeatureFusion) + .map(ReturnReasonClusterer) + .map(ReturnImprovementBuilder) + .sink(ReturnMiningSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/skill_gap_diagnosis/README.md b/apps/src/sage/apps/skill_gap_diagnosis/README.md new file mode 100644 index 0000000..8193760 --- /dev/null +++ b/apps/src/sage/apps/skill_gap_diagnosis/README.md @@ -0,0 +1,6 @@ +# 学员能力缺口诊断系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_skill_gap_diagnosis_pipeline` +- Entry script: `examples/run_skill_gap_diagnosis.py` diff --git a/apps/src/sage/apps/skill_gap_diagnosis/__init__.py b/apps/src/sage/apps/skill_gap_diagnosis/__init__.py new file mode 100644 index 0000000..fd46234 --- /dev/null +++ b/apps/src/sage/apps/skill_gap_diagnosis/__init__.py @@ -0,0 +1,5 @@ +"""学员能力缺口诊断系统 application.""" + +from .pipeline import run_skill_gap_diagnosis_pipeline + +__all__ = ["run_skill_gap_diagnosis_pipeline"] diff --git a/apps/src/sage/apps/skill_gap_diagnosis/operators.py b/apps/src/sage/apps/skill_gap_diagnosis/operators.py new file mode 100644 index 0000000..7a8b357 --- /dev/null +++ b/apps/src/sage/apps/skill_gap_diagnosis/operators.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _as_list(value: Any) -> list[str]: + if isinstance(value, list): + return [str(item).strip() for item in value if str(item).strip()] + return [part.strip() for part in str(value or "").split(",") if part.strip()] + + +class LearningRecordSource(ListBatchSource): + def __init__(self, record_dir: str, **kwargs): + super().__init__(**kwargs) + self.record_dir = record_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.record_dir) + for item in items: + item.setdefault("source_path", self.record_dir) + return items + + +class SkillMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["current_skills"] = _as_list(payload.get("current_skills") or payload.get("skills")) + payload["target_skills"] = _as_list( + payload.get("target_skills") or payload.get("goal_skills") + ) + payload["student_id"] = str( + payload.get("student_id") or payload.get("learner_id") or "unknown" + ) + return payload + + +class SkillGapDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + current = set(payload.get("current_skills") or []) + target = set(payload.get("target_skills") or []) + missing = sorted(target - current) + payload["missing_skills"] = missing + payload["gap_level"] = "high" if len(missing) >= 3 else "medium" if missing else "low" + return payload + + +class PracticePathBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + missing = payload.get("missing_skills") or [] + payload["practice_plan"] = [f"补练 {skill} 模块" for skill in missing[:3]] or [ + "维持当前学习节奏" + ] + payload["group_recommendation"] = ( + "advanced" + if payload.get("gap_level") == "low" + else "support_group" + if payload.get("gap_level") == "high" + else "standard" + ) + return payload + + +class SkillGapSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/skill_gap_diagnosis/pipeline.py b/apps/src/sage/apps/skill_gap_diagnosis/pipeline.py new file mode 100644 index 0000000..996859e --- /dev/null +++ b/apps/src/sage/apps/skill_gap_diagnosis/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + LearningRecordSource, + PracticePathBuilder, + SkillGapDetector, + SkillGapSink, + SkillMapper, +) + + +def run_skill_gap_diagnosis_pipeline(record_dir: str, output_file: str) -> None: + env = LocalEnvironment("skill_gap_diagnosis") + ( + env.from_batch(LearningRecordSource, record_dir=record_dir) + .map(SkillMapper) + .map(SkillGapDetector) + .map(PracticePathBuilder) + .sink(SkillGapSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/solar_alerting/README.md b/apps/src/sage/apps/solar_alerting/README.md new file mode 100644 index 0000000..cac650d --- /dev/null +++ b/apps/src/sage/apps/solar_alerting/README.md @@ -0,0 +1,6 @@ +# 光伏场站告警系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_solar_alerting_pipeline` +- Entry script: `examples/run_solar_alerting.py` diff --git a/apps/src/sage/apps/solar_alerting/__init__.py b/apps/src/sage/apps/solar_alerting/__init__.py new file mode 100644 index 0000000..e579566 --- /dev/null +++ b/apps/src/sage/apps/solar_alerting/__init__.py @@ -0,0 +1,5 @@ +"""光伏场站告警系统 application.""" + +from .pipeline import run_solar_alerting_pipeline + +__all__ = ["run_solar_alerting_pipeline"] diff --git a/apps/src/sage/apps/solar_alerting/operators.py b/apps/src/sage/apps/solar_alerting/operators.py new file mode 100644 index 0000000..712ab29 --- /dev/null +++ b/apps/src/sage/apps/solar_alerting/operators.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class SolarSignalSource(ListBatchSource): + def __init__(self, sensor_file: str, weather_file: str, **kwargs): + super().__init__(**kwargs) + self.sensor_file = sensor_file + self.weather_file = weather_file + + def load_items(self) -> list[dict[str, Any]]: + signals = _load_records(self.sensor_file) + weather = _load_records(self.weather_file) + weather_ref = weather[0] if weather else {} + for item in signals: + item.setdefault("app_slug", "solar_alerting") + item["weather_ref"] = weather_ref + item.setdefault("source_path", self.sensor_file) + return signals + + +class SolarWeatherJoiner(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + weather = payload.get("weather_ref") or {} + payload["power_kw"] = _to_float(payload.get("power_kw")) + payload["expected_irradiance"] = _to_float(weather.get("irradiance_wm2"), 800) + payload["cloud_cover_pct"] = _to_float(weather.get("cloud_cover_pct")) + return payload + + +class SolarAnomalyDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + issues: list[str] = [] + if payload.get("power_kw", 0.0) < 50 and payload.get("expected_irradiance", 0.0) > 700: + issues.append("under_generation") + if str(payload.get("device_status") or "").lower() in {"offline", "fault"}: + issues.append("device_fault") + if payload.get("cloud_cover_pct", 0.0) > 80: + issues.append("weather_limited") + payload["solar_flags"] = issues + return payload + + +class SolarPriorityScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + score = len(payload.get("solar_flags") or []) * 3 + priority = "normal" + if "device_fault" in (payload.get("solar_flags") or []): + score += 3 + if score >= 6: + priority = "high" + elif score >= 3: + priority = "watch" + payload["alert_priority"] = priority + payload["alert_summary"] = ( + f"Site alert priority {priority}, flags {', '.join(payload.get('solar_flags') or []) or 'none'}." + ) + return payload + + +class SolarAlertSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("weather_ref", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/solar_alerting/pipeline.py b/apps/src/sage/apps/solar_alerting/pipeline.py new file mode 100644 index 0000000..5dd1f82 --- /dev/null +++ b/apps/src/sage/apps/solar_alerting/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + SolarAlertSink, + SolarAnomalyDetector, + SolarPriorityScorer, + SolarSignalSource, + SolarWeatherJoiner, +) + + +def run_solar_alerting_pipeline(sensor_file: str, weather_file: str, output_file: str) -> None: + env = LocalEnvironment("solar_alerting") + ( + env.from_batch(SolarSignalSource, sensor_file=sensor_file, weather_file=weather_file) + .map(SolarWeatherJoiner) + .map(SolarAnomalyDetector) + .map(SolarPriorityScorer) + .sink(SolarAlertSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/store_daily_digest/README.md b/apps/src/sage/apps/store_daily_digest/README.md new file mode 100644 index 0000000..59cc639 --- /dev/null +++ b/apps/src/sage/apps/store_daily_digest/README.md @@ -0,0 +1,6 @@ +# 门店运营日报生成系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_store_daily_digest_pipeline` +- Entry script: `examples/run_store_daily_digest.py` diff --git a/apps/src/sage/apps/store_daily_digest/__init__.py b/apps/src/sage/apps/store_daily_digest/__init__.py new file mode 100644 index 0000000..62ad82f --- /dev/null +++ b/apps/src/sage/apps/store_daily_digest/__init__.py @@ -0,0 +1,5 @@ +"""门店运营日报生成系统 application.""" + +from .pipeline import run_store_daily_digest_pipeline + +__all__ = ["run_store_daily_digest_pipeline"] diff --git a/apps/src/sage/apps/store_daily_digest/operators.py b/apps/src/sage/apps/store_daily_digest/operators.py new file mode 100644 index 0000000..7bbd464 --- /dev/null +++ b/apps/src/sage/apps/store_daily_digest/operators.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class StoreOpsSource(ListBatchSource): + def __init__(self, input_dir: str, **kwargs): + super().__init__(**kwargs) + self.input_dir = input_dir + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_dir) + for item in items: + item.setdefault("app_slug", "store_daily_digest") + item.setdefault("source_path", self.input_dir) + return items + + +class StoreMetricAggregator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["sales"] = _to_float(payload.get("sales")) + payload["stockouts"] = int(_to_float(payload.get("stockouts"))) + payload["complaints"] = int(_to_float(payload.get("complaints"))) + payload["shrinkage"] = _to_float(payload.get("shrinkage")) + return payload + + +class StoreExceptionDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + issues: list[str] = [] + if payload.get("stockouts", 0) > 5: + issues.append("high_stockouts") + if payload.get("complaints", 0) > 3: + issues.append("high_customer_complaints") + if payload.get("shrinkage", 0.0) > 1000: + issues.append("high_shrinkage") + payload["daily_exceptions"] = issues + return payload + + +class StoreActionBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + actions: list[str] = [] + if "high_stockouts" in (payload.get("daily_exceptions") or []): + actions.append("replenish_fast_moving_items") + if "high_customer_complaints" in (payload.get("daily_exceptions") or []): + actions.append("review_service_shift_and_queue") + if "high_shrinkage" in (payload.get("daily_exceptions") or []): + actions.append("audit_inventory_and_loss_prevention") + payload["next_day_actions"] = actions or ["maintain_normal_operations"] + payload["digest_summary"] = ( + f"Sales {payload.get('sales')}, exceptions {', '.join(payload.get('daily_exceptions') or []) or 'none'}, " + f"actions {', '.join(payload.get('next_day_actions') or [])}." + ) + return payload + + +class StoreDigestSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/store_daily_digest/pipeline.py b/apps/src/sage/apps/store_daily_digest/pipeline.py new file mode 100644 index 0000000..fdac69c --- /dev/null +++ b/apps/src/sage/apps/store_daily_digest/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + StoreActionBuilder, + StoreDigestSink, + StoreExceptionDetector, + StoreMetricAggregator, + StoreOpsSource, +) + + +def run_store_daily_digest_pipeline(input_dir: str, output_file: str) -> None: + env = LocalEnvironment("store_daily_digest") + ( + env.from_batch(StoreOpsSource, input_dir=input_dir) + .map(StoreMetricAggregator) + .map(StoreExceptionDetector) + .map(StoreActionBuilder) + .sink(StoreDigestSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/subscription_dispatch/README.md b/apps/src/sage/apps/subscription_dispatch/README.md new file mode 100644 index 0000000..7d759e1 --- /dev/null +++ b/apps/src/sage/apps/subscription_dispatch/README.md @@ -0,0 +1,5 @@ +# Subscription Dispatch + +读取发布内容，匹配订阅关系并输出分发目标。 + +输入：内容文件，可选订阅用户文件。 输出：包含匹配用户和分发消息的 JSON 文件。 diff --git a/apps/src/sage/apps/subscription_dispatch/__init__.py b/apps/src/sage/apps/subscription_dispatch/__init__.py new file mode 100644 index 0000000..4a2312e --- /dev/null +++ b/apps/src/sage/apps/subscription_dispatch/__init__.py @@ -0,0 +1,5 @@ +"""Subscription dispatch application.""" + +from .pipeline import run_subscription_dispatch_pipeline + +__all__ = ["run_subscription_dispatch_pipeline"] diff --git a/apps/src/sage/apps/subscription_dispatch/operators.py b/apps/src/sage/apps/subscription_dispatch/operators.py new file mode 100644 index 0000000..f4dd422 --- /dev/null +++ b/apps/src/sage/apps/subscription_dispatch/operators.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class ContentPublishSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class SubscriptionMatcher(MapFunction): + def __init__(self, subscription_file: str | None = None, **kwargs): + super().__init__(**kwargs) + self.subscription_file = subscription_file + self._subscriptions = _load_records(subscription_file) if subscription_file else [] + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + topic = str(item.get("topic") or item.get("category") or "general").lower() + matches = [] + for subscription in self._subscriptions: + subscribed_topic = str( + subscription.get("topic") or subscription.get("interest") or "" + ).lower() + if topic and topic in subscribed_topic or subscribed_topic in topic: + matches.append(subscription) + item["matched_subscriptions"] = matches + return item + + +class PersonalizationFilter(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + headline = str(item.get("title") or item.get("headline") or "") + personalized = [] + for subscription in item.get("matched_subscriptions", []): + personalized.append( + { + "subscriber_id": subscription.get("subscriber_id") + or subscription.get("user_id") + or "unknown", + "channel": subscription.get("channel") or "email", + "message": f"{headline} | match_topic={subscription.get('topic') or subscription.get('interest')}", + } + ) + item["dispatch_targets"] = personalized + return item + + +class DispatchSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/subscription_dispatch/pipeline.py b/apps/src/sage/apps/subscription_dispatch/pipeline.py new file mode 100644 index 0000000..3100525 --- /dev/null +++ b/apps/src/sage/apps/subscription_dispatch/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ContentPublishSource, + DispatchSink, + PersonalizationFilter, + SubscriptionMatcher, +) + + +def run_subscription_dispatch_pipeline( + input_file: str, output_file: str, subscription_file: str | None = None +) -> None: + env = LocalEnvironment("subscription_dispatch") + ( + env.from_batch(ContentPublishSource, input_file=input_file) + .map(SubscriptionMatcher, subscription_file=subscription_file) + .map(PersonalizationFilter) + .sink(DispatchSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/subtitle_qc/README.md b/apps/src/sage/apps/subtitle_qc/README.md new file mode 100644 index 0000000..ef3bde0 --- /dev/null +++ b/apps/src/sage/apps/subtitle_qc/README.md @@ -0,0 +1,6 @@ +# 字幕术语质检系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_subtitle_qc_pipeline` +- Entry script: `examples/run_subtitle_qc.py` diff --git a/apps/src/sage/apps/subtitle_qc/__init__.py b/apps/src/sage/apps/subtitle_qc/__init__.py new file mode 100644 index 0000000..2dce5d9 --- /dev/null +++ b/apps/src/sage/apps/subtitle_qc/__init__.py @@ -0,0 +1,5 @@ +"""字幕术语质检系统 application.""" + +from .pipeline import run_subtitle_qc_pipeline + +__all__ = ["run_subtitle_qc_pipeline"] diff --git a/apps/src/sage/apps/subtitle_qc/operators.py b/apps/src/sage/apps/subtitle_qc/operators.py new file mode 100644 index 0000000..fe65bf0 --- /dev/null +++ b/apps/src/sage/apps/subtitle_qc/operators.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _parse_seconds(value: Any) -> float: + text = str(value or "").strip().replace(",", ".") + if not text: + return 0.0 + parts = text.split(":") + if len(parts) == 3: + hours, minutes, seconds = parts + return int(hours) * 3600 + int(minutes) * 60 + float(seconds) + try: + return float(text) + except ValueError: + return 0.0 + + +class SubtitleSource(ListBatchSource): + def __init__(self, subtitle_file: str, glossary_file: str, **kwargs): + super().__init__(**kwargs) + self.subtitle_file = subtitle_file + self.glossary_file = glossary_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.subtitle_file) + glossary = _load_records(self.glossary_file) + glossary_map = { + str(entry.get("term") or entry.get("source") or "").strip().lower(): str( + entry.get("approved") or entry.get("target") or "" + ).strip() + for entry in glossary + if str(entry.get("term") or entry.get("source") or "").strip() + } + for item in items: + item.setdefault("app_slug", "subtitle_qc") + item["glossary_map"] = glossary_map + item.setdefault("source_path", self.subtitle_file) + return items + + +class SubtitleBlockParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["start_seconds"] = _parse_seconds(payload.get("start") or payload.get("start_time")) + payload["end_seconds"] = _parse_seconds(payload.get("end") or payload.get("end_time")) + text = str(payload.get("text") or payload.get("subtitle") or "") + payload["subtitle_text"] = text.strip() + duration = max(payload["end_seconds"] - payload["start_seconds"], 0.0) + payload["duration_seconds"] = round(duration, 2) + payload["char_count"] = len(payload["subtitle_text"]) + return payload + + +class SubtitleGlossaryChecker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + text_lower = str(payload.get("subtitle_text") or "").lower() + glossary_map = payload.get("glossary_map") or {} + issues: list[str] = [] + matched_terms: list[str] = [] + for source_term, approved in glossary_map.items(): + if source_term and source_term in text_lower: + matched_terms.append(source_term) + if approved and approved.lower() not in text_lower: + issues.append(f"missing_approved_term:{source_term}->{approved}") + payload["matched_glossary_terms"] = matched_terms + payload["glossary_issues"] = issues + return payload + + +class SubtitleTimingChecker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + issues = list(payload.get("glossary_issues") or []) + duration = float(payload.get("duration_seconds") or 0.0) + char_count = int(payload.get("char_count") or 0) + cps = round(char_count / duration, 2) if duration > 0 else 999.0 + if duration <= 0: + issues.append("invalid_time_range") + if cps > 20: + issues.append("reading_speed_too_fast") + payload["reading_speed_cps"] = cps + payload["qc_status"] = "fail" if issues else "pass" + payload["qc_summary"] = ( + f"Block {payload.get('start_seconds')}-{payload.get('end_seconds')} status {payload.get('qc_status')}, " + f"issues {', '.join(issues) or 'none'}." + ) + return payload + + +class SubtitleQCSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + payload = dict(item) + payload.pop("glossary_map", None) + self.items.append(payload) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/subtitle_qc/pipeline.py b/apps/src/sage/apps/subtitle_qc/pipeline.py new file mode 100644 index 0000000..e99cad9 --- /dev/null +++ b/apps/src/sage/apps/subtitle_qc/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + SubtitleBlockParser, + SubtitleGlossaryChecker, + SubtitleQCSink, + SubtitleSource, + SubtitleTimingChecker, +) + + +def run_subtitle_qc_pipeline(subtitle_file: str, glossary_file: str, output_file: str) -> None: + env = LocalEnvironment("subtitle_qc") + ( + env.from_batch(SubtitleSource, subtitle_file=subtitle_file, glossary_file=glossary_file) + .map(SubtitleBlockParser) + .map(SubtitleGlossaryChecker) + .map(SubtitleTimingChecker) + .sink(SubtitleQCSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/supply_chain_tracker/README.md b/apps/src/sage/apps/supply_chain_tracker/README.md new file mode 100644 index 0000000..737b88b --- /dev/null +++ b/apps/src/sage/apps/supply_chain_tracker/README.md @@ -0,0 +1,5 @@ +# Supply Chain Tracker + +读取供应链状态更新，构建时间线并评估延误风险。 + +输入：CSV 或 JSON 状态记录。 输出：包含时间线和风险等级的 JSON 文件。 diff --git a/apps/src/sage/apps/supply_chain_tracker/__init__.py b/apps/src/sage/apps/supply_chain_tracker/__init__.py new file mode 100644 index 0000000..1ca89e5 --- /dev/null +++ b/apps/src/sage/apps/supply_chain_tracker/__init__.py @@ -0,0 +1,5 @@ +"""Supply chain tracker application.""" + +from .pipeline import run_supply_chain_tracker_pipeline + +__all__ = ["run_supply_chain_tracker_pipeline"] diff --git a/apps/src/sage/apps/supply_chain_tracker/operators.py b/apps/src/sage/apps/supply_chain_tracker/operators.py new file mode 100644 index 0000000..e2307ad --- /dev/null +++ b/apps/src/sage/apps/supply_chain_tracker/operators.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class SupplyStatusSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class StatusNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + raw_status = str(item.get("status") or item.get("supply_status") or "pending").lower() + mapping = { + "in transit": "in_transit", + "delayed": "delayed", + "arrived": "arrived", + "pending": "pending", + } + item["normalized_status"] = mapping.get(raw_status, raw_status.replace(" ", "_")) + item["delay_days"] = int(float(item.get("delay_days") or 0)) + return item + + +class TimelineBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["timeline"] = [ + {"stage": "order_created", "time": item.get("order_date") or "unknown"}, + { + "stage": item.get("normalized_status"), + "time": item.get("status_time") or item.get("update_time") or "unknown", + }, + ] + return item + + +class DelayRiskDetector(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = item.get("delay_days", 0) + if item.get("normalized_status") == "delayed": + score += 2 + item["delay_risk_level"] = "high" if score >= 5 else "medium" if score >= 2 else "low" + return item + + +class TrackingSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/supply_chain_tracker/pipeline.py b/apps/src/sage/apps/supply_chain_tracker/pipeline.py new file mode 100644 index 0000000..da934eb --- /dev/null +++ b/apps/src/sage/apps/supply_chain_tracker/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + DelayRiskDetector, + StatusNormalizer, + SupplyStatusSource, + TimelineBuilder, + TrackingSink, +) + + +def run_supply_chain_tracker_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("supply_chain_tracker") + ( + env.from_batch(SupplyStatusSource, input_file=input_file) + .map(StatusNormalizer) + .map(TimelineBuilder) + .map(DelayRiskDetector) + .sink(TrackingSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/ticket_router/README.md b/apps/src/sage/apps/ticket_router/README.md new file mode 100644 index 0000000..47336e8 --- /dev/null +++ b/apps/src/sage/apps/ticket_router/README.md @@ -0,0 +1,10 @@ +# Ticket Router + +客服工单自动路由应用。 + +## 功能 + +- 读取工单数据 +- 分类工单类别 +- 计算优先级 +- 按负载分配客服 diff --git a/apps/src/sage/apps/ticket_router/__init__.py b/apps/src/sage/apps/ticket_router/__init__.py new file mode 100644 index 0000000..f7c16bb --- /dev/null +++ b/apps/src/sage/apps/ticket_router/__init__.py @@ -0,0 +1,5 @@ +"""Ticket router application.""" + +from .pipeline import run_ticket_router_pipeline + +__all__ = ["run_ticket_router_pipeline"] diff --git a/apps/src/sage/apps/ticket_router/operators.py b/apps/src/sage/apps/ticket_router/operators.py new file mode 100644 index 0000000..d2f3925 --- /dev/null +++ b/apps/src/sage/apps/ticket_router/operators.py @@ -0,0 +1,86 @@ +"""Operators for routing service tickets.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class TicketSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class TicketParser(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["subject"] = str(item.get("subject", "")).strip() + item["description"] = str(item.get("description", "")).strip() + item["requester"] = str(item.get("requester", "unknown")).strip() + return item + + +class TicketClassifier(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = f"{item.get('subject', '')} {item.get('description', '')}".lower() + if any(keyword in text for keyword in ["refund", "invoice", "billing", "浠樻"]): + item["category"] = "billing" + elif any(keyword in text for keyword in ["bug", "error", "failure", "鏁呴殰"]): + item["category"] = "technical" + elif any(keyword in text for keyword in ["complaint", "angry", "鎶曡瘔"]): + item["category"] = "complaint" + else: + item["category"] = "general" + return item + + +class PriorityScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + text = f"{item.get('subject', '')} {item.get('description', '')}".lower() + score = 1 + if item.get("category") in {"complaint", "technical"}: + score += 1 + if any(keyword in text for keyword in ["urgent", "asap", "critical", "涓ラ噸"]): + score += 2 + item["priority_score"] = score + item["priority"] = "high" if score >= 4 else "medium" if score >= 2 else "low" + return item + + +class LoadBalancer(MapFunction): + def __init__(self, agents: list[str] | None = None, **kwargs): + super().__init__(**kwargs) + self.agents = agents or ["agent_a", "agent_b", "agent_c"] + self.load = dict.fromkeys(self.agents, 0) + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + assignee = min(self.load, key=self.load.get) + self.load[assignee] += 1 + int(item.get("priority_score", 1)) + item["assignee"] = assignee + return item + + +class NotificationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/ticket_router/pipeline.py b/apps/src/sage/apps/ticket_router/pipeline.py new file mode 100644 index 0000000..7331759 --- /dev/null +++ b/apps/src/sage/apps/ticket_router/pipeline.py @@ -0,0 +1,29 @@ +"""Ticket router pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + LoadBalancer, + NotificationSink, + PriorityScorer, + TicketClassifier, + TicketParser, + TicketSource, +) + + +def run_ticket_router_pipeline( + input_file: str, output_file: str, agents: list[str] | None = None +) -> None: + env = LocalEnvironment("ticket_router") + ( + env.from_batch(TicketSource, input_file=input_file) + .map(TicketParser) + .map(TicketClassifier) + .map(PriorityScorer) + .map(LoadBalancer, agents=agents) + .sink(NotificationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/traffic_briefing/README.md b/apps/src/sage/apps/traffic_briefing/README.md new file mode 100644 index 0000000..33e2e12 --- /dev/null +++ b/apps/src/sage/apps/traffic_briefing/README.md @@ -0,0 +1,6 @@ +# 交通突发事件简报系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_traffic_briefing_pipeline` +- Entry script: `examples/run_traffic_briefing.py` diff --git a/apps/src/sage/apps/traffic_briefing/__init__.py b/apps/src/sage/apps/traffic_briefing/__init__.py new file mode 100644 index 0000000..9ed4381 --- /dev/null +++ b/apps/src/sage/apps/traffic_briefing/__init__.py @@ -0,0 +1,5 @@ +"""交通突发事件简报系统 application.""" + +from .pipeline import run_traffic_briefing_pipeline + +__all__ = ["run_traffic_briefing_pipeline"] diff --git a/apps/src/sage/apps/traffic_briefing/operators.py b/apps/src/sage/apps/traffic_briefing/operators.py new file mode 100644 index 0000000..1aa3e47 --- /dev/null +++ b/apps/src/sage/apps/traffic_briefing/operators.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class TrafficEventSource(ListBatchSource): + def __init__(self, event_file: str, **kwargs): + super().__init__(**kwargs) + self.event_file = event_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.event_file) + for item in items: + item.setdefault("app_slug", "traffic_briefing") + item.setdefault("source_path", self.event_file) + return items + + +class TrafficEventMerger(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["event_type"] = str(payload.get("event_type") or payload.get("type") or "incident") + payload["location"] = str(payload.get("location") or payload.get("road") or "unknown") + payload["lanes_blocked"] = int(_to_float(payload.get("lanes_blocked"))) + payload["weather_impact"] = str(payload.get("weather_impact") or "none") + return payload + + +class TrafficImpactScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + score = 0.0 + if payload.get("event_type") in {"accident", "collision"}: + score += 4 + score += int(payload.get("lanes_blocked") or 0) * 2 + if payload.get("weather_impact") not in {"none", "clear", ""}: + score += 2 + payload["impact_score"] = score + payload["dispatch_priority"] = ( + "high" if score >= 6 else "medium" if score >= 3 else "normal" + ) + return payload + + +class TrafficBriefFormatter(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["brief_summary"] = ( + f"{payload.get('event_type')} at {payload.get('location')}, lanes blocked {payload.get('lanes_blocked')}, " + f"priority {payload.get('dispatch_priority')}." + ) + return payload + + +class TrafficBriefSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/traffic_briefing/pipeline.py b/apps/src/sage/apps/traffic_briefing/pipeline.py new file mode 100644 index 0000000..b6067d7 --- /dev/null +++ b/apps/src/sage/apps/traffic_briefing/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + TrafficBriefFormatter, + TrafficBriefSink, + TrafficEventMerger, + TrafficEventSource, + TrafficImpactScorer, +) + + +def run_traffic_briefing_pipeline(event_file: str, output_file: str) -> None: + env = LocalEnvironment("traffic_briefing") + ( + env.from_batch(TrafficEventSource, event_file=event_file) + .map(TrafficEventMerger) + .map(TrafficImpactScorer) + .map(TrafficBriefFormatter) + .sink(TrafficBriefSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/triage_structurer/README.md b/apps/src/sage/apps/triage_structurer/README.md new file mode 100644 index 0000000..f0648c0 --- /dev/null +++ b/apps/src/sage/apps/triage_structurer/README.md @@ -0,0 +1,6 @@ +# 门急诊分诊整理系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_triage_structurer_pipeline` +- Entry script: `examples/run_triage_structurer.py` diff --git a/apps/src/sage/apps/triage_structurer/__init__.py b/apps/src/sage/apps/triage_structurer/__init__.py new file mode 100644 index 0000000..625e870 --- /dev/null +++ b/apps/src/sage/apps/triage_structurer/__init__.py @@ -0,0 +1,5 @@ +"""门急诊分诊整理系统 application.""" + +from .pipeline import run_triage_structurer_pipeline + +__all__ = ["run_triage_structurer_pipeline"] diff --git a/apps/src/sage/apps/triage_structurer/operators.py b/apps/src/sage/apps/triage_structurer/operators.py new file mode 100644 index 0000000..4f05769 --- /dev/null +++ b/apps/src/sage/apps/triage_structurer/operators.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import csv +import json +import re +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + lines = [ + line.strip() for line in path.read_text(encoding="utf-8-sig").splitlines() if line.strip() + ] + return [ + {"text": line, "line_number": index + 1, "source_path": str(path)} + for index, line in enumerate(lines) + ] + + +def _parse_float(value: Any) -> float | None: + try: + return float(value) + except (TypeError, ValueError): + return None + + +class TriageRecordSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.input_file) + for item in items: + item.setdefault("app_slug", "triage_structurer") + item.setdefault("source_path", self.input_file) + return items + + +class TriageFieldExtractor(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + note = str( + payload.get("note") or payload.get("chief_complaint") or payload.get("text") or "" + ) + payload["chief_complaint"] = str(payload.get("chief_complaint") or note) + payload["temperature_c"] = _parse_float(payload.get("temperature_c")) + payload["heart_rate"] = _parse_float(payload.get("heart_rate")) + payload["oxygen_saturation"] = _parse_float(payload.get("oxygen_saturation")) + payload["systolic_bp"] = _parse_float(payload.get("systolic_bp")) + if payload["temperature_c"] is None: + match = re.search(r"temp(?:erature)?[^\d-]*(-?\d+(?:\.\d+)?)", note, re.I) + payload["temperature_c"] = _parse_float(match.group(1)) if match else None + return payload + + +class TriagePriorityAssigner(MapFunction): + CRITICAL_TERMS = {"chest pain", "stroke", "unconscious", "bleeding", "seizure"} + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + note = str(payload.get("chief_complaint") or "").lower() + priority = "standard" + reasons: list[str] = [] + if ( + payload.get("oxygen_saturation") is not None + and float(payload["oxygen_saturation"]) < 90 + ): + priority = "emergent" + reasons.append("low_oxygen") + if payload.get("systolic_bp") is not None and float(payload["systolic_bp"]) < 90: + priority = "emergent" + reasons.append("low_blood_pressure") + if any(term in note for term in self.CRITICAL_TERMS): + priority = "emergent" + reasons.append("critical_complaint") + elif priority != "emergent": + if (payload.get("temperature_c") or 0) >= 39 or (payload.get("heart_rate") or 0) >= 120: + priority = "urgent" + reasons.append("unstable_vitals") + payload["triage_priority"] = priority + payload["triage_reasons"] = reasons or ["routine_intake"] + return payload + + +class TriageSummaryBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + complaint = str(payload.get("chief_complaint") or "") + department = "general_medicine" + if "chest" in complaint.lower(): + department = "emergency_cardiology" + elif "breath" in complaint.lower() or "cough" in complaint.lower(): + department = "respiratory" + payload["recommended_department"] = department + payload["triage_summary"] = ( + f"主诉: {complaint}; 优先级: {payload.get('triage_priority')}; 建议科室: {department}." + ) + return payload + + +class TriageSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/triage_structurer/pipeline.py b/apps/src/sage/apps/triage_structurer/pipeline.py new file mode 100644 index 0000000..b596fb8 --- /dev/null +++ b/apps/src/sage/apps/triage_structurer/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + TriageFieldExtractor, + TriagePriorityAssigner, + TriageRecordSource, + TriageSink, + TriageSummaryBuilder, +) + + +def run_triage_structurer_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("triage_structurer") + ( + env.from_batch(TriageRecordSource, input_file=input_file) + .map(TriageFieldExtractor) + .map(TriagePriorityAssigner) + .map(TriageSummaryBuilder) + .sink(TriageSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/urban_repair_scheduler/README.md b/apps/src/sage/apps/urban_repair_scheduler/README.md new file mode 100644 index 0000000..96a0273 --- /dev/null +++ b/apps/src/sage/apps/urban_repair_scheduler/README.md @@ -0,0 +1,6 @@ +# 城市设施维修调度系统 + +对应的可运行 SAGE 示例应用。 + +- Pipeline export: `run_urban_repair_scheduler_pipeline` +- Entry script: `examples/run_urban_repair_scheduler.py` diff --git a/apps/src/sage/apps/urban_repair_scheduler/__init__.py b/apps/src/sage/apps/urban_repair_scheduler/__init__.py new file mode 100644 index 0000000..7bc97dd --- /dev/null +++ b/apps/src/sage/apps/urban_repair_scheduler/__init__.py @@ -0,0 +1,5 @@ +"""城市设施维修调度系统 application.""" + +from .pipeline import run_urban_repair_scheduler_pipeline + +__all__ = ["run_urban_repair_scheduler_pipeline"] diff --git a/apps/src/sage/apps/urban_repair_scheduler/operators.py b/apps/src/sage/apps/urban_repair_scheduler/operators.py new file mode 100644 index 0000000..bc5d372 --- /dev/null +++ b/apps/src/sage/apps/urban_repair_scheduler/operators.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(path_str: str) -> list[dict[str, Any]]: + path = Path(path_str) + if not path.exists(): + return [{"source_path": path_str, "text": path.name or path_str}] + if path.is_dir(): + items: list[dict[str, Any]] = [] + for child in sorted(path.iterdir()): + if child.is_file(): + items.extend(_load_records(str(child))) + return items + suffix = path.suffix.lower() + if suffix == ".json": + data = json.loads(path.read_text(encoding="utf-8-sig")) + if isinstance(data, list): + return [item if isinstance(item, dict) else {"value": item} for item in data] + if isinstance(data, dict): + return [data] + if suffix in {".csv", ".tsv"}: + delimiter = "\t" if suffix == ".tsv" else "," + with path.open("r", encoding="utf-8-sig", newline="") as handle: + return list(csv.DictReader(handle, delimiter=delimiter)) + return [{"text": path.read_text(encoding="utf-8-sig"), "source_path": str(path)}] + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + return float(value) + except (TypeError, ValueError): + return default + + +class RepairTicketSource(ListBatchSource): + def __init__(self, ticket_file: str, **kwargs): + super().__init__(**kwargs) + self.ticket_file = ticket_file + + def load_items(self) -> list[dict[str, Any]]: + items = _load_records(self.ticket_file) + for item in items: + item.setdefault("app_slug", "urban_repair_scheduler") + item.setdefault("source_path", self.ticket_file) + return items + + +class RepairGeoMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + payload["district"] = str(payload.get("district") or payload.get("zone") or "unknown") + payload["asset_type"] = str( + payload.get("asset_type") or payload.get("category") or "infrastructure" + ) + payload["street"] = str(payload.get("street") or payload.get("location") or "unknown") + return payload + + +class RepairPriorityScorer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + severity = _to_float(payload.get("severity"), 1.0) + public_safety = str(payload.get("public_safety_risk") or "false").lower() in { + "true", + "1", + "yes", + } + score = severity * 2 + (3 if public_safety else 0) + if payload.get("asset_type") in {"road", "manhole", "streetlight"}: + score += 1 + payload["priority_score"] = score + payload["dispatch_priority"] = ( + "high" if score >= 6 else "medium" if score >= 3 else "normal" + ) + return payload + + +class RepairRoutePlanner(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + payload = dict(item) + district = payload.get("district") + street = payload.get("street") + asset = payload.get("asset_type") + payload["crew_type"] = f"{asset}_crew" + payload["route_bucket"] = f"{district}:{street}" + payload["schedule_summary"] = ( + f"Dispatch {payload.get('crew_type')} to {payload.get('route_bucket')} with priority {payload.get('dispatch_priority')}." + ) + return payload + + +class RepairScheduleSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_target = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(dict(item)) + target = Path(self.output_target) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8") diff --git a/apps/src/sage/apps/urban_repair_scheduler/pipeline.py b/apps/src/sage/apps/urban_repair_scheduler/pipeline.py new file mode 100644 index 0000000..774dc36 --- /dev/null +++ b/apps/src/sage/apps/urban_repair_scheduler/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + RepairGeoMapper, + RepairPriorityScorer, + RepairRoutePlanner, + RepairScheduleSink, + RepairTicketSource, +) + + +def run_urban_repair_scheduler_pipeline(ticket_file: str, output_file: str) -> None: + env = LocalEnvironment("urban_repair_scheduler") + ( + env.from_batch(RepairTicketSource, ticket_file=ticket_file) + .map(RepairGeoMapper) + .map(RepairPriorityScorer) + .map(RepairRoutePlanner) + .sink(RepairScheduleSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/user_behavior_analytics/README.md b/apps/src/sage/apps/user_behavior_analytics/README.md new file mode 100644 index 0000000..0a1b113 --- /dev/null +++ b/apps/src/sage/apps/user_behavior_analytics/README.md @@ -0,0 +1,3 @@ +# User Behavior Analytics + +用户行为事件采集与分析应用。 diff --git a/apps/src/sage/apps/user_behavior_analytics/__init__.py b/apps/src/sage/apps/user_behavior_analytics/__init__.py new file mode 100644 index 0000000..bc792b3 --- /dev/null +++ b/apps/src/sage/apps/user_behavior_analytics/__init__.py @@ -0,0 +1,5 @@ +"""User behavior analytics application.""" + +from .pipeline import run_user_behavior_analytics_pipeline + +__all__ = ["run_user_behavior_analytics_pipeline"] diff --git a/apps/src/sage/apps/user_behavior_analytics/operators.py b/apps/src/sage/apps/user_behavior_analytics/operators.py new file mode 100644 index 0000000..e656bf2 --- /dev/null +++ b/apps/src/sage/apps/user_behavior_analytics/operators.py @@ -0,0 +1,69 @@ +"""Operators for user behavior analytics.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class EventSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class EventNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["user_id"] = str(item.get("user_id", "unknown")).strip() + item["event_type"] = str(item.get("event_type", "unknown")).strip().lower() + item["value"] = float(item.get("value") or 1) + return item + + +class UserAggregator(MapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.state: dict[str, dict[str, Any]] = {} + + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + user_id = item.get("user_id", "unknown") + profile = self.state.setdefault( + user_id, + {"user_id": user_id, "event_count": 0, "total_value": 0.0, "event_types": set()}, + ) + profile["event_count"] += 1 + profile["total_value"] += float(item.get("value", 0)) + profile["event_types"].add(item.get("event_type", "unknown")) + return { + "user_id": user_id, + "event_count": profile["event_count"], + "total_value": round(profile["total_value"], 2), + "event_types": sorted(profile["event_types"]), + } + + +class AnalyticsSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.latest_by_user: dict[str, dict[str, Any]] = {} + + def execute(self, item: dict[str, Any]) -> None: + self.latest_by_user[item["user_id"]] = item + + def teardown(self, context: Any) -> None: + payload = sorted(self.latest_by_user.values(), key=lambda value: value["user_id"]) + Path(self.output_file).write_text( + json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/user_behavior_analytics/pipeline.py b/apps/src/sage/apps/user_behavior_analytics/pipeline.py new file mode 100644 index 0000000..66dc5b1 --- /dev/null +++ b/apps/src/sage/apps/user_behavior_analytics/pipeline.py @@ -0,0 +1,18 @@ +"""User behavior analytics pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import AnalyticsSink, EventNormalizer, EventSource, UserAggregator + + +def run_user_behavior_analytics_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("user_behavior_analytics") + ( + env.from_batch(EventSource, input_file=input_file) + .map(EventNormalizer) + .map(UserAggregator) + .sink(AnalyticsSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/vendor_evaluation_standardizer/README.md b/apps/src/sage/apps/vendor_evaluation_standardizer/README.md new file mode 100644 index 0000000..197c6d4 --- /dev/null +++ b/apps/src/sage/apps/vendor_evaluation_standardizer/README.md @@ -0,0 +1,5 @@ +# Vendor Evaluation Standardizer + +读取供应商评估数据，统一字段、标准化评分并标记风险等级。 + +输入：CSV 或 JSON 评估记录。 输出：包含标准分、评估等级和风险标记的 JSON 文件。 diff --git a/apps/src/sage/apps/vendor_evaluation_standardizer/__init__.py b/apps/src/sage/apps/vendor_evaluation_standardizer/__init__.py new file mode 100644 index 0000000..7b57561 --- /dev/null +++ b/apps/src/sage/apps/vendor_evaluation_standardizer/__init__.py @@ -0,0 +1,5 @@ +"""Vendor evaluation standardizer application.""" + +from .pipeline import run_vendor_evaluation_standardizer_pipeline + +__all__ = ["run_vendor_evaluation_standardizer_pipeline"] diff --git a/apps/src/sage/apps/vendor_evaluation_standardizer/operators.py b/apps/src/sage/apps/vendor_evaluation_standardizer/operators.py new file mode 100644 index 0000000..223babb --- /dev/null +++ b/apps/src/sage/apps/vendor_evaluation_standardizer/operators.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +def _load_records(input_file: str) -> list[dict[str, Any]]: + with open(input_file, encoding="utf-8", newline="") as handle: + if input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class VendorEvaluationSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + return _load_records(self.input_file) + + +class EvaluationFieldMapper(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["vendor_id"] = ( + item.get("vendor_id") or item.get("supplier_id") or item.get("id") or "unknown" + ) + item["vendor_name"] = ( + item.get("vendor_name") or item.get("supplier_name") or item.get("name") or "unknown" + ) + item["raw_score"] = item.get("raw_score") or item.get("score") or item.get("rating") or 0 + item["evaluation_comment"] = ( + item.get("evaluation_comment") or item.get("comment") or item.get("remark") or "" + ) + return item + + +class EvaluationNormalizer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = float(item.get("raw_score") or 0) + normalized_score = max(0.0, min(score, 100.0)) + item["normalized_score"] = round(normalized_score, 2) + item["evaluation_level"] = ( + "excellent" + if normalized_score >= 85 + else "watch" + if normalized_score < 60 + else "stable" + ) + return item + + +class VendorRiskMarker(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + comment = str(item.get("evaluation_comment", "")).lower() + risk_terms = ("delay", "penalty", "quality", "complaint", "breach") + risk_hits = sum(1 for term in risk_terms if term in comment) + if item.get("normalized_score", 0) < 60: + risk_hits += 1 + item["risk_marker"] = "high" if risk_hits >= 2 else "medium" if risk_hits == 1 else "low" + return item + + +class VendorEvaluationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) diff --git a/apps/src/sage/apps/vendor_evaluation_standardizer/pipeline.py b/apps/src/sage/apps/vendor_evaluation_standardizer/pipeline.py new file mode 100644 index 0000000..8aeb751 --- /dev/null +++ b/apps/src/sage/apps/vendor_evaluation_standardizer/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + EvaluationFieldMapper, + EvaluationNormalizer, + VendorEvaluationSink, + VendorEvaluationSource, + VendorRiskMarker, +) + + +def run_vendor_evaluation_standardizer_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("vendor_evaluation_standardizer") + ( + env.from_batch(VendorEvaluationSource, input_file=input_file) + .map(EvaluationFieldMapper) + .map(EvaluationNormalizer) + .map(VendorRiskMarker) + .sink(VendorEvaluationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/voucher_classifier/README.md b/apps/src/sage/apps/voucher_classifier/README.md new file mode 100644 index 0000000..5e4fb90 --- /dev/null +++ b/apps/src/sage/apps/voucher_classifier/README.md @@ -0,0 +1,18 @@ +# Voucher Classifier + +财务凭证自动分类应用。 + +## 功能 + +- 读取 CSV 或文本凭证记录 +- 提取 OCR 文本中的金额、日期和类别线索 +- 基于规则进行费用分类 +- 输出 JSON 分类结果 + +## 用法 + +```bash +python examples/run_voucher_classifier.py \ + --input-file vouchers.csv \ + --output voucher_results.json +``` diff --git a/apps/src/sage/apps/voucher_classifier/__init__.py b/apps/src/sage/apps/voucher_classifier/__init__.py new file mode 100644 index 0000000..8cc643e --- /dev/null +++ b/apps/src/sage/apps/voucher_classifier/__init__.py @@ -0,0 +1,5 @@ +"""Voucher classification application.""" + +from .pipeline import run_voucher_classifier_pipeline + +__all__ = ["run_voucher_classifier_pipeline"] diff --git a/apps/src/sage/apps/voucher_classifier/operators.py b/apps/src/sage/apps/voucher_classifier/operators.py new file mode 100644 index 0000000..3644929 --- /dev/null +++ b/apps/src/sage/apps/voucher_classifier/operators.py @@ -0,0 +1,71 @@ +"""Operators for voucher classification.""" + +from __future__ import annotations + +import csv +import json +import re +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class VoucherSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".csv"): + return list(csv.DictReader(handle)) + return [{"raw_text": line.strip()} for line in handle if line.strip()] + + +class OcrExtractor(MapFunction): + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + data["ocr_text"] = data.get("ocr_text") or data.get("raw_text") or "" + return data + + +class FieldExtractor(MapFunction): + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + text = data.get("ocr_text", "") + amount_match = re.search( + r"(?:amount|金额)\s*[:：]?\s*([0-9]+(?:\.[0-9]{1,2})?)", text, re.IGNORECASE + ) + date_match = re.search(r"\b(20\d{2}[-/]\d{1,2}[-/]\d{1,2})\b", text) + category_hint = re.search(r"(travel|hotel|meal|office|taxi|fuel)", text, re.IGNORECASE) + data["amount"] = float(amount_match.group(1)) if amount_match else 0.0 + data["voucher_date"] = date_match.group(1) if date_match else "" + data["category_hint"] = category_hint.group(1) if category_hint else "" + return data + + +class RuleClassifier(MapFunction): + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + hint = str(data.get("category_hint", "")).lower() + if hint in {"travel", "hotel", "taxi", "fuel"}: + data["voucher_type"] = "travel_expense" + elif hint in {"meal"}: + data["voucher_type"] = "meal_expense" + elif hint in {"office"}: + data["voucher_type"] = "office_expense" + else: + data["voucher_type"] = "general_expense" + return data + + +class ClassificationSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, data: dict[str, Any]) -> None: + self.items.append(data) + + def teardown(self, context: Any) -> None: + with open(self.output_file, "w", encoding="utf-8") as handle: + json.dump(self.items, handle, ensure_ascii=False, indent=2) diff --git a/apps/src/sage/apps/voucher_classifier/pipeline.py b/apps/src/sage/apps/voucher_classifier/pipeline.py new file mode 100644 index 0000000..0eb6463 --- /dev/null +++ b/apps/src/sage/apps/voucher_classifier/pipeline.py @@ -0,0 +1,25 @@ +"""Voucher classifier pipeline.""" + +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + ClassificationSink, + FieldExtractor, + OcrExtractor, + RuleClassifier, + VoucherSource, +) + + +def run_voucher_classifier_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("voucher_classifier") + ( + env.from_batch(VoucherSource, input_file=input_file) + .map(OcrExtractor) + .map(FieldExtractor) + .map(RuleClassifier) + .sink(ClassificationSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/warehouse_slot_optimizer/README.md b/apps/src/sage/apps/warehouse_slot_optimizer/README.md new file mode 100644 index 0000000..8f1c51a --- /dev/null +++ b/apps/src/sage/apps/warehouse_slot_optimizer/README.md @@ -0,0 +1,3 @@ +# Warehouse Slot Optimizer + +仓库货位优化应用。 diff --git a/apps/src/sage/apps/warehouse_slot_optimizer/__init__.py b/apps/src/sage/apps/warehouse_slot_optimizer/__init__.py new file mode 100644 index 0000000..9d50dd5 --- /dev/null +++ b/apps/src/sage/apps/warehouse_slot_optimizer/__init__.py @@ -0,0 +1,5 @@ +"""Warehouse slot optimizer application.""" + +from .pipeline import run_warehouse_slot_optimizer_pipeline + +__all__ = ["run_warehouse_slot_optimizer_pipeline"] diff --git a/apps/src/sage/apps/warehouse_slot_optimizer/operators.py b/apps/src/sage/apps/warehouse_slot_optimizer/operators.py new file mode 100644 index 0000000..b221769 --- /dev/null +++ b/apps/src/sage/apps/warehouse_slot_optimizer/operators.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class SlotSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class PickingHistorySource(SlotSource): + pass + + +class SlotHeatAnalyzer(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + picks = int(float(item.get("pick_count") or 0)) + distance = float(item.get("distance_m") or 0) + item["slot_score"] = round(picks * 0.8 - distance * 0.1, 2) + return item + + +class SlotHeatCalculator(SlotHeatAnalyzer): + pass + + +class DistanceCostBuilder(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + item["distance_cost"] = round(float(item.get("distance_m") or 0) * 0.1, 2) + item["slot_score"] = round(float(item.get("slot_score") or 0) - item["distance_cost"], 2) + return item + + +class SlotAllocator(MapFunction): + def execute(self, item: dict[str, Any]) -> dict[str, Any]: + score = float(item.get("slot_score", 0)) + item["recommended_zone"] = "front" if score >= 50 else "middle" if score >= 20 else "rear" + return item + + +class SlotOptimizer(SlotAllocator): + pass + + +class SlotSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, item: dict[str, Any]) -> None: + self.items.append(item) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + +class SlotPlanSink(SlotSink): + pass diff --git a/apps/src/sage/apps/warehouse_slot_optimizer/pipeline.py b/apps/src/sage/apps/warehouse_slot_optimizer/pipeline.py new file mode 100644 index 0000000..31a0ccd --- /dev/null +++ b/apps/src/sage/apps/warehouse_slot_optimizer/pipeline.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + DistanceCostBuilder, + PickingHistorySource, + SlotHeatCalculator, + SlotOptimizer, + SlotPlanSink, +) + + +def run_warehouse_slot_optimizer_pipeline(input_file: str, output_file: str) -> None: + env = LocalEnvironment("warehouse_slot_optimizer") + ( + env.from_batch(PickingHistorySource, input_file=input_file) + .map(SlotHeatCalculator) + .map(DistanceCostBuilder) + .map(SlotOptimizer) + .sink(SlotPlanSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/weather_sales_forecast/README.md b/apps/src/sage/apps/weather_sales_forecast/README.md new file mode 100644 index 0000000..5a06c47 --- /dev/null +++ b/apps/src/sage/apps/weather_sales_forecast/README.md @@ -0,0 +1,3 @@ +# Weather Sales Forecast + +天气驱动销量预测应用。 diff --git a/apps/src/sage/apps/weather_sales_forecast/__init__.py b/apps/src/sage/apps/weather_sales_forecast/__init__.py new file mode 100644 index 0000000..f86ed8c --- /dev/null +++ b/apps/src/sage/apps/weather_sales_forecast/__init__.py @@ -0,0 +1,5 @@ +"""Weather sales forecast application.""" + +from .pipeline import run_weather_sales_forecast_pipeline + +__all__ = ["run_weather_sales_forecast_pipeline"] diff --git a/apps/src/sage/apps/weather_sales_forecast/operators.py b/apps/src/sage/apps/weather_sales_forecast/operators.py new file mode 100644 index 0000000..b311b71 --- /dev/null +++ b/apps/src/sage/apps/weather_sales_forecast/operators.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import MapFunction, SinkFunction + + +class SalesSource(ListBatchSource): + def __init__(self, input_file: str, **kwargs): + super().__init__(**kwargs) + self.input_file = input_file + + def load_items(self) -> list[dict[str, Any]]: + with open(self.input_file, encoding="utf-8", newline="") as handle: + if self.input_file.lower().endswith(".json"): + return json.load(handle) + return list(csv.DictReader(handle)) + + +class SalesHistorySource(SalesSource): + pass + + +class WeatherJoiner(MapFunction): + def __init__(self, weather_map: dict[str, float] | None = None, **kwargs): + super().__init__(**kwargs) + self.weather_map = weather_map or {"sunny": 1.1, "cloudy": 1.0, "rainy": 0.85, "snowy": 0.7} + + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + weather = str(data.get("weather") or "cloudy").lower() + data["weather_factor"] = self.weather_map.get(weather, 1.0) + return data + + +class WeatherFetcher(WeatherJoiner): + pass + + +class WeatherSalesFeatureBuilder(MapFunction): + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + baseline = float(data.get("baseline_sales") or data.get("sales") or 0) + data["baseline_sales"] = baseline + data["weather_adjusted_baseline"] = round( + baseline * float(data.get("weather_factor", 1.0)), 2 + ) + return data + + +class ForecastCalculator(MapFunction): + def execute(self, data: dict[str, Any]) -> dict[str, Any]: + baseline = float(data.get("baseline_sales") or data.get("sales") or 0) + forecast = baseline * float(data.get("weather_factor", 1.0)) + data["forecast_sales"] = round(forecast, 2) + return data + + +class SalesForecaster(ForecastCalculator): + pass + + +class ForecastSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self.items: list[dict[str, Any]] = [] + + def execute(self, data: dict[str, Any]) -> None: + self.items.append(data) + + def teardown(self, context: Any) -> None: + Path(self.output_file).write_text( + json.dumps(self.items, ensure_ascii=False, indent=2), encoding="utf-8" + ) + + +class RestockSuggestionSink(ForecastSink): + pass diff --git a/apps/src/sage/apps/weather_sales_forecast/pipeline.py b/apps/src/sage/apps/weather_sales_forecast/pipeline.py new file mode 100644 index 0000000..ac45f17 --- /dev/null +++ b/apps/src/sage/apps/weather_sales_forecast/pipeline.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from sage.runtime import LocalEnvironment + +from .operators import ( + RestockSuggestionSink, + SalesForecaster, + SalesHistorySource, + WeatherFetcher, + WeatherSalesFeatureBuilder, +) + + +def run_weather_sales_forecast_pipeline( + sales_file: str, + output_file: str, + city_config: str | None = None, + weather_map: dict[str, float] | None = None, +) -> None: + env = LocalEnvironment("weather_sales_forecast") + ( + env.from_batch(SalesHistorySource, input_file=sales_file) + .map(WeatherFetcher, weather_map=weather_map) + .map(WeatherSalesFeatureBuilder) + .map(SalesForecaster) + .sink(RestockSuggestionSink, output_file=output_file) + ) + env.submit(autostop=True) diff --git a/apps/src/sage/apps/web_scraper/README.md b/apps/src/sage/apps/web_scraper/README.md new file mode 100644 index 0000000..3a9f263 --- /dev/null +++ b/apps/src/sage/apps/web_scraper/README.md @@ -0,0 +1,20 @@ +# Web Scraper + +网页内容抓取与表格抽取应用。 + +## 功能 + +- 从文本文件读取 URL 列表 +- 抓取网页标题与 HTML +- 抽取 HTML table 内容 +- 输出 JSON 数组结果 + +## 用法 + +```bash +python examples/run_web_scraper.py \ + --url-file urls.txt \ + --output scraped_tables.json +``` + +输入文件每行一个 URL。 diff --git a/apps/src/sage/apps/web_scraper/__init__.py b/apps/src/sage/apps/web_scraper/__init__.py new file mode 100644 index 0000000..893c7ec --- /dev/null +++ b/apps/src/sage/apps/web_scraper/__init__.py @@ -0,0 +1,5 @@ +"""Web scraper application.""" + +from .pipeline import run_web_scraper_pipeline + +__all__ = ["run_web_scraper_pipeline"] diff --git a/apps/src/sage/apps/web_scraper/operators.py b/apps/src/sage/apps/web_scraper/operators.py new file mode 100644 index 0000000..434d229 --- /dev/null +++ b/apps/src/sage/apps/web_scraper/operators.py @@ -0,0 +1,101 @@ +""" +Web Scraper Operators + +Lightweight operators for web scraping and table extraction. +""" + +from __future__ import annotations + +import json +from typing import Any + +from sage.apps._batch import ListBatchSource +from sage.foundation import CustomLogger, FlatMapFunction, MapFunction, SinkFunction + + +class UrlSource(ListBatchSource): + def __init__(self, url_file: str, **kwargs): + super().__init__(**kwargs) + self.url_file = url_file + self._source_logger = CustomLogger("UrlSource") + + def load_items(self) -> list[str]: + urls = [] + try: + with open(self.url_file, encoding="utf-8") as f: + for line in f: + if line.strip(): + urls.append(line.strip()) + self.logger.info(f"Read {len(urls)} URLs from {self.url_file}") + except Exception as e: + self.logger.error(f"Error reading URL file: {e}") + return urls + + +class WebScraper(MapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._scraper_logger = CustomLogger("WebScraper") + + def execute(self, data: str) -> dict[str, Any]: + import requests + from bs4 import BeautifulSoup + + try: + resp = requests.get(data, timeout=10) + resp.raise_for_status() + soup = BeautifulSoup(resp.text, "html.parser") + title = soup.title.string if soup.title else "" + return {"url": data, "title": title, "html": resp.text} + except Exception as e: + self.logger.error(f"Failed to fetch {data}: {e}") + return {"url": data, "title": "", "html": ""} + + +class TableExtractor(FlatMapFunction): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._extractor_logger = CustomLogger("TableExtractor") + + def execute(self, data: dict[str, Any]) -> list[dict[str, Any]]: + from bs4 import BeautifulSoup + + html = data.get("html", "") + if not html: + return [] + + soup = BeautifulSoup(html, "html.parser") + tables = [] + for table in soup.find_all("table"): + rows = [] + for tr in table.find_all("tr"): + cols = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])] + if cols: + rows.append(cols) + if rows: + tables.append({"url": data.get("url"), "rows": rows}) + return tables + + +class DatabaseSink(SinkFunction): + def __init__(self, output_file: str, **kwargs): + super().__init__(**kwargs) + self.output_file = output_file + self._sink_logger = CustomLogger("DatabaseSink") + self.count = 0 + + def setup(self, context: Any) -> None: + with open(self.output_file, "w", encoding="utf-8") as f: + f.write("[\n") + + def execute(self, data: dict[str, Any]) -> None: + with open(self.output_file, "a", encoding="utf-8") as f: + if self.count > 0: + f.write(",\n") + json.dump(data, f, ensure_ascii=False) + self.count += 1 + + def teardown(self, context: Any) -> None: + with open(self.output_file, "a", encoding="utf-8") as f: + f.write("\n]") + self.logger.info(f"Written {self.count} tables to {self.output_file}") diff --git a/apps/src/sage/apps/web_scraper/pipeline.py b/apps/src/sage/apps/web_scraper/pipeline.py new file mode 100644 index 0000000..f46ce06 --- /dev/null +++ b/apps/src/sage/apps/web_scraper/pipeline.py @@ -0,0 +1,28 @@ +""" +Web Scraper Pipeline +""" + +from __future__ import annotations + +from sage.foundation import CustomLogger +from sage.runtime import LocalEnvironment + +from .operators import DatabaseSink, TableExtractor, UrlSource, WebScraper + + +def run_web_scraper_pipeline(url_file: str, output_file: str, verbose: bool = False) -> None: + logger = CustomLogger("WebScraperPipeline") + if verbose: + logger.info(f"Starting web scraper: {url_file} -> {output_file}") + + env = LocalEnvironment("web_scraper") + ( + env.from_batch(UrlSource, url_file=url_file) + .map(WebScraper) + .flatmap(TableExtractor) + .sink(DatabaseSink, output_file=output_file) + ) + env.submit(autostop=True) + + if verbose: + logger.info("Web scraper pipeline finished") diff --git a/apps/tests/conftest.py b/apps/tests/conftest.py new file mode 100644 index 0000000..b328a1f --- /dev/null +++ b/apps/tests/conftest.py @@ -0,0 +1,95 @@ +"""Shared test fixtures for locally developed generated apps.""" + +from __future__ import annotations + +import logging +import sys +import types +from pathlib import Path + + +def pytest_configure() -> None: + repo_root = Path(__file__).resolve().parents[1] + apps_src = repo_root / "src" + sage_root = apps_src / "sage" + apps_root = sage_root / "apps" + + for key in list(sys.modules): + if key == "sage" or key.startswith("sage."): + del sys.modules[key] + + sage_pkg = types.ModuleType("sage") + sage_pkg.__path__ = [str(sage_root)] + sys.modules["sage"] = sage_pkg + + foundation = types.ModuleType("sage.foundation") + + class _Base: + def __init__(self, **kwargs): + for key, value in kwargs.items(): + setattr(self, key, value) + + class BatchFunction(_Base): + pass + + class MapFunction(_Base): + pass + + class FlatMapFunction(_Base): + pass + + class SinkFunction(_Base): + pass + + class CustomLogger: + def __init__(self, name: str): + self._logger = logging.getLogger(name) + + def info(self, message: str) -> None: + self._logger.info(message) + + def error(self, message: str) -> None: + self._logger.error(message) + + foundation.BatchFunction = BatchFunction + foundation.MapFunction = MapFunction + foundation.FlatMapFunction = FlatMapFunction + foundation.SinkFunction = SinkFunction + foundation.CustomLogger = CustomLogger + sys.modules["sage.foundation"] = foundation + + runtime = types.ModuleType("sage.runtime") + + class _Pipeline: + def map(self, *args, **kwargs): + return self + + def flat_map(self, *args, **kwargs): + return self + + def filter(self, *args, **kwargs): + return self + + def sink(self, *args, **kwargs): + return self + + class LocalEnvironment: + def __init__(self, name: str): + self.name = name + + def from_batch(self, *args, **kwargs): + return _Pipeline() + + def submit(self, autostop: bool = True): + return None + + runtime.LocalEnvironment = LocalEnvironment + sys.modules["sage.runtime"] = runtime + + apps_pkg = types.ModuleType("sage.apps") + apps_pkg.__path__ = [str(apps_root)] + version_ns: dict[str, object] = {} + exec((apps_root / "_version.py").read_text(encoding="utf-8"), version_ns) + apps_pkg.__version__ = version_ns["__version__"] + sys.modules["sage.apps"] = apps_pkg + sage_pkg.apps = apps_pkg diff --git a/apps/tests/generated/test_generated_apps.py b/apps/tests/generated/test_generated_apps.py new file mode 100644 index 0000000..368ab99 --- /dev/null +++ b/apps/tests/generated/test_generated_apps.py @@ -0,0 +1,237 @@ +"""Smoke tests for generated SAGE example apps.""" + +from __future__ import annotations + +import importlib +import json +from pathlib import Path + +APP_PIPELINES = { + "log_parser": "run_log_parser_pipeline", + "data_cleaner": "run_data_cleaner_pipeline", + "resume_parser": "run_resume_parser_pipeline", + "feedback_analyzer": "run_feedback_analyzer_pipeline", + "web_scraper": "run_web_scraper_pipeline", + "academic_metadata": "run_academic_metadata_pipeline", + "customer_deduplication": "run_customer_deduplication_pipeline", + "voucher_classifier": "run_voucher_classifier_pipeline", + "product_sync": "run_product_sync_pipeline", + "doc_classifier": "run_doc_classifier_pipeline", + "contract_matcher": "run_contract_matcher_pipeline", + "news_aggregator": "run_news_aggregator_pipeline", + "ticket_router": "run_ticket_router_pipeline", + "content_moderation": "run_content_moderation_pipeline", + "contract_risk": "run_contract_risk_pipeline", + "user_behavior_analytics": "run_user_behavior_analytics_pipeline", + "inventory_alert": "run_inventory_alert_pipeline", + "quality_defect_filter": "run_quality_defect_filter_pipeline", + "permission_audit": "run_permission_audit_pipeline", + "order_anomaly_detector": "run_order_anomaly_detector_pipeline", + "attendance_alert": "run_attendance_alert_pipeline", + "lead_scoring": "run_lead_scoring_pipeline", + "arbitrage_detector": "run_arbitrage_detector_pipeline", + "weather_sales_forecast": "run_weather_sales_forecast_pipeline", + "geo_recommendation": "run_geo_recommendation_pipeline", + "company_credit": "run_company_credit_pipeline", + "movie_scheduling_optimizer": "run_movie_scheduling_optimizer_pipeline", + "real_estate_valuation": "run_real_estate_valuation_pipeline", + "multi_factor_credit_score": "run_multi_factor_credit_score_pipeline", + "logistics_cost_optimizer": "run_logistics_cost_optimizer_pipeline", + "medical_registration_optimizer": "run_medical_registration_optimizer_pipeline", + "exhibition_heatmap": "run_exhibition_heatmap_pipeline", + "dorm_energy_optimizer": "run_dorm_energy_optimizer_pipeline", + "restaurant_sales_analysis": "run_restaurant_sales_analysis_pipeline", + "warehouse_slot_optimizer": "run_warehouse_slot_optimizer_pipeline", + "paper_classifier": "run_paper_classifier_pipeline", + "vendor_evaluation_standardizer": "run_vendor_evaluation_standardizer_pipeline", + "content_tagger": "run_content_tagger_pipeline", + "partner_profile_hub": "run_partner_profile_hub_pipeline", + "subscription_dispatch": "run_subscription_dispatch_pipeline", + "contract_versioning": "run_contract_versioning_pipeline", + "invoice_reconciliation": "run_invoice_reconciliation_pipeline", + "project_risk_monitor": "run_project_risk_monitor_pipeline", + "learning_record_hub": "run_learning_record_hub_pipeline", + "supply_chain_tracker": "run_supply_chain_tracker_pipeline", + "compliance_doc_manager": "run_compliance_doc_manager_pipeline", + "mail_classifier": "run_mail_classifier_pipeline", + "meeting_minutes": "run_meeting_minutes_pipeline", + "policy_update_notifier": "run_policy_update_notifier_pipeline", + "export_transformer": "run_export_transformer_pipeline", + "api_log_analytics": "run_api_log_analytics_pipeline", + "backup_sync": "run_backup_sync_pipeline", + "patent_competition_monitor": "run_patent_competition_monitor_pipeline", + "grant_subscription": "run_grant_subscription_pipeline", + "experiment_review": "run_experiment_review_pipeline", + "repro_audit": "run_repro_audit_pipeline", + "benchmark_watch": "run_benchmark_watch_pipeline", + "course_qa_helper": "run_course_qa_helper_pipeline", + "lesson_scheduler": "run_lesson_scheduler_pipeline", + "assignment_feedback": "run_assignment_feedback_pipeline", + "skill_gap_diagnosis": "run_skill_gap_diagnosis_pipeline", + "interview_coach": "run_interview_coach_pipeline", + "campus_aid_gap_alert": "run_campus_aid_gap_alert_pipeline", + "triage_structurer": "run_triage_structurer_pipeline", + "radiology_followup_loop": "run_radiology_followup_loop_pipeline", + "drug_leaflet_extractor": "run_drug_leaflet_extractor_pipeline", + "lab_turnaround_alert": "run_lab_turnaround_alert_pipeline", + "quote_compare": "run_quote_compare_pipeline", + "policy_search_helper": "run_policy_search_helper_pipeline", + "knowledge_cleanup": "run_knowledge_cleanup_pipeline", + "podcast_highlight": "run_podcast_highlight_pipeline", + "brand_compliance_review": "run_brand_compliance_review_pipeline", + "subtitle_qc": "run_subtitle_qc_pipeline", + "content_scheduler": "run_content_scheduler_pipeline", + "media_archive_search": "run_media_archive_search_pipeline", + "factory_watch": "run_factory_watch_pipeline", + "cold_chain_watch": "run_cold_chain_watch_pipeline", + "greenhouse_assistant": "run_greenhouse_assistant_pipeline", + "community_hotspot_drift": "run_community_hotspot_drift_pipeline", + "traffic_briefing": "run_traffic_briefing_pipeline", + "urban_repair_scheduler": "run_urban_repair_scheduler_pipeline", + "permit_material_review": "run_permit_material_review_pipeline", + "municipal_search": "run_municipal_search_pipeline", + "budget_variance_alert": "run_budget_variance_alert_pipeline", + "cashflow_watch": "run_cashflow_watch_pipeline", + "return_reason_mining": "run_return_reason_mining_pipeline", + "store_daily_digest": "run_store_daily_digest_pipeline", + "carbon_collection": "run_carbon_collection_pipeline", + "solar_alerting": "run_solar_alerting_pipeline", + "campus_emission_report": "run_campus_emission_report_pipeline", + "data_center_watch": "run_data_center_watch_pipeline", +} + + +def test_generated_pipeline_exports(): + for app_name, symbol in APP_PIPELINES.items(): + module = importlib.import_module(f"sage.apps.{app_name}") + assert hasattr(module, symbol) + + +def test_operator_smoke_behaviors(tmp_path: Path): + from sage.apps.academic_metadata.operators import AuthorNormalizer, MetadataExtractor + from sage.apps.contract_matcher.operators import KeywordExtractor, TemplateMatcher + from sage.apps.customer_deduplication.operators import DuplicateDetector, SimilarityCalculator + from sage.apps.doc_classifier.operators import Classifier as DocClassifier + from sage.apps.doc_classifier.operators import FeatureExtractor + from sage.apps.doc_classifier.operators import TextExtractor as DocTextExtractor + from sage.apps.doc_classifier.operators import Tokenizer as DocTokenizer + from sage.apps.feedback_analyzer.operators import KeywordExtractor as FeedbackKeywordExtractor + from sage.apps.feedback_analyzer.operators import KeywordScorer, TextCleaner + from sage.apps.inventory_alert.operators import AlertGenerator, InventoryComparator + from sage.apps.log_parser.operators import ErrorFilter, LogEnricher, LogParser + from sage.apps.order_anomaly_detector.operators import FeatureCalculator, RuleScorer + from sage.apps.product_sync.operators import DataValidator, FieldMapper + from sage.apps.quality_defect_filter.operators import ( + DefectSeverityScorer, + DefectSplitter, + DefectStandardizer, + DefectTextExtractor, + ) + from sage.apps.restaurant_sales_analysis.operators import ( + DishSplitter, + InventoryJoiner, + MenuProfitScorer, + ) + from sage.apps.voucher_classifier.operators import FieldExtractor as VoucherFieldExtractor + from sage.apps.voucher_classifier.operators import RuleClassifier as VoucherRuleClassifier + + requirement = KeywordExtractor().execute( + {"text": "Need a confidentiality and disclosure contract"} + ) + matched = TemplateMatcher(top_k=2).execute(requirement) + assert matched["matches"] + + parsed_log = LogParser().execute("2026-04-21T10:00:00 ERROR [API] Request failed ERR500") + filtered_log = ErrorFilter(error_levels=["ERROR"]).execute(parsed_log) + enriched_log = LogEnricher().execute(filtered_log) + assert enriched_log["has_error_code"] is True + + metadata = MetadataExtractor().execute( + { + "text": "A Practical Medical Study\nAlice Smith, Bob Lee\nAbstract: This paper studies biomarkers.\nKeywords: medicine\nDOI 10.1234/ABC123" + } + ) + normalized_metadata = AuthorNormalizer().execute(metadata) + assert normalized_metadata["title"] == "A Practical Medical Study" + assert normalized_metadata["authors"] + + duplicate_detector = DuplicateDetector(threshold=0.8) + first = duplicate_detector.execute( + SimilarityCalculator().execute( + {"customer_id": "1", "name": "Alice", "email": "a@example.com", "phone": "13800000000"} + ) + ) + second = duplicate_detector.execute( + SimilarityCalculator().execute( + {"customer_id": "2", "name": "Alice", "email": "a@example.com", "phone": "13800000000"} + ) + ) + assert first["is_duplicate"] is False + assert second["is_duplicate"] is True + + cleaned_feedback = TextCleaner().execute( + {"text": "Great support, but refund is slow", "id": "1"} + ) + scored_feedback = FeedbackKeywordExtractor(top_n=5).execute( + KeywordScorer().execute( + {"feedback_id": "1", "token": cleaned_feedback["cleaned_text"].split()[0], "length": 5} + ) + ) + assert scored_feedback is not None + + voucher = VoucherRuleClassifier().execute( + VoucherFieldExtractor().execute( + {"ocr_text": "Amount: 200 Date: 2026-04-01 taxi reimbursement"} + ) + ) + assert voucher["voucher_type"] == "travel_expense" + + mapped_product = DataValidator().execute( + FieldMapper().execute( + {"sku": "S-1", "name": "Widget", "price": "8.5", "inventory": "12", "category": "tool"} + ) + ) + assert mapped_product["is_valid"] is True + + tokenized_doc = DocTokenizer().execute( + DocTextExtractor().execute( + {"text": "This contract includes agreement clauses and liability terms."} + ) + ) + classified_doc = DocClassifier().execute(FeatureExtractor().execute(tokenized_doc[0])) + assert classified_doc["label"] == "contract" + + inventory = AlertGenerator().execute( + InventoryComparator().execute( + {"sku": "A-1", "current_stock": 2, "reorder_point": 5, "max_stock": 20} + ) + ) + assert inventory["status"] == "low" + + defect_items = DefectSplitter().execute( + DefectTextExtractor().execute( + {"report_id": "R1", "description": "scratch; crack", "severity": "high"} + ) + ) + scored_defect = DefectSeverityScorer().execute(DefectStandardizer().execute(defect_items[1])) + assert scored_defect["severity_level"] in {"medium", "high"} + + dish_items = DishSplitter().execute( + { + "order_id": "O1", + "dishes": "noodles:2:48:20|tea:1:12:3", + "inventory_qty": 3, + "waste_rate": 0.1, + } + ) + advised_dish = MenuProfitScorer().execute(InventoryJoiner().execute(dish_items[0])) + assert advised_dish["recommendation"] in {"restock", "keep", "promote", "remove_or_reprice"} + + scored_order = RuleScorer().execute( + FeatureCalculator().execute({"amount": 12000, "quantity": 2}) + ) + assert scored_order["is_anomaly"] is True + + snapshot = tmp_path / "snapshot.json" + snapshot.write_text(json.dumps({"ok": True}), encoding="utf-8") + assert json.loads(snapshot.read_text(encoding="utf-8"))["ok"] is True diff --git a/compile_targets.txt b/compile_targets.txt new file mode 100644 index 0000000..f76fcfa --- /dev/null +++ b/compile_targets.txt @@ -0,0 +1,273 @@ +apps/src/sage/apps/log_parser/operators.py +apps/src/sage/apps/log_parser/pipeline.py +examples/run_log_parser.py +apps/src/sage/apps/data_cleaner/operators.py +apps/src/sage/apps/data_cleaner/pipeline.py +examples/run_data_cleaner.py +apps/src/sage/apps/resume_parser/operators.py +apps/src/sage/apps/resume_parser/pipeline.py +examples/run_resume_parser.py +apps/src/sage/apps/feedback_analyzer/operators.py +apps/src/sage/apps/feedback_analyzer/pipeline.py +examples/run_feedback_analyzer.py +apps/src/sage/apps/web_scraper/operators.py +apps/src/sage/apps/web_scraper/pipeline.py +examples/run_web_scraper.py +apps/src/sage/apps/academic_metadata/operators.py +apps/src/sage/apps/academic_metadata/pipeline.py +examples/run_academic_metadata.py +apps/src/sage/apps/customer_deduplication/operators.py +apps/src/sage/apps/customer_deduplication/pipeline.py +examples/run_customer_deduplication.py +apps/src/sage/apps/voucher_classifier/operators.py +apps/src/sage/apps/voucher_classifier/pipeline.py +examples/run_voucher_classifier.py +apps/src/sage/apps/product_sync/operators.py +apps/src/sage/apps/product_sync/pipeline.py +examples/run_product_sync.py +apps/src/sage/apps/doc_classifier/operators.py +apps/src/sage/apps/doc_classifier/pipeline.py +examples/run_doc_classifier.py +apps/src/sage/apps/contract_matcher/operators.py +apps/src/sage/apps/contract_matcher/pipeline.py +examples/run_contract_matcher.py +apps/src/sage/apps/news_aggregator/operators.py +apps/src/sage/apps/news_aggregator/pipeline.py +examples/run_news_aggregator.py +apps/src/sage/apps/ticket_router/operators.py +apps/src/sage/apps/ticket_router/pipeline.py +examples/run_ticket_router.py +apps/src/sage/apps/content_moderation/operators.py +apps/src/sage/apps/content_moderation/pipeline.py +examples/run_content_moderation.py +apps/src/sage/apps/contract_risk/operators.py +apps/src/sage/apps/contract_risk/pipeline.py +examples/run_contract_risk.py +apps/src/sage/apps/user_behavior_analytics/operators.py +apps/src/sage/apps/user_behavior_analytics/pipeline.py +examples/run_user_behavior_analytics.py +apps/src/sage/apps/inventory_alert/operators.py +apps/src/sage/apps/inventory_alert/pipeline.py +examples/run_inventory_alert.py +apps/src/sage/apps/quality_defect_filter/operators.py +apps/src/sage/apps/quality_defect_filter/pipeline.py +examples/run_quality_defect_filter.py +apps/src/sage/apps/permission_audit/operators.py +apps/src/sage/apps/permission_audit/pipeline.py +examples/run_permission_audit.py +apps/src/sage/apps/order_anomaly_detector/operators.py +apps/src/sage/apps/order_anomaly_detector/pipeline.py +examples/run_order_anomaly_detector.py +apps/src/sage/apps/attendance_alert/operators.py +apps/src/sage/apps/attendance_alert/pipeline.py +examples/run_attendance_alert.py +apps/src/sage/apps/lead_scoring/operators.py +apps/src/sage/apps/lead_scoring/pipeline.py +examples/run_lead_scoring.py +apps/src/sage/apps/arbitrage_detector/operators.py +apps/src/sage/apps/arbitrage_detector/pipeline.py +examples/run_arbitrage_detector.py +apps/src/sage/apps/weather_sales_forecast/operators.py +apps/src/sage/apps/weather_sales_forecast/pipeline.py +examples/run_weather_sales_forecast.py +apps/src/sage/apps/geo_recommendation/operators.py +apps/src/sage/apps/geo_recommendation/pipeline.py +examples/run_geo_recommendation.py +apps/src/sage/apps/company_credit/operators.py +apps/src/sage/apps/company_credit/pipeline.py +examples/run_company_credit.py +apps/src/sage/apps/movie_scheduling_optimizer/operators.py +apps/src/sage/apps/movie_scheduling_optimizer/pipeline.py +examples/run_movie_scheduling_optimizer.py +apps/src/sage/apps/real_estate_valuation/operators.py +apps/src/sage/apps/real_estate_valuation/pipeline.py +examples/run_real_estate_valuation.py +apps/src/sage/apps/multi_factor_credit_score/operators.py +apps/src/sage/apps/multi_factor_credit_score/pipeline.py +examples/run_multi_factor_credit_score.py +apps/src/sage/apps/logistics_cost_optimizer/operators.py +apps/src/sage/apps/logistics_cost_optimizer/pipeline.py +examples/run_logistics_cost_optimizer.py +apps/src/sage/apps/medical_registration_optimizer/operators.py +apps/src/sage/apps/medical_registration_optimizer/pipeline.py +examples/run_medical_registration_optimizer.py +apps/src/sage/apps/exhibition_heatmap/operators.py +apps/src/sage/apps/exhibition_heatmap/pipeline.py +examples/run_exhibition_heatmap.py +apps/src/sage/apps/dorm_energy_optimizer/operators.py +apps/src/sage/apps/dorm_energy_optimizer/pipeline.py +examples/run_dorm_energy_optimizer.py +apps/src/sage/apps/restaurant_sales_analysis/operators.py +apps/src/sage/apps/restaurant_sales_analysis/pipeline.py +examples/run_restaurant_sales_analysis.py +apps/src/sage/apps/warehouse_slot_optimizer/operators.py +apps/src/sage/apps/warehouse_slot_optimizer/pipeline.py +examples/run_warehouse_slot_optimizer.py +apps/src/sage/apps/paper_classifier/operators.py +apps/src/sage/apps/paper_classifier/pipeline.py +examples/run_paper_classifier.py +apps/src/sage/apps/vendor_evaluation_standardizer/operators.py +apps/src/sage/apps/vendor_evaluation_standardizer/pipeline.py +examples/run_vendor_evaluation_standardizer.py +apps/src/sage/apps/content_tagger/operators.py +apps/src/sage/apps/content_tagger/pipeline.py +examples/run_content_tagger.py +apps/src/sage/apps/partner_profile_hub/operators.py +apps/src/sage/apps/partner_profile_hub/pipeline.py +examples/run_partner_profile_hub.py +apps/src/sage/apps/subscription_dispatch/operators.py +apps/src/sage/apps/subscription_dispatch/pipeline.py +examples/run_subscription_dispatch.py +apps/src/sage/apps/contract_versioning/operators.py +apps/src/sage/apps/contract_versioning/pipeline.py +examples/run_contract_versioning.py +apps/src/sage/apps/invoice_reconciliation/operators.py +apps/src/sage/apps/invoice_reconciliation/pipeline.py +examples/run_invoice_reconciliation.py +apps/src/sage/apps/project_risk_monitor/operators.py +apps/src/sage/apps/project_risk_monitor/pipeline.py +examples/run_project_risk_monitor.py +apps/src/sage/apps/learning_record_hub/operators.py +apps/src/sage/apps/learning_record_hub/pipeline.py +examples/run_learning_record_hub.py +apps/src/sage/apps/supply_chain_tracker/operators.py +apps/src/sage/apps/supply_chain_tracker/pipeline.py +examples/run_supply_chain_tracker.py +apps/src/sage/apps/compliance_doc_manager/operators.py +apps/src/sage/apps/compliance_doc_manager/pipeline.py +examples/run_compliance_doc_manager.py +apps/src/sage/apps/mail_classifier/operators.py +apps/src/sage/apps/mail_classifier/pipeline.py +examples/run_mail_classifier.py +apps/src/sage/apps/meeting_minutes/operators.py +apps/src/sage/apps/meeting_minutes/pipeline.py +examples/run_meeting_minutes.py +apps/src/sage/apps/policy_update_notifier/operators.py +apps/src/sage/apps/policy_update_notifier/pipeline.py +examples/run_policy_update_notifier.py +apps/src/sage/apps/export_transformer/operators.py +apps/src/sage/apps/export_transformer/pipeline.py +examples/run_export_transformer.py +apps/src/sage/apps/api_log_analytics/operators.py +apps/src/sage/apps/api_log_analytics/pipeline.py +examples/run_api_log_analytics.py +apps/src/sage/apps/backup_sync/operators.py +apps/src/sage/apps/backup_sync/pipeline.py +examples/run_backup_sync.py +apps/src/sage/apps/patent_competition_monitor/operators.py +apps/src/sage/apps/patent_competition_monitor/pipeline.py +examples/run_patent_competition_monitor.py +apps/src/sage/apps/grant_subscription/operators.py +apps/src/sage/apps/grant_subscription/pipeline.py +examples/run_grant_subscription.py +apps/src/sage/apps/experiment_review/operators.py +apps/src/sage/apps/experiment_review/pipeline.py +examples/run_experiment_review.py +apps/src/sage/apps/repro_audit/operators.py +apps/src/sage/apps/repro_audit/pipeline.py +examples/run_repro_audit.py +apps/src/sage/apps/benchmark_watch/operators.py +apps/src/sage/apps/benchmark_watch/pipeline.py +examples/run_benchmark_watch.py +apps/src/sage/apps/course_qa_helper/operators.py +apps/src/sage/apps/course_qa_helper/pipeline.py +examples/run_course_qa_helper.py +apps/src/sage/apps/lesson_scheduler/operators.py +apps/src/sage/apps/lesson_scheduler/pipeline.py +examples/run_lesson_scheduler.py +apps/src/sage/apps/assignment_feedback/operators.py +apps/src/sage/apps/assignment_feedback/pipeline.py +examples/run_assignment_feedback.py +apps/src/sage/apps/skill_gap_diagnosis/operators.py +apps/src/sage/apps/skill_gap_diagnosis/pipeline.py +examples/run_skill_gap_diagnosis.py +apps/src/sage/apps/interview_coach/operators.py +apps/src/sage/apps/interview_coach/pipeline.py +examples/run_interview_coach.py +apps/src/sage/apps/campus_aid_gap_alert/operators.py +apps/src/sage/apps/campus_aid_gap_alert/pipeline.py +examples/run_campus_aid_gap_alert.py +apps/src/sage/apps/triage_structurer/operators.py +apps/src/sage/apps/triage_structurer/pipeline.py +examples/run_triage_structurer.py +apps/src/sage/apps/radiology_followup_loop/operators.py +apps/src/sage/apps/radiology_followup_loop/pipeline.py +examples/run_radiology_followup_loop.py +apps/src/sage/apps/drug_leaflet_extractor/operators.py +apps/src/sage/apps/drug_leaflet_extractor/pipeline.py +examples/run_drug_leaflet_extractor.py +apps/src/sage/apps/lab_turnaround_alert/operators.py +apps/src/sage/apps/lab_turnaround_alert/pipeline.py +examples/run_lab_turnaround_alert.py +apps/src/sage/apps/quote_compare/operators.py +apps/src/sage/apps/quote_compare/pipeline.py +examples/run_quote_compare.py +apps/src/sage/apps/policy_search_helper/operators.py +apps/src/sage/apps/policy_search_helper/pipeline.py +examples/run_policy_search_helper.py +apps/src/sage/apps/knowledge_cleanup/operators.py +apps/src/sage/apps/knowledge_cleanup/pipeline.py +examples/run_knowledge_cleanup.py +apps/src/sage/apps/podcast_highlight/operators.py +apps/src/sage/apps/podcast_highlight/pipeline.py +examples/run_podcast_highlight.py +apps/src/sage/apps/brand_compliance_review/operators.py +apps/src/sage/apps/brand_compliance_review/pipeline.py +examples/run_brand_compliance_review.py +apps/src/sage/apps/subtitle_qc/operators.py +apps/src/sage/apps/subtitle_qc/pipeline.py +examples/run_subtitle_qc.py +apps/src/sage/apps/content_scheduler/operators.py +apps/src/sage/apps/content_scheduler/pipeline.py +examples/run_content_scheduler.py +apps/src/sage/apps/media_archive_search/operators.py +apps/src/sage/apps/media_archive_search/pipeline.py +examples/run_media_archive_search.py +apps/src/sage/apps/factory_watch/operators.py +apps/src/sage/apps/factory_watch/pipeline.py +examples/run_factory_watch.py +apps/src/sage/apps/cold_chain_watch/operators.py +apps/src/sage/apps/cold_chain_watch/pipeline.py +examples/run_cold_chain_watch.py +apps/src/sage/apps/greenhouse_assistant/operators.py +apps/src/sage/apps/greenhouse_assistant/pipeline.py +examples/run_greenhouse_assistant.py +apps/src/sage/apps/community_hotspot_drift/operators.py +apps/src/sage/apps/community_hotspot_drift/pipeline.py +examples/run_community_hotspot_drift.py +apps/src/sage/apps/traffic_briefing/operators.py +apps/src/sage/apps/traffic_briefing/pipeline.py +examples/run_traffic_briefing.py +apps/src/sage/apps/urban_repair_scheduler/operators.py +apps/src/sage/apps/urban_repair_scheduler/pipeline.py +examples/run_urban_repair_scheduler.py +apps/src/sage/apps/permit_material_review/operators.py +apps/src/sage/apps/permit_material_review/pipeline.py +examples/run_permit_material_review.py +apps/src/sage/apps/municipal_search/operators.py +apps/src/sage/apps/municipal_search/pipeline.py +examples/run_municipal_search.py +apps/src/sage/apps/budget_variance_alert/operators.py +apps/src/sage/apps/budget_variance_alert/pipeline.py +examples/run_budget_variance_alert.py +apps/src/sage/apps/cashflow_watch/operators.py +apps/src/sage/apps/cashflow_watch/pipeline.py +examples/run_cashflow_watch.py +apps/src/sage/apps/return_reason_mining/operators.py +apps/src/sage/apps/return_reason_mining/pipeline.py +examples/run_return_reason_mining.py +apps/src/sage/apps/store_daily_digest/operators.py +apps/src/sage/apps/store_daily_digest/pipeline.py +examples/run_store_daily_digest.py +apps/src/sage/apps/carbon_collection/operators.py +apps/src/sage/apps/carbon_collection/pipeline.py +examples/run_carbon_collection.py +apps/src/sage/apps/solar_alerting/operators.py +apps/src/sage/apps/solar_alerting/pipeline.py +examples/run_solar_alerting.py +apps/src/sage/apps/campus_emission_report/operators.py +apps/src/sage/apps/campus_emission_report/pipeline.py +examples/run_campus_emission_report.py +apps/src/sage/apps/data_center_watch/operators.py +apps/src/sage/apps/data_center_watch/pipeline.py +examples/run_data_center_watch.py diff --git a/docs/100-app-roadmap.md b/docs/100-app-roadmap.md index 9378917..e3fe822 100644 --- a/docs/100-app-roadmap.md +++ b/docs/100-app-roadmap.md @@ -59,188 +59,184 @@ with the current repository structure. ### Education And Training -11. `student_improvement` | `existing` | Stateful score-improvement workflow with wrong-question - tracking. -01. `course_qa_tutor` | `B1` | Retrieval-based course tutor over notes, slides, and assignments. -01. `lesson_plan_designer` | `B1` | Generate lesson plans, weekly pacing, and differentiated - exercises. -01. `classroom_attention_monitor` | `B2` | Analyze classroom video for engagement, confusion, and - pacing signals. -01. `assignment_feedback_assistant` | `B1` | Provide rubric-based draft feedback before teacher - review. -01. `oral_exam_coach` | `B2` | Evaluate spoken answers and generate targeted follow-up drills. -01. `skill_gap_mapper` | `B1` | Map learner performance to a competency graph and recommend practice - paths. -01. `training_compliance_tracker` | `B1` | Monitor enterprise training completion and highlight - compliance risks. -01. `interview_practice_coach` | `B1` | Simulate interviews, score responses, and suggest - role-specific improvements. -01. `campus_service_desk` | `B2` | Route student requests for finance, housing, advising, and - administration. +1. `student_improvement` | `existing` | Stateful score-improvement workflow with wrong-question + tracking. +1. `course_qa_tutor` | `B1` | Retrieval-based course tutor over notes, slides, and assignments. +1. `lesson_plan_designer` | `B1` | Generate lesson plans, weekly pacing, and differentiated + exercises. +1. `classroom_attention_monitor` | `B2` | Analyze classroom video for engagement, confusion, and + pacing signals. +1. `assignment_feedback_assistant` | `B1` | Provide rubric-based draft feedback before teacher + review. +1. `oral_exam_coach` | `B2` | Evaluate spoken answers and generate targeted follow-up drills. +1. `skill_gap_mapper` | `B1` | Map learner performance to a competency graph and recommend practice + paths. +1. `training_compliance_tracker` | `B1` | Monitor enterprise training completion and highlight + compliance risks. +1. `interview_practice_coach` | `B1` | Simulate interviews, score responses, and suggest + role-specific improvements. +1. `campus_service_desk` | `B2` | Route student requests for finance, housing, advising, and + administration. ### Healthcare And Life Sciences -21. `medical_diagnosis` | `existing` | AI-assisted medical image analysis workflow. -01. `triage_intake_assistant` | `B2` | Structure patient intake information and prioritize triage - queues. -01. `radiology_followup_tracker` | `B2` | Detect follow-up recommendations in reports and track - unresolved cases. -01. `clinical_guideline_copilot` | `B2` | Link patient context to guideline sections and action - checklists. -01. `adverse_event_monitor` | `B2` | Monitor notes, reports, and signals for potential adverse - events. -01. `medical_claim_auditor` | `B3` | Review claim packets for coding gaps, missing evidence, and - denial risk. -01. `eldercare_home_monitor` | `B3` | Combine home-device signals and notes to flag decline or - safety issues. -01. `hospital_bed_flow_coordinator` | `B3` | Summarize bed status, discharge blockers, and - escalation priorities. -01. `drug_label_extractor` | `B1` | Extract dosage, contraindications, and warnings from drug - labels. -01. `telehealth_visit_summarizer` | `B1` | Turn telehealth transcripts into structured visit notes - and action items. +1. `medical_diagnosis` | `existing` | AI-assisted medical image analysis workflow. +1. `triage_intake_assistant` | `B2` | Structure patient intake information and prioritize triage + queues. +1. `radiology_followup_tracker` | `B2` | Detect follow-up recommendations in reports and track + unresolved cases. +1. `clinical_guideline_copilot` | `B2` | Link patient context to guideline sections and action + checklists. +1. `adverse_event_monitor` | `B2` | Monitor notes, reports, and signals for potential adverse + events. +1. `medical_claim_auditor` | `B3` | Review claim packets for coding gaps, missing evidence, and + denial risk. +1. `eldercare_home_monitor` | `B3` | Combine home-device signals and notes to flag decline or safety + issues. +1. `hospital_bed_flow_coordinator` | `B3` | Summarize bed status, discharge blockers, and escalation + priorities. +1. `drug_label_extractor` | `B1` | Extract dosage, contraindications, and warnings from drug labels. +1. `telehealth_visit_summarizer` | `B1` | Turn telehealth transcripts into structured visit notes + and action items. ### Enterprise Productivity And Operations -31. `work_report_generator` | `existing` | Generate work reports from activity history and project - context. -01. `meeting_action_tracker` | `B1` | Extract decisions, owners, deadlines, and unresolved blockers - from meetings. -01. `contract_review_assistant` | `B2` | Highlight risky clauses, obligations, and missing terms in - contracts. -01. `sales_call_insights` | `B1` | Summarize objections, buying signals, and follow-up priorities - from calls. -01. `customer_support_copilot` | `B1` | Draft grounded responses and route tickets by issue type and - urgency. -01. `procurement_analyzer` | `B2` | Compare supplier quotes, delivery risk, and negotiation - leverage. -01. `org_policy_search` | `B1` | Answer internal policy questions with source-grounded citations. -01. `internal_ticket_router` | `B1` | Route IT, HR, and finance requests to the right queue with - context. -01. `knowledge_base_curator` | `B1` | Deduplicate, summarize, and age-rank internal knowledge - articles. -01. `multi_team_status_board` | `B2` | Aggregate project updates into a leadership-level cross-team - digest. +1. `work_report_generator` | `existing` | Generate work reports from activity history and project + context. +1. `meeting_action_tracker` | `B1` | Extract decisions, owners, deadlines, and unresolved blockers + from meetings. +1. `contract_review_assistant` | `B2` | Highlight risky clauses, obligations, and missing terms in + contracts. +1. `sales_call_insights` | `B1` | Summarize objections, buying signals, and follow-up priorities + from calls. +1. `customer_support_copilot` | `B1` | Draft grounded responses and route tickets by issue type and + urgency. +1. `procurement_analyzer` | `B2` | Compare supplier quotes, delivery risk, and negotiation leverage. +1. `org_policy_search` | `B1` | Answer internal policy questions with source-grounded citations. +1. `internal_ticket_router` | `B1` | Route IT, HR, and finance requests to the right queue with + context. +1. `knowledge_base_curator` | `B1` | Deduplicate, summarize, and age-rank internal knowledge + articles. +1. `multi_team_status_board` | `B2` | Aggregate project updates into a leadership-level cross-team + digest. ### Media And Creative Operations -41. `video_intelligence` | `existing` | Multimodal video understanding and event summarization. -01. `podcast_clip_generator` | `B1` | Find high-value podcast segments and generate clips with - titles and notes. -01. `brand_asset_reviewer` | `B2` | Check copy, visuals, and metadata for brand consistency. -01. `livestream_incident_monitor` | `B3` | Detect incidents, dead air, and moderation risks in - livestreams. -01. `ad_creative_tester` | `B2` | Compare ad variants, messaging hooks, and likely audience - resonance. -01. `subtitle_qc_assistant` | `B1` | Detect subtitle timing, terminology, and translation quality - issues. -01. `content_calendar_planner` | `B1` | Plan multi-channel content schedules based on themes and - launches. -01. `social_trend_storyboard` | `B2` | Turn social trend signals into short-form content concepts. -01. `news_bias_analyzer` | `B2` | Compare framing and bias patterns across outlets on the same - event. -01. `media_archive_indexer` | `B1` | Build searchable indexes over large media collections with - summaries. +1. `video_intelligence` | `existing` | Multimodal video understanding and event summarization. +1. `podcast_clip_generator` | `B1` | Find high-value podcast segments and generate clips with titles + and notes. +1. `brand_asset_reviewer` | `B2` | Check copy, visuals, and metadata for brand consistency. +1. `livestream_incident_monitor` | `B3` | Detect incidents, dead air, and moderation risks in + livestreams. +1. `ad_creative_tester` | `B2` | Compare ad variants, messaging hooks, and likely audience + resonance. +1. `subtitle_qc_assistant` | `B1` | Detect subtitle timing, terminology, and translation quality + issues. +1. `content_calendar_planner` | `B1` | Plan multi-channel content schedules based on themes and + launches. +1. `social_trend_storyboard` | `B2` | Turn social trend signals into short-form content concepts. +1. `news_bias_analyzer` | `B2` | Compare framing and bias patterns across outlets on the same event. +1. `media_archive_indexer` | `B1` | Build searchable indexes over large media collections with + summaries. ### IoT, Edge, And Physical World -51. `smart_home` | `existing` | Stateful IoT automation and home-event orchestration. -01. `building_energy_manager` | `B2` | Optimize occupancy, HVAC schedules, and energy anomalies. -01. `factory_sensor_watchdog` | `B2` | Detect abnormal sensor patterns and summarize likely causes. -01. `cold_chain_monitor` | `B2` | Track storage temperature excursions and delivery quality risk. -01. `retail_queue_monitor` | `B3` | Analyze camera and POS signals to predict queue pressure. -01. `parking_lot_coordinator` | `B2` | Combine camera, gate, and event data to manage parking flow. -01. `edge_safety_alerting` | `B3` | Detect unsafe zones, restricted access, and PPE violations at - the edge. -01. `greenhouse_copilot` | `B2` | Orchestrate irrigation, climate signals, and crop alerting. -01. `fleet_dashcam_reviewer` | `B3` | Summarize risky driving events and coaching recommendations. -01. `warehouse_robot_supervisor` | `B3` | Detect stalled tasks, congestion, and robot-assist - escalation needs. +1. `smart_home` | `existing` | Stateful IoT automation and home-event orchestration. +1. `building_energy_manager` | `B2` | Optimize occupancy, HVAC schedules, and energy anomalies. +1. `factory_sensor_watchdog` | `B2` | Detect abnormal sensor patterns and summarize likely causes. +1. `cold_chain_monitor` | `B2` | Track storage temperature excursions and delivery quality risk. +1. `retail_queue_monitor` | `B3` | Analyze camera and POS signals to predict queue pressure. +1. `parking_lot_coordinator` | `B2` | Combine camera, gate, and event data to manage parking flow. +1. `edge_safety_alerting` | `B3` | Detect unsafe zones, restricted access, and PPE violations at the + edge. +1. `greenhouse_copilot` | `B2` | Orchestrate irrigation, climate signals, and crop alerting. +1. `fleet_dashcam_reviewer` | `B3` | Summarize risky driving events and coaching recommendations. +1. `warehouse_robot_supervisor` | `B3` | Detect stalled tasks, congestion, and robot-assist + escalation needs. ### Public Sector And Smart City -61. `emergency_signal_fusion` | `B3` | Fuse call-center, camera, and field-report signals into - incident views. -01. `citizen_feedback_router` | `B1` | Classify citizen complaints and route them to departments - with summaries. -01. `traffic_incident_briefing` | `B2` | Summarize traffic disruptions, likely impact, and response - priorities. -01. `urban_maintenance_scheduler` | `B2` | Cluster repair requests and recommend efficient - maintenance routing. -01. `environmental_complaint_analyzer` | `B1` | Detect recurring pollution complaints and hotspot - trends. -01. `permit_review_assistant` | `B2` | Check permit submissions for missing materials and policy - mismatches. -01. `utility_outage_triage` | `B3` | Correlate outage reports, asset telemetry, and restoration - priorities. -01. `public_health_signal_watch` | `B3` | Monitor weak signals from clinics, news, and reports for - outbreaks. -01. `disaster_resource_matcher` | `B3` | Match shelters, supplies, volunteers, and requests during - emergencies. -01. `municipal_doc_search` | `B1` | Search city policies, council decisions, and service procedures. +1. `emergency_signal_fusion` | `B3` | Fuse call-center, camera, and field-report signals into + incident views. +1. `citizen_feedback_router` | `B1` | Classify citizen complaints and route them to departments with + summaries. +1. `traffic_incident_briefing` | `B2` | Summarize traffic disruptions, likely impact, and response + priorities. +1. `urban_maintenance_scheduler` | `B2` | Cluster repair requests and recommend efficient + maintenance routing. +1. `environmental_complaint_analyzer` | `B1` | Detect recurring pollution complaints and hotspot + trends. +1. `permit_review_assistant` | `B2` | Check permit submissions for missing materials and policy + mismatches. +1. `utility_outage_triage` | `B3` | Correlate outage reports, asset telemetry, and restoration + priorities. +1. `public_health_signal_watch` | `B3` | Monitor weak signals from clinics, news, and reports for + outbreaks. +1. `disaster_resource_matcher` | `B3` | Match shelters, supplies, volunteers, and requests during + emergencies. +1. `municipal_doc_search` | `B1` | Search city policies, council decisions, and service procedures. ### Finance And Risk -71. `invoice_exception_detector` | `B1` | Flag unusual invoice amounts, duplicate risk, and missing - fields. -01. `expense_audit_assistant` | `B1` | Review reimbursements for policy violations and evidence - gaps. -01. `loan_application_screening` | `B2` | Structure applicant packets and surface review priorities. -01. `aml_case_prioritizer` | `B3` | Rank anti-money-laundering cases by signal strength and analyst - value. -01. `insurance_claim_triage` | `B2` | Classify claim complexity, fraud indicators, and next-review - steps. -01. `portfolio_news_risk_monitor` | `B2` | Connect news events to holdings and summarize exposure - impact. -01. `vendor_risk_scanner` | `B2` | Combine public signals, contracts, and incidents into vendor risk - views. -01. `cashflow_forecaster` | `B2` | Blend invoicing, pipeline, and payment signals into short-term - forecasts. -01. `pricing_anomaly_monitor` | `B2` | Detect suspicious price changes across products or accounts. -01. `billing_dispute_resolver` | `B1` | Organize billing evidence and draft dispute-resolution - responses. +1. `invoice_exception_detector` | `B1` | Flag unusual invoice amounts, duplicate risk, and missing + fields. +1. `expense_audit_assistant` | `B1` | Review reimbursements for policy violations and evidence gaps. +1. `loan_application_screening` | `B2` | Structure applicant packets and surface review priorities. +1. `aml_case_prioritizer` | `B3` | Rank anti-money-laundering cases by signal strength and analyst + value. +1. `insurance_claim_triage` | `B2` | Classify claim complexity, fraud indicators, and next-review + steps. +1. `portfolio_news_risk_monitor` | `B2` | Connect news events to holdings and summarize exposure + impact. +1. `vendor_risk_scanner` | `B2` | Combine public signals, contracts, and incidents into vendor risk + views. +1. `cashflow_forecaster` | `B2` | Blend invoicing, pipeline, and payment signals into short-term + forecasts. +1. `pricing_anomaly_monitor` | `B2` | Detect suspicious price changes across products or accounts. +1. `billing_dispute_resolver` | `B1` | Organize billing evidence and draft dispute-resolution + responses. ### Commerce And Supply Chain -81. `demand_sensing_assistant` | `B2` | Merge sales, promotions, weather, and events to improve - demand sensing. -01. `dynamic_catalog_enricher` | `B1` | Normalize product listings and generate richer catalog - attributes. -01. `return_reason_analyzer` | `B1` | Cluster return causes and identify avoidable quality or - expectation gaps. -01. `supplier_delay_predictor` | `B2` | Forecast likely supplier delays from signals across orders - and incidents. -01. `warehouse_pick_path_optimizer` | `B3` | Combine order waves and layout constraints for better - pick routing. -01. `aftersales_ticket_triage` | `B1` | Prioritize installation, warranty, and repair tickets. -01. `menu_margin_copilot` | `B1` | Help restaurant operators balance menu pricing, margin, and - demand. -01. `store_ops_digest` | `B1` | Summarize shift issues, stockouts, shrinkage, and action items. -01. `franchise_quality_monitor` | `B2` | Compare quality drift, service complaints, and compliance - across stores. -01. `marketplace_listing_guard` | `B2` | Detect duplicate listings, policy risk, and misleading - claims. +1. `demand_sensing_assistant` | `B2` | Merge sales, promotions, weather, and events to improve + demand sensing. +1. `dynamic_catalog_enricher` | `B1` | Normalize product listings and generate richer catalog + attributes. +1. `return_reason_analyzer` | `B1` | Cluster return causes and identify avoidable quality or + expectation gaps. +1. `supplier_delay_predictor` | `B2` | Forecast likely supplier delays from signals across orders + and incidents. +1. `warehouse_pick_path_optimizer` | `B3` | Combine order waves and layout constraints for better + pick routing. +1. `aftersales_ticket_triage` | `B1` | Prioritize installation, warranty, and repair tickets. +1. `menu_margin_copilot` | `B1` | Help restaurant operators balance menu pricing, margin, and + demand. +1. `store_ops_digest` | `B1` | Summarize shift issues, stockouts, shrinkage, and action items. +1. `franchise_quality_monitor` | `B2` | Compare quality drift, service complaints, and compliance + across stores. +1. `marketplace_listing_guard` | `B2` | Detect duplicate listings, policy risk, and misleading + claims. ### Sustainability And Infrastructure -91. `carbon_data_collector` | `B1` | Gather emissions data from documents, meters, and operational - systems. -01. `solar_farm_alerting` | `B2` | Summarize inverter anomalies, weather effects, and maintenance - needs. -01. `water_leak_monitor` | `B2` | Detect persistent leak signatures and prioritize field inspection. -01. `campus_emissions_reporter` | `B1` | Build periodic campus sustainability reports from - distributed data. -01. `hvac_fault_diagnosis` | `B2` | Diagnose HVAC performance issues from building telemetry. -01. `grid_event_summarizer` | `B3` | Turn grid alarms and operator logs into concise incident - timelines. -01. `recycling_sorting_analytics` | `B3` | Analyze contamination patterns in recycling streams using - vision inputs. -01. `construction_site_safety_monitor` | `B3` | Detect unsafe behavior, zone conflicts, and repeated - risk patterns. -01. `data_center_capacity_watch` | `B2` | Track capacity, cooling pressure, and incident risk in - data centers. -01. `pipeline_anomaly_review` | `B3` | Review inspection imagery, sensor events, and maintenance - history. +1. `carbon_data_collector` | `B1` | Gather emissions data from documents, meters, and operational + systems. +1. `solar_farm_alerting` | `B2` | Summarize inverter anomalies, weather effects, and maintenance + needs. +1. `water_leak_monitor` | `B2` | Detect persistent leak signatures and prioritize field inspection. +1. `campus_emissions_reporter` | `B1` | Build periodic campus sustainability reports from + distributed data. +1. `hvac_fault_diagnosis` | `B2` | Diagnose HVAC performance issues from building telemetry. +1. `grid_event_summarizer` | `B3` | Turn grid alarms and operator logs into concise incident + timelines. +1. `recycling_sorting_analytics` | `B3` | Analyze contamination patterns in recycling streams using + vision inputs. +1. `construction_site_safety_monitor` | `B3` | Detect unsafe behavior, zone conflicts, and repeated + risk patterns. +1. `data_center_capacity_watch` | `B2` | Track capacity, cooling pressure, and incident risk in data + centers. +1. `pipeline_anomaly_review` | `B3` | Review inspection imagery, sensor events, and maintenance + history. ## First 15 New Apps To Build diff --git a/docs/sage-100-app-roadmap.md b/docs/sage-100-app-roadmap.md new file mode 100644 index 0000000..e46f9ef --- /dev/null +++ b/docs/sage-100-app-roadmap.md @@ -0,0 +1,4135 @@ +# SAGE框架应用场景完整清单（91个应用） + +## 91个应用总览 + +| 序号 | 应用 | 对应项目 | 能做什么 | +| ---- | -------------------------- | -------------------------------- | --------------------------------------------------------------------- | +| 1 | 企业日志解析与结构化系统 | `log_parser` | 日志文件流 | +| 2 | CSV/Excel数据批量清洗系统 | `data_cleaner` | 批量CSV/Excel → 读取 → 类型转换 → 缺失值填充 → 异常检测 → 清洗输出 | +| 3 | 网页内容抓取与表格提取系统 | `web_scraper` | URL列表 → HTTP请求 → HTML解析 → 表格提取 → 结构化 → 数据库入库 | +| 4 | 医学文献元数据提取系统 | `academic_metadata` | 论文PDF → 文本提取 → 正则+规则提取元数据 → 数据库存储 | +| 5 | 客户数据去重系统 | `customer_deduplication` | 客户数据 → 按电话号码分组 → 计算相似度 → 标记重复 → 输出去重结果 | +| 6 | 财务凭证自动分类系统 | `voucher_classifier` | 凭证图片/PDF → OCR识别 → 提取关键信息 → 规则分类 → 输出分类结果 | +| 7 | 简历数据标准化系统 | `resume_parser` | 简历文件 → 格式识别和转换 → 信息提取 → 标准化 → 结构化简历数据 | +| 8 | 多平台商品数据同步系统 | `product_sync` | A平台商品数据 → 读取 → 字段映射 → 数据校验 → 写入B平台 | +| 9 | 企业知识库自动分类系统 | `doc_classifier` | 文档文件 → 文本提取 → 分词 → TF-IDF特征 → 分类 → 保存分类结果 | +| 10 | 客户反馈关键词提取系统 | `feedback_analyzer` | 反馈文本 → 文本清洗 → 分词 → 关键词提取 → 统计输出 | +| 11 | 法律文案模板匹配系统 | `contract_matcher` | 客户需求描述 → 关键词提取 → 与模板库余弦相似度计算 → 返回匹配模板 | +| 12 | 新闻聚合与去重系统 | `news_aggregator` | RSS源 → HTML解析 → 内容指纹计算 → 去重 → 输出聚合新闻 | +| 13 | 客服工单自动路由系统 | `ticket_router` | 客服工单流 → 工单分类 → 优先级评分 → 客服负载计算 → 最优分配 | +| 14 | 社交媒体内容审核系统 | `content_moderation` | 用户发布内容流 → 文本提取 → 敏感词匹配 → 违规评分 → 隔离违规内容 | +| 15 | 合同条款风险识别系统 | `contract_risk` | 合同文件 → 文本解析 → 条款拆分 → 风险规则匹配 → 输出风险提示 | +| 16 | 用户行为事件采集与分类系统 | `user_behavior_analytics` | 事件流 → 事件校验 → 字段规范化 → 用户行为分类 → 分流写入分析表 | +| 17 | 库存异常告警系统 | `inventory_alert` | 库存快照 → 特征计算 → 阈值规则比对 → 异常分级 → 告警输出 | +| 18 | 生产质量缺陷过滤系统 | `quality_defect_filter` | 质检报告 → 文本抽取 → 缺陷拆分 → 标准化分类 → 缺陷库入库 | +| 19 | 用户权限变更审计系统 | `permission_audit` | 权限变更日志 → 解析 → 敏感操作识别 → 风险打分 → 审计报告输出 | +| 20 | 订单异常检测系统 | `order_anomaly_detector` | 订单数据 → 特征构造 → 风险规则评分 → 异常筛选 → 输出复核队列 | +| 21 | 员工出勤异常预警系统 | `attendance_alert` | 打卡记录 → 班次映射 → 出勤规则校验 → 异常识别 → 预警输出 | +| 22 | 销售机会评分系统 | `lead_scoring` | 销售线索 → 特征抽取 → 规则评分 → 优先级排序 → 分配给销售 | +| 23 | 实时汇率转换与套利检测系统 | `arbitrage_detector` | 交易订单 → 汇率抓取 → 汇率转换 → 套利规则打分 → 告警输出 | +| 24 | 天气数据驱动的销量预测系统 | `weather_sales_forecast` | 销量历史 → 天气数据抓取 → 特征融合 → 销量预测 → 库存建议 | +| 25 | 地理位置智能推荐系统 | `geo_recommendation` | 用户位置 → 地图API查询 → 门店候选集 → 偏好打分 → 推荐结果输出 | +| 26 | 企业信用评估系统 | `company_credit` | 企业名单 → 企业信息抓取 → 风险因子抽取 → 信用评分 → 报告输出 | +| 27 | 电影库存与排片优化系统 | `movie_scheduling_optimizer` | 档期数据 → 电影热度抓取 → 收益特征计算 → 排片评分 → 方案输出 | +| 28 | 房产智能估价系统 | `real_estate_valuation` | 房产信息 → 周边房源抓取 → 特征构造 → 估价计算 → 报价说明输出 | +| 29 | 多维用户信用评分系统 | `multi_factor_credit_score` | 用户清单 → 多源信息抓取 → 特征融合 → 信用评分 → 审批结果输出 | +| 30 | 物流成本优化系统 | `logistics_cost_optimizer` | 订单数据 → 物流报价抓取 → 成本时效比对 → 方案打分 → 推荐输出 | +| 31 | 医疗挂号优化系统 | `medical_registration_optimizer` | 挂号请求 → 号源抓取 → 医患匹配评分 → 挂号建议 → 通知输出 | +| 32 | 展会热度分析系统 | `exhibition_heatmap` | 客流数据 → 区域映射 → 热度特征计算 → 排名输出 → 拥堵提醒 | +| 33 | 宿舍能耗监测优化系统 | `dorm_energy_optimizer` | 能耗数据 → 宿舍映射 → 基线对比 → 异常判断 → 建议输出 | +| 34 | 餐厅菜品销售分析系统 | `restaurant_sales_analysis` | 销售订单 → 菜品拆分 → 库存关联 → 利润计算 → 菜单建议输出 | +| 35 | 仓库货位优化系统 | `warehouse_slot_optimizer` | 拣货历史 → 热度统计 → 距离成本计算 → 货位评分 → 调整方案输出 | +| 36 | 医学论文分类系统 | `paper_classifier` | 论文文本 → 分词与关键词提取 → 相似度分类 → 结果输出 | +| 37 | 供应商评价数据标准化系统 | `vendor_evaluation_standardizer` | 评价数据 → 字段映射 → 评分标准统一 → 重复合并 → 主表输出 | +| 38 | 内容标签自动生成系统 | `content_tagger` | 内容文本 → 清洗 → 关键词提取 → 标签候选生成 → 标签筛选输出 | +| 39 | 合作伙伴信息聚合系统 | `partner_profile_hub` | 多源伙伴数据 → 字段对齐 → 重复合并 → 画像生成 → 统一视图输出 | +| 40 | 订阅制内容分发系统 | `subscription_dispatch` | 新内容 → 内容解析 → 订阅规则匹配 → 个性化筛选 → 分发输出 | +| 41 | 合同模板版本管理系统 | `contract_versioning` | 模板文件 → 版本解析 → 差异比对 → 变更记录生成 → 版本库更新 | +| 42 | 发票自动匹配与对账系统 | `invoice_reconciliation` | 发票数据 + 订单数据 → 字段标准化 → 匹配打分 → 差异识别 → 对账报告输出 | +| 43 | 项目风险日志监控系统 | `project_risk_monitor` | 项目日志 → 风险关键词抽取 → 风险评分 → 项目分组 → 预警输出 | +| 44 | 员工学习认证聚合系统 | `learning_record_hub` | 培训记录 → 员工映射 → 课程归一化 → 认证状态判断 → 学习档案输出 | +| 45 | 供应链订单追踪聚合系统 | `supply_chain_tracker` | 多系统状态 → 状态归一化 → 时间线拼接 → 延迟识别 → 追踪结果输出 | +| 46 | 合规文档自动整理系统 | `compliance_doc_manager` | 合规文档 → 分类整理 → 元数据抽取 → 更新检查 → 复审提醒输出 | +| 47 | 邮件智能分类系统 | `mail_classifier` | 邮件流 → 标题正文解析 → 分类规则匹配 → 优先级评分 → 分类结果输出 | +| 48 | 会议纪要自动生成系统 | `meeting_minutes` | 会议转写文本 → 议题切分 → 行动项提取 → 纪要结构化 → 分发输出 | +| 49 | 制度文件更新通知系统 | `policy_update_notifier` | 制度新旧版本 → 差异提取 → 影响范围识别 → 通知内容生成 → 分发输出 | +| 50 | 数据导出格式转换系统 | `export_transformer` | 源数据 → 查询读取 → 字段映射 → 格式转换 → 多目标输出 | +| 51 | API日志聚合分析系统 | `api_log_analytics` | 多源API日志 → 解析归一化 → 指标提取 → 异常识别 → 性能报告输出 | +| 52 | 数据备份与冗余同步系统 | `backup_sync` | 源数据清单 → 增量识别 → 多目标同步 → 校验比对 → 同步报告输出 | +| 53 | 专利侵权线索预警系统 | `patent_competition_monitor` | 专利文本流 → 权利要求抽取 → 技术要点比对 → 冲突线索评分 → 证据包输出 | +| 54 | 科研资助机会订阅系统 | `grant_subscription` | 资助公告 → 文本解析 → 条件结构化 → 团队画像匹配 → 订阅提醒输出 | +| 55 | 实验记录异常回顾系统 | `experiment_review` | 实验日志 → 记录切分 → 参数抽取 → 异常标记 → 回顾摘要输出 | +| 56 | 科研交付复现审计系统 | `repro_audit` | 交付清单 → 元数据抽取 → 一致性校验 → 缺失项标记 → 审计报告输出 | +| 57 | 模型评测榜单波动监控系统 | `benchmark_watch` | 榜单页面 → 结构化解析 → 版本对比 → 波动识别 → 监控简报输出 | +| 58 | 课程资料问答助手 | `course_qa_helper` | 课程资料 → 文本抽取 → 分段索引 → 问题匹配 → 引用答案输出 | +| 59 | 周课时计划排布系统 | `lesson_scheduler` | 教学要求 → 资源约束读取 → 周计划评分 → 冲突校验 → 排课草案输出 | +| 60 | 作业初稿反馈系统 | `assignment_feedback` | 作业初稿 → 段落解析 → rubric匹配 → 错误归类 → 反馈建议输出 | +| 61 | 班级分层编组系统 | `skill_gap_diagnosis` | 学习记录 → 分层指标构造 → 学员分群 → 班级编组评分 → 编组方案输出 | +| 62 | 岗位面试模拟教练系统 | `interview_coach` | 岗位题库 → 模拟回答记录 → 维度评分 → 弱项识别 → 训练报告输出 | +| 63 | 校园奖助申请缺口预警系统 | `campus_aid_gap_alert` | 申请材料 → 条件规则抽取 → 学生画像匹配 → 缺口识别 → 预警清单输出 | +| 64 | 门急诊分诊整理系统 | `triage_structurer` | 接诊记录 → 字段抽取 → 分诊规则判断 → 风险标签生成 → 摘要输出 | +| 65 | 影像报告随访闭环系统 | `radiology_followup_loop` | 影像报告 → 随访建议抽取 → 患者关联 → 超期判断 → 闭环清单输出 | +| 66 | 药品说明书结构化抽取系统 | `drug_leaflet_extractor` | 药品说明书 → OCR/文本提取 → 字段抽取 → 单位规范化 → 知识条目输出 | +| 67 | 检验样本周转异常预警系统 | `lab_turnaround_alert` | 样本流转记录 → 阶段映射 → 周转时长计算 → 异常打分 → 预警清单输出 | +| 68 | 供应商报价对比系统 | `quote_compare` | 报价单 → 字段标准化 → 条件比对 → 综合评分 → 推荐清单输出 | +| 69 | 企业制度检索助手 | `policy_search_helper` | 制度文档 → 文本分段 → 索引构建 → 问题匹配 → 引用答案输出 | +| 70 | 内部知识库去重整理系统 | `knowledge_cleanup` | 知识文章 → 文本指纹生成 → 相似度比对 → 新鲜度评估 → 整理清单输出 | +| 71 | 播客高光切片系统 | `podcast_highlight` | 播客转写 → 片段切分 → 高光评分 → 标题标签生成 → 切片清单输出 | +| 72 | 品牌物料合规审核系统 | `brand_compliance_review` | 品牌物料 → 文本/元数据抽取 → 规范规则匹配 → 风险标记 → 审核清单输出 | +| 73 | 字幕术语质检系统 | `subtitle_qc` | 字幕文件 → 块级解析 → 术语校验 → 时序检查 → 质检报告输出 | +| 74 | 多渠道内容排期系统 | `content_scheduler` | 活动计划 → 渠道规则映射 → 内容主题分配 → 冲突检查 → 排期表输出 | +| 75 | 媒体资料归档检索系统 | `media_archive_search` | 媒体素材 → 元数据抽取 → 标签生成 → 去重归档 → 检索索引输出 | +| 76 | 产线传感器异常看护系统 | `factory_watch` | 传感器流 → 设备映射 → 异常特征计算 → 告警分级 → 看护清单输出 | +| 77 | 冷链运输越界监控系统 | `cold_chain_watch` | 冷链记录 → 批次关联 → 越界检测 → 风险升级 → 监控报告输出 | +| 78 | 温室种植协同助手 | `greenhouse_assistant` | 温室数据 → 区域映射 → 环境异常判断 → 农事建议生成 → 协同清单输出 | +| 79 | 社区民生热点漂移监测系统 | `community_hotspot_drift` | 民生事件流 → 区域映射 → 问题主题聚合 → 漂移趋势识别 → 治理看板输出 | +| 80 | 交通突发事件简报系统 | `traffic_briefing` | 交通事件流 → 事件归并 → 影响评估 → 优先级排序 → 简报输出 | +| 81 | 城市设施维修调度系统 | `urban_repair_scheduler` | 维修工单 → 位置归并 → 紧急度打分 → 路线调度 → 计划输出 | +| 82 | 许可申报材料审查系统 | `permit_material_review` | 申报材料 → 材料类型识别 → 清单校验 → 缺失项标记 → 审查结果输出 | +| 83 | 市政协同文档检索系统 | `municipal_search` | 政务文档 → 分段索引 → 元数据归一化 → 问题匹配 → 引用结果输出 | +| 84 | 预算执行偏差预警系统 | `budget_variance_alert` | 预算数据 → 科目映射 → 计划实际比对 → 偏差趋势识别 → 预警报告输出 | +| 85 | 企业现金流预测系统 | `cashflow_watch` | 财务数据 → 收支特征构造 → 现金流预测 → 风险判断 → 周报输出 | +| 86 | 电商退货原因挖掘系统 | `return_reason_mining` | 退货记录 → 文本与属性融合 → 原因聚类 → 问题排序 → 改进清单输出 | +| 87 | 门店运营日报生成系统 | `store_daily_digest` | 门店数据 → 日指标汇总 → 异常识别 → 动作项整理 → 日报输出 | +| 88 | 碳排数据采集归集系统 | `carbon_collection` | 碳排数据源 → 字段抽取 → 单位换算 → 口径映射 → 归集台账输出 | +| 89 | 光伏场站告警系统 | `solar_alerting` | 场站数据 → 设备关联 → 发电异常识别 → 告警分级 → 运维清单输出 | +| 90 | 校园碳排报告系统 | `campus_emission_report` | 校园数据 → 排放因子映射 → 分项汇总 → 报告模板填充 → 年报输出 | +| 91 | 数据中心容量与冷却监测系统 | `data_center_watch` | 机房数据 → 机柜映射 → 容量与温度特征计算 → 风险打分 → 监测报告输出 | + +______________________________________________________________________ + +## 第一部分：10个数据清洗与处理应用（1-10） + +### 1. **企业日志解析与结构化系统** + +**现实场景痛点**：服务器日志海量（GB级每日），定位问题困难 + +**发现需求点**： + +- 需要快速从日志中提取特定错误信息 +- 日志格式多样（nginx/apache/应用日志） +- 需要实时处理，避免一次性加载全部日志 + +**解决方案**： + +``` +日志文件流 + → 逐行正则解析提取字段 + → 按错误级别过滤 + → 输出JSON结构化格式 +``` + +**需要的AI应用**： + +- 日志分类：区分业务错误 vs 系统错误 +- 异常检测：识别新型错误模式 + +**SAGE关键创新点**： + +- ✅ 流式处理避免内存溢出 +- ✅ map+filter链式表达简洁 +- ✅ 代码无改动即可扩展到分布式（FlowNetEnvironment） + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/log_parser/ + pipeline.py: + - def run_log_parser_pipeline(log_file, output_path) + - 使用 LocalEnvironment("log_parser") + - 链式操作: from_batch(LogSource) → map(Parser) → map(Filter) → sink(Output) + + operators.py: + - class LogSource(BatchFunction): 逐行读取日志文件 + - class LogParser(MapFunction): 正则解析提取字段（level, timestamp, message等） + - class ErrorFilter(MapFunction): 按错误级别筛选 + - class JsonSink(SinkFunction): 输出JSON行到文件 + + __init__.py: + - 导出 run_log_parser_pipeline 函数 + + README.md: + - 使用说明和配置示例 + +新建: examples/run_log_parser.py + - argparse 接收 --log-file --output-path --error-level参数 + - 调用pipeline.py的main函数 +``` + +**预期收益**：月费3-10万 | **实现周期**：2周 + +______________________________________________________________________ + +### 2. **CSV/Excel数据批量清洗系统** + +**现实场景痛点**：企业数据格式杂乱、缺失值多、类型混乱 + +**发现需求点**： + +- 数据来自不同来源（Excel、CSV、数据库导出） +- 需要统一数据类型、处理缺失、检测异常值 +- 批量处理效率要求高 + +**解决方案**： + +``` +批量CSV/Excel → 读取 → 类型转换 → 缺失值填充 → 异常检测 → 清洗输出 +``` + +**SAGE关键创新点**： + +- ✅ 批处理效率高 +- ✅ 易于添加新的清洗规则（map链式） + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/data_cleaner/ + pipeline.py: + - def run_data_cleaner_pipeline(input_file, output_file, rules_config) + + operators.py: + - class CsvSource(BatchFunction): 读取CSV/Excel + - class TypeConverter(MapFunction): 根据规则转换字段类型 + - class MissingValueFiller(MapFunction): 填充缺失值 + - class AnomalyDetector(MapFunction): 检测异常值 + - class CleanedDataSink(SinkFunction): 输出清洗数据 +``` + +**预期收益**：按数据量计费（0.01-0.1元/行） | **实现周期**：2周 + +______________________________________________________________________ + +### 3. **网页内容抓取与表格提取系统** + +**现实场景痛点**：竞品监测需要定期从网页抓取表格数据 + +**发现需求点**： + +- URL列表来自爬虫或手工维护 +- 需要解析HTML提取表格 +- 需要结构化存储 + +**解决方案**： + +``` +URL列表 → HTTP请求 → HTML解析 → 表格提取 → 结构化 → 数据库入库 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/web_scraper/ + operators.py: + - class UrlSource(BatchFunction): 从CSV读取URL列表 + - class WebScraper(MapFunction): 使用requests+BeautifulSoup抓取+解析 + - class TableExtractor(FlatMapFunction): 拆分多个表格（一个HTML多表） + - class DatabaseSink(SinkFunction): 写入数据库 +``` + +**预期收益**：月费2-8万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 4. **医学文献元数据提取系统** + +**现实场景痛点**：医学文献海量，手动整理元数据浪费时间 + +**发现需求点**： + +- 论文文本或PDF格式 +- 需要提取：作者、标题、摘要、发表日期、关键词等 +- 需要规范化作者格式 + +**解决方案**： + +``` +论文PDF → 文本提取 → 正则+规则提取元数据 → 数据库存储 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/academic_metadata/ + operators.py: + - class PdfSource(BatchFunction): 读取PDF文件 + - class TextExtractor(MapFunction): 提取文本（使用PyPDF2） + - class MetadataExtractor(MapFunction): 正则+规则提取作者/标题/摘要 + - class AuthorNormalizer(MapFunction): 规范化作者格式 + - class MetadataSink(SinkFunction): 输出JSON到数据库 +``` + +**预期收益**：月费3-12万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 5. **客户数据去重系统** + +**现实场景痛点**：CRM系统数据重复，影响营销和分析 + +**发现需求点**： + +- 同一客户可能有多条记录（电话/邮箱重复） +- 需要按多个维度去重 +- 需要保留完整的客户信息 + +**解决方案**： + +``` +客户数据 → 按电话号码分组 → 计算相似度 → 标记重复 → 输出去重结果 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/customer_deduplication/ + operators.py: + - class CustomerSource(BatchFunction): 读取客户数据 + - class SimilarityCalculator(MapFunction): 计算编辑距离（电话/邮箱） + - class DuplicateDetector(MapFunction): 规则判断是否重复 + - class DeduplicationSink(SinkFunction): 输出重复记录对 +``` + +**预期收益**：按去重记录计费 | **实现周期**：2周 + +______________________________________________________________________ + +### 6. **财务凭证自动分类系统** + +**现实场景痛点**：会计手动分类凭证工作量大 + +**发现需求点**： + +- 凭证来自扫描件（需要OCR）或PDF +- 需要分类：收入/支出/转账/其他 +- 分类规则基于金额、对方、摘要等 + +**解决方案**： + +``` +凭证图片/PDF → OCR识别 → 提取关键信息 → 规则分类 → 输出分类结果 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/voucher_classifier/ + operators.py: + - class VoucherSource(BatchFunction): 读取凭证图片/PDF + - class OcrExtractor(MapFunction): 使用pytesseract进行OCR + - class FieldExtractor(MapFunction): 提取金额、对方、摘要 + - class RuleClassifier(MapFunction): 应用分类规则 + - class ClassificationSink(SinkFunction): 输出分类结果 +``` + +**预期收益**：月费5-15万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 7. **简历数据标准化系统** + +**现实场景痛点**：招聘系统收到格式多样的简历，难以对比 + +**发现需求点**： + +- 简历格式多样（PDF、Word、图片） +- 需要提取：姓名、电话、邮箱、工作经历、教育背景 +- 需要标准化格式和日期 + +**解决方案**： + +``` +简历文件 → 格式识别和转换 → 信息提取 → 标准化 → 结构化简历数据 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/resume_parser/ + operators.py: + - class ResumeSource(BatchFunction): 读取简历文件（PDF/Word） + - class TextExtractor(MapFunction): 提取纯文本 + - class InfoExtractor(MapFunction): 提取个人信息、工作经历等 + - class DateNormalizer(MapFunction): 标准化日期格式 + - class ResumeSink(SinkFunction): 输出JSON结构化数据 +``` + +**预期收益**：月费2-8万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 8. **多平台商品数据同步系统** + +**现实场景痛点**：多渠道销售商品信息不一致 + +**发现需求点**： + +- 不同平台的字段名不同 +- 需要字段映射和转换 +- 需要数据校验 + +**解决方案**： + +``` +A平台商品数据 → 读取 → 字段映射 → 数据校验 → 写入B平台 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/product_sync/ + operators.py: + - class ProductSource(BatchFunction): 从A平台读取商品（API或数据库） + - class FieldMapper(MapFunction): 字段映射（A字段→B字段） + - class DataValidator(MapFunction): 校验数据有效性 + - class PlatformSink(SinkFunction): 写入B平台API/数据库 +``` + +**预期收益**：月费3-10万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 9. **企业知识库自动分类系统** + +**现实场景痛点**：公司文档海量，难以分类管理 + +**发现需求点**： + +- 文档来自不同部门、格式多样 +- 需要按主题/部门自动分类 +- 需要支持模糊查询 + +**解决方案**： + +``` +文档文件 → 文本提取 → 分词 → TF-IDF特征 → 分类 → 保存分类结果 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/doc_classifier/ + operators.py: + - class DocSource(BatchFunction): 读取文档文件 + - class TextExtractor(MapFunction): 提取纯文本（支持PDF/Word/txt） + - class Tokenizer(FlatMapFunction): 分词（使用jieba） + - class FeatureExtractor(MapFunction): 计算TF-IDF + - class Classifier(MapFunction): 基于规则或预训练模型分类 + - class DocSink(SinkFunction): 输出分类标签 +``` + +**预期收益**：月费5-15万 | **实现周期**：3周 + +______________________________________________________________________ + +### 10. **客户反馈关键词提取系统** + +**现实场景痛点**：客户评价成千上万，难以快速发现问题 + +**发现需求点**： + +- 反馈文本来自评价、投诉、问卷等多个渠道 +- 需要提取高频关键词 +- 需要统计词频和情感倾向 + +**解决方案**： + +``` +反馈文本 → 文本清洗 → 分词 → 关键词提取 → 统计输出 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/feedback_analyzer/ + operators.py: + - class FeedbackSource(BatchFunction): 读取反馈数据 + - class TextCleaner(MapFunction): 清洗文本（去标点、转小写等） + - class Tokenizer(FlatMapFunction): 分词 + - class KeywordExtractor(MapFunction): 提取关键词（TF-IDF/TextRank） + - class StatisticsSink(SinkFunction): 统计词频并输出排序结果 +``` + +**预期收益**：月费3-10万 | **实现周期**：2.5周 + +______________________________________________________________________ + +## 第二部分：11-20 文本与法律处理应用 + +### 11. **法律文案模板匹配系统** + +**现实场景痛点**：律师编写合同耗时，需要从模板库快速查找相似案例 + +**发现需求点**： + +- 律师描述需求，需要快速匹配相关模板 +- 需要计算需求与模板的相似度 +- 模板库内容丰富但难以检索 + +**解决方案**： + +``` +客户需求描述 → 关键词提取 → 与模板库余弦相似度计算 → 返回匹配模板 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/contract_matcher/ + operators.py: + - class RequirementSource(BatchFunction): 读取需求描述 + - class KeywordExtractor(MapFunction): 提取关键词 + - class TemplateMatcher(MapFunction): 计算相似度（余弦相似度） + - class MatchSink(SinkFunction): 输出匹配结果 +``` + +**预期收益**：月费5-15万 | **实现周期**：2周 + +______________________________________________________________________ + +### 12. **新闻聚合与去重系统** + +**现实场景痛点**：新闻网站和研究团队接入多个 RSS 或资讯源后，重复内容太多，人工整理会浪费大量时间。 + +**发现需求点**： + +- 多个 RSS 源内容高度重复 +- 需要快速识别重复内容 +- 需要聚合到统一平台给下游监测或分析系统使用 + +**解决方案**： + +``` +RSS源 → HTML解析 → 内容指纹计算 → 去重 → 输出聚合新闻 +``` + +**需要的AI应用**： + +- 新闻去重识别 +- 重复版本聚合 + +**AI应用关键解决问题的创新点**： + +- 该条目与原生 Article Monitoring 存在较高雷同风险，因此这里明确把自身边界限定在“多源聚合和去重入口层”，而不是做持续监测、主题推荐和告警 +- SAGE适合持续接入多源新闻流，并用 map/filter 串起抽取、指纹、去重和下游输出 +- 相比通用任务编排框架，更适合做稳定的新闻数据治理流水线 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/news_aggregator/ + pipeline.py: + - def run_news_aggregator_pipeline(feed_file, output_file) + operators.py: + - class RssSource(BatchFunction): 抓取RSS源 + - class NewsExtractor(MapFunction): 提取标题、摘要、URL + - class FingerprintCalculator(MapFunction): 计算SimHash指纹 + - class DeduplicationFilter(MapFunction): 去重判断 + - class NewsSink(SinkFunction): 输出聚合新闻到数据库 +新建: examples/run_news_aggregator.py +``` + +**预期收益**：月费2-8万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 13. **客服工单自动路由系统** + +**现实场景痛点**：工单分配不合理，客服响应慢 + +**发现需求点**： + +- 工单数量多，需要自动分配 +- 需要按客服负载均衡分配 +- 需要优先级排序 + +**解决方案**： + +``` +客服工单流 → 工单分类 → 优先级评分 → 客服负载计算 → 最优分配 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/ticket_router/ + operators.py: + - class TicketSource(BatchFunction): 读取工单 + - class TicketParser(MapFunction): 提取工单信息 + - class TicketClassifier(MapFunction): 分类（售前/售后/投诉等） + - class PriorityScorer(MapFunction): 优先级评分 + - class LoadBalancer(MapFunction): 按客服负载分配 + - class NotificationSink(SinkFunction): 发送分配通知 +``` + +**预期收益**：月费5-20万 | **实现周期**：3周 + +______________________________________________________________________ + +### 14. **社交媒体内容审核系统** + +**现实场景痛点**：平台需要快速删除违规内容 + +**发现需求点**： + +- 用户发布内容频繁，需要实时审核 +- 需要识别敏感词、违规图片等 +- 需要快速删除或隔离 + +**解决方案**： + +``` +用户发布内容流 → 文本提取 → 敏感词匹配 → 违规评分 → 隔离违规内容 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/content_moderation/ + operators.py: + - class ContentSource(BatchFunction): 读取用户发布内容 + - class TextExtractor(MapFunction): 提取文本 + - class Tokenizer(FlatMapFunction): 分词 + - class SensitiveFilter(MapFunction): 敏感词匹配 + - class ViolationScorer(MapFunction): 违规评分 + - class ModerationSink(SinkFunction): 输出违规内容 +``` + +**预期收益**：按审核量计费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 15. **合同条款风险识别系统** + +**现实场景痛点**：商务人员逐条阅读合同条款耗时 + +**发现需求点**： + +- 合同条款众多、复杂 +- 需要快速识别风险条款 +- 需要提示注意事项 + +**解决方案**： + +``` +合同文件 → 文本解析 → 条款拆分 → 风险规则匹配 → 输出风险提示 +``` + +**具体实现计划**： + +``` +新建: apps/src/sage/apps/contract_risk/ + operators.py: + - class ContractSource(BatchFunction): 读取合同文件 + - class TextExtractor(MapFunction): 提取合同文本 + - class ClauseSegmenter(FlatMapFunction): 拆分条款 + - class RiskScorer(MapFunction): 规则评分引擎 + - class RiskReportSink(SinkFunction): 输出风险报告 +``` + +**预期收益**：按合同数计费 | **实现周期**：3周 + +______________________________________________________________________ + +### 16. **用户行为事件采集与分类系统** + +**现实场景痛点**：产品、运营和增长团队无法持续获得结构化用户行为数据，导致漏斗分析和功能决策滞后。 + +**发现需求点**： + +- 埋点格式不统一，事件名和字段定义经常变化 +- 需要按用户、页面、事件类型做连续统计 +- 希望先本地跑通，再平滑扩展到更大吞吐 + +**解决方案**： + +``` +事件流 → 事件校验 → 字段规范化 → 用户行为分类 → 分流写入分析表 +``` + +**需要的AI应用**： + +- 事件语义归类 +- 异常行为模式识别 + +**AI应用关键解决问题的创新点**： + +- SAGE天然适合持续事件流，不需要把事件处理拆成离散任务节点 +- map/filter/flatmap可以把埋点清洗、分类、分流串成单一数据链路 +- 后续切到FlowNetEnvironment时，处理逻辑不需要重写 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/user_behavior_analytics/ + pipeline.py: + - def run_user_behavior_analytics_pipeline(input_path, output_dir) + operators.py: + - class EventSource(BatchFunction): 读取埋点日志或事件CSV + - class EventValidator(MapFunction): 校验字段完整性 + - class EventNormalizer(MapFunction): 统一事件结构 + - class BehaviorClassifier(MapFunction): 归类浏览/点击/转化等行为 + - class EventSink(SinkFunction): 写入结构化行为结果 + __init__.py: + - 导出 run_user_behavior_analytics_pipeline + README.md: + - 说明事件格式、字段映射、运行方式 +新建: examples/run_user_behavior_analytics.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2周 + +______________________________________________________________________ + +### 17. **库存异常告警系统** + +**现实场景痛点**：库存过多占压现金，库存过少导致断货，很多企业仍靠人工巡检库存报表。 + +**发现需求点**： + +- 需要把每日库存变动快速转成异常提醒 +- 需要把安全库存、补货阈值、周转率规则沉淀下来 +- 需要多仓库、多SKU统一处理 + +**解决方案**： + +``` +库存快照 → 特征计算 → 阈值规则比对 → 异常分级 → 告警输出 +``` + +**需要的AI应用**： + +- 库存异常识别 +- 告警优先级建议 + +**AI应用关键解决问题的创新点**： + +- SAGE可以把库存数据当连续数据流处理，而不是每天一次性离线脚本 +- 规则变更只需要替换MapFunction，不需要改整条流程 +- 相比通用工作流框架，更适合做高频、可复用的流式判断 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/inventory_alert/ + pipeline.py: + - def run_inventory_alert_pipeline(input_file, output_file, config_path) + operators.py: + - class InventorySource(BatchFunction): 读取库存报表 + - class InventoryFeatureBuilder(MapFunction): 计算周转天数、安全库存差值 + - class InventoryAnomalyScorer(MapFunction): 基于规则打分 + - class AlertLevelMapper(MapFunction): 映射高/中/低风险等级 + - class AlertSink(SinkFunction): 输出告警清单 +新建: examples/run_inventory_alert.py +``` + +**预期收益**：月费2-8万 | **实现周期**：2周 + +______________________________________________________________________ + +### 18. **生产质量缺陷过滤系统** + +**现实场景痛点**：质检报告描述混乱，同一类缺陷被多人写成不同表述，后续统计和追责困难。 + +**发现需求点**： + +- 需要把自由文本缺陷描述转成标准缺陷类型 +- 一个报告里往往包含多个缺陷点 +- 需要按产线、批次、工位输出缺陷记录 + +**解决方案**： + +``` +质检报告 → 文本抽取 → 缺陷拆分 → 标准化分类 → 缺陷库入库 +``` + +**需要的AI应用**： + +- 缺陷模式归类 +- 缺陷严重度评分 + +**AI应用关键解决问题的创新点**： + +- SAGE的flatmap很适合一份报告拆成多条缺陷记录 +- 处理链路天然透明，质检部门能看清每一步规则 +- 比黑箱式Agent流程更适合工业质量场景的可审计要求 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/quality_defect_filter/ + pipeline.py: + - def run_quality_defect_filter_pipeline(report_path, output_path) + operators.py: + - class QualityReportSource(BatchFunction): 读取质检报告 + - class DefectTextExtractor(MapFunction): 提取缺陷描述 + - class DefectSplitter(FlatMapFunction): 拆成单条缺陷 + - class DefectStandardizer(MapFunction): 归一化缺陷类型 + - class DefectSeverityScorer(MapFunction): 严重度打分 + - class DefectSink(SinkFunction): 输出标准缺陷记录 +新建: examples/run_quality_defect_filter.py +``` + +**预期收益**：月费3-12万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 19. **用户权限变更审计系统** + +**现实场景痛点**：权限提升、越权授权和敏感角色调整经常在事后才被发现，审计成本高。 + +**发现需求点**： + +- 需要持续收集权限变更日志 +- 需要识别高风险角色和敏感资源授权 +- 需要形成可追踪的审计报告 + +**解决方案**： + +``` +权限变更日志 → 解析 → 敏感操作识别 → 风险打分 → 审计报告输出 +``` + +**需要的AI应用**： + +- 权限变更风险识别 +- 审计告警排序 + +**AI应用关键解决问题的创新点**： + +- SAGE能直接处理权限事件流，适合持续审计而不是周期性批跑 +- 数据流链路清晰，满足审计解释性需求 +- 相比以对话为核心的框架，更适合做确定性安全规则执行 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/permission_audit/ + pipeline.py: + - def run_permission_audit_pipeline(log_file, output_dir) + operators.py: + - class AuditLogSource(BatchFunction): 读取权限变更日志 + - class AuditLogParser(MapFunction): 解析账号、角色、资源、动作 + - class SensitiveActionDetector(MapFunction): 识别高风险权限动作 + - class AuditRiskScorer(MapFunction): 风险评分 + - class AuditSink(SinkFunction): 输出审计结果 +新建: examples/run_permission_audit.py +``` + +**预期收益**：月费5-15万 | **实现周期**：2周 + +______________________________________________________________________ + +### 20. **订单异常检测系统** + +**现实场景痛点**：电商和零售平台存在刷单、撞库下单、异常退款等风险，人工排查成本很高。 + +**发现需求点**： + +- 需要从订单、用户、地址、支付方式中提取风险特征 +- 需要快速识别可疑订单进行人工复核 +- 需要可调整的规则体系 + +**解决方案**： + +``` +订单数据 → 特征构造 → 风险规则评分 → 异常筛选 → 输出复核队列 +``` + +**需要的AI应用**： + +- 异常交易识别 +- 复核优先级排序 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把订单当连续流处理，异常规则可以逐层叠加 +- 每个评分步骤都能单独解释和替换 +- 后续接入更多来源时，只需要扩展MapFunction而不是重写控制流 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/order_anomaly_detector/ + pipeline.py: + - def run_order_anomaly_detector_pipeline(order_file, output_file) + operators.py: + - class OrderSource(BatchFunction): 读取订单数据 + - class OrderFeatureBuilder(MapFunction): 构造金额、频次、地址等特征 + - class OrderRiskScorer(MapFunction): 规则评分 + - class OrderAnomalyFilter(MapFunction): 标记异常订单 + - class AnomalySink(SinkFunction): 输出异常订单与原因 +新建: examples/run_order_anomaly_detector.py +``` + +**预期收益**：按交易额分成 | **实现周期**：2.5周 + +______________________________________________________________________ + +## 第三部分：21-35 规则引擎与外部数据集成应用 + +### 21. **员工出勤异常预警系统** + +**现实场景痛点**：企业HR往往在月末才发现迟到、旷工和异常排班问题，处理滞后。 + +**发现需求点**： + +- 打卡数据格式不统一 +- 需要按员工、部门、班次做快速比对 +- 需要自动触发人事预警 + +**解决方案**： + +``` +打卡记录 → 班次映射 → 出勤规则校验 → 异常识别 → 预警输出 +``` + +**需要的AI应用**： + +- 出勤异常识别 +- 人事处理优先级建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合按记录流连续判断，不必等待月末汇总 +- keyby和map的组合适合做员工维度处理 +- 规则透明，适合HR和法务复核 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/attendance_alert/ + pipeline.py: + - def run_attendance_alert_pipeline(clock_file, schedule_file, output_file) + operators.py: + - class AttendanceSource(BatchFunction) + - class ScheduleMatcher(MapFunction) + - class AttendanceAnomalyDetector(MapFunction) + - class AttendanceAlertSink(SinkFunction) +新建: examples/run_attendance_alert.py +``` + +**预期收益**：月费2-5万 | **实现周期**：2周 + +______________________________________________________________________ + +### 22. **销售机会评分系统** + +**现实场景痛点**：销售线索过多，团队无法把精力集中在高转化机会。 + +**发现需求点**： + +- 需要对线索做自动评分和排序 +- 需要根据行业、规模、活跃度、历史触达情况综合判断 +- 需要输出给CRM或销售看板 + +**解决方案**： + +``` +销售线索 → 特征抽取 → 规则评分 → 优先级排序 → 分配给销售 +``` + +**需要的AI应用**： + +- 商机评分 +- 销售分派建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合在统一流里连续完成清洗、评分和下游分派 +- 当规则频繁调整时，只需替换评分算子 +- 相比以交互推理为主的框架，更适合高吞吐商机处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/lead_scoring/ + pipeline.py: + - def run_lead_scoring_pipeline(lead_file, output_file) + operators.py: + - class LeadSource(BatchFunction) + - class LeadFeatureBuilder(MapFunction) + - class OpportunityScorer(MapFunction) + - class SalesAssigner(MapFunction) + - class OpportunitySink(SinkFunction) + 新建: examples/run_lead_scoring.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2周 + +______________________________________________________________________ + +### 23. **实时汇率转换与套利检测系统** + +**现实场景痛点**：跨境交易和财务团队需要快速换算汇率，并及时发现短时套利空间。 + +**发现需求点**： + +- 汇率变化频繁 +- 需要接入多个汇率源做交叉比对 +- 需要把套利信号快速输出给交易或风控团队 + +**解决方案**： + +``` +交易订单 → 汇率抓取 → 汇率转换 → 套利规则打分 → 告警输出 +``` + +**需要的AI应用**： + +- 汇率异常识别 +- 套利机会检测 + +**AI应用关键解决问题的创新点**： + +- SAGE擅长处理持续到来的交易记录和报价更新 +- 将HTTP抓取、转换、打分串成稳定流水线，比离散脚本更可靠 +- 后续增加更多报价源时，可以继续追加MapFunction，不需要改架构 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/arbitrage_detector/ + pipeline.py: + - def run_arbitrage_detector_pipeline(order_file, output_file, provider_config) + operators.py: + - class OrderSource(BatchFunction): 读取交易订单 + - class ExchangeRateFetcher(MapFunction): 使用httpx调用汇率API + - class ConversionCalculator(MapFunction): 汇率转换计算 + - class ArbitrageMatcher(MapFunction): 检测套利机会 + - class ArbitrageSink(SinkFunction): 输出套利告警 +新建: examples/run_arbitrage_detector.py +``` + +**预期收益**：月费5-20万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 24. **天气数据驱动的销量预测系统** + +**现实场景痛点**：零售、生鲜、饮品等行业销量受天气显著影响，库存经常错配。 + +**发现需求点**： + +- 需要把销量历史和天气数据联合分析 +- 需要做门店级别预测 +- 需要输出补货建议 + +**解决方案**： + +``` +销量历史 → 天气数据抓取 → 特征融合 → 销量预测 → 库存建议 +``` + +**需要的AI应用**： + +- 天气影响销量预测 +- 补货建议生成 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把历史销量、天气API结果和门店配置统一进同一处理流 +- 可以逐步替换预测逻辑，而不改外层管道 +- 对比重型编排框架，更适合做高频批流混合预测 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/weather_sales_forecast/ + pipeline.py: + - def run_weather_sales_forecast_pipeline(sales_file, city_config, output_file) + operators.py: + - class SalesHistorySource(BatchFunction) + - class WeatherFetcher(MapFunction) + - class WeatherSalesFeatureBuilder(MapFunction) + - class SalesForecaster(MapFunction) + - class RestockSuggestionSink(SinkFunction) +新建: examples/run_weather_sales_forecast.py +``` + +**预期收益**：月费2-8万 | **实现周期**：3周 + +______________________________________________________________________ + +### 25. **地理位置智能推荐系统** + +**现实场景痛点**：O2O、连锁门店和本地生活平台难以根据用户位置及时推荐合适服务。 + +**发现需求点**： + +- 需要实时获取附近门店信息 +- 需要结合用户偏好做个性化推荐 +- 需要把推荐结果推送到APP或短信系统 + +**解决方案**： + +``` +用户位置 → 地图API查询 → 门店候选集 → 偏好打分 → 推荐结果输出 +``` + +**需要的AI应用**： + +- 位置感知推荐 +- 推荐排序 + +**AI应用关键解决问题的创新点**： + +- SAGE可以把位置流、偏好数据和门店信息做连续处理 +- 规则和排序逻辑独立在算子中，便于实验和替换 +- 相比以对话代理为中心的框架，更适合高频推荐流水线 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/geo_recommendation/ + pipeline.py: + - def run_geo_recommendation_pipeline(user_file, output_file) + operators.py: + - class UserLocationSource(BatchFunction) + - class NearbyStoreFetcher(MapFunction) + - class PreferenceMatcher(MapFunction) + - class RecommendationRanker(MapFunction) + - class RecommendationSink(SinkFunction) + 新建: examples/run_geo_recommendation.py +``` + +**预期收益**：按推荐转化计费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 26. **企业信用评估系统** + +**现实场景痛点**：B2B合作前需要快速识别企业诉讼、失信、经营异常等风险信息。 + +**发现需求点**： + +- 需要自动抓取企业公开信息 +- 需要提炼关键风险因子 +- 需要统一输出信用报告 + +**解决方案**： + +``` +企业名单 → 企业信息抓取 → 风险因子抽取 → 信用评分 → 报告输出 +``` + +**需要的AI应用**： + +- 企业风险评分 +- 尽调优先级建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把企业数据抓取和评分做成稳定可追踪的数据流 +- 每个风险因子可以拆成独立算子，便于审计和调优 +- 比一次性脚本更适合长期、批量尽调场景 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/company_credit/ + pipeline.py: + - def run_company_credit_pipeline(company_file, output_file, api_config) + operators.py: + - class CompanySource(BatchFunction) + - class CompanyInfoFetcher(MapFunction) + - class RiskFactorExtractor(MapFunction) + - class CreditScorer(MapFunction) + - class CreditReportSink(SinkFunction) +新建: examples/run_company_credit.py +``` + +**预期收益**：按查询数计费 | **实现周期**：2周 + +______________________________________________________________________ + +### 27. **电影库存与排片优化系统** + +**现实场景痛点**：影院排片依赖经验，热门片次分配不准，导致上座率和单厅收益不稳定。 + +**发现需求点**： + +- 需要结合档期、热度、评分、历史上座率决定排片 +- 需要快速生成不同档位方案 +- 需要输出可执行的排片建议 + +**解决方案**： + +``` +档期数据 → 电影热度抓取 → 收益特征计算 → 排片评分 → 方案输出 +``` + +**需要的AI应用**： + +- 场次收益预测 +- 排片建议生成 + +**AI应用关键解决问题的创新点**： + +- SAGE把多源数据汇总和评分放在同一流式管道中处理 +- 排片规则可替换，方便按影院策略调整 +- 相比面向交互的框架，更适合高吞吐、批量场次优化 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/movie_scheduling_optimizer/ + pipeline.py: + - def run_movie_scheduling_optimizer_pipeline(schedule_file, output_file) + operators.py: + - class ScheduleSource(BatchFunction) + - class MovieHeatFetcher(MapFunction) + - class RevenueFeatureBuilder(MapFunction) + - class SchedulingScorer(MapFunction) + - class SchedulingSink(SinkFunction) + 新建: examples/run_movie_scheduling_optimizer.py +``` + +**预期收益**：月费5-15万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 28. **房产智能估价系统** + +**现实场景痛点**：中介和经纪人需要快速给出房屋估值，但市场对比数据分散。 + +**发现需求点**： + +- 需要抓取周边成交、挂牌和地段信息 +- 需要做可解释的估价 +- 需要快速形成报价说明 + +**解决方案**： + +``` +房产信息 → 周边房源抓取 → 特征构造 → 估价计算 → 报价说明输出 +``` + +**需要的AI应用**： + +- 房价估计 +- 价格偏离风险识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多源房产数据的拼装与连续处理 +- 估价逻辑可拆分为可解释算子，便于经纪人复核 +- 相比通用流程引擎，更适合高重复、强规则的估价流水线 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/real_estate_valuation/ + pipeline.py: + - def run_real_estate_valuation_pipeline(property_file, output_file) + operators.py: + - class PropertySource(BatchFunction) + - class NearbyListingFetcher(MapFunction) + - class ValuationFeatureBuilder(MapFunction) + - class ValuationCalculator(MapFunction) + - class ValuationSink(SinkFunction) +新建: examples/run_real_estate_valuation.py +``` + +**预期收益**：按估价次数计费 | **实现周期**：3周 + +______________________________________________________________________ + +### 29. **多维用户信用评分系统** + +**现实场景痛点**：放贷、租赁和高价值服务开通前，需要快速评估用户履约风险。 + +**发现需求点**： + +- 需要融合征信、电商、运营商、设备等多维信息 +- 需要可配置的评分规则 +- 需要批量输出审批辅助结果 + +**解决方案**： + +``` +用户清单 → 多源信息抓取 → 特征融合 → 信用评分 → 审批结果输出 +``` + +**需要的AI应用**： + +- 用户信用评分 +- 风险分层 + +**AI应用关键解决问题的创新点**： + +- SAGE适合将多源数据依次接入、规整和评分 +- 可以先本地做批处理，后续放大到分布式评分 +- 对比面向Agent决策的框架，更适合标准化风控流水线 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/multi_factor_credit_score/ + pipeline.py: + - def run_multi_factor_credit_score_pipeline(user_file, output_file, provider_config) + operators.py: + - class UserSource(BatchFunction) + - class MultiSourceFetcher(MapFunction) + - class CreditFeatureFusion(MapFunction) + - class UserCreditScorer(MapFunction) + - class CreditDecisionSink(SinkFunction) + 新建: examples/run_multi_factor_credit_score.py +``` + +**预期收益**：按评分次数计费 | **实现周期**：3周 + +______________________________________________________________________ + +### 30. **物流成本优化系统** + +**现实场景痛点**：物流企业或商家经常无法在时效和成本之间找到最优方案。 + +**发现需求点**： + +- 需要对接多家运费和路线接口 +- 需要综合重量、距离、时效、破损率等因素 +- 需要批量给订单推荐承运商 + +**解决方案**： + +``` +订单数据 → 物流报价抓取 → 成本时效比对 → 方案打分 → 推荐输出 +``` + +**需要的AI应用**： + +- 物流方案评分 +- 承运商推荐 + +**AI应用关键解决问题的创新点**： + +- SAGE适合高批量订单逐条评分和分流 +- 新接入物流商时只需新增一个抓取或转换算子 +- 比脚本拼接式集成更稳定，也比交互式框架更适合流水处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/logistics_cost_optimizer/ + pipeline.py: + - def run_logistics_cost_optimizer_pipeline(order_file, output_file) + operators.py: + - class LogisticsOrderSource(BatchFunction) + - class FreightQuoteFetcher(MapFunction) + - class RouteOptionBuilder(MapFunction) + - class LogisticsScorer(MapFunction) + - class LogisticsSink(SinkFunction) + 新建: examples/run_logistics_cost_optimizer.py +``` + +**预期收益**：按省钱额度分成 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 31. **医疗挂号优化系统** + +**现实场景痛点**：医院挂号高峰时，患者和号源匹配效率低，优质号源浪费严重。 + +**发现需求点**： + +- 需要动态抓取医生排班和空余号源 +- 需要按科室、病情描述、距离做匹配 +- 需要输出排队和改约建议 + +**解决方案**： + +``` +挂号请求 → 号源抓取 → 医患匹配评分 → 挂号建议 → 通知输出 +``` + +**需要的AI应用**： + +- 就诊匹配推荐 +- 号源调度建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把持续到来的挂号请求与实时号源数据组合处理 +- 管道结构清晰，方便医院IT部门接入现有系统 +- 对比通用工作流，更适合稳定、高频的结构化处理任务 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/medical_registration_optimizer/ + pipeline.py: + - def run_medical_registration_optimizer_pipeline(request_file, output_file) + operators.py: + - class RegistrationRequestSource(BatchFunction) + - class DoctorSlotFetcher(MapFunction) + - class PatientDoctorMatcher(MapFunction) + - class RegistrationPlanBuilder(MapFunction) + - class RegistrationSink(SinkFunction) +新建: examples/run_medical_registration_optimizer.py +``` + +**预期收益**：月费5-20万 | **实现周期**：3周 + +______________________________________________________________________ + +### 32. **展会热度分析系统** + +**现实场景痛点**：展会组织方无法及时判断展位热度和人流分布，招商与定价缺少依据。 + +**发现需求点**： + +- 需要持续接入客流数据 +- 需要按区域、展位、时段输出热度 +- 需要对异常拥堵做提醒 + +**解决方案**： + +``` +客流数据 → 区域映射 → 热度特征计算 → 排名输出 → 拥堵提醒 +``` + +**需要的AI应用**： + +- 区域热度识别 +- 展位价值排序 + +**AI应用关键解决问题的创新点**： + +- SAGE的流式管道天然适合场内连续客流处理 +- 排名与告警逻辑可以解耦为多个算子 +- 相比传统BI离线报表，更适合现场实时决策 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/exhibition_heatmap/ + pipeline.py: + - def run_exhibition_heatmap_pipeline(flow_file, output_file) + operators.py: + - class VisitorFlowSource(BatchFunction) + - class ZoneMapper(MapFunction) + - class HeatScoreCalculator(MapFunction) + - class CongestionDetector(MapFunction) + - class HeatmapSink(SinkFunction) +新建: examples/run_exhibition_heatmap.py +``` + +**预期收益**：按场次计费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 33. **宿舍能耗监测优化系统** + +**现实场景痛点**：学校宿舍或园区公寓用电波动大，异常耗电和违规电器难以及时发现。 + +**发现需求点**： + +- 需要按宿舍持续采集能耗 +- 需要比对同楼层、同季节、同时间段均值 +- 需要输出节能建议和异常告警 + +**解决方案**： + +``` +能耗数据 → 宿舍映射 → 基线对比 → 异常判断 → 建议输出 +``` + +**需要的AI应用**： + +- 能耗异常识别 +- 节能建议生成 + +**AI应用关键解决问题的创新点**： + +- SAGE适合连续能耗流的低成本处理 +- keyby按宿舍分组后可自然衔接各类规则算子 +- 对比重型工业平台，更适合校园级轻量部署 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/dorm_energy_optimizer/ + pipeline.py: + - def run_dorm_energy_optimizer_pipeline(meter_file, output_file) + operators.py: + - class MeterSource(BatchFunction) + - class DormMapper(MapFunction) + - class EnergyBaselineComparer(MapFunction) + - class EnergyAnomalyDetector(MapFunction) + - class EnergyAdviceSink(SinkFunction) +新建: examples/run_dorm_energy_optimizer.py +``` + +**预期收益**：月费2-8万 | **实现周期**：2周 + +______________________________________________________________________ + +### 34. **餐厅菜品销售分析系统** + +**现实场景痛点**：餐厅难以同时兼顾销量、毛利和库存，菜单优化经常靠经验。 + +**发现需求点**： + +- 需要按菜品持续统计销量和毛利 +- 需要结合库存、损耗和时段偏好分析 +- 需要输出菜单调整建议 + +**解决方案**： + +``` +销售订单 → 菜品拆分 → 库存关联 → 利润计算 → 菜单建议输出 +``` + +**需要的AI应用**： + +- 菜品销售评分 +- 菜单优化建议 + +**AI应用关键解决问题的创新点**： + +- SAGE的flatmap适合把订单拆成菜品级记录 +- 多个评分步骤可以线性叠加，逻辑清晰 +- 相比纯报表工具，更适合形成可执行的自动化处理链路 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/restaurant_sales_analysis/ + pipeline.py: + - def run_restaurant_sales_analysis_pipeline(order_file, inventory_file, output_file) + operators.py: + - class RestaurantOrderSource(BatchFunction) + - class DishSplitter(FlatMapFunction) + - class InventoryJoiner(MapFunction) + - class MenuProfitScorer(MapFunction) + - class MenuAdviceSink(SinkFunction) + 新建: examples/run_restaurant_sales_analysis.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 35. **仓库货位优化系统** + +**现实场景痛点**：仓库热门SKU摆放不合理，拣货路径长，仓内作业效率低。 + +**发现需求点**： + +- 需要分析历史拣货频率和货位距离 +- 需要找出高频SKU的最优位置 +- 需要输出可执行的调整清单 + +**解决方案**： + +``` +拣货历史 → 热度统计 → 距离成本计算 → 货位评分 → 调整方案输出 +``` + +**需要的AI应用**： + +- 热门SKU识别 +- 货位调整建议 + +**AI应用关键解决问题的创新点**： + +- SAGE可以持续处理拣货流水，不需要频繁导出再离线分析 +- 评分逻辑天然模块化，适合仓储业务迭代 +- 相比通用自动化平台，更适合高频结构化物流数据 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/warehouse_slot_optimizer/ + pipeline.py: + - def run_warehouse_slot_optimizer_pipeline(pick_file, output_file) + operators.py: + - class PickingHistorySource(BatchFunction) + - class SlotHeatCalculator(MapFunction) + - class DistanceCostBuilder(MapFunction) + - class SlotOptimizer(MapFunction) + - class SlotPlanSink(SinkFunction) +新建: examples/run_warehouse_slot_optimizer.py +``` + +**预期收益**：月费3-12万 | **实现周期**：2.5周 + +______________________________________________________________________ + +## 第四部分：36-52 文本处理、业务自动化与数据管理应用 + +### 36. **医学论文分类系统** + +**现实场景痛点**：研究机构和医学信息团队要面对海量论文，手工分类速度跟不上新增文献。 + +**发现需求点**： + +- 需要按学科、病种、研究方法快速归类 +- 需要支持批量处理新文献 +- 需要把分类结果写回检索系统 + +**解决方案**： + +``` +论文文本 → 分词与关键词提取 → 相似度分类 → 结果输出 +``` + +**需要的AI应用**： + +- 论文主题分类 +- 学科标签推荐 + +**AI应用关键解决问题的创新点**： + +- SAGE适合批量论文流的连续分类 +- 可以把文本提取、关键词抽取、分类拆成清晰算子 +- 相比通用DAG框架，更适合可持续追加的新文献处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/paper_classifier/ + pipeline.py: + - def run_paper_classifier_pipeline(input_path, output_path) + operators.py: + - class PaperSource(BatchFunction) + - class PaperKeywordExtractor(MapFunction) + - class PaperTopicClassifier(MapFunction) + - class PaperClassificationSink(SinkFunction) +新建: examples/run_paper_classifier.py +``` + +**预期收益**：月费5-15万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 37. **供应商评价数据标准化系统** + +**现实场景痛点**：采购、质量、交付团队各自维护供应商评价表，字段和评分标准不统一。 + +**发现需求点**： + +- 需要统一字段和评分口径 +- 需要处理多表来源和重复记录 +- 需要输出统一的供应商评价主表 + +**解决方案**： + +``` +评价数据 → 字段映射 → 评分标准统一 → 重复合并 → 主表输出 +``` + +**需要的AI应用**： + +- 供应商评价归一化 +- 风险供应商识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多来源表格标准化和连续清洗 +- 链式算子结构便于逐步加规则 +- 相比人工Excel整合，能把流程固化为可复用数据管道 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/vendor_evaluation_standardizer/ + pipeline.py: + - def run_vendor_evaluation_standardizer_pipeline(input_dir, output_file) + operators.py: + - class VendorEvaluationSource(BatchFunction) + - class EvaluationFieldMapper(MapFunction) + - class EvaluationNormalizer(MapFunction) + - class VendorRiskMarker(MapFunction) + - class VendorEvaluationSink(SinkFunction) +新建: examples/run_vendor_evaluation_standardizer.py +``` + +**预期收益**：月费2-8万 | **实现周期**：2周 + +______________________________________________________________________ + +### 38. **内容标签自动生成系统** + +**现实场景痛点**：内容平台标签维护成本高，缺少标签会直接影响检索和推荐效果。 + +**发现需求点**： + +- 需要从文本中提取主题和关键词 +- 需要控制标签数量和质量 +- 需要支持文章、帖子、商品描述等不同内容类型 + +**解决方案**： + +``` +内容文本 → 清洗 → 关键词提取 → 标签候选生成 → 标签筛选输出 +``` + +**需要的AI应用**： + +- 内容主题识别 +- 标签推荐 + +**AI应用关键解决问题的创新点**： + +- SAGE适合内容持续入库时自动完成标签化处理 +- 处理链清晰，可轻松替换关键词或标签规则 +- 相比一次性脚本，更适合内容平台持续生产场景 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/content_tagger/ + pipeline.py: + - def run_content_tagger_pipeline(content_file, output_file) + operators.py: + - class ContentSource(BatchFunction) + - class ContentCleaner(MapFunction) + - class TagCandidateExtractor(MapFunction) + - class TagSelector(MapFunction) + - class TagSink(SinkFunction) +新建: examples/run_content_tagger.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 39. **合作伙伴信息聚合系统** + +**现实场景痛点**：合作伙伴资料散落在销售、法务、财务多个系统，缺少统一视图。 + +**发现需求点**： + +- 需要统一主键与字段命名 +- 需要合并重复企业记录 +- 需要输出单一伙伴画像 + +**解决方案**： + +``` +多源伙伴数据 → 字段对齐 → 重复合并 → 画像生成 → 统一视图输出 +``` + +**需要的AI应用**： + +- 合作伙伴画像构建 +- 数据冲突识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多源数据按顺序规整、去重和融合 +- 每一步映射和合并可独立验证 +- 比手工ETL更轻量，也比通用编排框架更直接 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/partner_profile_hub/ + pipeline.py: + - def run_partner_profile_hub_pipeline(input_dir, output_file) + operators.py: + - class PartnerSource(BatchFunction) + - class PartnerFieldMapper(MapFunction) + - class PartnerDeduplicator(MapFunction) + - class PartnerProfileBuilder(MapFunction) + - class PartnerSink(SinkFunction) +新建: examples/run_partner_profile_hub.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 40. **订阅制内容分发系统** + +**现实场景痛点**：媒体、研究和资讯平台很难把新内容精准分发给真正关心的订阅用户。 + +**发现需求点**： + +- 需要维护用户订阅条件 +- 需要把新内容快速匹配到订阅群体 +- 需要支持多渠道分发 + +**解决方案**： + +``` +新内容 → 内容解析 → 订阅规则匹配 → 个性化筛选 → 分发输出 +``` + +**需要的AI应用**： + +- 内容订阅匹配 +- 分发优先级排序 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把新内容当流处理并立即分发 +- map/filter结构直接表达匹配逻辑和下游分流 +- 相比轮询式脚本，延迟更低、链路更稳定 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/subscription_dispatch/ + pipeline.py: + - def run_subscription_dispatch_pipeline(content_file, subscription_file, output_dir) + operators.py: + - class ContentPublishSource(BatchFunction) + - class SubscriptionMatcher(MapFunction) + - class PersonalizationFilter(MapFunction) + - class DispatchSink(SinkFunction) +新建: examples/run_subscription_dispatch.py +``` + +**预期收益**：按订阅数或推送次数计费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 41. **合同模板版本管理系统** + +**现实场景痛点**：法务模板多版本并存，业务部门常常误用旧版本合同。 + +**发现需求点**： + +- 需要比较模板差异 +- 需要生成版本变更记录 +- 需要统一输出当前有效版本清单 + +**解决方案**： + +``` +模板文件 → 版本解析 → 差异比对 → 变更记录生成 → 版本库更新 +``` + +**需要的AI应用**： + +- 模板差异识别 +- 版本风险提示 + +**AI应用关键解决问题的创新点**： + +- SAGE适合批量模板的持续检测和记录更新 +- 差异、版本号、风险提示能拆成独立算子 +- 相比人工比对文档，处理链更稳定且可重复 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/contract_versioning/ + pipeline.py: + - def run_contract_versioning_pipeline(template_dir, output_dir) + operators.py: + - class ContractTemplateSource(BatchFunction) + - class VersionParser(MapFunction) + - class TemplateDiffAnalyzer(MapFunction) + - class VersionRegistrySink(SinkFunction) +新建: examples/run_contract_versioning.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2周 + +______________________________________________________________________ + +### 42. **发票自动匹配与对账系统** + +**现实场景痛点**：财务部门需要手工核对发票、订单和付款记录，工作量大且容易错账。 + +**发现需求点**： + +- 需要统一订单号、金额、税号等字段 +- 需要批量匹配发票和业务单据 +- 需要输出异常对账项 + +**解决方案**： + +``` +发票数据 + 订单数据 → 字段标准化 → 匹配打分 → 差异识别 → 对账报告输出 +``` + +**需要的AI应用**： + +- 发票匹配评分 +- 异常对账识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合在一条链里完成标准化、匹配和异常输出 +- 规则打分结果可解释，方便财务复核 +- 相比多脚本串联，对账链路更稳定、维护成本更低 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/invoice_reconciliation/ + pipeline.py: + - def run_invoice_reconciliation_pipeline(invoice_file, order_file, output_file) + operators.py: + - class InvoiceSource(BatchFunction) + - class OrderSource(BatchFunction) + - class ReconciliationFieldNormalizer(MapFunction) + - class InvoiceMatcher(MapFunction) + - class ReconciliationSink(SinkFunction) +新建: examples/run_invoice_reconciliation.py +``` + +**预期收益**：月费5-15万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 43. **项目风险日志监控系统** + +**现实场景痛点**：项目风险往往散落在周报、日报、需求变更和缺陷日志里，管理层很难及时发现。 + +**发现需求点**： + +- 需要从日志或文本记录中提取风险信号 +- 需要按项目、模块、负责人输出风险等级 +- 需要形成预警摘要 + +**解决方案**： + +``` +项目日志 → 风险关键词抽取 → 风险评分 → 项目分组 → 预警输出 +``` + +**需要的AI应用**： + +- 风险事件识别 +- 项目风险排序 + +**AI应用关键解决问题的创新点**： + +- SAGE适合持续接收项目文本更新并做增量分析 +- 风险判断链条清晰，适合项目管理场景复盘 +- 相比聊天式助手，更适合结构化风险流水处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/project_risk_monitor/ + pipeline.py: + - def run_project_risk_monitor_pipeline(log_file, output_file) + operators.py: + - class ProjectLogSource(BatchFunction) + - class RiskKeywordExtractor(MapFunction) + - class ProjectRiskScorer(MapFunction) + - class ProjectRiskSink(SinkFunction) +新建: examples/run_project_risk_monitor.py +``` + +**预期收益**：月费3-12万 | **实现周期**：2周 + +______________________________________________________________________ + +### 44. **员工学习认证聚合系统** + +**现实场景痛点**：员工培训和认证记录散落在不同平台，人力和合规部门难以形成统一档案。 + +**发现需求点**： + +- 需要同步多平台学习记录 +- 需要按员工统一主档 +- 需要标记过期认证和缺失课程 + +**解决方案**： + +``` +培训记录 → 员工映射 → 课程归一化 → 认证状态判断 → 学习档案输出 +``` + +**需要的AI应用**： + +- 认证缺口识别 +- 学习进度提醒 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多源学习记录的标准化和持续汇总 +- 规则明确，MapFunction即可表达大多数业务逻辑 +- 比人工导表和VLOOKUP更稳定可复用 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/learning_record_hub/ + pipeline.py: + - def run_learning_record_hub_pipeline(input_dir, output_file) + operators.py: + - class LearningRecordSource(BatchFunction) + - class EmployeeMapper(MapFunction) + - class CourseNormalizer(MapFunction) + - class CertificationGapDetector(MapFunction) + - class LearningProfileSink(SinkFunction) +新建: examples/run_learning_record_hub.py +``` + +**预期收益**：月费2-8万 | **实现周期**：2周 + +______________________________________________________________________ + +### 45. **供应链订单追踪聚合系统** + +**现实场景痛点**：订单状态分散在供应商、仓库、物流多个系统，供应链团队经常靠电话催问。 + +**发现需求点**： + +- 需要整合多个状态来源 +- 需要统一状态枚举 +- 需要持续更新订单追踪看板 + +**解决方案**： + +``` +多系统状态 → 状态归一化 → 时间线拼接 → 延迟识别 → 追踪结果输出 +``` + +**需要的AI应用**： + +- 订单状态整合 +- 延迟风险识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多来源订单状态的持续汇聚和加工 +- 状态转换链条透明，便于排查问题源头 +- 相比人工对账和多系统跳转，能形成单一数据流视图 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/supply_chain_tracker/ + pipeline.py: + - def run_supply_chain_tracker_pipeline(input_dir, output_file) + operators.py: + - class SupplyStatusSource(BatchFunction) + - class StatusNormalizer(MapFunction) + - class TimelineBuilder(MapFunction) + - class DelayRiskDetector(MapFunction) + - class TrackingSink(SinkFunction) +新建: examples/run_supply_chain_tracker.py +``` + +**预期收益**：月费5-15万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 46. **合规文档自动整理系统** + +**现实场景痛点**：合规制度、制度附件和更新记录分散，审查前往往需要人工重新整理。 + +**发现需求点**： + +- 需要按制度类型、更新时间、适用范围整理文档 +- 需要检查是否缺失更新 +- 需要提醒复审节点 + +**解决方案**： + +``` +合规文档 → 分类整理 → 元数据抽取 → 更新检查 → 复审提醒输出 +``` + +**需要的AI应用**： + +- 文档归档分类 +- 复审风险识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合对大批文档做持续规整和检查 +- 处理链透明，便于合规团队确认依据 +- 相比纯文档管理工具，更容易补充规则算子 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/compliance_doc_manager/ + pipeline.py: + - def run_compliance_doc_manager_pipeline(doc_dir, output_dir) + operators.py: + - class ComplianceDocSource(BatchFunction) + - class ComplianceDocClassifier(MapFunction) + - class ReviewDeadlineChecker(MapFunction) + - class ComplianceReminderSink(SinkFunction) +新建: examples/run_compliance_doc_manager.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 47. **邮件智能分类系统** + +**现实场景痛点**：企业员工每天收到大量邮件，重要邮件容易被普通通知淹没。 + +**发现需求点**： + +- 需要根据发件人、关键词、主题自动分类 +- 需要区分紧急、待办、通知、垃圾等优先级 +- 需要接入现有邮箱规则或待办系统 + +**解决方案**： + +``` +邮件流 → 标题正文解析 → 分类规则匹配 → 优先级评分 → 分类结果输出 +``` + +**需要的AI应用**： + +- 邮件分类 +- 优先级排序 + +**AI应用关键解决问题的创新点**： + +- SAGE适合处理持续到来的邮件流 +- 文本解析、规则评分、下游分流可以形成单一链路 +- 相比单次交互式总结，更适合稳定自动化落地 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/mail_classifier/ + pipeline.py: + - def run_mail_classifier_pipeline(mail_file, output_file) + operators.py: + - class MailSource(BatchFunction) + - class MailParser(MapFunction) + - class MailCategoryClassifier(MapFunction) + - class MailPriorityScorer(MapFunction) + - class MailSink(SinkFunction) +新建: examples/run_mail_classifier.py +``` + +**预期收益**：月费2-8万 | **实现周期**：2周 + +______________________________________________________________________ + +### 48. **会议纪要自动生成系统** + +**现实场景痛点**：会议结束后整理纪要耗时，责任人、截止时间和关键结论容易遗漏。 + +**发现需求点**： + +- 需要从录音转写文本中抽取议题和行动项 +- 需要按会议主题生成结构化纪要 +- 需要支持邮件或协同工具分发 + +**解决方案**： + +``` +会议转写文本 → 议题切分 → 行动项提取 → 纪要结构化 → 分发输出 +``` + +**需要的AI应用**： + +- 行动项抽取 +- 纪要结构化生成 + +**AI应用关键解决问题的创新点**： + +- SAGE适合将转写文本处理成连续的结构化流水线 +- 议题拆分和行动项提取天然适合flatmap与map组合 +- 相比人工整理，更适合高频会议场景批量化处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/meeting_minutes/ + pipeline.py: + - def run_meeting_minutes_pipeline(transcript_file, output_dir) + operators.py: + - class TranscriptSource(BatchFunction) + - class AgendaSegmenter(FlatMapFunction) + - class ActionItemExtractor(MapFunction) + - class MinutesFormatter(MapFunction) + - class MinutesSink(SinkFunction) +新建: examples/run_meeting_minutes.py +``` + +**预期收益**：月费3-12万 | **实现周期**：3周 + +______________________________________________________________________ + +### 49. **制度文件更新通知系统** + +**现实场景痛点**：制度更新后员工不知道改了什么，执行口径不一致。 + +**发现需求点**： + +- 需要对比制度版本差异 +- 需要提炼受影响部门和岗位 +- 需要输出更新通知与阅读清单 + +**解决方案**： + +``` +制度新旧版本 → 差异提取 → 影响范围识别 → 通知内容生成 → 分发输出 +``` + +**需要的AI应用**： + +- 变更点提取 +- 影响范围识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合持续跟踪文件变更流并快速下发结果 +- 差异提取与通知分发都能放进单一管道 +- 相比手工比对文档，更稳定且便于审计 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/policy_update_notifier/ + pipeline.py: + - def run_policy_update_notifier_pipeline(old_dir, new_dir, output_dir) + operators.py: + - class PolicyVersionSource(BatchFunction) + - class PolicyDiffExtractor(MapFunction) + - class PolicyImpactAnalyzer(MapFunction) + - class PolicyNoticeSink(SinkFunction) +新建: examples/run_policy_update_notifier.py +``` + +**预期收益**：月费2-8万 | **实现周期**：2周 + +______________________________________________________________________ + +### 50. **数据导出格式转换系统** + +**现实场景痛点**：业务部门需要定期把数据导出成不同格式发给外部伙伴，重复劳动高且易出错。 + +**发现需求点**： + +- 需要定时导出 +- 需要支持CSV、Excel、JSON等格式 +- 需要适配不同字段模板 + +**解决方案**： + +``` +源数据 → 查询读取 → 字段映射 → 格式转换 → 多目标输出 +``` + +**需要的AI应用**： + +- 导出模板匹配 +- 数据质量校验 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把导出流程做成固定数据管道 +- 新增格式只需要新增转换算子 +- 比手工导出或脚本碎片化处理更可维护 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/export_transformer/ + pipeline.py: + - def run_export_transformer_pipeline(source_config, output_dir) + operators.py: + - class ExportQuerySource(BatchFunction) + - class ExportFieldMapper(MapFunction) + - class FormatTransformer(MapFunction) + - class ExportSink(SinkFunction) +新建: examples/run_export_transformer.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2周 + +______________________________________________________________________ + +### 51. **API日志聚合分析系统** + +**现实场景痛点**：API日志分散在多个服务，排查性能问题时需要人工汇总多个日志源。 + +**发现需求点**： + +- 需要统一日志格式 +- 需要统计接口耗时、错误码和调用频次 +- 需要输出接口级性能分析 + +**解决方案**： + +``` +多源API日志 → 解析归一化 → 指标提取 → 异常识别 → 性能报告输出 +``` + +**需要的AI应用**： + +- 接口异常识别 +- 性能热点排序 + +**AI应用关键解决问题的创新点**： + +- SAGE天然适合处理多源日志流并持续归并 +- 日志解析与统计拆分成多个算子后更利于维护 +- 相比手工grep与离线统计，更适合持续服务监控 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/api_log_analytics/ + pipeline.py: + - def run_api_log_analytics_pipeline(log_dir, output_file) + operators.py: + - class ApiLogSource(BatchFunction) + - class ApiLogParser(MapFunction) + - class ApiMetricExtractor(MapFunction) + - class ApiAnomalyDetector(MapFunction) + - class ApiAnalyticsSink(SinkFunction) +新建: examples/run_api_log_analytics.py +``` + +**预期收益**：月费5-15万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 52. **数据备份与冗余同步系统** + +**现实场景痛点**：企业做数据备份时常见漏备份、重复备份和副本不一致问题，恢复演练成本高。 + +**发现需求点**： + +- 需要检测增量变化 +- 需要向多个备份目标同步 +- 需要校验备份一致性 + +**解决方案**： + +``` +源数据清单 → 增量识别 → 多目标同步 → 校验比对 → 同步报告输出 +``` + +**需要的AI应用**： + +- 备份异常识别 +- 副本一致性检查 + +**AI应用关键解决问题的创新点**： + +- SAGE适合持续处理备份任务流和校验任务流 +- 同步、校验、告警可以串成稳定流水线 +- 相比分散脚本，更适合长期运维和批量任务管理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/backup_sync/ + pipeline.py: + - def run_backup_sync_pipeline(source_manifest, target_config, output_file) + operators.py: + - class BackupSource(BatchFunction) + - class IncrementDetector(MapFunction) + - class BackupDispatcher(MapFunction) + - class ConsistencyChecker(MapFunction) + - class BackupReportSink(SinkFunction) +新建: examples/run_backup_sync.py +``` + +**预期收益**：按数据量或副本数计费 | **实现周期**：2.5周 + +______________________________________________________________________ + +## 第五部分：53-65 科研、教育与医疗应用 + +### 53. **专利侵权线索预警系统** + +**现实场景痛点**：制造、材料、生物医药等企业在推新产品前，法务和研发往往无法及时识别新方案是否与竞争对手专利权利要求发生冲突。 + +**发现需求点**： + +- 需要把自有技术要点和外部新公开专利持续比对 +- 需要识别高风险权利要求碰撞点 +- 需要输出法务初筛所需的证据清单 + +**解决方案**： + +``` +专利文本流 → 权利要求抽取 → 技术要点比对 → 冲突线索评分 → 证据包输出 +``` + +**需要的AI应用**： + +- 权利要求冲突识别 +- 侵权线索优先级排序 + +**AI应用关键解决问题的创新点**： + +- 这项应用和原生 Patent Landscape Mapper 的区别在于：原生示例做宏观专利版图与空白机会分析，本项做微观权利要求冲突预警和法务线索整理 +- SAGE适合把持续更新的专利公开流、规则比对和风险输出放在一条可审计的数据链路里 +- 相比 LangGraph 这类偏报告/代理编排的框架，SAGE更适合高频结构化比对和规则打分 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/patent_competition_monitor/ + pipeline.py: + - def run_patent_competition_monitor_pipeline(patent_file, profile_file, output_dir) + operators.py: + - class PatentWatchSource(BatchFunction) + - class ClaimExtractor(MapFunction) + - class TechnologyProfileMatcher(MapFunction) + - class InfringementRiskScorer(MapFunction) + - class EvidenceBundleSink(SinkFunction) + 新建: examples/run_patent_competition_monitor.py +``` + +**预期收益**：单项目10-50万 | **实现周期**：3周 + +______________________________________________________________________ + +### 54. **科研资助机会订阅系统** + +**现实场景痛点**：高校课题组和科技企业经常错过申报窗口，因为没人持续盯政策站点和基金发布渠道。 + +**发现需求点**： + +- 需要定期抓取资助公告和申报指南 +- 需要按团队方向、地区、预算范围做匹配 +- 需要第一时间生成订阅提醒 + +**解决方案**： + +``` +资助公告 → 文本解析 → 条件结构化 → 团队画像匹配 → 订阅提醒输出 +``` + +**需要的AI应用**： + +- 资助机会匹配 +- 申报优先级推荐 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把公告抓取、条件抽取、画像匹配串成稳定流水线 +- 匹配规则可用 MapFunction 持续迭代，而不必重写控制流 +- 相比以多轮推理为主的框架，SAGE更适合做公告流的批流混合处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/grant_subscription/ + pipeline.py: + - def run_grant_subscription_pipeline(announcement_file, profile_file, output_file) + operators.py: + - class GrantAnnouncementSource(BatchFunction) + - class GrantRuleExtractor(MapFunction) + - class TeamProfileMatcher(MapFunction) + - class GrantPriorityScorer(MapFunction) + - class GrantAlertSink(SinkFunction) +新建: examples/run_grant_subscription.py +``` + +**预期收益**：年费10-30万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 55. **实验记录异常回顾系统** + +**现实场景痛点**：实验室每天积累大量实验记录，异常样本和失败条件很容易散落在文本里，复盘成本高。 + +**发现需求点**： + +- 需要把实验日志切分成步骤级记录 +- 需要识别异常实验条件和结果偏差 +- 需要形成可回顾的异常摘要 + +**解决方案**： + +``` +实验日志 → 记录切分 → 参数抽取 → 异常标记 → 回顾摘要输出 +``` + +**需要的AI应用**： + +- 实验异常识别 +- 实验回顾摘要生成 + +**AI应用关键解决问题的创新点**： + +- SAGE的 flatmap 很适合把一份长实验记录拆成多条步骤记录 +- 整个链路可追踪，适合科研场景对过程可复核的要求 +- 相比面向会话的框架，SAGE更适合持续处理规范化日志流 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/experiment_review/ + pipeline.py: + - def run_experiment_review_pipeline(log_file, output_file) + operators.py: + - class ExperimentLogSource(BatchFunction) + - class ExperimentStepSplitter(FlatMapFunction) + - class ExperimentParameterExtractor(MapFunction) + - class ExperimentAnomalyMarker(MapFunction) + - class ExperimentReviewSink(SinkFunction) +新建: examples/run_experiment_review.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2周 + +______________________________________________________________________ + +### 56. **科研交付复现审计系统** + +**现实场景痛点**：论文附带代码、数据和配置经常缺失版本对应关系，导致内部复现和对外交付风险很高。 + +**发现需求点**： + +- 需要检查数据、脚本、环境、模型版本是否齐全 +- 需要识别缺失项和不一致项 +- 需要输出复现审计清单 + +**解决方案**： + +``` +交付清单 → 元数据抽取 → 一致性校验 → 缺失项标记 → 审计报告输出 +``` + +**需要的AI应用**： + +- 复现缺口识别 +- 审计风险分级 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把多类元数据文件统一成同一处理流 +- 每一类校验逻辑都能模块化成算子，适合长期迭代 +- 相比通用 agent 链，更适合做高确定性的审计型任务 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/repro_audit/ + pipeline.py: + - def run_repro_audit_pipeline(manifest_file, output_dir) + operators.py: + - class ReproManifestSource(BatchFunction) + - class ReproMetadataExtractor(MapFunction) + - class ReproConsistencyChecker(MapFunction) + - class ReproRiskScorer(MapFunction) + - class ReproAuditSink(SinkFunction) +新建: examples/run_repro_audit.py +``` + +**预期收益**：单项目5-20万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 57. **模型评测榜单波动监控系统** + +**现实场景痛点**：模型团队和投资研究团队很难及时掌握 benchmark 榜单变化，竞争情报反应慢。 + +**发现需求点**： + +- 需要定期抓取榜单与测评文章 +- 需要识别排名、分数、模型名称变化 +- 需要输出波动原因和关注对象 + +**解决方案**： + +``` +榜单页面 → 结构化解析 → 版本对比 → 波动识别 → 监控简报输出 +``` + +**需要的AI应用**： + +- 榜单波动识别 +- 竞争对象追踪 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把定期抓取和版本比对做成持续流水线 +- 解析、对比、告警互相独立，维护成本低 +- 相比依赖人工轮询或临时脚本，更适合长期情报监控场景 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/benchmark_watch/ + pipeline.py: + - def run_benchmark_watch_pipeline(input_file, output_file) + operators.py: + - class BenchmarkSource(BatchFunction) + - class BenchmarkParser(MapFunction) + - class BenchmarkDiffDetector(MapFunction) + - class BenchmarkTrendTagger(MapFunction) + - class BenchmarkWatchSink(SinkFunction) +新建: examples/run_benchmark_watch.py +``` + +**预期收益**：月费3-12万 | **实现周期**：2周 + +______________________________________________________________________ + +### 58. **课程资料问答助手** + +**现实场景痛点**：学生面对讲义、作业说明和往届资料时，经常找不到准确答案，教师重复答疑耗时。 + +**发现需求点**： + +- 需要把讲义、题目、答案解析统一入库 +- 需要按课程章节检索与定位 +- 需要给出来源明确的答复 + +**解决方案**： + +``` +课程资料 → 文本抽取 → 分段索引 → 问题匹配 → 引用答案输出 +``` + +**需要的AI应用**： + +- 课程问题匹配 +- 来源定位答复 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把资料清洗、切段、索引构建做成标准数据流 +- 问题匹配与答案组装都能以确定性规则或轻量模型落地 +- 相比重 agent 路由，更适合教育场景中高频、可解释的问答支持 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/course_qa_helper/ + pipeline.py: + - def run_course_qa_helper_pipeline(doc_dir, question_file, output_file) + operators.py: + - class CourseDocSource(BatchFunction) + - class CourseChunker(FlatMapFunction) + - class CourseQuestionMatcher(MapFunction) + - class CourseAnswerFormatter(MapFunction) + - class CourseAnswerSink(SinkFunction) +新建: examples/run_course_qa_helper.py +``` + +**预期收益**：按学校或课程包年收费 | **实现周期**：3周 + +______________________________________________________________________ + +### 59. **周课时计划排布系统** + +**现实场景痛点**：培训机构和学校排课依赖人工协调，容易出现课时不均、内容进度失衡和资源冲突。 + +**发现需求点**： + +- 需要结合教师、教室、章节进度做排布 +- 需要按不同班型生成周计划 +- 需要输出可执行排课草案 + +**解决方案**： + +``` +教学要求 → 资源约束读取 → 周计划评分 → 冲突校验 → 排课草案输出 +``` + +**需要的AI应用**： + +- 教学节奏推荐 +- 排课冲突识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合将规则校验、评分、输出串成一条处理链 +- 教学规则变化时只需替换评分和校验算子 +- 相比会话式助手，SAGE更适合批量生成并校验结构化排课结果 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/lesson_scheduler/ + pipeline.py: + - def run_lesson_scheduler_pipeline(plan_file, resource_file, output_file) + operators.py: + - class TeachingRequirementSource(BatchFunction) + - class TeachingConstraintParser(MapFunction) + - class LessonPlanScorer(MapFunction) + - class LessonConflictChecker(MapFunction) + - class LessonScheduleSink(SinkFunction) +新建: examples/run_lesson_scheduler.py +``` + +**预期收益**：学期服务费5-15万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 60. **作业初稿反馈系统** + +**现实场景痛点**：教师无法在正式批改前给每个学生提供足够多的过程性反馈，学生修改成本高。 + +**发现需求点**： + +- 需要按 rubric 对初稿进行结构化检查 +- 需要识别常见错误类型 +- 需要输出修改建议和复查清单 + +**解决方案**： + +``` +作业初稿 → 段落解析 → rubric匹配 → 错误归类 → 反馈建议输出 +``` + +**需要的AI应用**： + +- 作业问题识别 +- rubric 反馈生成 + +**AI应用关键解决问题的创新点**： + +- SAGE适合将作业解析、评分规则、反馈模板沉淀成可复用管道 +- 每个评分点都可解释，便于教师校验和修正规则 +- 相比完全生成式 Agent，更适合教育反馈中的一致性要求 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/assignment_feedback/ + pipeline.py: + - def run_assignment_feedback_pipeline(draft_file, rubric_file, output_file) + operators.py: + - class AssignmentDraftSource(BatchFunction) + - class AssignmentSectionParser(MapFunction) + - class RubricMatcher(MapFunction) + - class FeedbackComposer(MapFunction) + - class AssignmentFeedbackSink(SinkFunction) +新建: examples/run_assignment_feedback.py +``` + +**预期收益**：按班级或学期收费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 61. **班级分层编组系统** + +**现实场景痛点**：学校和培训机构在做分班、分层教学和助教资源配置时，常常只能依靠教师经验，导致班级差异过大、教学节奏失衡。 + +**发现需求点**： + +- 需要把测验、作业、出勤和课堂表现转成分层指标 +- 需要按班级整体效果而不是单个学生提分来做编组 +- 需要输出可执行的分班和助教配置建议 + +**解决方案**： + +``` +学习记录 → 分层指标构造 → 学员分群 → 班级编组评分 → 编组方案输出 +``` + +**需要的AI应用**： + +- 学员分层聚类 +- 班级编组推荐 + +**AI应用关键解决问题的创新点**： + +- 这项应用和原生 Student Improvement 的区别在于：原生示例聚焦单个学生的持续提分与错题跟踪，本项聚焦班级层面的分组、排布和资源配置 +- SAGE适合持续汇总多源学习记录，并把分群、评分、方案输出放在同一条批流处理链上 +- 相比一次性表格分班或交互式助手，SAGE更适合周期性、大批量教学编组任务 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/skill_gap_diagnosis/ + pipeline.py: + - def run_skill_gap_diagnosis_pipeline(record_dir, constraint_file, output_file) + operators.py: + - class CohortRecordSource(BatchFunction) + - class CohortFeatureBuilder(MapFunction) + - class StudentClusterer(MapFunction) + - class CohortPlanScorer(MapFunction) + - class CohortPlanSink(SinkFunction) + 新建: examples/run_skill_gap_diagnosis.py +``` + +**预期收益**：学期服务费5-15万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 62. **岗位面试模拟教练系统** + +**现实场景痛点**：求职者练习面试往往缺少结构化反馈，职业培训机构难以规模化提供高质量陪练。 + +**发现需求点**： + +- 需要按岗位题库生成模拟面试流程 +- 需要根据回答内容打分并给出改进建议 +- 需要记录多轮练习进步情况 + +**解决方案**： + +``` +岗位题库 → 模拟回答记录 → 维度评分 → 弱项识别 → 训练报告输出 +``` + +**需要的AI应用**： + +- 回答质量评分 +- 面试改进建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把题库、回答、评分、报告串成稳定训练流水线 +- 训练记录可以持续流入同一管道做进步跟踪 +- 相比多 agent 对话树，更适合标准化的岗位训练场景 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/interview_coach/ + pipeline.py: + - def run_interview_coach_pipeline(answer_file, rubric_file, output_file) + operators.py: + - class InterviewAnswerSource(BatchFunction) + - class InterviewQuestionMapper(MapFunction) + - class InterviewScorer(MapFunction) + - class InterviewAdviceBuilder(MapFunction) + - class InterviewReportSink(SinkFunction) +新建: examples/run_interview_coach.py +``` + +**预期收益**：按用户订阅收费 | **实现周期**：3周 + +______________________________________________________________________ + +### 63. **校园奖助申请缺口预警系统** + +**现实场景痛点**：学校每学期都要处理大量奖学金、助学金和困难补助申请，学生经常因为材料不全、条件不符或截止时间遗漏而错失机会。 + +**发现需求点**： + +- 需要持续检查申请材料是否齐全 +- 需要把成绩、家庭情况、奖惩记录等条件自动比对到申请规则 +- 需要提前向辅导员和学生输出缺口提醒 + +**解决方案**： + +``` +申请材料 → 条件规则抽取 → 学生画像匹配 → 缺口识别 → 预警清单输出 +``` + +**需要的AI应用**： + +- 申请缺口识别 +- 截止风险预警 + +**AI应用关键解决问题的创新点**： + +- 这项应用替换了原来的校园事务分派，避免与已实现的工单路由类能力重复 +- SAGE适合把学生档案、申请表和规则条件放进同一条批流处理链里 +- 相比人工逐份核对，更适合学期集中、批量化的校园资助管理流程 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/campus_aid_gap_alert/ + pipeline.py: + - def run_campus_aid_gap_alert_pipeline(application_file, profile_file, output_file) + operators.py: + - class AidApplicationSource(BatchFunction) + - class AidRuleExtractor(MapFunction) + - class StudentEligibilityMatcher(MapFunction) + - class AidGapDetector(MapFunction) + - class AidAlertSink(SinkFunction) +新建: examples/run_campus_aid_gap_alert.py +``` + +**预期收益**：年费8-20万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 64. **门急诊分诊整理系统** + +**现实场景痛点**：门急诊高峰期患者信息记录杂乱，护士分诊压力大，轻重缓急容易判断失真。 + +**发现需求点**： + +- 需要整理主诉、生命体征、既往史 +- 需要按规则进行分诊优先级判断 +- 需要给分诊台输出结构化摘要 + +**解决方案**： + +``` +接诊记录 → 字段抽取 → 分诊规则判断 → 风险标签生成 → 摘要输出 +``` + +**需要的AI应用**： + +- 接诊信息结构化 +- 分诊优先级建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合处理连续到来的接诊记录流 +- 规则分诊可完全透明化，符合医疗场景审计要求 +- 相比黑箱式 agent 决策，更适合临床前台的高确定性流程 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/triage_structurer/ + pipeline.py: + - def run_triage_structurer_pipeline(input_file, output_file) + operators.py: + - class TriageRecordSource(BatchFunction) + - class TriageFieldExtractor(MapFunction) + - class TriagePriorityAssigner(MapFunction) + - class TriageSummaryBuilder(MapFunction) + - class TriageSink(SinkFunction) +新建: examples/run_triage_structurer.py +``` + +**预期收益**：科室年费15-40万 | **实现周期**：3周 + +______________________________________________________________________ + +### 65. **影像报告随访闭环系统** + +**现实场景痛点**：影像报告里写了“建议复查”后，很多医院无法追踪患者是否真正完成随访，带来医疗风险。 + +**发现需求点**： + +- 需要从影像报告中识别随访建议 +- 需要关联患者、时间和检查类型 +- 需要输出待办清单和超期提醒 + +**解决方案**： + +``` +影像报告 → 随访建议抽取 → 患者关联 → 超期判断 → 闭环清单输出 +``` + +**需要的AI应用**： + +- 随访建议识别 +- 随访超期预警 + +**AI应用关键解决问题的创新点**： + +- SAGE擅长把报告流、患者清单和提醒规则放进单一数据流 +- 流程可解释，适合医疗质控部门追溯原因 +- 相比仅做问答的框架，更适合随访管理这种持续运营型任务 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/radiology_followup_loop/ + pipeline.py: + - def run_radiology_followup_loop_pipeline(report_file, patient_file, output_file) + operators.py: + - class RadiologyReportSource(BatchFunction) + - class FollowupExtractor(MapFunction) + - class PatientMatcher(MapFunction) + - class FollowupDeadlineChecker(MapFunction) + - class FollowupSink(SinkFunction) +新建: examples/run_radiology_followup_loop.py +``` + +**预期收益**：年费20-50万 | **实现周期**：3周 + +______________________________________________________________________ + +## 第六部分：66-78 企业内容、媒体与物联应用 + +### 66. **药品说明书结构化抽取系统** + +**现实场景痛点**：药企、医院和药房在整理说明书时常要手工提取剂量、禁忌和警示，效率低且容易漏项。 + +**发现需求点**： + +- 需要从 PDF 或扫描件中抽取关键字段 +- 需要统一剂量单位和用药场景 +- 需要输出结构化药品知识条目 + +**解决方案**： + +``` +药品说明书 → OCR/文本提取 → 字段抽取 → 单位规范化 → 知识条目输出 +``` + +**需要的AI应用**： + +- 药品字段抽取 +- 用药风险标签识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把说明书清洗、字段抽取、规范化串成批处理流水线 +- 每个字段规则透明，便于药学团队复核 +- 相比面向对话的系统，更适合稳定的大批量文档处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/drug_leaflet_extractor/ + pipeline.py: + - def run_drug_leaflet_extractor_pipeline(input_dir, output_file) + operators.py: + - class DrugLeafletSource(BatchFunction) + - class DrugLeafletTextExtractor(MapFunction) + - class DrugFieldExtractor(MapFunction) + - class DrugUnitNormalizer(MapFunction) + - class DrugLeafletSink(SinkFunction) +新建: examples/run_drug_leaflet_extractor.py +``` + +**预期收益**：按药品条目计费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 67. **检验样本周转异常预警系统** + +**现实场景痛点**：医院检验科和第三方实验室每天要处理大量样本，一旦出现采样后超时、运输延迟或科室回传缓慢，就会影响报告时效和医疗安全。 + +**发现需求点**： + +- 需要跟踪样本从采集、送检到出报告的全流程时间戳 +- 需要识别不同科室、不同检验项目的周转异常 +- 需要提前输出超时风险和堵点位置 + +**解决方案**： + +``` +样本流转记录 → 阶段映射 → 周转时长计算 → 异常打分 → 预警清单输出 +``` + +**需要的AI应用**： + +- 周转异常识别 +- 堵点环节定位 + +**AI应用关键解决问题的创新点**： + +- 这项应用替换了远程问诊纪要归档，避免与已实现的纪要/摘要生成类能力重复 +- SAGE适合处理样本流转这种天然时序型、状态型数据链路 +- 相比人工在 LIS 或 Excel 中回查，更适合做持续告警和流程瓶颈监测 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/lab_turnaround_alert/ + pipeline.py: + - def run_lab_turnaround_alert_pipeline(record_file, output_file) + operators.py: + - class LabRecordSource(BatchFunction) + - class LabStageMapper(MapFunction) + - class TurnaroundTimeBuilder(MapFunction) + - class TurnaroundAnomalyDetector(MapFunction) + - class LabAlertSink(SinkFunction) +新建: examples/run_lab_turnaround_alert.py +``` + +**预期收益**：科室年费12-30万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 68. **供应商报价对比系统** + +**现实场景痛点**：采购人员每天要比较多家供应商报价、账期和交付条件，Excel 对比效率低。 + +**发现需求点**： + +- 需要把报价单字段统一 +- 需要对价格、交付周期、历史质量做综合评分 +- 需要输出推荐排序和风险提示 + +**解决方案**： + +``` +报价单 → 字段标准化 → 条件比对 → 综合评分 → 推荐清单输出 +``` + +**需要的AI应用**： + +- 报价优选评分 +- 风险供应商提示 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多份报价文件的连续标准化和评分 +- 评分链条清晰，方便采购和审计部门复盘 +- 相比临时脚本或人工表格，更适合长期采购流程沉淀 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/quote_compare/ + pipeline.py: + - def run_quote_compare_pipeline(input_dir, output_file) + operators.py: + - class QuoteSource(BatchFunction) + - class QuoteNormalizer(MapFunction) + - class QuoteConditionComparer(MapFunction) + - class QuoteScorer(MapFunction) + - class QuoteCompareSink(SinkFunction) +新建: examples/run_quote_compare.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2周 + +______________________________________________________________________ + +### 69. **企业制度检索助手** + +**现实场景痛点**：员工和中层主管经常不知道制度条文在哪里，HR 与行政反复回复相同问题。 + +**发现需求点**： + +- 需要统一整理制度、附件和历史版本 +- 需要按问题快速定位条款和出处 +- 需要输出引用明确的答复 + +**解决方案**： + +``` +制度文档 → 文本分段 → 索引构建 → 问题匹配 → 引用答案输出 +``` + +**需要的AI应用**： + +- 制度问答匹配 +- 条款定位引用 + +**AI应用关键解决问题的创新点**： + +- SAGE适合先把制度文档做持续清洗和索引更新，再服务问答链路 +- 文档更新时只需重跑局部数据流，维护成本低 +- 相比以 agent 为中心的检索链，SAGE更适合稳定的企业知识处理流水线 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/policy_search_helper/ + pipeline.py: + - def run_policy_search_helper_pipeline(doc_dir, question_file, output_file) + operators.py: + - class PolicyDocSource(BatchFunction) + - class PolicyChunker(FlatMapFunction) + - class PolicyQuestionMatcher(MapFunction) + - class PolicyAnswerComposer(MapFunction) + - class PolicyAnswerSink(SinkFunction) +新建: examples/run_policy_search_helper.py +``` + +**预期收益**：年费10-25万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 70. **内部知识库去重整理系统** + +**现实场景痛点**：内部知识库文章越积越多，重复答案和过期资料大量存在，员工搜索体验差。 + +**发现需求点**： + +- 需要识别重复文章和相似知识点 +- 需要标记过期内容和缺失元数据 +- 需要输出整理建议 + +**解决方案**： + +``` +知识文章 → 文本指纹生成 → 相似度比对 → 新鲜度评估 → 整理清单输出 +``` + +**需要的AI应用**： + +- 重复文章识别 +- 知识新鲜度评估 + +**AI应用关键解决问题的创新点**： + +- SAGE适合批量知识条目的持续清洗与比对 +- 去重、分组、老化评估都能独立成算子，便于维护 +- 相比纯聊天检索，更适合先把底层知识质量治理好 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/knowledge_cleanup/ + pipeline.py: + - def run_knowledge_cleanup_pipeline(article_dir, output_file) + operators.py: + - class KnowledgeArticleSource(BatchFunction) + - class KnowledgeFingerprintBuilder(MapFunction) + - class KnowledgeDuplicateDetector(MapFunction) + - class KnowledgeFreshnessScorer(MapFunction) + - class KnowledgeCleanupSink(SinkFunction) +新建: examples/run_knowledge_cleanup.py +``` + +**预期收益**：月费3-10万 | **实现周期**：2周 + +______________________________________________________________________ + +### 71. **播客高光切片系统** + +**现实场景痛点**：播客团队需要从长音频中人工挑选高光片段用于分发，耗时且依赖主观经验。 + +**发现需求点**： + +- 需要按转写文本和时间轴识别高价值片段 +- 需要输出片段标题和话题标签 +- 需要支持多节目批量处理 + +**解决方案**： + +``` +播客转写 → 片段切分 → 高光评分 → 标题标签生成 → 切片清单输出 +``` + +**需要的AI应用**： + +- 高光片段识别 +- 分发标题生成 + +**AI应用关键解决问题的创新点**： + +- SAGE的 flatmap 适合把长节目拆成多个候选片段 +- 评分和标题生成是线性处理链，适合批量生产流程 +- 相比为每期节目单独跑 agent，更适合媒体团队规模化运营 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/podcast_highlight/ + pipeline.py: + - def run_podcast_highlight_pipeline(transcript_file, output_file) + operators.py: + - class PodcastTranscriptSource(BatchFunction) + - class PodcastSegmenter(FlatMapFunction) + - class HighlightScorer(MapFunction) + - class HighlightTitleBuilder(MapFunction) + - class HighlightSink(SinkFunction) +新建: examples/run_podcast_highlight.py +``` + +**预期收益**：按节目或包月收费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 72. **品牌物料合规审核系统** + +**现实场景痛点**：市场团队发布海报、推文、宣传页时，经常出现品牌话术不统一、素材过期或免责声明缺失。 + +**发现需求点**： + +- 需要检查标题、正文、版本号和素材元数据 +- 需要按品牌规范输出风险项 +- 需要形成审核记录 + +**解决方案**： + +``` +品牌物料 → 文本/元数据抽取 → 规范规则匹配 → 风险标记 → 审核清单输出 +``` + +**需要的AI应用**： + +- 品牌违规识别 +- 审核优先级推荐 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把多来源素材批量审核做成标准流水线 +- 规则清晰，便于品牌团队维护规范库 +- 相比依赖人工逐项检查，更适合营销高频产出场景 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/brand_compliance_review/ + pipeline.py: + - def run_brand_compliance_review_pipeline(asset_dir, output_file) + operators.py: + - class BrandAssetSource(BatchFunction) + - class BrandAssetParser(MapFunction) + - class BrandRuleMatcher(MapFunction) + - class BrandRiskScorer(MapFunction) + - class BrandReviewSink(SinkFunction) +新建: examples/run_brand_compliance_review.py +``` + +**预期收益**：月费5-15万 | **实现周期**：2周 + +______________________________________________________________________ + +### 73. **字幕术语质检系统** + +**现实场景痛点**：字幕团队在多语种、长节目项目中常出现术语不一致、时序错位和人名误译问题。 + +**发现需求点**： + +- 需要按时间轴检查字幕块 +- 需要核对术语表和常错词 +- 需要输出可复核的质检报告 + +**解决方案**： + +``` +字幕文件 → 块级解析 → 术语校验 → 时序检查 → 质检报告输出 +``` + +**需要的AI应用**： + +- 术语一致性检查 +- 时序异常识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把字幕块当流式记录逐条检查 +- 各类校验规则都可拆成 MapFunction，适合翻译团队维护 +- 相比一次性人工质检，更适合规模化字幕生产流程 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/subtitle_qc/ + pipeline.py: + - def run_subtitle_qc_pipeline(subtitle_file, glossary_file, output_file) + operators.py: + - class SubtitleSource(BatchFunction) + - class SubtitleBlockParser(MapFunction) + - class SubtitleGlossaryChecker(MapFunction) + - class SubtitleTimingChecker(MapFunction) + - class SubtitleQCSink(SinkFunction) +新建: examples/run_subtitle_qc.py +``` + +**预期收益**：按片时或项目收费 | **实现周期**：2周 + +______________________________________________________________________ + +### 74. **多渠道内容排期系统** + +**现实场景痛点**：品牌和内容团队需要同时运营公众号、短视频、社媒和官网，人工排期容易漏档和撞题。 + +**发现需求点**： + +- 需要统一活动节点、产品节奏和渠道约束 +- 需要生成周/月内容日历 +- 需要提示主题重复和资源冲突 + +**解决方案**： + +``` +活动计划 → 渠道规则映射 → 内容主题分配 → 冲突检查 → 排期表输出 +``` + +**需要的AI应用**： + +- 内容主题分配 +- 排期冲突识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把多渠道规则、主题池和时间计划放入统一数据流 +- 冲突检查和推荐逻辑可快速替换，适合运营迭代 +- 相比灵感型 agent 生成，更适合真正可执行的排期生产 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/content_scheduler/ + pipeline.py: + - def run_content_scheduler_pipeline(plan_file, channel_file, output_file) + operators.py: + - class CampaignPlanSource(BatchFunction) + - class ChannelRuleMapper(MapFunction) + - class TopicAllocator(MapFunction) + - class ScheduleConflictDetector(MapFunction) + - class ContentScheduleSink(SinkFunction) +新建: examples/run_content_scheduler.py +``` + +**预期收益**：月费3-12万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 75. **媒体资料归档检索系统** + +**现实场景痛点**：媒体公司和品牌内容团队积累了大量音视频、图片和文稿素材，但后期几乎找不到。 + +**发现需求点**： + +- 需要统一抽取标题、人物、主题、日期等元数据 +- 需要去重并支持主题检索 +- 需要输出可复用的素材索引 + +**解决方案**： + +``` +媒体素材 → 元数据抽取 → 标签生成 → 去重归档 → 检索索引输出 +``` + +**需要的AI应用**： + +- 媒体标签生成 +- 素材去重检索 + +**AI应用关键解决问题的创新点**： + +- SAGE适合对海量素材做持续索引更新，而不是离线一次性入库 +- 标签、去重、归档可拆成可复核算子，避免黑箱管理 +- 相比只做聊天式搜索，先把资料治理好更符合商业落地价值 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/media_archive_search/ + pipeline.py: + - def run_media_archive_search_pipeline(asset_dir, output_dir) + operators.py: + - class MediaAssetSource(BatchFunction) + - class MediaMetadataExtractor(MapFunction) + - class MediaTagger(MapFunction) + - class MediaDuplicateDetector(MapFunction) + - class MediaArchiveSink(SinkFunction) +新建: examples/run_media_archive_search.py +``` + +**预期收益**：年费15-40万 | **实现周期**：3周 + +______________________________________________________________________ + +### 76. **产线传感器异常看护系统** + +**现实场景痛点**：制造企业产线传感器告警多、误报多，班组长难以及时判断真正需要处理的异常。 + +**发现需求点**： + +- 需要持续接收温度、压力、振动等传感器记录 +- 需要按设备和工位识别异常模式 +- 需要输出告警级别和排查建议 + +**解决方案**： + +``` +传感器流 → 设备映射 → 异常特征计算 → 告警分级 → 看护清单输出 +``` + +**需要的AI应用**： + +- 传感器异常识别 +- 告警优先级排序 + +**AI应用关键解决问题的创新点**： + +- SAGE天然适合连续传感器流，Local 到 FlowNet 的扩展成本低 +- 告警规则可持续替换，不需要重建全链路 +- 相比 LangGraph 这种面向对话和任务编排的框架，SAGE更贴合工业信号流处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/factory_watch/ + pipeline.py: + - def run_factory_watch_pipeline(sensor_file, output_file) + operators.py: + - class SensorSource(BatchFunction) + - class SensorDeviceMapper(MapFunction) + - class SensorAnomalyScorer(MapFunction) + - class SensorAlertLeveler(MapFunction) + - class SensorWatchSink(SinkFunction) +新建: examples/run_factory_watch.py +``` + +**预期收益**：单厂年费20-60万 | **实现周期**：3周 + +______________________________________________________________________ + +### 77. **冷链运输越界监控系统** + +**现实场景痛点**：冷链运输一旦温度越界，药品和生鲜可能整批报废，但很多企业只能事后看报表。 + +**发现需求点**： + +- 需要处理车辆、箱体、批次温度流 +- 需要识别持续越界和恢复失败情况 +- 需要输出批次级风险清单 + +**解决方案**： + +``` +冷链记录 → 批次关联 → 越界检测 → 风险升级 → 监控报告输出 +``` + +**需要的AI应用**： + +- 温度越界识别 +- 冷链质量风险预警 + +**AI应用关键解决问题的创新点**： + +- SAGE适合连续记录流和批次关联处理，不必靠多段脚本拼接 +- 越界规则、升级规则都可模块化，符合质量审计需求 +- 相比通用 agent，不需要复杂对话，重点是稳定处理时序记录 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/cold_chain_watch/ + pipeline.py: + - def run_cold_chain_watch_pipeline(record_file, output_file) + operators.py: + - class ColdChainRecordSource(BatchFunction) + - class ColdChainBatchMatcher(MapFunction) + - class TemperatureExcursionDetector(MapFunction) + - class ColdChainRiskScorer(MapFunction) + - class ColdChainSink(SinkFunction) +新建: examples/run_cold_chain_watch.py +``` + +**预期收益**：月费5-20万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 78. **温室种植协同助手** + +**现实场景痛点**：温室种植需要同时兼顾温湿度、灌溉、病虫预警和人工巡检，人工协调成本高。 + +**发现需求点**： + +- 需要持续接入环境和巡检数据 +- 需要识别异常环境组合 +- 需要输出灌溉与巡检建议 + +**解决方案**： + +``` +温室数据 → 区域映射 → 环境异常判断 → 农事建议生成 → 协同清单输出 +``` + +**需要的AI应用**： + +- 环境异常识别 +- 农事协同建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多源农场数据的持续流式治理和建议生成 +- 告警、建议、分派都可以沿同一数据流继续下游处理 +- 相比以聊天为中心的框架，更适合农业运营中的高频自动化链路 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/greenhouse_assistant/ + pipeline.py: + - def run_greenhouse_assistant_pipeline(sensor_file, task_file, output_file) + operators.py: + - class GreenhouseSensorSource(BatchFunction) + - class GreenhouseZoneMapper(MapFunction) + - class ClimateAnomalyDetector(MapFunction) + - class GreenhouseAdviceBuilder(MapFunction) + - class GreenhouseSink(SinkFunction) +新建: examples/run_greenhouse_assistant.py +``` + +**预期收益**：单园区年费15-40万 | **实现周期**：3周 + +______________________________________________________________________ + +## 第七部分：79-91 公共服务、财务与可持续应用 + +### 79. **社区民生热点漂移监测系统** + +**现实场景痛点**：街道、社区和城运中心不仅要知道“当前有什么诉求”，更要及时发现某个片区的问题类型是否正在持续漂移，比如噪声投诉突然转向停车矛盾或物业服务失衡。 + +**发现需求点**： + +- 需要把不同时间段、不同片区的民生问题做结构化聚合 +- 需要识别热点问题类型的变化趋势 +- 需要输出片区治理优先级而不是具体工单分派 + +**解决方案**： + +``` +民生事件流 → 区域映射 → 问题主题聚合 → 漂移趋势识别 → 治理看板输出 +``` + +**需要的AI应用**： + +- 热点主题漂移识别 +- 片区治理优先级建议 + +**AI应用关键解决问题的创新点**： + +- 这项应用替换了市民诉求智能分派，避免与已实现的诉求路由/工单分派类能力重复 +- SAGE适合把多周、多月的诉求流和巡检流放在同一处理链里做趋势识别 +- 相比只做单条工单分派，更能直接服务街道治理和资源投放决策 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/community_hotspot_drift/ + pipeline.py: + - def run_community_hotspot_drift_pipeline(event_file, output_file) + operators.py: + - class CommunityEventSource(BatchFunction) + - class CommunityZoneMapper(MapFunction) + - class CommunityTopicAggregator(MapFunction) + - class HotspotDriftDetector(MapFunction) + - class CommunityInsightSink(SinkFunction) +新建: examples/run_community_hotspot_drift.py +``` + +**预期收益**：单区年费15-35万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 80. **交通突发事件简报系统** + +**现实场景痛点**：交管部门要同时汇总事故、拥堵、施工和天气信息，人工写简报滞后。 + +**发现需求点**： + +- 需要接入路况、报警、施工公告等多源数据 +- 需要识别影响范围和优先级 +- 需要输出简报给指挥中心 + +**解决方案**： + +``` +交通事件流 → 事件归并 → 影响评估 → 优先级排序 → 简报输出 +``` + +**需要的AI应用**： + +- 交通事件聚合 +- 处置优先级推荐 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多源事件流持续归并和加工 +- 影响评估与简报生成是天然的线性处理链 +- 相比单次问答系统，更适合指挥中心的持续态势产出 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/traffic_briefing/ + pipeline.py: + - def run_traffic_briefing_pipeline(event_file, output_file) + operators.py: + - class TrafficEventSource(BatchFunction) + - class TrafficEventMerger(MapFunction) + - class TrafficImpactScorer(MapFunction) + - class TrafficBriefFormatter(MapFunction) + - class TrafficBriefSink(SinkFunction) +新建: examples/run_traffic_briefing.py +``` + +**预期收益**：单城年费15-40万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 81. **城市设施维修调度系统** + +**现实场景痛点**：路灯、井盖、道路破损等设施问题分散在多个渠道，维修工单排程缺少统一视图。 + +**发现需求点**： + +- 需要聚合巡检、投诉和传感器工单 +- 需要按区域和紧急程度排队 +- 需要输出日调度计划 + +**解决方案**： + +``` +维修工单 → 位置归并 → 紧急度打分 → 路线调度 → 计划输出 +``` + +**需要的AI应用**： + +- 维修工单聚类 +- 调度优先级推荐 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把工单聚类、打分、输出放进单一数据流 +- 新增维修规则只需替换算子，不影响整体链路 +- 相比传统审批流工具，更适合持续流入的城市工单场景 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/urban_repair_scheduler/ + pipeline.py: + - def run_urban_repair_scheduler_pipeline(ticket_file, output_file) + operators.py: + - class RepairTicketSource(BatchFunction) + - class RepairGeoMapper(MapFunction) + - class RepairPriorityScorer(MapFunction) + - class RepairRoutePlanner(MapFunction) + - class RepairScheduleSink(SinkFunction) +新建: examples/run_urban_repair_scheduler.py +``` + +**预期收益**：单城年费20-60万 | **实现周期**：3周 + +______________________________________________________________________ + +### 82. **许可申报材料审查系统** + +**现实场景痛点**：企业或政府窗口在处理许可申请时，最耗时间的是反复核对材料是否齐全、格式是否合规。 + +**发现需求点**： + +- 需要按申报类型检查材料清单 +- 需要识别缺页、缺章、缺字段等问题 +- 需要生成退回原因和补充建议 + +**解决方案**： + +``` +申报材料 → 材料类型识别 → 清单校验 → 缺失项标记 → 审查结果输出 +``` + +**需要的AI应用**： + +- 材料完备性检查 +- 退回原因生成 + +**AI应用关键解决问题的创新点**： + +- SAGE适合将材料批处理、规则校验、结果输出固定成标准流程 +- 校验依据清晰，便于窗口部门和申请方对齐 +- 相比对话型助手，更适合政务申报中高确定性的材料审查 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/permit_material_review/ + pipeline.py: + - def run_permit_material_review_pipeline(input_dir, output_file) + operators.py: + - class PermitMaterialSource(BatchFunction) + - class PermitDocumentClassifier(MapFunction) + - class PermitChecklistChecker(MapFunction) + - class PermitReviewFormatter(MapFunction) + - class PermitReviewSink(SinkFunction) +新建: examples/run_permit_material_review.py +``` + +**预期收益**：按窗口或年费收费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 83. **市政协同文档检索系统** + +**现实场景痛点**：政务人员查找政策文件、会议纪要和办事流程时，经常在多个系统之间来回切换。 + +**发现需求点**： + +- 需要把政策、通知、决议、流程文件统一索引 +- 需要按问题返回出处和版本 +- 需要支持部门内部共用 + +**解决方案**： + +``` +政务文档 → 分段索引 → 元数据归一化 → 问题匹配 → 引用结果输出 +``` + +**需要的AI应用**： + +- 政务文档检索 +- 条款出处定位 + +**AI应用关键解决问题的创新点**： + +- SAGE擅长先把杂乱文档做底层结构化治理，再服务上层检索 +- 索引更新与问答输出可以拆成两段稳定管道 +- 相比聊天式知识库，更适合政务场景对来源确定性的要求 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/municipal_search/ + pipeline.py: + - def run_municipal_search_pipeline(doc_dir, question_file, output_file) + operators.py: + - class MunicipalDocSource(BatchFunction) + - class MunicipalChunker(FlatMapFunction) + - class MunicipalQuestionMatcher(MapFunction) + - class MunicipalAnswerFormatter(MapFunction) + - class MunicipalSearchSink(SinkFunction) +新建: examples/run_municipal_search.py +``` + +**预期收益**：年费15-40万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 84. **预算执行偏差预警系统** + +**现实场景痛点**：企业和事业单位在月中、季中很难及时发现预算执行偏离，往往等到月末或季度复盘时才暴露超支、预算挪用和项目进度失衡问题。 + +**发现需求点**： + +- 需要把预算计划、实际发生额和项目进度统一比对 +- 需要识别持续超支、执行滞后和异常科目偏移 +- 需要输出责任中心级的偏差预警 + +**解决方案**： + +``` +预算数据 → 科目映射 → 计划实际比对 → 偏差趋势识别 → 预警报告输出 +``` + +**需要的AI应用**： + +- 预算偏差识别 +- 超支趋势预警 + +**AI应用关键解决问题的创新点**： + +- 这项应用替换了费用报销合规审核，避免与已实现的票据审核、对账和凭证类应用重复 +- SAGE适合持续接收预算、费用、项目进度等多源财务记录，并做增量偏差计算 +- 相比只审核单笔单据，更能直接服务财务管理层的过程控制和预算纠偏 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/budget_variance_alert/ + pipeline.py: + - def run_budget_variance_alert_pipeline(plan_file, actual_file, output_file) + operators.py: + - class BudgetPlanSource(BatchFunction) + - class BudgetActualSource(BatchFunction) + - class BudgetCategoryMapper(MapFunction) + - class BudgetVarianceDetector(MapFunction) + - class BudgetAlertSink(SinkFunction) +新建: examples/run_budget_variance_alert.py +``` + +**预期收益**：月费5-18万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 85. **企业现金流预测系统** + +**现实场景痛点**：中小企业财务往往只能在月底回看现金流，难以及时识别短期资金压力。 + +**发现需求点**： + +- 需要融合订单、回款、开票和应付计划 +- 需要按周输出现金流预测 +- 需要提示短缺风险和资金缺口时间点 + +**解决方案**： + +``` +财务数据 → 收支特征构造 → 现金流预测 → 风险判断 → 周报输出 +``` + +**需要的AI应用**： + +- 短期现金流预测 +- 资金风险预警 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把多源财务记录统一为同一数据流做持续预测 +- 预测与阈值告警可以独立迭代，不影响上下游 +- 相比临时表格分析，更适合财务周期性运营需求 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/cashflow_watch/ + pipeline.py: + - def run_cashflow_watch_pipeline(input_dir, output_file) + operators.py: + - class CashflowSource(BatchFunction) + - class CashflowFeatureBuilder(MapFunction) + - class CashflowForecaster(MapFunction) + - class CashflowRiskMarker(MapFunction) + - class CashflowSink(SinkFunction) +新建: examples/run_cashflow_watch.py +``` + +**预期收益**：月费5-20万 | **实现周期**：3周 + +______________________________________________________________________ + +### 86. **电商退货原因挖掘系统** + +**现实场景痛点**：电商平台退货量高，但商家往往只看到“七天无理由”，真正问题无法沉淀。 + +**发现需求点**： + +- 需要融合退货文本、客服记录和商品属性 +- 需要聚类高频退货原因 +- 需要输出改进建议给运营和供应链 + +**解决方案**： + +``` +退货记录 → 文本与属性融合 → 原因聚类 → 问题排序 → 改进清单输出 +``` + +**需要的AI应用**： + +- 退货原因聚类 +- 可改善问题识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合订单后链路数据的批量规整和连续分析 +- 聚类前的数据清洗、原因打标、分组输出可串成同一流程 +- 相比只做客服问答，更能直接服务商家经营提效 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/return_reason_mining/ + pipeline.py: + - def run_return_reason_mining_pipeline(return_file, output_file) + operators.py: + - class ReturnRecordSource(BatchFunction) + - class ReturnFeatureFusion(MapFunction) + - class ReturnReasonClusterer(MapFunction) + - class ReturnImprovementBuilder(MapFunction) + - class ReturnMiningSink(SinkFunction) +新建: examples/run_return_reason_mining.py +``` + +**预期收益**：月费3-12万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 87. **门店运营日报生成系统** + +**现实场景痛点**：连锁门店每天要汇总缺货、客诉、损耗和人员异常，店长花很多时间在整理日报上。 + +**发现需求点**： + +- 需要整合 POS、库存、值班和工单数据 +- 需要自动生成门店日报结构 +- 需要突出异常事项和次日动作 + +**解决方案**： + +``` +门店数据 → 日指标汇总 → 异常识别 → 动作项整理 → 日报输出 +``` + +**需要的AI应用**： + +- 门店异常总结 +- 次日动作建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合把多来源门店数据做日终批处理并沉淀成固定产出链路 +- 汇总逻辑和异常规则分离，便于区域运营统一维护 +- 相比人工汇报或聊天式总结，更适合规模化连锁运营 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/store_daily_digest/ + pipeline.py: + - def run_store_daily_digest_pipeline(input_dir, output_file) + operators.py: + - class StoreOpsSource(BatchFunction) + - class StoreMetricAggregator(MapFunction) + - class StoreExceptionDetector(MapFunction) + - class StoreActionBuilder(MapFunction) + - class StoreDigestSink(SinkFunction) +新建: examples/run_store_daily_digest.py +``` + +**预期收益**：按门店数订阅收费 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 88. **碳排数据采集归集系统** + +**现实场景痛点**：企业做碳盘查时，能耗、运输、采购和生产数据散落在多个系统里，人工汇总费时费力。 + +**发现需求点**： + +- 需要统一采集多系统碳排相关数据 +- 需要按口径做字段映射和单位换算 +- 需要输出可审计的归集明细 + +**解决方案**： + +``` +碳排数据源 → 字段抽取 → 单位换算 → 口径映射 → 归集台账输出 +``` + +**需要的AI应用**： + +- 碳排数据归一化 +- 缺失数据识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多源业务数据的持续采集与清洗，是典型的数据流型应用 +- 归集规则可以独立为算子，便于不同企业口径切换 +- 相比通用 agent，SAGE更适合强结构化、强审计的 ESG 数据处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/carbon_collection/ + pipeline.py: + - def run_carbon_collection_pipeline(input_dir, output_file) + operators.py: + - class CarbonDataSource(BatchFunction) + - class CarbonFieldExtractor(MapFunction) + - class CarbonUnitNormalizer(MapFunction) + - class CarbonLedgerBuilder(MapFunction) + - class CarbonCollectionSink(SinkFunction) +新建: examples/run_carbon_collection.py +``` + +**预期收益**：项目费10-30万 | **实现周期**：3周 + +______________________________________________________________________ + +### 89. **光伏场站告警系统** + +**现实场景痛点**：光伏运维团队面对逆变器、组件和天气信号时，很难及时定位真正影响发电的异常。 + +**发现需求点**： + +- 需要处理设备告警与气象数据 +- 需要识别低发电、停机和环境因素异常 +- 需要输出运维工单线索 + +**解决方案**： + +``` +场站数据 → 设备关联 → 发电异常识别 → 告警分级 → 运维清单输出 +``` + +**需要的AI应用**： + +- 发电异常识别 +- 运维告警优先级排序 + +**AI应用关键解决问题的创新点**： + +- SAGE适合将设备信号和天气因素放在同一数据流处理 +- Local 环境可先在单站点验证，再扩展到多站点 FlowNet +- 相比 agent 式巡检描述，更适合连续设备数据的稳定处理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/solar_alerting/ + pipeline.py: + - def run_solar_alerting_pipeline(sensor_file, weather_file, output_file) + operators.py: + - class SolarSignalSource(BatchFunction) + - class SolarWeatherJoiner(MapFunction) + - class SolarAnomalyDetector(MapFunction) + - class SolarPriorityScorer(MapFunction) + - class SolarAlertSink(SinkFunction) +新建: examples/run_solar_alerting.py +``` + +**预期收益**：单场站年费10-30万 | **实现周期**：2.5周 + +______________________________________________________________________ + +### 90. **校园碳排报告系统** + +**现实场景痛点**：高校做碳排申报和年度披露时，教学楼、宿舍、食堂和校车数据分散，报告制作周期长。 + +**发现需求点**： + +- 需要整合建筑、交通和活动数据 +- 需要按校区和部门汇总排放量 +- 需要输出报告附件和年度摘要 + +**解决方案**： + +``` +校园数据 → 排放因子映射 → 分项汇总 → 报告模板填充 → 年报输出 +``` + +**需要的AI应用**： + +- 校园碳排汇总 +- 报告缺项识别 + +**AI应用关键解决问题的创新点**： + +- SAGE适合多校区、多系统数据的持续归集和汇总 +- 报告填充和缺项检查都能作为下游算子复用 +- 相比手工年度拼报表，更适合长期校园治理场景 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/campus_emission_report/ + pipeline.py: + - def run_campus_emission_report_pipeline(input_dir, output_file) + operators.py: + - class CampusEmissionSource(BatchFunction) + - class EmissionFactorMapper(MapFunction) + - class CampusEmissionAggregator(MapFunction) + - class CampusReportFormatter(MapFunction) + - class CampusReportSink(SinkFunction) +新建: examples/run_campus_emission_report.py +``` + +**预期收益**：项目费8-25万 | **实现周期**：3周 + +______________________________________________________________________ + +### 91. **数据中心容量与冷却监测系统** + +**现实场景痛点**：数据中心运维需要同时关注机柜容量、冷却压力和异常事件，很多团队仍然靠多个系统切换查看。 + +**发现需求点**： + +- 需要整合容量、功耗、温度和告警日志 +- 需要识别热点机柜和冷却风险 +- 需要输出容量预警与运维建议 + +**解决方案**： + +``` +机房数据 → 机柜映射 → 容量与温度特征计算 → 风险打分 → 监测报告输出 +``` + +**需要的AI应用**： + +- 机柜风险识别 +- 容量预警建议 + +**AI应用关键解决问题的创新点**： + +- SAGE适合处理持续到来的监控指标和日志流 +- 容量评估、冷却判断、告警输出能沿同一管道完成 +- 相比偏任务编排的框架，SAGE更适合高吞吐的运维数据流治理 + +**具体AI应用的实现计划**： + +``` +新建: apps/src/sage/apps/data_center_watch/ + pipeline.py: + - def run_data_center_watch_pipeline(metric_file, alert_file, output_file) + operators.py: + - class DataCenterMetricSource(BatchFunction) + - class RackMapper(MapFunction) + - class CapacityCoolingScorer(MapFunction) + - class DataCenterRiskMarker(MapFunction) + - class DataCenterWatchSink(SinkFunction) +新建: examples/run_data_center_watch.py +``` + +**预期收益**：单机房年费20-50万 | **实现周期**：3周 + +______________________________________________________________________ diff --git a/examples/README.md b/examples/README.md index 049d3e9..5f077c1 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,6 +13,98 @@ This directory contains maintained entry scripts for the `sage.apps` package. - `run_work_report.py` — Work report generation demo. - `run_patent_landscape_mapper.py` — Patent clustering and whitespace opportunity mapping demo. - `run_student_improvement.py` — Personalized score improvement system MVP. +- `run_log_parser.py` — Enterprise log parsing demo. +- `run_data_cleaner.py` — CSV and tabular data cleaning demo. +- `run_resume_parser.py` — Resume standardization demo. +- `run_feedback_analyzer.py` — Customer feedback keyword analysis demo. +- `run_web_scraper.py` — Web page scraping and table extraction demo. +- `run_academic_metadata.py` — Academic metadata extraction demo. +- `run_customer_deduplication.py` — Customer deduplication demo. +- `run_voucher_classifier.py` — Financial voucher classification demo. +- `run_product_sync.py` — Product synchronization demo. +- `run_doc_classifier.py` — Rule-based document classification demo. +- `run_contract_matcher.py` — Contract template matching demo. +- `run_news_aggregator.py` — News aggregation and deduplication demo. +- `run_ticket_router.py` — Service ticket routing demo. +- `run_content_moderation.py` — Content moderation demo. +- `run_contract_risk.py` — Contract clause risk analysis demo. +- `run_user_behavior_analytics.py` — User behavior analytics demo. +- `run_inventory_alert.py` — Inventory alerting demo. +- `run_quality_defect_filter.py` — Quality defect filtering demo. +- `run_permission_audit.py` — Permission audit demo. +- `run_order_anomaly_detector.py` — Order anomaly detection demo. +- `run_attendance_alert.py` — Attendance anomaly alert demo. +- `run_lead_scoring.py` — Sales opportunity scoring demo. +- `run_arbitrage_detector.py` — Exchange-rate arbitrage demo. +- `run_weather_sales_forecast.py` — Weather-driven sales forecast demo. +- `run_geo_recommendation.py` — Geo recommendation demo. +- `run_company_credit.py` — Company credit assessment demo. +- `run_movie_scheduling_optimizer.py` — Movie scheduling optimization demo. +- `run_real_estate_valuation.py` — Real estate valuation demo. +- `run_multi_factor_credit_score.py` — Multi-source user credit scoring demo. +- `run_logistics_cost_optimizer.py` — Logistics cost optimization demo. +- `run_medical_registration_optimizer.py` — Medical registration optimization demo. +- `run_exhibition_heatmap.py` — Exhibition heat and congestion analysis demo. +- `run_dorm_energy_optimizer.py` — Dormitory energy optimization demo. +- `run_restaurant_sales_analysis.py` — Menu and dish sales optimization demo. +- `run_warehouse_slot_optimizer.py` — Warehouse slot optimization demo. +- `run_paper_classifier.py` — Paper topic and keyword classification demo. +- `run_vendor_evaluation_standardizer.py` — Vendor evaluation normalization and risk marking demo. +- `run_content_tagger.py` — Content tagging and label selection demo. +- `run_partner_profile_hub.py` — Partner profile consolidation demo. +- `run_subscription_dispatch.py` — Subscription matching and dispatch demo. +- `run_contract_versioning.py` — Contract version parsing and diff analysis demo. +- `run_invoice_reconciliation.py` — Invoice and order reconciliation demo. +- `run_project_risk_monitor.py` — Project risk monitoring demo. +- `run_learning_record_hub.py` — Learning record aggregation demo. +- `run_supply_chain_tracker.py` — Supply chain timeline and delay risk demo. +- `run_compliance_doc_manager.py` — Compliance review reminder demo. +- `run_mail_classifier.py` — Mail category and priority classification demo. +- `run_meeting_minutes.py` — Meeting transcript summarization demo. +- `run_policy_update_notifier.py` — Policy update impact notification demo. +- `run_export_transformer.py` — Export payload transformation demo. +- `run_api_log_analytics.py` — API log metric and anomaly analysis demo. +- `run_backup_sync.py` — Backup increment and consistency sync demo. +- `run_patent_competition_monitor.py` — Patent infringement signal monitoring and competitor digest + demo. +- `run_grant_subscription.py` — Grant subscription matching and alert prioritization demo. +- `run_experiment_review.py` — Experiment review and anomaly tagging demo. +- `run_repro_audit.py` — Reproducibility audit and manifest risk scoring demo. +- `run_benchmark_watch.py` — Benchmark drift watch and trend tagging demo. +- `run_course_qa_helper.py` — Course QA retrieval and answer drafting demo. +- `run_lesson_scheduler.py` — Lesson scheduling and conflict detection demo. +- `run_assignment_feedback.py` — Assignment feedback structuring demo. +- `run_skill_gap_diagnosis.py` — Skill gap diagnosis and coaching priority demo. +- `run_interview_coach.py` — Interview coaching and improvement planning demo. +- `run_campus_aid_gap_alert.py` — Campus aid eligibility gap alert demo. +- `run_triage_structurer.py` — Clinical triage structuring and priority assignment demo. +- `run_radiology_followup_loop.py` — Radiology follow-up loop monitoring demo. +- `run_drug_leaflet_extractor.py` — Drug leaflet extraction and dosage normalization demo. +- `run_lab_turnaround_alert.py` — Lab turnaround anomaly alert demo. +- `run_quote_compare.py` — Quote comparison and preferred vendor scoring demo. +- `run_policy_search_helper.py` — Policy search helper and citation drafting demo. +- `run_knowledge_cleanup.py` — Knowledge cleanup and duplicate detection demo. +- `run_podcast_highlight.py` — Podcast highlight extraction demo. +- `run_brand_compliance_review.py` — Brand compliance review and risk scoring demo. +- `run_subtitle_qc.py` — Subtitle quality control demo. +- `run_content_scheduler.py` — Content scheduling and capacity conflict demo. +- `run_media_archive_search.py` — Media archive tagging and duplicate search demo. +- `run_factory_watch.py` — Factory sensor watch and alert leveling demo. +- `run_cold_chain_watch.py` — Cold chain excursion monitoring demo. +- `run_greenhouse_assistant.py` — Greenhouse climate anomaly and action planning demo. +- `run_community_hotspot_drift.py` — Community hotspot drift monitoring demo. +- `run_traffic_briefing.py` — Traffic briefing and dispatch priority demo. +- `run_urban_repair_scheduler.py` — Urban repair scheduling and crew routing demo. +- `run_permit_material_review.py` — Permit material review and missing item detection demo. +- `run_municipal_search.py` — Municipal document search and citation drafting demo. +- `run_budget_variance_alert.py` — Budget variance alert and overspend detection demo. +- `run_cashflow_watch.py` — Cashflow watch and projected balance risk demo. +- `run_return_reason_mining.py` — Return reason mining and improvement suggestion demo. +- `run_store_daily_digest.py` — Store daily digest and action planning demo. +- `run_carbon_collection.py` — Carbon collection and ledger generation demo. +- `run_solar_alerting.py` — Solar plant alert prioritization demo. +- `run_campus_emission_report.py` — Campus emission aggregation and report drafting demo. +- `run_data_center_watch.py` — Data center capacity and cooling risk watch demo. - `run_supply_chain_alert.py` — Supply chain anomaly alert dashboard MVP. - `run_supply_chain_alert_api.py` — FastAPI service for the supply chain anomaly alert dashboard, including a multi-role browser dashboard at `/dashboard/ui`. @@ -35,6 +127,97 @@ python examples/run_literature_report_assistant.py python examples/run_work_report.py python examples/run_patent_landscape_mapper.py python examples/run_student_improvement.py +python examples/run_log_parser.py +python examples/run_data_cleaner.py +python examples/run_resume_parser.py +python examples/run_feedback_analyzer.py +python examples/run_web_scraper.py +python examples/run_academic_metadata.py +python examples/run_customer_deduplication.py +python examples/run_voucher_classifier.py +python examples/run_product_sync.py +python examples/run_doc_classifier.py +python examples/run_contract_matcher.py +python examples/run_news_aggregator.py +python examples/run_ticket_router.py +python examples/run_content_moderation.py +python examples/run_contract_risk.py +python examples/run_user_behavior_analytics.py +python examples/run_inventory_alert.py +python examples/run_quality_defect_filter.py +python examples/run_permission_audit.py +python examples/run_order_anomaly_detector.py +python examples/run_attendance_alert.py +python examples/run_lead_scoring.py +python examples/run_arbitrage_detector.py +python examples/run_weather_sales_forecast.py +python examples/run_geo_recommendation.py +python examples/run_company_credit.py +python examples/run_movie_scheduling_optimizer.py +python examples/run_real_estate_valuation.py +python examples/run_multi_factor_credit_score.py +python examples/run_logistics_cost_optimizer.py +python examples/run_medical_registration_optimizer.py +python examples/run_exhibition_heatmap.py +python examples/run_dorm_energy_optimizer.py +python examples/run_restaurant_sales_analysis.py +python examples/run_warehouse_slot_optimizer.py +python examples/run_paper_classifier.py +python examples/run_vendor_evaluation_standardizer.py +python examples/run_content_tagger.py +python examples/run_partner_profile_hub.py +python examples/run_subscription_dispatch.py +python examples/run_contract_versioning.py +python examples/run_invoice_reconciliation.py +python examples/run_project_risk_monitor.py +python examples/run_learning_record_hub.py +python examples/run_supply_chain_tracker.py +python examples/run_compliance_doc_manager.py +python examples/run_mail_classifier.py +python examples/run_meeting_minutes.py +python examples/run_policy_update_notifier.py +python examples/run_export_transformer.py +python examples/run_api_log_analytics.py +python examples/run_backup_sync.py +python examples/run_patent_competition_monitor.py +python examples/run_grant_subscription.py +python examples/run_experiment_review.py +python examples/run_repro_audit.py +python examples/run_benchmark_watch.py +python examples/run_course_qa_helper.py +python examples/run_lesson_scheduler.py +python examples/run_assignment_feedback.py +python examples/run_skill_gap_diagnosis.py +python examples/run_interview_coach.py +python examples/run_campus_aid_gap_alert.py +python examples/run_triage_structurer.py +python examples/run_radiology_followup_loop.py +python examples/run_drug_leaflet_extractor.py +python examples/run_lab_turnaround_alert.py +python examples/run_quote_compare.py +python examples/run_policy_search_helper.py +python examples/run_knowledge_cleanup.py +python examples/run_podcast_highlight.py +python examples/run_brand_compliance_review.py +python examples/run_subtitle_qc.py +python examples/run_content_scheduler.py +python examples/run_media_archive_search.py +python examples/run_factory_watch.py +python examples/run_cold_chain_watch.py +python examples/run_greenhouse_assistant.py +python examples/run_community_hotspot_drift.py +python examples/run_traffic_briefing.py +python examples/run_urban_repair_scheduler.py +python examples/run_permit_material_review.py +python examples/run_municipal_search.py +python examples/run_budget_variance_alert.py +python examples/run_cashflow_watch.py +python examples/run_return_reason_mining.py +python examples/run_store_daily_digest.py +python examples/run_carbon_collection.py +python examples/run_solar_alerting.py +python examples/run_campus_emission_report.py +python examples/run_data_center_watch.py python examples/run_supply_chain_alert.py python examples/run_supply_chain_alert_api.py python examples/run_ticket_triage.py diff --git a/examples/run_academic_metadata.py b/examples/run_academic_metadata.py new file mode 100644 index 0000000..99ced59 --- /dev/null +++ b/examples/run_academic_metadata.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Run the academic metadata application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.academic_metadata import run_academic_metadata_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.academic_metadata: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run academic metadata extraction") + parser.add_argument("--input-path", required=True, help="Input file, CSV, or directory") + parser.add_argument("--output", required=True, help="Output JSON file") + parser.add_argument("--verbose", action="store_true", help="Enable verbose logging") + args = parser.parse_args() + run_academic_metadata_pipeline(args.input_path, args.output, verbose=args.verbose) + + +if __name__ == "__main__": + main() diff --git a/examples/run_api_log_analytics.py b/examples/run_api_log_analytics.py new file mode 100644 index 0000000..bf637db --- /dev/null +++ b/examples/run_api_log_analytics.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.api_log_analytics import run_api_log_analytics_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.api_log_analytics: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run API log analytics pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_api_log_analytics_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_arbitrage_detector.py b/examples/run_arbitrage_detector.py new file mode 100644 index 0000000..9a503e8 --- /dev/null +++ b/examples/run_arbitrage_detector.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.arbitrage_detector import run_arbitrage_detector_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.arbitrage_detector: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run arbitrage detector") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_arbitrage_detector_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_assignment_feedback.py b/examples/run_assignment_feedback.py new file mode 100644 index 0000000..3196f51 --- /dev/null +++ b/examples/run_assignment_feedback.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.assignment_feedback import run_assignment_feedback_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.assignment_feedback: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run assignment feedback") + parser.add_argument("--draft-file", required=True) + parser.add_argument("--rubric-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_assignment_feedback_pipeline(args.draft_file, args.rubric_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_attendance_alert.py b/examples/run_attendance_alert.py new file mode 100644 index 0000000..18bc509 --- /dev/null +++ b/examples/run_attendance_alert.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.attendance_alert import run_attendance_alert_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.attendance_alert: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run attendance alerting") + parser.add_argument("--input-file", required=True) + parser.add_argument("--schedule-file") + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_attendance_alert_pipeline(args.input_file, args.output, schedule_file=args.schedule_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_backup_sync.py b/examples/run_backup_sync.py new file mode 100644 index 0000000..e9bc9a2 --- /dev/null +++ b/examples/run_backup_sync.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.backup_sync import run_backup_sync_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.backup_sync: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run backup sync pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_backup_sync_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_benchmark_watch.py b/examples/run_benchmark_watch.py new file mode 100644 index 0000000..a1cf890 --- /dev/null +++ b/examples/run_benchmark_watch.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.benchmark_watch import run_benchmark_watch_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.benchmark_watch: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run benchmark watch") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_benchmark_watch_pipeline(args.input_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_brand_compliance_review.py b/examples/run_brand_compliance_review.py new file mode 100644 index 0000000..54ef072 --- /dev/null +++ b/examples/run_brand_compliance_review.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.brand_compliance_review import run_brand_compliance_review_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.brand_compliance_review: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run brand compliance review") + parser.add_argument("--asset-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_brand_compliance_review_pipeline(args.asset_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_budget_variance_alert.py b/examples/run_budget_variance_alert.py new file mode 100644 index 0000000..91b743b --- /dev/null +++ b/examples/run_budget_variance_alert.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.budget_variance_alert import run_budget_variance_alert_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.budget_variance_alert: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run budget variance alert pipeline") + parser.add_argument("--plan-file", required=True) + parser.add_argument("--actual-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_budget_variance_alert_pipeline(args.plan_file, args.actual_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_campus_aid_gap_alert.py b/examples/run_campus_aid_gap_alert.py new file mode 100644 index 0000000..2d6b00b --- /dev/null +++ b/examples/run_campus_aid_gap_alert.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.campus_aid_gap_alert import run_campus_aid_gap_alert_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.campus_aid_gap_alert: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run campus aid gap alert pipeline") + parser.add_argument("--application-file", required=True) + parser.add_argument("--profile-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_campus_aid_gap_alert_pipeline(args.application_file, args.profile_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_campus_emission_report.py b/examples/run_campus_emission_report.py new file mode 100644 index 0000000..bd687e0 --- /dev/null +++ b/examples/run_campus_emission_report.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.campus_emission_report import run_campus_emission_report_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.campus_emission_report: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run campus emission report") + parser.add_argument("--input-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_campus_emission_report_pipeline(args.input_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_carbon_collection.py b/examples/run_carbon_collection.py new file mode 100644 index 0000000..a8307f7 --- /dev/null +++ b/examples/run_carbon_collection.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.carbon_collection import run_carbon_collection_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.carbon_collection: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run carbon collection") + parser.add_argument("--input-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_carbon_collection_pipeline(args.input_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_cashflow_watch.py b/examples/run_cashflow_watch.py new file mode 100644 index 0000000..1f1d7e3 --- /dev/null +++ b/examples/run_cashflow_watch.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.cashflow_watch import run_cashflow_watch_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.cashflow_watch: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run cashflow watch") + parser.add_argument("--input-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_cashflow_watch_pipeline(args.input_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_cold_chain_watch.py b/examples/run_cold_chain_watch.py new file mode 100644 index 0000000..734f5f0 --- /dev/null +++ b/examples/run_cold_chain_watch.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.cold_chain_watch import run_cold_chain_watch_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.cold_chain_watch: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run cold chain watch") + parser.add_argument("--record-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_cold_chain_watch_pipeline(args.record_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_community_hotspot_drift.py b/examples/run_community_hotspot_drift.py new file mode 100644 index 0000000..e7012cf --- /dev/null +++ b/examples/run_community_hotspot_drift.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.community_hotspot_drift import run_community_hotspot_drift_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.community_hotspot_drift: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run community hotspot drift pipeline") + parser.add_argument("--event-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_community_hotspot_drift_pipeline(args.event_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_company_credit.py b/examples/run_company_credit.py new file mode 100644 index 0000000..14a125f --- /dev/null +++ b/examples/run_company_credit.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.company_credit import run_company_credit_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.company_credit: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run company credit scoring") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--api-config") + args = parser.parse_args() + run_company_credit_pipeline(args.input_file, args.output, api_config=args.api_config) + + +if __name__ == "__main__": + main() diff --git a/examples/run_compliance_doc_manager.py b/examples/run_compliance_doc_manager.py new file mode 100644 index 0000000..640e19d --- /dev/null +++ b/examples/run_compliance_doc_manager.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.compliance_doc_manager import run_compliance_doc_manager_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.compliance_doc_manager: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run compliance doc manager pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--reference-date") + args = parser.parse_args() + run_compliance_doc_manager_pipeline( + args.input_file, args.output, reference_date=args.reference_date + ) + + +if __name__ == "__main__": + main() diff --git a/examples/run_content_moderation.py b/examples/run_content_moderation.py new file mode 100644 index 0000000..efcf90b --- /dev/null +++ b/examples/run_content_moderation.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the content moderation application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.content_moderation import run_content_moderation_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.content_moderation: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run content moderation") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_content_moderation_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_content_scheduler.py b/examples/run_content_scheduler.py new file mode 100644 index 0000000..2124edc --- /dev/null +++ b/examples/run_content_scheduler.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.content_scheduler import run_content_scheduler_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.content_scheduler: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run content scheduler") + parser.add_argument("--plan-file", required=True) + parser.add_argument("--channel-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_content_scheduler_pipeline(args.plan_file, args.channel_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_content_tagger.py b/examples/run_content_tagger.py new file mode 100644 index 0000000..c2f4ab2 --- /dev/null +++ b/examples/run_content_tagger.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.content_tagger import run_content_tagger_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.content_tagger: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run content tagger pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--top-k", type=int, default=8) + args = parser.parse_args() + run_content_tagger_pipeline(args.input_file, args.output, top_k=args.top_k) + + +if __name__ == "__main__": + main() diff --git a/examples/run_contract_matcher.py b/examples/run_contract_matcher.py new file mode 100644 index 0000000..a4fe47b --- /dev/null +++ b/examples/run_contract_matcher.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Run the contract matcher application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.contract_matcher import run_contract_matcher_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.contract_matcher: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run contract matcher") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--template-file") + parser.add_argument("--top-k", type=int, default=3) + args = parser.parse_args() + run_contract_matcher_pipeline( + args.input_file, args.output, template_file=args.template_file, top_k=args.top_k + ) + + +if __name__ == "__main__": + main() diff --git a/examples/run_contract_risk.py b/examples/run_contract_risk.py new file mode 100644 index 0000000..8763066 --- /dev/null +++ b/examples/run_contract_risk.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the contract risk application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.contract_risk import run_contract_risk_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.contract_risk: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run contract risk analysis") + parser.add_argument("--input-path", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_contract_risk_pipeline(args.input_path, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_contract_versioning.py b/examples/run_contract_versioning.py new file mode 100644 index 0000000..1aab250 --- /dev/null +++ b/examples/run_contract_versioning.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.contract_versioning import run_contract_versioning_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.contract_versioning: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run contract versioning pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_contract_versioning_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_course_qa_helper.py b/examples/run_course_qa_helper.py new file mode 100644 index 0000000..bb10ce4 --- /dev/null +++ b/examples/run_course_qa_helper.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.course_qa_helper import run_course_qa_helper_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.course_qa_helper: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run course QA helper") + parser.add_argument("--doc-dir", required=True) + parser.add_argument("--question-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_course_qa_helper_pipeline(args.doc_dir, args.question_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_customer_deduplication.py b/examples/run_customer_deduplication.py new file mode 100644 index 0000000..916f5ed --- /dev/null +++ b/examples/run_customer_deduplication.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Run the customer deduplication application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.customer_deduplication import run_customer_deduplication_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.customer_deduplication: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run customer deduplication") + parser.add_argument("--input-file", required=True, help="Input CSV file") + parser.add_argument("--output", required=True, help="Output JSON file") + parser.add_argument("--threshold", type=float, default=0.9, help="Duplicate threshold") + args = parser.parse_args() + run_customer_deduplication_pipeline(args.input_file, args.output, threshold=args.threshold) + + +if __name__ == "__main__": + main() diff --git a/examples/run_data_center_watch.py b/examples/run_data_center_watch.py new file mode 100644 index 0000000..c624d2c --- /dev/null +++ b/examples/run_data_center_watch.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.data_center_watch import run_data_center_watch_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.data_center_watch: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run data center watch") + parser.add_argument("--metric-file", required=True) + parser.add_argument("--alert-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_data_center_watch_pipeline(args.metric_file, args.alert_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_data_cleaner.py b/examples/run_data_cleaner.py new file mode 100644 index 0000000..a3817e6 --- /dev/null +++ b/examples/run_data_cleaner.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +""" +Data Cleaner System Example + +This script demonstrates the Data Cleaner application from sage-apps. +It reads CSV files, cleans data (type conversion, missing values, anomalies), +and outputs structured data in CSV or JSON format. + +Requirements: + pip install -e sage-apps + # Or: pip install isage-apps + +Usage: + python examples/run_data_cleaner.py --input raw.csv --output cleaned.csv + python examples/run_data_cleaner.py --input raw.csv --output cleaned.json --output-format json + python examples/run_data_cleaner.py --help + +Test Configuration: + @test_category: apps + @test_speed: fast +""" + +import argparse +import sys + +try: + from sage.apps.data_cleaner import run_data_cleaner_pipeline + from sage.foundation import CustomLogger +except ImportError as e: + print(f"Error importing sage.apps.data_cleaner: {e}") + print("\nPlease install sage-apps:") + print(" cd sage-apps && pip install -e .") + print(" Or: pip install isage-apps") + sys.exit(1) + + +def parse_key_value_list(s: str) -> dict[str, str]: + """Parse key:value,key:value format.""" + result = {} + if not s: + return result + + for pair in s.split(","): + if ":" in pair: + k, v = pair.split(":", 1) + result[k.strip()] = v.strip() + return result + + +def parse_list(s: str) -> list[str]: + """Parse comma-separated list.""" + if not s: + return [] + return [item.strip() for item in s.split(",")] + + +def main(): + """Run the data cleaner pipeline.""" + parser = argparse.ArgumentParser( + description="SAGE Data Cleaner System - Clean and standardize CSV data", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic cleaning + python %(prog)s --input raw.csv --output cleaned.csv + + # Type conversion + python %(prog)s \\ + --input raw.csv \\ + --output cleaned.csv \\ + --type-rules age:int,salary:float,active:bool,hire_date:date + + # With anomaly detection + python %(prog)s \\ + --input raw.csv \\ + --output cleaned.csv \\ + --numeric-fields age,salary + + # JSON output with all features + python %(prog)s \\ + --input raw.csv \\ + --output cleaned.json \\ + --output-format json \\ + --key-fields email,phone \\ + --numeric-fields age,salary + + # Custom missing value strategy + python %(prog)s \\ + --input raw.csv \\ + --output cleaned.csv \\ + --fill-strategy drop + """, + ) + + parser.add_argument( + "--input", + type=str, + required=True, + help="Path to input CSV file", + ) + + parser.add_argument( + "--output", + type=str, + required=True, + help="Path to output file", + ) + + parser.add_argument( + "--type-rules", + type=str, + default="", + help="Type conversion rules (field:type,field:type). Types: int, float, bool, date", + ) + + parser.add_argument( + "--numeric-fields", + type=str, + default="", + help="Numeric fields for anomaly detection (comma-separated)", + ) + + parser.add_argument( + "--key-fields", + type=str, + default="", + help="Key fields for duplicate detection (comma-separated)", + ) + + parser.add_argument( + "--fill-strategy", + type=str, + default="drop", + help="Strategy for missing values: 'drop', 'forward', or 'field:value,field:value'", + ) + + parser.add_argument( + "--output-format", + type=str, + choices=["csv", "json"], + default="csv", + help="Output format (default: csv)", + ) + + parser.add_argument( + "--verbose", + action="store_true", + help="Enable verbose logging", + ) + + args = parser.parse_args() + + # Parse arguments + type_rules = parse_key_value_list(args.type_rules) + numeric_fields = parse_list(args.numeric_fields) + key_fields = parse_list(args.key_fields) + + # Parse fill strategy + if args.fill_strategy in ["drop", "forward"]: + fill_strategy = args.fill_strategy + else: + fill_strategy = parse_key_value_list(args.fill_strategy) + + if args.verbose: + logger = CustomLogger("DataCleanerExample") + logger.info("Starting Data Cleaner with:") + logger.info(f" Input file: {args.input}") + logger.info(f" Output file: {args.output}") + logger.info(f" Type rules: {type_rules}") + logger.info(f" Numeric fields: {numeric_fields}") + logger.info(f" Key fields: {key_fields}") + + try: + run_data_cleaner_pipeline( + input_file=args.input, + output_file=args.output, + type_rules=type_rules or None, + fill_strategy=fill_strategy, + numeric_fields=numeric_fields or None, + key_fields=key_fields or None, + output_format=args.output_format, + verbose=args.verbose, + ) + except KeyboardInterrupt: + print("\n\nData cleaner interrupted by user") + sys.exit(0) + except Exception as e: + print(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/run_doc_classifier.py b/examples/run_doc_classifier.py new file mode 100644 index 0000000..f65c229 --- /dev/null +++ b/examples/run_doc_classifier.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the document classifier application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.doc_classifier import run_doc_classifier_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.doc_classifier: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run document classification") + parser.add_argument("--input-file", required=True, help="Input text, CSV, or JSON file") + parser.add_argument("--output", required=True, help="Output JSON file") + args = parser.parse_args() + run_doc_classifier_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_dorm_energy_optimizer.py b/examples/run_dorm_energy_optimizer.py new file mode 100644 index 0000000..3893f43 --- /dev/null +++ b/examples/run_dorm_energy_optimizer.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.dorm_energy_optimizer import run_dorm_energy_optimizer_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.dorm_energy_optimizer: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run dorm energy optimization") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_dorm_energy_optimizer_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_drug_leaflet_extractor.py b/examples/run_drug_leaflet_extractor.py new file mode 100644 index 0000000..7f6486b --- /dev/null +++ b/examples/run_drug_leaflet_extractor.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.drug_leaflet_extractor import run_drug_leaflet_extractor_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.drug_leaflet_extractor: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run drug leaflet extractor") + parser.add_argument("--input-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_drug_leaflet_extractor_pipeline(args.input_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_exhibition_heatmap.py b/examples/run_exhibition_heatmap.py new file mode 100644 index 0000000..c0a9f0e --- /dev/null +++ b/examples/run_exhibition_heatmap.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.exhibition_heatmap import run_exhibition_heatmap_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.exhibition_heatmap: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run exhibition heatmap") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_exhibition_heatmap_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_experiment_review.py b/examples/run_experiment_review.py new file mode 100644 index 0000000..caa5b73 --- /dev/null +++ b/examples/run_experiment_review.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.experiment_review import run_experiment_review_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.experiment_review: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run experiment review") + parser.add_argument("--log-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_experiment_review_pipeline(args.log_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_export_transformer.py b/examples/run_export_transformer.py new file mode 100644 index 0000000..5fd46ef --- /dev/null +++ b/examples/run_export_transformer.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.export_transformer import run_export_transformer_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.export_transformer: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run export transformer pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--output-format", default="json") + args = parser.parse_args() + run_export_transformer_pipeline(args.input_file, args.output, output_format=args.output_format) + + +if __name__ == "__main__": + main() diff --git a/examples/run_factory_watch.py b/examples/run_factory_watch.py new file mode 100644 index 0000000..3297463 --- /dev/null +++ b/examples/run_factory_watch.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.factory_watch import run_factory_watch_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.factory_watch: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run factory watch") + parser.add_argument("--sensor-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_factory_watch_pipeline(args.sensor_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_feedback_analyzer.py b/examples/run_feedback_analyzer.py new file mode 100644 index 0000000..b29ad8e --- /dev/null +++ b/examples/run_feedback_analyzer.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +""" +Feedback Analyzer System Example + +This script demonstrates the Feedback Analyzer application from sage-apps. +It processes customer feedback, extracts keywords, and generates statistics. + +Requirements: + pip install -e sage-apps + # Or: pip install isage-apps + +Usage: + python examples/run_feedback_analyzer.py --feedback-file feedback.txt --output keywords.json + python examples/run_feedback_analyzer.py --feedback-file feedback.csv --delimiter "," --output keywords.json + python examples/run_feedback_analyzer.py --help + +Test Configuration: + @test_category: apps + @test_speed: fast +""" + +import argparse +import sys + +try: + from sage.apps.feedback_analyzer import run_feedback_analyzer_pipeline + from sage.foundation import CustomLogger +except ImportError as e: + print(f"Error importing sage.apps.feedback_analyzer: {e}") + print("\nPlease install sage-apps:") + print(" cd sage-apps && pip install -e .") + print(" Or: pip install isage-apps") + sys.exit(1) + + +def main(): + """Run the feedback analyzer pipeline.""" + parser = argparse.ArgumentParser( + description="SAGE Feedback Analyzer System - Extract keywords from customer feedback", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Basic feedback analysis + python %(prog)s --feedback-file feedback.txt --output keywords.json + + # CSV format feedback + python %(prog)s \\ + --feedback-file feedback.csv \\ + --delimiter "," \\ + --output keywords.json + + # Custom top keywords count + python %(prog)s \\ + --feedback-file feedback.txt \\ + --output keywords.json \\ + --top-n 100 \\ + --verbose + """, + ) + + parser.add_argument( + "--feedback-file", + type=str, + required=True, + help="Path to feedback file (text or CSV)", + ) + + parser.add_argument( + "--output", + type=str, + default="feedback_keywords.json", + help="Output JSON file path (default: feedback_keywords.json)", + ) + + parser.add_argument( + "--delimiter", + type=str, + default="\t", + help="CSV delimiter (default: tab)", + ) + + parser.add_argument( + "--top-n", + type=int, + default=50, + help="Number of top keywords to include (default: 50)", + ) + + parser.add_argument( + "--verbose", + action="store_true", + help="Enable verbose logging", + ) + + args = parser.parse_args() + + if args.verbose: + logger = CustomLogger("FeedbackAnalyzerExample") + logger.info("Starting Feedback Analyzer with:") + logger.info(f" Feedback file: {args.feedback_file}") + logger.info(f" Output file: {args.output}") + logger.info(f" Top keywords: {args.top_n}") + + try: + run_feedback_analyzer_pipeline( + feedback_file=args.feedback_file, + output_file=args.output, + top_n=args.top_n, + verbose=args.verbose, + ) + except KeyboardInterrupt: + print("\n\nFeedback analyzer interrupted by user") + sys.exit(0) + except Exception as e: + print(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/run_geo_recommendation.py b/examples/run_geo_recommendation.py new file mode 100644 index 0000000..e1b13f9 --- /dev/null +++ b/examples/run_geo_recommendation.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.geo_recommendation import run_geo_recommendation_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.geo_recommendation: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run geo recommendation") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_geo_recommendation_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_grant_subscription.py b/examples/run_grant_subscription.py new file mode 100644 index 0000000..f80da17 --- /dev/null +++ b/examples/run_grant_subscription.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.grant_subscription import run_grant_subscription_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.grant_subscription: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run grant subscription") + parser.add_argument("--announcement-file", required=True) + parser.add_argument("--profile-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_grant_subscription_pipeline(args.announcement_file, args.profile_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_greenhouse_assistant.py b/examples/run_greenhouse_assistant.py new file mode 100644 index 0000000..f2b77b7 --- /dev/null +++ b/examples/run_greenhouse_assistant.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.greenhouse_assistant import run_greenhouse_assistant_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.greenhouse_assistant: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run greenhouse assistant") + parser.add_argument("--sensor-file", required=True) + parser.add_argument("--task-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_greenhouse_assistant_pipeline(args.sensor_file, args.task_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_interview_coach.py b/examples/run_interview_coach.py new file mode 100644 index 0000000..5303d75 --- /dev/null +++ b/examples/run_interview_coach.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.interview_coach import run_interview_coach_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.interview_coach: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run interview coach") + parser.add_argument("--answer-file", required=True) + parser.add_argument("--rubric-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_interview_coach_pipeline(args.answer_file, args.rubric_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_inventory_alert.py b/examples/run_inventory_alert.py new file mode 100644 index 0000000..c61cbdb --- /dev/null +++ b/examples/run_inventory_alert.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the inventory alert application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.inventory_alert import run_inventory_alert_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.inventory_alert: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run inventory alerting") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_inventory_alert_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_invoice_reconciliation.py b/examples/run_invoice_reconciliation.py new file mode 100644 index 0000000..e9fc46a --- /dev/null +++ b/examples/run_invoice_reconciliation.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.invoice_reconciliation import run_invoice_reconciliation_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.invoice_reconciliation: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run invoice reconciliation pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--order-file") + parser.add_argument("--tolerance", type=float, default=1.0) + args = parser.parse_args() + run_invoice_reconciliation_pipeline( + args.input_file, args.output, order_file=args.order_file, tolerance=args.tolerance + ) + + +if __name__ == "__main__": + main() diff --git a/examples/run_knowledge_cleanup.py b/examples/run_knowledge_cleanup.py new file mode 100644 index 0000000..55e9101 --- /dev/null +++ b/examples/run_knowledge_cleanup.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.knowledge_cleanup import run_knowledge_cleanup_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.knowledge_cleanup: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run knowledge cleanup") + parser.add_argument("--article-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_knowledge_cleanup_pipeline(args.article_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_lab_turnaround_alert.py b/examples/run_lab_turnaround_alert.py new file mode 100644 index 0000000..d0a9b8a --- /dev/null +++ b/examples/run_lab_turnaround_alert.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.lab_turnaround_alert import run_lab_turnaround_alert_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.lab_turnaround_alert: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run lab turnaround alert pipeline") + parser.add_argument("--record-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_lab_turnaround_alert_pipeline(args.record_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_lead_scoring.py b/examples/run_lead_scoring.py new file mode 100644 index 0000000..4d1b085 --- /dev/null +++ b/examples/run_lead_scoring.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.lead_scoring import run_lead_scoring_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.lead_scoring: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run lead scoring") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_lead_scoring_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_learning_record_hub.py b/examples/run_learning_record_hub.py new file mode 100644 index 0000000..37114ba --- /dev/null +++ b/examples/run_learning_record_hub.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.learning_record_hub import run_learning_record_hub_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.learning_record_hub: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run learning record hub pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_learning_record_hub_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_lesson_scheduler.py b/examples/run_lesson_scheduler.py new file mode 100644 index 0000000..fc721d3 --- /dev/null +++ b/examples/run_lesson_scheduler.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.lesson_scheduler import run_lesson_scheduler_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.lesson_scheduler: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run lesson scheduler") + parser.add_argument("--plan-file", required=True) + parser.add_argument("--resource-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_lesson_scheduler_pipeline(args.plan_file, args.resource_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_log_parser.py b/examples/run_log_parser.py new file mode 100644 index 0000000..237b161 --- /dev/null +++ b/examples/run_log_parser.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +""" +Log Parser System Example + +This script demonstrates the Log Parser application from sage-apps. +It reads log files in various formats, parses them, filters by error level, +and outputs structured JSON. + +Requirements: + pip install -e sage-apps + # Or: pip install isage-apps + +Usage: + python examples/run_log_parser.py --log-file app.log --output structured.json + python examples/run_log_parser.py --log-file app.log --console --verbose + python examples/run_log_parser.py --help + +Test Configuration: + @test_category: apps + @test_speed: fast +""" + +import argparse +import sys + +try: + from sage.apps.log_parser import run_log_parser_pipeline + from sage.foundation import CustomLogger +except ImportError as e: + print(f"Error importing sage.apps.log_parser: {e}") + print("\nPlease install sage-apps:") + print(" cd sage-apps && pip install -e .") + print(" Or: pip install isage-apps") + sys.exit(1) + + +def main(): + """Run the log parser pipeline.""" + parser = argparse.ArgumentParser( + description="SAGE Log Parser System - Parse and structure enterprise logs", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Parse log file and output to JSON + python %(prog)s --log-file app.log --output structured.json + + # Parse and display to console + python %(prog)s --log-file app.log --console --verbose + + # Custom error levels + python %(prog)s \\ + --log-file app.log \\ + --output structured.json \\ + --error-levels ERROR,CRITICAL + + # Both file and console output + python %(prog)s \\ + --log-file app.log \\ + --output structured.json \\ + --console + """, + ) + + parser.add_argument( + "--log-file", + type=str, + required=True, + help="Path to the log file to parse", + ) + + parser.add_argument( + "--output", + type=str, + default=None, + help="Output JSON file path (optional)", + ) + + parser.add_argument( + "--error-levels", + type=str, + default="ERROR,CRITICAL,WARN", + help="Comma-separated error levels to filter (default: ERROR,CRITICAL,WARN)", + ) + + parser.add_argument( + "--console", + action="store_true", + help="Also output to console", + ) + + parser.add_argument( + "--verbose", + action="store_true", + help="Enable verbose logging", + ) + + args = parser.parse_args() + + # Parse error levels + error_levels = [level.strip().upper() for level in args.error_levels.split(",")] + + if args.verbose: + logger = CustomLogger("LogParserExample") + logger.info("Starting Log Parser with:") + logger.info(f" Log file: {args.log_file}") + logger.info(f" Output file: {args.output}") + logger.info(f" Error levels: {error_levels}") + + try: + run_log_parser_pipeline( + log_file=args.log_file, + output_file=args.output, + error_levels=error_levels, + verbose=args.verbose, + console_output=args.console, + ) + except KeyboardInterrupt: + print("\n\nLog parser interrupted by user") + sys.exit(0) + except Exception as e: + print(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/run_logistics_cost_optimizer.py b/examples/run_logistics_cost_optimizer.py new file mode 100644 index 0000000..f6b622e --- /dev/null +++ b/examples/run_logistics_cost_optimizer.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.logistics_cost_optimizer import run_logistics_cost_optimizer_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.logistics_cost_optimizer: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run logistics cost optimization") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_logistics_cost_optimizer_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_mail_classifier.py b/examples/run_mail_classifier.py new file mode 100644 index 0000000..5e39bf1 --- /dev/null +++ b/examples/run_mail_classifier.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.mail_classifier import run_mail_classifier_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.mail_classifier: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run mail classifier pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_mail_classifier_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_media_archive_search.py b/examples/run_media_archive_search.py new file mode 100644 index 0000000..98c43f3 --- /dev/null +++ b/examples/run_media_archive_search.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.media_archive_search import run_media_archive_search_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.media_archive_search: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run media archive search") + parser.add_argument("--asset-dir", required=True) + parser.add_argument("--output-dir", required=True) + args = parser.parse_args() + run_media_archive_search_pipeline(args.asset_dir, args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/examples/run_medical_registration_optimizer.py b/examples/run_medical_registration_optimizer.py new file mode 100644 index 0000000..ec32167 --- /dev/null +++ b/examples/run_medical_registration_optimizer.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.medical_registration_optimizer import run_medical_registration_optimizer_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.medical_registration_optimizer: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run medical registration optimization") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_medical_registration_optimizer_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_meeting_minutes.py b/examples/run_meeting_minutes.py new file mode 100644 index 0000000..1bd6763 --- /dev/null +++ b/examples/run_meeting_minutes.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.meeting_minutes import run_meeting_minutes_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.meeting_minutes: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run meeting minutes pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_meeting_minutes_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_movie_scheduling_optimizer.py b/examples/run_movie_scheduling_optimizer.py new file mode 100644 index 0000000..04fcd64 --- /dev/null +++ b/examples/run_movie_scheduling_optimizer.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.movie_scheduling_optimizer import run_movie_scheduling_optimizer_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.movie_scheduling_optimizer: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run movie scheduling optimization") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_movie_scheduling_optimizer_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_multi_factor_credit_score.py b/examples/run_multi_factor_credit_score.py new file mode 100644 index 0000000..daa3ec1 --- /dev/null +++ b/examples/run_multi_factor_credit_score.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.multi_factor_credit_score import run_multi_factor_credit_score_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.multi_factor_credit_score: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run multi-factor credit scoring") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_multi_factor_credit_score_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_municipal_search.py b/examples/run_municipal_search.py new file mode 100644 index 0000000..924a03d --- /dev/null +++ b/examples/run_municipal_search.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.municipal_search import run_municipal_search_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.municipal_search: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run municipal search") + parser.add_argument("--doc-dir", required=True) + parser.add_argument("--question-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_municipal_search_pipeline(args.doc_dir, args.question_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_news_aggregator.py b/examples/run_news_aggregator.py new file mode 100644 index 0000000..d3ddec0 --- /dev/null +++ b/examples/run_news_aggregator.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the news aggregator application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.news_aggregator import run_news_aggregator_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.news_aggregator: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run news aggregation") + parser.add_argument("--source-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_news_aggregator_pipeline(args.source_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_order_anomaly_detector.py b/examples/run_order_anomaly_detector.py new file mode 100644 index 0000000..a6c6e36 --- /dev/null +++ b/examples/run_order_anomaly_detector.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the order anomaly detector application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.order_anomaly_detector import run_order_anomaly_detector_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.order_anomaly_detector: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run order anomaly detection") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_order_anomaly_detector_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_paper_classifier.py b/examples/run_paper_classifier.py new file mode 100644 index 0000000..5b37840 --- /dev/null +++ b/examples/run_paper_classifier.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.paper_classifier import run_paper_classifier_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.paper_classifier: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run paper classifier pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--top-k", type=int, default=5) + args = parser.parse_args() + run_paper_classifier_pipeline(args.input_file, args.output, top_k=args.top_k) + + +if __name__ == "__main__": + main() diff --git a/examples/run_partner_profile_hub.py b/examples/run_partner_profile_hub.py new file mode 100644 index 0000000..1a5f1fd --- /dev/null +++ b/examples/run_partner_profile_hub.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.partner_profile_hub import run_partner_profile_hub_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.partner_profile_hub: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run partner profile hub pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_partner_profile_hub_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_patent_competition_monitor.py b/examples/run_patent_competition_monitor.py new file mode 100644 index 0000000..b9fdd25 --- /dev/null +++ b/examples/run_patent_competition_monitor.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.patent_competition_monitor import run_patent_competition_monitor_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.patent_competition_monitor: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run patent competition monitor") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output-dir", required=True) + args = parser.parse_args() + run_patent_competition_monitor_pipeline(args.input_file, args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/examples/run_permission_audit.py b/examples/run_permission_audit.py new file mode 100644 index 0000000..8d382b8 --- /dev/null +++ b/examples/run_permission_audit.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the permission audit application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.permission_audit import run_permission_audit_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.permission_audit: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run permission audit") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_permission_audit_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_permit_material_review.py b/examples/run_permit_material_review.py new file mode 100644 index 0000000..86be74f --- /dev/null +++ b/examples/run_permit_material_review.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.permit_material_review import run_permit_material_review_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.permit_material_review: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run permit material review") + parser.add_argument("--input-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_permit_material_review_pipeline(args.input_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_podcast_highlight.py b/examples/run_podcast_highlight.py new file mode 100644 index 0000000..4b64f98 --- /dev/null +++ b/examples/run_podcast_highlight.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.podcast_highlight import run_podcast_highlight_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.podcast_highlight: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run podcast highlight") + parser.add_argument("--transcript-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_podcast_highlight_pipeline(args.transcript_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_policy_search_helper.py b/examples/run_policy_search_helper.py new file mode 100644 index 0000000..c646376 --- /dev/null +++ b/examples/run_policy_search_helper.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.policy_search_helper import run_policy_search_helper_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.policy_search_helper: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run policy search helper") + parser.add_argument("--doc-dir", required=True) + parser.add_argument("--question-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_policy_search_helper_pipeline(args.doc_dir, args.question_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_policy_update_notifier.py b/examples/run_policy_update_notifier.py new file mode 100644 index 0000000..6264d25 --- /dev/null +++ b/examples/run_policy_update_notifier.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.policy_update_notifier import run_policy_update_notifier_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.policy_update_notifier: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run policy update notifier pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_policy_update_notifier_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_product_sync.py b/examples/run_product_sync.py new file mode 100644 index 0000000..f65751d --- /dev/null +++ b/examples/run_product_sync.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the product sync application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.product_sync import run_product_sync_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.product_sync: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run product synchronization") + parser.add_argument("--input-file", required=True, help="Input CSV or JSON file") + parser.add_argument("--output", required=True, help="Output JSON file") + args = parser.parse_args() + run_product_sync_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_project_risk_monitor.py b/examples/run_project_risk_monitor.py new file mode 100644 index 0000000..825127f --- /dev/null +++ b/examples/run_project_risk_monitor.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.project_risk_monitor import run_project_risk_monitor_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.project_risk_monitor: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run project risk monitor pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_project_risk_monitor_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_quality_defect_filter.py b/examples/run_quality_defect_filter.py new file mode 100644 index 0000000..fbd151e --- /dev/null +++ b/examples/run_quality_defect_filter.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the quality defect filter application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.quality_defect_filter import run_quality_defect_filter_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.quality_defect_filter: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run quality defect filtering") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_quality_defect_filter_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_quote_compare.py b/examples/run_quote_compare.py new file mode 100644 index 0000000..b98ba4d --- /dev/null +++ b/examples/run_quote_compare.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.quote_compare import run_quote_compare_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.quote_compare: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run quote compare") + parser.add_argument("--input-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_quote_compare_pipeline(args.input_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_radiology_followup_loop.py b/examples/run_radiology_followup_loop.py new file mode 100644 index 0000000..67181a3 --- /dev/null +++ b/examples/run_radiology_followup_loop.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.radiology_followup_loop import run_radiology_followup_loop_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.radiology_followup_loop: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run radiology followup loop") + parser.add_argument("--report-file", required=True) + parser.add_argument("--patient-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_radiology_followup_loop_pipeline(args.report_file, args.patient_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_real_estate_valuation.py b/examples/run_real_estate_valuation.py new file mode 100644 index 0000000..929c490 --- /dev/null +++ b/examples/run_real_estate_valuation.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.real_estate_valuation import run_real_estate_valuation_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.real_estate_valuation: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run real estate valuation") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_real_estate_valuation_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_repro_audit.py b/examples/run_repro_audit.py new file mode 100644 index 0000000..b19363f --- /dev/null +++ b/examples/run_repro_audit.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.repro_audit import run_repro_audit_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.repro_audit: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run repro audit") + parser.add_argument("--manifest-file", required=True) + parser.add_argument("--output-dir", required=True) + args = parser.parse_args() + run_repro_audit_pipeline(args.manifest_file, args.output_dir) + + +if __name__ == "__main__": + main() diff --git a/examples/run_restaurant_sales_analysis.py b/examples/run_restaurant_sales_analysis.py new file mode 100644 index 0000000..b9eb75c --- /dev/null +++ b/examples/run_restaurant_sales_analysis.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.restaurant_sales_analysis import run_restaurant_sales_analysis_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.restaurant_sales_analysis: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run restaurant sales analysis") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_restaurant_sales_analysis_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_resume_parser.py b/examples/run_resume_parser.py new file mode 100644 index 0000000..02068fa --- /dev/null +++ b/examples/run_resume_parser.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +""" +Resume Parser System Example + +This script demonstrates the Resume Parser application from sage-apps. +It reads resume files, extracts structured information, and outputs JSON. + +Requirements: + pip install -e sage-apps + # Or: pip install isage-apps + +Usage: + python examples/run_resume_parser.py --resume-dir ./resumes --output parsed.json + python examples/run_resume_parser.py --resume-files resume1.txt resume2.txt --output parsed.json + python examples/run_resume_parser.py --help + +Test Configuration: + @test_category: apps + @test_speed: fast +""" + +import argparse +import sys + +try: + from sage.apps.resume_parser import run_resume_parser_pipeline + from sage.foundation import CustomLogger +except ImportError as e: + print(f"Error importing sage.apps.resume_parser: {e}") + print("\nPlease install sage-apps:") + print(" cd sage-apps && pip install -e .") + print(" Or: pip install isage-apps") + sys.exit(1) + + +def main(): + """Run the resume parser pipeline.""" + parser = argparse.ArgumentParser( + description="SAGE Resume Parser System - Parse and structure resume data", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Parse resumes from directory + python %(prog)s --resume-dir ./resumes --output parsed.json + + # Parse specific files + python %(prog)s \\ + --resume-files resume1.txt resume2.txt resume3.txt \\ + --output parsed.json + + # With verbose logging + python %(prog)s \\ + --resume-dir ./resumes \\ + --output parsed.json \\ + --verbose + """, + ) + + parser.add_argument( + "--resume-dir", + type=str, + default=None, + help="Directory containing resume files", + ) + + parser.add_argument( + "--resume-files", + type=str, + nargs="*", + default=None, + help="List of resume file paths", + ) + + parser.add_argument( + "--output", + type=str, + default="parsed_resumes.json", + help="Output JSON file path (default: parsed_resumes.json)", + ) + + parser.add_argument( + "--verbose", + action="store_true", + help="Enable verbose logging", + ) + + args = parser.parse_args() + + if not args.resume_dir and not args.resume_files: + parser.print_help() + print("\nError: Please specify either --resume-dir or --resume-files") + sys.exit(1) + + if args.verbose: + logger = CustomLogger("ResumeParserExample") + logger.info("Starting Resume Parser with:") + logger.info(f" Resume dir: {args.resume_dir}") + logger.info(f" Resume files: {args.resume_files}") + logger.info(f" Output file: {args.output}") + + try: + run_resume_parser_pipeline( + resume_dir=args.resume_dir, + resume_files=args.resume_files, + output_file=args.output, + verbose=args.verbose, + ) + except KeyboardInterrupt: + print("\n\nResume parser interrupted by user") + sys.exit(0) + except Exception as e: + print(f"Error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/examples/run_return_reason_mining.py b/examples/run_return_reason_mining.py new file mode 100644 index 0000000..6b81462 --- /dev/null +++ b/examples/run_return_reason_mining.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.return_reason_mining import run_return_reason_mining_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.return_reason_mining: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run return reason mining") + parser.add_argument("--return-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_return_reason_mining_pipeline(args.return_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_skill_gap_diagnosis.py b/examples/run_skill_gap_diagnosis.py new file mode 100644 index 0000000..bfa6f06 --- /dev/null +++ b/examples/run_skill_gap_diagnosis.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.skill_gap_diagnosis import run_skill_gap_diagnosis_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.skill_gap_diagnosis: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run skill gap diagnosis") + parser.add_argument("--record-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_skill_gap_diagnosis_pipeline(args.record_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_solar_alerting.py b/examples/run_solar_alerting.py new file mode 100644 index 0000000..b8db4a4 --- /dev/null +++ b/examples/run_solar_alerting.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.solar_alerting import run_solar_alerting_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.solar_alerting: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run solar alerting") + parser.add_argument("--sensor-file", required=True) + parser.add_argument("--weather-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_solar_alerting_pipeline(args.sensor_file, args.weather_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_store_daily_digest.py b/examples/run_store_daily_digest.py new file mode 100644 index 0000000..ad27b82 --- /dev/null +++ b/examples/run_store_daily_digest.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.store_daily_digest import run_store_daily_digest_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.store_daily_digest: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run store daily digest") + parser.add_argument("--input-dir", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_store_daily_digest_pipeline(args.input_dir, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_subscription_dispatch.py b/examples/run_subscription_dispatch.py new file mode 100644 index 0000000..85a5ab8 --- /dev/null +++ b/examples/run_subscription_dispatch.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.subscription_dispatch import run_subscription_dispatch_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.subscription_dispatch: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run subscription dispatch pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--subscription-file") + args = parser.parse_args() + run_subscription_dispatch_pipeline( + args.input_file, args.output, subscription_file=args.subscription_file + ) + + +if __name__ == "__main__": + main() diff --git a/examples/run_subtitle_qc.py b/examples/run_subtitle_qc.py new file mode 100644 index 0000000..2622c90 --- /dev/null +++ b/examples/run_subtitle_qc.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.subtitle_qc import run_subtitle_qc_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.subtitle_qc: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run subtitle qc") + parser.add_argument("--subtitle-file", required=True) + parser.add_argument("--glossary-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_subtitle_qc_pipeline(args.subtitle_file, args.glossary_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_supply_chain_tracker.py b/examples/run_supply_chain_tracker.py new file mode 100644 index 0000000..7563b64 --- /dev/null +++ b/examples/run_supply_chain_tracker.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.supply_chain_tracker import run_supply_chain_tracker_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.supply_chain_tracker: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run supply chain tracker pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_supply_chain_tracker_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_ticket_router.py b/examples/run_ticket_router.py new file mode 100644 index 0000000..026991e --- /dev/null +++ b/examples/run_ticket_router.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Run the ticket router application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.ticket_router import run_ticket_router_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.ticket_router: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run ticket routing") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--agents", default="agent_a,agent_b,agent_c") + args = parser.parse_args() + agents = [value.strip() for value in args.agents.split(",") if value.strip()] + run_ticket_router_pipeline(args.input_file, args.output, agents=agents) + + +if __name__ == "__main__": + main() diff --git a/examples/run_traffic_briefing.py b/examples/run_traffic_briefing.py new file mode 100644 index 0000000..66a1e0f --- /dev/null +++ b/examples/run_traffic_briefing.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.traffic_briefing import run_traffic_briefing_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.traffic_briefing: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run traffic briefing") + parser.add_argument("--event-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_traffic_briefing_pipeline(args.event_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_triage_structurer.py b/examples/run_triage_structurer.py new file mode 100644 index 0000000..176c942 --- /dev/null +++ b/examples/run_triage_structurer.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.triage_structurer import run_triage_structurer_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.triage_structurer: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run triage structurer") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_triage_structurer_pipeline(args.input_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_urban_repair_scheduler.py b/examples/run_urban_repair_scheduler.py new file mode 100644 index 0000000..edbf7cf --- /dev/null +++ b/examples/run_urban_repair_scheduler.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.urban_repair_scheduler import run_urban_repair_scheduler_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.urban_repair_scheduler: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run urban repair scheduler") + parser.add_argument("--ticket-file", required=True) + parser.add_argument("--output-file", required=True) + args = parser.parse_args() + run_urban_repair_scheduler_pipeline(args.ticket_file, args.output_file) + + +if __name__ == "__main__": + main() diff --git a/examples/run_user_behavior_analytics.py b/examples/run_user_behavior_analytics.py new file mode 100644 index 0000000..23e5822 --- /dev/null +++ b/examples/run_user_behavior_analytics.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the user behavior analytics application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.user_behavior_analytics import run_user_behavior_analytics_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.user_behavior_analytics: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run user behavior analytics") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_user_behavior_analytics_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_vendor_evaluation_standardizer.py b/examples/run_vendor_evaluation_standardizer.py new file mode 100644 index 0000000..7940704 --- /dev/null +++ b/examples/run_vendor_evaluation_standardizer.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.vendor_evaluation_standardizer import run_vendor_evaluation_standardizer_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.vendor_evaluation_standardizer: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run vendor evaluation standardizer pipeline") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_vendor_evaluation_standardizer_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_voucher_classifier.py b/examples/run_voucher_classifier.py new file mode 100644 index 0000000..e32b707 --- /dev/null +++ b/examples/run_voucher_classifier.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +"""Run the voucher classifier application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.voucher_classifier import run_voucher_classifier_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.voucher_classifier: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run voucher classification") + parser.add_argument("--input-file", required=True, help="Input CSV or text file") + parser.add_argument("--output", required=True, help="Output JSON file") + args = parser.parse_args() + run_voucher_classifier_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_warehouse_slot_optimizer.py b/examples/run_warehouse_slot_optimizer.py new file mode 100644 index 0000000..d90a710 --- /dev/null +++ b/examples/run_warehouse_slot_optimizer.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.warehouse_slot_optimizer import run_warehouse_slot_optimizer_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.warehouse_slot_optimizer: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run warehouse slot optimization") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_warehouse_slot_optimizer_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_weather_sales_forecast.py b/examples/run_weather_sales_forecast.py new file mode 100644 index 0000000..c8825b2 --- /dev/null +++ b/examples/run_weather_sales_forecast.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.weather_sales_forecast import run_weather_sales_forecast_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.weather_sales_forecast: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run weather sales forecast") + parser.add_argument("--input-file", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + run_weather_sales_forecast_pipeline(args.input_file, args.output) + + +if __name__ == "__main__": + main() diff --git a/examples/run_web_scraper.py b/examples/run_web_scraper.py new file mode 100644 index 0000000..e20fe32 --- /dev/null +++ b/examples/run_web_scraper.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Run the web scraper application.""" + +from __future__ import annotations + +import argparse +import sys + +try: + from sage.apps.web_scraper import run_web_scraper_pipeline +except ImportError as exc: + print(f"Error importing sage.apps.web_scraper: {exc}") + sys.exit(1) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run the SAGE web scraper") + parser.add_argument("--url-file", required=True, help="Text file with one URL per line") + parser.add_argument("--output", required=True, help="Output JSON file") + parser.add_argument("--verbose", action="store_true", help="Enable verbose logging") + args = parser.parse_args() + run_web_scraper_pipeline(args.url_file, args.output, verbose=args.verbose) + + +if __name__ == "__main__": + main()