diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0245856..a6869a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,10 +21,7 @@ jobs: - run: python -c "from app.main import app; print('import ok')" - name: Init test DB run: | - cd .. - python scripts/init_db.py - python scripts/seed_data.py - python scripts/compute_baseline.py + python -c "from app.database import init_database; from app.main import DB_PATH; init_database(DB_PATH)" - run: python -m pytest tests/ -v --tb=short frontend: diff --git a/Makefile b/Makefile index 7af39b4..70a7e7d 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,11 @@ # 一键初始化数据库并生成 baseline data: - python3 scripts/init_db.py - python3 scripts/seed_data.py - python3 scripts/compute_baseline.py + cd backend && python3 -c "from app.database import init_database; from pathlib import Path; init_database(Path('data') / 'baseline.db')" + +# 清空种子和基线后重新初始化 +data-refresh: + cd backend && python3 -c "from app.database import init_database; from pathlib import Path; init_database(Path('data') / 'baseline.db', force=True)" # 安装所有依赖 install: diff --git a/backend/app/database/__init__.py b/backend/app/database/__init__.py new file mode 100644 index 0000000..788c60e --- /dev/null +++ b/backend/app/database/__init__.py @@ -0,0 +1,19 @@ +import sqlite3 +from pathlib import Path + +from app.database.schema import create_tables + + +def init_database(db_path: Path, force: bool = False): + """创建数据库表结构（如不存在）；force=True 时清空种子和基线后重写""" + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + create_tables(conn) + conn.commit() + conn.close() + + from app.database.seed_data import seed_data + from app.database.baseline import compute_baseline + + seed_data(db_path, force=force) + compute_baseline(db_path, force=force) diff --git a/scripts/compute_baseline.py b/backend/app/database/baseline.py similarity index 93% rename from scripts/compute_baseline.py rename to backend/app/database/baseline.py index 8b7fc6d..5ce98fa 100644 --- a/scripts/compute_baseline.py +++ b/backend/app/database/baseline.py @@ -1,15 +1,7 @@ -""" -基于 notes 表数据，预计算各垂类的 baseline 统计指标并写入 baseline_stats 表。 - -Usage: - python scripts/compute_baseline.py -""" import sqlite3 import json -import os from collections import Counter - -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "backend", "data", "baseline.db") +from pathlib import Path def upsert_stat(cursor, category, metric_name, metric_value=None, metric_json=None): @@ -174,12 +166,18 @@ def compute_for_category(cursor, category): print(f" [{category}] 已计算 baseline 指标（含粉丝分层与标签分桶）") -def main(): - """计算所有垂类的 baseline 统计指标""" - conn = sqlite3.connect(DB_PATH) +def compute_baseline(db_path: Path, force: bool = False): + """计算所有垂类的 baseline 统计指标；force=True 时清空后重算""" + conn = sqlite3.connect(db_path) cursor = conn.cursor() - cursor.execute("DELETE FROM baseline_stats") + if force: + cursor.execute("DELETE FROM baseline_stats") + else: + cursor.execute("SELECT COUNT(*) FROM baseline_stats") + if cursor.fetchone()[0] > 0: + conn.close() + return for cat in ["food", "fashion", "tech", "travel", "beauty", "fitness", "lifestyle", "home"]: compute_for_category(cursor, cat) @@ -187,7 +185,3 @@ def main(): conn.commit() conn.close() print("所有 baseline 统计指标已计算完毕") - - -if __name__ == "__main__": - main() diff --git a/backend/app/database/schema.py b/backend/app/database/schema.py new file mode 100644 index 0000000..ad3a734 --- /dev/null +++ b/backend/app/database/schema.py @@ -0,0 +1,84 @@ +import sqlite3 + + +def create_tables(conn: sqlite3.Connection): + conn.execute(""" + CREATE TABLE IF NOT EXISTS notes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, + title TEXT NOT NULL, + title_length INTEGER, + content TEXT, + tags TEXT, + publish_hour INTEGER, + likes INTEGER DEFAULT 0, + collects INTEGER DEFAULT 0, + comments INTEGER DEFAULT 0, + followers INTEGER DEFAULT 0, + is_viral INTEGER DEFAULT 0, + cover_has_face INTEGER DEFAULT 0, + cover_text_ratio REAL DEFAULT 0, + cover_saturation REAL DEFAULT 0, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_notes_category ON notes(category)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_notes_viral ON notes(category, is_viral)") + + conn.execute(""" + CREATE TABLE IF NOT EXISTS baseline_stats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + category TEXT NOT NULL, + metric_name TEXT NOT NULL, + metric_value REAL, + metric_json TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + UNIQUE(category, metric_name) + ) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS diagnosis_history ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL, + category TEXT NOT NULL, + overall_score REAL, + grade TEXT, + report_json TEXT NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute(""" + CREATE INDEX IF NOT EXISTS idx_history_created + ON diagnosis_history(created_at DESC) + """) + + conn.execute(""" + CREATE TABLE IF NOT EXISTS usage_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ip TEXT NOT NULL, + action TEXT NOT NULL DEFAULT 'diagnose', + title TEXT DEFAULT '', + category TEXT DEFAULT '', + total_tokens INTEGER DEFAULT 0, + duration_sec REAL DEFAULT 0, + status TEXT DEFAULT 'ok', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_created ON usage_log(created_at DESC)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_ip ON usage_log(ip)") + + conn.execute(""" + CREATE TABLE IF NOT EXISTS visit_log ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + visitor_hash TEXT NOT NULL, + user_agent_hash TEXT DEFAULT '', + path TEXT NOT NULL, + referrer TEXT DEFAULT '', + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_created ON visit_log(created_at DESC)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_visitor ON visit_log(visitor_hash)") + conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_path ON visit_log(path)") diff --git a/scripts/seed_data.py b/backend/app/database/seed_data.py similarity index 95% rename from scripts/seed_data.py rename to backend/app/database/seed_data.py index 3354721..23946d4 100644 --- a/scripts/seed_data.py +++ b/backend/app/database/seed_data.py @@ -1,16 +1,8 @@ -""" -生成模拟 baseline 种子数据用于开发和演示。 -实际比赛前应替换为真实采集的小红书笔记数据。 - -Usage: - python scripts/seed_data.py -""" import sqlite3 import json import random -import os +from pathlib import Path -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "backend", "data", "baseline.db") FOOD_TITLES = [ "手把手教你做日式溏心蛋！零失败！", "一周减脂餐分享｜好吃不胖", @@ -41,7 +33,7 @@ "这个APP改变了我的学习方式", "数码产品年度盘点｜好用到哭", "iPad学习法｜从学渣到学霸", "耳机横评｜千元内最值得买的5款", "NAS入门指南｜打造私人云存储", "手机摄影技巧｜拍出电影质感", - "机械键盘入坑指南｜新手必看", "二手数码避坑指南‼️", + "机械键盘入坑指南｜新手必看", "二手数码避坑指南‼️", "AI工具合集｜效率提升10倍", "极简桌面布置｜打造高效工作台", ] @@ -153,12 +145,18 @@ def generate_notes(category, titles, tags_pool, count=500): return notes -def seed(): - """写入种子数据""" - conn = sqlite3.connect(DB_PATH) +def seed_data(db_path: Path, force: bool = False): + """若 notes 表为空，则填充种子数据；force=True 时清空后重写""" + conn = sqlite3.connect(db_path) cursor = conn.cursor() - cursor.execute("DELETE FROM notes") + if force: + cursor.execute("DELETE FROM notes") + else: + cursor.execute("SELECT COUNT(*) FROM notes") + if cursor.fetchone()[0] > 0: + conn.close() + return all_notes = [] all_notes.extend(generate_notes("food", FOOD_TITLES, FOOD_TAGS, 500)) @@ -181,7 +179,3 @@ def seed(): conn.commit() print(f"已插入 {len(all_notes)} 条种子数据") conn.close() - - -if __name__ == "__main__": - seed() diff --git a/backend/app/local_memory.py b/backend/app/local_memory.py index 4131cab..774cdd2 100644 --- a/backend/app/local_memory.py +++ b/backend/app/local_memory.py @@ -12,10 +12,11 @@ import logging import os from datetime import datetime +from pathlib import Path logger = logging.getLogger("noterx.local_memory") -_DATA_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "data")) +_DATA_ROOT = Path(__file__).parent.parent / "data" WORKSPACE_ROOT = os.path.join(_DATA_ROOT, "noterx_workspace") MEMORY_MD = os.path.join(WORKSPACE_ROOT, "MEMORY.md") MEMORY_DIR = os.path.join(WORKSPACE_ROOT, "memory") diff --git a/backend/app/main.py b/backend/app/main.py index 113bf9d..3bbeb04 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -3,7 +3,7 @@ """ import logging import os -import sqlite3 +from pathlib import Path from contextlib import asynccontextmanager from fastapi import FastAPI @@ -13,69 +13,17 @@ from app.api.routes import router as api_router from app import local_memory +from app.database import init_database +DB_PATH = Path(__file__).parent.parent / "data" / "baseline.db" FRONTEND_DIST = os.path.join(os.path.dirname(__file__), "..", "..", "frontend", "dist") -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "data", "baseline.db") - - -def _ensure_history_table(): - """启动时自动创建 diagnosis_history 表（如不存在）""" - os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) - conn = sqlite3.connect(DB_PATH) - conn.execute(""" - CREATE TABLE IF NOT EXISTS diagnosis_history ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL, - category TEXT NOT NULL, - overall_score REAL, - grade TEXT, - report_json TEXT NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute(""" - CREATE INDEX IF NOT EXISTS idx_history_created - ON diagnosis_history(created_at DESC) - """) - # Usage tracking table - conn.execute(""" - CREATE TABLE IF NOT EXISTS usage_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - ip TEXT NOT NULL, - action TEXT NOT NULL DEFAULT 'diagnose', - title TEXT DEFAULT '', - category TEXT DEFAULT '', - total_tokens INTEGER DEFAULT 0, - duration_sec REAL DEFAULT 0, - status TEXT DEFAULT 'ok', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_created ON usage_log(created_at DESC)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_usage_ip ON usage_log(ip)") - conn.execute(""" - CREATE TABLE IF NOT EXISTS visit_log ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - visitor_hash TEXT NOT NULL, - user_agent_hash TEXT DEFAULT '', - path TEXT NOT NULL, - referrer TEXT DEFAULT '', - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_created ON visit_log(created_at DESC)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_visitor ON visit_log(visitor_hash)") - conn.execute("CREATE INDEX IF NOT EXISTS idx_visit_path ON visit_log(path)") - conn.commit() - conn.close() - local_memory.ensure_memory_md() - @asynccontextmanager async def lifespan(_app: FastAPI): """应用生命周期：启动时自动建表""" - _ensure_history_table() + init_database(DB_PATH) + local_memory.ensure_memory_md() yield logging.basicConfig( @@ -178,12 +126,10 @@ async def serve_app(): async def health(): """详细健康检查，含数据库探测""" import sqlite3 - import os - db_path = os.path.join(os.path.dirname(__file__), "..", "data", "baseline.db") db_ok = False note_count = 0 try: - conn = sqlite3.connect(db_path) + conn = sqlite3.connect(DB_PATH) cur = conn.cursor() cur.execute("SELECT COUNT(*) FROM notes") note_count = cur.fetchone()[0] diff --git a/deploy_backend.py b/deploy_backend.py index b12907a..ae3a379 100644 --- a/deploy_backend.py +++ b/deploy_backend.py @@ -89,10 +89,7 @@ def run(ssh, cmd, check=True): run(ssh, f"{REMOTE_DIR}/backend/venv/bin/pip install -r {REMOTE_DIR}/backend/requirements.txt") # Init DB -print(" Initializing database...") -run(ssh, f"cd {REMOTE_DIR} && {REMOTE_DIR}/backend/venv/bin/python scripts/init_db.py", check=False) -run(ssh, f"cd {REMOTE_DIR} && {REMOTE_DIR}/backend/venv/bin/python scripts/seed_data.py", check=False) -run(ssh, f"cd {REMOTE_DIR} && {REMOTE_DIR}/backend/venv/bin/python scripts/compute_baseline.py", check=False) +print(" Database tables / seed data / baseline will be handled by app startup") # Upload .env print(" Uploading .env...") diff --git a/scripts/init_db.py b/scripts/init_db.py deleted file mode 100644 index ecf746c..0000000 --- a/scripts/init_db.py +++ /dev/null @@ -1,82 +0,0 @@ -""" -初始化 SQLite 数据库，创建 baseline 数据表结构。 - -Usage: - python scripts/init_db.py -""" -import sqlite3 -import os - -DB_PATH = os.path.join(os.path.dirname(__file__), "..", "backend", "data", "baseline.db") - - -def init_database(): - """创建数据库表结构""" - os.makedirs(os.path.dirname(DB_PATH), exist_ok=True) - conn = sqlite3.connect(DB_PATH) - cursor = conn.cursor() - - cursor.execute(""" - CREATE TABLE IF NOT EXISTS notes ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category TEXT NOT NULL, -- food / fashion / tech - title TEXT NOT NULL, - title_length INTEGER, - content TEXT, - tags TEXT, -- JSON array - publish_hour INTEGER, -- 0-23 - likes INTEGER DEFAULT 0, - collects INTEGER DEFAULT 0, - comments INTEGER DEFAULT 0, - followers INTEGER DEFAULT 0, - is_viral INTEGER DEFAULT 0, -- 1=爆款, 0=普通 - cover_has_face INTEGER DEFAULT 0, - cover_text_ratio REAL DEFAULT 0, - cover_saturation REAL DEFAULT 0, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - - cursor.execute(""" - CREATE TABLE IF NOT EXISTS baseline_stats ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - category TEXT NOT NULL, - metric_name TEXT NOT NULL, -- e.g. avg_title_length - metric_value REAL, - metric_json TEXT, -- JSON for complex metrics - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - UNIQUE(category, metric_name) - ) - """) - - cursor.execute(""" - CREATE TABLE IF NOT EXISTS diagnosis_history ( - id TEXT PRIMARY KEY, - title TEXT NOT NULL, - category TEXT NOT NULL, - overall_score REAL, - grade TEXT, - report_json TEXT NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - """) - - cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_history_created - ON diagnosis_history(created_at DESC) - """) - - cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_notes_category ON notes(category) - """) - cursor.execute(""" - CREATE INDEX IF NOT EXISTS idx_notes_viral ON notes(category, is_viral) - """) - - conn.commit() - conn.close() - print(f"数据库已初始化: {os.path.abspath(DB_PATH)}") - - -if __name__ == "__main__": - init_database()