From 3054aacf3dd15503df4403ffbb34250abd7a57f4 Mon Sep 17 00:00:00 2001 From: warkcod Date: Sat, 11 Apr 2026 11:42:53 +0800 Subject: [PATCH] feat(scys): migrate adapters onto latest upstream --- cli-manifest.json | 1321 ++++++++++++++++------ clis/scys/activity.js | 20 + clis/scys/article.js | 22 + clis/scys/common.js | 99 ++ clis/scys/common.test.js | 68 ++ clis/scys/course-download.js | 104 ++ clis/scys/course-download.test.js | 81 ++ clis/scys/course-utils.js | 137 +++ clis/scys/course-utils.test.js | 93 ++ clis/scys/course.js | 50 + clis/scys/extractors.js | 1227 ++++++++++++++++++++ clis/scys/extractors.toc.test.js | 75 ++ clis/scys/feed.js | 23 + clis/scys/opportunity-utils.js | 88 ++ clis/scys/opportunity-utils.test.js | 60 + clis/scys/opportunity.js | 67 ++ clis/scys/read.js | 57 + clis/scys/toc.js | 20 + docs/adapters/browser/scys.md | 55 + docs/adapters/index.md | 1 + docs/developer/scys-schema-guidelines.md | 59 + 21 files changed, 3411 insertions(+), 316 deletions(-) create mode 100644 clis/scys/activity.js create mode 100644 clis/scys/article.js create mode 100644 clis/scys/common.js create mode 100644 clis/scys/common.test.js create mode 100644 clis/scys/course-download.js create mode 100644 clis/scys/course-download.test.js create mode 100644 clis/scys/course-utils.js create mode 100644 clis/scys/course-utils.test.js create mode 100644 clis/scys/course.js create mode 100644 clis/scys/extractors.js create mode 100644 clis/scys/extractors.toc.test.js create mode 100644 clis/scys/feed.js create mode 100644 clis/scys/opportunity-utils.js create mode 100644 clis/scys/opportunity-utils.test.js create mode 100644 clis/scys/opportunity.js create mode 100644 clis/scys/read.js create mode 100644 clis/scys/toc.js create mode 100644 docs/adapters/browser/scys.md create mode 100644 docs/developer/scys-schema-guidelines.md diff --git a/cli-manifest.json b/cli-manifest.json index fea0100f5..ecf86cc1b 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -25,7 +25,8 @@ ], "type": "js", "modulePath": "1688/assets.js", - "sourceFile": "1688/assets.js" + "sourceFile": "1688/assets.js", + "navigateBefore": "https://www.1688.com" }, { "site": "1688", @@ -58,7 +59,8 @@ ], "type": "js", "modulePath": "1688/download.js", - "sourceFile": "1688/download.js" + "sourceFile": "1688/download.js", + "navigateBefore": "https://www.1688.com" }, { "site": "1688", @@ -174,7 +176,8 @@ ], "type": "js", "modulePath": "36kr/article.js", - "sourceFile": "36kr/article.js" + "sourceFile": "36kr/article.js", + "navigateBefore": true }, { "site": "36kr", @@ -519,7 +522,8 @@ ], "type": "js", "modulePath": "antigravity/dump.js", - "sourceFile": "antigravity/dump.js" + "sourceFile": "antigravity/dump.js", + "navigateBefore": true }, { "site": "antigravity", @@ -534,7 +538,8 @@ ], "type": "js", "modulePath": "antigravity/extract-code.js", - "sourceFile": "antigravity/extract-code.js" + "sourceFile": "antigravity/extract-code.js", + "navigateBefore": true }, { "site": "antigravity", @@ -557,7 +562,8 @@ ], "type": "js", "modulePath": "antigravity/model.js", - "sourceFile": "antigravity/model.js" + "sourceFile": "antigravity/model.js", + "navigateBefore": true }, { "site": "antigravity", @@ -572,7 +578,8 @@ ], "type": "js", "modulePath": "antigravity/new.js", - "sourceFile": "antigravity/new.js" + "sourceFile": "antigravity/new.js", + "navigateBefore": true }, { "site": "antigravity", @@ -595,7 +602,8 @@ ], "type": "js", "modulePath": "antigravity/read.js", - "sourceFile": "antigravity/read.js" + "sourceFile": "antigravity/read.js", + "navigateBefore": true }, { "site": "antigravity", @@ -619,7 +627,8 @@ ], "type": "js", "modulePath": "antigravity/send.js", - "sourceFile": "antigravity/send.js" + "sourceFile": "antigravity/send.js", + "navigateBefore": true }, { "site": "antigravity", @@ -636,7 +645,8 @@ ], "type": "js", "modulePath": "antigravity/status.js", - "sourceFile": "antigravity/status.js" + "sourceFile": "antigravity/status.js", + "navigateBefore": true }, { "site": "antigravity", @@ -650,7 +660,8 @@ "timeout": 86400, "type": "js", "modulePath": "antigravity/watch.js", - "sourceFile": "antigravity/watch.js" + "sourceFile": "antigravity/watch.js", + "navigateBefore": true }, { "site": "apple-podcasts", @@ -824,7 +835,8 @@ ], "type": "js", "modulePath": "band/bands.js", - "sourceFile": "band/bands.js" + "sourceFile": "band/bands.js", + "navigateBefore": "https://www.band.us" }, { "site": "band", @@ -872,7 +884,8 @@ ], "type": "js", "modulePath": "band/mentions.js", - "sourceFile": "band/mentions.js" + "sourceFile": "band/mentions.js", + "navigateBefore": true }, { "site": "band", @@ -998,7 +1011,8 @@ ], "type": "js", "modulePath": "barchart/flow.js", - "sourceFile": "barchart/flow.js" + "sourceFile": "barchart/flow.js", + "navigateBefore": "https://www.barchart.com" }, { "site": "barchart", @@ -1045,7 +1059,8 @@ ], "type": "js", "modulePath": "barchart/greeks.js", - "sourceFile": "barchart/greeks.js" + "sourceFile": "barchart/greeks.js", + "navigateBefore": "https://www.barchart.com" }, { "site": "barchart", @@ -1098,7 +1113,8 @@ ], "type": "js", "modulePath": "barchart/options.js", - "sourceFile": "barchart/options.js" + "sourceFile": "barchart/options.js", + "navigateBefore": "https://www.barchart.com" }, { "site": "barchart", @@ -1134,7 +1150,8 @@ ], "type": "js", "modulePath": "barchart/quote.js", - "sourceFile": "barchart/quote.js" + "sourceFile": "barchart/quote.js", + "navigateBefore": "https://www.barchart.com" }, { "site": "bbc", @@ -1195,7 +1212,8 @@ ], "type": "js", "modulePath": "bilibili/comments.js", - "sourceFile": "bilibili/comments.js" + "sourceFile": "bilibili/comments.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1235,7 +1253,8 @@ ], "type": "js", "modulePath": "bilibili/download.js", - "sourceFile": "bilibili/download.js" + "sourceFile": "bilibili/download.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1262,7 +1281,8 @@ ], "type": "js", "modulePath": "bilibili/dynamic.js", - "sourceFile": "bilibili/dynamic.js" + "sourceFile": "bilibili/dynamic.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1296,7 +1316,8 @@ ], "type": "js", "modulePath": "bilibili/favorite.js", - "sourceFile": "bilibili/favorite.js" + "sourceFile": "bilibili/favorite.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1330,7 +1351,8 @@ ], "type": "js", "modulePath": "bilibili/feed.js", - "sourceFile": "bilibili/feed.js" + "sourceFile": "bilibili/feed.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1370,7 +1392,8 @@ ], "type": "js", "modulePath": "bilibili/following.js", - "sourceFile": "bilibili/following.js" + "sourceFile": "bilibili/following.js", + "navigateBefore": true }, { "site": "bilibili", @@ -1397,7 +1420,8 @@ ], "type": "js", "modulePath": "bilibili/history.js", - "sourceFile": "bilibili/history.js" + "sourceFile": "bilibili/history.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1424,7 +1448,8 @@ ], "type": "js", "modulePath": "bilibili/hot.js", - "sourceFile": "bilibili/hot.js" + "sourceFile": "bilibili/hot.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1444,7 +1469,8 @@ ], "type": "js", "modulePath": "bilibili/me.js", - "sourceFile": "bilibili/me.js" + "sourceFile": "bilibili/me.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1471,7 +1497,8 @@ ], "type": "js", "modulePath": "bilibili/ranking.js", - "sourceFile": "bilibili/ranking.js" + "sourceFile": "bilibili/ranking.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1519,7 +1546,8 @@ ], "type": "js", "modulePath": "bilibili/search.js", - "sourceFile": "bilibili/search.js" + "sourceFile": "bilibili/search.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bilibili", @@ -1550,7 +1578,8 @@ ], "type": "js", "modulePath": "bilibili/subtitle.js", - "sourceFile": "bilibili/subtitle.js" + "sourceFile": "bilibili/subtitle.js", + "navigateBefore": true }, { "site": "bilibili", @@ -1599,7 +1628,8 @@ ], "type": "js", "modulePath": "bilibili/user-videos.js", - "sourceFile": "bilibili/user-videos.js" + "sourceFile": "bilibili/user-videos.js", + "navigateBefore": "https://www.bilibili.com" }, { "site": "bloomberg", @@ -1772,7 +1802,8 @@ ], "type": "js", "modulePath": "bloomberg/news.js", - "sourceFile": "bloomberg/news.js" + "sourceFile": "bloomberg/news.js", + "navigateBefore": "https://www.bloomberg.com" }, { "site": "bloomberg", @@ -2727,7 +2758,8 @@ "timeout": 90, "type": "js", "modulePath": "chaoxing/assignments.js", - "sourceFile": "chaoxing/assignments.js" + "sourceFile": "chaoxing/assignments.js", + "navigateBefore": "https://mooc2-ans.chaoxing.com" }, { "site": "chaoxing", @@ -2776,7 +2808,8 @@ "timeout": 90, "type": "js", "modulePath": "chaoxing/exams.js", - "sourceFile": "chaoxing/exams.js" + "sourceFile": "chaoxing/exams.js", + "navigateBefore": "https://mooc2-ans.chaoxing.com" }, { "site": "chatgpt", @@ -2964,7 +2997,8 @@ ], "type": "js", "modulePath": "chatwise/ask.js", - "sourceFile": "chatwise/ask.js" + "sourceFile": "chatwise/ask.js", + "navigateBefore": true }, { "site": "chatwise", @@ -2988,7 +3022,8 @@ ], "type": "js", "modulePath": "chatwise/export.js", - "sourceFile": "chatwise/export.js" + "sourceFile": "chatwise/export.js", + "navigateBefore": true }, { "site": "chatwise", @@ -3004,7 +3039,8 @@ ], "type": "js", "modulePath": "chatwise/history.js", - "sourceFile": "chatwise/history.js" + "sourceFile": "chatwise/history.js", + "navigateBefore": true }, { "site": "chatwise", @@ -3028,7 +3064,8 @@ ], "type": "js", "modulePath": "chatwise/model.js", - "sourceFile": "chatwise/model.js" + "sourceFile": "chatwise/model.js", + "navigateBefore": true }, { "site": "chatwise", @@ -3043,7 +3080,8 @@ ], "type": "js", "modulePath": "chatwise/read.js", - "sourceFile": "chatwise/read.js" + "sourceFile": "chatwise/read.js", + "navigateBefore": true }, { "site": "chatwise", @@ -3067,7 +3105,8 @@ ], "type": "js", "modulePath": "chatwise/send.js", - "sourceFile": "chatwise/send.js" + "sourceFile": "chatwise/send.js", + "navigateBefore": true }, { "site": "cnki", @@ -3134,7 +3173,8 @@ ], "type": "js", "modulePath": "codex/ask.js", - "sourceFile": "codex/ask.js" + "sourceFile": "codex/ask.js", + "navigateBefore": true }, { "site": "codex", @@ -3158,7 +3198,8 @@ ], "type": "js", "modulePath": "codex/export.js", - "sourceFile": "codex/export.js" + "sourceFile": "codex/export.js", + "navigateBefore": true }, { "site": "codex", @@ -3174,7 +3215,8 @@ ], "type": "js", "modulePath": "codex/extract-diff.js", - "sourceFile": "codex/extract-diff.js" + "sourceFile": "codex/extract-diff.js", + "navigateBefore": true }, { "site": "codex", @@ -3190,7 +3232,8 @@ ], "type": "js", "modulePath": "codex/history.js", - "sourceFile": "codex/history.js" + "sourceFile": "codex/history.js", + "navigateBefore": true }, { "site": "codex", @@ -3214,7 +3257,8 @@ ], "type": "js", "modulePath": "codex/model.js", - "sourceFile": "codex/model.js" + "sourceFile": "codex/model.js", + "navigateBefore": true }, { "site": "codex", @@ -3229,7 +3273,8 @@ ], "type": "js", "modulePath": "codex/read.js", - "sourceFile": "codex/read.js" + "sourceFile": "codex/read.js", + "navigateBefore": true }, { "site": "codex", @@ -3253,7 +3298,8 @@ ], "type": "js", "modulePath": "codex/send.js", - "sourceFile": "codex/send.js" + "sourceFile": "codex/send.js", + "navigateBefore": true }, { "site": "coupang", @@ -3285,7 +3331,8 @@ ], "type": "js", "modulePath": "coupang/add-to-cart.js", - "sourceFile": "coupang/add-to-cart.js" + "sourceFile": "coupang/add-to-cart.js", + "navigateBefore": "https://www.coupang.com" }, { "site": "coupang", @@ -3337,7 +3384,8 @@ ], "type": "js", "modulePath": "coupang/search.js", - "sourceFile": "coupang/search.js" + "sourceFile": "coupang/search.js", + "navigateBefore": "https://www.coupang.com" }, { "site": "ctrip", @@ -3402,7 +3450,8 @@ ], "type": "js", "modulePath": "cursor/ask.js", - "sourceFile": "cursor/ask.js" + "sourceFile": "cursor/ask.js", + "navigateBefore": true }, { "site": "cursor", @@ -3426,7 +3475,8 @@ ], "type": "js", "modulePath": "cursor/composer.js", - "sourceFile": "cursor/composer.js" + "sourceFile": "cursor/composer.js", + "navigateBefore": true }, { "site": "cursor", @@ -3450,7 +3500,8 @@ ], "type": "js", "modulePath": "cursor/export.js", - "sourceFile": "cursor/export.js" + "sourceFile": "cursor/export.js", + "navigateBefore": true }, { "site": "cursor", @@ -3465,7 +3516,8 @@ ], "type": "js", "modulePath": "cursor/extract-code.js", - "sourceFile": "cursor/extract-code.js" + "sourceFile": "cursor/extract-code.js", + "navigateBefore": true }, { "site": "cursor", @@ -3481,7 +3533,8 @@ ], "type": "js", "modulePath": "cursor/history.js", - "sourceFile": "cursor/history.js" + "sourceFile": "cursor/history.js", + "navigateBefore": true }, { "site": "cursor", @@ -3505,7 +3558,8 @@ ], "type": "js", "modulePath": "cursor/model.js", - "sourceFile": "cursor/model.js" + "sourceFile": "cursor/model.js", + "navigateBefore": true }, { "site": "cursor", @@ -3521,7 +3575,8 @@ ], "type": "js", "modulePath": "cursor/read.js", - "sourceFile": "cursor/read.js" + "sourceFile": "cursor/read.js", + "navigateBefore": true }, { "site": "cursor", @@ -3545,7 +3600,8 @@ ], "type": "js", "modulePath": "cursor/send.js", - "sourceFile": "cursor/send.js" + "sourceFile": "cursor/send.js", + "navigateBefore": true }, { "site": "devto", @@ -3733,7 +3789,8 @@ ], "type": "js", "modulePath": "discord-app/channels.js", - "sourceFile": "discord-app/channels.js" + "sourceFile": "discord-app/channels.js", + "navigateBefore": true }, { "site": "discord-app", @@ -3750,7 +3807,8 @@ ], "type": "js", "modulePath": "discord-app/members.js", - "sourceFile": "discord-app/members.js" + "sourceFile": "discord-app/members.js", + "navigateBefore": true }, { "site": "discord-app", @@ -3775,7 +3833,8 @@ ], "type": "js", "modulePath": "discord-app/read.js", - "sourceFile": "discord-app/read.js" + "sourceFile": "discord-app/read.js", + "navigateBefore": true }, { "site": "discord-app", @@ -3800,7 +3859,8 @@ ], "type": "js", "modulePath": "discord-app/search.js", - "sourceFile": "discord-app/search.js" + "sourceFile": "discord-app/search.js", + "navigateBefore": true }, { "site": "discord-app", @@ -3823,7 +3883,8 @@ ], "type": "js", "modulePath": "discord-app/send.js", - "sourceFile": "discord-app/send.js" + "sourceFile": "discord-app/send.js", + "navigateBefore": true }, { "site": "discord-app", @@ -3839,7 +3900,8 @@ ], "type": "js", "modulePath": "discord-app/servers.js", - "sourceFile": "discord-app/servers.js" + "sourceFile": "discord-app/servers.js", + "navigateBefore": true }, { "site": "discord-app", @@ -3856,7 +3918,8 @@ ], "type": "js", "modulePath": "discord-app/status.js", - "sourceFile": "discord-app/status.js" + "sourceFile": "discord-app/status.js", + "navigateBefore": true }, { "site": "douban", @@ -3886,7 +3949,8 @@ ], "type": "js", "modulePath": "douban/book-hot.js", - "sourceFile": "douban/book-hot.js" + "sourceFile": "douban/book-hot.js", + "navigateBefore": "https://book.douban.com" }, { "site": "douban", @@ -3939,7 +4003,8 @@ ], "type": "js", "modulePath": "douban/download.js", - "sourceFile": "douban/download.js" + "sourceFile": "douban/download.js", + "navigateBefore": "https://movie.douban.com" }, { "site": "douban", @@ -3987,7 +4052,8 @@ ], "type": "js", "modulePath": "douban/marks.js", - "sourceFile": "douban/marks.js" + "sourceFile": "douban/marks.js", + "navigateBefore": "https://movie.douban.com" }, { "site": "douban", @@ -4017,7 +4083,8 @@ ], "type": "js", "modulePath": "douban/movie-hot.js", - "sourceFile": "douban/movie-hot.js" + "sourceFile": "douban/movie-hot.js", + "navigateBefore": "https://movie.douban.com" }, { "site": "douban", @@ -4057,7 +4124,8 @@ ], "type": "js", "modulePath": "douban/photos.js", - "sourceFile": "douban/photos.js" + "sourceFile": "douban/photos.js", + "navigateBefore": "https://movie.douban.com" }, { "site": "douban", @@ -4098,7 +4166,8 @@ ], "type": "js", "modulePath": "douban/reviews.js", - "sourceFile": "douban/reviews.js" + "sourceFile": "douban/reviews.js", + "navigateBefore": "https://movie.douban.com" }, { "site": "douban", @@ -4144,7 +4213,8 @@ ], "type": "js", "modulePath": "douban/search.js", - "sourceFile": "douban/search.js" + "sourceFile": "douban/search.js", + "navigateBefore": "https://search.douban.com" }, { "site": "douban", @@ -4179,7 +4249,8 @@ ], "type": "js", "modulePath": "douban/subject.js", - "sourceFile": "douban/subject.js" + "sourceFile": "douban/subject.js", + "navigateBefore": "https://movie.douban.com" }, { "site": "douban", @@ -4206,7 +4277,8 @@ ], "type": "js", "modulePath": "douban/top250.js", - "sourceFile": "douban/top250.js" + "sourceFile": "douban/top250.js", + "navigateBefore": "https://movie.douban.com" }, { "site": "doubao", @@ -4465,7 +4537,8 @@ ], "type": "js", "modulePath": "doubao-app/ask.js", - "sourceFile": "doubao-app/ask.js" + "sourceFile": "doubao-app/ask.js", + "navigateBefore": true }, { "site": "doubao-app", @@ -4481,7 +4554,8 @@ ], "type": "js", "modulePath": "doubao-app/dump.js", - "sourceFile": "doubao-app/dump.js" + "sourceFile": "doubao-app/dump.js", + "navigateBefore": true }, { "site": "doubao-app", @@ -4496,7 +4570,8 @@ ], "type": "js", "modulePath": "doubao-app/new.js", - "sourceFile": "doubao-app/new.js" + "sourceFile": "doubao-app/new.js", + "navigateBefore": true }, { "site": "doubao-app", @@ -4512,7 +4587,8 @@ ], "type": "js", "modulePath": "doubao-app/read.js", - "sourceFile": "doubao-app/read.js" + "sourceFile": "doubao-app/read.js", + "navigateBefore": true }, { "site": "doubao-app", @@ -4535,7 +4611,8 @@ ], "type": "js", "modulePath": "doubao-app/screenshot.js", - "sourceFile": "doubao-app/screenshot.js" + "sourceFile": "doubao-app/screenshot.js", + "navigateBefore": true }, { "site": "doubao-app", @@ -4559,7 +4636,8 @@ ], "type": "js", "modulePath": "doubao-app/send.js", - "sourceFile": "doubao-app/send.js" + "sourceFile": "doubao-app/send.js", + "navigateBefore": true }, { "site": "doubao-app", @@ -4576,7 +4654,8 @@ ], "type": "js", "modulePath": "doubao-app/status.js", - "sourceFile": "doubao-app/status.js" + "sourceFile": "doubao-app/status.js", + "navigateBefore": true }, { "site": "douyin", @@ -4593,7 +4672,8 @@ ], "type": "js", "modulePath": "douyin/activities.js", - "sourceFile": "douyin/activities.js" + "sourceFile": "douyin/activities.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -4618,7 +4698,8 @@ ], "type": "js", "modulePath": "douyin/collections.js", - "sourceFile": "douyin/collections.js" + "sourceFile": "douyin/collections.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -4641,7 +4722,8 @@ ], "type": "js", "modulePath": "douyin/delete.js", - "sourceFile": "douyin/delete.js" + "sourceFile": "douyin/delete.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -4723,7 +4805,8 @@ ], "type": "js", "modulePath": "douyin/drafts.js", - "sourceFile": "douyin/drafts.js" + "sourceFile": "douyin/drafts.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -4774,7 +4857,8 @@ ], "type": "js", "modulePath": "douyin/hashtag.js", - "sourceFile": "douyin/hashtag.js" + "sourceFile": "douyin/hashtag.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -4807,7 +4891,8 @@ ], "type": "js", "modulePath": "douyin/location.js", - "sourceFile": "douyin/location.js" + "sourceFile": "douyin/location.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -4826,7 +4911,8 @@ ], "type": "js", "modulePath": "douyin/profile.js", - "sourceFile": "douyin/profile.js" + "sourceFile": "douyin/profile.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -4946,7 +5032,8 @@ ], "type": "js", "modulePath": "douyin/publish.js", - "sourceFile": "douyin/publish.js" + "sourceFile": "douyin/publish.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -4970,7 +5057,8 @@ ], "type": "js", "modulePath": "douyin/stats.js", - "sourceFile": "douyin/stats.js" + "sourceFile": "douyin/stats.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -5007,7 +5095,8 @@ ], "type": "js", "modulePath": "douyin/update.js", - "sourceFile": "douyin/update.js" + "sourceFile": "douyin/update.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "douyin", @@ -5057,7 +5146,8 @@ ], "type": "js", "modulePath": "douyin/user-videos.js", - "sourceFile": "douyin/user-videos.js" + "sourceFile": "douyin/user-videos.js", + "navigateBefore": "https://www.douyin.com" }, { "site": "douyin", @@ -5105,7 +5195,8 @@ ], "type": "js", "modulePath": "douyin/videos.js", - "sourceFile": "douyin/videos.js" + "sourceFile": "douyin/videos.js", + "navigateBefore": "https://creator.douyin.com" }, { "site": "facebook", @@ -5129,7 +5220,8 @@ ], "type": "js", "modulePath": "facebook/add-friend.js", - "sourceFile": "facebook/add-friend.js" + "sourceFile": "facebook/add-friend.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5153,7 +5245,8 @@ ], "type": "js", "modulePath": "facebook/events.js", - "sourceFile": "facebook/events.js" + "sourceFile": "facebook/events.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5181,7 +5274,8 @@ ], "type": "js", "modulePath": "facebook/feed.js", - "sourceFile": "facebook/feed.js" + "sourceFile": "facebook/feed.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5206,7 +5300,8 @@ ], "type": "js", "modulePath": "facebook/friends.js", - "sourceFile": "facebook/friends.js" + "sourceFile": "facebook/friends.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5232,7 +5327,8 @@ ], "type": "js", "modulePath": "facebook/groups.js", - "sourceFile": "facebook/groups.js" + "sourceFile": "facebook/groups.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5256,7 +5352,8 @@ ], "type": "js", "modulePath": "facebook/join-group.js", - "sourceFile": "facebook/join-group.js" + "sourceFile": "facebook/join-group.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5282,7 +5379,8 @@ ], "type": "js", "modulePath": "facebook/memories.js", - "sourceFile": "facebook/memories.js" + "sourceFile": "facebook/memories.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5307,7 +5405,8 @@ ], "type": "js", "modulePath": "facebook/notifications.js", - "sourceFile": "facebook/notifications.js" + "sourceFile": "facebook/notifications.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5334,7 +5433,8 @@ ], "type": "js", "modulePath": "facebook/profile.js", - "sourceFile": "facebook/profile.js" + "sourceFile": "facebook/profile.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "facebook", @@ -5367,7 +5467,8 @@ ], "type": "js", "modulePath": "facebook/search.js", - "sourceFile": "facebook/search.js" + "sourceFile": "facebook/search.js", + "navigateBefore": "https://www.facebook.com" }, { "site": "gemini", @@ -5527,7 +5628,7 @@ { "name": "op", "type": "str", - "default": "/Users/jakevin/tmp/gemini-images", + "default": "/Users/mac/tmp/gemini-images", "required": false, "help": "Output directory shorthand" }, @@ -5840,7 +5941,8 @@ ], "type": "js", "modulePath": "grok/ask.js", - "sourceFile": "grok/ask.js" + "sourceFile": "grok/ask.js", + "navigateBefore": "https://grok.com" }, { "site": "hackernews", @@ -6179,7 +6281,8 @@ ], "type": "js", "modulePath": "hupu/hot.js", - "sourceFile": "hupu/hot.js" + "sourceFile": "hupu/hot.js", + "navigateBefore": "https://bbs.hupu.com" }, { "site": "hupu", @@ -6620,7 +6723,8 @@ ], "type": "js", "modulePath": "instagram/comment.js", - "sourceFile": "instagram/comment.js" + "sourceFile": "instagram/comment.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -6640,7 +6744,7 @@ { "name": "path", "type": "str", - "default": "/Users/jakevin/Downloads/Instagram", + "default": "/Users/mac/Downloads/Instagram", "required": false, "help": "Download directory" } @@ -6676,7 +6780,8 @@ ], "type": "js", "modulePath": "instagram/explore.js", - "sourceFile": "instagram/explore.js" + "sourceFile": "instagram/explore.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -6700,7 +6805,8 @@ ], "type": "js", "modulePath": "instagram/follow.js", - "sourceFile": "instagram/follow.js" + "sourceFile": "instagram/follow.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -6734,7 +6840,8 @@ ], "type": "js", "modulePath": "instagram/followers.js", - "sourceFile": "instagram/followers.js" + "sourceFile": "instagram/followers.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -6768,7 +6875,8 @@ ], "type": "js", "modulePath": "instagram/following.js", - "sourceFile": "instagram/following.js" + "sourceFile": "instagram/following.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -6800,7 +6908,8 @@ ], "type": "js", "modulePath": "instagram/like.js", - "sourceFile": "instagram/like.js" + "sourceFile": "instagram/like.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -6826,7 +6935,8 @@ "timeout": 120, "type": "js", "modulePath": "instagram/note.js", - "sourceFile": "instagram/note.js" + "sourceFile": "instagram/note.js", + "navigateBefore": true }, { "site": "instagram", @@ -6859,7 +6969,8 @@ "timeout": 300, "type": "js", "modulePath": "instagram/post.js", - "sourceFile": "instagram/post.js" + "sourceFile": "instagram/post.js", + "navigateBefore": true }, { "site": "instagram", @@ -6888,7 +6999,8 @@ ], "type": "js", "modulePath": "instagram/profile.js", - "sourceFile": "instagram/profile.js" + "sourceFile": "instagram/profile.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -6921,7 +7033,8 @@ "timeout": 600, "type": "js", "modulePath": "instagram/reel.js", - "sourceFile": "instagram/reel.js" + "sourceFile": "instagram/reel.js", + "navigateBefore": true }, { "site": "instagram", @@ -6953,7 +7066,8 @@ ], "type": "js", "modulePath": "instagram/save.js", - "sourceFile": "instagram/save.js" + "sourceFile": "instagram/save.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -6981,7 +7095,8 @@ ], "type": "js", "modulePath": "instagram/saved.js", - "sourceFile": "instagram/saved.js" + "sourceFile": "instagram/saved.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -7016,7 +7131,8 @@ ], "type": "js", "modulePath": "instagram/search.js", - "sourceFile": "instagram/search.js" + "sourceFile": "instagram/search.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -7042,7 +7158,8 @@ "timeout": 300, "type": "js", "modulePath": "instagram/story.js", - "sourceFile": "instagram/story.js" + "sourceFile": "instagram/story.js", + "navigateBefore": true }, { "site": "instagram", @@ -7066,7 +7183,8 @@ ], "type": "js", "modulePath": "instagram/unfollow.js", - "sourceFile": "instagram/unfollow.js" + "sourceFile": "instagram/unfollow.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -7098,7 +7216,8 @@ ], "type": "js", "modulePath": "instagram/unlike.js", - "sourceFile": "instagram/unlike.js" + "sourceFile": "instagram/unlike.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -7130,7 +7249,8 @@ ], "type": "js", "modulePath": "instagram/unsave.js", - "sourceFile": "instagram/unsave.js" + "sourceFile": "instagram/unsave.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "instagram", @@ -7165,7 +7285,8 @@ ], "type": "js", "modulePath": "instagram/user.js", - "sourceFile": "instagram/user.js" + "sourceFile": "instagram/user.js", + "navigateBefore": "https://www.instagram.com" }, { "site": "jd", @@ -7285,7 +7406,8 @@ ], "type": "js", "modulePath": "jd/item.js", - "sourceFile": "jd/item.js" + "sourceFile": "jd/item.js", + "navigateBefore": "https://item.jd.com" }, { "site": "jd", @@ -7390,7 +7512,8 @@ ], "type": "js", "modulePath": "jianyu/detail.js", - "sourceFile": "jianyu/detail.js" + "sourceFile": "jianyu/detail.js", + "navigateBefore": "https://www.jianyu360.cn" }, { "site": "jianyu", @@ -7426,7 +7549,8 @@ ], "type": "js", "modulePath": "jianyu/search.js", - "sourceFile": "jianyu/search.js" + "sourceFile": "jianyu/search.js", + "navigateBefore": "https://www.jianyu360.cn" }, { "site": "jike", @@ -7457,7 +7581,8 @@ ], "type": "js", "modulePath": "jike/comment.js", - "sourceFile": "jike/comment.js" + "sourceFile": "jike/comment.js", + "navigateBefore": true }, { "site": "jike", @@ -7481,7 +7606,8 @@ ], "type": "js", "modulePath": "jike/create.js", - "sourceFile": "jike/create.js" + "sourceFile": "jike/create.js", + "navigateBefore": true }, { "site": "jike", @@ -7509,7 +7635,8 @@ ], "type": "js", "modulePath": "jike/feed.js", - "sourceFile": "jike/feed.js" + "sourceFile": "jike/feed.js", + "navigateBefore": "https://web.okjike.com" }, { "site": "jike", @@ -7533,7 +7660,8 @@ ], "type": "js", "modulePath": "jike/like.js", - "sourceFile": "jike/like.js" + "sourceFile": "jike/like.js", + "navigateBefore": true }, { "site": "jike", @@ -7559,7 +7687,8 @@ ], "type": "js", "modulePath": "jike/notifications.js", - "sourceFile": "jike/notifications.js" + "sourceFile": "jike/notifications.js", + "navigateBefore": "https://web.okjike.com" }, { "site": "jike", @@ -7586,7 +7715,8 @@ ], "type": "js", "modulePath": "jike/post.js", - "sourceFile": "jike/post.js" + "sourceFile": "jike/post.js", + "navigateBefore": "https://m.okjike.com" }, { "site": "jike", @@ -7617,7 +7747,8 @@ ], "type": "js", "modulePath": "jike/repost.js", - "sourceFile": "jike/repost.js" + "sourceFile": "jike/repost.js", + "navigateBefore": true }, { "site": "jike", @@ -7652,7 +7783,8 @@ ], "type": "js", "modulePath": "jike/search.js", - "sourceFile": "jike/search.js" + "sourceFile": "jike/search.js", + "navigateBefore": "https://web.okjike.com" }, { "site": "jike", @@ -7687,7 +7819,8 @@ ], "type": "js", "modulePath": "jike/topic.js", - "sourceFile": "jike/topic.js" + "sourceFile": "jike/topic.js", + "navigateBefore": "https://m.okjike.com" }, { "site": "jike", @@ -7722,7 +7855,8 @@ ], "type": "js", "modulePath": "jike/user.js", - "sourceFile": "jike/user.js" + "sourceFile": "jike/user.js", + "navigateBefore": "https://m.okjike.com" }, { "site": "jimeng", @@ -7762,7 +7896,8 @@ ], "type": "js", "modulePath": "jimeng/generate.js", - "sourceFile": "jimeng/generate.js" + "sourceFile": "jimeng/generate.js", + "navigateBefore": "https://jimeng.jianying.com" }, { "site": "jimeng", @@ -7789,7 +7924,8 @@ ], "type": "js", "modulePath": "jimeng/history.js", - "sourceFile": "jimeng/history.js" + "sourceFile": "jimeng/history.js", + "navigateBefore": "https://jimeng.jianying.com" }, { "site": "jimeng", @@ -7805,7 +7941,8 @@ ], "type": "js", "modulePath": "jimeng/new.js", - "sourceFile": "jimeng/new.js" + "sourceFile": "jimeng/new.js", + "navigateBefore": "https://jimeng.jianying.com" }, { "site": "jimeng", @@ -7823,7 +7960,8 @@ ], "type": "js", "modulePath": "jimeng/workspaces.js", - "sourceFile": "jimeng/workspaces.js" + "sourceFile": "jimeng/workspaces.js", + "navigateBefore": "https://jimeng.jianying.com" }, { "site": "lesswrong", @@ -8339,7 +8477,8 @@ ], "type": "js", "modulePath": "linkedin/search.js", - "sourceFile": "linkedin/search.js" + "sourceFile": "linkedin/search.js", + "navigateBefore": "https://www.linkedin.com" }, { "site": "linkedin", @@ -8370,7 +8509,8 @@ ], "type": "js", "modulePath": "linkedin/timeline.js", - "sourceFile": "linkedin/timeline.js" + "sourceFile": "linkedin/timeline.js", + "navigateBefore": "https://www.linkedin.com" }, { "site": "linux-do", @@ -8404,7 +8544,8 @@ ], "type": "js", "modulePath": "linux-do/categories.js", - "sourceFile": "linux-do/categories.js" + "sourceFile": "linux-do/categories.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8448,7 +8589,8 @@ "replacedBy": "opencli linux-do feed --category ", "type": "js", "modulePath": "linux-do/category.js", - "sourceFile": "linux-do/category.js" + "sourceFile": "linux-do/category.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8539,7 +8681,8 @@ ], "type": "js", "modulePath": "linux-do/categories.js", - "sourceFile": "linux-do/categories.js" + "sourceFile": "linux-do/categories.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8584,7 +8727,8 @@ "replacedBy": "opencli linux-do feed --view top --period ", "type": "js", "modulePath": "linux-do/hot.js", - "sourceFile": "linux-do/hot.js" + "sourceFile": "linux-do/hot.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8614,7 +8758,8 @@ "replacedBy": "opencli linux-do feed --view latest", "type": "js", "modulePath": "linux-do/latest.js", - "sourceFile": "linux-do/latest.js" + "sourceFile": "linux-do/latest.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8649,7 +8794,8 @@ ], "type": "js", "modulePath": "linux-do/search.js", - "sourceFile": "linux-do/search.js" + "sourceFile": "linux-do/search.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8675,7 +8821,8 @@ ], "type": "js", "modulePath": "linux-do/tags.js", - "sourceFile": "linux-do/tags.js" + "sourceFile": "linux-do/tags.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8708,7 +8855,8 @@ ], "type": "js", "modulePath": "linux-do/topic.js", - "sourceFile": "linux-do/topic.js" + "sourceFile": "linux-do/topic.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8731,7 +8879,8 @@ ], "type": "js", "modulePath": "linux-do/topic-content.js", - "sourceFile": "linux-do/topic-content.js" + "sourceFile": "linux-do/topic-content.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8766,7 +8915,8 @@ ], "type": "js", "modulePath": "linux-do/user-posts.js", - "sourceFile": "linux-do/user-posts.js" + "sourceFile": "linux-do/user-posts.js", + "navigateBefore": "https://linux.do" }, { "site": "linux-do", @@ -8802,7 +8952,8 @@ ], "type": "js", "modulePath": "linux-do/user-topics.js", - "sourceFile": "linux-do/user-topics.js" + "sourceFile": "linux-do/user-topics.js", + "navigateBefore": "https://linux.do" }, { "site": "lobsters", @@ -8956,7 +9107,8 @@ ], "type": "js", "modulePath": "medium/feed.js", - "sourceFile": "medium/feed.js" + "sourceFile": "medium/feed.js", + "navigateBefore": "https://medium.com" }, { "site": "medium", @@ -8992,7 +9144,8 @@ ], "type": "js", "modulePath": "medium/search.js", - "sourceFile": "medium/search.js" + "sourceFile": "medium/search.js", + "navigateBefore": "https://medium.com" }, { "site": "medium", @@ -9027,7 +9180,8 @@ ], "type": "js", "modulePath": "medium/user.js", - "sourceFile": "medium/user.js" + "sourceFile": "medium/user.js", + "navigateBefore": "https://medium.com" }, { "site": "notebooklm", @@ -9368,7 +9522,8 @@ ], "type": "js", "modulePath": "notion/export.js", - "sourceFile": "notion/export.js" + "sourceFile": "notion/export.js", + "navigateBefore": true }, { "site": "notion", @@ -9385,7 +9540,8 @@ ], "type": "js", "modulePath": "notion/favorites.js", - "sourceFile": "notion/favorites.js" + "sourceFile": "notion/favorites.js", + "navigateBefore": true }, { "site": "notion", @@ -9408,7 +9564,8 @@ ], "type": "js", "modulePath": "notion/new.js", - "sourceFile": "notion/new.js" + "sourceFile": "notion/new.js", + "navigateBefore": true }, { "site": "notion", @@ -9424,7 +9581,8 @@ ], "type": "js", "modulePath": "notion/read.js", - "sourceFile": "notion/read.js" + "sourceFile": "notion/read.js", + "navigateBefore": true }, { "site": "notion", @@ -9448,7 +9606,8 @@ ], "type": "js", "modulePath": "notion/search.js", - "sourceFile": "notion/search.js" + "sourceFile": "notion/search.js", + "navigateBefore": true }, { "site": "notion", @@ -9464,7 +9623,8 @@ ], "type": "js", "modulePath": "notion/sidebar.js", - "sourceFile": "notion/sidebar.js" + "sourceFile": "notion/sidebar.js", + "navigateBefore": true }, { "site": "notion", @@ -9481,7 +9641,8 @@ ], "type": "js", "modulePath": "notion/status.js", - "sourceFile": "notion/status.js" + "sourceFile": "notion/status.js", + "navigateBefore": true }, { "site": "notion", @@ -9504,7 +9665,8 @@ ], "type": "js", "modulePath": "notion/write.js", - "sourceFile": "notion/write.js" + "sourceFile": "notion/write.js", + "navigateBefore": true }, { "site": "ones", @@ -9971,7 +10133,8 @@ ], "type": "js", "modulePath": "pixiv/detail.js", - "sourceFile": "pixiv/detail.js" + "sourceFile": "pixiv/detail.js", + "navigateBefore": "https://www.pixiv.net" }, { "site": "pixiv", @@ -10004,7 +10167,8 @@ ], "type": "js", "modulePath": "pixiv/download.js", - "sourceFile": "pixiv/download.js" + "sourceFile": "pixiv/download.js", + "navigateBefore": "https://www.pixiv.net" }, { "site": "pixiv", @@ -10040,7 +10204,8 @@ ], "type": "js", "modulePath": "pixiv/illusts.js", - "sourceFile": "pixiv/illusts.js" + "sourceFile": "pixiv/illusts.js", + "navigateBefore": "https://www.pixiv.net" }, { "site": "pixiv", @@ -10093,7 +10258,8 @@ ], "type": "js", "modulePath": "pixiv/ranking.js", - "sourceFile": "pixiv/ranking.js" + "sourceFile": "pixiv/ranking.js", + "navigateBefore": "https://www.pixiv.net" }, { "site": "pixiv", @@ -10162,7 +10328,8 @@ ], "type": "js", "modulePath": "pixiv/search.js", - "sourceFile": "pixiv/search.js" + "sourceFile": "pixiv/search.js", + "navigateBefore": "https://www.pixiv.net" }, { "site": "pixiv", @@ -10192,7 +10359,8 @@ ], "type": "js", "modulePath": "pixiv/user.js", - "sourceFile": "pixiv/user.js" + "sourceFile": "pixiv/user.js", + "navigateBefore": "https://www.pixiv.net" }, { "site": "producthunt", @@ -10226,7 +10394,8 @@ ], "type": "js", "modulePath": "producthunt/browse.js", - "sourceFile": "producthunt/browse.js" + "sourceFile": "producthunt/browse.js", + "navigateBefore": true }, { "site": "producthunt", @@ -10252,7 +10421,8 @@ ], "type": "js", "modulePath": "producthunt/hot.js", - "sourceFile": "producthunt/hot.js" + "sourceFile": "producthunt/hot.js", + "navigateBefore": true }, { "site": "producthunt", @@ -10356,7 +10526,8 @@ ], "type": "js", "modulePath": "quark/ls.js", - "sourceFile": "quark/ls.js" + "sourceFile": "quark/ls.js", + "navigateBefore": "https://pan.quark.cn" }, { "site": "quark", @@ -10388,7 +10559,8 @@ ], "type": "js", "modulePath": "quark/mkdir.js", - "sourceFile": "quark/mkdir.js" + "sourceFile": "quark/mkdir.js", + "navigateBefore": "https://pan.quark.cn" }, { "site": "quark", @@ -10423,7 +10595,8 @@ "timeout": 120, "type": "js", "modulePath": "quark/mv.js", - "sourceFile": "quark/mv.js" + "sourceFile": "quark/mv.js", + "navigateBefore": "https://pan.quark.cn" }, { "site": "quark", @@ -10449,7 +10622,8 @@ ], "type": "js", "modulePath": "quark/rename.js", - "sourceFile": "quark/rename.js" + "sourceFile": "quark/rename.js", + "navigateBefore": "https://pan.quark.cn" }, { "site": "quark", @@ -10469,7 +10643,8 @@ ], "type": "js", "modulePath": "quark/rm.js", - "sourceFile": "quark/rm.js" + "sourceFile": "quark/rm.js", + "navigateBefore": "https://pan.quark.cn" }, { "site": "quark", @@ -10525,7 +10700,8 @@ "timeout": 120, "type": "js", "modulePath": "quark/save.js", - "sourceFile": "quark/save.js" + "sourceFile": "quark/save.js", + "navigateBefore": "https://pan.quark.cn" }, { "site": "quark", @@ -10559,7 +10735,8 @@ ], "type": "js", "modulePath": "quark/share-tree.js", - "sourceFile": "quark/share-tree.js" + "sourceFile": "quark/share-tree.js", + "navigateBefore": "https://pan.quark.cn" }, { "site": "reddit", @@ -10590,7 +10767,8 @@ ], "type": "js", "modulePath": "reddit/comment.js", - "sourceFile": "reddit/comment.js" + "sourceFile": "reddit/comment.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10618,7 +10796,8 @@ ], "type": "js", "modulePath": "reddit/frontpage.js", - "sourceFile": "reddit/frontpage.js" + "sourceFile": "reddit/frontpage.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10652,7 +10831,8 @@ ], "type": "js", "modulePath": "reddit/hot.js", - "sourceFile": "reddit/hot.js" + "sourceFile": "reddit/hot.js", + "navigateBefore": "https://www.reddit.com" }, { "site": "reddit", @@ -10680,7 +10860,8 @@ ], "type": "js", "modulePath": "reddit/popular.js", - "sourceFile": "reddit/popular.js" + "sourceFile": "reddit/popular.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10741,7 +10922,8 @@ ], "type": "js", "modulePath": "reddit/read.js", - "sourceFile": "reddit/read.js" + "sourceFile": "reddit/read.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10772,7 +10954,8 @@ ], "type": "js", "modulePath": "reddit/save.js", - "sourceFile": "reddit/save.js" + "sourceFile": "reddit/save.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10799,7 +10982,8 @@ ], "type": "js", "modulePath": "reddit/saved.js", - "sourceFile": "reddit/saved.js" + "sourceFile": "reddit/saved.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10855,7 +11039,8 @@ ], "type": "js", "modulePath": "reddit/search.js", - "sourceFile": "reddit/search.js" + "sourceFile": "reddit/search.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10903,7 +11088,8 @@ ], "type": "js", "modulePath": "reddit/subreddit.js", - "sourceFile": "reddit/subreddit.js" + "sourceFile": "reddit/subreddit.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10934,7 +11120,8 @@ ], "type": "js", "modulePath": "reddit/subscribe.js", - "sourceFile": "reddit/subscribe.js" + "sourceFile": "reddit/subscribe.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10965,7 +11152,8 @@ ], "type": "js", "modulePath": "reddit/upvote.js", - "sourceFile": "reddit/upvote.js" + "sourceFile": "reddit/upvote.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -10992,7 +11180,8 @@ ], "type": "js", "modulePath": "reddit/upvoted.js", - "sourceFile": "reddit/upvoted.js" + "sourceFile": "reddit/upvoted.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -11016,7 +11205,8 @@ ], "type": "js", "modulePath": "reddit/user.js", - "sourceFile": "reddit/user.js" + "sourceFile": "reddit/user.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -11049,7 +11239,8 @@ ], "type": "js", "modulePath": "reddit/user-comments.js", - "sourceFile": "reddit/user-comments.js" + "sourceFile": "reddit/user-comments.js", + "navigateBefore": "https://reddit.com" }, { "site": "reddit", @@ -11083,7 +11274,8 @@ ], "type": "js", "modulePath": "reddit/user-posts.js", - "sourceFile": "reddit/user-posts.js" + "sourceFile": "reddit/user-posts.js", + "navigateBefore": "https://reddit.com" }, { "site": "reuters", @@ -11117,7 +11309,383 @@ ], "type": "js", "modulePath": "reuters/search.js", - "sourceFile": "reuters/search.js" + "sourceFile": "reuters/search.js", + "navigateBefore": "https://www.reuters.com" + }, + { + "site": "scys", + "name": "activity", + "description": "Extract SCYS activity landing page structure (tabs, stages, tasks)", + "domain": "scys.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "url", + "type": "str", + "required": true, + "positional": true, + "help": "Activity landing URL: /activity/landing/:id" + }, + { + "name": "wait", + "type": "int", + "default": 3, + "required": false, + "help": "Seconds to wait after page load" + } + ], + "columns": [ + "title", + "subtitle", + "tabs", + "stages", + "url" + ], + "type": "js", + "modulePath": "scys/activity.js", + "sourceFile": "scys/activity.js", + "navigateBefore": false + }, + { + "site": "scys", + "name": "article", + "description": "Extract SCYS article detail page content and metadata", + "domain": "scys.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "url", + "type": "str", + "required": true, + "positional": true, + "help": "Article URL or topic id: /articleDetail//" + }, + { + "name": "wait", + "type": "int", + "default": 5, + "required": false, + "help": "Seconds to wait after page load" + }, + { + "name": "max-length", + "type": "int", + "default": 4000, + "required": false, + "help": "Max content length for long text fields" + } + ], + "columns": [ + "topic_id", + "entity_type", + "title", + "author", + "time", + "tags", + "flags", + "image_count", + "external_link_count", + "content", + "ai_summary", + "url" + ], + "type": "js", + "modulePath": "scys/article.js", + "sourceFile": "scys/article.js", + "navigateBefore": false + }, + { + "site": "scys", + "name": "course", + "description": "Read SCYS course detail content and chapter context", + "domain": "scys.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "url", + "type": "str", + "required": true, + "positional": true, + "help": "Course URL: /course/detail/:id[?chapterId=...]" + }, + { + "name": "wait", + "type": "int", + "default": 3, + "required": false, + "help": "Seconds to wait after page load" + }, + { + "name": "max-length", + "type": "int", + "default": 4000, + "required": false, + "help": "Max content length" + }, + { + "name": "all", + "type": "boolean", + "default": false, + "required": false, + "help": "Export all deterministic chapter ids from TOC" + }, + { + "name": "download-images", + "type": "boolean", + "default": false, + "required": false, + "help": "Download course page images to local directory" + }, + { + "name": "output", + "type": "str", + "default": "./scys-course-downloads", + "required": false, + "help": "Image output directory" + } + ], + "columns": [ + "course_title", + "chapter_title", + "breadcrumb", + "updated_at_text", + "participant_count", + "image_count", + "content_image_count", + "prev_chapter", + "next_chapter", + "chapter_id", + "course_id", + "url", + "image_dir" + ], + "type": "js", + "modulePath": "scys/course.js", + "sourceFile": "scys/course.js", + "navigateBefore": false + }, + { + "site": "scys", + "name": "feed", + "description": "Extract SCYS feed cards (home essence or profile posts)", + "domain": "scys.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "url", + "type": "str", + "default": "https://scys.com/?filter=essence", + "required": false, + "positional": true, + "help": "Feed URL (default: home essence feed)" + }, + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Max number of cards" + }, + { + "name": "wait", + "type": "int", + "default": 3, + "required": false, + "help": "Seconds to wait after page load" + } + ], + "columns": [ + "rank", + "author", + "time", + "flags", + "title", + "summary", + "tags", + "interactions_display", + "image_count", + "url" + ], + "type": "js", + "modulePath": "scys/feed.js", + "sourceFile": "scys/feed.js", + "navigateBefore": false + }, + { + "site": "scys", + "name": "opportunity", + "description": "Extract SCYS opportunity feed with flags, summaries, and tags", + "domain": "scys.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "url", + "type": "str", + "default": "https://scys.com/opportunity", + "required": false, + "positional": true, + "help": "Opportunity URL" + }, + { + "name": "tab", + "type": "str", + "default": "all", + "required": false, + "help": "Filter tab: all/全部, hot/热门, winning/中标" + }, + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Max number of cards" + }, + { + "name": "wait", + "type": "int", + "default": 3, + "required": false, + "help": "Seconds to wait after page load" + }, + { + "name": "download-images", + "type": "boolean", + "default": false, + "required": false, + "help": "Download post images to local directory" + }, + { + "name": "output", + "type": "str", + "default": "./scys-opportunity-downloads", + "required": false, + "help": "Image output directory" + } + ], + "columns": [ + "rank", + "author", + "time", + "flags", + "title", + "summary", + "ai_summary", + "tags", + "interactions_display", + "image_count", + "url", + "image_dir" + ], + "type": "js", + "modulePath": "scys/opportunity.js", + "sourceFile": "scys/opportunity.js", + "navigateBefore": false + }, + { + "site": "scys", + "name": "read", + "description": "Read a SCYS page with automatic page-type routing", + "domain": "scys.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "url", + "type": "str", + "required": true, + "positional": true, + "help": "Any scys.com URL" + }, + { + "name": "wait", + "type": "int", + "default": 3, + "required": false, + "help": "Seconds to wait after page load" + }, + { + "name": "limit", + "type": "int", + "default": 20, + "required": false, + "help": "Max rows for list pages" + }, + { + "name": "max-length", + "type": "int", + "default": 4000, + "required": false, + "help": "Max content length for long text fields" + }, + { + "name": "all", + "type": "boolean", + "default": false, + "required": false, + "help": "For course pages, export all deterministic chapter ids from TOC" + }, + { + "name": "download-images", + "type": "boolean", + "default": false, + "required": false, + "help": "For course pages, download page images to local directory" + }, + { + "name": "output", + "type": "str", + "default": "./scys-course-downloads", + "required": false, + "help": "Image output directory for course pages" + } + ], + "type": "js", + "modulePath": "scys/read.js", + "sourceFile": "scys/read.js", + "navigateBefore": false + }, + { + "site": "scys", + "name": "toc", + "description": "Extract chapter table of contents from a SCYS course", + "domain": "scys.com", + "strategy": "cookie", + "browser": true, + "args": [ + { + "name": "course", + "type": "str", + "required": true, + "positional": true, + "help": "Course URL or numeric course id" + }, + { + "name": "wait", + "type": "int", + "default": 2, + "required": false, + "help": "Seconds to wait after page load" + } + ], + "columns": [ + "rank", + "entry_type", + "section", + "group", + "chapter_id", + "chapter_title", + "status", + "is_current" + ], + "type": "js", + "modulePath": "scys/toc.js", + "sourceFile": "scys/toc.js", + "navigateBefore": false }, { "site": "sinablog", @@ -11145,7 +11713,8 @@ ], "type": "js", "modulePath": "sinablog/article.js", - "sourceFile": "sinablog/article.js" + "sourceFile": "sinablog/article.js", + "navigateBefore": "https://blog.sina.com.cn" }, { "site": "sinablog", @@ -11173,7 +11742,8 @@ ], "type": "js", "modulePath": "sinablog/hot.js", - "sourceFile": "sinablog/hot.js" + "sourceFile": "sinablog/hot.js", + "navigateBefore": "https://blog.sina.com.cn" }, { "site": "sinablog", @@ -11243,7 +11813,8 @@ ], "type": "js", "modulePath": "sinablog/user.js", - "sourceFile": "sinablog/user.js" + "sourceFile": "sinablog/user.js", + "navigateBefore": "https://blog.sina.com.cn" }, { "site": "sinafinance", @@ -11294,7 +11865,8 @@ ], "type": "js", "modulePath": "sinafinance/rolling-news.js", - "sourceFile": "sinafinance/rolling-news.js" + "sourceFile": "sinafinance/rolling-news.js", + "navigateBefore": "https://finance.sina.com.cn/roll" }, { "site": "sinafinance", @@ -11405,7 +11977,8 @@ ], "type": "js", "modulePath": "smzdm/search.js", - "sourceFile": "smzdm/search.js" + "sourceFile": "smzdm/search.js", + "navigateBefore": "https://www.smzdm.com" }, { "site": "spotify", @@ -11812,7 +12385,8 @@ ], "type": "js", "modulePath": "substack/feed.js", - "sourceFile": "substack/feed.js" + "sourceFile": "substack/feed.js", + "navigateBefore": "https://substack.com" }, { "site": "substack", @@ -11846,7 +12420,8 @@ ], "type": "js", "modulePath": "substack/publication.js", - "sourceFile": "substack/publication.js" + "sourceFile": "substack/publication.js", + "navigateBefore": "https://substack.com" }, { "site": "substack", @@ -12257,7 +12832,8 @@ ], "type": "js", "modulePath": "tiktok/comment.js", - "sourceFile": "tiktok/comment.js" + "sourceFile": "tiktok/comment.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12283,7 +12859,8 @@ ], "type": "js", "modulePath": "tiktok/explore.js", - "sourceFile": "tiktok/explore.js" + "sourceFile": "tiktok/explore.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12307,7 +12884,8 @@ ], "type": "js", "modulePath": "tiktok/follow.js", - "sourceFile": "tiktok/follow.js" + "sourceFile": "tiktok/follow.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12332,7 +12910,8 @@ ], "type": "js", "modulePath": "tiktok/following.js", - "sourceFile": "tiktok/following.js" + "sourceFile": "tiktok/following.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12357,7 +12936,8 @@ ], "type": "js", "modulePath": "tiktok/friends.js", - "sourceFile": "tiktok/friends.js" + "sourceFile": "tiktok/friends.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12382,7 +12962,8 @@ ], "type": "js", "modulePath": "tiktok/like.js", - "sourceFile": "tiktok/like.js" + "sourceFile": "tiktok/like.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12408,7 +12989,8 @@ ], "type": "js", "modulePath": "tiktok/live.js", - "sourceFile": "tiktok/live.js" + "sourceFile": "tiktok/live.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12446,7 +13028,8 @@ ], "type": "js", "modulePath": "tiktok/notifications.js", - "sourceFile": "tiktok/notifications.js" + "sourceFile": "tiktok/notifications.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12476,7 +13059,8 @@ ], "type": "js", "modulePath": "tiktok/profile.js", - "sourceFile": "tiktok/profile.js" + "sourceFile": "tiktok/profile.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12500,7 +13084,8 @@ ], "type": "js", "modulePath": "tiktok/save.js", - "sourceFile": "tiktok/save.js" + "sourceFile": "tiktok/save.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12537,7 +13122,8 @@ ], "type": "js", "modulePath": "tiktok/search.js", - "sourceFile": "tiktok/search.js" + "sourceFile": "tiktok/search.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12561,7 +13147,8 @@ ], "type": "js", "modulePath": "tiktok/unfollow.js", - "sourceFile": "tiktok/unfollow.js" + "sourceFile": "tiktok/unfollow.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12586,7 +13173,8 @@ ], "type": "js", "modulePath": "tiktok/unlike.js", - "sourceFile": "tiktok/unlike.js" + "sourceFile": "tiktok/unlike.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12610,7 +13198,8 @@ ], "type": "js", "modulePath": "tiktok/unsave.js", - "sourceFile": "tiktok/unsave.js" + "sourceFile": "tiktok/unsave.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "tiktok", @@ -12642,7 +13231,8 @@ ], "type": "js", "modulePath": "tiktok/user.js", - "sourceFile": "tiktok/user.js" + "sourceFile": "tiktok/user.js", + "navigateBefore": "https://www.tiktok.com" }, { "site": "twitter", @@ -12676,7 +13266,8 @@ "timeout": 600, "type": "js", "modulePath": "twitter/accept.js", - "sourceFile": "twitter/accept.js" + "sourceFile": "twitter/accept.js", + "navigateBefore": true }, { "site": "twitter", @@ -12702,7 +13293,8 @@ ], "type": "js", "modulePath": "twitter/article.js", - "sourceFile": "twitter/article.js" + "sourceFile": "twitter/article.js", + "navigateBefore": "https://x.com" }, { "site": "twitter", @@ -12726,7 +13318,8 @@ ], "type": "js", "modulePath": "twitter/block.js", - "sourceFile": "twitter/block.js" + "sourceFile": "twitter/block.js", + "navigateBefore": true }, { "site": "twitter", @@ -12750,7 +13343,8 @@ ], "type": "js", "modulePath": "twitter/bookmark.js", - "sourceFile": "twitter/bookmark.js" + "sourceFile": "twitter/bookmark.js", + "navigateBefore": true }, { "site": "twitter", @@ -12776,7 +13370,8 @@ ], "type": "js", "modulePath": "twitter/bookmarks.js", - "sourceFile": "twitter/bookmarks.js" + "sourceFile": "twitter/bookmarks.js", + "navigateBefore": "https://x.com" }, { "site": "twitter", @@ -12800,7 +13395,8 @@ ], "type": "js", "modulePath": "twitter/delete.js", - "sourceFile": "twitter/delete.js" + "sourceFile": "twitter/delete.js", + "navigateBefore": true }, { "site": "twitter", @@ -12846,7 +13442,8 @@ ], "type": "js", "modulePath": "twitter/download.js", - "sourceFile": "twitter/download.js" + "sourceFile": "twitter/download.js", + "navigateBefore": "https://x.com" }, { "site": "twitter", @@ -12870,7 +13467,8 @@ ], "type": "js", "modulePath": "twitter/follow.js", - "sourceFile": "twitter/follow.js" + "sourceFile": "twitter/follow.js", + "navigateBefore": true }, { "site": "twitter", @@ -12903,7 +13501,8 @@ ], "type": "js", "modulePath": "twitter/followers.js", - "sourceFile": "twitter/followers.js" + "sourceFile": "twitter/followers.js", + "navigateBefore": true }, { "site": "twitter", @@ -12936,7 +13535,8 @@ ], "type": "js", "modulePath": "twitter/following.js", - "sourceFile": "twitter/following.js" + "sourceFile": "twitter/following.js", + "navigateBefore": true }, { "site": "twitter", @@ -12960,7 +13560,8 @@ ], "type": "js", "modulePath": "twitter/hide-reply.js", - "sourceFile": "twitter/hide-reply.js" + "sourceFile": "twitter/hide-reply.js", + "navigateBefore": true }, { "site": "twitter", @@ -12984,7 +13585,8 @@ ], "type": "js", "modulePath": "twitter/like.js", - "sourceFile": "twitter/like.js" + "sourceFile": "twitter/like.js", + "navigateBefore": true }, { "site": "twitter", @@ -13018,7 +13620,8 @@ ], "type": "js", "modulePath": "twitter/likes.js", - "sourceFile": "twitter/likes.js" + "sourceFile": "twitter/likes.js", + "navigateBefore": "https://x.com" }, { "site": "twitter", @@ -13045,7 +13648,8 @@ ], "type": "js", "modulePath": "twitter/notifications.js", - "sourceFile": "twitter/notifications.js" + "sourceFile": "twitter/notifications.js", + "navigateBefore": true }, { "site": "twitter", @@ -13076,7 +13680,8 @@ ], "type": "js", "modulePath": "twitter/post.js", - "sourceFile": "twitter/post.js" + "sourceFile": "twitter/post.js", + "navigateBefore": true }, { "site": "twitter", @@ -13109,7 +13714,8 @@ ], "type": "js", "modulePath": "twitter/profile.js", - "sourceFile": "twitter/profile.js" + "sourceFile": "twitter/profile.js", + "navigateBefore": "https://x.com" }, { "site": "twitter", @@ -13153,7 +13759,8 @@ ], "type": "js", "modulePath": "twitter/reply.js", - "sourceFile": "twitter/reply.js" + "sourceFile": "twitter/reply.js", + "navigateBefore": true }, { "site": "twitter", @@ -13194,7 +13801,8 @@ "timeout": 600, "type": "js", "modulePath": "twitter/reply-dm.js", - "sourceFile": "twitter/reply-dm.js" + "sourceFile": "twitter/reply-dm.js", + "navigateBefore": true }, { "site": "twitter", @@ -13241,7 +13849,8 @@ ], "type": "js", "modulePath": "twitter/search.js", - "sourceFile": "twitter/search.js" + "sourceFile": "twitter/search.js", + "navigateBefore": true }, { "site": "twitter", @@ -13276,7 +13885,8 @@ ], "type": "js", "modulePath": "twitter/thread.js", - "sourceFile": "twitter/thread.js" + "sourceFile": "twitter/thread.js", + "navigateBefore": "https://x.com" }, { "site": "twitter", @@ -13318,7 +13928,8 @@ ], "type": "js", "modulePath": "twitter/timeline.js", - "sourceFile": "twitter/timeline.js" + "sourceFile": "twitter/timeline.js", + "navigateBefore": "https://x.com" }, { "site": "twitter", @@ -13344,7 +13955,8 @@ ], "type": "js", "modulePath": "twitter/trending.js", - "sourceFile": "twitter/trending.js" + "sourceFile": "twitter/trending.js", + "navigateBefore": "https://x.com" }, { "site": "twitter", @@ -13368,7 +13980,8 @@ ], "type": "js", "modulePath": "twitter/unblock.js", - "sourceFile": "twitter/unblock.js" + "sourceFile": "twitter/unblock.js", + "navigateBefore": true }, { "site": "twitter", @@ -13392,7 +14005,8 @@ ], "type": "js", "modulePath": "twitter/unbookmark.js", - "sourceFile": "twitter/unbookmark.js" + "sourceFile": "twitter/unbookmark.js", + "navigateBefore": true }, { "site": "twitter", @@ -13416,7 +14030,8 @@ ], "type": "js", "modulePath": "twitter/unfollow.js", - "sourceFile": "twitter/unfollow.js" + "sourceFile": "twitter/unfollow.js", + "navigateBefore": true }, { "site": "v2ex", @@ -13432,7 +14047,8 @@ ], "type": "js", "modulePath": "v2ex/daily.js", - "sourceFile": "v2ex/daily.js" + "sourceFile": "v2ex/daily.js", + "navigateBefore": "https://www.v2ex.com" }, { "site": "v2ex", @@ -13506,7 +14122,8 @@ ], "type": "js", "modulePath": "v2ex/me.js", - "sourceFile": "v2ex/me.js" + "sourceFile": "v2ex/me.js", + "navigateBefore": "https://www.v2ex.com" }, { "site": "v2ex", @@ -13620,7 +14237,8 @@ ], "type": "js", "modulePath": "v2ex/notifications.js", - "sourceFile": "v2ex/notifications.js" + "sourceFile": "v2ex/notifications.js", + "navigateBefore": "https://www.v2ex.com" }, { "site": "v2ex", @@ -13798,7 +14416,8 @@ ], "type": "js", "modulePath": "weibo/comments.js", - "sourceFile": "weibo/comments.js" + "sourceFile": "weibo/comments.js", + "navigateBefore": "https://weibo.com" }, { "site": "weibo", @@ -13827,7 +14446,8 @@ ], "type": "js", "modulePath": "weibo/feed.js", - "sourceFile": "weibo/feed.js" + "sourceFile": "weibo/feed.js", + "navigateBefore": "https://weibo.com" }, { "site": "weibo", @@ -13855,7 +14475,8 @@ ], "type": "js", "modulePath": "weibo/hot.js", - "sourceFile": "weibo/hot.js" + "sourceFile": "weibo/hot.js", + "navigateBefore": "https://weibo.com" }, { "site": "weibo", @@ -13876,7 +14497,8 @@ ], "type": "js", "modulePath": "weibo/me.js", - "sourceFile": "weibo/me.js" + "sourceFile": "weibo/me.js", + "navigateBefore": "https://weibo.com" }, { "site": "weibo", @@ -13900,7 +14522,8 @@ ], "type": "js", "modulePath": "weibo/post.js", - "sourceFile": "weibo/post.js" + "sourceFile": "weibo/post.js", + "navigateBefore": "https://weibo.com" }, { "site": "weibo", @@ -13934,7 +14557,8 @@ ], "type": "js", "modulePath": "weibo/search.js", - "sourceFile": "weibo/search.js" + "sourceFile": "weibo/search.js", + "navigateBefore": "https://weibo.com" }, { "site": "weibo", @@ -13965,7 +14589,8 @@ ], "type": "js", "modulePath": "weibo/user.js", - "sourceFile": "weibo/user.js" + "sourceFile": "weibo/user.js", + "navigateBefore": "https://weibo.com" }, { "site": "weixin", @@ -14005,7 +14630,8 @@ ], "type": "js", "modulePath": "weixin/download.js", - "sourceFile": "weixin/download.js" + "sourceFile": "weixin/download.js", + "navigateBefore": "https://mp.weixin.qq.com" }, { "site": "weread", @@ -14033,7 +14659,8 @@ ], "type": "js", "modulePath": "weread/book.js", - "sourceFile": "weread/book.js" + "sourceFile": "weread/book.js", + "navigateBefore": "https://weread.qq.com" }, { "site": "weread", @@ -14065,7 +14692,8 @@ ], "type": "js", "modulePath": "weread/highlights.js", - "sourceFile": "weread/highlights.js" + "sourceFile": "weread/highlights.js", + "navigateBefore": "https://weread.qq.com" }, { "site": "weread", @@ -14083,7 +14711,8 @@ ], "type": "js", "modulePath": "weread/notebooks.js", - "sourceFile": "weread/notebooks.js" + "sourceFile": "weread/notebooks.js", + "navigateBefore": "https://weread.qq.com" }, { "site": "weread", @@ -14116,7 +14745,8 @@ ], "type": "js", "modulePath": "weread/notes.js", - "sourceFile": "weread/notes.js" + "sourceFile": "weread/notes.js", + "navigateBefore": "https://weread.qq.com" }, { "site": "weread", @@ -14212,7 +14842,8 @@ ], "type": "js", "modulePath": "weread/shelf.js", - "sourceFile": "weread/shelf.js" + "sourceFile": "weread/shelf.js", + "navigateBefore": "https://weread.qq.com" }, { "site": "wikipedia", @@ -14480,7 +15111,8 @@ ], "type": "js", "modulePath": "xiaoe/catalog.js", - "sourceFile": "xiaoe/catalog.js" + "sourceFile": "xiaoe/catalog.js", + "navigateBefore": "https://h5.xet.citv.cn" }, { "site": "xiaoe", @@ -14505,7 +15137,8 @@ ], "type": "js", "modulePath": "xiaoe/content.js", - "sourceFile": "xiaoe/content.js" + "sourceFile": "xiaoe/content.js", + "navigateBefore": "https://h5.xet.citv.cn" }, { "site": "xiaoe", @@ -14522,7 +15155,8 @@ ], "type": "js", "modulePath": "xiaoe/courses.js", - "sourceFile": "xiaoe/courses.js" + "sourceFile": "xiaoe/courses.js", + "navigateBefore": "https://study.xiaoe-tech.com" }, { "site": "xiaoe", @@ -14549,7 +15183,8 @@ ], "type": "js", "modulePath": "xiaoe/detail.js", - "sourceFile": "xiaoe/detail.js" + "sourceFile": "xiaoe/detail.js", + "navigateBefore": "https://h5.xet.citv.cn" }, { "site": "xiaoe", @@ -14576,7 +15211,8 @@ ], "type": "js", "modulePath": "xiaoe/play-url.js", - "sourceFile": "xiaoe/play-url.js" + "sourceFile": "xiaoe/play-url.js", + "navigateBefore": "https://h5.xet.citv.cn" }, { "site": "xiaohongshu", @@ -14619,7 +15255,8 @@ ], "type": "js", "modulePath": "xiaohongshu/comments.js", - "sourceFile": "xiaohongshu/comments.js" + "sourceFile": "xiaohongshu/comments.js", + "navigateBefore": "https://www.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14645,7 +15282,8 @@ ], "type": "js", "modulePath": "xiaohongshu/creator-note-detail.js", - "sourceFile": "xiaohongshu/creator-note-detail.js" + "sourceFile": "xiaohongshu/creator-note-detail.js", + "navigateBefore": "https://creator.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14676,7 +15314,8 @@ ], "type": "js", "modulePath": "xiaohongshu/creator-notes-summary.js", - "sourceFile": "xiaohongshu/creator-notes-summary.js" + "sourceFile": "xiaohongshu/creator-notes-summary.js", + "navigateBefore": "https://creator.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14712,7 +15351,8 @@ "timeout": 180, "type": "js", "modulePath": "xiaohongshu/creator-notes-summary.js", - "sourceFile": "xiaohongshu/creator-notes-summary.js" + "sourceFile": "xiaohongshu/creator-notes-summary.js", + "navigateBefore": "https://creator.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14728,7 +15368,8 @@ ], "type": "js", "modulePath": "xiaohongshu/creator-profile.js", - "sourceFile": "xiaohongshu/creator-profile.js" + "sourceFile": "xiaohongshu/creator-profile.js", + "navigateBefore": "https://creator.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14757,7 +15398,8 @@ ], "type": "js", "modulePath": "xiaohongshu/creator-stats.js", - "sourceFile": "xiaohongshu/creator-stats.js" + "sourceFile": "xiaohongshu/creator-stats.js", + "navigateBefore": "https://creator.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14790,7 +15432,8 @@ ], "type": "js", "modulePath": "xiaohongshu/download.js", - "sourceFile": "xiaohongshu/download.js" + "sourceFile": "xiaohongshu/download.js", + "navigateBefore": "https://www.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14817,7 +15460,8 @@ ], "type": "js", "modulePath": "xiaohongshu/feed.js", - "sourceFile": "xiaohongshu/feed.js" + "sourceFile": "xiaohongshu/feed.js", + "navigateBefore": true }, { "site": "xiaohongshu", @@ -14841,7 +15485,8 @@ ], "type": "js", "modulePath": "xiaohongshu/note.js", - "sourceFile": "xiaohongshu/note.js" + "sourceFile": "xiaohongshu/note.js", + "navigateBefore": "https://www.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14876,7 +15521,8 @@ ], "type": "js", "modulePath": "xiaohongshu/notifications.js", - "sourceFile": "xiaohongshu/notifications.js" + "sourceFile": "xiaohongshu/notifications.js", + "navigateBefore": true }, { "site": "xiaohongshu", @@ -14925,7 +15571,8 @@ ], "type": "js", "modulePath": "xiaohongshu/publish.js", - "sourceFile": "xiaohongshu/publish.js" + "sourceFile": "xiaohongshu/publish.js", + "navigateBefore": "https://creator.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14960,7 +15607,8 @@ ], "type": "js", "modulePath": "xiaohongshu/search.js", - "sourceFile": "xiaohongshu/search.js" + "sourceFile": "xiaohongshu/search.js", + "navigateBefore": "https://www.xiaohongshu.com" }, { "site": "xiaohongshu", @@ -14994,7 +15642,8 @@ ], "type": "js", "modulePath": "xiaohongshu/user.js", - "sourceFile": "xiaohongshu/user.js" + "sourceFile": "xiaohongshu/user.js", + "navigateBefore": "https://www.xiaohongshu.com" }, { "site": "xiaoyuzhou", @@ -15161,7 +15810,8 @@ ], "type": "js", "modulePath": "xueqiu/earnings-date.js", - "sourceFile": "xueqiu/earnings-date.js" + "sourceFile": "xueqiu/earnings-date.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "xueqiu", @@ -15195,7 +15845,8 @@ ], "type": "js", "modulePath": "xueqiu/feed.js", - "sourceFile": "xueqiu/feed.js" + "sourceFile": "xueqiu/feed.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "xueqiu", @@ -15264,7 +15915,8 @@ ], "type": "js", "modulePath": "xueqiu/groups.js", - "sourceFile": "xueqiu/groups.js" + "sourceFile": "xueqiu/groups.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "xueqiu", @@ -15291,7 +15943,8 @@ ], "type": "js", "modulePath": "xueqiu/hot.js", - "sourceFile": "xueqiu/hot.js" + "sourceFile": "xueqiu/hot.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "xueqiu", @@ -15326,7 +15979,8 @@ ], "type": "js", "modulePath": "xueqiu/hot-stock.js", - "sourceFile": "xueqiu/hot-stock.js" + "sourceFile": "xueqiu/hot-stock.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "xueqiu", @@ -15361,7 +16015,8 @@ ], "type": "js", "modulePath": "xueqiu/kline.js", - "sourceFile": "xueqiu/kline.js" + "sourceFile": "xueqiu/kline.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "xueqiu", @@ -15396,7 +16051,8 @@ ], "type": "js", "modulePath": "xueqiu/search.js", - "sourceFile": "xueqiu/search.js" + "sourceFile": "xueqiu/search.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "xueqiu", @@ -15423,7 +16079,8 @@ ], "type": "js", "modulePath": "xueqiu/stock.js", - "sourceFile": "xueqiu/stock.js" + "sourceFile": "xueqiu/stock.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "xueqiu", @@ -15456,7 +16113,8 @@ ], "type": "js", "modulePath": "xueqiu/watchlist.js", - "sourceFile": "xueqiu/watchlist.js" + "sourceFile": "xueqiu/watchlist.js", + "navigateBefore": "https://xueqiu.com" }, { "site": "yahoo-finance", @@ -15488,7 +16146,8 @@ ], "type": "js", "modulePath": "yahoo-finance/quote.js", - "sourceFile": "yahoo-finance/quote.js" + "sourceFile": "yahoo-finance/quote.js", + "navigateBefore": "https://finance.yahoo.com" }, { "site": "yollomi", @@ -15535,7 +16194,8 @@ ], "type": "js", "modulePath": "yollomi/background.js", - "sourceFile": "yollomi/background.js" + "sourceFile": "yollomi/background.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -15594,7 +16254,8 @@ ], "type": "js", "modulePath": "yollomi/edit.js", - "sourceFile": "yollomi/edit.js" + "sourceFile": "yollomi/edit.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -15639,7 +16300,8 @@ ], "type": "js", "modulePath": "yollomi/face-swap.js", - "sourceFile": "yollomi/face-swap.js" + "sourceFile": "yollomi/face-swap.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -15707,7 +16369,8 @@ ], "type": "js", "modulePath": "yollomi/generate.js", - "sourceFile": "yollomi/generate.js" + "sourceFile": "yollomi/generate.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -15785,7 +16448,8 @@ ], "type": "js", "modulePath": "yollomi/object-remover.js", - "sourceFile": "yollomi/object-remover.js" + "sourceFile": "yollomi/object-remover.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -15825,7 +16489,8 @@ ], "type": "js", "modulePath": "yollomi/remove-bg.js", - "sourceFile": "yollomi/remove-bg.js" + "sourceFile": "yollomi/remove-bg.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -15865,7 +16530,8 @@ ], "type": "js", "modulePath": "yollomi/restore.js", - "sourceFile": "yollomi/restore.js" + "sourceFile": "yollomi/restore.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -15922,7 +16588,8 @@ ], "type": "js", "modulePath": "yollomi/try-on.js", - "sourceFile": "yollomi/try-on.js" + "sourceFile": "yollomi/try-on.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -15948,7 +16615,8 @@ ], "type": "js", "modulePath": "yollomi/upload.js", - "sourceFile": "yollomi/upload.js" + "sourceFile": "yollomi/upload.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -16000,7 +16668,8 @@ ], "type": "js", "modulePath": "yollomi/upscale.js", - "sourceFile": "yollomi/upscale.js" + "sourceFile": "yollomi/upscale.js", + "navigateBefore": "https://yollomi.com" }, { "site": "yollomi", @@ -16068,7 +16737,8 @@ ], "type": "js", "modulePath": "yollomi/video.js", - "sourceFile": "yollomi/video.js" + "sourceFile": "yollomi/video.js", + "navigateBefore": "https://yollomi.com" }, { "site": "youtube", @@ -16099,7 +16769,8 @@ ], "type": "js", "modulePath": "youtube/channel.js", - "sourceFile": "youtube/channel.js" + "sourceFile": "youtube/channel.js", + "navigateBefore": "https://www.youtube.com" }, { "site": "youtube", @@ -16134,7 +16805,8 @@ ], "type": "js", "modulePath": "youtube/comments.js", - "sourceFile": "youtube/comments.js" + "sourceFile": "youtube/comments.js", + "navigateBefore": "https://www.youtube.com" }, { "site": "youtube", @@ -16191,7 +16863,8 @@ ], "type": "js", "modulePath": "youtube/search.js", - "sourceFile": "youtube/search.js" + "sourceFile": "youtube/search.js", + "navigateBefore": "https://www.youtube.com" }, { "site": "youtube", @@ -16224,7 +16897,8 @@ ], "type": "js", "modulePath": "youtube/transcript.js", - "sourceFile": "youtube/transcript.js" + "sourceFile": "youtube/transcript.js", + "navigateBefore": "https://www.youtube.com" }, { "site": "youtube", @@ -16248,7 +16922,8 @@ ], "type": "js", "modulePath": "youtube/video.js", - "sourceFile": "youtube/video.js" + "sourceFile": "youtube/video.js", + "navigateBefore": "https://www.youtube.com" }, { "site": "yuanbao", @@ -16361,7 +17036,8 @@ ], "type": "js", "modulePath": "zhihu/answer.js", - "sourceFile": "zhihu/answer.js" + "sourceFile": "zhihu/answer.js", + "navigateBefore": true }, { "site": "zhihu", @@ -16410,7 +17086,8 @@ ], "type": "js", "modulePath": "zhihu/comment.js", - "sourceFile": "zhihu/comment.js" + "sourceFile": "zhihu/comment.js", + "navigateBefore": true }, { "site": "zhihu", @@ -16450,7 +17127,8 @@ ], "type": "js", "modulePath": "zhihu/download.js", - "sourceFile": "zhihu/download.js" + "sourceFile": "zhihu/download.js", + "navigateBefore": "https://zhuanlan.zhihu.com" }, { "site": "zhihu", @@ -16497,7 +17175,8 @@ ], "type": "js", "modulePath": "zhihu/favorite.js", - "sourceFile": "zhihu/favorite.js" + "sourceFile": "zhihu/favorite.js", + "navigateBefore": true }, { "site": "zhihu", @@ -16530,7 +17209,8 @@ ], "type": "js", "modulePath": "zhihu/follow.js", - "sourceFile": "zhihu/follow.js" + "sourceFile": "zhihu/follow.js", + "navigateBefore": true }, { "site": "zhihu", @@ -16556,7 +17236,8 @@ ], "type": "js", "modulePath": "zhihu/hot.js", - "sourceFile": "zhihu/hot.js" + "sourceFile": "zhihu/hot.js", + "navigateBefore": "https://www.zhihu.com" }, { "site": "zhihu", @@ -16589,7 +17270,8 @@ ], "type": "js", "modulePath": "zhihu/like.js", - "sourceFile": "zhihu/like.js" + "sourceFile": "zhihu/like.js", + "navigateBefore": true }, { "site": "zhihu", @@ -16622,7 +17304,8 @@ ], "type": "js", "modulePath": "zhihu/question.js", - "sourceFile": "zhihu/question.js" + "sourceFile": "zhihu/question.js", + "navigateBefore": "https://www.zhihu.com" }, { "site": "zhihu", @@ -16657,7 +17340,8 @@ ], "type": "js", "modulePath": "zhihu/search.js", - "sourceFile": "zhihu/search.js" + "sourceFile": "zhihu/search.js", + "navigateBefore": "https://www.zhihu.com" }, { "site": "zsxq", @@ -16686,7 +17370,8 @@ ], "type": "js", "modulePath": "zsxq/dynamics.js", - "sourceFile": "zsxq/dynamics.js" + "sourceFile": "zsxq/dynamics.js", + "navigateBefore": "https://wx.zsxq.com" }, { "site": "zsxq", @@ -16715,7 +17400,8 @@ ], "type": "js", "modulePath": "zsxq/groups.js", - "sourceFile": "zsxq/groups.js" + "sourceFile": "zsxq/groups.js", + "navigateBefore": "https://wx.zsxq.com" }, { "site": "zsxq", @@ -16758,7 +17444,8 @@ ], "type": "js", "modulePath": "zsxq/search.js", - "sourceFile": "zsxq/search.js" + "sourceFile": "zsxq/search.js", + "navigateBefore": "https://wx.zsxq.com" }, { "site": "zsxq", @@ -16795,7 +17482,8 @@ ], "type": "js", "modulePath": "zsxq/topic.js", - "sourceFile": "zsxq/topic.js" + "sourceFile": "zsxq/topic.js", + "navigateBefore": "https://wx.zsxq.com" }, { "site": "zsxq", @@ -16831,6 +17519,7 @@ ], "type": "js", "modulePath": "zsxq/topics.js", - "sourceFile": "zsxq/topics.js" + "sourceFile": "zsxq/topics.js", + "navigateBefore": "https://wx.zsxq.com" } ] \ No newline at end of file diff --git a/clis/scys/activity.js b/clis/scys/activity.js new file mode 100644 index 000000000..f9f608397 --- /dev/null +++ b/clis/scys/activity.js @@ -0,0 +1,20 @@ +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { extractScysActivity } from './extractors.js'; +cli({ + site: 'scys', + name: 'activity', + description: 'Extract SCYS activity landing page structure (tabs, stages, tasks)', + domain: 'scys.com', + strategy: Strategy.COOKIE, + navigateBefore: false, + args: [ + { name: 'url', required: true, positional: true, help: 'Activity landing URL: /activity/landing/:id' }, + { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, + ], + columns: ['title', 'subtitle', 'tabs', 'stages', 'url'], + func: async (page, kwargs) => { + return extractScysActivity(page, String(kwargs.url), { + waitSeconds: Number(kwargs.wait ?? 3), + }); + }, +}); diff --git a/clis/scys/article.js b/clis/scys/article.js new file mode 100644 index 000000000..f81867515 --- /dev/null +++ b/clis/scys/article.js @@ -0,0 +1,22 @@ +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { extractScysArticle } from './extractors.js'; +cli({ + site: 'scys', + name: 'article', + description: 'Extract SCYS article detail page content and metadata', + domain: 'scys.com', + strategy: Strategy.COOKIE, + navigateBefore: false, + args: [ + { name: 'url', required: true, positional: true, help: 'Article URL or topic id: /articleDetail//' }, + { name: 'wait', type: 'int', default: 5, help: 'Seconds to wait after page load' }, + { name: 'max-length', type: 'int', default: 4000, help: 'Max content length for long text fields' }, + ], + columns: ['topic_id', 'entity_type', 'title', 'author', 'time', 'tags', 'flags', 'image_count', 'external_link_count', 'content', 'ai_summary', 'url'], + func: async (page, kwargs) => { + return extractScysArticle(page, String(kwargs.url), { + waitSeconds: Number(kwargs.wait ?? 5), + maxLength: Number(kwargs['max-length'] ?? 4000), + }); + }, +}); diff --git a/clis/scys/common.js b/clis/scys/common.js new file mode 100644 index 000000000..89bdea518 --- /dev/null +++ b/clis/scys/common.js @@ -0,0 +1,99 @@ +import { ArgumentError } from '@jackwener/opencli/errors'; +const SCYS_ORIGIN = 'https://scys.com'; +export function normalizeScysUrl(input) { + const raw = String(input ?? '').trim(); + if (!raw) { + throw new ArgumentError('SCYS URL is required'); + } + if (/^https?:\/\//i.test(raw)) { + return raw; + } + if (raw.startsWith('/')) { + return `${SCYS_ORIGIN}${raw}`; + } + if (raw.startsWith('scys.com')) { + return `https://${raw}`; + } + return `${SCYS_ORIGIN}/${raw.replace(/^\/+/, '')}`; +} +export function toScysCourseUrl(input) { + const raw = String(input ?? '').trim(); + if (!raw) + throw new ArgumentError('Course URL or course id is required'); + if (/^\d+$/.test(raw)) { + return `${SCYS_ORIGIN}/course/detail/${raw}`; + } + return normalizeScysUrl(raw); +} +export function toScysArticleUrl(input) { + const raw = String(input ?? '').trim(); + if (!raw) + throw new ArgumentError('Article URL is required'); + if (/^\d{8,}$/.test(raw)) { + return `${SCYS_ORIGIN}/articleDetail/xq_topic/${raw}`; + } + const url = normalizeScysUrl(raw); + const parsed = new URL(url); + const match = parsed.pathname.match(/^\/articleDetail\/([^/]+)\/([^/]+)$/); + if (!match) { + throw new ArgumentError(`Unsupported SCYS article URL: ${input}`, 'Use /articleDetail// or pass a numeric topic id'); + } + return url; +} +export function detectScysPageType(input) { + const url = new URL(normalizeScysUrl(input)); + const pathname = url.pathname; + if (pathname.startsWith('/course/detail/')) + return 'course'; + if (pathname.startsWith('/opportunity')) + return 'opportunity'; + if (pathname.startsWith('/activity/landing/')) + return 'activity'; + if (/^\/articleDetail\/[^/]+\/[^/]+$/.test(pathname)) + return 'article'; + if (pathname.startsWith('/personal/')) { + const tab = (url.searchParams.get('tab') || '').toLowerCase(); + if (tab === 'posts') + return 'feed'; + } + if (pathname === '/' || pathname === '') { + const filter = (url.searchParams.get('filter') || '').toLowerCase(); + if (filter === 'essence') + return 'feed'; + } + return 'unknown'; +} +export function extractScysCourseId(input) { + const url = new URL(toScysCourseUrl(input)); + const match = url.pathname.match(/\/course\/detail\/(\d+)/); + return match?.[1] ?? ''; +} +export function extractScysArticleMeta(input) { + const url = new URL(toScysArticleUrl(input)); + const match = url.pathname.match(/^\/articleDetail\/([^/]+)\/([^/]+)$/); + return { + entityType: match?.[1] ?? '', + topicId: match?.[2] ?? '', + }; +} +export function cleanText(value) { + return String(value ?? '').replace(/\s+/g, ' ').trim(); +} +export function extractInteractions(raw) { + const text = cleanText(raw); + if (!text) + return ''; + const pieces = text.match(/[0-9]+(?:\.[0-9]+)?(?:万|亿)?/g); + if (!pieces || pieces.length === 0) + return text; + return pieces.join(' '); +} +export function inferScysReadUrl(input) { + return normalizeScysUrl(input); +} +export function buildScysHomeEssenceUrl() { + return `${SCYS_ORIGIN}/?filter=essence`; +} +export function buildScysOpportunityUrl() { + return `${SCYS_ORIGIN}/opportunity`; +} diff --git a/clis/scys/common.test.js b/clis/scys/common.test.js new file mode 100644 index 000000000..75f3efec6 --- /dev/null +++ b/clis/scys/common.test.js @@ -0,0 +1,68 @@ +import { describe, expect, it } from 'vitest'; +import { cleanText, detectScysPageType, extractScysArticleMeta, extractInteractions, normalizeScysUrl, toScysArticleUrl, toScysCourseUrl, } from './common.js'; +describe('normalizeScysUrl', () => { + it('normalizes bare domain and keeps path/query', () => { + expect(normalizeScysUrl('scys.com/course/detail/142?chapterId=9445')).toBe('https://scys.com/course/detail/142?chapterId=9445'); + }); + it('normalizes root-relative paths', () => { + expect(normalizeScysUrl('/opportunity')).toBe('https://scys.com/opportunity'); + }); +}); +describe('toScysCourseUrl', () => { + it('accepts numeric course id', () => { + expect(toScysCourseUrl('92')).toBe('https://scys.com/course/detail/92'); + }); + it('keeps full course detail URL unchanged', () => { + expect(toScysCourseUrl('https://scys.com/course/detail/142?chapterId=9445')).toBe('https://scys.com/course/detail/142?chapterId=9445'); + }); +}); +describe('toScysArticleUrl', () => { + it('accepts numeric topic id', () => { + expect(toScysArticleUrl('55188458224514554')).toBe('https://scys.com/articleDetail/xq_topic/55188458224514554'); + }); + it('keeps full article detail url', () => { + expect(toScysArticleUrl('https://scys.com/articleDetail/xq_topic/55188458224514554')).toBe('https://scys.com/articleDetail/xq_topic/55188458224514554'); + }); +}); +describe('extractScysArticleMeta', () => { + it('extracts entity type and topic id from url', () => { + expect(extractScysArticleMeta('https://scys.com/articleDetail/xq_topic/55188458224514554')).toEqual({ + entityType: 'xq_topic', + topicId: '55188458224514554', + }); + }); +}); +describe('detectScysPageType', () => { + it('detects course detail with chapterId', () => { + expect(detectScysPageType('https://scys.com/course/detail/142?chapterId=9445')).toBe('course'); + }); + it('detects course detail without chapterId', () => { + expect(detectScysPageType('https://scys.com/course/detail/92')).toBe('course'); + }); + it('detects essence feed on homepage', () => { + expect(detectScysPageType('https://scys.com/?filter=essence')).toBe('feed'); + }); + it('detects profile posts feed', () => { + expect(detectScysPageType('https://scys.com/personal/421122582111848?number=18563&tab=posts')).toBe('feed'); + }); + it('detects opportunity page', () => { + expect(detectScysPageType('https://scys.com/opportunity')).toBe('opportunity'); + }); + it('detects activity landing page', () => { + expect(detectScysPageType('https://scys.com/activity/landing/5505?tabIndex=1')).toBe('activity'); + }); + it('detects article detail page', () => { + expect(detectScysPageType('https://scys.com/articleDetail/xq_topic/55188458224514554')).toBe('article'); + }); + it('returns unknown for unsupported pages', () => { + expect(detectScysPageType('https://scys.com/help')).toBe('unknown'); + }); +}); +describe('text helpers', () => { + it('cleanText collapses whitespace', () => { + expect(cleanText(' hello\n\nworld ')).toBe('hello world'); + }); + it('extractInteractions keeps compact numeric text', () => { + expect(extractInteractions('赞 1.2万 评论 35')).toBe('1.2万 35'); + }); +}); diff --git a/clis/scys/course-download.js b/clis/scys/course-download.js new file mode 100644 index 000000000..68e0b5e90 --- /dev/null +++ b/clis/scys/course-download.js @@ -0,0 +1,104 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { createHash } from 'node:crypto'; +import { formatCookieHeader, httpDownload } from '@jackwener/opencli/download'; +function sanitizeExtname(url) { + try { + const pathname = new URL(url).pathname || ''; + const ext = path.extname(pathname).toLowerCase(); + if (ext && ext.length <= 6) + return ext; + } + catch { + // ignore invalid URL and fall back + } + return '.jpg'; +} +function hashUrl(url) { + return createHash('sha1').update(url).digest('hex'); +} +function buildDownloadPlan(rows, output) { + const cacheDir = path.join(output, '.cache'); + const byUrl = new Map(); + rows.forEach((row, rowIndex) => { + const courseId = row.course_id || 'course'; + const chapterId = row.chapter_id || 'root'; + const imageUrls = Array.isArray(row.images) ? row.images.filter(Boolean) : []; + imageUrls.forEach((url, imageIndex) => { + const ext = sanitizeExtname(url); + const cachePath = path.join(cacheDir, `${hashUrl(url)}${ext}`); + const destPath = path.join(output, courseId, chapterId, `${courseId}_${chapterId}_${imageIndex + 1}${ext}`); + const existing = byUrl.get(url); + if (existing) { + existing.copies.push({ rowIndex, destPath }); + return; + } + byUrl.set(url, { + url, + cachePath, + copies: [{ rowIndex, destPath }], + }); + }); + }); + return Array.from(byUrl.values()); +} +async function runWithConcurrency(items, concurrency, worker) { + const limit = Math.max(1, Math.floor(concurrency)); + let cursor = 0; + async function consume() { + while (cursor < items.length) { + const index = cursor; + cursor += 1; + await worker(items[index]); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => consume())); +} +function createDefaultDeps() { + return { + concurrency: 8, + downloadToPath: async (url, destPath, cookies) => { + const result = await httpDownload(url, destPath, { + cookies, + timeout: 60_000, + }); + return result.success; + }, + }; +} +export async function downloadScysCourseImagesInternal(data, output, cookies, overrides = {}) { + const rows = Array.isArray(data) ? data : [data]; + const deps = { ...createDefaultDeps(), ...overrides }; + const withDownloads = rows.map((row) => ({ ...row, image_count: 0, image_dir: '' })); + const plan = buildDownloadPlan(withDownloads, output); + const successCounts = new Array(withDownloads.length).fill(0); + await fs.promises.mkdir(path.join(output, '.cache'), { recursive: true }); + await runWithConcurrency(plan, deps.concurrency, async (entry) => { + let available = false; + try { + await fs.promises.access(entry.cachePath, fs.constants.F_OK); + available = true; + } + catch { + await fs.promises.mkdir(path.dirname(entry.cachePath), { recursive: true }); + available = await deps.downloadToPath(entry.url, entry.cachePath, cookies); + } + if (!available) + return; + await Promise.all(entry.copies.map(async (copy) => { + await fs.promises.mkdir(path.dirname(copy.destPath), { recursive: true }); + await fs.promises.copyFile(entry.cachePath, copy.destPath); + successCounts[copy.rowIndex] += 1; + })); + }); + const result = withDownloads.map((row, index) => ({ + ...row, + image_count: successCounts[index] ?? 0, + image_dir: row.images.length > 0 ? path.join(output, row.course_id || 'course', row.chapter_id || 'root') : '', + })); + return Array.isArray(data) ? result : result[0]; +} +export async function downloadScysCourseImages(page, data, output) { + const cookies = formatCookieHeader(await page.getCookies({ domain: 'scys.com' })); + return downloadScysCourseImagesInternal(data, output, cookies); +} diff --git a/clis/scys/course-download.test.js b/clis/scys/course-download.test.js new file mode 100644 index 000000000..462826d54 --- /dev/null +++ b/clis/scys/course-download.test.js @@ -0,0 +1,81 @@ +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { downloadScysCourseImagesInternal } from './course-download.js'; +function makeRow(overrides) { + return { + course_title: 'Course', + chapter_title: 'Chapter', + breadcrumb: 'A > B > C', + content: 'body', + chapter_id: '1', + course_id: '92', + toc_summary: '', + url: 'https://scys.com/course/detail/92?chapterId=1', + raw_url: 'https://scys.com/course/detail/92?chapterId=1', + updated_at_text: '', + copyright_text: '', + prev_chapter: '', + next_chapter: '', + participant_count: 0, + discussion_hint: '', + links: [], + images: [], + image_count: 0, + content_images: [], + content_image_count: 0, + image_dir: '', + ...overrides, + }; +} +describe('downloadScysCourseImagesInternal', () => { + it('deduplicates repeated image urls across chapters and copies cached files', async () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), 'scys-course-download-')); + const rows = [ + makeRow({ chapter_id: '4038', images: ['https://cdn.example.com/shared.png', 'https://cdn.example.com/unique-a.png'] }), + makeRow({ chapter_id: '4039', images: ['https://cdn.example.com/shared.png'] }), + ]; + const calls = []; + const result = await downloadScysCourseImagesInternal(rows, output, 'cookie=a', { + concurrency: 2, + downloadToPath: async (url, destPath) => { + calls.push(url); + await fs.promises.mkdir(path.dirname(destPath), { recursive: true }); + await fs.promises.writeFile(destPath, `downloaded:${url}`); + return true; + }, + }); + expect(calls).toEqual([ + 'https://cdn.example.com/shared.png', + 'https://cdn.example.com/unique-a.png', + ]); + expect(result[0]?.image_count).toBe(2); + expect(result[1]?.image_count).toBe(1); + expect(fs.existsSync(path.join(output, '92', '4038', '92_4038_1.png'))).toBe(true); + expect(fs.existsSync(path.join(output, '92', '4038', '92_4038_2.png'))).toBe(true); + expect(fs.existsSync(path.join(output, '92', '4039', '92_4039_1.png'))).toBe(true); + }); + it('downloads unique image urls concurrently instead of one-by-one', async () => { + const output = fs.mkdtempSync(path.join(os.tmpdir(), 'scys-course-download-')); + const rows = [ + makeRow({ chapter_id: '4038', images: ['https://cdn.example.com/a.png', 'https://cdn.example.com/b.png'] }), + makeRow({ chapter_id: '4039', images: ['https://cdn.example.com/c.png', 'https://cdn.example.com/d.png'] }), + ]; + let active = 0; + let maxActive = 0; + await downloadScysCourseImagesInternal(rows, output, 'cookie=a', { + concurrency: 3, + downloadToPath: async (_url, destPath) => { + active += 1; + maxActive = Math.max(maxActive, active); + await new Promise((resolve) => setTimeout(resolve, 30)); + await fs.promises.mkdir(path.dirname(destPath), { recursive: true }); + await fs.promises.writeFile(destPath, 'x'); + active -= 1; + return true; + }, + }); + expect(maxActive).toBeGreaterThan(1); + }); +}); diff --git a/clis/scys/course-utils.js b/clis/scys/course-utils.js new file mode 100644 index 000000000..02506ae22 --- /dev/null +++ b/clis/scys/course-utils.js @@ -0,0 +1,137 @@ +import { cleanText, normalizeScysUrl, toScysCourseUrl } from './common.js'; +function safeNormalizeScysUrl(url) { + const cleaned = cleanText(url); + if (!cleaned) + return ''; + return normalizeScysUrl(cleaned); +} +function dedupeStrings(values) { + return Array.from(new Set(values.map((value) => cleanText(value)).filter(Boolean))); +} +function normalizeImageUrl(url) { + if (!url) + return ''; + if (url.startsWith('http://') || url.startsWith('https://')) + return url; + if (url.startsWith('/')) + return `https://scys.com${url}`; + return ''; +} +function normalizeImages(values) { + return Array.from(new Set(values + .map((value) => normalizeImageUrl(cleanText(value))) + .filter(Boolean) + .filter((value) => !value.startsWith('data:')) + .filter((value) => !/\/images\/pic_empty\.png$/i.test(value)))); +} +function chooseCourseChapterTitle(payload) { + const explicit = cleanText(payload.chapterTitle); + if (explicit) + return explicit; + const byToc = (payload.tocRows ?? []).find((row) => cleanText(row.chapter_id) === cleanText(payload.chapterId)); + if (byToc?.chapter_title) + return cleanText(byToc.chapter_title); + const current = cleanText(payload.currentChapter); + if (current) + return current; + const breadcrumbLast = cleanText((payload.breadcrumb ?? []).at(-1)); + return breadcrumbLast; +} +function normalizeBreadcrumb(payload, chapterTitle) { + const byToc = (payload.tocRows ?? []).find((row) => cleanText(row.chapter_id) === cleanText(payload.chapterId)); + if (byToc) { + const tocParts = [cleanText(byToc.section), cleanText(byToc.group), chapterTitle || cleanText(byToc.chapter_title)].filter(Boolean); + if (tocParts.length > 0) + return tocParts.join(' > '); + } + const parts = dedupeStrings(payload.breadcrumb ?? []); + if (parts.length === 0) + return chapterTitle; + if (!chapterTitle) + return parts.join(' > '); + return [...parts.slice(0, -1), chapterTitle].filter(Boolean).join(' > '); +} +function parseParticipantCount(input) { + const match = cleanText(input).match(/(\d+)\s*人参与/); + return match?.[1] ? Number(match[1]) : 0; +} +// Course正文是由多个内联节点拼接而成，URL 常在 DOM 文本合并时被打散。 +export function repairScysBrokenUrls(input) { + let output = cleanText(input); + if (!output) + return ''; + output = output.replace(/\b(https?)\s*:\s*\/\//gi, '$1://'); + output = output.replace(/(https?:\/\/)\s+/gi, '$1'); + let previous = ''; + while (output !== previous) { + previous = output; + output = output.replace(/(https?:\/\/[A-Za-z0-9._-]*[./])\s+([A-Za-z0-9._/-]+)/gi, '$1$2'); + output = output.replace(/(https?:\/\/[A-Za-z0-9._-]+)\s+([A-Za-z0-9._-]+\.[A-Za-z]{2,}(?:\/[A-Za-z0-9._/-]*)?)/gi, '$1$2'); + } + output = output.replace(/https?:\/\/www\.curor\.com\//gi, 'http://www.cursor.com/'); + output = output.replace(/https?:\/\/github\.com\/ignup/gi, 'http://github.com/signup'); + output = output.replace(/https?:\/\/iliconflow\.cn\//gi, 'http://siliconflow.cn/'); + return output; +} +export function summarizeScysToc(rows) { + return rows + .slice(0, 24) + .map((row, index) => { + const section = cleanText(row.section); + const group = cleanText(row.group); + const chapterTitle = cleanText(row.chapter_title); + const entryType = cleanText(row.entry_type); + const left = entryType === 'section' + ? section || chapterTitle || group + : [section, group, chapterTitle].filter(Boolean).join(' > ').replace(/ > ([^>]+)$/, '/$1'); + return `${index + 1}.${left}${row.chapter_id ? `(${cleanText(row.chapter_id)})` : ''}`; + }) + .join(' | '); +} +export function buildScysCourseChapterUrls(baseUrl, rows) { + const courseUrl = new URL(toScysCourseUrl(baseUrl)); + const seen = new Set(); + const urls = []; + for (const row of rows) { + const chapterId = cleanText(row.chapter_id); + if (!chapterId) + continue; + if (cleanText(row.entry_type) && cleanText(row.entry_type) !== 'chapter') + continue; + if (seen.has(chapterId)) + continue; + seen.add(chapterId); + const url = new URL(courseUrl.toString()); + url.searchParams.set('chapterId', chapterId); + urls.push(url.toString()); + } + return urls; +} +export function normalizeScysCoursePayload(payload) { + const chapterTitle = chooseCourseChapterTitle(payload); + const images = normalizeImages(payload.images ?? []); + const contentImages = normalizeImages(payload.contentImages ?? []); + const links = dedupeStrings(payload.links ?? []).map((value) => normalizeScysUrl(value)).filter(Boolean); + return { + course_title: cleanText(payload.courseTitle), + chapter_title: chapterTitle, + breadcrumb: normalizeBreadcrumb(payload, chapterTitle), + content: repairScysBrokenUrls(cleanText(payload.content)), + chapter_id: cleanText(payload.chapterId), + toc_summary: summarizeScysToc(payload.tocRows ?? []), + url: safeNormalizeScysUrl(payload.pageUrl || ''), + raw_url: safeNormalizeScysUrl(payload.pageUrl || ''), + updated_at_text: cleanText(payload.updatedAtText), + copyright_text: cleanText(payload.copyrightText), + prev_chapter: cleanText(payload.prevChapter), + next_chapter: cleanText(payload.nextChapter), + participant_count: parseParticipantCount(cleanText(payload.participantText)), + discussion_hint: cleanText(payload.discussionHint), + links, + images, + image_count: images.length, + content_images: contentImages, + content_image_count: contentImages.length, + image_dir: '', + }; +} diff --git a/clis/scys/course-utils.test.js b/clis/scys/course-utils.test.js new file mode 100644 index 000000000..f8768fc18 --- /dev/null +++ b/clis/scys/course-utils.test.js @@ -0,0 +1,93 @@ +import { describe, expect, it } from 'vitest'; +import { buildScysCourseChapterUrls, normalizeScysCoursePayload, repairScysBrokenUrls, summarizeScysToc, } from './course-utils.js'; +describe('repairScysBrokenUrls', () => { + it('repairs spaced and split urls in extracted course text', () => { + const input = [ + '工具地址：http ://raphael.app', + '编辑器：http ://www.cur or.com/', + '注册页：http ://github.com/ ignup', + '平台：http :// iliconflow.cn/', + ].join(' '); + expect(repairScysBrokenUrls(input)).toContain('http://raphael.app'); + expect(repairScysBrokenUrls(input)).toContain('http://www.cursor.com/'); + expect(repairScysBrokenUrls(input)).toContain('http://github.com/signup'); + expect(repairScysBrokenUrls(input)).toContain('http://siliconflow.cn/'); + }); +}); +describe('normalizeScysCoursePayload', () => { + it('prefers the content title over a stale active chapter when chapterId is explicit', () => { + const result = normalizeScysCoursePayload({ + courseTitle: '【深海圈】AI产品出海', + chapterTitle: '课程目标', + currentChapter: '课程前言', + breadcrumb: ['预备篇', '图文', '课程前言'], + content: '课程目标：正本清源', + chapterId: '4038', + pageUrl: 'https://scys.com/course/detail/92?chapterId=4038', + images: [ + 'https://cdn.example.com/cover.jpg', + '/assets/logo.png', + 'data:image/png;base64,abc', + '/images/pic_empty.png', + ], + contentImages: ['https://cdn.example.com/content-1.jpg'], + updatedAtText: '更新于：2025.12.02 08:03', + copyrightText: '版权归生财有术及手册出品人所有', + prevChapter: '上一节 课程前言', + nextChapter: '下一节 基础篇', + participantText: '146人参与', + discussionHint: '发起讨论', + links: [' https://scys.com/course/detail/92?chapterId=4038 ', 'https://example.com/a '], + tocRows: [ + { section: '预备篇', group: '图文', chapter_id: '4137', chapter_title: '课程前言', status: '737人学过', is_current: false, rank: 1, entry_type: 'chapter' }, + { section: '预备篇', group: '图文', chapter_id: '4038', chapter_title: '课程目标', status: '508人学过', is_current: true, rank: 2, entry_type: 'chapter' }, + ], + }); + expect(result.chapter_title).toBe('课程目标'); + expect(result.breadcrumb).toBe('预备篇 > 图文 > 课程目标'); + expect(result.updated_at_text).toBe('更新于：2025.12.02 08:03'); + expect(result.participant_count).toBe(146); + expect(result.image_count).toBe(2); + expect(result.images).toEqual(['https://cdn.example.com/cover.jpg', 'https://scys.com/assets/logo.png']); + expect(result.content_image_count).toBe(1); + expect(result.links).toEqual([ + 'https://scys.com/course/detail/92?chapterId=4038', + 'https://example.com/a', + ]); + }); + it('prefers toc-based section and group when breadcrumb is polluted by sidebar state', () => { + const result = normalizeScysCoursePayload({ + courseTitle: '【深海圈】AI产品出海', + chapterTitle: '课程前言', + breadcrumb: ['问答（持续更新）', '图文', '课程前言'], + content: '课程前言正文', + chapterId: '4137', + pageUrl: 'https://scys.com/course/detail/92?chapterId=4137', + tocRows: [ + { section: '预备篇', group: '图文', chapter_id: '4137', chapter_title: '课程前言', status: '737人学过', is_current: false, rank: 1, entry_type: 'chapter' }, + ], + }); + expect(result.breadcrumb).toBe('预备篇 > 图文 > 课程前言'); + }); +}); +describe('summarizeScysToc', () => { + it('includes all visible groups and chapters in the summary', () => { + expect(summarizeScysToc([ + { rank: 1, entry_type: 'chapter', section: '预备篇', group: '图文', chapter_id: '4137', chapter_title: '课程前言', status: '', is_current: false }, + { rank: 2, entry_type: 'chapter', section: '预备篇', group: '图文', chapter_id: '4038', chapter_title: '课程目标', status: '', is_current: true }, + { rank: 3, entry_type: 'section', section: '基础篇', group: '基础篇', chapter_id: '', chapter_title: '基础篇', status: '', is_current: false }, + ])).toBe('1.预备篇 > 图文/课程前言(4137) | 2.预备篇 > 图文/课程目标(4038) | 3.基础篇'); + }); +}); +describe('buildScysCourseChapterUrls', () => { + it('builds deterministic chapter urls from toc rows', () => { + expect(buildScysCourseChapterUrls('https://scys.com/course/detail/92', [ + { rank: 1, entry_type: 'section', section: '预备篇', group: '预备篇', chapter_id: '', chapter_title: '预备篇', status: '', is_current: false }, + { rank: 2, entry_type: 'chapter', section: '预备篇', group: '图文', chapter_id: '4137', chapter_title: '课程前言', status: '', is_current: false }, + { rank: 3, entry_type: 'chapter', section: '预备篇', group: '图文', chapter_id: '4038', chapter_title: '课程目标', status: '', is_current: true }, + ])).toEqual([ + 'https://scys.com/course/detail/92?chapterId=4137', + 'https://scys.com/course/detail/92?chapterId=4038', + ]); + }); +}); diff --git a/clis/scys/course.js b/clis/scys/course.js new file mode 100644 index 000000000..3568b8829 --- /dev/null +++ b/clis/scys/course.js @@ -0,0 +1,50 @@ +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { downloadScysCourseImages } from './course-download.js'; +import { extractScysCourse, extractScysCourseAll } from './extractors.js'; +cli({ + site: 'scys', + name: 'course', + description: 'Read SCYS course detail content and chapter context', + domain: 'scys.com', + strategy: Strategy.COOKIE, + navigateBefore: false, + args: [ + { name: 'url', required: true, positional: true, help: 'Course URL: /course/detail/:id[?chapterId=...]' }, + { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, + { name: 'max-length', type: 'int', default: 4000, help: 'Max content length' }, + { name: 'all', type: 'boolean', default: false, help: 'Export all deterministic chapter ids from TOC' }, + { name: 'download-images', type: 'boolean', default: false, help: 'Download course page images to local directory' }, + { name: 'output', default: './scys-course-downloads', help: 'Image output directory' }, + ], + columns: [ + 'course_title', + 'chapter_title', + 'breadcrumb', + 'updated_at_text', + 'participant_count', + 'image_count', + 'content_image_count', + 'prev_chapter', + 'next_chapter', + 'chapter_id', + 'course_id', + 'url', + 'image_dir', + ], + func: async (page, kwargs) => { + const all = kwargs.all === true || String(kwargs.all) === 'true'; + const data = all + ? await extractScysCourseAll(page, String(kwargs.url), { + waitSeconds: Number(kwargs.wait ?? 3), + maxLength: Number(kwargs['max-length'] ?? 4000), + }) + : await extractScysCourse(page, String(kwargs.url), { + waitSeconds: Number(kwargs.wait ?? 3), + maxLength: Number(kwargs['max-length'] ?? 4000), + }); + const downloadImages = kwargs['download-images'] === true || String(kwargs['download-images']) === 'true'; + if (!downloadImages) + return data; + return downloadScysCourseImages(page, data, String(kwargs.output ?? './scys-course-downloads')); + }, +}); diff --git a/clis/scys/extractors.js b/clis/scys/extractors.js new file mode 100644 index 000000000..67c7f5abb --- /dev/null +++ b/clis/scys/extractors.js @@ -0,0 +1,1227 @@ +import { AuthRequiredError, EmptyResultError } from '@jackwener/opencli/errors'; +import { cleanText, extractScysArticleMeta, extractScysCourseId, normalizeScysUrl, toScysArticleUrl, toScysCourseUrl, } from './common.js'; +import { buildScysTopicLink, formatScysRelativeTime, inferTopicIdFromImageUrls, normalizeOpportunityTab, parseAiSummaryText, stripScysRichText, splitOpportunityFlagsAndTags, } from './opportunity-utils.js'; +import { buildScysCourseChapterUrls, normalizeScysCoursePayload, repairScysBrokenUrls, } from './course-utils.js'; +const SCYS_DOMAIN = 'scys.com'; +const SCYS_TEXT_FIXUPS = [ + [/\bCur\s*or\b/g, 'Cursor'], + [/\bBu\s*ine\b/g, 'Business'], + [/\bJava\s*cript\b/g, 'Javascript'], + [/\bSupaba\s*e\b/g, 'Supabase'], + [/\bcreen\s*haring\b/gi, 'screensharing'], + [/\bfa\s*t3d\b/gi, 'fast3d'], +]; +async function gotoAndWait(page, url, waitSeconds) { + await page.goto(url); + await page.wait(waitSeconds); +} +function pickPreferredScysLink(candidates) { + const links = Array.from(new Set(candidates + .map((value) => cleanText(value)) + .filter(Boolean) + .map((value) => value.replace(/\s+/g, '')))); + if (links.length === 0) + return ''; + const detail = links.find((link) => /^https?:\/\/(?:www\.)?scys\.com\/articleDetail\//i.test(link)); + if (detail) + return detail; + const internal = links.find((link) => /^https?:\/\/(?:www\.)?scys\.com\//i.test(link)); + if (internal) + return internal; + return links[0] ?? ''; +} +function parseCnNumberToken(token) { + const raw = cleanText(token); + if (!raw) + return 0; + const numeric = Number(raw.replace(/[万亿]/g, '')); + if (!Number.isFinite(numeric)) + return 0; + if (raw.endsWith('万')) + return Math.floor(numeric * 10_000); + if (raw.endsWith('亿')) + return Math.floor(numeric * 100_000_000); + return Math.floor(numeric); +} +function parseInteractionCounts(raw) { + const text = cleanText(raw); + if (!text) + return { likes: 0, comments: 0, favorites: 0 }; + const matched = text.match(/[0-9]+(?:\.[0-9]+)?(?:万|亿)?/g) ?? []; + return { + likes: parseCnNumberToken(matched[0] ?? ''), + comments: parseCnNumberToken(matched[1] ?? ''), + favorites: parseCnNumberToken(matched[2] ?? ''), + }; +} +function buildScysInteractions(like, comments, favorites, fallback) { + const likeCount = Number(like); + const commentCount = Number(comments); + const favoriteCount = Number(favorites); + if ([likeCount, commentCount, favoriteCount].every((n) => Number.isFinite(n) && n >= 0)) { + const likes = Math.floor(likeCount); + const commentsValue = Math.floor(commentCount); + const favoritesValue = Math.floor(favoriteCount); + return { + likes, + comments: commentsValue, + favorites: favoritesValue, + display: `点赞${likes} 评论${commentsValue} 收藏${favoritesValue}`, + }; + } + const parsed = parseInteractionCounts(fallback); + return { + ...parsed, + display: `点赞${parsed.likes} 评论${parsed.comments} 收藏${parsed.favorites}`, + }; +} +function trimWithLimit(value, maxLength) { + const text = polishScysText(value); + if (!text) + return ''; + return text.slice(0, maxLength); +} +function polishScysText(value) { + let text = cleanText(value); + if (!text) + return ''; + for (const [pattern, replacement] of SCYS_TEXT_FIXUPS) { + text = text.replace(pattern, replacement); + } + return text; +} +function extractFirstNumber(value) { + const text = cleanText(value); + if (!text) + return 0; + const match = text.match(/[0-9]+(?:\.[0-9]+)?(?:万|亿)?/); + if (!match?.[0]) + return 0; + const raw = match[0]; + const numeric = Number(raw.replace(/[万亿]/g, '')); + if (!Number.isFinite(numeric)) + return 0; + if (raw.endsWith('万')) + return Math.floor(numeric * 10_000); + if (raw.endsWith('亿')) + return Math.floor(numeric * 100_000_000); + return Math.floor(numeric); +} +function isLikelyExternalLink(url) { + if (!url) + return false; + return /^https?:\/\//i.test(url) && !/^https?:\/\/(?:www\.)?scys\.com\//i.test(url); +} +function normalizeMaybeBrokenUrl(raw) { + return cleanText(raw).replace(/\s+/g, ''); +} +function isLikelyFalsePositiveLink(url) { + const normalized = normalizeMaybeBrokenUrl(url); + if (!/^https?:\/\//i.test(normalized)) + return false; + // Heuristic: markdown/autolink-like false positives such as "7.AI" in numbered lists. + // Example false extraction: http://7.AI + return /^https?:\/\/\d+\.[a-z]{2,}\/?$/i.test(normalized); +} +function normalizeScysTocRows(rows) { + const seen = new Set(); + const out = []; + for (const row of rows ?? []) { + const entryType = cleanText(row.entry_type || 'chapter'); + const section = cleanText(row.section ?? ''); + const group = cleanText(row.group ?? ''); + const chapterId = cleanText(row.chapter_id ?? ''); + const chapterTitle = cleanText(row.chapter_title ?? ''); + const status = cleanText(row.status ?? ''); + const isCurrent = !!row.is_current; + if (!section && !group && !chapterTitle && !chapterId) + continue; + const key = [ + entryType || 'chapter', + section, + group, + chapterId, + chapterTitle, + status, + isCurrent ? '1' : '0', + ].join('|'); + if (seen.has(key)) + continue; + seen.add(key); + out.push({ + rank: out.length + 1, + entry_type: entryType === 'section' ? 'section' : 'chapter', + section, + group, + chapter_id: chapterId, + chapter_title: chapterTitle, + status, + is_current: isCurrent, + }); + } + return out; +} +async function evaluateScysTocRows(page, opts = {}) { + const shouldExpand = opts.expandCollapsedSections === true; + const rows = await page.evaluate(` + (async () => { + const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim(); + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + const out = []; + const seen = new Set(); + const pushRow = (row) => { + const normalized = { + entry_type: clean(row.entry_type || 'chapter'), + section: clean(row.section || ''), + group: clean(row.group || ''), + chapter_id: clean(row.chapter_id || ''), + chapter_title: clean(row.chapter_title || ''), + status: clean(row.status || ''), + is_current: !!row.is_current, + }; + if (!normalized.section && !normalized.group && !normalized.chapter_id && !normalized.chapter_title) return; + const key = [ + normalized.entry_type || 'chapter', + normalized.section, + normalized.group, + normalized.chapter_id, + normalized.chapter_title, + normalized.status, + normalized.is_current ? '1' : '0', + ].join('|'); + if (seen.has(key)) return; + seen.add(key); + out.push(normalized); + }; + + const chapterSelector = '.vc-chapter-item[data-item-id], .chapter-list .vc-chapter-item, .vc-chapter-item'; + + ${shouldExpand ? ` + const expandSections = async () => { + const sections = Array.from(document.querySelectorAll('.catalogue-section')); + for (const section of sections) { + const currentCount = section.querySelectorAll(chapterSelector).length; + const isExpanded = + section.classList.contains('expanded') || + !!section.querySelector('.vc-section-header.expanded'); + if (currentCount > 0 || isExpanded) continue; + + const sectionTitleEl = + section.querySelector('.section-title') || + section.querySelector('.vc-section-header') || + section; + + if (!sectionTitleEl) continue; + + if (typeof sectionTitleEl.click === 'function') { + sectionTitleEl.click(); + } else { + sectionTitleEl.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true })); + } + + for (let i = 0; i < 12; i += 1) { + await sleep(200); + const count = section.querySelectorAll(chapterSelector).length; + const expandedNow = + section.classList.contains('expanded') || + !!section.querySelector('.vc-section-header.expanded'); + if (count > 0 || expandedNow) break; + } + } + }; + + await expandSections(); + ` : ''} + + const groups = Array.from(document.querySelectorAll('.vc-chapter-group')); + const sections = Array.from(document.querySelectorAll('.catalogue-section')); + + if (sections.length > 0) { + sections.forEach((section) => { + const sectionTitle = clean( + section.querySelector('.section-title, .catalogue-section-title, .title')?.textContent || '' + ); + if (sectionTitle) { + pushRow({ + entry_type: 'section', + section: sectionTitle, + group: sectionTitle, + chapter_title: sectionTitle, + chapter_id: '', + status: '', + is_current: false, + }); + } + + const sectionGroups = Array.from(section.querySelectorAll('.vc-chapter-group')); + if (sectionGroups.length > 0) { + sectionGroups.forEach((group) => { + const groupTitle = clean( + group.querySelector('.group-title, .chapter-group-title, .vc-group-title')?.textContent || '' + ); + const items = Array.from(group.querySelectorAll(chapterSelector)); + items.forEach((item) => { + const title = clean( + item.querySelector('.chapter-title')?.textContent || + item.querySelector('.chapter-content')?.textContent || + item.textContent || + '' + ); + const status = clean(item.querySelector('.chapter-status, .chapter-meta')?.textContent || ''); + const cls = item.className || ''; + const isCurrent = /active|current|selected|is-active/.test(cls) || item.getAttribute('aria-current') === 'true'; + if (!title) return; + pushRow({ + entry_type: 'chapter', + section: sectionTitle, + group: groupTitle || sectionTitle, + chapter_id: item.getAttribute('data-item-id') || '', + chapter_title: title, + status, + is_current: isCurrent, + }); + }); + }); + } + }); + } + + if (out.length === 0 && groups.length > 0) { + groups.forEach((group) => { + const groupTitle = clean( + group.querySelector('.group-title, .chapter-group-title, .vc-group-title')?.textContent || '' + ); + const sectionTitle = clean( + group.closest('.catalogue-section')?.querySelector('.section-title, .catalogue-section-title, .title')?.textContent || '' + ); + const items = Array.from(group.querySelectorAll(chapterSelector)); + items.forEach((item) => { + const title = clean( + item.querySelector('.chapter-title')?.textContent || + item.querySelector('.chapter-content')?.textContent || + item.textContent || + '' + ); + const status = clean(item.querySelector('.chapter-status, .chapter-meta')?.textContent || ''); + const cls = item.className || ''; + const isCurrent = /active|current|selected|is-active/.test(cls) || item.getAttribute('aria-current') === 'true'; + if (!title) return; + pushRow({ + entry_type: 'chapter', + section: sectionTitle, + group: groupTitle, + chapter_id: item.getAttribute('data-item-id') || '', + chapter_title: title, + status, + is_current: isCurrent, + }); + }); + }); + } + + if (out.length === 0) { + const items = Array.from(document.querySelectorAll(chapterSelector)); + items.forEach((item) => { + const title = clean( + item.querySelector('.chapter-title')?.textContent || + item.querySelector('.chapter-content')?.textContent || + item.textContent || + '' + ); + const status = clean(item.querySelector('.chapter-status, .chapter-meta')?.textContent || ''); + const cls = item.className || ''; + const isCurrent = /active|current|selected|is-active/.test(cls) || item.getAttribute('aria-current') === 'true'; + if (!title) return; + pushRow({ + entry_type: 'chapter', + section: '', + group: '', + chapter_id: item.getAttribute('data-item-id') || '', + chapter_title: title, + status, + is_current: isCurrent, + }); + }); + } + + return out; + })() + `); + return normalizeScysTocRows(rows); +} +function polishScysCourseSummary(summary, courseId, maxLength) { + return { + ...summary, + course_id: courseId, + course_title: polishScysText(summary.course_title), + chapter_title: polishScysText(summary.chapter_title), + breadcrumb: polishScysText(summary.breadcrumb), + content: polishScysText(repairScysBrokenUrls(summary.content)).slice(0, maxLength), + toc_summary: polishScysText(summary.toc_summary), + updated_at_text: polishScysText(summary.updated_at_text), + copyright_text: polishScysText(summary.copyright_text), + prev_chapter: polishScysText(summary.prev_chapter), + next_chapter: polishScysText(summary.next_chapter), + discussion_hint: polishScysText(summary.discussion_hint), + url: summary.url || '', + raw_url: summary.raw_url || '', + links: Array.from(new Set((summary.links ?? []).map((link) => cleanText(link)).filter(Boolean))), + images: Array.from(new Set((summary.images ?? []).map((link) => cleanText(link)).filter(Boolean))), + content_images: Array.from(new Set((summary.content_images ?? []).map((link) => cleanText(link)).filter(Boolean))), + image_count: Array.isArray(summary.images) ? summary.images.length : 0, + content_image_count: Array.isArray(summary.content_images) ? summary.content_images.length : 0, + image_dir: summary.image_dir || '', + }; +} +async function extractScysCourseSingle(page, inputUrl, opts = {}) { + const url = toScysCourseUrl(inputUrl); + const waitSeconds = Math.max(1, Number(opts.waitSeconds ?? 3)); + const maxLength = Math.max(300, Number(opts.maxLength ?? 4000)); + await gotoAndWait(page, url, waitSeconds); + await ensureScysLogin(page); + const tocRows = opts.tocRows ?? await evaluateScysTocRows(page); + const payload = await page.evaluate(` + (() => { + const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim(); + const normalizeUrl = (value) => clean(value).replace(/\\s+/g, ''); + const abs = (href) => { + const raw = normalizeUrl(href); + if (!raw) return ''; + if (raw.startsWith('http://') || raw.startsWith('https://')) return raw; + if (raw.startsWith('//')) return location.protocol + raw; + if (raw.startsWith('/')) return location.origin + raw; + return ''; + }; + const uniq = (list) => Array.from(new Set(list.filter(Boolean))); + const pickFirstText = (selectors) => { + for (const selector of selectors) { + const el = document.querySelector(selector); + const text = clean(el?.textContent || el?.innerText || ''); + if (text) return text; + } + return ''; + }; + const pickFirstEl = (selectors) => { + for (const selector of selectors) { + const el = document.querySelector(selector); + if (el) return el; + } + return null; + }; + const bodyText = clean(document.body?.innerText || ''); + const capture = (matcher) => clean(bodyText.match(matcher)?.[0] || ''); + + const contentEl = pickFirstEl([ + '.feishu-doc-content', + '.document-container', + '.vc-course-content', + '.course-content-container', + '.content-container', + '.vc-course-main', + ]); + + const breadcrumbTexts = Array.from( + document.querySelectorAll( + '.simple-catalog-toggle .breadcrumb-item, .breadcrumb-item, .breadcrumb a, .breadcrumb span, .vc-breadcrumb a, .vc-breadcrumb span' + ) + ) + .map((el) => clean(el.textContent || '')) + .filter(Boolean); + + const chapterItems = Array.from(document.querySelectorAll('.vc-chapter-item[data-item-id], .chapter-list .vc-chapter-item')).map((el) => { + const item = el; + const id = clean(item.getAttribute('data-item-id') || ''); + const title = clean( + item.querySelector('.chapter-title')?.textContent || + item.querySelector('.chapter-content')?.textContent || + item.textContent || + '' + ); + const cls = item.className || ''; + const isCurrent = /active|current|selected|is-active/.test(cls) || item.getAttribute('aria-current') === 'true'; + return { id, title, isCurrent }; + }).filter((row) => row.title); + + const chapterIdFromQuery = new URL(location.href).searchParams.get('chapterId') || ''; + const chapterId = chapterIdFromQuery || chapterItems.find((item) => item.isCurrent)?.id || ''; + const activeChapterEl = + document.querySelector('.vc-chapter-item.is-active, .vc-chapter-item.is-current, .vc-chapter-item.active') || + null; + const activeGroupTitle = clean( + activeChapterEl?.closest('.vc-chapter-group')?.querySelector('.group-title, .chapter-group-title')?.textContent || '' + ); + const activeSectionTitle = clean( + activeChapterEl?.closest('.catalogue-section')?.querySelector('.section-title, .catalogue-section-title, .title')?.textContent || '' + ); + const activeChapterTitle = clean(activeChapterEl?.querySelector('.chapter-title')?.textContent || ''); + const catalogBreadcrumb = [activeSectionTitle, activeGroupTitle, activeChapterTitle].filter(Boolean); + + const courseTitle = + pickFirstText([ + '.vc-course-main .course-name', + '.course-name', + '.vc-course-sidebar .course-title', + '.course-header .course-title', + '.course-title', + ]) || + clean((document.title || '').split(' - ')[0] || ''); + + const chapterTitleFromContent = pickFirstText([ + '.vc-course-content .content-title', + '.course-content-container .content-title', + '.content-title', + '.current-chapter', + '.vc-course-main h1', + 'h1', + ]); + + const allImages = uniq( + Array.from(document.querySelectorAll('img')) + .map((img) => abs(img.currentSrc || img.getAttribute('src') || img.getAttribute('data-src') || '')) + ); + const contentImages = uniq( + Array.from(contentEl?.querySelectorAll?.('img') || []) + .map((img) => abs(img.currentSrc || img.getAttribute('src') || img.getAttribute('data-src') || '')) + ); + const links = uniq( + Array.from(contentEl?.querySelectorAll?.('a[href]') || []) + .map((link) => abs(link.getAttribute('href') || '')) + ); + + return { + courseTitle, + chapterTitle: chapterTitleFromContent, + currentChapter: + chapterItems.find((item) => item.id === chapterId)?.title || + chapterItems.find((item) => item.isCurrent)?.title || + activeChapterTitle || + chapterTitleFromContent, + breadcrumb: catalogBreadcrumb.length >= 2 ? catalogBreadcrumb : breadcrumbTexts, + content: clean(contentEl?.innerText || ''), + chapterId, + pageUrl: location.href, + images: allImages, + contentImages, + links, + updatedAtText: capture(/更新于[:：]?\\s*[0-9]{4}[./-][0-9]{2}[./-][0-9]{2}\\s*[0-9]{2}:[0-9]{2}/), + copyrightText: capture(/版权归[^。！？]{0,120}(?:。|$)/), + prevChapter: bodyText.includes('上一节') ? '上一节' : '', + nextChapter: bodyText.includes('下一节') ? '下一节' : '', + participantText: capture(/\\d+\\s*人参与/), + discussionHint: bodyText.includes('发起讨论') ? '发起讨论' : (bodyText.includes('讨论区') ? '讨论区' : ''), + }; + })() + `); + if (!payload) { + throw new EmptyResultError('scys/course', 'Failed to extract course page content'); + } + const courseId = extractScysCourseId(url); + const normalized = normalizeScysCoursePayload({ + courseTitle: payload.courseTitle, + chapterTitle: payload.chapterTitle, + currentChapter: payload.currentChapter, + breadcrumb: Array.isArray(payload.breadcrumb) ? payload.breadcrumb : [], + content: payload.content, + chapterId: payload.chapterId, + pageUrl: String(payload.pageUrl || url), + images: Array.isArray(payload.images) ? payload.images : [], + contentImages: Array.isArray(payload.contentImages) ? payload.contentImages : [], + links: Array.isArray(payload.links) ? payload.links : [], + tocRows, + updatedAtText: payload.updatedAtText, + copyrightText: payload.copyrightText, + prevChapter: payload.prevChapter, + nextChapter: payload.nextChapter, + discussionHint: payload.discussionHint, + participantText: payload.participantText, + }); + const result = polishScysCourseSummary({ + ...normalized, + url: normalized.url || normalizeScysUrl(url), + raw_url: normalized.raw_url || normalizeScysUrl(url), + }, courseId, maxLength); + if (!result.content && tocRows.length === 0) { + throw new EmptyResultError('scys/course', 'No course content or table of contents was detected'); + } + return result; +} +export async function ensureScysFeedReady(page) { + await page.evaluate(` + (async () => { + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + for (let i = 0; i < 12; i += 1) { + const hasCards = document.querySelectorAll('.post-list-container .compact-card, .compact-card').length > 0; + const hasControls = document.querySelector('.vc-secondary-filter .filter-item, .titles.selector .button, .select.wrap .button'); + if (hasCards || hasControls) return; + await sleep(250); + } + })() + `); +} +export async function ensureScysLogin(page) { + const state = await page.evaluate(` + (() => { + const text = (document.body?.innerText || '').slice(0, 12000); + const strongLoginText = /扫码登录|手机号登录|验证码登录|微信登录|账号登录|登录\\/注册/.test(text); + const genericLoginText = /请登录|登录后/.test(text); + const loginCtaText = /立即登录|去登录|重新登录|登录查看|登录后查看|登录可见|请先登录/.test(text); + const loginByDom = !!document.querySelector( + '.login-container, .login-box, .qrcode-login, .login-btn, .btn-login, .auth-mask, .auth-dialog, form[action*="login"], input[type="password"], input[type="tel"][placeholder*="手机号"], button[class*="login"], a[href*="login"]' + ); + const hasContentSignals = !!document.querySelector( + '.course-detail-page, .vc-course-main, .post-list-container, .compact-card, .activity-left, .week-card, .vc-secondary-filter' + ); + const routeLooksLikeLogin = location.pathname.includes('/login'); + return { strongLoginText, genericLoginText, loginCtaText, loginByDom, hasContentSignals, routeLooksLikeLogin }; + })() + `); + if (!state) + return; + const shouldBlock = !!state.routeLooksLikeLogin + || !!state.loginByDom + || (!!state.loginCtaText && !state.hasContentSignals) + || (!!state.strongLoginText && !state.hasContentSignals) + || (!!state.genericLoginText && !state.hasContentSignals); + if (shouldBlock) { + throw new AuthRequiredError(SCYS_DOMAIN, 'SCYS content requires a logged-in browser session'); + } +} +export async function extractScysCourse(page, inputUrl, opts = {}) { + return extractScysCourseSingle(page, inputUrl, opts); +} +export async function extractScysCourseAll(page, inputUrl, opts = {}) { + const tocRows = await extractScysToc(page, inputUrl, opts); + const urls = buildScysCourseChapterUrls(inputUrl, tocRows); + if (urls.length === 0) { + throw new EmptyResultError('scys/course', 'No chapter ids were detected for deterministic full-course export'); + } + const out = []; + for (const url of urls) { + out.push(await extractScysCourseSingle(page, url, { ...opts, tocRows })); + } + return out; +} +export async function extractScysToc(page, courseInput, opts = {}) { + const url = toScysCourseUrl(courseInput); + const waitSeconds = Math.max(1, Number(opts.waitSeconds ?? 2)); + await gotoAndWait(page, url, waitSeconds); + await ensureScysLogin(page); + const normalized = await evaluateScysTocRows(page, { expandCollapsedSections: true }); + if (normalized.length === 0) { + await ensureScysLogin(page); + throw new EmptyResultError('scys/toc', 'No chapter list was detected on this course page. If your SCYS browser session expired, reopen scys.com in Chrome, log in again, then retry.'); + } + return normalized; +} +export async function extractScysArticle(page, inputUrl, opts = {}) { + const url = toScysArticleUrl(inputUrl); + const waitSeconds = Math.max(1, Number(opts.waitSeconds ?? 5)); + const maxLength = Math.max(300, Number(opts.maxLength ?? 4000)); + const fromUrl = extractScysArticleMeta(url); + await gotoAndWait(page, url, waitSeconds); + await ensureScysLogin(page); + const payload = await page.evaluate(` + (() => { + const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim(); + const normalizeUrl = (value) => clean(value).replace(/\\s+/g, ''); + const abs = (href) => { + const raw = normalizeUrl(href); + if (!raw) return ''; + if (raw.startsWith('http://') || raw.startsWith('https://')) return raw; + if (raw.startsWith('/')) return location.origin + raw; + return ''; + }; + const uniq = (list) => Array.from(new Set(list.filter(Boolean))); + const pickText = (selectors) => { + for (const selector of selectors) { + const text = clean(document.querySelector(selector)?.textContent || ''); + if (text) return text; + } + return ''; + }; + + const articleMatch = location.pathname.match(/^\\/articleDetail\\/([^/]+)\\/([^/]+)/); + const entityType = clean(articleMatch?.[1] || ''); + const topicId = clean(articleMatch?.[2] || ''); + + const title = pickText([ + '.title-line .post-title', + '.post-title', + '.article-title', + '.topic-title', + 'h1', + ]) || clean(document.title || ''); + const author = pickText([ + '.post-item-top-right .name', + '.post-item-top .name', + '.post-item-top-right .user-name', + '.post-item-top .user-name', + ]); + const time = pickText([ + '.post-item-top-right .date', + '.post-item-top .date', + '.post-item-top-right .time', + '.post-item-top .time', + ]); + + const content = pickText([ + '.post-content', + '.content-container .post-content', + '.content-container', + ]); + const aiSummary = pickText([ + '.ai-summary-container .content', + '.ai-summary-container .content-stream', + '.ai-summary-container', + ]); + + const flags = uniq( + Array.from(document.querySelectorAll('.title-line .icon, .title-line .tag, .title-line .flag')) + .map((el) => clean(el.textContent || '')) + ); + const tags = uniq( + Array.from(document.querySelectorAll('.label-box .tag-item, .tag-label-box .tag-item, .label-box .tag')) + .map((el) => clean(el.textContent || '')) + ); + + const interactionNodes = Array.from( + document.querySelectorAll('.interactions .item, .interactions .favorite-wrapper, .interactions .favorite-wrapper .item') + ).map((el) => ({ + cls: (el.className || '').toString(), + text: clean(el.textContent || ''), + })); + + const likeText = clean(document.querySelector('.interactions .like-item')?.textContent || ''); + const favoriteText = clean( + document.querySelector('.interactions .favorite-wrapper .item')?.textContent || + document.querySelector('.interactions .favorite-wrapper')?.textContent || + '' + ); + const commentText = clean( + interactionNodes.find((node) => /item/.test(node.cls) && !/like/.test(node.cls) && /^[0-9]/.test(node.text))?.text || '' + ); + + const imageCandidates = Array.from( + document.querySelectorAll('.image-list-container img, .arco-carousel img, .post-content img, .content-container img') + ) + .map((img) => abs(img.getAttribute('src') || img.getAttribute('data-src') || '')) + .map((src) => normalizeUrl(src)) + .filter(Boolean) + .filter((src) => !src.startsWith('data:')) + .filter((src) => !src.includes('/upload/avatar/')) + .filter((src) => !src.includes('/images/img_bg_empty')) + .filter((src) => /\\/xq\\/images\\/|\\.(jpg|jpeg|png|webp|gif)(\\?|$)/i.test(src)); + const images = uniq(imageCandidates); + + const sourceLinks = uniq( + Array.from(document.querySelectorAll('.post-content a[href], .content-container a[href]')) + .map((a) => abs(a.getAttribute('href') || '')) + .map((href) => normalizeUrl(href)) + ); + const externalLinks = sourceLinks.filter((href) => /^https?:\\/\\//i.test(href) && !/^https?:\\/\\/(?:www\\.)?scys\\.com\\//i.test(href)); + + return { + entityType, + topicId, + title, + author, + time, + flags, + tags, + content, + aiSummary, + likeText, + commentText, + favoriteText, + images, + sourceLinks, + externalLinks, + pageUrl: location.href, + }; + })() + `); + if (!payload) { + throw new EmptyResultError('scys/article', 'Failed to extract article detail page'); + } + const rawFlags = (payload.flags ?? []).map((value) => polishScysText(value)).filter(Boolean); + const rawTags = (payload.tags ?? []).map((value) => polishScysText(value)).filter(Boolean); + const split = splitOpportunityFlagsAndTags([...rawFlags, ...rawTags]); + const flags = Array.from(new Set([ + ...rawFlags, + ...split.flags.map((value) => polishScysText(value)).filter(Boolean), + ])); + const tags = Array.from(new Set([ + ...rawTags, + ...split.tags.map((value) => polishScysText(value)).filter(Boolean), + ])).filter((tag) => !flags.includes(tag)); + const interactions = buildScysInteractions(extractFirstNumber(payload.likeText), extractFirstNumber(payload.commentText), extractFirstNumber(payload.favoriteText)); + const sourceLinks = Array.from(new Set((payload.sourceLinks ?? []) + .map((href) => normalizeMaybeBrokenUrl(href)) + .filter(Boolean) + .filter((href) => !isLikelyFalsePositiveLink(href)))); + const externalLinks = Array.from(new Set((payload.externalLinks ?? []) + .map((href) => normalizeMaybeBrokenUrl(href)) + .filter(isLikelyExternalLink) + .filter((href) => !isLikelyFalsePositiveLink(href)))); + const images = Array.from(new Set((payload.images ?? []).map((src) => normalizeMaybeBrokenUrl(src)).filter(Boolean))); + const content = polishScysText(stripScysRichText(payload.content ?? '')).slice(0, maxLength); + const aiSummary = polishScysText(stripScysRichText(payload.aiSummary ?? '')).slice(0, maxLength); + const title = polishScysText(payload.title ?? ''); + const author = polishScysText(payload.author ?? ''); + if (!title && !content && !aiSummary) { + throw new EmptyResultError('scys/article', 'No title/content was detected on this article page'); + } + return { + entity_type: polishScysText(payload.entityType || fromUrl.entityType), + topic_id: polishScysText(payload.topicId || fromUrl.topicId), + url: normalizeScysUrl(payload.pageUrl || url), + title, + author, + time: polishScysText(payload.time ?? ''), + tags, + flags, + content, + ai_summary: aiSummary, + interactions, + image_count: images.length, + images, + external_link_count: externalLinks.length, + external_links: externalLinks, + source_links: sourceLinks, + raw_url: normalizeScysUrl(payload.pageUrl || url), + }; +} +export async function extractScysFeed(page, inputUrl, opts = {}) { + const url = normalizeScysUrl(inputUrl); + const waitSeconds = Math.max(1, Number(opts.waitSeconds ?? 3)); + const limit = Math.max(1, Number(opts.limit ?? 20)); + const maxLength = Math.max(120, Number(opts.maxLength ?? 600)); + await gotoAndWait(page, url, waitSeconds); + await ensureScysLogin(page); + await ensureScysFeedReady(page); + // API-first extraction: + // feed pages use /shengcai-web/client/homePage/searchTopic as list source. + await page.installInterceptor('shengcai-web/client'); + await page.evaluate(` + (async () => { + const clean = (v) => (v || '').replace(/\\s+/g, ' ').trim(); + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + const isActive = (el) => /active|is-active|selected/.test(el?.className || ''); + + const search = new URL(location.href).searchParams; + const expectedFilter = (search.get('filter') || '').toLowerCase(); + + const homeFilters = Array.from(document.querySelectorAll('.vc-secondary-filter .filter-item')); + if (homeFilters.length > 0) { + const targetLabel = expectedFilter === 'essence' ? '精华' : '全部'; + const target = homeFilters.find((el) => clean(el.textContent || '') === targetLabel) || homeFilters[0]; + const active = homeFilters.find((el) => isActive(el)); + const alt = homeFilters.find((el) => el !== target); + if (active && target && active === target && alt) { + alt.click(); + await sleep(900); + } + if (target) { + target.click(); + await sleep(1200); + } + } + + const profileTabs = Array.from(document.querySelectorAll('.titles.selector .button, .select.wrap .button, .button')) + .filter((el) => ['帖子', '收藏'].includes(clean(el.textContent || ''))); + if (profileTabs.length > 0) { + const posts = profileTabs.find((el) => clean(el.textContent || '') === '帖子') || profileTabs[0]; + const alt = profileTabs.find((el) => el !== posts); + if (posts && isActive(posts) && alt) { + alt.click(); + await sleep(1000); + } + if (posts) { + posts.click(); + await sleep(1200); + } + } + + window.scrollTo(0, document.body.scrollHeight); + await sleep(800); + window.scrollTo(0, 0); + await sleep(300); + })() + `); + const intercepted = await page.getInterceptedRequests(); + const latest = intercepted + .filter((entry) => { + const data = entry?.data; + return data && Array.isArray(data.items) && data.items.some((item) => item?.topicDTO); + }) + .at(-1); + let normalized = []; + if (latest?.data?.items?.length) { + normalized = latest.data.items.slice(0, limit).map((item, index) => { + const topic = item?.topicDTO ?? {}; + const user = item?.topicUserDTO ?? {}; + const menuValues = Array.isArray(topic.menuList) + ? topic.menuList.map((m) => cleanText(m?.value)).filter(Boolean) + : []; + const tags = Array.from(new Set(menuValues.map((v) => polishScysText(v)).filter(Boolean))); + const topicId = cleanText(topic.topicId || topic.entityId); + const entityType = cleanText(topic.entityType || 'xq_topic'); + const url = pickPreferredScysLink([ + item?.detailUrl, + buildScysTopicLink(entityType, topicId), + topic?.externalLink, + ]); + const images = Array.isArray(topic.imageList) + ? topic.imageList.map((u) => cleanText(u)).filter(Boolean) + : []; + const interactions = buildScysInteractions(topic.likeCount, topic.commentsCount, topic.favoriteCount); + const flags = topic.isDigested ? ['精华'] : []; + const summary = trimWithLimit(stripScysRichText(topic.articleContent), maxLength); + return { + rank: index + 1, + author: polishScysText(user.name), + time: formatScysRelativeTime(topic.gmtCreate), + flags, + title: polishScysText(stripScysRichText(topic.showTitle)), + summary, + tags, + interactions, + interactions_display: interactions.display, + url, + raw_url: url, + images, + image_count: images.length, + }; + }).filter((row) => row.title || row.summary); + } + // DOM fallback for cases where interceptor is blocked or request timing misses. + if (normalized.length === 0) { + await page.autoScroll({ times: 2, delayMs: 1200 }); + const rows = await page.evaluate(` + (() => { + const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim(); + const abs = (href) => { + if (!href) return ''; + if (href.startsWith('http://') || href.startsWith('https://')) return href; + if (href.startsWith('/')) return location.origin + href; + return ''; + }; + + const cards = Array.from(document.querySelectorAll('.post-list-container .compact-card, .compact-card')); + return cards.map((card) => { + const userLine = clean(card.querySelector('.user-line')?.textContent || ''); + const author = clean( + card.querySelector('.user-line .user-name, .avatar-group .user-name, .author-name')?.textContent || '' + ); + const time = clean(card.querySelector('.user-line .time-label, .user-line .time')?.textContent || ''); + const badge = clean(card.querySelector('.vc-essence-badge, .badge')?.textContent || ''); + const title = clean(card.querySelector('.title-text, .title-line .title, .title-line')?.textContent || ''); + const preview = clean(card.querySelector('.content-preview, .preview, .content')?.textContent || ''); + const tags = Array.from(card.querySelectorAll('.tags .tag, .tags span, .tag-list .tag')) + .map((el) => clean(el.textContent || '')) + .filter(Boolean); + const interactions = clean(card.querySelector('.compact-interactions, .interactions')?.textContent || ''); + const metaLine = clean(card.querySelector('.meta-line')?.textContent || ''); + const links = Array.from(card.querySelectorAll('a[href]')) + .map((el) => abs(el.getAttribute('href') || '')) + .filter(Boolean); + + return { + author, + time, + user_line: userLine, + badge, + title, + preview, + tags, + interactions, + meta_line: metaLine, + links, + }; + }).filter((item) => item.title || item.preview); + })() + `); + normalized = (rows ?? []).slice(0, limit).map((row, index) => { + const userLine = cleanText(row.user_line ?? '') + .replace(/复制链接|跳转星球|投诉建议/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + const [authorByLine, timeByLine] = userLine.split('·').map((part) => cleanText(part)); + const tags = Array.from(new Set((row.tags ?? []).map((tag) => polishScysText(tag)).filter(Boolean))); + const flags = row.badge ? [polishScysText(row.badge)] : []; + const summary = trimWithLimit(row.preview ?? '', maxLength); + const interactions = buildScysInteractions(undefined, undefined, undefined, row.interactions || row.meta_line); + const url = pickPreferredScysLink(row.links ?? []); + return { + rank: index + 1, + author: polishScysText(row.author ?? authorByLine), + time: cleanText(row.time ?? timeByLine), + flags, + title: polishScysText(row.title ?? '').replace(/^(精华|热门)\s*/, ''), + summary, + tags, + interactions, + interactions_display: interactions.display, + url, + raw_url: url, + images: [], + image_count: 0, + }; + }).filter((row) => row.title || row.summary); + } + if (normalized.length === 0) { + throw new EmptyResultError('scys/feed', 'No feed cards were detected on this page'); + } + return normalized; +} +export async function extractScysOpportunity(page, inputUrl, opts = {}) { + const url = normalizeScysUrl(inputUrl); + const waitSeconds = Math.max(1, Number(opts.waitSeconds ?? 3)); + const limit = Math.max(1, Number(opts.limit ?? 20)); + const tab = normalizeOpportunityTab(opts.tab); + await gotoAndWait(page, url, waitSeconds); + await ensureScysLogin(page); + // API-first extraction. The page internally requests: + // /shengcai-web/client/homePage/searchTopic + // We intercept this payload to get stable fields (time, tags, images, topic ids). + await page.installInterceptor('shengcai-web/client/homePage/searchTopic'); + await page.evaluate(` + (async () => { + const clean = (v) => (v || '').replace(/\\s+/g, ' ').trim(); + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + const target = ${JSON.stringify(tab.label)}; + const filters = Array.from(document.querySelectorAll('.vc-secondary-filter .filter-item')); + const hit = filters.find((el) => clean(el.textContent || '') === target); + const active = filters.find((el) => (el.className || '').includes('active')); + const alt = filters.find((el) => el !== hit); + + // Trigger request even when the current tab is already active: + // switch away once, then switch back to target. + if (active && hit && active === hit && alt) { + alt.click(); + await sleep(1000); + } + + if (hit) { + hit.click(); + } else if (filters.length > 0) { + filters[0].click(); + } + + await sleep(1400); + window.scrollTo(0, document.body.scrollHeight); + await sleep(800); + })() + `); + const intercepted = await page.getInterceptedRequests(); + const latest = intercepted + .filter((entry) => { + const data = entry?.data; + return data && Array.isArray(data.items) && data.items.length > 0; + }) + .at(-1); + let normalized = []; + if (latest?.data?.items?.length) { + normalized = latest.data.items.slice(0, limit).map((item, index) => { + const topic = item?.topicDTO ?? {}; + const user = item?.topicUserDTO ?? {}; + const menuValues = Array.isArray(topic.menuList) + ? topic.menuList.map((m) => cleanText(m?.value)).filter(Boolean) + : []; + const { flags, tags } = splitOpportunityFlagsAndTags(menuValues); + const interactions = buildScysInteractions(topic.likeCount, topic.commentsCount, topic.favoriteCount); + const entityType = cleanText(topic.entityType); + const topicId = cleanText(topic.topicId || topic.entityId); + const images = Array.isArray(topic.imageList) + ? topic.imageList.map((u) => cleanText(u)).filter(Boolean) + : []; + const url = cleanText(item.detailUrl) || buildScysTopicLink(entityType, topicId); + const normalizedFlags = flags.map((f) => polishScysText(f)).filter(Boolean); + const normalizedTags = tags.map((t) => polishScysText(t)).filter(Boolean); + const summary = polishScysText(stripScysRichText(topic.articleContent)); + return { + rank: index + 1, + author: polishScysText(user.name), + time: formatScysRelativeTime(topic.gmtCreate), + flags: normalizedFlags, + title: polishScysText(stripScysRichText(topic.showTitle)), + summary, + ai_summary: polishScysText(parseAiSummaryText(topic.aiSummaryContent)), + tags: normalizedTags, + interactions, + interactions_display: interactions.display, + url, + raw_url: url, + topic_id: topicId, + entity_type: entityType, + images, + image_count: images.length, + }; + }); + } + // DOM fallback: keep the previous extractor as backup when the API payload is blocked. + if (normalized.length === 0) { + await page.autoScroll({ times: 2, delayMs: 1200 }); + const rows = await page.evaluate(` + (() => { + const clean = (value) => (value || '').replace(/\\s+/g, ' ').trim(); + const abs = (href) => { + if (!href) return ''; + if (href.startsWith('http://') || href.startsWith('https://')) return href; + if (href.startsWith('/')) return location.origin + href; + return ''; + }; + const cards = Array.from(document.querySelectorAll('.post-list-container .post-item, .post-item')); + return cards.map((card) => { + const top = card.querySelector('.post-item-top') || card; + const author = clean(top.querySelector('.name, .author, .nickname, .user-name')?.textContent || ''); + const time = clean(top.querySelector('.date, .time, .meta-time')?.textContent || ''); + const flags = Array.from(card.querySelectorAll('.hit-icon, .icon, .post-title .tag, .post-title .flag')) + .map((el) => clean(el.textContent || '')) + .filter(Boolean); + const title = clean(card.querySelector('.post-title, .title-line')?.textContent || ''); + const content = clean(card.querySelector('.content-stream, .post-content, .content-preview')?.textContent || ''); + const aiSummary = clean(card.querySelector('.ai-summary-container .content, .ai-summary-container, .ai-summary')?.textContent || ''); + const tags = Array.from(card.querySelectorAll('.label-box .tag-item, .label-box span, .tags .tag')) + .map((el) => clean(el.textContent || '')) + .filter(Boolean); + const interactions = clean(card.querySelector('.interactions, .compact-interactions')?.textContent || ''); + const images = Array.from(card.querySelectorAll('.image-list img, img.multi-img')) + .map((img) => clean(img.getAttribute('src') || img.getAttribute('data-src') || '')) + .filter(Boolean); + const link = abs(card.querySelector('a[href]')?.getAttribute('href') || ''); + return { author, time, flags, title, content, ai_summary: aiSummary, tags, interactions, link, image_urls: images }; + }).filter((item) => item.title || item.content); + })() + `); + normalized = (rows ?? []).slice(0, limit).map((row, index) => { + const images = (row.image_urls ?? []).map((u) => cleanText(u)).filter(Boolean); + const topicId = inferTopicIdFromImageUrls(images); + const tags = Array.from(new Set((row.tags ?? []).map((tag) => cleanText(tag)).filter(Boolean))); + const interactions = buildScysInteractions(undefined, undefined, undefined, row.interactions ?? ''); + const summary = polishScysText(stripScysRichText(row.content ?? '')); + const url = cleanText(row.link ?? '') || buildScysTopicLink('xq_topic', topicId); + const normalizedFlags = (row.flags ?? []).map((f) => polishScysText(f)).filter(Boolean); + return { + rank: index + 1, + author: polishScysText(row.author ?? ''), + time: cleanText(row.time ?? ''), + flags: normalizedFlags, + title: polishScysText(stripScysRichText(row.title ?? '')), + summary, + ai_summary: polishScysText(stripScysRichText(row.ai_summary ?? '')), + tags: tags.map((tag) => polishScysText(tag)).filter(Boolean), + interactions, + interactions_display: interactions.display, + url, + raw_url: url, + topic_id: topicId, + entity_type: topicId ? 'xq_topic' : '', + images, + image_count: images.length, + }; + }); + } + if (normalized.length === 0) { + throw new EmptyResultError('scys/opportunity', 'No opportunity cards were detected on this page'); + } + return normalized; +} +export async function extractScysActivity(page, inputUrl, opts = {}) { + const url = normalizeScysUrl(inputUrl); + const waitSeconds = Math.max(1, Number(opts.waitSeconds ?? 3)); + await gotoAndWait(page, url, waitSeconds); + await ensureScysLogin(page); + const payload = await page.evaluate(` + (async () => { + const clean = (value) => (value || '').replace(/\s+/g, ' ').trim(); + const normalizeTab = (value) => clean(value).replace(/\s*New$/i, '').trim(); + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + + const contentTabs = Array.from(document.querySelectorAll('.activity-left .container.v-no-scrollbar span, .container.v-no-scrollbar span')) + .filter((el) => clean(el.textContent || '')); + const roadmapTab = contentTabs.find((el) => clean(el.textContent || '').includes('航线图')); + if (roadmapTab && typeof roadmapTab.click === 'function') { + roadmapTab.click(); + await sleep(500); + } + + const title = clean( + document.querySelector('.activity-left .name, h1, .activity-title, .landing-title')?.textContent || + document.title || + '' + ); + const subtitle = clean( + document.querySelector('.activity-left .des, .subtitle, .sub-title, .activity-subtitle, .landing-subtitle')?.textContent || + '' + ); + + const tabGroups = Array.from( + document.querySelectorAll('.activity-left .tabs, .activity-left .container.v-no-scrollbar, .tabs') + ) + .map((group) => + Array.from(group.querySelectorAll('.tab-item, .tab, [role="tab"], .item, span')) + .map((el) => normalizeTab(el.textContent || '')) + .filter(Boolean) + ) + .filter((group) => group.length > 0); + const tabsRaw = + tabGroups.find((group) => group.some((text) => /简介|航线图|问答/.test(text))) || + tabGroups[0] || + []; + const tabs = Array.from(new Set(tabsRaw)); + + const stageEls = Array.from( + document.querySelectorAll('.activity-line-content .week-card, .activity-left .week-card, .week-card') + ); + const stages = stageEls.map((stage) => { + const phaseTitle = clean(stage.querySelector('.title-name, .stage-name')?.textContent || ''); + const stageTitleRaw = clean(stage.querySelector('.title-week .text, .title-week, .week-title, .stage-title')?.textContent || ''); + const duration = clean( + stage.querySelector('.title-week .highlightInActivity, .duration, .time, .date-range, .stage-duration')?.textContent || '' + ); + const stageTitle = clean( + [phaseTitle, stageTitleRaw.replace(duration, '').trim()] + .map((v) => clean(v)) + .filter(Boolean) + .join(' ') + ); + const tasks = Array.from(stage.querySelectorAll('.card .row, .row')) + .map((row) => { + const key = clean(row.querySelector('.key')?.textContent || ''); + const text = clean(row.querySelector('.card-title')?.textContent || row.textContent || ''); + if (!text) return ''; + if (key && !text.startsWith(key)) return key + '. ' + text; + return text; + }) + .filter(Boolean); + return { title: stageTitle, duration, tasks }; + }).filter((stage) => stage.title || stage.tasks.length > 0); + + return { + title, + subtitle, + tabs, + stages, + url: location.href, + }; + })() + `); + if (!payload) { + throw new EmptyResultError('scys/activity', 'Failed to extract activity page content'); + } + if (!payload.title && (!payload.stages || payload.stages.length === 0)) { + throw new EmptyResultError('scys/activity', 'No activity title or stages were detected'); + } + return { + title: polishScysText(payload.title), + subtitle: polishScysText(payload.subtitle), + tabs: (payload.tabs ?? []).map((tab) => polishScysText(tab)).filter(Boolean), + stages: (payload.stages ?? []).map((stage) => ({ + title: polishScysText(stage.title), + duration: polishScysText(stage.duration), + tasks: (stage.tasks ?? []).map((task) => polishScysText(task)).filter(Boolean), + })), + url: normalizeScysUrl(payload.url || url), + }; +} diff --git a/clis/scys/extractors.toc.test.js b/clis/scys/extractors.toc.test.js new file mode 100644 index 000000000..f905159bd --- /dev/null +++ b/clis/scys/extractors.toc.test.js @@ -0,0 +1,75 @@ +import { describe, expect, it } from 'vitest'; +import { extractScysToc } from './extractors.js'; +function createScysTocPageMock(loginState, tocRows) { + return { + goto: async () => { }, + wait: async () => { }, + evaluate: async (js) => { + if (js.includes('const text = (document.body?.innerText ||') && js.includes('hasContentSignals')) { + return loginState ?? { + strongLoginText: false, + genericLoginText: false, + loginByDom: false, + hasContentSignals: true, + routeLooksLikeLogin: false, + }; + } + if (js.includes('sectionTitleEl.click') || js.includes('sectionTitleEl.dispatchEvent')) { + return [ + { entry_type: 'section', section: '预备篇', group: '预备篇', chapter_id: '', chapter_title: '预备篇', status: '', is_current: false }, + { entry_type: 'chapter', section: '预备篇', group: '图文', chapter_id: '4137', chapter_title: '课程前言', status: '737人学过', is_current: false }, + { entry_type: 'chapter', section: '预备篇', group: '图文', chapter_id: '4038', chapter_title: '课程目标', status: '508人学过', is_current: true }, + { entry_type: 'section', section: '基础篇', group: '基础篇', chapter_id: '', chapter_title: '基础篇', status: '', is_current: false }, + { entry_type: 'chapter', section: '基础篇', group: '一、玩起来！ 通过 AI，10 分钟发布你的第一款网站产品！', chapter_id: '4039', chapter_title: '视频', status: '624人学过', is_current: false }, + { entry_type: 'chapter', section: '基础篇', group: '一、玩起来！ 通过 AI，10 分钟发布你的第一款网站产品！', chapter_id: '4040', chapter_title: '图文', status: '674人学过', is_current: false }, + ]; + } + return tocRows ?? [ + { entry_type: 'section', section: '预备篇', group: '预备篇', chapter_id: '', chapter_title: '预备篇', status: '', is_current: false }, + { entry_type: 'chapter', section: '预备篇', group: '图文', chapter_id: '4137', chapter_title: '课程前言', status: '737人学过', is_current: false }, + { entry_type: 'chapter', section: '预备篇', group: '图文', chapter_id: '4038', chapter_title: '课程目标', status: '508人学过', is_current: true }, + { entry_type: 'section', section: '基础篇', group: '基础篇', chapter_id: '', chapter_title: '基础篇', status: '', is_current: false }, + ]; + }, + getCookies: async () => [], + snapshot: async () => null, + click: async () => { }, + typeText: async () => { }, + pressKey: async () => { }, + scrollTo: async () => null, + getFormState: async () => null, + tabs: async () => [], + closeTab: async () => { }, + newTab: async () => { }, + selectTab: async () => { }, + networkRequests: async () => [], + consoleMessages: async () => [], + scroll: async () => { }, + autoScroll: async () => { }, + installInterceptor: async () => { }, + getInterceptedRequests: async () => [], + waitForCapture: async () => { }, + screenshot: async () => '', + getCurrentUrl: async () => 'https://scys.com/course/detail/92', + }; +} +describe('extractScysToc', () => { + it('expands collapsed sections to recover deterministic chapter ids', async () => { + const page = createScysTocPageMock(); + const rows = await extractScysToc(page, '92', { waitSeconds: 1 }); + expect(rows.some((row) => row.chapter_id === '4039')).toBe(true); + expect(rows.some((row) => row.chapter_id === '4040')).toBe(true); + expect(rows.find((row) => row.chapter_id === '4039')?.group).toBe('一、玩起来！ 通过 AI，10 分钟发布你的第一款网站产品！'); + }); + it('treats login CTA walls as auth failures instead of outdated adapter errors', async () => { + const page = createScysTocPageMock({ + strongLoginText: false, + genericLoginText: false, + loginByDom: false, + hasContentSignals: false, + routeLooksLikeLogin: false, + loginCtaText: true, + }, []); + await expect(extractScysToc(page, '92', { waitSeconds: 1 })).rejects.toThrow('SCYS content requires a logged-in browser session'); + }); +}); diff --git a/clis/scys/feed.js b/clis/scys/feed.js new file mode 100644 index 000000000..e8327a3a8 --- /dev/null +++ b/clis/scys/feed.js @@ -0,0 +1,23 @@ +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { buildScysHomeEssenceUrl } from './common.js'; +import { extractScysFeed } from './extractors.js'; +cli({ + site: 'scys', + name: 'feed', + description: 'Extract SCYS feed cards (home essence or profile posts)', + domain: 'scys.com', + strategy: Strategy.COOKIE, + navigateBefore: false, + args: [ + { name: 'url', positional: true, default: buildScysHomeEssenceUrl(), help: 'Feed URL (default: home essence feed)' }, + { name: 'limit', type: 'int', default: 20, help: 'Max number of cards' }, + { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, + ], + columns: ['rank', 'author', 'time', 'flags', 'title', 'summary', 'tags', 'interactions_display', 'image_count', 'url'], + func: async (page, kwargs) => { + return extractScysFeed(page, String(kwargs.url ?? buildScysHomeEssenceUrl()), { + waitSeconds: Number(kwargs.wait ?? 3), + limit: Number(kwargs.limit ?? 20), + }); + }, +}); diff --git a/clis/scys/opportunity-utils.js b/clis/scys/opportunity-utils.js new file mode 100644 index 000000000..800742738 --- /dev/null +++ b/clis/scys/opportunity-utils.js @@ -0,0 +1,88 @@ +import { ArgumentError } from '@jackwener/opencli/errors'; +const FLAG_SET = new Set(['中标', '热门', '信息差', '新玩法', '市场洞察', '风向标']); +export function normalizeOpportunityTab(input) { + const raw = String(input ?? '').trim().toLowerCase(); + if (!raw || raw === 'all' || raw === '全部') + return { key: 'all', label: '全部' }; + if (raw === 'hot' || raw === '热门') + return { key: 'hot', label: '热门' }; + if (raw === 'winning' || raw === 'win' || raw === 'zhongbiao' || raw === '中标') { + return { key: 'winning', label: '中标' }; + } + throw new ArgumentError(`Unsupported tab: ${String(input)}`, 'Use one of: all/全部, hot/热门, winning/中标'); +} +export function splitOpportunityFlagsAndTags(values) { + const cleaned = values.map((v) => String(v || '').trim()).filter(Boolean); + const flags = Array.from(new Set(cleaned.filter((v) => FLAG_SET.has(v)))); + const tags = Array.from(new Set(cleaned.filter((v) => !FLAG_SET.has(v)))); + return { flags, tags }; +} +export function buildScysTopicLink(entityType, entityId) { + const type = String(entityType ?? '').trim(); + const id = String(entityId ?? '').trim(); + if (!type || !id) + return ''; + return `https://scys.com/articleDetail/${encodeURIComponent(type)}/${encodeURIComponent(id)}`; +} +export function inferTopicIdFromImageUrls(urls) { + if (!Array.isArray(urls)) + return ''; + for (const raw of urls) { + const text = String(raw || ''); + const m = text.match(/\/images\/(\d{8,})\//); + if (m?.[1]) + return m[1]; + } + return ''; +} +export function parseAiSummaryText(input) { + return stripScysRichText(input); +} +function decodeUriSafe(value) { + try { + return decodeURIComponent(value); + } + catch { + return value; + } +} +/** + * SCYS 富文本常见格式: + * - + * - 常规 HTML 标签

//... + */ +export function stripScysRichText(input) { + const raw = String(input ?? ''); + if (!raw) + return ''; + const withHashtagText = raw.replace(/]*\btitle="([^"]+)"[^>]*\/?>/gi, (_full, title) => ` ${decodeUriSafe(title)} `); + return withHashtagText + .replace(/]*\/?>/gi, ' ') + .replace(/<[^>]*>/g, ' ') + .replace(/ /gi, ' ') + .replace(/&/gi, '&') + .replace(/\s+/g, ' ') + .trim(); +} +export function formatScysRelativeTime(tsSeconds, nowMs = Date.now()) { + const ts = Number(tsSeconds); + if (!Number.isFinite(ts) || ts <= 0) + return ''; + const targetMs = ts * 1000; + const deltaSec = Math.floor((nowMs - targetMs) / 1000); + if (deltaSec < 0) + return ''; + if (deltaSec < 60) + return '刚刚'; + if (deltaSec < 3600) + return `${Math.max(1, Math.floor(deltaSec / 60))}分钟前`; + if (deltaSec < 86400) + return `${Math.max(1, Math.floor(deltaSec / 3600))}小时前`; + if (deltaSec < 86400 * 30) + return `${Math.max(1, Math.floor(deltaSec / 86400))}天前`; + const d = new Date(targetMs); + const y = d.getFullYear(); + const m = String(d.getMonth() + 1).padStart(2, '0'); + const day = String(d.getDate()).padStart(2, '0'); + return `${y}-${m}-${day}`; +} diff --git a/clis/scys/opportunity-utils.test.js b/clis/scys/opportunity-utils.test.js new file mode 100644 index 000000000..7dcc212e2 --- /dev/null +++ b/clis/scys/opportunity-utils.test.js @@ -0,0 +1,60 @@ +import { describe, expect, it } from 'vitest'; +import { buildScysTopicLink, formatScysRelativeTime, inferTopicIdFromImageUrls, normalizeOpportunityTab, parseAiSummaryText, stripScysRichText, splitOpportunityFlagsAndTags, } from './opportunity-utils.js'; +describe('normalizeOpportunityTab', () => { + it('maps all aliases', () => { + expect(normalizeOpportunityTab('')).toEqual({ key: 'all', label: '全部' }); + expect(normalizeOpportunityTab('all')).toEqual({ key: 'all', label: '全部' }); + expect(normalizeOpportunityTab('全部')).toEqual({ key: 'all', label: '全部' }); + }); + it('maps hot aliases', () => { + expect(normalizeOpportunityTab('hot')).toEqual({ key: 'hot', label: '热门' }); + expect(normalizeOpportunityTab('热门')).toEqual({ key: 'hot', label: '热门' }); + }); + it('maps winning aliases', () => { + expect(normalizeOpportunityTab('winning')).toEqual({ key: 'winning', label: '中标' }); + expect(normalizeOpportunityTab('win')).toEqual({ key: 'winning', label: '中标' }); + expect(normalizeOpportunityTab('中标')).toEqual({ key: 'winning', label: '中标' }); + }); +}); +describe('splitOpportunityFlagsAndTags', () => { + it('splits system flags and custom tags', () => { + expect(splitOpportunityFlagsAndTags(['中标', '市场洞察', '垂直小号', '00后/大学生'])).toEqual({ + flags: ['中标', '市场洞察'], + tags: ['垂直小号', '00后/大学生'], + }); + }); +}); +describe('buildScysTopicLink', () => { + it('builds canonical article detail link', () => { + expect(buildScysTopicLink('xq_topic', '45811252552251118')).toBe('https://scys.com/articleDetail/xq_topic/45811252552251118'); + }); +}); +describe('inferTopicIdFromImageUrls', () => { + it('extracts topic id from signed oss image urls', () => { + expect(inferTopicIdFromImageUrls([ + 'https://sphere-sh.oss-cn-shanghai.aliyuncs.com/private/xq/images/45811252552251118/Fmrm4.jpg?Expires=1', + ])).toBe('45811252552251118'); + }); +}); +describe('parseAiSummaryText', () => { + it('strips html tags', () => { + expect(parseAiSummaryText('

细分需求：测试

')).toBe('细分需求： 测试'); + }); +}); +describe('stripScysRichText', () => { + it('converts SCYS hashtag marker and strips tags', () => { + expect(stripScysRichText('蹭热度：

备考

')).toBe('蹭热度： #全国计算机考试# 备考'); + }); +}); +describe('formatScysRelativeTime', () => { + const now = new Date('2026-03-28T12:00:00Z').getTime(); + it('formats recent intervals', () => { + expect(formatScysRelativeTime(Math.floor((now - 30_000) / 1000), now)).toBe('刚刚'); + expect(formatScysRelativeTime(Math.floor((now - 10 * 60_000) / 1000), now)).toBe('10分钟前'); + expect(formatScysRelativeTime(Math.floor((now - 3 * 3600_000) / 1000), now)).toBe('3小时前'); + expect(formatScysRelativeTime(Math.floor((now - 5 * 86400_000) / 1000), now)).toBe('5天前'); + }); + it('falls back to absolute date for old timestamps', () => { + expect(formatScysRelativeTime(Math.floor((now - 40 * 86400_000) / 1000), now)).toBe('2026-02-16'); + }); +}); diff --git a/clis/scys/opportunity.js b/clis/scys/opportunity.js new file mode 100644 index 000000000..0115955a3 --- /dev/null +++ b/clis/scys/opportunity.js @@ -0,0 +1,67 @@ +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { buildScysOpportunityUrl } from './common.js'; +import { extractScysOpportunity } from './extractors.js'; +import { normalizeOpportunityTab } from './opportunity-utils.js'; +import { formatCookieHeader } from '@jackwener/opencli/download'; +import { downloadMedia } from '@jackwener/opencli/download/media-download'; +import * as path from 'node:path'; +cli({ + site: 'scys', + name: 'opportunity', + description: 'Extract SCYS opportunity feed with flags, summaries, and tags', + domain: 'scys.com', + strategy: Strategy.COOKIE, + navigateBefore: false, + args: [ + { name: 'url', positional: true, default: buildScysOpportunityUrl(), help: 'Opportunity URL' }, + { name: 'tab', default: 'all', help: 'Filter tab: all/全部, hot/热门, winning/中标' }, + { name: 'limit', type: 'int', default: 20, help: 'Max number of cards' }, + { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, + { name: 'download-images', type: 'boolean', default: false, help: 'Download post images to local directory' }, + { name: 'output', default: './scys-opportunity-downloads', help: 'Image output directory' }, + ], + columns: ['rank', 'author', 'time', 'flags', 'title', 'summary', 'ai_summary', 'tags', 'interactions_display', 'image_count', 'url', 'image_dir'], + func: async (page, kwargs) => { + const tab = normalizeOpportunityTab(kwargs.tab); + const rows = await extractScysOpportunity(page, String(kwargs.url ?? buildScysOpportunityUrl()), { + waitSeconds: Number(kwargs.wait ?? 3), + limit: Number(kwargs.limit ?? 20), + tab: tab.label, + }); + const downloadImages = kwargs['download-images'] === true || String(kwargs['download-images']) === 'true'; + if (!downloadImages) + return rows; + const output = String(kwargs.output ?? './scys-opportunity-downloads'); + const cookies = formatCookieHeader(await page.getCookies({ domain: 'scys.com' })); + const withDownloads = []; + for (const row of rows) { + const imageUrls = Array.isArray(row.images) ? row.images.filter(Boolean) : []; + if (imageUrls.length === 0) { + withDownloads.push({ ...row, image_count: 0, image_dir: '' }); + continue; + } + const topicId = row.topic_id || `opportunity_${row.rank}`; + const subdir = path.join(tab.label, topicId); + const media = imageUrls.map((url, idx) => ({ + type: 'image', + url, + filename: `${topicId}_${idx + 1}.jpg`, + })); + const results = await downloadMedia(media, { + output, + subdir, + cookies, + filenamePrefix: topicId, + timeout: 60_000, + verbose: false, + }); + const successCount = results.filter((r) => r.status === 'success').length; + withDownloads.push({ + ...row, + image_count: successCount, + image_dir: path.join(output, subdir), + }); + } + return withDownloads; + }, +}); diff --git a/clis/scys/read.js b/clis/scys/read.js new file mode 100644 index 000000000..595c4b75d --- /dev/null +++ b/clis/scys/read.js @@ -0,0 +1,57 @@ +import { ArgumentError } from '@jackwener/opencli/errors'; +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { downloadScysCourseImages } from './course-download.js'; +import { detectScysPageType, inferScysReadUrl } from './common.js'; +import { extractScysActivity, extractScysArticle, extractScysCourse, extractScysCourseAll, extractScysFeed, extractScysOpportunity, } from './extractors.js'; +cli({ + site: 'scys', + name: 'read', + description: 'Read a SCYS page with automatic page-type routing', + domain: 'scys.com', + strategy: Strategy.COOKIE, + navigateBefore: false, + args: [ + { name: 'url', required: true, positional: true, help: 'Any scys.com URL' }, + { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' }, + { name: 'limit', type: 'int', default: 20, help: 'Max rows for list pages' }, + { name: 'max-length', type: 'int', default: 4000, help: 'Max content length for long text fields' }, + { name: 'all', type: 'boolean', default: false, help: 'For course pages, export all deterministic chapter ids from TOC' }, + { name: 'download-images', type: 'boolean', default: false, help: 'For course pages, download page images to local directory' }, + { name: 'output', default: './scys-course-downloads', help: 'Image output directory for course pages' }, + ], + func: async (page, kwargs) => { + const url = inferScysReadUrl(String(kwargs.url)); + const waitSeconds = Math.max(1, Number(kwargs.wait ?? 3)); + const limit = Math.max(1, Number(kwargs.limit ?? 20)); + const maxLength = Math.max(300, Number(kwargs['max-length'] ?? 4000)); + const all = kwargs.all === true || String(kwargs.all) === 'true'; + const downloadImages = kwargs['download-images'] === true || String(kwargs['download-images']) === 'true'; + const pageType = detectScysPageType(url); + if (pageType === 'course') { + const extracted = all + ? await extractScysCourseAll(page, url, { waitSeconds, maxLength }) + : await extractScysCourse(page, url, { waitSeconds, maxLength }); + const data = downloadImages + ? await downloadScysCourseImages(page, extracted, String(kwargs.output ?? './scys-course-downloads')) + : extracted; + return { page_type: pageType, data }; + } + if (pageType === 'feed') { + const data = await extractScysFeed(page, url, { waitSeconds, limit, maxLength }); + return { page_type: pageType, data }; + } + if (pageType === 'opportunity') { + const data = await extractScysOpportunity(page, url, { waitSeconds, limit, maxLength }); + return { page_type: pageType, data }; + } + if (pageType === 'activity') { + const data = await extractScysActivity(page, url, { waitSeconds, maxLength }); + return { page_type: pageType, data }; + } + if (pageType === 'article') { + const data = await extractScysArticle(page, url, { waitSeconds, maxLength }); + return { page_type: pageType, data }; + } + throw new ArgumentError(`Unsupported SCYS page for scys/read: ${url}`, 'Supported patterns: /course/detail/:id, /?filter=essence, /personal/:id?tab=posts, /opportunity, /activity/landing/:id, /articleDetail/:entityType/:topicId'); + }, +}); diff --git a/clis/scys/toc.js b/clis/scys/toc.js new file mode 100644 index 000000000..2a189bf49 --- /dev/null +++ b/clis/scys/toc.js @@ -0,0 +1,20 @@ +import { cli, Strategy } from '@jackwener/opencli/registry'; +import { extractScysToc } from './extractors.js'; +cli({ + site: 'scys', + name: 'toc', + description: 'Extract chapter table of contents from a SCYS course', + domain: 'scys.com', + strategy: Strategy.COOKIE, + navigateBefore: false, + args: [ + { name: 'course', required: true, positional: true, help: 'Course URL or numeric course id' }, + { name: 'wait', type: 'int', default: 2, help: 'Seconds to wait after page load' }, + ], + columns: ['rank', 'entry_type', 'section', 'group', 'chapter_id', 'chapter_title', 'status', 'is_current'], + func: async (page, kwargs) => { + return extractScysToc(page, String(kwargs.course), { + waitSeconds: Number(kwargs.wait ?? 2), + }); + }, +}); diff --git a/docs/adapters/browser/scys.md b/docs/adapters/browser/scys.md new file mode 100644 index 000000000..ecb787f3e --- /dev/null +++ b/docs/adapters/browser/scys.md @@ -0,0 +1,55 @@ +# SCYS + +**Mode**: 🔐 Browser · **Domain**: `scys.com` + +## Commands + +| Command | Description | +|---------|-------------| +| `opencli scys course ` | Read SCYS course detail content and chapter context | +| `opencli scys toc ` | Extract the chapter outline from a SCYS course detail page | +| `opencli scys read ` | Auto-detect the SCYS page type and dispatch to the right extractor | +| `opencli scys feed [url]` | Read SCYS 精华 feed cards with summaries and interactions | +| `opencli scys opportunity [url]` | Read SCYS opportunity cards with AI summaries and tags | +| `opencli scys activity ` | Read a SCYS activity landing page timeline | +| `opencli scys article ` | Read a SCYS article detail page | + +## Usage Examples + +```bash +# Read one course chapter +opencli scys course "https://scys.com/course/detail/92" + +# Export all deterministic chapters from the TOC +opencli scys course "https://scys.com/course/detail/92" --all -f json + +# Download course images while exporting all chapters +opencli scys course "https://scys.com/course/detail/92" --all --download-images --output ./scys-course-downloads -f json + +# Extract just the table of contents +opencli scys toc "https://scys.com/course/detail/92" -f json + +# Read the essence feed +opencli scys feed "https://scys.com/?filter=essence" -f json + +# Read the opportunity page +opencli scys opportunity "https://scys.com/opportunity" -f json + +# Read an article detail page +opencli scys article "https://scys.com/articleDetail/xq_topic/55188458224514554" -f json + +# Let read auto-dispatch based on URL +opencli scys read "https://scys.com/articleDetail/xq_topic/55188458224514554" -f json +``` + +## Prerequisites + +- Chrome logged into `scys.com` +- [Browser Bridge extension](/guide/browser-bridge) installed + +## Notes + +- `read` dispatches by URL shape and supports course, feed, opportunity, activity, and article pages +- `course --all` expands deterministic chapter IDs from the page TOC and exports each chapter as a separate row +- `course --download-images` stores course images under the output directory and adds `image_dir`/`image_count` fields to the result +- `article`, `feed`, and `opportunity` normalize output to stable JSON field names such as `url`, `raw_url`, `summary`, `content`, and structured `interactions` diff --git a/docs/adapters/index.md b/docs/adapters/index.md index 080f43251..08f857ef9 100644 --- a/docs/adapters/index.md +++ b/docs/adapters/index.md @@ -14,6 +14,7 @@ Run `opencli list` for the live registry. | **[zhihu](./browser/zhihu.md)** | `hot` `search` `question` `download` `follow` `like` `favorite` `comment` `answer` | 🔐 Browser | | **[xiaohongshu](./browser/xiaohongshu.md)** | `search` `notifications` `feed` `user` `note` `comments` `download` `publish` `creator-notes` `creator-note-detail` `creator-notes-summary` `creator-profile` `creator-stats` | 🔐 Browser | | **[xiaoe](./browser/xiaoe.md)** | `courses` `detail` `catalog` `play-url` `content` | 🔐 Browser | +| **[scys](./browser/scys.md)** | `course` `toc` `read` `feed` `opportunity` `activity` `article` | 🔐 Browser | | **[xueqiu](./browser/xueqiu.md)** | `feed` `hot-stock` `hot` `search` `stock` `comments` `watchlist` `earnings-date` `fund-holdings` `fund-snapshot` | 🔐 Browser | | **[youtube](./browser/youtube.md)** | `search` `video` `transcript` | 🔐 Browser | | **[v2ex](./browser/v2ex.md)** | `hot` `latest` `topic` `node` `user` `member` `replies` `nodes` `daily` `me` `notifications` | 🌐 / 🔐 | diff --git a/docs/developer/scys-schema-guidelines.md b/docs/developer/scys-schema-guidelines.md new file mode 100644 index 000000000..83d67f9ed --- /dev/null +++ b/docs/developer/scys-schema-guidelines.md @@ -0,0 +1,59 @@ +# SCYS Schema Guidelines + +This document defines JSON output conventions for `opencli scys` commands to keep `feed`, `opportunity`, `article`, and `read` consistent for pipeline consumers. + +## 1) Canonical Naming + +Use these canonical field names for the same semantics: + +- `url`: canonical page/detail URL +- `raw_url`: original URL used to fetch data (before normalization/fallback) +- `images`: image URL list (`string[]`) +- `summary`: list/card preview text +- `content`: full detail text (detail pages) + +Deprecated aliases (`link`, `raw_link`, `image_urls`, `preview`) are no longer part of canonical SCYS JSON output. New code must not reintroduce them. + +## 2) Canonical Types + +For all SCYS JSON outputs: + +- `tags`: always `string[]` +- `flags`: always `string[]` +- `images`: always `string[]` +- `external_links`: always `string[]` +- `source_links`: always `string[]` +- `interactions`: always object: + +```json +{ + "likes": 16, + "comments": 0, + "favorites": 4, + "display": "点赞16 评论0 收藏4" +} +``` + +## 3) Structured vs Display Fields + +Keep machine fields and display fields separate: + +- Machine fields: `interactions.likes/comments/favorites`, `tags`, `flags`, `images` +- Display field: `interactions.display` +- Table-oriented helper fields (for CLI table only), e.g. `interactions_display`, are allowed but should mirror structured fields exactly. + +## 4) List vs Detail Semantics + +- List commands (`feed`, `opportunity`) should prioritize `summary`. +- Detail command (`article`) should prioritize `content`. +- Do not expose duplicated legacy aliases in normal command output. + +## 5) Change Checklist + +When adding or changing SCYS commands: + +1. Reuse canonical field names and types from this document. +2. Do not add new semantic duplicates. +3. Keep `scys read` routing output schema aligned with direct command output. +4. Run `npm run typecheck` and adapter tests before commit. +5. If compatibility aliases are changed or removed, document it in PR notes explicitly.