From e8dda04730dedf381bbfd3d01cec5e1a3521b179 Mon Sep 17 00:00:00 2001 From: Kloping <3474006766@qq.com> Date: Wed, 10 Jun 2026 14:53:31 +0800 Subject: [PATCH] =?UTF-8?q?fix(parse):=20=E5=A2=9E=E5=BC=BAURL=E8=A7=A3?= =?UTF-8?q?=E6=9E=90=E9=94=99=E8=AF=AF=E5=A4=84=E7=90=86=E4=B8=8E=E9=87=8D?= =?UTF-8?q?=E5=AE=9A=E5=90=91=E8=B0=83=E8=AF=95;=20=E5=BF=AB=E6=89=8B?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加对空输入和无效重定向链接的错误校验，修复视频ID提取来源，优化重定向跟踪的日志输出，并补充快手域名映射。 --- configs/business_config.json | 25 +++++++++++++------------ src/api/parse.py | 22 ++++++++++++++++------ utils/web_fetcher.py | 30 +++++++++++++++++++++--------- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/configs/business_config.json b/configs/business_config.json index 60ac44b..8ddc2c5 100755 --- a/configs/business_config.json +++ b/configs/business_config.json @@ -2,12 +2,13 @@ "DOMAIN_TO_NAME": { "www.xiaohongshu.com": "小红书", "www.douyin.com": "抖音", - "www.iesdouyin.com": "抖音", - "www.kuaishou.com": "快手", - "v.kuaishou.com": "快手", - "m.kuaishou.com": "快手", - "v.m.chenzhongtech.com": "快手", - "m.chenzhongtech.com": "快手", + "www.iesdouyin.com": "抖音", + "www.kuaishou.com": "快手", + "v.kuaishou.com": "快手", + "m.kuaishou.com": "快手", + "c.kuaishou.com": "快手", + "v.m.chenzhongtech.com": "快手", + "m.chenzhongtech.com": "快手", "www.bilibili.com": "哔哩哔哩", "haokan.baidu.com": "好看视频", "haokan.hao123.com": "好看视频", @@ -17,11 +18,11 @@ "h5.pipigx.com": "皮皮搞笑", "m.acfun.cn": "AcFun", "www.acfun.cn": "AcFun", - "www.tiktok.com": "TikTok", - "vt.tiktok.com": "TikTok", - "vm.tiktok.com": "TikTok", - "weibo.com": "微博", - "m.weibo.cn": "微博", + "www.tiktok.com": "TikTok", + "vt.tiktok.com": "TikTok", + "vm.tiktok.com": "TikTok", + "weibo.com": "微博", + "m.weibo.cn": "微博", "www.ixigua.com": "西瓜视频", "v.ixigua.com": "西瓜视频", "www.youtube.com": "YouTube", @@ -80,4 +81,4 @@ "Mozilla/5.0 (Android 10; Tablet; rv:90.0) Gecko/90.0 Firefox/90.0", "Mozilla/5.0 (iPod; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1" ] -} +} diff --git a/src/api/parse.py b/src/api/parse.py index 1629ee3..f11392a 100644 --- a/src/api/parse.py +++ b/src/api/parse.py @@ -12,14 +12,24 @@ def parse(): try: data = request.json - text = data.get('text') + text = data.get('text', '') - # 1. 解析基础信息 - redirect_url = WebFetcher.fetch_redirect_url(UrlParser.get_url(text)) - platform = DOMAIN_TO_NAME.get(UrlParser.get_domain(redirect_url)) - real_url = UrlParser.extract_video_address(redirect_url) + # 1. 从用户输入中提取URL + raw_url = UrlParser.get_url(text) + if not raw_url: + logger.error(f'No valid URL found in text: {text}') + return make_response(400, '未检测到有效链接', None, False), 400 + + # 2. 跟踪重定向，获取真实URL（fetch_redirect_url 内部已调用 extract_video_address） + real_url = WebFetcher.fetch_redirect_url(raw_url) + if not real_url: + logger.error(f'Failed to resolve redirect for: {raw_url}') + return make_response(400, '无法解析该链接，请检查链接是否有效', None, False), 400 + logger.debug(f'real_url {real_url}') + # 3. 识别平台 + platform = DOMAIN_TO_NAME.get(UrlParser.get_domain(real_url)) if not platform: logger.error(f'This link is not supported for extraction: {real_url}') return make_response(400, '该链接尚未支持提取', None, False), 400 @@ -46,7 +56,7 @@ def parse(): # 4. 统一转换 HTTPS data_dict = { - 'video_id': UrlParser.get_video_id(redirect_url), + 'video_id': UrlParser.get_video_id(real_url), 'platform': platform, 'title': content_data['title'], 'video_url': UrlParser.convert_to_https(content_data['video_url']), diff --git a/utils/web_fetcher.py b/utils/web_fetcher.py index ad7230d..9b2b464 100755 --- a/utils/web_fetcher.py +++ b/utils/web_fetcher.py @@ -14,33 +14,45 @@ class WebFetcher: @staticmethod def fetch_redirect_url(url, max_redirects=5): + logger.debug(f"fetch_redirect_url start, url={url}") try: current_url = url - for _ in range(max_redirects): - # 发送请求，禁止重定向 - resp = requests.get(current_url, headers=WebFetcher.headers, allow_redirects=False, timeout=5) + for step in range(max_redirects): + logger.debug(f" [step {step}] requesting: {current_url}") + resp = requests.get(current_url, headers=WebFetcher.headers, allow_redirects=False, timeout=10) + logger.debug(f" [step {step}] status={resp.status_code}, headers={dict(resp.headers)}") resp.raise_for_status() - # 获取重定向后的URL + redirect_url = resp.headers.get("location") if redirect_url: + logger.debug(f" [step {step}] location header: {redirect_url}") redirect_url = urljoin(current_url, redirect_url) + logger.debug(f" [step {step}] after urljoin: {redirect_url}") current_url = redirect_url - if DOMAIN_TO_NAME.get(UrlParser.get_domain(current_url)): + domain = UrlParser.get_domain(current_url) + logger.debug(f" [step {step}] domain={domain}, known={bool(DOMAIN_TO_NAME.get(domain))}") + if DOMAIN_TO_NAME.get(domain): break else: + logger.debug(f" [step {step}] no location header, stopping redirect follow") break else: + logger.warning(f"redirect chain exhausted after {max_redirects} hops, last url: {current_url}") return None - if not DOMAIN_TO_NAME.get(UrlParser.get_domain(current_url)): + final_domain = UrlParser.get_domain(current_url) + if not DOMAIN_TO_NAME.get(final_domain): + logger.warning(f"final domain not recognized: {final_domain}, url={current_url}") return None - return UrlParser.extract_video_address(current_url) + result = UrlParser.extract_video_address(current_url) + logger.debug(f"fetch_redirect_url success: {result}") + return result except requests.RequestException as e: - logger.error(f"Failed to get the page: {e}") + logger.error(f"Request failed for url={current_url}: {type(e).__name__}: {e}") return None except Exception as e: - logger.error(f"An error occurred: {e}") + logger.exception(f"Unexpected error for url={current_url}") return None