Skip to content

fix(parse): 增强URL解析错误处理与重定向调试; 快手解析修复 #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Kloping wants to merge 1 commit into from
+50 −27
Closed

fix(parse): 增强URL解析错误处理与重定向调试; 快手解析修复 #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 13 additions & 12 deletions configs/business_config.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,13 @@
"DOMAIN_TO_NAME": {
"www.xiaohongshu.com": "小红书",
"www.douyin.com": "抖音",
"www.iesdouyin.com": "抖音",
"www.kuaishou.com": "快手",
"v.kuaishou.com": "快手",
"m.kuaishou.com": "快手",
"v.m.chenzhongtech.com": "快手",
"m.chenzhongtech.com": "快手",
"www.iesdouyin.com": "抖音",
"www.kuaishou.com": "快手",
"v.kuaishou.com": "快手",
"m.kuaishou.com": "快手",
"c.kuaishou.com": "快手",
"v.m.chenzhongtech.com": "快手",
"m.chenzhongtech.com": "快手",
"www.bilibili.com": "哔哩哔哩",
"haokan.baidu.com": "好看视频",
"haokan.hao123.com": "好看视频",
Expand All @@ -17,11 +18,11 @@
"h5.pipigx.com": "皮皮搞笑",
"m.acfun.cn": "AcFun",
"www.acfun.cn": "AcFun",
"www.tiktok.com": "TikTok",
"vt.tiktok.com": "TikTok",
"vm.tiktok.com": "TikTok",
"weibo.com": "微博",
"m.weibo.cn": "微博",
"www.tiktok.com": "TikTok",
"vt.tiktok.com": "TikTok",
"vm.tiktok.com": "TikTok",
"weibo.com": "微博",
"m.weibo.cn": "微博",
"www.ixigua.com": "西瓜视频",
"v.ixigua.com": "西瓜视频",
"www.youtube.com": "YouTube",
Expand Down Expand Up @@ -80,4 +81,4 @@
"Mozilla/5.0 (Android 10; Tablet; rv:90.0) Gecko/90.0 Firefox/90.0",
"Mozilla/5.0 (iPod; CPU iPhone OS 14_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Mobile/15E148 Safari/604.1"
]
}
}
22 changes: 16 additions & 6 deletions src/api/parse.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,24 @@
def parse():
try:
data = request.json
text = data.get('text')
text = data.get('text', '')

# 1. 解析基础信息
redirect_url = WebFetcher.fetch_redirect_url(UrlParser.get_url(text))
platform = DOMAIN_TO_NAME.get(UrlParser.get_domain(redirect_url))
real_url = UrlParser.extract_video_address(redirect_url)
# 1. 从用户输入中提取URL
raw_url = UrlParser.get_url(text)
if not raw_url:
logger.error(f'No valid URL found in text: {text}')
return make_response(400, '未检测到有效链接', None, False), 400

# 2. 跟踪重定向，获取真实URL（fetch_redirect_url 内部已调用 extract_video_address）
real_url = WebFetcher.fetch_redirect_url(raw_url)
if not real_url:
logger.error(f'Failed to resolve redirect for: {raw_url}')
return make_response(400, '无法解析该链接，请检查链接是否有效', None, False), 400

logger.debug(f'real_url {real_url}')

# 3. 识别平台
platform = DOMAIN_TO_NAME.get(UrlParser.get_domain(real_url))
if not platform:
logger.error(f'This link is not supported for extraction: {real_url}')
return make_response(400, '该链接尚未支持提取', None, False), 400
Expand All @@ -46,7 +56,7 @@ def parse():

# 4. 统一转换 HTTPS
data_dict = {
'video_id': UrlParser.get_video_id(redirect_url),
'video_id': UrlParser.get_video_id(real_url),
'platform': platform,
'title': content_data['title'],
'video_url': UrlParser.convert_to_https(content_data['video_url']),
Expand Down
30 changes: 21 additions & 9 deletions utils/web_fetcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,33 +14,45 @@ class WebFetcher:

@staticmethod
def fetch_redirect_url(url, max_redirects=5):
logger.debug(f"fetch_redirect_url start, url={url}")
try:
current_url = url
for _ in range(max_redirects):
# 发送请求，禁止重定向
resp = requests.get(current_url, headers=WebFetcher.headers, allow_redirects=False, timeout=5)
for step in range(max_redirects):
logger.debug(f" [step {step}] requesting: {current_url}")
resp = requests.get(current_url, headers=WebFetcher.headers, allow_redirects=False, timeout=10)
logger.debug(f" [step {step}] status={resp.status_code}, headers={dict(resp.headers)}")
resp.raise_for_status()
# 获取重定向后的URL

redirect_url = resp.headers.get("location")
if redirect_url:
logger.debug(f" [step {step}] location header: {redirect_url}")
redirect_url = urljoin(current_url, redirect_url)
logger.debug(f" [step {step}] after urljoin: {redirect_url}")
current_url = redirect_url
if DOMAIN_TO_NAME.get(UrlParser.get_domain(current_url)):
domain = UrlParser.get_domain(current_url)
logger.debug(f" [step {step}] domain={domain}, known={bool(DOMAIN_TO_NAME.get(domain))}")
if DOMAIN_TO_NAME.get(domain):
break
else:
logger.debug(f" [step {step}] no location header, stopping redirect follow")
break
else:
logger.warning(f"redirect chain exhausted after {max_redirects} hops, last url: {current_url}")
return None

if not DOMAIN_TO_NAME.get(UrlParser.get_domain(current_url)):
final_domain = UrlParser.get_domain(current_url)
if not DOMAIN_TO_NAME.get(final_domain):
logger.warning(f"final domain not recognized: {final_domain}, url={current_url}")
return None

return UrlParser.extract_video_address(current_url)
result = UrlParser.extract_video_address(current_url)
logger.debug(f"fetch_redirect_url success: {result}")
return result
except requests.RequestException as e:
logger.error(f"Failed to get the page: {e}")
logger.error(f"Request failed for url={current_url}: {type(e).__name__}: {e}")
return None
except Exception as e:
logger.error(f"An error occurred: {e}")
logger.exception(f"Unexpected error for url={current_url}")
return None


Expand Down