对外py编写
This commit is contained in:
@@ -104,7 +104,7 @@ class XxqgCrawler(BaseCrawler):
|
||||
'sec-ch-ua-platform': '"Windows"'
|
||||
}
|
||||
),
|
||||
"zongheshiping": UrlConfig(
|
||||
"zonghexinwen": UrlConfig(
|
||||
url="https://www.xuexi.cn/7097477a9643eacffe4cc101e4906fdb/9a3668c13f6e303932b5e0e100fc248b.html",
|
||||
method="GET",
|
||||
params={
|
||||
@@ -359,7 +359,7 @@ class XxqgCrawler(BaseCrawler):
|
||||
"type": "text",
|
||||
"content": text_content
|
||||
})
|
||||
logger.debug(f"提取文字: {text_content[:50]}...")
|
||||
# logger.debug(f"提取文字: {text_content[:50]}...")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"处理内容元素失败: {str(e)}")
|
||||
@@ -369,7 +369,7 @@ class XxqgCrawler(BaseCrawler):
|
||||
|
||||
if is_page():
|
||||
pass
|
||||
logger.info(f"解析文章详情完成: {news_item.model_dump()}")
|
||||
logger.info(f"解析文章详情完成: {news_item.url}")
|
||||
return news_item
|
||||
|
||||
|
||||
@@ -577,7 +577,7 @@ class XxqgCrawler(BaseCrawler):
|
||||
# 解析meta请求响应获取channelId
|
||||
try:
|
||||
meta_data = json.loads(target_request.response.body)
|
||||
logger.info(f"Meta响应数据: {meta_data}")
|
||||
# logger.info(f"Meta响应数据: {meta_data}")
|
||||
|
||||
# 提取channelId
|
||||
if 'pageData' in meta_data and 'channel' in meta_data['pageData']:
|
||||
|
||||
Reference in New Issue
Block a user