179 lines
14 KiB
Plaintext
179 lines
14 KiB
Plaintext
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 18,
|
||
"id": "948be230",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"项目根目录: f:\\Project\\schoolNews\n",
|
||
"✓ 已启用自动重载模块功能 - 修改 .py 文件后会自动生效\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"# 自动重载模块（当文件修改后自动刷新）\n",
|
||
"%reload_ext autoreload\n",
|
||
"%autoreload 2\n",
|
||
"\n",
|
||
"import sys\n",
|
||
"import os\n",
|
||
"\n",
|
||
"# 先添加项目根目录到路径（必须在导入之前）\n",
|
||
"project_root = os.path.abspath(os.path.join(os.getcwd(), '..'))\n",
|
||
"if project_root not in sys.path:\n",
|
||
" sys.path.insert(0, project_root)\n",
|
||
"\n",
|
||
"# 然后再导入模块\n",
|
||
"from crawler.xhw.XhwCrawler import XhwCrawler\n",
|
||
"from crawler.BaseCrawler import NewsItem\n",
|
||
"from loguru import logger\n",
|
||
"import json\n",
|
||
"from pprint import pprint\n",
|
||
"\n",
|
||
"print(f\"项目根目录: {project_root}\")\n",
|
||
"print(\"✓ 已启用自动重载模块功能 - 修改 .py 文件后会自动生效\")\n"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 19,
|
||
"id": "31a8a0dd",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\u001b[32m2025-11-20 14:39:07.858\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.BaseCrawler\u001b[0m:\u001b[36m__init__\u001b[0m:\u001b[36m71\u001b[0m - \u001b[1m初始化爬虫: XhwCrawler\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 14:39:08.884\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36m_init_driver\u001b[0m:\u001b[36m20\u001b[0m - \u001b[1mChrome浏览器初始化成功\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 14:39:08.884\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36m_init_driver\u001b[0m:\u001b[36m24\u001b[0m - \u001b[1m访问主页获取初始Cookie\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 14:39:08.885\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36m_init_driver\u001b[0m:\u001b[36m25\u001b[0m - \u001b[1m准备访问URL: https://xhsz.news.cn/\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 14:39:10.309\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36m_init_driver\u001b[0m:\u001b[36m28\u001b[0m - \u001b[1m成功访问URL: https://xhsz.news.cn/\u001b[0m\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"crawler = XhwCrawler()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "e5a6e91c",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"\u001b[32m2025-11-20 13:19:51.853\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36msearch\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m请求URL: https://xhsz.news.cn/s?k=%E5%A4%A7%E5%AD%A6&action=news&page=1\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:20:15.300\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:20:20.310\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.8B0C5F90441ED5455E088CF6DF7032DE.e.84\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:20:36.428\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:20:41.434\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.D41E40A40777EF2D881878B18F35342A.e.114\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:20:57.656\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:21:02.664\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.2BA293A49BA4DA88D492D8BDC1E07365.e.157\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:21:18.808\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:21:23.814\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.DDC416596722BE8B22A5E84011EA59C3.e.198\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:22:32.631\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:22:37.642\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.B9E24DEEF281C700F90635CABAA2B108.e.230\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:22:53.636\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:22:58.643\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.EECC90A746E37A0994443791EFF7C402.e.290\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:23:15.189\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:23:20.196\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.0188441312BE753DFF48394C16A44F8F.e.330\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:23:36.050\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:23:41.057\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.F7A148D8A30D006FFCDAC45B01A2E7B5.e.374\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:23:56.819\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:24:01.826\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.5A632E0B79568A5FFC8E29FFD5B09507.e.396\")>\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:24:17.976\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m26\u001b[0m - \u001b[1m未发现滑动验证，直接继续\u001b[0m\n",
|
||
"\u001b[32m2025-11-20 13:24:22.983\u001b[0m | \u001b[1mINFO \u001b[0m | \u001b[36mcrawler.xhw.XhwCrawler\u001b[0m:\u001b[36mparse_xhsz_news_detail\u001b[0m:\u001b[36m33\u001b[0m - \u001b[1m找到新闻主体部分: <selenium.webdriver.remote.webelement.WebElement (session=\"11360ade0a59af3938c0f8faa9b88abf\", element=\"f.6B13A7AB92BA3CB5CE0964EB246896F9.d.6B5B529215D1C2221EEF1597FF0C3D0A.e.445\")>\u001b[0m\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"ResultDomain(code=0, message='', success=True, data=None, dataList=[])"
|
||
]
|
||
},
|
||
"execution_count": 16,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"crawler.search(\"大学\", 1)\n",
|
||
"# crawler.search(\"中国\", 10, \"xhsz\")\n",
|
||
"# crawler.search(\"中国\", 10, \"news\")\n",
|
||
"# crawler.search(\"中国\", 10, \"xhsz\")\n",
|
||
"# crawler.search(\"中国\", 10, \"news\")\n",
|
||
"# crawler.search(\"中国\", 10, \"news\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 17,
|
||
"id": "7e0f56fa",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": [
|
||
"# crawler.parse_xhsz_news_detail(\"https://xhsz.news.cn/focus_news/detail?id=9752\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 20,
|
||
"id": "47327ebf",
|
||
"metadata": {},
|
||
"outputs": [
|
||
{
|
||
"data": {
|
||
"text/plain": [
|
||
"NewsItem(title='《习近平总书记关于党的建设的重要思想概论》出版座谈会在北京召开', contentRows=[{'tag': 'p', 'content': '<p>\\u3000\\u3000新华社北京2月24日电\\u3000《习近平总书记关于党的建设的重要思想概论》出版座谈会2月24日在京召开。与会代表结合《概论》主要内容，交流学习贯彻习近平总书记关于党的建设的重要思想的认识和体会。</p>'}, {'tag': 'p', 'content': '<p>\\u3000\\u3000会议认为，《概论》是广大党员、干部深入学习领会习近平总书记关于党的建设的重要思想的权威辅助读物。习近平总书记关于党的建设的重要思想，是一个逻辑严密、内涵丰富、系统全面、博大精深的科学体系，是对中国化的马克思主义党建理论体系的继承发展，构成习近平新时代中国特色社会主义思想的“党建篇”。在这一重要思想的科学指引下，我们党成功开辟百年大党自我革命新境界，推动党和国家事业取得历史性成就、发生历史性变革，为世界政党建设提供了重要借鉴。</p>'}, {'tag': 'p', 'content': '<p>\\u3000\\u3000会议指出，要以学好用好《概论》为契机，进一步把习近平总书记关于党的建设的重要思想领会深、把握准、落到位，深刻领会其科学体系、理论品质和实践指向，更加深刻领悟“两个确立”的决定性意义，增强“四个意识”、坚定“四个自信”、做到“两个维护”。要不断深化体系化研究、学理化阐释，深刻把握这一重要思想蕴含的深刻道理、透彻学理、深邃哲理。要坚持用这一重要思想武装头脑、指导实践、推动工作，把学习成果转化为工作实效，推进党建研究高质量发展，以党建研究新成果推进党的建设和组织工作高质量发展，为以中国式现代化全面推进强国建设、民族复兴伟业提供坚强组织保证。</p>'}, {'tag': 'p', 'content': '<p>\\u3000\\u3000座谈会由全国党建研究会举办，中央和国家机关有关部门，各省区市和新疆生产建设兵团党建研究会（学会），部分中管企业、高校有关负责同志，党史党建专家代表参加座谈会。</p>'}], url='https://www.news.cn/politics/leaders/20250224/5384be3d47c643b3a68e3bb724656152/c.html', viewCount=None, publishTime='2025-02-24 22:44:25', author=None, source='新华网', category=None, executeStatus=0, executeMessage=None)"
|
||
]
|
||
},
|
||
"execution_count": 20,
|
||
"metadata": {},
|
||
"output_type": "execute_result"
|
||
}
|
||
],
|
||
"source": [
|
||
"crawler.parse_xh_news_detail(\"https://www.news.cn/politics/leaders/20250224/5384be3d47c643b3a68e3bb724656152/c.html\")"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": null,
|
||
"id": "fa359d5b",
|
||
"metadata": {},
|
||
"outputs": [],
|
||
"source": []
|
||
}
|
||
],
|
||
"metadata": {
|
||
"kernelspec": {
|
||
"display_name": "schoolNewsCrawler",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"codemirror_mode": {
|
||
"name": "ipython",
|
||
"version": 3
|
||
},
|
||
"file_extension": ".py",
|
||
"mimetype": "text/x-python",
|
||
"name": "python",
|
||
"nbconvert_exporter": "python",
|
||
"pygments_lexer": "ipython3",
|
||
"version": "3.12.12"
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 5
|
||
}
|