From 2982d5380095630f48d4a37a8ccad3f8a514e80c Mon Sep 17 00:00:00 2001 From: wangys <3401275564@qq.com> Date: Thu, 13 Nov 2025 11:04:40 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E9=97=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- schoolNewsCrawler/crawler/RmrbCrawler.py | 24 +++++++++++---- schoolNewsCrawler/crawler/RmrbTrending.py | 30 +++++-------------- .../org/xyzh/common/core/page/PageParam.java | 10 +++++++ .../org/xyzh/crontab/pojo/CrontabItem.java | 1 + .../src/main/resources/application.yml | 4 +++ schoolNewsWeb/src/apis/crontab/index.ts | 2 +- schoolNewsWeb/src/types/crontab/index.ts | 1 + .../admin/manage/crontab/NewsCrawlerView.vue | 4 +-- 8 files changed, 45 insertions(+), 31 deletions(-) diff --git a/schoolNewsCrawler/crawler/RmrbCrawler.py b/schoolNewsCrawler/crawler/RmrbCrawler.py index 701ee3d..eaee0c6 100644 --- a/schoolNewsCrawler/crawler/RmrbCrawler.py +++ b/schoolNewsCrawler/crawler/RmrbCrawler.py @@ -287,11 +287,14 @@ class RmrbCrawler(BaseCrawler): if parser_func is None: logger.error(f"未找到对应解析器，category={category}, url={url}") - return NewsItem( - url=url, - executeStatus=0, - executeMessage=f"不支持的新闻类型: {category}" - ) + parser_func = self.parse_base_news_detail + # return NewsItem( + # url=url, + # contentRows=[], + # title="", + # executeStatus=0, + # executeMessage=f"不支持的新闻类型: {category}" + # ) # 调用对应的解析方法（注意：这些方法是实例方法，需通过 self 调用） return parser_func(url) @@ -638,5 +641,16 @@ class RmrbCrawler(BaseCrawler): except Exception as e: logger.error(f"解析新闻详情失败 [{url}]: {str(e)}") + return NewsItem( + title="", + contentRows=[], + url=url, + publishTime="", + author="", + source="人民网", + category="", + executeStatus=0, + executeMessage=f"解析新闻详情失败: {str(e)}" + ) \ No newline at end of file diff --git a/schoolNewsCrawler/crawler/RmrbTrending.py b/schoolNewsCrawler/crawler/RmrbTrending.py index eee39da..c27ae53 100644 --- a/schoolNewsCrawler/crawler/RmrbTrending.py +++ b/schoolNewsCrawler/crawler/RmrbTrending.py @@ -18,7 +18,7 @@ sys.path.insert(0, str(Path(__file__).parent.parent)) from crawler.RmrbCrawler import RmrbCrawler from loguru import logger - +from core.ResultDomain import ResultDomain def parse_date(date_str) -> datetime: """ @@ -65,7 +65,6 @@ def main(): """ ) - parser.add_argument('--date', '-d', type=str, help='指定日期 (格式: YYYYMMDD)') parser.add_argument('--startDate', '-s', type=str, help='开始日期 (需与--end-date一起使用)') parser.add_argument('--endDate', '-e', type=str, help='结束日期 (需与--start-date一起使用)') parser.add_argument('--yesterday', '-y', action='store_true', help='查询昨日 (默认行为)') @@ -75,7 +74,6 @@ def main(): # 初始化变量 output_file = args.output - date = args.date start_date = args.startDate end_date = args.endDate is_yesterday = args.yesterday if args.yesterday else True # 默认查昨日 @@ -86,25 +84,19 @@ def main(): def clean(s): return s.strip() if s and isinstance(s, str) and s.strip() else None - date = clean(date) start_date = clean(start_date) end_date = clean(end_date) try: crawler = RmrbCrawler() - # 单日模式 - if date: - if start_date or end_date: - raise ValueError("不能同时使用 date 和 startDate/endDate 参数") - target_date = parse_date(date) - logger.info(f"获取单日热点新闻: {target_date.strftime('%Y-%m-%d')}") + if is_yesterday: + target_date = datetime.now() - timedelta(days=1) + date_str = target_date.strftime('%Y%m%d') + logger.info(f"获取昨日热点新闻: {target_date.strftime('%Y-%m-%d')} (参数格式: {date_str})") result = crawler.getOneDayTrendingNews(target_date) - # 日期范围模式 elif start_date and end_date: - if date: - raise ValueError("不能同时使用 date 和 startDate/endDate 参数") start_dt = parse_date(start_date) end_dt = parse_date(end_date) if start_dt > end_dt: @@ -114,17 +106,9 @@ def main(): # 只给一个边界 elif start_date or end_date: - raise ValueError("--start-date 和 --end-date 必须同时指定") - - # 默认模式 + raise ValueError("--startDate 和 --endDate 必须同时指定") else: - if is_yesterday: - target_date = datetime.now() - timedelta(days=1) - logger.info(f"获取昨日热点新闻: {target_date.strftime('%Y-%m-%d')}") - else: - target_date = datetime.now() - logger.info(f"获取今日热点新闻: {target_date.strftime('%Y-%m-%d')}") - result = crawler.getOneDayTrendingNews(target_date) + result = ResultDomain(code=0, message="参数错误", success=False, data=None, dataList=None) # 构造输出 output = { diff --git a/schoolNewsServ/common/common-core/src/main/java/org/xyzh/common/core/page/PageParam.java b/schoolNewsServ/common/common-core/src/main/java/org/xyzh/common/core/page/PageParam.java index 9b243f2..3c69009 100644 --- a/schoolNewsServ/common/common-core/src/main/java/org/xyzh/common/core/page/PageParam.java +++ b/schoolNewsServ/common/common-core/src/main/java/org/xyzh/common/core/page/PageParam.java @@ -77,6 +77,11 @@ public class PageParam implements Serializable { */ public void setPageNumber(int pageNumber) { this.pageNumber = pageNumber; + if (pageSize > 0) { + this.offset = (pageNumber - 1) * pageSize; + } else { + this.offset = 0; + } } /** @@ -98,6 +103,11 @@ public class PageParam implements Serializable { */ public void setPageSize(int pageSize) { this.pageSize = pageSize; + if (pageNumber > 0) { + this.offset = (pageNumber - 1) * pageSize; + } else { + this.offset = 0; + } } /** diff --git a/schoolNewsServ/crontab/src/main/java/org/xyzh/crontab/pojo/CrontabItem.java b/schoolNewsServ/crontab/src/main/java/org/xyzh/crontab/pojo/CrontabItem.java index 05fade7..8b1fc87 100644 --- a/schoolNewsServ/crontab/src/main/java/org/xyzh/crontab/pojo/CrontabItem.java +++ b/schoolNewsServ/crontab/src/main/java/org/xyzh/crontab/pojo/CrontabItem.java @@ -28,5 +28,6 @@ public class CrontabItem { private String description; private String type; private Object value; + private Boolean required; } } diff --git a/schoolNewsServ/crontab/src/main/resources/application.yml b/schoolNewsServ/crontab/src/main/resources/application.yml index a2e2841..d8470e5 100644 --- a/schoolNewsServ/crontab/src/main/resources/application.yml +++ b/schoolNewsServ/crontab/src/main/resources/application.yml @@ -18,10 +18,12 @@ crontab: description: 搜索关键字 type: String value: "" + required: true - name: total description: 总新闻数量 type: Integer value: 10 + required: true - name: 排行榜爬取 clazz: newsCrewerTask excuete_method: execute @@ -35,10 +37,12 @@ crontab: description: 开始日期 type: String value: "" + required: false - name: endDate description: 结束日期 type: String value: "" + required: false - name: yesterday description: 是否是昨天 type: Boolean diff --git a/schoolNewsWeb/src/apis/crontab/index.ts b/schoolNewsWeb/src/apis/crontab/index.ts index 0f976fd..123aab6 100644 --- a/schoolNewsWeb/src/apis/crontab/index.ts +++ b/schoolNewsWeb/src/apis/crontab/index.ts @@ -117,7 +117,7 @@ export const crontabApi = { * @returns Promise> */ async executeTaskOnce(taskId: string): Promise> { - const response = await api.post(`${this.baseUrl}/task/execute/${taskId}`); + const response = await api.post(`${this.baseUrl}/task/execute/${taskId}`,{},{showLoading: false}); return response.data; }, diff --git a/schoolNewsWeb/src/types/crontab/index.ts b/schoolNewsWeb/src/types/crontab/index.ts index ea37b9a..0a3775b 100644 --- a/schoolNewsWeb/src/types/crontab/index.ts +++ b/schoolNewsWeb/src/types/crontab/index.ts @@ -157,6 +157,7 @@ export interface CrontabParam { type: string; /** 默认值 */ value: any; + required: boolean; } /** diff --git a/schoolNewsWeb/src/views/admin/manage/crontab/NewsCrawlerView.vue b/schoolNewsWeb/src/views/admin/manage/crontab/NewsCrawlerView.vue index 8cbfb0a..29a2013 100644 --- a/schoolNewsWeb/src/views/admin/manage/crontab/NewsCrawlerView.vue +++ b/schoolNewsWeb/src/views/admin/manage/crontab/NewsCrawlerView.vue @@ -632,11 +632,11 @@ async function handleSubmit() { for (const param of selectedMethod.value.params) { const value = dynamicParams.value[param.name]; - if (param.type === 'String' && (!value || value.trim() === '')) { + if (param.required && param.type === 'String' && (!value || value.trim() === '')) { ElMessage.warning(`请输入${param.description}`); return; } - if (param.type === 'Integer' && (value === undefined || value === null || value === '')) { + if (param.required && param.type === 'Integer' && (value === undefined || value === null || value === '')) { ElMessage.warning(`请输入${param.description}`); return; }