新闻采集修改,完成发送邮件

This commit is contained in:
2025-11-18 17:56:10 +08:00
parent 049b6f2cf3
commit 9f3176194b
50 changed files with 3929 additions and 322 deletions

View File

@@ -7,6 +7,8 @@ CREATE TABLE `tb_crontab_task` (
`task_id` VARCHAR(64) NOT NULL COMMENT '任务ID',
`task_name` VARCHAR(100) NOT NULL COMMENT '任务名称',
`task_group` VARCHAR(50) NOT NULL DEFAULT 'DEFAULT' COMMENT '任务分组',
`meta_id` VARCHAR(64) NOT NULL COMMENT '任务元数据ID',
`default_recipient` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否使用默认接收人0:否 1:是)',
`bean_name` VARCHAR(100) NOT NULL COMMENT 'Bean名称',
`method_name` VARCHAR(100) NOT NULL COMMENT '方法名称',
`method_params` VARCHAR(500) DEFAULT NULL COMMENT '方法参数',
@@ -94,4 +96,78 @@ CREATE TABLE `tb_data_collection_item` (
KEY `idx_status` (`status`),
KEY `idx_publish_time` (`publish_time`),
KEY `idx_source_url` (`source_url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='数据采集项表';
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='数据采集项表';
-- ====================================================
-- 定时任务元数据表(存储爬虫任务的元数据配置)
-- ====================================================
DROP TABLE IF EXISTS `tb_crontab_task_meta`;
CREATE TABLE `tb_crontab_task_meta` (
`id` VARCHAR(64) NOT NULL COMMENT '主键ID',
`meta_id` VARCHAR(64) NOT NULL COMMENT '元数据ID',
`name` VARCHAR(100) NOT NULL COMMENT '任务名称',
`description` VARCHAR(500) DEFAULT NULL COMMENT '任务描述',
`category` VARCHAR(50) NOT NULL COMMENT '任务分类(如:人民日报新闻爬取)',
`bean_name` VARCHAR(100) NOT NULL COMMENT 'Bean名称执行器类名',
`method_name` VARCHAR(100) NOT NULL COMMENT '执行方法名',
`script_path` VARCHAR(255) DEFAULT NULL COMMENT 'Python脚本路径相对于basePath',
`param_schema` TEXT DEFAULT NULL COMMENT '参数模板JSON格式定义参数名、类型、描述、默认值等',
`auto_publish` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否自动发布',
`sort_order` INT DEFAULT 0 COMMENT '排序号',
`creator` VARCHAR(64) DEFAULT NULL COMMENT '创建者',
`updater` VARCHAR(64) DEFAULT NULL COMMENT '更新者',
`create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` DATETIME DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
`delete_time` DATETIME DEFAULT NULL COMMENT '删除时间',
`deleted` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否删除0:否 1:是)',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_meta_id` (`meta_id`),
KEY `idx_category` (`category`),
KEY `idx_deleted` (`deleted`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='定时任务元数据表';
-- ====================================================
-- 定时任务邮件通知默认接收人员表(关联任务元数据,定义默认接收人员)
-- ====================================================
DROP TABLE IF EXISTS `tb_crontab_email_default`;
CREATE TABLE `tb_crontab_email_default` (
`id` VARCHAR(64) NOT NULL COMMENT '主键ID',
`default_id` VARCHAR(64) NOT NULL COMMENT '默认ID',
`meta_id` VARCHAR(64) NOT NULL COMMENT '关联任务元数据ID',
`user_id` VARCHAR(64) NOT NULL COMMENT '关联用户ID',
`creator` VARCHAR(64) DEFAULT NULL COMMENT '创建者',
`updater` VARCHAR(64) DEFAULT NULL COMMENT '更新者',
`create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` DATETIME DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
`delete_time` DATETIME DEFAULT NULL COMMENT '删除时间',
`deleted` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否删除0:否 1:是)',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_default_id` (`default_id`),
UNIQUE KEY `uk_meta_id` (`meta_id`),
KEY `idx_deleted` (`deleted`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='定时任务邮件通知默认接收人员表';
-- ====================================================
-- 定时任务邮件接收人表1对多
-- ====================================================
DROP TABLE IF EXISTS `tb_crontab_email_recipient`;
CREATE TABLE `tb_crontab_email_recipient` (
`id` VARCHAR(64) NOT NULL COMMENT '主键ID',
`recipient_id` VARCHAR(64) NOT NULL COMMENT '接收人ID',
`task_id` VARCHAR(64) DEFAULT NULL COMMENT '关联任务IDNULL表示不属于任何任务',
`user_id` VARCHAR(64) DEFAULT NULL COMMENT '关联用户ID',
`email` VARCHAR(100) NOT NULL COMMENT '邮箱地址',
`name` VARCHAR(100) DEFAULT NULL COMMENT '接收人姓名',
`creator` VARCHAR(64) DEFAULT NULL COMMENT '创建者',
`updater` VARCHAR(64) DEFAULT NULL COMMENT '更新者',
`create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` DATETIME DEFAULT NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
`delete_time` DATETIME DEFAULT NULL COMMENT '删除时间',
`deleted` TINYINT(1) NOT NULL DEFAULT 0 COMMENT '是否删除0:否 1:是)',
PRIMARY KEY (`id`),
UNIQUE KEY `uk_recipient_id` (`recipient_id`),
KEY `idx_task_id` (`task_id`),
KEY `idx_user_id` (`user_id`),
KEY `idx_email` (`email`),
KEY `idx_deleted` (`deleted`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='定时任务邮件接收人表';

View File

@@ -38,7 +38,9 @@ INSERT INTO `tb_sys_config` (id, config_key, config_value, config_type, config_g
('10', 'system.banner.interval', '5000', 'number', 'banner', 'Banner切换间隔毫秒', 0, '1', now()),
('11', 'system.resource.auto_publish', 'false', 'boolean', 'resource', '资源自动发布', 0, '1', now()),
('12', 'system.resource.auto_publish_time', '08:00', 'string', 'resource', '自动发布时间', 0, '1', now()),
('13', 'system.ai.enabled', 'true', 'boolean', 'ai', '是否启用智能体', 0, '1', now());
('13', 'system.ai.enabled', 'true', 'boolean', 'ai', '是否启用智能体', 0, '1', now()),
('14', 'crawler.pythonPath', 'F:/Environment/Conda/envs/schoolNewsCrawler/python.exe', 'string', 'crawler', 'Python可执行文件路径', 1, '1', now()),
('15', 'crawler.basePath', 'F:/Project/schoolNews/schoolNewsCrawler', 'string', 'crawler', '爬虫脚本根目录', 1, '1', now());
-- 注意默认superadmin用户已在 initMenuData.sql 中创建,此处无需重复创建

View File

@@ -0,0 +1,102 @@
-- ====================================================
-- 定时任务元数据初始化脚本
-- ====================================================
-- 插入人民日报新闻爬取任务的元数据
-- 1. 关键字搜索爬取
INSERT INTO `tb_crontab_task_meta` (
`id`, `meta_id`, `name`, `description`, `category`,
`bean_name`, `method_name`, `script_path`, `param_schema`,
`sort_order`, `creator`, `create_time`
) VALUES (
'1',
'rmbr_keyword_search',
'关键字搜索爬取',
'根据关键字搜索人民日报新闻内容',
'人民日报新闻爬取',
'newsCrewerTask',
'execute',
'crawler/RmrbSearch.py',
'[
{
"name": "query",
"description": "搜索关键字",
"type": "String",
"value": "",
"required": true
},
{
"name": "total",
"description": "总新闻数量",
"type": "Integer",
"value": 10,
"required": true
}
]',
1,
'system',
NOW()
);
-- 2. 排行榜爬取
INSERT INTO `tb_crontab_task_meta` (
`id`, `meta_id`, `name`, `description`, `category`,
`bean_name`, `method_name`, `script_path`, `param_schema`,
`sort_order`, `creator`, `create_time`
) VALUES (
'2',
'rmbr_hotpoint',
'排行榜爬取',
'爬取人民日报热门排行榜新闻',
'人民日报新闻爬取',
'newsCrewerTask',
'execute',
'crawler/RmrbHotPoint.py',
'[]',
2,
'system',
NOW()
);
-- 3. 往日精彩头条爬取
INSERT INTO `tb_crontab_task_meta` (
`id`, `meta_id`, `name`, `description`, `category`,
`bean_name`, `method_name`, `script_path`, `param_schema`,
`sort_order`, `creator`, `create_time`
) VALUES (
'3',
'rmbr_trending',
'往日精彩头条爬取',
'爬取人民日报往日精彩头条新闻',
'人民日报新闻爬取',
'newsCrewerTask',
'execute',
'crawler/RmrbTrending.py',
'[
{
"name": "startDate",
"description": "开始日期",
"type": "String",
"value": "",
"required": false
},
{
"name": "endDate",
"description": "结束日期",
"type": "String",
"value": "",
"required": false
},
{
"name": "yesterday",
"description": "是否是昨天",
"type": "Boolean",
"value": true,
"required": false
}
]',
3,
'system',
NOW()
);