278 lines
7.1 KiB
SQL
278 lines
7.1 KiB
SQL
-- ====================================================
|
|
-- 定时任务元数据初始化脚本
|
|
-- ====================================================
|
|
|
|
-- 插入人民日报新闻爬取任务的元数据
|
|
|
|
-- 1. 关键字搜索爬取
|
|
INSERT INTO `tb_crontab_task_meta` (
|
|
`id`, `meta_id`, `name`, `description`, `category`,
|
|
`bean_name`, `method_name`, `script_path`, `param_schema`, `auto_publish`,
|
|
`sort_order`, `creator`, `create_time`
|
|
) VALUES (
|
|
'1',
|
|
'rmbr_keyword_search',
|
|
'关键字搜索爬取',
|
|
'根据关键字搜索人民日报新闻内容',
|
|
'人民日报新闻爬取',
|
|
'newsCrewerTask',
|
|
'execute',
|
|
'crawler/rmrb/RmrbSearch.py',
|
|
'[
|
|
{
|
|
"name": "query",
|
|
"description": "搜索关键字",
|
|
"type": "Input",
|
|
"valueType": "String",
|
|
"value": "",
|
|
"required": true
|
|
},
|
|
{
|
|
"name": "total",
|
|
"description": "总新闻数量",
|
|
"type": "InputNumber",
|
|
"valueType": "Integer",
|
|
"value": 10,
|
|
"required": true
|
|
}
|
|
]',
|
|
1,
|
|
1,
|
|
'system',
|
|
NOW()
|
|
);
|
|
|
|
-- 2. 排行榜爬取
|
|
INSERT INTO `tb_crontab_task_meta` (
|
|
`id`, `meta_id`, `name`, `description`, `category`,
|
|
`bean_name`, `method_name`, `script_path`, `param_schema`, `auto_publish`,
|
|
`sort_order`, `creator`, `create_time`
|
|
) VALUES (
|
|
'2',
|
|
'rmbr_hotpoint',
|
|
'排行榜爬取',
|
|
'爬取人民日报热门排行榜新闻',
|
|
'人民日报新闻爬取',
|
|
'newsCrewerTask',
|
|
'execute',
|
|
'crawler/rmrb/RmrbHotPoint.py',
|
|
'[]',
|
|
1,
|
|
2,
|
|
'system',
|
|
NOW()
|
|
);
|
|
|
|
-- 3. 往日精彩头条爬取
|
|
INSERT INTO `tb_crontab_task_meta` (
|
|
`id`, `meta_id`, `name`, `description`, `category`,
|
|
`bean_name`, `method_name`, `script_path`, `param_schema`, `auto_publish`,
|
|
`sort_order`, `creator`, `create_time`
|
|
) VALUES (
|
|
'3',
|
|
'rmbr_trending',
|
|
'往日精彩头条爬取',
|
|
'爬取人民日报往日精彩头条新闻',
|
|
'人民日报新闻爬取',
|
|
'newsCrewerTask',
|
|
'execute',
|
|
'crawler/rmrb/RmrbTrending.py',
|
|
'[
|
|
{
|
|
"name": "dateRange",
|
|
"description": "日期范围",
|
|
"type": "DateRangePicker",
|
|
"valueType": "String",
|
|
"value": "",
|
|
"required": false,
|
|
"startKey": "startDate",
|
|
"endKey": "endDate"
|
|
},
|
|
{
|
|
"name": "yesterday",
|
|
"description": "是否是昨天",
|
|
"type": "Switch",
|
|
"valueType": "Boolean",
|
|
"value": true,
|
|
"required": false
|
|
}
|
|
]',
|
|
1,
|
|
3,
|
|
'system',
|
|
NOW()
|
|
);
|
|
|
|
-- 4. 新华网关键字搜索爬取
|
|
INSERT INTO `tb_crontab_task_meta` (
|
|
`id`, `meta_id`, `name`, `description`, `category`,
|
|
`bean_name`, `method_name`, `script_path`, `param_schema`, `auto_publish`,
|
|
`sort_order`, `creator`, `create_time`
|
|
) VALUES (
|
|
'4',
|
|
'xhw_keyword_search',
|
|
'关键字搜索爬取',
|
|
'根据关键字搜索新华网新闻内容',
|
|
'新华网新闻爬取',
|
|
'newsCrewerTask',
|
|
'execute',
|
|
'crawler/xhw/XhwSearch.py',
|
|
'[
|
|
{
|
|
"name": "query",
|
|
"description": "搜索关键字",
|
|
"type": "Input",
|
|
"valueType": "String",
|
|
"value": "",
|
|
"required": true
|
|
},
|
|
{
|
|
"name": "total",
|
|
"description": "抓取数量",
|
|
"type": "InputNumber",
|
|
"valueType": "Integer",
|
|
"value": 10,
|
|
"required": true
|
|
}
|
|
]',
|
|
1,
|
|
4,
|
|
'system',
|
|
NOW()
|
|
);
|
|
|
|
-- 5. 新华网热点新闻爬取
|
|
INSERT INTO `tb_crontab_task_meta` (
|
|
`id`, `meta_id`, `name`, `description`, `category`,
|
|
`bean_name`, `method_name`, `script_path`, `param_schema`, `auto_publish`,
|
|
`sort_order`, `creator`, `create_time`
|
|
) VALUES (
|
|
'5',
|
|
'xhw_hot_point',
|
|
'热点新闻爬取',
|
|
'爬取新华网热点新闻',
|
|
'新华网新闻爬取',
|
|
'newsCrewerTask',
|
|
'execute',
|
|
'crawler/xhw/XhwHotPoint.py',
|
|
'[]',
|
|
1,
|
|
5,
|
|
'system',
|
|
NOW()
|
|
);
|
|
|
|
-- 6. 新华网推荐新闻爬取
|
|
INSERT INTO `tb_crontab_task_meta` (
|
|
`id`, `meta_id`, `name`, `description`, `category`,
|
|
`bean_name`, `method_name`, `script_path`, `param_schema`, `auto_publish`,
|
|
`sort_order`, `creator`, `create_time`
|
|
) VALUES (
|
|
'6',
|
|
'xhw_commend',
|
|
'推荐新闻爬取',
|
|
'爬取新华网推荐新闻',
|
|
'新华网新闻爬取',
|
|
'newsCrewerTask',
|
|
'execute',
|
|
'crawler/xhw/XhwCommend.py',
|
|
'[]',
|
|
1,
|
|
6,
|
|
'system',
|
|
NOW()
|
|
);
|
|
|
|
-- 7. 学习强国关键字搜索爬取
|
|
INSERT INTO `tb_crontab_task_meta` (
|
|
`id`, `meta_id`, `name`, `description`, `category`,
|
|
`bean_name`, `method_name`, `script_path`, `param_schema`, `auto_publish`,
|
|
`sort_order`, `creator`, `create_time`
|
|
) VALUES (
|
|
'7',
|
|
'xxqg_keyword_search',
|
|
'关键字搜索爬取',
|
|
'根据关键字搜索学习强国新闻内容',
|
|
'学习强国新闻爬取',
|
|
'newsCrewerTask',
|
|
'execute',
|
|
'crawler/xxqg/XxqgSearch.py',
|
|
'[
|
|
{
|
|
"name": "query",
|
|
"description": "搜索关键字",
|
|
"type": "Input",
|
|
"valueType": "String",
|
|
"value": "",
|
|
"required": true
|
|
},
|
|
{
|
|
"name": "total",
|
|
"description": "抓取数量",
|
|
"type": "InputNumber",
|
|
"valueType": "Integer",
|
|
"value": 10,
|
|
"required": true
|
|
}
|
|
]',
|
|
1,
|
|
7,
|
|
'system',
|
|
NOW()
|
|
);
|
|
|
|
-- 8. 学习强国栏目新闻爬取
|
|
INSERT INTO `tb_crontab_task_meta` (
|
|
`id`, `meta_id`, `name`, `description`, `category`,
|
|
`bean_name`, `method_name`, `script_path`, `param_schema`, `auto_publish`,
|
|
`sort_order`, `creator`, `create_time`
|
|
) VALUES (
|
|
'8',
|
|
'xxqg_column_crawl',
|
|
'栏目新闻爬取',
|
|
'爬取学习强国指定栏目的新闻内容',
|
|
'学习强国新闻爬取',
|
|
'newsCrewerTask',
|
|
'execute',
|
|
'crawler/xxqg/XxqgColumn.py',
|
|
'[
|
|
{
|
|
"name": "column",
|
|
"description": "栏目名称",
|
|
"type": "Select",
|
|
"valueType": "String",
|
|
"value": "important",
|
|
"required": true,
|
|
"options": [
|
|
{"label": "重要新闻", "value": "important"},
|
|
{"label": "学习时评", "value": "xuexishiping"},
|
|
{"label": "综合新闻", "value": "zonghexinwen"},
|
|
{"label": "中宣部发布", "value": "zhongxuanbu"}
|
|
]
|
|
},
|
|
{
|
|
"name": "yesterday",
|
|
"description": "是否抓取昨天的数据",
|
|
"type": "Switch",
|
|
"valueType": "Boolean",
|
|
"value": true,
|
|
"required": false
|
|
},
|
|
{
|
|
"name": "dateRange",
|
|
"description": "日期范围",
|
|
"type": "DateRangePicker",
|
|
"valueType": "String",
|
|
"value": "",
|
|
"required": false,
|
|
"startKey": "start",
|
|
"endKey": "end"
|
|
}
|
|
]',
|
|
1,
|
|
8,
|
|
'system',
|
|
NOW()
|
|
);
|
|
|