样式修改
This commit is contained in:
File diff suppressed because one or more lines are too long
@@ -490,6 +490,7 @@ class XhwCrawler(BaseCrawler):
|
|||||||
search_data = search_config.params.copy()
|
search_data = search_config.params.copy()
|
||||||
search_data["k"] = key
|
search_data["k"] = key
|
||||||
search_data["action"] = action
|
search_data["action"] = action
|
||||||
|
max_page = 1
|
||||||
|
|
||||||
try:
|
try:
|
||||||
# 获取新闻url
|
# 获取新闻url
|
||||||
@@ -524,6 +525,19 @@ class XhwCrawler(BaseCrawler):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.info(f"滑动验证处理失败或未出现: {e}")
|
logger.info(f"滑动验证处理失败或未出现: {e}")
|
||||||
|
|
||||||
|
# 获取最大分页数并校验
|
||||||
|
try:
|
||||||
|
search_foot = self.driver.find_element(By.CSS_SELECTOR, "div.pagebar")
|
||||||
|
if search_foot:
|
||||||
|
page_nums = search_foot.find_elements(By.CSS_SELECTOR, "a.num")
|
||||||
|
if page_nums:
|
||||||
|
max_page = int(page_nums[-1].text)
|
||||||
|
if page > max_page:
|
||||||
|
logger.info(f"当前页 {page} 超过最大页数 {max_page},停止翻页")
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"获取最大分页失败: {e}")
|
||||||
|
|
||||||
# 提取新闻列表
|
# 提取新闻列表
|
||||||
try:
|
try:
|
||||||
search_main = self.driver.find_element(By.CSS_SELECTOR, "div.page-search-main")
|
search_main = self.driver.find_element(By.CSS_SELECTOR, "div.page-search-main")
|
||||||
@@ -549,6 +563,8 @@ class XhwCrawler(BaseCrawler):
|
|||||||
|
|
||||||
# 从新闻url中获取新闻详情
|
# 从新闻url中获取新闻详情
|
||||||
count = 0
|
count = 0
|
||||||
|
total_urls = len(news_urls)
|
||||||
|
logger.info(f"开始解析新闻详情,共 {total_urls} 条URL,目标获取 {total} 条")
|
||||||
for news_url in news_urls:
|
for news_url in news_urls:
|
||||||
try:
|
try:
|
||||||
news = self.parse_news_detail(news_url)
|
news = self.parse_news_detail(news_url)
|
||||||
@@ -557,11 +573,14 @@ class XhwCrawler(BaseCrawler):
|
|||||||
news.publishTime = url_base_map.get(news_url, {}).get("date") or news.publishTime
|
news.publishTime = url_base_map.get(news_url, {}).get("date") or news.publishTime
|
||||||
news_list.append(news)
|
news_list.append(news)
|
||||||
count += 1
|
count += 1
|
||||||
|
if count % 10 == 0 or count >= total:
|
||||||
|
logger.info(f"解析进度: {count}/{total} 条")
|
||||||
if count >= total:
|
if count >= total:
|
||||||
break
|
break
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning(f"解析新闻失败: {news_url}, {e}")
|
logger.warning(f"解析新闻失败: {news_url}, {e}")
|
||||||
continue
|
continue
|
||||||
|
logger.info(f"新闻详情解析完成,共获取 {count} 条")
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"搜索过程整体异常: {e}")
|
logger.error(f"搜索过程整体异常: {e}")
|
||||||
|
|||||||
@@ -104,17 +104,20 @@ public class SchedulerManager {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @description 立即执行一次任务
|
* @description 立即执行一次任务(异步)
|
||||||
* @param task 任务对象
|
* @param task 任务对象
|
||||||
* @author yslg
|
* @author yslg
|
||||||
* @since 2025-10-25
|
* @since 2025-10-25
|
||||||
*/
|
*/
|
||||||
public void executeTaskOnce(TbCrontabTask task) {
|
public void executeTaskOnce(TbCrontabTask task) {
|
||||||
try {
|
try {
|
||||||
taskExecutor.executeTask(task);
|
taskScheduler.schedule(
|
||||||
logger.info("立即执行任务: {}", task.getTaskName());
|
() -> taskExecutor.executeTask(task),
|
||||||
|
java.time.Instant.now()
|
||||||
|
);
|
||||||
|
logger.info("立即执行任务已提交: {}", task.getTaskName());
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("立即执行任务失败: {}", task.getTaskName(), e);
|
logger.error("立即执行任务提交失败: {}", task.getTaskName(), e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -532,7 +532,7 @@ public class NewsCrawlerTask extends PythonCommandTask {
|
|||||||
Map<String, String> urlToResourceId = new HashMap<>();
|
Map<String, String> urlToResourceId = new HashMap<>();
|
||||||
|
|
||||||
for (TbDataCollectionItem item : itemList) {
|
for (TbDataCollectionItem item : itemList) {
|
||||||
if(item.getContent().isEmpty()) continue;
|
if(NonUtils.isEmpty(item.getContent())) continue;
|
||||||
try {
|
try {
|
||||||
TbResource resource = new TbResource();
|
TbResource resource = new TbResource();
|
||||||
resource.setId(IDUtils.generateID());
|
resource.setId(IDUtils.generateID());
|
||||||
|
|||||||
@@ -178,9 +178,13 @@
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|
||||||
<el-form-item prop="agree">
|
<el-form-item prop="agree">
|
||||||
<p class="agreement-text">
|
<div class="agreement-wrapper">
|
||||||
<el-checkbox v-model="loginForm.agree">登录即为同意<span class="agreement-link" style="color: red">《红色思政智能体平台》</span></el-checkbox>
|
<el-checkbox v-model="loginForm.agree" />
|
||||||
</p>
|
<span class="agreement-text">
|
||||||
|
登录即为同意 <span class="agreement-link">《红色思政智能体平台》</span><span class="agreement-link">《用户协议》</span>
|
||||||
|
<span class="agreement-link">《隐私协议》</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
</el-form>
|
</el-form>
|
||||||
|
|
||||||
@@ -806,19 +810,30 @@ onMounted(() => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.agreement-text {
|
.agreement-wrapper {
|
||||||
font-family: "Source Han Sans SC";
|
display: flex;
|
||||||
font-weight: 400;
|
align-items: flex-start;
|
||||||
font-size: 10px;
|
gap: 4px;
|
||||||
line-height: 1.8;
|
width: 100%;
|
||||||
color: rgba(0, 0, 0, 0.3);
|
|
||||||
text-align: center;
|
|
||||||
margin: 0;
|
|
||||||
|
|
||||||
:deep(.el-checkbox) {
|
.el-checkbox {
|
||||||
.el-checkbox__label {
|
margin-right: 0;
|
||||||
font-size: 10px;
|
}
|
||||||
color: rgba(0, 0, 0, 0.3);
|
|
||||||
|
.agreement-text {
|
||||||
|
font-family: "Source Han Sans SC";
|
||||||
|
font-weight: 400;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.8;
|
||||||
|
color: #C62828;
|
||||||
|
|
||||||
|
.agreement-link {
|
||||||
|
color: #C62828;
|
||||||
|
cursor: pointer;
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -178,9 +178,14 @@
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|
||||||
<el-form-item prop="agree">
|
<el-form-item prop="agree">
|
||||||
<p class="agreement-text">
|
<div class="agreement-wrapper">
|
||||||
<el-checkbox v-model="loginForm.agree">登录即为同意<span class="agreement-link" style="color: red">《红色思政智能体平台》</span></el-checkbox>
|
<el-checkbox v-model="loginForm.agree" />
|
||||||
</p>
|
<span class="agreement-text">
|
||||||
|
登录即为同意 <span class="agreement-link">《红色思政智能体平台》</span>
|
||||||
|
<span class="agreement-link">《用户协议》</span>
|
||||||
|
<span class="agreement-link">《隐私协议》</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
</el-form>
|
</el-form>
|
||||||
|
|
||||||
@@ -328,7 +333,7 @@ const loginRules: FormRules = {
|
|||||||
{
|
{
|
||||||
validator: (rule: any, value: boolean, callback: any) => {
|
validator: (rule: any, value: boolean, callback: any) => {
|
||||||
if (!value) {
|
if (!value) {
|
||||||
callback(new Error('请同意《红色思政智能体平台》用户协议'));
|
callback(new Error('请同意各项协议'));
|
||||||
} else {
|
} else {
|
||||||
callback();
|
callback();
|
||||||
}
|
}
|
||||||
@@ -803,14 +808,32 @@ onMounted(() => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.agreement-text {
|
.agreement-wrapper {
|
||||||
font-family: "Source Han Sans SC";
|
display: flex;
|
||||||
font-weight: 400;
|
align-items: flex-start;
|
||||||
font-size: 10px;
|
gap: 4px;
|
||||||
line-height: 1.8;
|
width: 100%;
|
||||||
color: rgba(0, 0, 0, 0.3);
|
|
||||||
text-align: center;
|
.el-checkbox {
|
||||||
margin: 0;
|
margin-right: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.agreement-text {
|
||||||
|
font-family: "Source Han Sans SC";
|
||||||
|
font-weight: 400;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.8;
|
||||||
|
color: #C62828;
|
||||||
|
|
||||||
|
.agreement-link {
|
||||||
|
color: #C62828;
|
||||||
|
cursor: pointer;
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -164,9 +164,13 @@
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|
||||||
<el-form-item prop="agree">
|
<el-form-item prop="agree">
|
||||||
<p class="agreement-text">
|
<div class="agreement-wrapper">
|
||||||
<el-checkbox v-model="registerForm.agree">注册即为同意<span class="agreement-link" style="color: red">《红色思政智能体平台》</span></el-checkbox>
|
<el-checkbox v-model="registerForm.agree" />
|
||||||
</p>
|
<span class="agreement-text">
|
||||||
|
注册即为同意 <span class="agreement-link">《红色思政智能体平台》</span><span class="agreement-link">《用户协议》</span>
|
||||||
|
<span class="agreement-link">《隐私协议》</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
</el-form>
|
</el-form>
|
||||||
</div>
|
</div>
|
||||||
@@ -748,19 +752,30 @@ onUnmounted(() => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.agreement-text {
|
.agreement-wrapper {
|
||||||
font-family: "Source Han Sans SC";
|
display: flex;
|
||||||
font-weight: 400;
|
align-items: flex-start;
|
||||||
font-size: 10px;
|
gap: 4px;
|
||||||
line-height: 1.8;
|
width: 100%;
|
||||||
color: rgba(0, 0, 0, 0.3);
|
|
||||||
text-align: center;
|
|
||||||
margin: 0;
|
|
||||||
|
|
||||||
:deep(.el-checkbox) {
|
.el-checkbox {
|
||||||
.el-checkbox__label {
|
margin-right: 0;
|
||||||
font-size: 10px;
|
}
|
||||||
color: rgba(0, 0, 0, 0.3);
|
|
||||||
|
.agreement-text {
|
||||||
|
font-family: "Source Han Sans SC";
|
||||||
|
font-weight: 400;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.8;
|
||||||
|
color: #C62828;
|
||||||
|
|
||||||
|
.agreement-link {
|
||||||
|
color: #C62828;
|
||||||
|
cursor: pointer;
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -171,9 +171,13 @@
|
|||||||
</el-form-item>
|
</el-form-item>
|
||||||
|
|
||||||
<el-form-item prop="agree">
|
<el-form-item prop="agree">
|
||||||
<p class="agreement-text">
|
<div class="agreement-wrapper">
|
||||||
<el-checkbox v-model="registerForm.agree">注册即为同意<span class="agreement-link" style="color: red">《红色思政智能体平台》</span></el-checkbox>
|
<el-checkbox v-model="registerForm.agree" />
|
||||||
</p>
|
<span class="agreement-text">
|
||||||
|
注册即为同意 <span class="agreement-link">《红色思政智能体平台》</span><span class="agreement-link">《用户协议》</span>
|
||||||
|
<span class="agreement-link">《隐私协议》</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
</el-form-item>
|
</el-form-item>
|
||||||
</el-form>
|
</el-form>
|
||||||
|
|
||||||
@@ -747,14 +751,32 @@ onUnmounted(() => {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.agreement-text {
|
.agreement-wrapper {
|
||||||
font-family: "Source Han Sans SC";
|
display: flex;
|
||||||
font-weight: 400;
|
align-items: flex-start;
|
||||||
font-size: 10px;
|
gap: 4px;
|
||||||
line-height: 1.8;
|
width: 100%;
|
||||||
color: rgba(0, 0, 0, 0.3);
|
|
||||||
text-align: center;
|
.el-checkbox {
|
||||||
margin: 0;
|
margin-right: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.agreement-text {
|
||||||
|
font-family: "Source Han Sans SC";
|
||||||
|
font-weight: 400;
|
||||||
|
font-size: 12px;
|
||||||
|
line-height: 1.8;
|
||||||
|
color: #C62828;
|
||||||
|
|
||||||
|
.agreement-link {
|
||||||
|
color: #C62828;
|
||||||
|
cursor: pointer;
|
||||||
|
|
||||||
|
&:hover {
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user