78 lines
3.0 KiB
Python
78 lines
3.0 KiB
Python
|
"""
|
|||
|
|
爬虫配置文件
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
import os
|
|||
|
|
from typing import Dict
|
|||
|
|
from pydantic_settings import BaseSettings
|
|||
|
|
from pydantic import Field
|
|||
|
|
|
|||
|
|
|
|||
|
|
class Settings(BaseSettings):
|
|||
|
|
"""应用配置"""
|
|||
|
|
|
|||
|
|
# 爬虫配置
|
|||
|
|
REQUEST_TIMEOUT: int = Field(default=30, description="请求超时时间（秒）")
|
|||
|
|
RETRY_TIMES: int = Field(default=3, description="重试次数")
|
|||
|
|
CONCURRENT_REQUESTS: int = Field(default=5, description="并发请求数")
|
|||
|
|
DOWNLOAD_DELAY: float = Field(default=1.0, description="下载延迟（秒）")
|
|||
|
|
|
|||
|
|
# 代理配置
|
|||
|
|
USE_PROXY: bool = Field(default=False, description="是否使用代理")
|
|||
|
|
PROXY_URL: str = Field(default="", description="代理地址")
|
|||
|
|
|
|||
|
|
# 数据库配置（用于存储爬取的数据）
|
|||
|
|
DB_HOST: str = Field(default="localhost", description="数据库主机")
|
|||
|
|
DB_PORT: int = Field(default=3306, description="数据库端口")
|
|||
|
|
DB_USER: str = Field(default="root", description="数据库用户名")
|
|||
|
|
DB_PASSWORD: str = Field(default="", description="数据库密码")
|
|||
|
|
DB_NAME: str = Field(default="school_news", description="数据库名称")
|
|||
|
|
|
|||
|
|
# 日志配置
|
|||
|
|
LOG_LEVEL: str = Field(default="INFO", description="日志级别")
|
|||
|
|
LOG_DIR: str = Field(default="logs", description="日志目录")
|
|||
|
|
LOG_RETENTION: str = Field(default="30 days", description="日志保留时间")
|
|||
|
|
|
|||
|
|
# 输出配置
|
|||
|
|
OUTPUT_DIR: str = Field(default="output", description="输出目录")
|
|||
|
|
OUTPUT_FORMAT: str = Field(default="json", description="输出格式(json/csv)")
|
|||
|
|
|
|||
|
|
# 人民日报配置
|
|||
|
|
RMRB_BASE_URL: str = Field(default="http://www.people.com.cn", description="人民日报基础URL")
|
|||
|
|
RMRB_CATEGORIES: Dict[str, str] = Field(
|
|||
|
|
default_factory=lambda: {
|
|||
|
|
"politics": "时政",
|
|||
|
|
"society": "社会",
|
|||
|
|
"world": "国际",
|
|||
|
|
"finance": "财经",
|
|||
|
|
"tech": "科技",
|
|||
|
|
"culture": "文化",
|
|||
|
|
"education": "教育"
|
|||
|
|
},
|
|||
|
|
description="人民日报新闻分类"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# API配置（用于将爬取的数据推送到后端）
|
|||
|
|
API_BASE_URL: str = Field(default="http://localhost:8080", description="后端API地址")
|
|||
|
|
API_TOKEN: str = Field(default="", description="API认证Token")
|
|||
|
|
|
|||
|
|
class Config:
|
|||
|
|
env_file = ".env"
|
|||
|
|
env_file_encoding = "utf-8"
|
|||
|
|
case_sensitive = True
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 创建全局配置实例
|
|||
|
|
settings = Settings()
|
|||
|
|
|
|||
|
|
|
|||
|
|
# 常用User-Agent列表
|
|||
|
|
USER_AGENTS = [
|
|||
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|||
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',
|
|||
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|||
|
|
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:121.0) Gecko/20100101 Firefox/121.0',
|
|||
|
|
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.1 Safari/605.1.15',
|
|||
|
|
]
|
|||
|
|
|