306 lines
11 KiB
Python
306 lines
11 KiB
Python
|
# -*- coding: utf-8 -*-
|
|||
|
|
"""
|
|||
|
|
浙江省公共资源交易中心爬虫 —— 基于 API + requests
|
|||
|
|
"""
|
|||
|
|
import json
|
|||
|
|
import logging
|
|||
|
|
import os
|
|||
|
|
import re
|
|||
|
|
from datetime import datetime, timedelta
|
|||
|
|
from .base import BaseSpider
|
|||
|
|
from utils.attachment import AttachmentHandler
|
|||
|
|
|
|||
|
|
logger = logging.getLogger("ztb")
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ZhejiangSpider(BaseSpider):
|
|||
|
|
"""浙江省公共资源交易中心爬虫"""
|
|||
|
|
|
|||
|
|
# ---------- API 列表 ----------
|
|||
|
|
|
|||
|
|
def _build_payload(self, page_index: int, page_size: int,
|
|||
|
|
category_code: str, notice_code: str,
|
|||
|
|
start_date: str, end_date: str) -> dict:
|
|||
|
|
"""构建浙江省 API 请求体"""
|
|||
|
|
condition = []
|
|||
|
|
if notice_code:
|
|||
|
|
condition.append({
|
|||
|
|
"fieldName": "categorynum",
|
|||
|
|
"isLike": True,
|
|||
|
|
"likeType": 2,
|
|||
|
|
"equal": notice_code,
|
|||
|
|
})
|
|||
|
|
elif category_code:
|
|||
|
|
condition.append({
|
|||
|
|
"fieldName": "categorynum",
|
|||
|
|
"isLike": True,
|
|||
|
|
"likeType": 2,
|
|||
|
|
"equal": category_code,
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
time_cond = []
|
|||
|
|
if start_date and end_date:
|
|||
|
|
time_cond.append({
|
|||
|
|
"fieldName": "webdate",
|
|||
|
|
"startTime": f"{start_date} 00:00:00",
|
|||
|
|
"endTime": f"{end_date} 23:59:59",
|
|||
|
|
})
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"token": "",
|
|||
|
|
"pn": page_index * page_size,
|
|||
|
|
"rn": page_size,
|
|||
|
|
"sdt": "", "edt": "",
|
|||
|
|
"wd": "", "inc_wd": "", "exc_wd": "",
|
|||
|
|
"fields": "title",
|
|||
|
|
"cnum": "001",
|
|||
|
|
"sort": '{"webdate":"0"}',
|
|||
|
|
"ssort": "title",
|
|||
|
|
"cl": 5000,
|
|||
|
|
"terminal": "",
|
|||
|
|
"condition": condition or None,
|
|||
|
|
"time": time_cond or None,
|
|||
|
|
"highlights": "",
|
|||
|
|
"statistics": None,
|
|||
|
|
"unionCondition": None,
|
|||
|
|
"accuracy": "",
|
|||
|
|
"noParticiple": "0",
|
|||
|
|
"searchRange": None,
|
|||
|
|
"isBusiness": "1",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
def fetch_list_page(self, page_index: int, page_size: int,
|
|||
|
|
category_code: str, notice_code: str,
|
|||
|
|
start_date: str, end_date: str) -> list:
|
|||
|
|
"""通过 API 获取一页列表数据"""
|
|||
|
|
payload = self._build_payload(
|
|||
|
|
page_index, page_size, category_code, notice_code,
|
|||
|
|
start_date, end_date,
|
|||
|
|
)
|
|||
|
|
resp = self.fetch(
|
|||
|
|
self.config["api_url"],
|
|||
|
|
method="POST",
|
|||
|
|
json=payload,
|
|||
|
|
headers={"Referer": self.config["base_url"] + "/jyxxgk/list.html"},
|
|||
|
|
)
|
|||
|
|
if resp is None:
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
try:
|
|||
|
|
data = resp.json()
|
|||
|
|
return data.get("result", {}).get("records", [])
|
|||
|
|
except Exception as e:
|
|||
|
|
logger.error(f"解析 API 响应失败: {e}")
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
# ---------- 解析记录 ----------
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _parse_record(record: dict, source: str) -> dict:
|
|||
|
|
"""将 API 原始记录转换为结果字典"""
|
|||
|
|
title = record.get("title", "").strip()
|
|||
|
|
link = record.get("linkurl", "")
|
|||
|
|
if link and not link.startswith("http"):
|
|||
|
|
link = "https://ggzy.zj.gov.cn" + link
|
|||
|
|
|
|||
|
|
date_str = record.get("webdate", "")
|
|||
|
|
date_short = date_str.split(" ")[0] if date_str else ""
|
|||
|
|
|
|||
|
|
item = {
|
|||
|
|
"标题": title,
|
|||
|
|
"发布日期": date_short,
|
|||
|
|
"地区": record.get("infod", ""),
|
|||
|
|
"公告类型": record.get("categoryname", ""),
|
|||
|
|
"链接": link,
|
|||
|
|
"来源": source,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# 解析特定格式的标题：[招标文件]项目名称[批准文号]
|
|||
|
|
import re
|
|||
|
|
# 改进的正则表达式，确保正确匹配标题格式
|
|||
|
|
title_pattern = r"\[(?:招标文件|招标公告)\]\s*(.*?)\s*\[([A-Z0-9]+)\]\s*$"
|
|||
|
|
match = re.search(title_pattern, title)
|
|||
|
|
if match:
|
|||
|
|
project_name = match.group(1).strip()
|
|||
|
|
# 删除结尾的"招标文件公示"、"招标文件预公示"等后缀
|
|||
|
|
suffixes = ["招标文件公示", "招标文件预公示", "招标公告", "招标预公告"]
|
|||
|
|
for suffix in suffixes:
|
|||
|
|
if project_name.endswith(suffix):
|
|||
|
|
project_name = project_name[:-len(suffix)].strip()
|
|||
|
|
item["项目名称"] = project_name
|
|||
|
|
item["项目批准文号"] = match.group(2).strip()
|
|||
|
|
else:
|
|||
|
|
# 如果正则匹配失败，直接使用标题作为项目名称
|
|||
|
|
project_name = title
|
|||
|
|
# 删除结尾的"招标文件公示"、"招标文件预公示"等后缀
|
|||
|
|
suffixes = ["招标文件公示", "招标文件预公示", "招标公告", "招标预公告"]
|
|||
|
|
for suffix in suffixes:
|
|||
|
|
if project_name.endswith(suffix):
|
|||
|
|
project_name = project_name[:-len(suffix)].strip()
|
|||
|
|
# 尝试从标题中提取批准文号
|
|||
|
|
number_pattern = r"\[([A-Z0-9]+)\]\s*$"
|
|||
|
|
match = re.search(number_pattern, project_name)
|
|||
|
|
if match:
|
|||
|
|
item["项目批准文号"] = match.group(1).strip()
|
|||
|
|
# 从项目名称中删除批准文号部分
|
|||
|
|
project_name = project_name[:match.start()].strip()
|
|||
|
|
item["项目名称"] = project_name
|
|||
|
|
|
|||
|
|
return item
|
|||
|
|
|
|||
|
|
@staticmethod
|
|||
|
|
def _parse_content_fields(content: str) -> dict:
|
|||
|
|
"""从 API content 字段提取结构化信息"""
|
|||
|
|
if not content:
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
# 清理 HTML 实体
|
|||
|
|
import html as html_mod
|
|||
|
|
text = html_mod.unescape(content)
|
|||
|
|
text = re.sub(r"<[^>]+>", "", text) # 去 HTML 标签
|
|||
|
|
text = re.sub(r"\s+", " ", text).strip()
|
|||
|
|
|
|||
|
|
fields = {}
|
|||
|
|
patterns = {
|
|||
|
|
"项目名称": r"项目名称[::：]\s*(.+?)\s{2,}",
|
|||
|
|
"项目代码": r"项目代码[::：]\s*(.+?)\s{2,}",
|
|||
|
|
"招标人": r"招标人[::：].*?名称[::：]\s*(.+?)\s{2,}",
|
|||
|
|
"招标代理": r"代理机构[::：].*?名称[::：]\s*(.+?)\s{2,}",
|
|||
|
|
"联系电话": r"电\s*话[::：]\s*([\d\-]+)",
|
|||
|
|
"招标估算金额": r"招标估算金额[::：]\s*([\d,\.]+\s*元)",
|
|||
|
|
}
|
|||
|
|
for key, pat in patterns.items():
|
|||
|
|
m = re.search(pat, text)
|
|||
|
|
if m:
|
|||
|
|
fields[key] = m.group(1).strip()
|
|||
|
|
|
|||
|
|
return fields
|
|||
|
|
|
|||
|
|
# ---------- 附件 ----------
|
|||
|
|
|
|||
|
|
def _extract_attachments_from_detail(self, url: str) -> list:
|
|||
|
|
"""访问详情页，提取附件链接"""
|
|||
|
|
resp = self.fetch(url)
|
|||
|
|
if resp is None:
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
attachments = []
|
|||
|
|
# PDF
|
|||
|
|
for href in re.findall(r'href=["\']([^"\']*\.pdf[^"\']*)', resp.text):
|
|||
|
|
if not href.startswith("http"):
|
|||
|
|
href = self.config["base_url"] + href
|
|||
|
|
name = href.split("/")[-1]
|
|||
|
|
attachments.append({"name": name, "url": href})
|
|||
|
|
# Word
|
|||
|
|
for href in re.findall(r'href=["\']([^"\']*\.docx?[^"\']*)', resp.text):
|
|||
|
|
if not href.startswith("http"):
|
|||
|
|
href = self.config["base_url"] + href
|
|||
|
|
name = href.split("/")[-1]
|
|||
|
|
attachments.append({"name": name, "url": href})
|
|||
|
|
|
|||
|
|
return attachments
|
|||
|
|
|
|||
|
|
# ---------- 主流程 ----------
|
|||
|
|
|
|||
|
|
def crawl(self, max_pages: int = None, category: str = None,
|
|||
|
|
notice_type: str = None, date_filter: str = None,
|
|||
|
|
download_attachment: bool = False, **kwargs):
|
|||
|
|
"""
|
|||
|
|
执行爬取
|
|||
|
|
|
|||
|
|
Args:
|
|||
|
|
max_pages: 最大爬取页数
|
|||
|
|
category: 交易领域（如 "工程建设"）
|
|||
|
|
notice_type: 公告类型（如 "招标公告"）
|
|||
|
|
date_filter: 日期过滤（"yesterday" 或 "2026-02-03"）
|
|||
|
|
download_attachment: 是否下载附件
|
|||
|
|
"""
|
|||
|
|
if max_pages is None:
|
|||
|
|
max_pages = self.spider_config.get("max_pages", 10)
|
|||
|
|
page_size = self.spider_config.get("page_size", 20)
|
|||
|
|
|
|||
|
|
# 日期范围
|
|||
|
|
if date_filter == "yesterday":
|
|||
|
|
d = datetime.now() - timedelta(days=1)
|
|||
|
|
start_date = end_date = d.strftime("%Y-%m-%d")
|
|||
|
|
logger.info(f"过滤日期: {start_date}（昨天）")
|
|||
|
|
elif date_filter:
|
|||
|
|
start_date = end_date = date_filter
|
|||
|
|
logger.info(f"过滤日期: {start_date}")
|
|||
|
|
else:
|
|||
|
|
# 默认近一个月
|
|||
|
|
end_date = datetime.now().strftime("%Y-%m-%d")
|
|||
|
|
start_date = (datetime.now() - timedelta(days=30)).strftime("%Y-%m-%d")
|
|||
|
|
|
|||
|
|
category_code = self.config.get("categories", {}).get(category, "")
|
|||
|
|
notice_code = self.config.get("notice_types", {}).get(notice_type, "")
|
|||
|
|
|
|||
|
|
# 附件处理器
|
|||
|
|
attachment_handler = None
|
|||
|
|
if download_attachment:
|
|||
|
|
attachment_dir = os.path.join(self.data_dir, "attachments")
|
|||
|
|
attachment_handler = AttachmentHandler(attachment_dir)
|
|||
|
|
logger.info(f"启用附件下载，保存到: {attachment_dir}")
|
|||
|
|
|
|||
|
|
logger.info(f"开始爬取: {self.config['name']}")
|
|||
|
|
if category:
|
|||
|
|
logger.info(f"交易领域: {category}")
|
|||
|
|
if notice_type:
|
|||
|
|
logger.info(f"公告类型: {notice_type}")
|
|||
|
|
|
|||
|
|
for page_idx in range(max_pages):
|
|||
|
|
if self._check_limits():
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
logger.info(f"正在爬取第 {page_idx + 1} 页...")
|
|||
|
|
records = self.fetch_list_page(
|
|||
|
|
page_idx, page_size, category_code, notice_code,
|
|||
|
|
start_date, end_date,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
if not records:
|
|||
|
|
logger.info("没有更多数据")
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
count = 0
|
|||
|
|
for rec in records:
|
|||
|
|
link = rec.get("linkurl", "")
|
|||
|
|
if link and not link.startswith("http"):
|
|||
|
|
link = self.config["base_url"] + link
|
|||
|
|
if self.is_duplicate(link):
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
item = self._parse_record(rec, self.config["name"])
|
|||
|
|
# 从 content 提取详情字段
|
|||
|
|
detail = self._parse_content_fields(rec.get("content", ""))
|
|||
|
|
item.update(detail)
|
|||
|
|
|
|||
|
|
# 附件
|
|||
|
|
if download_attachment and attachment_handler:
|
|||
|
|
self.detail_delay()
|
|||
|
|
atts = self._extract_attachments_from_detail(link)
|
|||
|
|
if atts:
|
|||
|
|
item["附件数量"] = len(atts)
|
|||
|
|
att_names = []
|
|||
|
|
for att in atts:
|
|||
|
|
att_names.append(att["name"])
|
|||
|
|
result = attachment_handler.download_and_extract(att["url"])
|
|||
|
|
if result["success"] and result["text"]:
|
|||
|
|
item["附件内容摘要"] = result["text"][:2000]
|
|||
|
|
item["附件名称"] = " | ".join(att_names)
|
|||
|
|
|
|||
|
|
self.results.append(item)
|
|||
|
|
count += 1
|
|||
|
|
|
|||
|
|
logger.info(f" 获取 {count} 条数据")
|
|||
|
|
|
|||
|
|
if count == 0:
|
|||
|
|
logger.info("当前页无新数据，停止翻页")
|
|||
|
|
break
|
|||
|
|
|
|||
|
|
self.delay()
|
|||
|
|
|
|||
|
|
self.print_stats()
|
|||
|
|
logger.info(f"爬取完成，共 {len(self.results)} 条数据")
|
|||
|
|
return self.results
|