101 lines
3.4 KiB
Python
101 lines
3.4 KiB
Python
# -*- coding: utf-8 -*-
|
||
"""
|
||
测试单条数据处理
|
||
"""
|
||
import logging
|
||
import sys
|
||
import os
|
||
|
||
# 添加当前目录到模块搜索路径
|
||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
# 导入配置和处理器
|
||
from config import REGION_CONFIGS, PROCESSING_CONFIG
|
||
from spiders.zhejiang import ZhejiangSpider
|
||
from processors.pipeline import ProcessingPipeline
|
||
|
||
# 配置日志
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s - %(levelname)s - %(message)s'
|
||
)
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 测试URL
|
||
TEST_URL = "https://ggzy.zj.gov.cn/jyxxgk/002001/002001011/20260212/9a7966d8-80f4-475b-897e-f7631bc64d0c.html"
|
||
|
||
# 模拟爬虫结果
|
||
TEST_ITEM = {
|
||
"标题": "[招标文件]（测-试）临海市房建施工0212-2招标文件公示[A3300000090000695005001]",
|
||
"发布日期": "2026-02-12",
|
||
"地区": "临海市",
|
||
"公告类型": "招标文件公示",
|
||
"链接": TEST_URL,
|
||
"来源": "浙江省公共资源交易中心"
|
||
}
|
||
|
||
def main():
|
||
"""主函数"""
|
||
logger.info("开始单条数据测试")
|
||
|
||
# 1. 测试标题解析
|
||
from spiders.zhejiang import ZhejiangSpider
|
||
# 模拟标题解析
|
||
import re
|
||
title = TEST_ITEM["标题"]
|
||
# 使用修复后的正则表达式
|
||
title_pattern = r"\[(?:招标文件|招标公告)\]\s*(.*?)\s*\[([A-Z0-9]+)\]\s*$"
|
||
match = re.search(title_pattern, title)
|
||
if match:
|
||
project_name = match.group(1).strip()
|
||
project_approval = match.group(2).strip()
|
||
logger.info(f"标题解析结果:")
|
||
logger.info(f" 项目名称: {project_name}")
|
||
logger.info(f" 项目批准文号: {project_approval}")
|
||
else:
|
||
logger.warning("标题解析失败")
|
||
|
||
# 2. 测试处理管道
|
||
logger.info("\n测试处理管道:")
|
||
pipeline = ProcessingPipeline()
|
||
|
||
# 模拟ZhejiangSpider的处理过程，添加项目名称和项目批准文号
|
||
test_item_with_fields = TEST_ITEM.copy()
|
||
# 使用修复后的标题解析
|
||
title = test_item_with_fields["标题"]
|
||
title_pattern = r"\[(?:招标文件|招标公告)\]\s*(.*?)\s*\[([A-Z0-9]+)\]\s*$"
|
||
match = re.search(title_pattern, title)
|
||
if match:
|
||
test_item_with_fields["项目名称"] = match.group(1).strip()
|
||
test_item_with_fields["项目批准文号"] = match.group(2).strip()
|
||
|
||
logger.info(f"添加字段后的测试项:")
|
||
logger.info(f" 项目名称: {test_item_with_fields.get('项目名称', '无')}")
|
||
logger.info(f" 项目批准文号: {test_item_with_fields.get('项目批准文号', '无')}")
|
||
|
||
# 模拟爬虫结果列表
|
||
results = [test_item_with_fields]
|
||
|
||
# 处理结果
|
||
processed = pipeline.process_results(
|
||
results,
|
||
site="zhejiang",
|
||
notice_type="招标文件公示",
|
||
upload=False
|
||
)
|
||
|
||
# 分析结果
|
||
if processed:
|
||
record = processed[0]
|
||
logger.info("\n处理结果:")
|
||
logger.info(f" 项目名称: {record.get('项目名称', '文档未提及')}")
|
||
logger.info(f" 项目批准文号: {record.get('项目批准文号', '文档未提及')}")
|
||
logger.info(f" 批准文号: {record.get('批准文号', '文档未提及')}")
|
||
logger.info(f" 最高投标限价: {record.get('最高投标限价', '文档未提及')}")
|
||
logger.info(f" 最高限价: {record.get('最高限价', '文档未提及')}")
|
||
else:
|
||
logger.error("处理失败")
|
||
|
||
if __name__ == "__main__":
|
||
main()
|