104 lines
3.9 KiB
Python
104 lines
3.9 KiB
Python
|
"""
|
|||
|
|
清理旧数据 + 补充缺失的AI总结
|
|||
|
|
"""
|
|||
|
|
import asyncio
|
|||
|
|
import sys
|
|||
|
|
sys.path.insert(0, ".")
|
|||
|
|
|
|||
|
|
from app.database import AsyncSessionLocal, init_db
|
|||
|
|
from app.models import Report, AnalysisResult, ExtractedContent
|
|||
|
|
from app.services.ai_analyzer import ai_analyzer
|
|||
|
|
from sqlalchemy import select, delete
|
|||
|
|
from sqlalchemy.orm import selectinload
|
|||
|
|
|
|||
|
|
async def cleanup_and_fix():
|
|||
|
|
print("=" * 60)
|
|||
|
|
print("数据清理和修复")
|
|||
|
|
print("=" * 60)
|
|||
|
|
|
|||
|
|
await init_db()
|
|||
|
|
|
|||
|
|
async with AsyncSessionLocal() as db:
|
|||
|
|
# 1. 删除2024年之前的报告（保留2024和2025年）
|
|||
|
|
print("\n1. 清理2024年之前的旧数据...")
|
|||
|
|
|
|||
|
|
stmt = select(Report).where(Report.report_year < 2024)
|
|||
|
|
result = await db.execute(stmt)
|
|||
|
|
old_reports = result.scalars().all()
|
|||
|
|
|
|||
|
|
print(f" 找到 {len(old_reports)} 份2024年前的报告")
|
|||
|
|
|
|||
|
|
for report in old_reports:
|
|||
|
|
# 删除关联数据
|
|||
|
|
await db.execute(delete(ExtractedContent).where(ExtractedContent.report_id == report.id))
|
|||
|
|
await db.execute(delete(AnalysisResult).where(AnalysisResult.report_id == report.id))
|
|||
|
|
await db.delete(report)
|
|||
|
|
|
|||
|
|
await db.commit()
|
|||
|
|
print(f" ✓ 已删除 {len(old_reports)} 份旧报告")
|
|||
|
|
|
|||
|
|
# 2. 为缺失AI总结的报告补充汇总
|
|||
|
|
print("\n2. 为缺失AI总结的报告生成汇总...")
|
|||
|
|
|
|||
|
|
stmt = select(Report).where(
|
|||
|
|
Report.is_extracted == True,
|
|||
|
|
Report.report_year >= 2024
|
|||
|
|
).options(
|
|||
|
|
selectinload(Report.company),
|
|||
|
|
selectinload(Report.analysis_results)
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
result = await db.execute(stmt)
|
|||
|
|
reports = result.scalars().all()
|
|||
|
|
|
|||
|
|
missing_summary_count = 0
|
|||
|
|
fixed_count = 0
|
|||
|
|
|
|||
|
|
for report in reports:
|
|||
|
|
# 检查是否缺少summary
|
|||
|
|
has_summary = any(r.analysis_type == "summary" for r in report.analysis_results)
|
|||
|
|
section_analyses = [r for r in report.analysis_results if r.analysis_type == "section"]
|
|||
|
|
|
|||
|
|
if not has_summary and section_analyses:
|
|||
|
|
missing_summary_count += 1
|
|||
|
|
print(f"\n [{missing_summary_count}] 修复: {report.company.short_name} - {report.title[:20]}...")
|
|||
|
|
|
|||
|
|
# 生成汇总
|
|||
|
|
valid_results = [{
|
|||
|
|
"section_name": s.section_name,
|
|||
|
|
"analysis": s.summary,
|
|||
|
|
"success": True
|
|||
|
|
} for s in section_analyses]
|
|||
|
|
|
|||
|
|
company_name = report.company.short_name or report.company.company_name
|
|||
|
|
summary = await ai_analyzer.summarize_analyses(valid_results, company_name, report.title)
|
|||
|
|
|
|||
|
|
if summary.get("success"):
|
|||
|
|
final_analysis = AnalysisResult(
|
|||
|
|
report_id=report.id,
|
|||
|
|
analysis_type="summary",
|
|||
|
|
section_name="综合分析",
|
|||
|
|
ai_model=ai_analyzer.model,
|
|||
|
|
summary=summary["summary"],
|
|||
|
|
token_count=summary.get("tokens", 0),
|
|||
|
|
is_final=True
|
|||
|
|
)
|
|||
|
|
db.add(final_analysis)
|
|||
|
|
await db.commit()
|
|||
|
|
fixed_count += 1
|
|||
|
|
print(f" ✓ 已生成AI总结")
|
|||
|
|
else:
|
|||
|
|
print(f" ✗ 生成失败")
|
|||
|
|
|
|||
|
|
print(f"\n修复完成: 共 {fixed_count}/{missing_summary_count} 份报告已补充AI总结")
|
|||
|
|
|
|||
|
|
# 3. 统计当前数据
|
|||
|
|
print("\n3. 当前数据统计:")
|
|||
|
|
stmt = select(Report).where(Report.report_year >= 2024)
|
|||
|
|
result = await db.execute(stmt)
|
|||
|
|
current_reports = result.scalars().all()
|
|||
|
|
print(f" 2024年及以后的报告: {len(current_reports)} 份")
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
asyncio.run(cleanup_and_fix())
|