后端设计治理：批次 0-4 全部完成（安全/部署/一致性/结构/架构）
按 ROADMAP §3.1 治理批次推进的后端设计审查整改：
- 批次 0（安全）：/api/status/{task_id} 补 JWT 鉴权与任务归属校验；
pythonocc_available 真实探测；bcrypt 超 72 字节显式拒绝；
SECRET_KEY/RUSTFS_* 惰性校验，代码侧弱默认移除
- 批次 1（部署正确性）：主处理链路改走 RustFS（分派入参 stp_file_id 化，
worker 按 object_key 下载）；AUTO_MIGRATE 开关 + 迁移目录 alembic/→migrations/
修复包遮蔽（自动迁移此前从未真正生效）；OCC 镜像改 conda 原生执行 +
基础镜像 tag 锁定；compose 关键项改 ${VAR:?} 强制显式配置
- 批次 2（任务一致性）：删除 Redis 进程内存回退，PG 为任务状态单一事实源；
批量元数据入库（processing_tasks.batch_id，迁移 a3f8c2d91e47）；
型腔失败任务标 failed 不再静默 completed；事务边界收口
（数据本体写 flush-only、失败先回滚再置 failed、进度更新保留即时 commit）
- 批次 3（API 与代码结构）：592 行 advanced_router 拆为 design/cost/machining/
export 四子路由，请求体全量 Pydantic 化；ROUTE_MODULES + route_registry
（/api/health 呈现 degraded，DEBUG fail fast）；纯计算端点统一 to_thread；
StorageIntegrationService 按职责三拆；MAX_FILE_SIZE 接线生效、
celery 复用 Settings.redis_url；管理员重置密码改 JSON body（端到端断裂修复）；
openapi.json 重导出（76 paths）+ 前端 gen:api
- 批次 4（架构演进）：共享 ORM 按模块拆分（shared/models/base.py + identity.py、
moldinsight/models/、inventory/models/，删除三条无使用方的跨模块
relationship，跨模块桥接收敛为裸 FK 硬规则，无兼容 facade）；
OCC executor 重建补 cancel_futures=True（消除旧队列被慢恢复线程
并行消化的数据竞争）；OCC 吞吐方案设计先行
（docs/topics/performance/OCC_THROUGHPUT.md）；顺手清偿 D15
（vite.config.ts 未用参数致 npm run build 失败）
测试基线：125 passed, 2 skipped（pytest + sqlite+aiosqlite；归属边界、
路由契约、配置治理、鉴权回归等随批新增）
文档同步：STATUS / TECH_DEBT / ROADMAP / ARCHITECTURE / API_CONTRACT /
OPERATIONS / AGENTS
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
+15
-363
@@ -1,27 +1,25 @@
|
||||
# services/storage_integration_rustfs.py
|
||||
"""存储集成服务 - 协调 PostgreSQL 和 RustFS"""
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy import select, update
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
import json
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
# services/analysis_storage_service.py
|
||||
"""分析结果数据存储——几何/网格/型腔/HTML/特征的 RustFS 上传与 PG 元数据，
|
||||
以及任务完整数据视图的组装。
|
||||
|
||||
from shared.models.database import (
|
||||
STPFile, GeometryData, MeshData, MoldCavityData,
|
||||
HTMLFile, ProcessingTask, User,
|
||||
FeatureDetection, DesignRecommendation,
|
||||
UserActivity, SystemLog
|
||||
)
|
||||
批次 3 自 storage_integration_rustfs.py 按职责拆分（见 task_storage_service.py 头注）。
|
||||
"""
|
||||
import json
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from moldinsight.models import STPFile, GeometryData, MeshData, MoldCavityData, HTMLFile, FeatureDetection, DesignRecommendation
|
||||
from moldinsight.storage.rustfs_storage import rustfs_manager
|
||||
from shared.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class StorageIntegrationService:
|
||||
"""存储集成服务 - PostgreSQL + RustFS"""
|
||||
class AnalysisStorageService:
|
||||
"""分析结果数据（几何/网格/型腔/HTML/特征）存储与视图组装"""
|
||||
|
||||
@staticmethod
|
||||
def _resolve_best_scheme_payload(cavity_json: Dict[str, Any]) -> Dict[str, Any]:
|
||||
@@ -80,169 +78,6 @@ class StorageIntegrationService:
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
async def save_stp_file(self, session: AsyncSession,
|
||||
file_path: Path,
|
||||
original_filename: str,
|
||||
user_id: Optional[int] = None,
|
||||
upload_batch: Optional[str] = None) -> STPFile:
|
||||
"""保存STP文件到PostgreSQL元数据 + RustFS对象存储
|
||||
|
||||
支持同一文件多次上传，每次上传都会创建新记录
|
||||
"""
|
||||
|
||||
# 1. 上传到RustFS
|
||||
upload_result = await rustfs_manager.upload_file(
|
||||
file_type='stp_files',
|
||||
file_path=file_path,
|
||||
original_filename=original_filename,
|
||||
metadata={
|
||||
'original_filename': original_filename,
|
||||
'user_id': str(user_id) if user_id else 'anonymous',
|
||||
'upload_batch': upload_batch or str(uuid.uuid4())
|
||||
}
|
||||
)
|
||||
|
||||
file_hash = upload_result['file_hash']
|
||||
batch_id = upload_batch or str(uuid.uuid4())
|
||||
|
||||
# 2. 创建新PostgreSQL记录（每次上传都创建新记录）
|
||||
from datetime import datetime
|
||||
stp_file = STPFile(
|
||||
user_id=user_id,
|
||||
object_key=upload_result['object_key'],
|
||||
storage_bucket=upload_result['bucket'],
|
||||
original_filename=original_filename,
|
||||
file_size=upload_result['file_size'],
|
||||
file_hash=file_hash,
|
||||
upload_batch=batch_id,
|
||||
status="uploaded",
|
||||
file_path=str(file_path),
|
||||
upload_time=datetime.now()
|
||||
)
|
||||
|
||||
session.add(stp_file)
|
||||
# D9：仅 flush，与 ProcessingTask 由路由层一并原子提交（避免孤儿文件记录）
|
||||
await session.flush()
|
||||
await session.refresh(stp_file)
|
||||
|
||||
logger.info(f"STP文件保存成功 RustFS: {stp_file.id}, 批次: {batch_id}")
|
||||
return stp_file
|
||||
|
||||
async def create_processing_task(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
task_id: str,
|
||||
stp_file_id: int,
|
||||
task_type: str = "stp_parsing",
|
||||
parameters: Optional[Dict[str, Any]] = None,
|
||||
batch_id: Optional[str] = None,
|
||||
) -> ProcessingTask:
|
||||
"""创建处理任务记录（D9：仅 flush 不 commit，事务由调用方收口——
|
||||
与 STPFile 记录同批提交，避免留下无任务的孤儿文件记录；batch_id 用于批量任务聚合查询）
|
||||
"""
|
||||
try:
|
||||
task = ProcessingTask(
|
||||
task_id=task_id,
|
||||
stp_file_id=stp_file_id,
|
||||
task_type=task_type,
|
||||
status="pending",
|
||||
started_time=datetime.now(),
|
||||
parameters=parameters or {},
|
||||
batch_id=batch_id,
|
||||
)
|
||||
|
||||
session.add(task)
|
||||
await session.flush()
|
||||
|
||||
logger.info(f"处理任务创建成功: {task_id}")
|
||||
return task
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"创建处理任务失败: {e}")
|
||||
raise
|
||||
|
||||
async def update_task_status(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
task_id: str,
|
||||
status: str,
|
||||
progress: Optional[int] = None,
|
||||
current_step: Optional[str] = None,
|
||||
error_message: Optional[str] = None
|
||||
):
|
||||
"""更新任务状态（保留即时 commit：进度/状态需跨事务对外可见，
|
||||
处理链路中的各阶段进度依赖它落库——D9 收口仅针对数据本体写方法）"""
|
||||
try:
|
||||
update_data = {
|
||||
"status": status,
|
||||
"completed_time": datetime.now() if status in ["completed", "failed"] else None,
|
||||
"error_message": error_message
|
||||
}
|
||||
|
||||
if progress is not None:
|
||||
update_data["progress"] = progress
|
||||
if current_step is not None:
|
||||
update_data["current_step"] = current_step
|
||||
|
||||
await session.execute(
|
||||
update(ProcessingTask)
|
||||
.where(ProcessingTask.task_id == task_id)
|
||||
.values(**update_data)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"任务状态更新: {task_id} -> {status}")
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"更新任务状态失败: {e}")
|
||||
raise
|
||||
|
||||
async def update_task_parameters(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
task_id: str,
|
||||
parameters: Dict[str, Any],
|
||||
):
|
||||
"""合并更新任务参数，便于保存阶段耗时等元数据。（D9：flush 不 commit，事务由调用方收口）"""
|
||||
try:
|
||||
task = await session.execute(
|
||||
select(ProcessingTask).where(ProcessingTask.task_id == task_id)
|
||||
)
|
||||
task = task.scalar_one_or_none()
|
||||
if task is None:
|
||||
return
|
||||
|
||||
merged = dict(task.parameters or {})
|
||||
merged.update(parameters or {})
|
||||
task.parameters = merged
|
||||
await session.flush()
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"更新任务参数失败: {e}")
|
||||
raise
|
||||
|
||||
async def update_stp_file_status(self, session: AsyncSession, stp_file_id: int, status: str):
|
||||
"""更新STP文件状态（保留即时 commit，理由同 update_task_status）"""
|
||||
try:
|
||||
await session.execute(
|
||||
update(STPFile)
|
||||
.where(STPFile.id == stp_file_id)
|
||||
.values(
|
||||
status=status,
|
||||
processed_time=datetime.now() if status in ["completed", "failed"] else None
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"STP文件状态更新: ID {stp_file_id} -> {status}")
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"更新STP文件状态失败: {e}")
|
||||
raise
|
||||
|
||||
async def save_geometry_data(self, session: AsyncSession,
|
||||
stp_file_id: int,
|
||||
geometry_json: Dict[str, Any],
|
||||
@@ -517,37 +352,9 @@ class StorageIntegrationService:
|
||||
await session.flush()
|
||||
logger.info(f"保存了 {len(features)} 个特征和 {len(recommendations)} 个建议")
|
||||
|
||||
async def log_user_activity(self, session: AsyncSession,
|
||||
user_id: int,
|
||||
activity_type: str,
|
||||
resource_type: Optional[str] = None,
|
||||
resource_id: Optional[int] = None,
|
||||
description: Optional[str] = None,
|
||||
metadata: Optional[Dict] = None,
|
||||
ip_address: Optional[str] = None,
|
||||
user_agent: Optional[str] = None):
|
||||
"""记录用户活动"""
|
||||
|
||||
activity = UserActivity(
|
||||
user_id=user_id,
|
||||
activity_type=activity_type,
|
||||
resource_type=resource_type,
|
||||
resource_id=resource_id,
|
||||
description=description,
|
||||
meta_data=metadata,
|
||||
ip_address=ip_address,
|
||||
user_agent=user_agent
|
||||
)
|
||||
|
||||
session.add(activity)
|
||||
await session.commit()
|
||||
logger.debug(f"用户活动记录: {activity_type} by user {user_id}")
|
||||
|
||||
async def get_stp_file_with_data(self, session: AsyncSession,
|
||||
stp_file_id: int) -> Dict[str, Any]:
|
||||
"""获取STP文件及其所有关联数据"""
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
try:
|
||||
# 1. 获取STP文件记录（使用 joinedload 预加载关联数据）
|
||||
result = await session.execute(
|
||||
@@ -666,157 +473,6 @@ class StorageIntegrationService:
|
||||
|
||||
return result
|
||||
|
||||
async def get_file_history_by_filename(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
filename: str,
|
||||
user_id: Optional[int] = None,
|
||||
limit: int = 50
|
||||
) -> list:
|
||||
"""获取同一文件名的所有上传历史记录"""
|
||||
from shared.models.database import ProcessingTask
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
query = select(STPFile).options(
|
||||
joinedload(STPFile.processing_tasks)
|
||||
).where(
|
||||
STPFile.original_filename == filename
|
||||
).order_by(STPFile.upload_time.desc())
|
||||
|
||||
if user_id:
|
||||
query = query.where(STPFile.user_id == user_id)
|
||||
|
||||
query = query.limit(limit)
|
||||
|
||||
result = await session.execute(query)
|
||||
files = result.unique().scalars().all()
|
||||
|
||||
return [
|
||||
{
|
||||
'id': f.id,
|
||||
'task_id': f.processing_tasks[0].task_id if f.processing_tasks else None,
|
||||
'upload_batch': f.upload_batch,
|
||||
'upload_time': f.upload_time.strftime('%Y-%m-%d %H:%M:%S') if f.upload_time else None,
|
||||
'file_size': f.file_size,
|
||||
'status': f.status,
|
||||
'volume': f.volume,
|
||||
'surface_area': f.surface_area,
|
||||
'product_weight': f.product_weight,
|
||||
'has_analysis': f.status == 'completed'
|
||||
}
|
||||
for f in files
|
||||
]
|
||||
|
||||
async def get_all_file_groups(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
user_id: Optional[int] = None,
|
||||
limit: int = 100
|
||||
) -> list:
|
||||
"""获取所有文件分组（按文件名分组），包含每个文件的最新分析结果"""
|
||||
|
||||
from sqlalchemy import func, desc
|
||||
from sqlalchemy.orm import joinedload
|
||||
from shared.models.database import ProcessingTask
|
||||
|
||||
# 子查询：获取每个文件名的最新上传
|
||||
subquery = (
|
||||
select(
|
||||
STPFile.original_filename,
|
||||
func.max(STPFile.upload_time).label('latest_upload')
|
||||
)
|
||||
.group_by(STPFile.original_filename)
|
||||
.order_by(desc('latest_upload'))
|
||||
.limit(limit)
|
||||
)
|
||||
|
||||
if user_id:
|
||||
subquery = subquery.where(STPFile.user_id == user_id)
|
||||
|
||||
subquery = subquery.subquery()
|
||||
|
||||
# 主查询：获取最新记录和统计信息
|
||||
query = (
|
||||
select(STPFile).options(
|
||||
joinedload(STPFile.processing_tasks)
|
||||
)
|
||||
.join(
|
||||
subquery,
|
||||
(STPFile.original_filename == subquery.c.original_filename) &
|
||||
(STPFile.upload_time == subquery.c.latest_upload)
|
||||
)
|
||||
.order_by(STPFile.upload_time.desc())
|
||||
)
|
||||
|
||||
result = await session.execute(query)
|
||||
latest_files = result.unique().scalars().all()
|
||||
|
||||
# 一次性聚合每个文件名的上传次数（替代逐文件 count 的 N+1 查询）
|
||||
count_subquery = (
|
||||
select(STPFile.original_filename, func.count().label("upload_count"))
|
||||
.group_by(STPFile.original_filename)
|
||||
)
|
||||
if user_id:
|
||||
count_subquery = count_subquery.where(STPFile.user_id == user_id)
|
||||
count_result = await session.execute(count_subquery)
|
||||
upload_counts = {
|
||||
row.original_filename: row.upload_count for row in count_result
|
||||
}
|
||||
|
||||
# 获取每个文件名的上传次数
|
||||
file_groups = []
|
||||
for f in latest_files:
|
||||
task_id = f.processing_tasks[0].task_id if f.processing_tasks else None
|
||||
upload_count = upload_counts.get(f.original_filename, 1)
|
||||
|
||||
file_groups.append({
|
||||
'filename': f.original_filename,
|
||||
'latest_id': f.id,
|
||||
'latest_task_id': task_id,
|
||||
'latest_upload_time': f.upload_time.strftime('%Y-%m-%d %H:%M:%S') if f.upload_time else None,
|
||||
'latest_status': f.status,
|
||||
'upload_count': upload_count,
|
||||
'file_size': f.file_size,
|
||||
'volume': f.volume,
|
||||
'surface_area': f.surface_area,
|
||||
'product_weight': f.product_weight
|
||||
})
|
||||
|
||||
return file_groups
|
||||
|
||||
async def update_stp_file_analysis_summary(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
stp_file_id: int,
|
||||
volume: Optional[float] = None,
|
||||
surface_area: Optional[float] = None,
|
||||
product_weight: Optional[float] = None
|
||||
):
|
||||
"""更新STP文件的分析摘要字段（用于快速查询）"""
|
||||
try:
|
||||
update_data = {}
|
||||
if volume is not None:
|
||||
update_data['volume'] = volume
|
||||
if surface_area is not None:
|
||||
update_data['surface_area'] = surface_area
|
||||
if product_weight is not None:
|
||||
update_data['product_weight'] = product_weight
|
||||
|
||||
if update_data:
|
||||
await session.execute(
|
||||
update(STPFile)
|
||||
.where(STPFile.id == stp_file_id)
|
||||
.values(**update_data)
|
||||
)
|
||||
# D9：flush 不 commit，随结果包由编排层统一提交
|
||||
await session.flush()
|
||||
logger.info(f"STP文件分析摘要更新: ID {stp_file_id}")
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"更新STP文件分析摘要失败: {e}")
|
||||
raise
|
||||
|
||||
async def delete_stp_file_cascade(self, session: AsyncSession,
|
||||
stp_file_id: int):
|
||||
"""级联删除STP文件及其所有关联数据"""
|
||||
@@ -861,7 +517,3 @@ class StorageIntegrationService:
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"STP文件及其关联数据已删除: {stp_file_id}")
|
||||
|
||||
|
||||
# 全局存储集成服务实例
|
||||
storage_integration = StorageIntegrationService()
|
||||
@@ -0,0 +1,131 @@
|
||||
# services/file_history_service.py
|
||||
"""文件历史查询视图——按文件名分组的多版本上传历史。
|
||||
|
||||
批次 3 自 storage_integration_rustfs.py 按职责拆分（见 task_storage_service.py 头注）。
|
||||
"""
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import select, func, desc
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from moldinsight.models import STPFile, ProcessingTask
|
||||
from shared.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class FileHistoryService:
|
||||
"""按文件名聚合的上传历史查询"""
|
||||
|
||||
async def get_file_history_by_filename(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
filename: str,
|
||||
user_id: Optional[int] = None,
|
||||
limit: int = 50
|
||||
) -> list:
|
||||
"""获取同一文件名的所有上传历史记录"""
|
||||
|
||||
query = select(STPFile).options(
|
||||
joinedload(STPFile.processing_tasks)
|
||||
).where(
|
||||
STPFile.original_filename == filename
|
||||
).order_by(STPFile.upload_time.desc())
|
||||
|
||||
if user_id:
|
||||
query = query.where(STPFile.user_id == user_id)
|
||||
|
||||
query = query.limit(limit)
|
||||
|
||||
result = await session.execute(query)
|
||||
files = result.unique().scalars().all()
|
||||
|
||||
return [
|
||||
{
|
||||
'id': f.id,
|
||||
'task_id': f.processing_tasks[0].task_id if f.processing_tasks else None,
|
||||
'upload_batch': f.upload_batch,
|
||||
'upload_time': f.upload_time.strftime('%Y-%m-%d %H:%M:%S') if f.upload_time else None,
|
||||
'file_size': f.file_size,
|
||||
'status': f.status,
|
||||
'volume': f.volume,
|
||||
'surface_area': f.surface_area,
|
||||
'product_weight': f.product_weight,
|
||||
'has_analysis': f.status == 'completed'
|
||||
}
|
||||
for f in files
|
||||
]
|
||||
|
||||
async def get_all_file_groups(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
user_id: Optional[int] = None,
|
||||
limit: int = 100
|
||||
) -> list:
|
||||
"""获取所有文件分组（按文件名分组），包含每个文件的最新分析结果"""
|
||||
|
||||
# 子查询：获取每个文件名的最新上传
|
||||
subquery = (
|
||||
select(
|
||||
STPFile.original_filename,
|
||||
func.max(STPFile.upload_time).label('latest_upload')
|
||||
)
|
||||
.group_by(STPFile.original_filename)
|
||||
.order_by(desc('latest_upload'))
|
||||
.limit(limit)
|
||||
)
|
||||
|
||||
if user_id:
|
||||
subquery = subquery.where(STPFile.user_id == user_id)
|
||||
|
||||
subquery = subquery.subquery()
|
||||
|
||||
# 主查询：获取最新记录和统计信息
|
||||
query = (
|
||||
select(STPFile).options(
|
||||
joinedload(STPFile.processing_tasks)
|
||||
)
|
||||
.join(
|
||||
subquery,
|
||||
(STPFile.original_filename == subquery.c.original_filename) &
|
||||
(STPFile.upload_time == subquery.c.latest_upload)
|
||||
)
|
||||
.order_by(STPFile.upload_time.desc())
|
||||
)
|
||||
|
||||
result = await session.execute(query)
|
||||
latest_files = result.unique().scalars().all()
|
||||
|
||||
# 一次性聚合每个文件名的上传次数（替代逐文件 count 的 N+1 查询）
|
||||
count_subquery = (
|
||||
select(STPFile.original_filename, func.count().label("upload_count"))
|
||||
.group_by(STPFile.original_filename)
|
||||
)
|
||||
if user_id:
|
||||
count_subquery = count_subquery.where(STPFile.user_id == user_id)
|
||||
count_result = await session.execute(count_subquery)
|
||||
upload_counts = {
|
||||
row.original_filename: row.upload_count for row in count_result
|
||||
}
|
||||
|
||||
# 获取每个文件名的上传次数
|
||||
file_groups = []
|
||||
for f in latest_files:
|
||||
task_id = f.processing_tasks[0].task_id if f.processing_tasks else None
|
||||
upload_count = upload_counts.get(f.original_filename, 1)
|
||||
|
||||
file_groups.append({
|
||||
'filename': f.original_filename,
|
||||
'latest_id': f.id,
|
||||
'latest_task_id': task_id,
|
||||
'latest_upload_time': f.upload_time.strftime('%Y-%m-%d %H:%M:%S') if f.upload_time else None,
|
||||
'latest_status': f.status,
|
||||
'upload_count': upload_count,
|
||||
'file_size': f.file_size,
|
||||
'volume': f.volume,
|
||||
'surface_area': f.surface_area,
|
||||
'product_weight': f.product_weight
|
||||
})
|
||||
|
||||
return file_groups
|
||||
@@ -20,14 +20,15 @@ from moldinsight.core.geometry_analyzer import GeometryAnalyzer
|
||||
from moldinsight.core.mesh_generator import MeshGenerator
|
||||
from moldinsight.core.multi_scheme_planner import MultiSchemeMoldPlanner
|
||||
from moldinsight.core.cad_exporter import CADExporter
|
||||
from moldinsight.services.storage_integration_rustfs import StorageIntegrationService
|
||||
from moldinsight.services.task_storage_service import TaskStorageService
|
||||
from moldinsight.services.analysis_storage_service import AnalysisStorageService
|
||||
from moldinsight.storage.rustfs_storage import rustfs_manager
|
||||
from shared.services.redis_task_manager import redis_task_manager
|
||||
from moldinsight.services.material_service import MaterialService
|
||||
from moldinsight.services.calculation_service import CalculationService
|
||||
from moldinsight.services.llm_service import llm_service
|
||||
from shared.models.schemas import ProcessingStatus
|
||||
from shared.models.database import STPFile
|
||||
from moldinsight.models import STPFile
|
||||
from shared.database.database import db_manager
|
||||
from shared.utils.html_generator import HTMLGenerator
|
||||
from shared.utils.logger import get_logger
|
||||
@@ -43,7 +44,9 @@ class ProcessingService:
|
||||
self.geometry_analyzer = GeometryAnalyzer()
|
||||
self.mesh_generator = MeshGenerator(quality="medium")
|
||||
self.html_generator = HTMLGenerator()
|
||||
self.storage_service = StorageIntegrationService()
|
||||
# 批次 3 按职责拆分：任务/文件生命周期 与 分析结果数据（原 StorageIntegrationService）
|
||||
self.task_storage = TaskStorageService()
|
||||
self.analysis_storage = AnalysisStorageService()
|
||||
self.multi_scheme_planner = MultiSchemeMoldPlanner()
|
||||
self.cad_exporter = CADExporter()
|
||||
# TopoDS_Shape 为 C++ 原生内存对象，LRU 上限防止长期运行内存只涨不降
|
||||
@@ -59,12 +62,15 @@ class ProcessingService:
|
||||
|
||||
asyncio.wait_for 只能取消协程，正在执行 OCC 布尔运算的线程无法中断；
|
||||
单 worker executor 中一个挂死线程会让后续任务永久排队直至重启。
|
||||
代价是泄漏 1 个线程，收益是恢复服务可用性。
|
||||
cancel_futures=True 丢弃旧 executor 中尚未开跑的排队任务（否则旧线程
|
||||
恢复后仍会继续消化旧队列，与新 executor 并发操作 OCC 必然崩溃）。
|
||||
已在运行中的 C++ 线程在 Python 层不可杀，仍会滞留——这是已知残留
|
||||
泄漏（每次超时 1 线程），根治需进程级 OCC 隔离，见 docs/OCC_THROUGHPUT.md。
|
||||
"""
|
||||
old = self._occ_executor
|
||||
self._occ_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="occ")
|
||||
old.shutdown(wait=False)
|
||||
logger.warning("OCC executor 已因处理超时重建（放弃等待旧线程，可能泄漏 1 个线程）")
|
||||
old.shutdown(wait=False, cancel_futures=True)
|
||||
logger.warning("OCC executor 已因处理超时重建（排队任务已丢弃，运行中线程可能滞留 1 个）")
|
||||
|
||||
async def run_occ(self, fn, *args):
|
||||
"""在 OCC 单线程 executor 中执行同步几何操作。
|
||||
@@ -169,8 +175,8 @@ class ProcessingService:
|
||||
# 避免 failed 更新把半成品 flush 数据一起带上
|
||||
await db_session.rollback()
|
||||
|
||||
await self.storage_service.update_stp_file_status(db_session, stp_file_id, "failed")
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_stp_file_status(db_session, stp_file_id, "failed")
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "failed", error_message=str(e)
|
||||
)
|
||||
|
||||
@@ -203,7 +209,7 @@ class ProcessingService:
|
||||
stage_timings: Dict[str, float] = {}
|
||||
|
||||
# 1. 解析STP文件
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "processing", 20, "解析STP文件"
|
||||
)
|
||||
|
||||
@@ -218,7 +224,7 @@ class ProcessingService:
|
||||
stage_timings["parse_stp"] = round(time.perf_counter() - stage_started, 3)
|
||||
|
||||
# 2. 生成网格数据并持久化
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "processing", 30, "生成网格数据"
|
||||
)
|
||||
|
||||
@@ -229,7 +235,7 @@ class ProcessingService:
|
||||
stage_timings["generate_mesh"] = round(time.perf_counter() - stage_started, 3)
|
||||
|
||||
# 3. 生成模具型腔
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "processing", 40, "生成模具型腔"
|
||||
)
|
||||
|
||||
@@ -257,7 +263,7 @@ class ProcessingService:
|
||||
)
|
||||
|
||||
# 4. 生成详细JSON数据 — 委托 CalculationService
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "processing", 60, "生成型腔详细数据"
|
||||
)
|
||||
|
||||
@@ -284,12 +290,12 @@ class ProcessingService:
|
||||
cavity_key_info = best_key_info
|
||||
|
||||
# 6. 保存几何数据到数据库
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "processing", 70, "保存几何数据"
|
||||
)
|
||||
|
||||
stage_started = time.perf_counter()
|
||||
await self.storage_service.save_geometry_data(
|
||||
await self.analysis_storage.save_geometry_data(
|
||||
db_session,
|
||||
stp_file_id,
|
||||
geometry_data,
|
||||
@@ -301,7 +307,7 @@ class ProcessingService:
|
||||
await db_session.commit()
|
||||
|
||||
# 7. 生成HTML可视化
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "processing", 85, "生成可视化报告"
|
||||
)
|
||||
|
||||
@@ -337,7 +343,7 @@ class ProcessingService:
|
||||
best_key_info = best_scheme.get("key_info", {}) if best_scheme else best_key_info
|
||||
|
||||
# 8. 保存模具型腔数据（包含方案级预览链接）
|
||||
await self.storage_service.save_mold_cavity_data(
|
||||
await self.analysis_storage.save_mold_cavity_data(
|
||||
db_session, stp_file_id, detailed_cavity_json
|
||||
)
|
||||
|
||||
@@ -349,7 +355,7 @@ class ProcessingService:
|
||||
lod_data=lod_data,
|
||||
)
|
||||
|
||||
await self.storage_service.save_html_file(
|
||||
await self.analysis_storage.save_html_file(
|
||||
db_session,
|
||||
stp_file_id,
|
||||
Path(html_file_path).name,
|
||||
@@ -370,7 +376,7 @@ class ProcessingService:
|
||||
)
|
||||
|
||||
if analysis_result:
|
||||
await self.storage_service.save_features_and_recommendations(
|
||||
await self.analysis_storage.save_features_and_recommendations(
|
||||
db_session,
|
||||
stp_file_id,
|
||||
analysis_result.get("detected_features", []),
|
||||
@@ -381,7 +387,7 @@ class ProcessingService:
|
||||
stage_timings["analyze_design"] = round(time.perf_counter() - stage_started, 3)
|
||||
|
||||
# 9.6 更新STP文件的分析摘要字段
|
||||
await self.storage_service.update_stp_file_analysis_summary(
|
||||
await self.task_storage.update_stp_file_analysis_summary(
|
||||
db_session,
|
||||
stp_file_id,
|
||||
volume=geometry_data.get("volume", 0),
|
||||
@@ -419,7 +425,7 @@ class ProcessingService:
|
||||
stage_timings["generate_llm_report"] = round(time.perf_counter() - stage_started, 3)
|
||||
|
||||
# 10. 完成处理——先 flush 任务参数，完成状态提交时一并原子落库（D9）
|
||||
await self.storage_service.update_task_parameters(
|
||||
await self.task_storage.update_task_parameters(
|
||||
db_session,
|
||||
task_id,
|
||||
{
|
||||
@@ -431,8 +437,8 @@ class ProcessingService:
|
||||
**process_params,
|
||||
},
|
||||
)
|
||||
await self.storage_service.update_stp_file_status(db_session, stp_file_id, "completed")
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_stp_file_status(db_session, stp_file_id, "completed")
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "completed", 100, "模具型腔生成完成"
|
||||
)
|
||||
|
||||
@@ -464,8 +470,8 @@ class ProcessingService:
|
||||
# D9：先丢弃未提交的数据本体再置失败（同外层说明）
|
||||
await db_session.rollback()
|
||||
|
||||
await self.storage_service.update_stp_file_status(db_session, stp_file_id, "failed")
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_stp_file_status(db_session, stp_file_id, "failed")
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "failed", error_message=str(e)
|
||||
)
|
||||
|
||||
@@ -524,7 +530,7 @@ class ProcessingService:
|
||||
"bounding_box": bbox,
|
||||
}
|
||||
|
||||
await self.storage_service.save_mesh_data(
|
||||
await self.analysis_storage.save_mesh_data(
|
||||
db_session,
|
||||
stp_file_id=stp_file_id,
|
||||
mesh_json=mesh_json,
|
||||
@@ -732,7 +738,7 @@ class ProcessingService:
|
||||
logger.info("FreeCAD验证已禁用（设置 ENABLE_FREECAD_VERIFICATION=true 启用）")
|
||||
return {"status": "disabled", "reason": "FreeCAD验证已禁用"}
|
||||
|
||||
await self.storage_service.update_task_status(
|
||||
await self.task_storage.update_task_status(
|
||||
db_session, task_id, "processing", 90, "FreeCAD几何验证"
|
||||
)
|
||||
|
||||
@@ -787,7 +793,7 @@ class ProcessingService:
|
||||
|
||||
async def _save_analysis_metrics(self, session: AsyncSession, stp_file_id: int, analysis_result: dict):
|
||||
"""保存分析指标到数据库"""
|
||||
from shared.models.database import AnalysisMetrics
|
||||
from moldinsight.models import AnalysisMetrics
|
||||
|
||||
quality_metrics = analysis_result.get("quality_metrics", {})
|
||||
analysis_summary = analysis_result.get("analysis_summary", "")
|
||||
@@ -807,7 +813,7 @@ class ProcessingService:
|
||||
|
||||
async def _save_verification_metrics(self, session: AsyncSession, stp_file_id: int, verification_result: dict):
|
||||
"""保存验证指标到数据库"""
|
||||
from shared.models.database import AnalysisMetrics
|
||||
from moldinsight.models import AnalysisMetrics
|
||||
from sqlalchemy import select
|
||||
|
||||
result = await session.execute(
|
||||
|
||||
@@ -12,7 +12,7 @@ from typing import Optional
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from shared.models.database import ProcessingTask, STPFile
|
||||
from moldinsight.models import ProcessingTask, STPFile
|
||||
from shared.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
@@ -10,9 +10,9 @@ from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from sqlalchemy.orm import joinedload
|
||||
|
||||
from moldinsight.services.storage_integration_rustfs import StorageIntegrationService
|
||||
from moldinsight.services.analysis_storage_service import AnalysisStorageService
|
||||
from shared.services.redis_task_manager import redis_task_manager
|
||||
from shared.models.database import ProcessingTask, STPFile, MeshData, HTMLFile
|
||||
from moldinsight.models import ProcessingTask, STPFile, MeshData, HTMLFile
|
||||
from shared.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
@@ -103,7 +103,7 @@ class TaskQueryService:
|
||||
return cached
|
||||
|
||||
# 3. 持久化任务（已完成/失败，或服务重启后的任务）
|
||||
storage_service = StorageIntegrationService()
|
||||
storage_service = AnalysisStorageService()
|
||||
|
||||
# 查询任务和文件元数据（预加载 html_file 关联）
|
||||
result = await db_session.execute(
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
# services/task_storage_service.py
|
||||
"""任务与源文件生命周期存储——PostgreSQL（+ 源文件 RustFS 上传）。
|
||||
|
||||
批次 3 自 storage_integration_rustfs.py 按职责拆分（原 867 行混杂
|
||||
写入/查询/历史三类职责）：
|
||||
- 本模块：STPFile 生命周期 + ProcessingTask 创建/状态/参数
|
||||
- 分析结果数据：analysis_storage_service.AnalysisStorageService
|
||||
- 历史查询视图：file_history_service.FileHistoryService
|
||||
"""
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict, Any
|
||||
from datetime import datetime
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import update, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from moldinsight.models import STPFile, ProcessingTask
|
||||
from moldinsight.storage.rustfs_storage import rustfs_manager
|
||||
from shared.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class TaskStorageService:
|
||||
"""STP 文件与处理任务的生命周期存储"""
|
||||
|
||||
async def save_stp_file(self, session: AsyncSession,
|
||||
file_path: Path,
|
||||
original_filename: str,
|
||||
user_id: Optional[int] = None,
|
||||
upload_batch: Optional[str] = None) -> STPFile:
|
||||
"""保存STP文件到PostgreSQL元数据 + RustFS对象存储
|
||||
|
||||
支持同一文件多次上传，每次上传都会创建新记录
|
||||
"""
|
||||
|
||||
# 1. 上传到RustFS
|
||||
upload_result = await rustfs_manager.upload_file(
|
||||
file_type='stp_files',
|
||||
file_path=file_path,
|
||||
original_filename=original_filename,
|
||||
metadata={
|
||||
'original_filename': original_filename,
|
||||
'user_id': str(user_id) if user_id else 'anonymous',
|
||||
'upload_batch': upload_batch or str(uuid.uuid4())
|
||||
}
|
||||
)
|
||||
|
||||
file_hash = upload_result['file_hash']
|
||||
batch_id = upload_batch or str(uuid.uuid4())
|
||||
|
||||
# 2. 创建新PostgreSQL记录（每次上传都创建新记录）
|
||||
stp_file = STPFile(
|
||||
user_id=user_id,
|
||||
object_key=upload_result['object_key'],
|
||||
storage_bucket=upload_result['bucket'],
|
||||
original_filename=original_filename,
|
||||
file_size=upload_result['file_size'],
|
||||
file_hash=file_hash,
|
||||
upload_batch=batch_id,
|
||||
status="uploaded",
|
||||
file_path=str(file_path),
|
||||
upload_time=datetime.now()
|
||||
)
|
||||
|
||||
session.add(stp_file)
|
||||
# D9：仅 flush，与 ProcessingTask 由路由层一并原子提交（避免孤儿文件记录）
|
||||
await session.flush()
|
||||
await session.refresh(stp_file)
|
||||
|
||||
logger.info(f"STP文件保存成功 RustFS: {stp_file.id}, 批次: {batch_id}")
|
||||
return stp_file
|
||||
|
||||
async def create_processing_task(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
task_id: str,
|
||||
stp_file_id: int,
|
||||
task_type: str = "stp_parsing",
|
||||
parameters: Optional[Dict[str, Any]] = None,
|
||||
batch_id: Optional[str] = None,
|
||||
) -> ProcessingTask:
|
||||
"""创建处理任务记录（D9：仅 flush 不 commit，事务由调用方收口——
|
||||
与 STPFile 记录同批提交，避免留下无任务的孤儿文件记录；batch_id 用于批量任务聚合查询）
|
||||
"""
|
||||
try:
|
||||
task = ProcessingTask(
|
||||
task_id=task_id,
|
||||
stp_file_id=stp_file_id,
|
||||
task_type=task_type,
|
||||
status="pending",
|
||||
started_time=datetime.now(),
|
||||
parameters=parameters or {},
|
||||
batch_id=batch_id,
|
||||
)
|
||||
|
||||
session.add(task)
|
||||
await session.flush()
|
||||
|
||||
logger.info(f"处理任务创建成功: {task_id}")
|
||||
return task
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"创建处理任务失败: {e}")
|
||||
raise
|
||||
|
||||
async def update_task_status(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
task_id: str,
|
||||
status: str,
|
||||
progress: Optional[int] = None,
|
||||
current_step: Optional[str] = None,
|
||||
error_message: Optional[str] = None
|
||||
):
|
||||
"""更新任务状态（保留即时 commit：进度/状态需跨事务对外可见，
|
||||
处理链路中的各阶段进度依赖它落库——D9 收口仅针对数据本体写方法）"""
|
||||
try:
|
||||
update_data = {
|
||||
"status": status,
|
||||
"completed_time": datetime.now() if status in ["completed", "failed"] else None,
|
||||
"error_message": error_message
|
||||
}
|
||||
|
||||
if progress is not None:
|
||||
update_data["progress"] = progress
|
||||
if current_step is not None:
|
||||
update_data["current_step"] = current_step
|
||||
|
||||
await session.execute(
|
||||
update(ProcessingTask)
|
||||
.where(ProcessingTask.task_id == task_id)
|
||||
.values(**update_data)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"任务状态更新: {task_id} -> {status}")
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"更新任务状态失败: {e}")
|
||||
raise
|
||||
|
||||
async def update_task_parameters(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
task_id: str,
|
||||
parameters: Dict[str, Any],
|
||||
):
|
||||
"""合并更新任务参数，便于保存阶段耗时等元数据。（D9：flush 不 commit，事务由调用方收口）"""
|
||||
try:
|
||||
task = await session.execute(
|
||||
select(ProcessingTask).where(ProcessingTask.task_id == task_id)
|
||||
)
|
||||
task = task.scalar_one_or_none()
|
||||
if task is None:
|
||||
return
|
||||
|
||||
merged = dict(task.parameters or {})
|
||||
merged.update(parameters or {})
|
||||
task.parameters = merged
|
||||
await session.flush()
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"更新任务参数失败: {e}")
|
||||
raise
|
||||
|
||||
async def update_stp_file_status(self, session: AsyncSession, stp_file_id: int, status: str):
|
||||
"""更新STP文件状态（保留即时 commit，理由同 update_task_status）"""
|
||||
try:
|
||||
await session.execute(
|
||||
update(STPFile)
|
||||
.where(STPFile.id == stp_file_id)
|
||||
.values(
|
||||
status=status,
|
||||
processed_time=datetime.now() if status in ["completed", "failed"] else None
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
logger.info(f"STP文件状态更新: ID {stp_file_id} -> {status}")
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"更新STP文件状态失败: {e}")
|
||||
raise
|
||||
|
||||
async def update_stp_file_analysis_summary(
|
||||
self,
|
||||
session: AsyncSession,
|
||||
stp_file_id: int,
|
||||
volume: Optional[float] = None,
|
||||
surface_area: Optional[float] = None,
|
||||
product_weight: Optional[float] = None
|
||||
):
|
||||
"""更新STP文件的分析摘要字段（用于快速查询）"""
|
||||
try:
|
||||
update_data = {}
|
||||
if volume is not None:
|
||||
update_data['volume'] = volume
|
||||
if surface_area is not None:
|
||||
update_data['surface_area'] = surface_area
|
||||
if product_weight is not None:
|
||||
update_data['product_weight'] = product_weight
|
||||
|
||||
if update_data:
|
||||
await session.execute(
|
||||
update(STPFile)
|
||||
.where(STPFile.id == stp_file_id)
|
||||
.values(**update_data)
|
||||
)
|
||||
# D9：flush 不 commit，随结果包由编排层统一提交
|
||||
await session.flush()
|
||||
logger.info(f"STP文件分析摘要更新: ID {stp_file_id}")
|
||||
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
logger.error(f"更新STP文件分析摘要失败: {e}")
|
||||
raise
|
||||
Reference in New Issue
Block a user