""" 消息存储服务 - 将每次查询的消息记录存储到 MySQL """ from __future__ import annotations import json from typing import Any, Dict, List, Optional import pymysql from config import Config from services.common.datetime_utils import DateTimeGenerator from schemas.messages import MessagesDTO class MessageStorage: """消息存储服务""" def __init__(self): cfg = Config.get_section("logging_mysql") self.enabled = str(cfg.get("enabled", "false")).lower() in ("1", "true", "yes") self.entity_debug_enabled = str(cfg.get("entity_debug_enabled", "false")).lower() in ("1", "true", "yes") self.host = cfg.get("host", "127.0.0.1") self.port = int(cfg.get("port", 3306)) self.user = cfg.get("user", "root") self.password = cfg.get("password", "") self.database = cfg.get("database", "more_dots") # 消息落库与结构化日志分表，避免误用 logging_mysql.table=structured_logs self.table = cfg.get("messages_table", "ipc_apbo.messages") self.conversation_table = cfg.get("conversation_table", "ipc_apbo.conversations") self.connect_timeout = int(cfg.get("connect_timeout", 5)) self._inited = False self._conversation_schema_checked = False def _get_conn(self): """获取数据库连接""" return pymysql.connect( host=self.host, port=self.port, user=self.user, password=self.password, database=self.database, charset="utf8mb4", autocommit=True, connect_timeout=self.connect_timeout, ) @staticmethod def _log_local(event: str, payload: Optional[Dict[str, Any]] = None) -> None: print(json.dumps({ "level": "ERROR", "event": event, "payload": payload or {}, "created_at": DateTimeGenerator.now().epoch_millis, }, ensure_ascii=False)) def _log_entity_debug(self, event: str, payload: Optional[Dict[str, Any]] = None) -> None: if not self.entity_debug_enabled: return print(json.dumps({ "level": "DEBUG", "event": event, "payload": payload or {}, "created_at": DateTimeGenerator.now().epoch_millis, }, ensure_ascii=False)) @staticmethod def _now_ms() -> int: return DateTimeGenerator.now().epoch_millis @staticmethod def _build_audit_fields( *, random_code: str, user: Optional[str], created_value: Any = None, updated_value: Any = None, ) -> Dict[str, Any]: created_ms = MessageStorage._resolve_epoch_millis(created_value) updated_ms = MessageStorage._resolve_epoch_millis(updated_value, fallback=created_ms) created_bundle = DateTimeGenerator.bundle(created_ms, default_to_now=True) updated_bundle = DateTimeGenerator.bundle(updated_ms, default_to_now=True) operator = (user or "system").strip() if isinstance(user, str) else "system" return { "random_code": random_code, "create_user": operator, "create_date": created_bundle.db_datetime, "update_user": operator, "update_date": updated_bundle.db_datetime, "create_user_name": operator, "update_user_name": operator, "created_at": created_ms, "updated_at": updated_ms, } @staticmethod def _resolve_epoch_millis(value: Any, fallback: Optional[int] = None) -> int: if value is None: return fallback if fallback is not None else DateTimeGenerator.now().epoch_millis if isinstance(value, (int, float)): raw = int(value) digits = len(str(abs(raw))) if digits == 10: return raw * 1000 return raw if isinstance(value, str): text = value.strip() if text.isdigit(): raw = int(text) digits = len(text) if digits == 10: return raw * 1000 if digits == 13: return raw parsed = DateTimeGenerator.bundle(value, default_to_now=True) return parsed.epoch_millis parsed = DateTimeGenerator.bundle(value, default_to_now=True) return parsed.epoch_millis def _log_entity_stage(self, entity: str, action: str, stage: str, started_at: int, payload: Optional[Dict[str, Any]] = None) -> None: debug_payload = dict(payload or {}) debug_payload.update({ "entity": entity, "action": action, "stage": stage, "elapsed_ms": max(0, self._now_ms() - started_at), }) self._log_entity_debug(f"message_storage.{entity}.{action}.{stage}", debug_payload) @staticmethod def _split_table_reference(table_name: str, default_schema: str) -> tuple[str, str]: cleaned = str(table_name or "").strip() if "." in cleaned: schema_name, physical_table_name = cleaned.split(".", 1) else: schema_name, physical_table_name = default_schema, cleaned return schema_name.strip().strip("`"), physical_table_name.strip().strip("`") def _ensure_conversation_schema(self, conn) -> None: if self._conversation_schema_checked or not self.enabled: return started_at = self._now_ms() self._log_entity_stage( "conversations", "ensure_schema", "start", started_at, {"conversation_table": self.conversation_table}, ) schema_name, table_name = self._split_table_reference(self.conversation_table, self.database) probe_sql = """ SELECT 1 FROM information_schema.columns WHERE table_schema = %s AND table_name = %s AND column_name = %s LIMIT 1 """ with conn.cursor() as cur: cur.execute(probe_sql, (schema_name, table_name, "name")) if cur.fetchone() is None: self._log_entity_stage( "conversations", "ensure_schema", "alter_needed", started_at, {"conversation_table": self.conversation_table, "missing_column": "name"}, ) alter_sql = f""" ALTER TABLE {self.conversation_table} ADD COLUMN name VARCHAR(255) NULL COMMENT '会话名称' AFTER user """ cur.execute(alter_sql) self._conversation_schema_checked = True self._log_entity_stage( "conversations", "ensure_schema", "success", started_at, {"conversation_table": self.conversation_table, "schema_checked": True}, ) def _ensure_table(self) -> None: """确保消息表存在""" if self._inited or not self.enabled: return started_at = self._now_ms() self._log_entity_stage( "storage", "ensure_table", "start", started_at, {"messages_table": self.table, "conversation_table": self.conversation_table}, ) message_sql = f""" CREATE TABLE IF NOT EXISTS {self.table} ( id BIGINT PRIMARY KEY AUTO_INCREMENT, random_code VARCHAR(100) NULL COMMENT '业务主键', create_user VARCHAR(100) NULL COMMENT '创建人', create_date DATETIME NULL COMMENT '创建时间', update_user VARCHAR(100) NULL COMMENT '修改人', update_date DATETIME NULL COMMENT '修改时间', create_user_name VARCHAR(255) NULL COMMENT '创建人姓名', update_user_name VARCHAR(255) NULL COMMENT '修改人姓名', message_id VARCHAR(255) NULL COMMENT '消息 ID', conversation_id VARCHAR(255) NULL COMMENT '会话 ID', user VARCHAR(255) NULL COMMENT '用户标识', query LONGTEXT NULL COMMENT '用户查询', answer LONGTEXT NULL COMMENT '回答消息内容', feedback VARCHAR(255) NULL COMMENT '点赞 like / 点踩 dislike', feedback_content TEXT NULL COMMENT '点踩内容', created_at BIGINT NULL COMMENT '创建时间(毫秒时间戳)', updated_at BIGINT NULL COMMENT '更新时间(毫秒时间戳)', `log` JSON NULL COMMENT '当前对话日志(JSON字符串)', UNIQUE KEY uk_random_code (random_code), UNIQUE KEY uk_message_id (message_id), INDEX idx_conversation_id (conversation_id), INDEX idx_created_at (created_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='消息记录表'; """ conversation_sql = f""" CREATE TABLE IF NOT EXISTS {self.conversation_table} ( id BIGINT PRIMARY KEY AUTO_INCREMENT, random_code VARCHAR(100) NULL COMMENT '业务主键', create_user VARCHAR(100) NULL COMMENT '创建人', create_date DATETIME NULL COMMENT '创建时间', update_user VARCHAR(100) NULL COMMENT '修改人', update_date DATETIME NULL COMMENT '修改时间', create_user_name VARCHAR(255) NULL COMMENT '创建人姓名', update_user_name VARCHAR(255) NULL COMMENT '修改人姓名', conversation_id VARCHAR(255) NULL COMMENT '会话 ID', user VARCHAR(255) NULL COMMENT '用户', name VARCHAR(512) NULL COMMENT '会话名称', status VARCHAR(255) NULL COMMENT '状态', introduction VARCHAR(255) NULL COMMENT '开场白', created_at BIGINT NULL COMMENT '创建时间(毫秒时间戳)', updated_at BIGINT NULL COMMENT '更新时间(毫秒时间戳)', UNIQUE KEY uk_conversation_random_code (random_code), UNIQUE KEY uk_conversation_id (conversation_id), INDEX idx_conversation_updated_at (updated_at) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='会话记录表'; """ try: with self._get_conn() as conn: with conn.cursor() as cur: cur.execute(message_sql) cur.execute(conversation_sql) self._ensure_conversation_schema(conn) self._inited = True self._log_entity_stage( "storage", "ensure_table", "success", started_at, {"messages_table": self.table, "conversation_table": self.conversation_table}, ) except Exception as exc: self._log_entity_stage( "storage", "ensure_table", "failed", started_at, {"messages_table": self.table, "conversation_table": self.conversation_table, "error": str(exc)}, ) self._log_local("message_storage.ensure_table_failed", { "error": str(exc), "messages_table": self.table, "conversation_table": self.conversation_table, }) # 开发阶段容错，避免初始化失败影响主流程 self.enabled = False def create_conversation( self, conversation_id: str, user: Optional[str], name: Optional[str], status: str, introduction: Optional[str], created_at: int, updated_at: int, ) -> bool: started_at = self._now_ms() self._log_entity_stage( "conversations", "create", "start", started_at, { "conversation_id": conversation_id, "user": user, "name_len": len(name or ""), "status": status, }, ) if not self.enabled: self._log_entity_stage( "conversations", "create", "skipped", started_at, {"conversation_id": conversation_id, "reason": "storage_disabled"}, ) return False self._ensure_table() if not self.enabled: self._log_entity_stage( "conversations", "create", "skipped", started_at, {"conversation_id": conversation_id, "reason": "storage_disabled_after_init"}, ) return False insert_sql = f""" INSERT INTO {self.conversation_table}( random_code, create_user, create_date, update_user, update_date, create_user_name, update_user_name, conversation_id, user, name, status, introduction, created_at, updated_at ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """ try: audit = self._build_audit_fields( random_code=conversation_id, user=user, created_value=created_at, updated_value=updated_at, ) with self._get_conn() as conn: self._ensure_conversation_schema(conn) with conn.cursor() as cur: self._log_entity_stage( "conversations", "create", "sql_execute", started_at, {"conversation_id": conversation_id, "table": self.conversation_table}, ) cur.execute( insert_sql, ( audit["random_code"], audit["create_user"], audit["create_date"], audit["update_user"], audit["update_date"], audit["create_user_name"], audit["update_user_name"], conversation_id, user, name, status, introduction, audit["created_at"], audit["updated_at"], ), ) self._log_entity_stage( "conversations", "create", "success", started_at, {"conversation_id": conversation_id, "table": self.conversation_table}, ) return True except Exception as exc: self._log_entity_stage( "conversations", "create", "failed", started_at, {"conversation_id": conversation_id, "table": self.conversation_table, "error": str(exc)}, ) self._log_local("message_storage.create_conversation_failed", { "error": str(exc), "conversation_table": self.conversation_table, "conversation_id": conversation_id, }) return False def get_conversation_by_id(self, conversation_id: str) -> Optional[Dict[str, Any]]: started_at = self._now_ms() self._log_entity_stage( "conversations", "get_by_id", "start", started_at, {"conversation_id": conversation_id}, ) if not self.enabled: self._log_entity_stage( "conversations", "get_by_id", "skipped", started_at, {"conversation_id": conversation_id, "reason": "storage_disabled"}, ) return None self._ensure_table() if not self.enabled: self._log_entity_stage( "conversations", "get_by_id", "skipped", started_at, {"conversation_id": conversation_id, "reason": "storage_disabled_after_init"}, ) return None select_sql = f""" SELECT conversation_id, user, name, status, introduction, created_at, updated_at FROM {self.conversation_table} WHERE conversation_id = %s LIMIT 1 """ try: with self._get_conn() as conn: self._ensure_conversation_schema(conn) with conn.cursor(pymysql.cursors.DictCursor) as cur: self._log_entity_stage( "conversations", "get_by_id", "sql_execute", started_at, {"conversation_id": conversation_id, "table": self.conversation_table}, ) cur.execute(select_sql, (conversation_id,)) result = cur.fetchone() self._log_entity_stage( "conversations", "get_by_id", "success", started_at, { "conversation_id": conversation_id, "table": self.conversation_table, "found": bool(result), }, ) return dict(result) if result else None except Exception as exc: self._log_entity_stage( "conversations", "get_by_id", "failed", started_at, {"conversation_id": conversation_id, "table": self.conversation_table, "error": str(exc)}, ) self._log_local("message_storage.get_conversation_failed", { "error": str(exc), "conversation_table": self.conversation_table, "conversation_id": conversation_id, }) return None def update_conversation_updated_at(self, conversation_id: str, updated_at: int) -> bool: started_at = self._now_ms() self._log_entity_stage( "conversations", "update_updated_at", "start", started_at, {"conversation_id": conversation_id, "updated_at": updated_at}, ) if not self.enabled: self._log_entity_stage( "conversations", "update_updated_at", "skipped", started_at, {"conversation_id": conversation_id, "reason": "storage_disabled"}, ) return False self._ensure_table() if not self.enabled: self._log_entity_stage( "conversations", "update_updated_at", "skipped", started_at, {"conversation_id": conversation_id, "reason": "storage_disabled_after_init"}, ) return False update_sql = f""" UPDATE {self.conversation_table} SET updated_at = %s, update_date = %s WHERE conversation_id = %s """ try: update_ms = self._resolve_epoch_millis(updated_at) update_bundle = DateTimeGenerator.bundle(update_ms, default_to_now=True) with self._get_conn() as conn: self._ensure_conversation_schema(conn) with conn.cursor() as cur: self._log_entity_stage( "conversations", "update_updated_at", "sql_execute", started_at, {"conversation_id": conversation_id, "table": self.conversation_table}, ) affected_rows = cur.execute( update_sql, (update_ms, update_bundle.db_datetime, conversation_id), ) self._log_entity_stage( "conversations", "update_updated_at", "success", started_at, {"conversation_id": conversation_id, "affected_rows": int(affected_rows or 0)}, ) return bool(affected_rows) except Exception as exc: self._log_entity_stage( "conversations", "update_updated_at", "failed", started_at, {"conversation_id": conversation_id, "table": self.conversation_table, "error": str(exc)}, ) self._log_local("message_storage.update_conversation_failed", { "error": str(exc), "conversation_table": self.conversation_table, "conversation_id": conversation_id, }) return False def save_message( self, conversation_id: str, message_id: str, query: str, answer: Optional[str] = None, workflow_type: Optional[str] = None, user: Optional[str] = None, sql_query: Optional[str] = None, execution_result: Optional[Dict[str, Any]] = None, metadata: Optional[Dict[str, Any]] = None, created_at: Optional[int] = None, updated_at: Optional[int] = None, logs: Optional[List[str]] = None, ) -> bool: started_at = self._now_ms() self._log_entity_stage( "messages", "create", "start", started_at, { "conversation_id": conversation_id, "message_id": message_id, "query_len": len(query or ""), "answer_len": len(answer or ""), "workflow_type": workflow_type, }, ) """ 保存消息记录 Args: conversation_id: 会话 ID message_id: 消息 ID query: 用户查询 answer: AI 回复 workflow_type: 工作流类型 (conversation/tool_using) user: 用户标识 sql_query: 生成的 SQL execution_result: SQL 执行结果 metadata: 其他元数据 logs: 过程日志（兼容 Java saveMessageToDB） Returns: bool: 是否保存成功 """ if not self.enabled: self._log_entity_stage( "messages", "create", "skipped", started_at, {"conversation_id": conversation_id, "message_id": message_id, "reason": "storage_disabled"}, ) return False self._ensure_table() if not self.enabled: self._log_entity_stage( "messages", "create", "skipped", started_at, {"conversation_id": conversation_id, "message_id": message_id, "reason": "storage_disabled_after_init"}, ) return False audit = self._build_audit_fields( random_code=message_id, user=user, created_value=created_at, updated_value=updated_at, ) log_payload = { "workflow_type": workflow_type, "sql_query": sql_query, "execution_result": execution_result or {}, "metadata": metadata or {}, } normalized_logs = [str(item) for item in (logs or []) if str(item).strip()] if normalized_logs: log_payload["data"] = "\n".join(normalized_logs) message_record = MessagesDTO( message_id=message_id, conversation_id=conversation_id, user=user, query=query, answer=answer, feedback=None, feedback_content=None, created_at=audit["created_at"], updated_at=audit["updated_at"], log=log_payload, ) insert_sql = f""" INSERT INTO {self.table}( random_code, create_user, create_date, update_user, update_date, create_user_name, update_user_name, message_id, conversation_id, user, query, answer, feedback, feedback_content, created_at, updated_at, `log` ) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """ try: with self._get_conn() as conn: with conn.cursor() as cur: self._log_entity_stage( "messages", "create", "sql_execute", started_at, {"conversation_id": conversation_id, "message_id": message_id, "table": self.table}, ) cur.execute( insert_sql, ( audit["random_code"], audit["create_user"], audit["create_date"], audit["update_user"], audit["update_date"], audit["create_user_name"], audit["update_user_name"], message_record.message_id, message_record.conversation_id, message_record.user, message_record.query, message_record.answer, message_record.feedback, message_record.feedback_content, message_record.created_at, message_record.updated_at, json.dumps(message_record.log, ensure_ascii=False), ), ) self._log_entity_stage( "messages", "create", "success", started_at, {"conversation_id": conversation_id, "message_id": message_id, "table": self.table}, ) return True except Exception as exc: self._log_entity_stage( "messages", "create", "failed", started_at, {"conversation_id": conversation_id, "message_id": message_id, "table": self.table, "error": str(exc)}, ) self._log_local("message_storage.save_failed", { "error": str(exc), "messages_table": self.table, "message_id": message_id, "conversation_id": conversation_id, }) # 开发阶段容错，避免日志失败影响主流程 return False def get_conversation_history( self, conversation_id: str, limit: int = 20 ) -> List[Dict[str, Any]]: """ 获取会话历史消息 Args: conversation_id: 会话 ID limit: 返回消息数量 Returns: List[Dict[str, Any]]: 消息列表 """ started_at = self._now_ms() self._log_entity_stage( "messages", "get_history", "start", started_at, {"conversation_id": conversation_id, "limit": limit}, ) if not self.enabled: self._log_entity_stage( "messages", "get_history", "skipped", started_at, {"conversation_id": conversation_id, "reason": "storage_disabled"}, ) return [] self._ensure_table() if not self.enabled: self._log_entity_stage( "messages", "get_history", "skipped", started_at, {"conversation_id": conversation_id, "reason": "storage_disabled_after_init"}, ) return [] select_sql = f""" SELECT * FROM {self.table} WHERE conversation_id = %s ORDER BY created_at DESC LIMIT %s """ try: with self._get_conn() as conn: with conn.cursor(pymysql.cursors.DictCursor) as cur: self._log_entity_stage( "messages", "get_history", "sql_execute", started_at, {"conversation_id": conversation_id, "limit": limit, "table": self.table}, ) cur.execute(select_sql, (conversation_id, limit)) results = cur.fetchall() self._log_entity_stage( "messages", "get_history", "success", started_at, {"conversation_id": conversation_id, "count": len(results or [])}, ) return list(results) except Exception as exc: self._log_entity_stage( "messages", "get_history", "failed", started_at, {"conversation_id": conversation_id, "error": str(exc)}, ) return [] def update_feedback_by_message_id( self, message_id: str, feedback: str, feedback_content: Optional[str] = None, ) -> bool: """按 message_id 回写点赞/点踩反馈。""" started_at = self._now_ms() self._log_entity_stage( "messages", "update_feedback", "start", started_at, {"message_id": message_id, "feedback": feedback}, ) if not self.enabled: self._log_entity_stage( "messages", "update_feedback", "skipped", started_at, {"message_id": message_id, "reason": "storage_disabled"}, ) return False self._ensure_table() if not self.enabled: self._log_entity_stage( "messages", "update_feedback", "skipped", started_at, {"message_id": message_id, "reason": "storage_disabled_after_init"}, ) return False update_sql = f""" UPDATE {self.table} SET feedback = %s, feedback_content = %s, updated_at = %s, update_date = %s WHERE message_id = %s """ normalized_feedback_content = (feedback_content or "").strip() or None now_bundle = DateTimeGenerator.now() try: with self._get_conn() as conn: with conn.cursor() as cur: self._log_entity_stage( "messages", "update_feedback", "sql_execute", started_at, {"message_id": message_id, "table": self.table}, ) affected_rows = cur.execute( update_sql, ( feedback, normalized_feedback_content, now_bundle.epoch_millis, now_bundle.db_datetime, message_id, ), ) self._log_entity_stage( "messages", "update_feedback", "success", started_at, {"message_id": message_id, "affected_rows": int(affected_rows or 0)}, ) return bool(affected_rows) except Exception as exc: self._log_entity_stage( "messages", "update_feedback", "failed", started_at, {"message_id": message_id, "error": str(exc)}, ) # 开发阶段容错，避免日志失败影响主流程 return False # 全局单例 _GLOBAL_MESSAGE_STORAGE: Optional[MessageStorage] = None def get_message_storage() -> MessageStorage: """获取消息存储服务实例""" global _GLOBAL_MESSAGE_STORAGE if _GLOBAL_MESSAGE_STORAGE is None: _GLOBAL_MESSAGE_STORAGE = MessageStorage() return _GLOBAL_MESSAGE_STORAGE