init
This commit is contained in:
@@ -0,0 +1,14 @@
|
||||
"""数据存储模块"""
|
||||
|
||||
from .message_storage import get_message_storage, MessageStorage
|
||||
from .structured_logger import get_structured_logger
|
||||
from .cache import CacheBase, NoopCache, RedisCache
|
||||
|
||||
__all__ = [
|
||||
"get_message_storage",
|
||||
"MessageStorage",
|
||||
"get_structured_logger",
|
||||
"CacheBase",
|
||||
"NoopCache",
|
||||
"RedisCache",
|
||||
]
|
||||
@@ -0,0 +1,64 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional
|
||||
|
||||
try:
|
||||
import redis
|
||||
except Exception:
|
||||
redis = None
|
||||
|
||||
|
||||
class CacheBase:
|
||||
"""缓存接口"""
|
||||
|
||||
def get(self, key: str) -> Optional[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
def set(self, key: str, value: str, ttl: int = None) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
def keys(self, pattern: str) -> List[str]:
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class NoopCache(CacheBase):
|
||||
"""空实现缓存"""
|
||||
|
||||
def get(self, key: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
def set(self, key: str, value: str, ttl: int = None) -> None:
|
||||
return None
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
return None
|
||||
|
||||
def keys(self, pattern: str) -> List[str]:
|
||||
return []
|
||||
|
||||
|
||||
class RedisCache(CacheBase):
|
||||
"""Redis 缓存实现"""
|
||||
|
||||
def __init__(self, url: str, db: int = 0):
|
||||
if redis is None:
|
||||
raise ImportError("未安装 redis 依赖")
|
||||
self._client = redis.Redis.from_url(url, db=db, decode_responses=True)
|
||||
|
||||
def get(self, key: str) -> Optional[str]:
|
||||
return self._client.get(key)
|
||||
|
||||
def set(self, key: str, value: str, ttl: int = None) -> None:
|
||||
if ttl:
|
||||
self._client.set(key, value, ex=ttl)
|
||||
else:
|
||||
self._client.set(key, value)
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self._client.delete(key)
|
||||
|
||||
def keys(self, pattern: str) -> List[str]:
|
||||
return self._client.keys(pattern)
|
||||
@@ -0,0 +1,885 @@
|
||||
"""
|
||||
消息存储服务 - 将每次查询的消息记录存储到 MySQL
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import pymysql
|
||||
|
||||
from config import Config
|
||||
from services.common.datetime_utils import DateTimeGenerator
|
||||
from schemas.messages import MessagesDTO
|
||||
|
||||
|
||||
class MessageStorage:
|
||||
"""消息存储服务"""
|
||||
|
||||
def __init__(self):
|
||||
cfg = Config.get_section("logging_mysql")
|
||||
self.enabled = str(cfg.get("enabled", "false")).lower() in ("1", "true", "yes")
|
||||
self.entity_debug_enabled = str(cfg.get("entity_debug_enabled", "false")).lower() in ("1", "true", "yes")
|
||||
self.host = cfg.get("host", "127.0.0.1")
|
||||
self.port = int(cfg.get("port", 3306))
|
||||
self.user = cfg.get("user", "root")
|
||||
self.password = cfg.get("password", "")
|
||||
self.database = cfg.get("database", "more_dots")
|
||||
# 消息落库与结构化日志分表，避免误用 logging_mysql.table=structured_logs
|
||||
self.table = cfg.get("messages_table", "ipc_apbo.messages")
|
||||
self.conversation_table = cfg.get("conversation_table", "ipc_apbo.conversations")
|
||||
self.connect_timeout = int(cfg.get("connect_timeout", 5))
|
||||
self._inited = False
|
||||
self._conversation_schema_checked = False
|
||||
|
||||
def _get_conn(self):
|
||||
"""获取数据库连接"""
|
||||
return pymysql.connect(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
user=self.user,
|
||||
password=self.password,
|
||||
database=self.database,
|
||||
charset="utf8mb4",
|
||||
autocommit=True,
|
||||
connect_timeout=self.connect_timeout,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _log_local(event: str, payload: Optional[Dict[str, Any]] = None) -> None:
|
||||
print(json.dumps({
|
||||
"level": "ERROR",
|
||||
"event": event,
|
||||
"payload": payload or {},
|
||||
"created_at": DateTimeGenerator.now().epoch_millis,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
def _log_entity_debug(self, event: str, payload: Optional[Dict[str, Any]] = None) -> None:
|
||||
if not self.entity_debug_enabled:
|
||||
return
|
||||
print(json.dumps({
|
||||
"level": "DEBUG",
|
||||
"event": event,
|
||||
"payload": payload or {},
|
||||
"created_at": DateTimeGenerator.now().epoch_millis,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
@staticmethod
|
||||
def _now_ms() -> int:
|
||||
return DateTimeGenerator.now().epoch_millis
|
||||
|
||||
@staticmethod
|
||||
def _build_audit_fields(
|
||||
*,
|
||||
random_code: str,
|
||||
user: Optional[str],
|
||||
created_value: Any = None,
|
||||
updated_value: Any = None,
|
||||
) -> Dict[str, Any]:
|
||||
created_ms = MessageStorage._resolve_epoch_millis(created_value)
|
||||
updated_ms = MessageStorage._resolve_epoch_millis(updated_value, fallback=created_ms)
|
||||
created_bundle = DateTimeGenerator.bundle(created_ms, default_to_now=True)
|
||||
updated_bundle = DateTimeGenerator.bundle(updated_ms, default_to_now=True)
|
||||
operator = (user or "system").strip() if isinstance(user, str) else "system"
|
||||
return {
|
||||
"random_code": random_code,
|
||||
"create_user": operator,
|
||||
"create_date": created_bundle.db_datetime,
|
||||
"update_user": operator,
|
||||
"update_date": updated_bundle.db_datetime,
|
||||
"create_user_name": operator,
|
||||
"update_user_name": operator,
|
||||
"created_at": created_ms,
|
||||
"updated_at": updated_ms,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _resolve_epoch_millis(value: Any, fallback: Optional[int] = None) -> int:
|
||||
if value is None:
|
||||
return fallback if fallback is not None else DateTimeGenerator.now().epoch_millis
|
||||
|
||||
if isinstance(value, (int, float)):
|
||||
raw = int(value)
|
||||
digits = len(str(abs(raw)))
|
||||
if digits == 10:
|
||||
return raw * 1000
|
||||
return raw
|
||||
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if text.isdigit():
|
||||
raw = int(text)
|
||||
digits = len(text)
|
||||
if digits == 10:
|
||||
return raw * 1000
|
||||
if digits == 13:
|
||||
return raw
|
||||
|
||||
parsed = DateTimeGenerator.bundle(value, default_to_now=True)
|
||||
return parsed.epoch_millis
|
||||
|
||||
parsed = DateTimeGenerator.bundle(value, default_to_now=True)
|
||||
return parsed.epoch_millis
|
||||
|
||||
def _log_entity_stage(self, entity: str, action: str, stage: str, started_at: int, payload: Optional[Dict[str, Any]] = None) -> None:
|
||||
debug_payload = dict(payload or {})
|
||||
debug_payload.update({
|
||||
"entity": entity,
|
||||
"action": action,
|
||||
"stage": stage,
|
||||
"elapsed_ms": max(0, self._now_ms() - started_at),
|
||||
})
|
||||
self._log_entity_debug(f"message_storage.{entity}.{action}.{stage}", debug_payload)
|
||||
|
||||
@staticmethod
|
||||
def _split_table_reference(table_name: str, default_schema: str) -> tuple[str, str]:
|
||||
cleaned = str(table_name or "").strip()
|
||||
if "." in cleaned:
|
||||
schema_name, physical_table_name = cleaned.split(".", 1)
|
||||
else:
|
||||
schema_name, physical_table_name = default_schema, cleaned
|
||||
return schema_name.strip().strip("`"), physical_table_name.strip().strip("`")
|
||||
|
||||
def _ensure_conversation_schema(self, conn) -> None:
|
||||
if self._conversation_schema_checked or not self.enabled:
|
||||
return
|
||||
|
||||
started_at = self._now_ms()
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"ensure_schema",
|
||||
"start",
|
||||
started_at,
|
||||
{"conversation_table": self.conversation_table},
|
||||
)
|
||||
|
||||
schema_name, table_name = self._split_table_reference(self.conversation_table, self.database)
|
||||
probe_sql = """
|
||||
SELECT 1
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = %s AND table_name = %s AND column_name = %s
|
||||
LIMIT 1
|
||||
"""
|
||||
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(probe_sql, (schema_name, table_name, "name"))
|
||||
if cur.fetchone() is None:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"ensure_schema",
|
||||
"alter_needed",
|
||||
started_at,
|
||||
{"conversation_table": self.conversation_table, "missing_column": "name"},
|
||||
)
|
||||
alter_sql = f"""
|
||||
ALTER TABLE {self.conversation_table}
|
||||
ADD COLUMN name VARCHAR(255) NULL COMMENT '会话名称' AFTER user
|
||||
"""
|
||||
cur.execute(alter_sql)
|
||||
|
||||
self._conversation_schema_checked = True
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"ensure_schema",
|
||||
"success",
|
||||
started_at,
|
||||
{"conversation_table": self.conversation_table, "schema_checked": True},
|
||||
)
|
||||
|
||||
def _ensure_table(self) -> None:
|
||||
"""确保消息表存在"""
|
||||
if self._inited or not self.enabled:
|
||||
return
|
||||
|
||||
started_at = self._now_ms()
|
||||
self._log_entity_stage(
|
||||
"storage",
|
||||
"ensure_table",
|
||||
"start",
|
||||
started_at,
|
||||
{"messages_table": self.table, "conversation_table": self.conversation_table},
|
||||
)
|
||||
|
||||
message_sql = f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.table} (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
random_code VARCHAR(100) NULL COMMENT '业务主键',
|
||||
create_user VARCHAR(100) NULL COMMENT '创建人',
|
||||
create_date DATETIME NULL COMMENT '创建时间',
|
||||
update_user VARCHAR(100) NULL COMMENT '修改人',
|
||||
update_date DATETIME NULL COMMENT '修改时间',
|
||||
create_user_name VARCHAR(255) NULL COMMENT '创建人姓名',
|
||||
update_user_name VARCHAR(255) NULL COMMENT '修改人姓名',
|
||||
message_id VARCHAR(255) NULL COMMENT '消息 ID',
|
||||
conversation_id VARCHAR(255) NULL COMMENT '会话 ID',
|
||||
user VARCHAR(255) NULL COMMENT '用户标识',
|
||||
query LONGTEXT NULL COMMENT '用户查询',
|
||||
answer LONGTEXT NULL COMMENT '回答消息内容',
|
||||
feedback VARCHAR(255) NULL COMMENT '点赞 like / 点踩 dislike',
|
||||
feedback_content TEXT NULL COMMENT '点踩内容',
|
||||
created_at BIGINT NULL COMMENT '创建时间(毫秒时间戳)',
|
||||
updated_at BIGINT NULL COMMENT '更新时间(毫秒时间戳)',
|
||||
`log` JSON NULL COMMENT '当前对话日志(JSON字符串)',
|
||||
UNIQUE KEY uk_random_code (random_code),
|
||||
UNIQUE KEY uk_message_id (message_id),
|
||||
INDEX idx_conversation_id (conversation_id),
|
||||
INDEX idx_created_at (created_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='消息记录表';
|
||||
"""
|
||||
|
||||
conversation_sql = f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.conversation_table} (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
random_code VARCHAR(100) NULL COMMENT '业务主键',
|
||||
create_user VARCHAR(100) NULL COMMENT '创建人',
|
||||
create_date DATETIME NULL COMMENT '创建时间',
|
||||
update_user VARCHAR(100) NULL COMMENT '修改人',
|
||||
update_date DATETIME NULL COMMENT '修改时间',
|
||||
create_user_name VARCHAR(255) NULL COMMENT '创建人姓名',
|
||||
update_user_name VARCHAR(255) NULL COMMENT '修改人姓名',
|
||||
conversation_id VARCHAR(255) NULL COMMENT '会话 ID',
|
||||
user VARCHAR(255) NULL COMMENT '用户',
|
||||
name VARCHAR(512) NULL COMMENT '会话名称',
|
||||
status VARCHAR(255) NULL COMMENT '状态',
|
||||
introduction VARCHAR(255) NULL COMMENT '开场白',
|
||||
created_at BIGINT NULL COMMENT '创建时间(毫秒时间戳)',
|
||||
updated_at BIGINT NULL COMMENT '更新时间(毫秒时间戳)',
|
||||
UNIQUE KEY uk_conversation_random_code (random_code),
|
||||
UNIQUE KEY uk_conversation_id (conversation_id),
|
||||
INDEX idx_conversation_updated_at (updated_at)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COMMENT='会话记录表';
|
||||
"""
|
||||
|
||||
try:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(message_sql)
|
||||
cur.execute(conversation_sql)
|
||||
self._ensure_conversation_schema(conn)
|
||||
self._inited = True
|
||||
self._log_entity_stage(
|
||||
"storage",
|
||||
"ensure_table",
|
||||
"success",
|
||||
started_at,
|
||||
{"messages_table": self.table, "conversation_table": self.conversation_table},
|
||||
)
|
||||
except Exception as exc:
|
||||
self._log_entity_stage(
|
||||
"storage",
|
||||
"ensure_table",
|
||||
"failed",
|
||||
started_at,
|
||||
{"messages_table": self.table, "conversation_table": self.conversation_table, "error": str(exc)},
|
||||
)
|
||||
self._log_local("message_storage.ensure_table_failed", {
|
||||
"error": str(exc),
|
||||
"messages_table": self.table,
|
||||
"conversation_table": self.conversation_table,
|
||||
})
|
||||
# 开发阶段容错，避免初始化失败影响主流程
|
||||
self.enabled = False
|
||||
|
||||
def create_conversation(
|
||||
self,
|
||||
conversation_id: str,
|
||||
user: Optional[str],
|
||||
name: Optional[str],
|
||||
status: str,
|
||||
introduction: Optional[str],
|
||||
created_at: int,
|
||||
updated_at: int,
|
||||
) -> bool:
|
||||
started_at = self._now_ms()
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"create",
|
||||
"start",
|
||||
started_at,
|
||||
{
|
||||
"conversation_id": conversation_id,
|
||||
"user": user,
|
||||
"name_len": len(name or ""),
|
||||
"status": status,
|
||||
},
|
||||
)
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"create",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "reason": "storage_disabled"},
|
||||
)
|
||||
return False
|
||||
|
||||
self._ensure_table()
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"create",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "reason": "storage_disabled_after_init"},
|
||||
)
|
||||
return False
|
||||
|
||||
insert_sql = f"""
|
||||
INSERT INTO {self.conversation_table}(
|
||||
random_code, create_user, create_date, update_user, update_date,
|
||||
create_user_name, update_user_name,
|
||||
conversation_id, user, name, status, introduction, created_at, updated_at
|
||||
) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
"""
|
||||
try:
|
||||
audit = self._build_audit_fields(
|
||||
random_code=conversation_id,
|
||||
user=user,
|
||||
created_value=created_at,
|
||||
updated_value=updated_at,
|
||||
)
|
||||
with self._get_conn() as conn:
|
||||
self._ensure_conversation_schema(conn)
|
||||
with conn.cursor() as cur:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"create",
|
||||
"sql_execute",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "table": self.conversation_table},
|
||||
)
|
||||
cur.execute(
|
||||
insert_sql,
|
||||
(
|
||||
audit["random_code"],
|
||||
audit["create_user"],
|
||||
audit["create_date"],
|
||||
audit["update_user"],
|
||||
audit["update_date"],
|
||||
audit["create_user_name"],
|
||||
audit["update_user_name"],
|
||||
conversation_id,
|
||||
user,
|
||||
name,
|
||||
status,
|
||||
introduction,
|
||||
audit["created_at"],
|
||||
audit["updated_at"],
|
||||
),
|
||||
)
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"create",
|
||||
"success",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "table": self.conversation_table},
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"create",
|
||||
"failed",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "table": self.conversation_table, "error": str(exc)},
|
||||
)
|
||||
self._log_local("message_storage.create_conversation_failed", {
|
||||
"error": str(exc),
|
||||
"conversation_table": self.conversation_table,
|
||||
"conversation_id": conversation_id,
|
||||
})
|
||||
return False
|
||||
|
||||
def get_conversation_by_id(self, conversation_id: str) -> Optional[Dict[str, Any]]:
|
||||
started_at = self._now_ms()
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"get_by_id",
|
||||
"start",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id},
|
||||
)
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"get_by_id",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "reason": "storage_disabled"},
|
||||
)
|
||||
return None
|
||||
|
||||
self._ensure_table()
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"get_by_id",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "reason": "storage_disabled_after_init"},
|
||||
)
|
||||
return None
|
||||
|
||||
select_sql = f"""
|
||||
SELECT conversation_id, user, name, status, introduction, created_at, updated_at
|
||||
FROM {self.conversation_table}
|
||||
WHERE conversation_id = %s
|
||||
LIMIT 1
|
||||
"""
|
||||
try:
|
||||
with self._get_conn() as conn:
|
||||
self._ensure_conversation_schema(conn)
|
||||
with conn.cursor(pymysql.cursors.DictCursor) as cur:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"get_by_id",
|
||||
"sql_execute",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "table": self.conversation_table},
|
||||
)
|
||||
cur.execute(select_sql, (conversation_id,))
|
||||
result = cur.fetchone()
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"get_by_id",
|
||||
"success",
|
||||
started_at,
|
||||
{
|
||||
"conversation_id": conversation_id,
|
||||
"table": self.conversation_table,
|
||||
"found": bool(result),
|
||||
},
|
||||
)
|
||||
return dict(result) if result else None
|
||||
except Exception as exc:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"get_by_id",
|
||||
"failed",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "table": self.conversation_table, "error": str(exc)},
|
||||
)
|
||||
self._log_local("message_storage.get_conversation_failed", {
|
||||
"error": str(exc),
|
||||
"conversation_table": self.conversation_table,
|
||||
"conversation_id": conversation_id,
|
||||
})
|
||||
return None
|
||||
|
||||
def update_conversation_updated_at(self, conversation_id: str, updated_at: int) -> bool:
|
||||
started_at = self._now_ms()
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"update_updated_at",
|
||||
"start",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "updated_at": updated_at},
|
||||
)
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"update_updated_at",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "reason": "storage_disabled"},
|
||||
)
|
||||
return False
|
||||
|
||||
self._ensure_table()
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"update_updated_at",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "reason": "storage_disabled_after_init"},
|
||||
)
|
||||
return False
|
||||
|
||||
update_sql = f"""
|
||||
UPDATE {self.conversation_table}
|
||||
SET updated_at = %s,
|
||||
update_date = %s
|
||||
WHERE conversation_id = %s
|
||||
"""
|
||||
try:
|
||||
update_ms = self._resolve_epoch_millis(updated_at)
|
||||
update_bundle = DateTimeGenerator.bundle(update_ms, default_to_now=True)
|
||||
with self._get_conn() as conn:
|
||||
self._ensure_conversation_schema(conn)
|
||||
with conn.cursor() as cur:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"update_updated_at",
|
||||
"sql_execute",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "table": self.conversation_table},
|
||||
)
|
||||
affected_rows = cur.execute(
|
||||
update_sql,
|
||||
(update_ms, update_bundle.db_datetime, conversation_id),
|
||||
)
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"update_updated_at",
|
||||
"success",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "affected_rows": int(affected_rows or 0)},
|
||||
)
|
||||
return bool(affected_rows)
|
||||
except Exception as exc:
|
||||
self._log_entity_stage(
|
||||
"conversations",
|
||||
"update_updated_at",
|
||||
"failed",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "table": self.conversation_table, "error": str(exc)},
|
||||
)
|
||||
self._log_local("message_storage.update_conversation_failed", {
|
||||
"error": str(exc),
|
||||
"conversation_table": self.conversation_table,
|
||||
"conversation_id": conversation_id,
|
||||
})
|
||||
return False
|
||||
|
||||
def save_message(
|
||||
self,
|
||||
conversation_id: str,
|
||||
message_id: str,
|
||||
query: str,
|
||||
answer: Optional[str] = None,
|
||||
workflow_type: Optional[str] = None,
|
||||
user: Optional[str] = None,
|
||||
sql_query: Optional[str] = None,
|
||||
execution_result: Optional[Dict[str, Any]] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
created_at: Optional[int] = None,
|
||||
updated_at: Optional[int] = None,
|
||||
logs: Optional[List[str]] = None,
|
||||
) -> bool:
|
||||
started_at = self._now_ms()
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"create",
|
||||
"start",
|
||||
started_at,
|
||||
{
|
||||
"conversation_id": conversation_id,
|
||||
"message_id": message_id,
|
||||
"query_len": len(query or ""),
|
||||
"answer_len": len(answer or ""),
|
||||
"workflow_type": workflow_type,
|
||||
},
|
||||
)
|
||||
"""
|
||||
保存消息记录
|
||||
|
||||
Args:
|
||||
conversation_id: 会话 ID
|
||||
message_id: 消息 ID
|
||||
query: 用户查询
|
||||
answer: AI 回复
|
||||
workflow_type: 工作流类型 (conversation/tool_using)
|
||||
user: 用户标识
|
||||
sql_query: 生成的 SQL
|
||||
execution_result: SQL 执行结果
|
||||
metadata: 其他元数据
|
||||
logs: 过程日志（兼容 Java saveMessageToDB）
|
||||
|
||||
Returns:
|
||||
bool: 是否保存成功
|
||||
"""
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"create",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "message_id": message_id, "reason": "storage_disabled"},
|
||||
)
|
||||
return False
|
||||
|
||||
self._ensure_table()
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"create",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "message_id": message_id, "reason": "storage_disabled_after_init"},
|
||||
)
|
||||
return False
|
||||
|
||||
audit = self._build_audit_fields(
|
||||
random_code=message_id,
|
||||
user=user,
|
||||
created_value=created_at,
|
||||
updated_value=updated_at,
|
||||
)
|
||||
log_payload = {
|
||||
"workflow_type": workflow_type,
|
||||
"sql_query": sql_query,
|
||||
"execution_result": execution_result or {},
|
||||
"metadata": metadata or {},
|
||||
}
|
||||
normalized_logs = [str(item) for item in (logs or []) if str(item).strip()]
|
||||
if normalized_logs:
|
||||
log_payload["data"] = "\n".join(normalized_logs)
|
||||
message_record = MessagesDTO(
|
||||
message_id=message_id,
|
||||
conversation_id=conversation_id,
|
||||
user=user,
|
||||
query=query,
|
||||
answer=answer,
|
||||
feedback=None,
|
||||
feedback_content=None,
|
||||
created_at=audit["created_at"],
|
||||
updated_at=audit["updated_at"],
|
||||
log=log_payload,
|
||||
)
|
||||
|
||||
insert_sql = f"""
|
||||
INSERT INTO {self.table}(
|
||||
random_code, create_user, create_date, update_user, update_date,
|
||||
create_user_name, update_user_name,
|
||||
message_id, conversation_id, user, query, answer,
|
||||
feedback, feedback_content, created_at, updated_at, `log`
|
||||
) VALUES(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
"""
|
||||
|
||||
try:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"create",
|
||||
"sql_execute",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "message_id": message_id, "table": self.table},
|
||||
)
|
||||
cur.execute(
|
||||
insert_sql,
|
||||
(
|
||||
audit["random_code"],
|
||||
audit["create_user"],
|
||||
audit["create_date"],
|
||||
audit["update_user"],
|
||||
audit["update_date"],
|
||||
audit["create_user_name"],
|
||||
audit["update_user_name"],
|
||||
message_record.message_id,
|
||||
message_record.conversation_id,
|
||||
message_record.user,
|
||||
message_record.query,
|
||||
message_record.answer,
|
||||
message_record.feedback,
|
||||
message_record.feedback_content,
|
||||
message_record.created_at,
|
||||
message_record.updated_at,
|
||||
json.dumps(message_record.log, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"create",
|
||||
"success",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "message_id": message_id, "table": self.table},
|
||||
)
|
||||
return True
|
||||
except Exception as exc:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"create",
|
||||
"failed",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "message_id": message_id, "table": self.table, "error": str(exc)},
|
||||
)
|
||||
self._log_local("message_storage.save_failed", {
|
||||
"error": str(exc),
|
||||
"messages_table": self.table,
|
||||
"message_id": message_id,
|
||||
"conversation_id": conversation_id,
|
||||
})
|
||||
# 开发阶段容错，避免日志失败影响主流程
|
||||
return False
|
||||
|
||||
def get_conversation_history(
|
||||
self,
|
||||
conversation_id: str,
|
||||
limit: int = 20
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
获取会话历史消息
|
||||
|
||||
Args:
|
||||
conversation_id: 会话 ID
|
||||
limit: 返回消息数量
|
||||
|
||||
Returns:
|
||||
List[Dict[str, Any]]: 消息列表
|
||||
"""
|
||||
started_at = self._now_ms()
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"get_history",
|
||||
"start",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "limit": limit},
|
||||
)
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"get_history",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "reason": "storage_disabled"},
|
||||
)
|
||||
return []
|
||||
|
||||
self._ensure_table()
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"get_history",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "reason": "storage_disabled_after_init"},
|
||||
)
|
||||
return []
|
||||
|
||||
select_sql = f"""
|
||||
SELECT * FROM {self.table}
|
||||
WHERE conversation_id = %s
|
||||
ORDER BY created_at DESC
|
||||
LIMIT %s
|
||||
"""
|
||||
|
||||
try:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor(pymysql.cursors.DictCursor) as cur:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"get_history",
|
||||
"sql_execute",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "limit": limit, "table": self.table},
|
||||
)
|
||||
cur.execute(select_sql, (conversation_id, limit))
|
||||
results = cur.fetchall()
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"get_history",
|
||||
"success",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "count": len(results or [])},
|
||||
)
|
||||
return list(results)
|
||||
except Exception as exc:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"get_history",
|
||||
"failed",
|
||||
started_at,
|
||||
{"conversation_id": conversation_id, "error": str(exc)},
|
||||
)
|
||||
return []
|
||||
|
||||
def update_feedback_by_message_id(
|
||||
self,
|
||||
message_id: str,
|
||||
feedback: str,
|
||||
feedback_content: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""按 message_id 回写点赞/点踩反馈。"""
|
||||
started_at = self._now_ms()
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"update_feedback",
|
||||
"start",
|
||||
started_at,
|
||||
{"message_id": message_id, "feedback": feedback},
|
||||
)
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"update_feedback",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"message_id": message_id, "reason": "storage_disabled"},
|
||||
)
|
||||
return False
|
||||
|
||||
self._ensure_table()
|
||||
if not self.enabled:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"update_feedback",
|
||||
"skipped",
|
||||
started_at,
|
||||
{"message_id": message_id, "reason": "storage_disabled_after_init"},
|
||||
)
|
||||
return False
|
||||
|
||||
update_sql = f"""
|
||||
UPDATE {self.table}
|
||||
SET feedback = %s,
|
||||
feedback_content = %s,
|
||||
updated_at = %s,
|
||||
update_date = %s
|
||||
WHERE message_id = %s
|
||||
"""
|
||||
|
||||
normalized_feedback_content = (feedback_content or "").strip() or None
|
||||
now_bundle = DateTimeGenerator.now()
|
||||
try:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"update_feedback",
|
||||
"sql_execute",
|
||||
started_at,
|
||||
{"message_id": message_id, "table": self.table},
|
||||
)
|
||||
affected_rows = cur.execute(
|
||||
update_sql,
|
||||
(
|
||||
feedback,
|
||||
normalized_feedback_content,
|
||||
now_bundle.epoch_millis,
|
||||
now_bundle.db_datetime,
|
||||
message_id,
|
||||
),
|
||||
)
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"update_feedback",
|
||||
"success",
|
||||
started_at,
|
||||
{"message_id": message_id, "affected_rows": int(affected_rows or 0)},
|
||||
)
|
||||
return bool(affected_rows)
|
||||
except Exception as exc:
|
||||
self._log_entity_stage(
|
||||
"messages",
|
||||
"update_feedback",
|
||||
"failed",
|
||||
started_at,
|
||||
{"message_id": message_id, "error": str(exc)},
|
||||
)
|
||||
# 开发阶段容错，避免日志失败影响主流程
|
||||
return False
|
||||
|
||||
|
||||
# 全局单例
|
||||
_GLOBAL_MESSAGE_STORAGE: Optional[MessageStorage] = None
|
||||
|
||||
|
||||
def get_message_storage() -> MessageStorage:
|
||||
"""获取消息存储服务实例"""
|
||||
global _GLOBAL_MESSAGE_STORAGE
|
||||
if _GLOBAL_MESSAGE_STORAGE is None:
|
||||
_GLOBAL_MESSAGE_STORAGE = MessageStorage()
|
||||
return _GLOBAL_MESSAGE_STORAGE
|
||||
@@ -0,0 +1,104 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
import pymysql
|
||||
|
||||
from config import Config
|
||||
from services.common.datetime_utils import DateTimeGenerator
|
||||
|
||||
|
||||
class StructuredLogger:
|
||||
def __init__(self):
|
||||
cfg = Config.get_section("logging_mysql")
|
||||
self.enabled = str(cfg.get("enabled", "false")).lower() in ("1", "true", "yes")
|
||||
self.host = cfg.get("host", "127.0.0.1")
|
||||
self.port = int(cfg.get("port", 3306))
|
||||
self.user = cfg.get("user", "root")
|
||||
self.password = cfg.get("password", "")
|
||||
self.database = cfg.get("database", "more_dots")
|
||||
self.table = cfg.get("table", "structured_logs")
|
||||
self.connect_timeout = int(cfg.get("connect_timeout", 5))
|
||||
self._inited = False
|
||||
|
||||
def _get_conn(self):
|
||||
return pymysql.connect(
|
||||
host=self.host,
|
||||
port=self.port,
|
||||
user=self.user,
|
||||
password=self.password,
|
||||
database=self.database,
|
||||
charset="utf8mb4",
|
||||
autocommit=True,
|
||||
connect_timeout=self.connect_timeout,
|
||||
)
|
||||
|
||||
def _ensure_table(self) -> None:
|
||||
if self._inited or not self.enabled:
|
||||
return
|
||||
sql = f"""
|
||||
CREATE TABLE IF NOT EXISTS {self.table} (
|
||||
id BIGINT PRIMARY KEY AUTO_INCREMENT,
|
||||
trace_id VARCHAR(64) NOT NULL,
|
||||
level VARCHAR(16) NOT NULL,
|
||||
event VARCHAR(128) NOT NULL,
|
||||
error_code VARCHAR(64) NULL,
|
||||
payload JSON NULL,
|
||||
created_at DATETIME NOT NULL
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
||||
"""
|
||||
try:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(sql)
|
||||
self._inited = True
|
||||
except Exception:
|
||||
# 开发阶段容错，避免日志失败影响主流程
|
||||
self.enabled = False
|
||||
|
||||
def log(self, level: str, event: str, trace_id: str, payload: Optional[Dict[str, Any]] = None, error_code: Optional[str] = None) -> None:
|
||||
print(json.dumps({
|
||||
"trace_id": trace_id,
|
||||
"level": level,
|
||||
"event": event,
|
||||
"error_code": error_code,
|
||||
"payload": payload or {},
|
||||
"created_at": DateTimeGenerator.now().iso_str,
|
||||
}, ensure_ascii=False))
|
||||
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
self._ensure_table()
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
insert_sql = f"INSERT INTO {self.table}(trace_id, level, event, error_code, payload, created_at) VALUES(%s,%s,%s,%s,%s,%s)"
|
||||
try:
|
||||
with self._get_conn() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
insert_sql,
|
||||
(
|
||||
trace_id,
|
||||
level,
|
||||
event,
|
||||
error_code,
|
||||
json.dumps(payload or {}, ensure_ascii=False),
|
||||
DateTimeGenerator.now().db_datetime,
|
||||
),
|
||||
)
|
||||
except Exception:
|
||||
# 开发阶段容错，避免日志失败影响主流程
|
||||
return
|
||||
|
||||
|
||||
_GLOBAL_STRUCTURED_LOGGER: Optional[StructuredLogger] = None
|
||||
|
||||
|
||||
def get_structured_logger() -> StructuredLogger:
|
||||
global _GLOBAL_STRUCTURED_LOGGER
|
||||
if _GLOBAL_STRUCTURED_LOGGER is None:
|
||||
_GLOBAL_STRUCTURED_LOGGER = StructuredLogger()
|
||||
return _GLOBAL_STRUCTURED_LOGGER
|
||||
Reference in New Issue
Block a user