more_dots/services/integrations/ragflow_sync.py

import json
import os
from typing import Any, Dict, List
from pathlib import Path

import httpx

from config import Config


def _dump_json_content(data: Dict[str, Any]) -> str:
    return json.dumps(data, ensure_ascii=False, indent=2)


def _extract_tables_map(data: Dict[str, Any]) -> Dict[str, Any]:
    """兼容两种结构：{"tables": {...}} 或直接 {...}"""
    tables = data.get("tables") if isinstance(data, dict) else None
    if isinstance(tables, dict):
        return tables
    if isinstance(data, dict):
        return data
    return {}


class RagflowSync:
    """RAGFlow 同步工具"""

    def __init__(self):
        cfg = Config.get_section("ragflow")
        self._base_url = cfg.get("url", "").rstrip("/")
        self._api_key = cfg.get("api_key", "")
        self._table_retrieval_dataset_id = (cfg.get("table_retrieval_dataset_id") or "").strip()
        self._sql_gen_dataset_id = (cfg.get("sql_gen_dataset_id") or "").strip()

    @staticmethod
    def _project_root() -> Path:
        """返回项目根目录。"""
        return Path(__file__).resolve().parents[2]

    def _collect_sql_gen_documents(self) -> tuple[List[Dict[str, Any]], List[Dict[str, str]]]:
        """收集 SQL 生成文档，并跳过空文件/非法 JSON。"""
        prompts_dir = self._project_root() / "config" / "sql_gen_prompts"
        documents: List[Dict[str, Any]] = []
        warnings: List[Dict[str, str]] = []

        for name in os.listdir(prompts_dir):
            if not name.endswith(".json"):
                continue

            path = prompts_dir / name
            raw_text = path.read_text(encoding="utf-8")
            if not raw_text.strip():
                warnings.append({"file": name, "reason": "empty_file"})
                continue

            try:
                prompt = json.loads(raw_text)
            except json.JSONDecodeError as exc:
                warnings.append({"file": name, "reason": f"invalid_json:{exc}"})
                continue

            table = prompt.get("table") or path.stem
            documents.append({"filename": f"{table}.txt", "content": _dump_json_content(prompt)})

        return documents, warnings

    def upload_documents(self, dataset_id: str, documents: List[Dict[str, Any]]) -> Dict[str, Any]:
        """上传文档到指定知识库（每个文档单独上传）"""
        if not self._base_url:
            raise RuntimeError("未配置 ragflow.url")
        if not dataset_id:
            raise RuntimeError("dataset_id 为空，无法上传文档")
        if not documents:
            raise RuntimeError("没有可上传的文档内容")
        
        # 构建正确的 URL
        url = f"{self._base_url}/api/v1/datasets/{dataset_id}/documents"
        
        headers = {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {}

        results: List[Dict[str, Any]] = []
        with httpx.Client(timeout=60) as client:
            for idx, doc in enumerate(documents, start=1):
                content = str(doc.get("content", ""))
                filename = str(doc.get("filename") or f"doc_{idx}.txt")
                files = {"file": (filename, content.encode("utf-8"), "text/plain")}

                print(f"上传文档 URL: {url}")
                print(f"上传文件名: {filename}")

                response = client.post(url, files=files, headers=headers)
                response.raise_for_status()
                result = response.json()
                print(f"RAGFlow 上传响应: {result}")
                results.append(result)

        dataset_detail = self._get_dataset_detail(dataset_id)
        chunk_method = self._extract_chunk_method(dataset_detail)
        if chunk_method is None:
            chunk_method = self._extract_chunk_method_from_upload_results(results)
        # 参考 Java 实现：查询知识库文档 ID 后统一调用 chunks 解析
        doc_ids = self._list_document_ids(dataset_id)
        parse_results = self._auto_parse_documents(dataset_id, doc_ids)
        upload_status = self._build_parse_status_from_upload_results(results)

        return {
            "ok": True,
            "count": len(results),
            "results": results,
            "chunk_method": chunk_method,
            "upload_status": upload_status,
            "parse": parse_results,
        }

    def _get_dataset_detail(self, dataset_id: str) -> Dict[str, Any]:
        """查询知识库详情（用于读取 chunk_method）"""
        url = f"{self._base_url}/api/v1/datasets/{dataset_id}"
        headers = {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {}
        with httpx.Client(timeout=30) as client:
            resp = client.get(url, headers=headers)
        resp.raise_for_status()
        return resp.json()

    @staticmethod
    def _extract_chunk_method(dataset_detail: Dict[str, Any]) -> Any:
        """从知识库详情提取 chunk_method"""
        data = dataset_detail.get("data")
        if isinstance(data, dict):
            if "chunk_method" in data:
                return data.get("chunk_method")
            parser_cfg = data.get("parser_config") or {}
            if isinstance(parser_cfg, dict):
                return parser_cfg.get("chunk_method")
        return None

    @staticmethod
    def _extract_chunk_method_from_upload_results(upload_results: List[Dict[str, Any]]) -> Any:
        """从上传响应中提取 chunk_method（兼容不同版本返回结构）"""
        for item in upload_results:
            data = item.get("data")
            records = data if isinstance(data, list) else [data] if isinstance(data, dict) else []
            for rec in records:
                if not isinstance(rec, dict):
                    continue
                if rec.get("chunk_method"):
                    return rec.get("chunk_method")
                parser_cfg = rec.get("parser_config") or {}
                if isinstance(parser_cfg, dict) and parser_cfg.get("chunk_method"):
                    return parser_cfg.get("chunk_method")
        return None

    @staticmethod
    def _extract_uploaded_doc_ids(upload_results: List[Dict[str, Any]]) -> List[str]:
        """从上传结果中提取文档 ID"""
        ids: List[str] = []
        for item in upload_results:
            data = item.get("data")
            if isinstance(data, list):
                for d in data:
                    if isinstance(d, dict) and d.get("id"):
                        ids.append(str(d.get("id")))
            elif isinstance(data, dict) and data.get("id"):
                ids.append(str(data.get("id")))
        return ids

    @staticmethod
    def _build_parse_status_from_upload_results(upload_results: List[Dict[str, Any]]) -> Dict[str, Any]:
        """根据上传返回构造解析状态（上传接口已触发解析，无需额外 parse API）"""
        details: List[Dict[str, Any]] = []
        for item in upload_results:
            data = item.get("data")
            records = data if isinstance(data, list) else [data] if isinstance(data, dict) else []
            for rec in records:
                if not isinstance(rec, dict):
                    continue
                details.append(
                    {
                        "doc_id": rec.get("id"),
                        "name": rec.get("name") or rec.get("location"),
                        "run": rec.get("run"),
                        "chunk_method": rec.get("chunk_method")
                        or (rec.get("parser_config") or {}).get("chunk_method"),
                    }
                )
        return {
            "ok": True,
            "trigger": "upload_endpoint",
            "message": "文档上传接口已触发解析流程，无需单独调用 parse API",
            "count": len(details),
            "details": details,
        }

    def _auto_parse_documents(self, dataset_id: str, doc_ids: List[str]) -> Dict[str, Any]:
        """调用官方 chunks 接口触发解析"""
        if not doc_ids:
            return {"ok": False, "message": "未提取到文档ID，无法触发解析", "count": 0, "details": []}

        url = f"{self._base_url}/api/v1/datasets/{dataset_id}/chunks"
        headers = {
            "Content-Type": "application/json",
            "Authorization": f"Bearer {self._api_key}",
        } if self._api_key else {"Content-Type": "application/json"}
        payload = {"document_ids": doc_ids}

        with httpx.Client(timeout=60) as client:
            resp = client.post(url, headers=headers, json=payload)

        if resp.status_code >= 400:
            return {
                "ok": False,
                "trigger": "chunks_api",
                "status": resp.status_code,
                "message": resp.text,
                "count": len(doc_ids),
                "details": [{"doc_id": d} for d in doc_ids],
            }

        body: Any
        try:
            body = resp.json()
        except Exception:
            body = resp.text

        return {
            "ok": True,
            "trigger": "chunks_api",
            "count": len(doc_ids),
            "details": [{"doc_id": d} for d in doc_ids],
            "response": body,
        }

    def _list_document_ids(self, dataset_id: str) -> List[str]:
        """获取知识库中的全部文档 ID（用于覆盖更新）"""
        if not self._base_url:
            raise RuntimeError("未配置 ragflow.url")
        if not dataset_id:
            raise RuntimeError("dataset_id 为空，无法查询文档")

        url = f"{self._base_url}/api/v1/datasets/{dataset_id}/documents"
        headers = {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {}

        ids: List[str] = []
        page = 1
        page_size = 100

        with httpx.Client(timeout=60) as client:
            while True:
                resp = client.get(url, headers=headers, params={"page": page, "page_size": page_size})
                resp.raise_for_status()
                body = resp.json()
                data = body.get("data")
                if isinstance(data, dict):
                    docs = data.get("docs") or data.get("list") or []
                elif isinstance(data, list):
                    docs = data
                else:
                    docs = []

                if not docs:
                    break

                for item in docs:
                    if isinstance(item, dict) and item.get("id"):
                        ids.append(str(item.get("id")))

                if len(docs) < page_size:
                    break
                page += 1

        return ids

    def _delete_documents(self, dataset_id: str, doc_ids: List[str]) -> Dict[str, Any]:
        """按 ID 删除文档"""
        if not doc_ids:
            return {"ok": True, "deleted": 0}

        url = f"{self._base_url}/api/v1/datasets/{dataset_id}/documents"
        headers = {"Authorization": f"Bearer {self._api_key}"} if self._api_key else {}
        payload = {"ids": doc_ids}

        with httpx.Client(timeout=60) as client:
            resp = client.request("DELETE", url, headers=headers, json=payload)
        resp.raise_for_status()
        return resp.json()

    def replace_documents(self, dataset_id: str, documents: List[Dict[str, Any]]) -> Dict[str, Any]:
        """覆盖更新：先删后传，避免“update 变新增”"""
        ids = self._list_document_ids(dataset_id)
        if ids:
            self._delete_documents(dataset_id, ids)
        return self.upload_documents(dataset_id, documents)

    def update_table_retrieval_documents(self) -> Dict[str, Any]:
        """更新表名检索文档（仅文档内容）"""
        if not self._table_retrieval_dataset_id:
            raise RuntimeError("未配置 ragflow.table_retrieval_dataset_id，无法更新表名检索文档")

        tables_file = self._project_root() / "config" / "table_retrieval_prompts" / "tables.json"
        with open(tables_file, "r", encoding="utf-8") as f:
            data = json.load(f)

        tables = _extract_tables_map(data)
        documents = [
            {
                "filename": f"{k}.txt",
                "content": _dump_json_content({"table": k, "templates": v}),
            }
            for k, v in tables.items()
        ]
        return self.replace_documents(self._table_retrieval_dataset_id, documents)

    def sync_table_retrieval(self) -> Dict[str, Any]:
        """兼容旧脚本：同步表名检索文档，采用覆盖更新避免旧表残留。"""
        return self.update_table_retrieval_documents()

    def update_sql_gen_documents(self) -> Dict[str, Any]:
        """更新 SQL 生成文档（仅文档内容）"""
        if not self._sql_gen_dataset_id:
            raise RuntimeError("未配置 ragflow.sql_gen_dataset_id，无法更新 SQL 生成文档")

        documents, warnings = self._collect_sql_gen_documents()
        if not documents:
            raise RuntimeError(f"SQL 生成提示词目录中没有可同步的有效 JSON 文档，warnings={warnings}")

        result = self.replace_documents(self._sql_gen_dataset_id, documents)
        result["warnings"] = warnings
        result["valid_document_count"] = len(documents)
        return result

    def sync_sql_gen_prompts(self) -> Dict[str, Any]:
        """兼容旧脚本：同步 SQL 生成提示词文档，采用覆盖更新避免旧 prompt 残留。"""
        return self.update_sql_gen_documents()

    def upload_table_retrieval(self) -> Dict[str, Any]:
        """上传表名检索模板文档 - 直接上传整个 JSON 文件"""
        if not self._table_retrieval_dataset_id:
            raise RuntimeError("未配置 ragflow.table_retrieval_dataset_id，无法上传表名检索模板")
        
        tables_file = self._project_root() / "config" / "table_retrieval_prompts" / "tables.json"

        if not os.path.exists(tables_file):
            raise RuntimeError(f"表名检索模板文件不存在: {tables_file}")
        
        # 读取整个 JSON 文件内容
        with open(tables_file, "r", encoding="utf-8") as f:
            data = json.load(f)

        tables = _extract_tables_map(data)
        # 每个 key 一个文档，配合 One 解析时每个表单独成块
        documents = [
            {
                "filename": f"{k}.txt",
                "content": _dump_json_content({"table": k, "templates": v}),
            }
            for k, v in tables.items()
        ]
        return self.upload_documents(self._table_retrieval_dataset_id, documents)

    def upload_sql_gen(self) -> Dict[str, Any]:
        """上传 SQL 生成提示词文档"""
        if not self._sql_gen_dataset_id:
            raise RuntimeError("未配置 ragflow.sql_gen_dataset_id，无法上传 SQL 生成提示词")
        
        prompts_dir = self._project_root() / "config" / "sql_gen_prompts"

        if not os.path.exists(prompts_dir):
            raise RuntimeError(f"SQL 生成提示词目录不存在: {prompts_dir}")

        documents, warnings = self._collect_sql_gen_documents()
        if not documents:
            raise RuntimeError(f"SQL 生成提示词目录中没有可上传的有效 JSON 文档，warnings={warnings}")

        result = self.upload_documents(self._sql_gen_dataset_id, documents)
        result["warnings"] = warnings
        result["valid_document_count"] = len(documents)
        return result