# 使用 Docker Compose 部署文档解析服务
cat > ~/rag-doc-parser/docker-compose.yml << 'EOF'
version: '3.8'
services:
  parser:
    image: ghcr.io/nickhould/docling-server:latest
    ports:
      - "5001:5000"
    volumes:
      - ./uploads:/app/uploads
      - ./output:/app/output
    environment:
      - MAX_WORKERS=2
      - TIMEOUT=300

  # 可选：OCR 服务（处理扫描件 PDF）
  ocr:
    image: ghcr.io/nickhould/docling-server:latest
    command: ["--ocr"]
    ports:
      - "5002:5000"
EOF

cd ~/rag-doc-parser && docker compose up -d

# 推荐的智能切分策略
from langchain_text_splitters import RecursiveCharacterTextSplitter

splitter = RecursiveCharacterTextSplitter(
    chunk_size=512,          # 每块 512 tokens
    chunk_overlap=64,        # 相邻块重叠 64 tokens（保留上下文）
    separators=[             # 优先在这些位置切分
        "\n\n",              # 段落之间
        "\n",                # 行之间
        ". ",                # 句子之间
        " ",                 # 单词之间
        "",                  # 强制切分
    ],
    length_function=len,
)

# 每个 chunk 携带元数据
chunks = [
    {
        "text": "退款政策：订单签收后 7 天内可申请全额退款...",
        "metadata": {
            "source": "company-handbook.pdf",
            "section": "财务政策 > 退款流程",
            "page": 12,
            "document_type": "policy"
        }
    }
]

mkdir -p ~/rag-chroma && cd ~/rag-chroma

cat > docker-compose.yml << 'EOF'
version: '3.8'
services:
  chroma:
    image: chromadb/chroma:latest
    ports:
      - "8000:8000"
    volumes:
      - chroma_data:/chroma/chroma
    environment:
      - ANONYMIZED_TELEMETRY=false
      - CHROMA_SERVER_AUTH_CREDENTIALS=admin:secure-password
      - CHROMA_SERVER_AUTH_CREDENTIALS_PROVIDER=chromadb.auth.simple_authn.SimpleAuthServerCredentialsProvider
      - CHROMA_SERVER_AUTH_PROVIDER=chromadb.auth.simple_authn.SimpleAuthServerAuthenticationProvider

volumes:
  chroma_data:
EOF

docker compose up -d

mkdir -p ~/rag-qdrant && cd ~/rag-qdrant

cat > docker-compose.yml << 'EOF'
version: '3.8'
services:
  qdrant:
    image: qdrant/qdrant:latest
    ports:
      - "6333:6333"   # REST API
      - "6334:6334"   # gRPC
    volumes:
      - qdrant_data:/qdrant/storage
    environment:
      - QDRANT__SERVICE__API_KEY=your-secret-key

  # Qdrant 面板（可选）
  qdrant_dashboard:
    image: ghcr.io/qdrant/dashboard:latest
    ports:
      - "8080:80"
    environment:
      - QDRANT__API_KEY=your-secret-key

volumes:
  qdrant_data:
EOF

docker compose up -d

from qdrant_client import QdrantClient
from qdrant_client.models import (
    Distance,
    VectorParams,
    PayloadSchemaType,
)

client = QdrantClient(url="http://localhost:6333", api_key="your-secret-key")

client.create_collection(
    collection_name="knowledge_base",
    vectors_config=VectorParams(
        size=1024,              # Embedding 维度（bge-large 是 1024）
        distance=Distance.COSINE,  # 余弦相似度
    ),
)

# 为常用过滤字段创建_payload_ 索引
client.create_payload_index(
    collection_name="knowledge_base",
    field_name="document_type",
    field_schema=PayloadSchemaType.KEYWORD,
)

client.create_payload_index(
    collection_name="knowledge_base",
    field_name="source",
    field_schema=PayloadSchemaType.KEYWORD,
)

# 使用 FastAPI + Sentence Transformers 部署 Embedding API
mkdir -p ~/rag-embedding && cd ~/rag-embedding

cat > requirements.txt << 'EOF'
fastapi==0.115.0
uvicorn==0.30.6
sentence-transformers==3.2.0
pydantic==2.9.0
EOF

cat > main.py << 'PYEOF'
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from sentence_transformers import SentenceTransformer
from typing import List
import numpy as np

app = FastAPI(title="RAG Embedding Service")

# 加载模型（首次启动会下载，约 1.3GB）
model = SentenceTransformer("BAAI/bge-large-zh-v1.5")

class EmbedRequest(BaseModel):
    texts: List[str]
    normalize: bool = True

class EmbedResponse(BaseModel):
    embeddings: List[List[float]]
    model: str = "bge-large-zh-v1.5"

@app.post("/embed", response_model=EmbedResponse)
def embed_texts(request: EmbedRequest):
    try:
        embeddings = model.encode(
            request.texts,
            normalize_embeddings=request.normalize,
            batch_size=32,
        )
        return EmbedResponse(embeddings=embeddings.tolist())
    except Exception as e:
        raise HTTPException(status_code=500, detail=str(e))

@app.get("/health")
def health():
    return {"status": "ok", "model": "bge-large-zh-v1.5"}
PYEOF

# 启动服务
pip install -r requirements.txt
uvicorn main:app --host 0.0.0.0 --port 8001 --workers 2

# 测试 Embedding 服务
curl http://localhost:8001/embed \
  -H "Content-Type: application/json" \
  -d '{"texts": ["什么是退款政策？", "公司年假规定"], "normalize": true}'

cat > docker-compose.yml << 'EOF'
version: '3.8'
services:
  embedding:
    build: .
    ports:
      - "8001:8001"
    volumes:
      - ~/.cache/huggingface:/root/.cache/huggingface  # 缓存模型文件
    deploy:
      resources:
        limits:
          memory: 4G
EOF

cat > Dockerfile << 'EOF'
FROM python:3.11-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
# 预下载模型，避免首次请求超时
RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTransformer('BAAI/bge-large-zh-v1.5')"
COPY main.py .
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "2"]
EOF

mkdir -p ~/rag-pipeline && cd ~/rag-pipeline

cat > requirements.txt << 'EOF'
langchain==0.3.0
langchain-community==0.3.0
langchain-core==0.3.0
langchain-text-splitters==0.3.0
chromadb==0.5.0
openai==1.40.0
tiktoken==0.7.0
EOF

cat > rag_pipeline.py << 'PYEOF'
"""
完整的 RAG 检索管线
"""
import os
from langchain_community.vectorstores import Chroma
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_core.documents import Document
from langchain_openai import OpenAIEmbeddings, ChatOpenAI
from langchain.chains import RetrievalQA
from langchain.retrievers import EnsembleRetriever

# 配置
CHROMA_PATH = "./chroma_db"
EMBEDDING_URL = "http://embedding-service:8001"  # 自托管 Embedding
CHAT_MODEL = "gpt-4o-mini"  # 或使用自托管的 Qwen2.5-7B

class RAGPipeline:
    def __init__(self):
        # 1. 初始化 Embedding（优先使用自托管）
        self.embeddings = OpenAIEmbeddings(
            openai_api_key="self-hosted",
            base_url=EMBEDDING_URL,
            model="bge-large-zh-v1.5",
            dimensions=1024,
        )

        # 2. 文档切分器
        self.splitter = RecursiveCharacterTextSplitter(
            chunk_size=512,
            chunk_overlap=64,
            separators=["\n\n", "\n", ". ", " ", ""],
        )

        # 3. 向量数据库
        self.vectorstore = Chroma(
            persist_directory=CHROMA_PATH,
            embedding_function=self.embeddings,
        )

    def ingest_documents(self, texts: List[str], metadata: List[dict]):
        """导入文档到向量库"""
        documents = [
            Document(page_content=text, metadata=meta)
            for text, meta in zip(texts, metadata)
        ]

        chunks = self.splitter.split_documents(documents)
        self.vectorstore.add_documents(chunks)
        print(f"✅ 已导入 {len(chunks)} 个文档块")

    def retrieve(self, query: str, top_k: int = 5) -> list:
        """
        检索相关文档块
        使用混合检索：向量检索 + 关键词检索
        """
        # 向量检索
        vector_retriever = self.vectorstore.as_retriever(
            search_type="similarity",
            search_kwargs={"k": top_k},
        )

        # 相似度检索（考虑相关性）
        mmr_retriever = self.vectorstore.as_retriever(
            search_type="mmr",
            search_kwargs={"k": top_k, "fetch_k": 20},
        )

        # 混合检索
        ensemble = EnsembleRetriever(
            retrievers=[vector_retriever, mmr_retriever],
            weights=[0.7, 0.3],
        )

        return ensemble.invoke(query)

    def qa_chain(self, query: str, temperature: float = 0.1) -> str:
        """生成基于检索结果的回答"""
        docs = self.retrieve(query)

        # 构建上下文
        context = "\n\n".join([doc.page_content for doc in docs])

        # 调用 LLM 生成回答
        llm = ChatOpenAI(
            model=CHAT_MODEL,
            temperature=temperature,
            openai_api_key=os.getenv("OPENAI_API_KEY", "sk-placeholder"),
            base_url=os.getenv("OPENAI_API_BASE", "https://api.openai.com/v1"),
        )

        prompt = f"""基于以下参考资料回答问题。如果资料中没有相关信息，请如实说明。

参考资料：
{context}

问题：{query}

请给出详细、准确的回答，并在回答末尾标注引用的文档来源。"""

        response = llm.invoke(prompt)
        return response.content

# 使用示例
if __name__ == "__main__":
    pipeline = RAGPipeline()

    # 导入文档
    pipeline.ingest_documents(
        texts=["公司的退款政策是签收后 7 天内可申请全额退款..."],
        metadata=[{"source": "handbook.pdf", "document_type": "policy"}],
    )

    # 检索并回答
    answer = pipeline.qa_chain("我们的退款政策是什么？")
    print(answer)
PYEOF

def evaluate_retrieval(query: str, expected_docs: List[str]) -> dict:
    """
    评估检索质量
    - Precision@K: 检索结果中有多少是相关的
    - Recall@K: 所有相关文档中有多少被检索到了
    - MRR: 第一个相关文档的排名倒数平均
    """
    retrieved = pipeline.retrieve(query, top_k=5)

    relevant_count = sum(1 for doc in retrieved if doc.metadata.get("relevant"))
    precision = relevant_count / len(retrieved) if retrieved else 0

    return {
        "precision_at_5": precision,
        "retrieved_count": len(retrieved),
        "documents": [doc.metadata for doc in retrieved],
    }

mkdir -p ~/rag-webui && cd ~/rag-webui

cat > docker-compose.yml << 'EOF'
version: '3.8'
services:
  open-webui:
    image: ghcr.io/open-webui/open-webui:main
    ports:
      - "3000:8080"
    environment:
      - OLLAMA_BASE_URL=http://embedding:11434
      - WEBUI_SECRET_KEY=your-secret-key
      - ENABLE_RAG_HYBRID_SEARCH=true
      - RAG_TOP_K=5
      - RAG_CHUNK_SIZE=512
      - RAG_CHUNK_OVERLAP=64
    volumes:
      - webui_data:/app/backend/data

volumes:
  webui_data:
EOF

docker compose up -d

# Dify 部署需要 4GB+ RAM
curl -o dify-env.txt https://raw.githubusercontent.com/langgenius/dify/main/docker/.env
# 修改配置指向你的自托管 Embedding 服务和向量数据库
sed -i 's/EMBEDDING_API_KEY=.*/EMBEDDING_API_KEY=your-key/' dify-env.txt
sed -i 's/VECTOR_STORE=.*/VECTOR_STORE=qdrant/' dify-env.txt
sed -i 's/QDRANT_URI=.*/QDRANT_URI=http://qdrant:6333/' dify-env.txt

docker compose -f docker/Compose.yaml up -d

<!-- simple-rag.html - 直接部署到 Nginx -->
<!DOCTYPE html>
<html lang="zh-CN">
  <head>
    <meta charset="UTF-8" />
    <title>AI 知识库</title>
    <style>
      body {
        font-family: -apple-system, sans-serif;
        max-width: 800px;
        margin: 0 auto;
        padding: 20px;
      }
      .chat {
        border: 1px solid #ddd;
        border-radius: 8px;
        padding: 16px;
        min-height: 400px;
      }
      .message {
        margin: 8px 0;
        padding: 8px 12px;
        border-radius: 4px;
      }
      .user {
        background: #e3f2fd;
        text-align: right;
      }
      .ai {
        background: #f5f5f5;
      }
      input {
        width: 100%;
        padding: 12px;
        border: 1px solid #ddd;
        border-radius: 4px;
        box-sizing: border-box;
      }
      button {
        margin-top: 8px;
        padding: 12px 24px;
        background: #1976d2;
        color: white;
        border: none;
        border-radius: 4px;
        cursor: pointer;
      }
    </style>
  </head>
  <body>
    <h1>📚 AI 知识库</h1>
    <div class="chat" id="chat"></div>
    <input
      type="text"
      id="query"
      placeholder="输入你的问题..."
      onkeypress="if(event.key==='Enter')ask()"
    />
    <button onclick="ask()">提问</button>

    <script>
      async function ask() {
        const query = document.getElementById("query").value;
        if (!query) return;

        // 显示用户问题
        addMessage(query, "user");
        document.getElementById("query").value = "";

        // 调用 RAG API
        const resp = await fetch("/api/rag/query", {
          method: "POST",
          headers: { "Content-Type": "application/json" },
          body: JSON.stringify({ query }),
        });
        const data = await resp.json();
        addMessage(data.answer, "ai");
      }

      function addMessage(text, role) {
        const chat = document.getElementById("chat");
        const div = document.createElement("div");
        div.className = `message ${role}`;
        div.textContent = text;
        chat.appendChild(div);
        chat.scrollTop = chat.scrollHeight;
      }
    </script>
  </body>
</html>

def rewrite_query(original_query: str) -> List[str]:
    """
    将用户问题改写为多个角度的查询
    例如："退款政策" → ["如何申请退款", "退款条件是什么", "退款流程"]
    """
    llm = ChatOpenAI(model="gpt-4o-mini", temperature=0.5)
    prompt = f"""
    用户的问题是："{original_query}"

    请将其改写为 3 个不同角度但意图相同的问题，用于文档检索。
    只输出问题列表，每行一个问题，不要其他内容。
    """
    response = llm.invoke(prompt)
    return [q.strip() for q in response.content.strip().split("\n")]

def hyde_retrieve(query: str) -> List[Document]:
    """
    HyDE: Hypothetical Document Embeddings
    先生成一个假设性文档，再用它做检索
    """
    llm = ChatOpenAI(model="gpt-4o-mini")
    prompt = f"""
    假设你有一份关于以下问题的完美文档，请生成这个文档的关键段落。
    问题：{query}

    只输出文档内容，不要解释。
    """
    hypothetical_doc = llm.invoke(prompt).content

    # 用假设性文档做检索
    return vectorstore.similarity_search(hypothetical_doc, k=5)

# 部署 BGE Reranker
docker run -d --name reranker \
  -p 8002:8000 \
  -e MODEL_PATH="BAAI/bge-reranker-v2-m3" \
  bge-reranker:latest

# 检索 + 重排序管线
def retrieve_with_rerank(query: str, top_k: int = 5) -> List[Document]:
    # 第一步：向量检索召回 Top-50
    candidates = vectorstore.similarity_search(query, k=50)

    # 第二步：用重排序模型精选 Top-5
    passages = [{"query": query, "passage": doc.page_content} for doc in candidates]
    scores = reranker_service.rank(passages)

    # 按分数排序，取 Top-K
    ranked_indices = sorted(range(len(scores)), key=lambda i: scores[i], reverse=True)
    return [candidates[i] for i in ranked_indices[:top_k]]

def manage_document_lifecycle():
    """
    文档生命周期管理
    - 新文档：解析 → Embed → 入库
    - 更新文档：删除旧版本 → 重新入库
    - 过期文档：标记为 deprecated，降低检索权重
    """
    # 检查文档最后更新时间
    # 超过 90 天的文档标记为待审核
    # 超过 180 天的文档归档
    pass

# ChromaDB 备份
tar czf chroma-backup-$(date +%Y%m%d).tar.gz ./chroma_db/

# Qdrant 备份（在线热备）
curl -X POST http://localhost:6333/collections/knowledge_base/backup
# 备份文件存储在 ./backups/ 目录

# 建议：每日自动备份到对象存储（S3 兼容）
aws s3 cp ./backups/ s3://rag-backups/ --recursive