3

backend/app/processors/base.py

@@ -0,0 +1,12 @@
+"""文档处理器抽象（扩展接口 3：同步 → 异步任务）。"""
+
+from typing import Protocol, runtime_checkable
+
+
+@runtime_checkable
+class DocumentProcessor(Protocol):
+    """文档处理统一接口。业务代码通过此接口处理文档，不直接写解析逻辑。"""
+
+    def process(self, document_id: str) -> None:
+        """处理文档：解析 → 提取元数据 → 保存 Markdown。"""
+        ...

backend/app/processors/local_processor.py

@@ -0,0 +1,164 @@
+"""本地文档处理器（MVP：同步处理）。"""
+
+from pathlib import Path
+
+from app.core.logging import get_logger
+from app.core.security import decrypt_token, encrypt_token, generate_token, hash_token
+from app.models.document import Document
+from app.processors.parsers.docx_parser import parse_docx_with_python_docx
+from app.processors.parsers.markitdown_parser import parse_with_markitdown
+from app.processors.parsers.pdf_parser import has_text_layer, parse_pdf_with_pymupdf
+from app.repositories.doc_repo import DocumentRepository
+from app.storage.local_storage import get_storage
+from app.storage.object_keys import markdown_object_key
+from sqlalchemy.orm import Session
+
+logger = get_logger(__name__)
+
+
+class LocalDocumentProcessor:
+    """本地文档处理器：在同一进程中完成解析。"""
+
+    def __init__(self, session: Session) -> None:
+        self._session = session
+        self._doc_repo = DocumentRepository(session)
+
+    def process(self, document_id: str) -> None:
+        """处理文档：解析 → 提取元数据 → 保存 Markdown → 更新 DB。"""
+        doc = self._doc_repo.get_by_id(document_id)
+        if doc is None:
+            logger.error("Document not found: %s", document_id)
+            return
+
+        # 更新状态为 PROCESSING
+        doc.status = "PROCESSING"
+        self._session.commit()
+
+        try:
+            # 读取原始文件
+            storage = get_storage()
+            file_content = storage.read(doc.storage_path)
+
+            # 解析
+            markdown = self._parse_file(file_content, doc.file_ext, doc.original_filename)
+
+            if markdown is None:
+                # 解析失败
+                if doc.file_ext == ".pdf" and not has_text_layer(file_content):
+                    doc.error_code = "SCANNED_PDF_NO_TEXT_LAYER"
+                    doc.status = "FAILED"
+                else:
+                    doc.error_code = "PARSING_FAILED"
+                    doc.status = "FAILED"
+                self._session.commit()
+                logger.warning("Document parsing failed: %s (%s)", doc.id, doc.error_code)
+                return
+
+            # 清洗 Markdown
+            markdown = self._clean_markdown(markdown)
+
+            # 保存 Markdown 文件
+            md_key = markdown_object_key(
+                user_id=doc.user_id,
+                knowledge_base_id=doc.knowledge_base_id,
+                document_id=doc.id,
+            )
+            storage.save(md_key, markdown.encode("utf-8"))
+            doc.markdown_path = md_key
+
+            # 提取元数据
+            title = self._extract_title(markdown, doc.original_filename)
+            summary = self._extract_summary(markdown)
+            keywords = self._extract_keywords(markdown)
+
+            doc.title = title
+            doc.content_summary = summary
+            doc.keywords = keywords
+            doc.status = "READY"
+            self._session.commit()
+
+            logger.info("Document processed: %s → READY", doc.id)
+
+        except Exception as e:
+            logger.error("Document processing error: %s - %s", doc.id, e, exc_info=True)
+            doc.status = "FAILED"
+            doc.error_code = "PARSING_FAILED"
+            self._session.commit()
+
+    def _parse_file(self, content: bytes, ext: str, filename: str) -> str | None:
+        """按扩展名路由到对应解析器。"""
+        # 优先 MarkItDown
+        markdown = parse_with_markitdown(content, filename)
+        if markdown and markdown.strip():
+            return markdown
+
+        # Fallback
+        if ext == ".pdf":
+            return parse_pdf_with_pymupdf(content)
+        elif ext == ".docx":
+            return parse_docx_with_python_docx(content)
+
+        return None
+
+    def _clean_markdown(self, markdown: str) -> str:
+        """清洗 Markdown：压缩空行、规整标题层级。"""
+        import re
+
+        # 压缩连续空行为最多 2 个
+        markdown = re.sub(r"\n{3,}", "\n\n", markdown)
+        # 截断超长行（> 1000 字符的行）
+        lines = markdown.split("\n")
+        cleaned = []
+        for line in lines:
+            if len(line) > 1000:
+                line = line[:1000] + "..."
+            cleaned.append(line)
+        return "\n".join(cleaned)
+
+    def _extract_title(self, markdown: str, fallback: str) -> str:
+        """提取标题：H1 → 文件名（去扩展名）。"""
+        import re
+
+        # 找第一个 H1
+        match = re.search(r"^#\s+(.+)$", markdown, re.MULTILINE)
+        if match:
+            return match.group(1).strip()
+
+        # Fallback：文件名去扩展名
+        return Path(fallback).stem
+
+    def _extract_summary(self, markdown: str, max_len: int = 200) -> str:
+        """抽取式摘要：取正文前 ~200 字纯文本。"""
+        import re
+
+        # 去掉 Markdown 标记
+        text = re.sub(r"[#*_`\[\]()>]", "", markdown)
+        text = re.sub(r"\s+", " ", text).strip()
+
+        if len(text) <= max_len:
+            return text
+
+        # 在 max_len 附近找句号/逗号断句
+        cut = text[:max_len]
+        for sep in ["。", "！", "？", "；", ".", "!", "?", "；"]:
+            idx = cut.rfind(sep)
+            if idx > max_len * 0.5:
+                return cut[: idx + 1]
+        return cut + "..."
+
+    def _extract_keywords(self, markdown: str, top_k: int = 10) -> str:
+        """提取关键词（jieba TF-IDF）。"""
+        try:
+            import jieba.analyse
+
+            # 去掉 Markdown 标记
+            import re
+
+            text = re.sub(r"[#*_`\[\]()>]", "", markdown)
+            text = re.sub(r"\s+", " ", text).strip()
+
+            keywords = jieba.analyse.extract_tags(text, topK=top_k)
+            return ",".join(keywords)
+        except Exception as e:
+            logger.warning("关键词提取失败: %s", e)
+            return ""

backend/app/processors/parsers/docx_parser.py

@@ -0,0 +1,54 @@
+"""DOCX 解析器（python-docx fallback）。"""
+
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def parse_docx_with_python_docx(file_content: bytes) -> str | None:
+    """用 python-docx 解析 DOCX，返回 Markdown 或 None（失败时）。"""
+    try:
+        import io
+
+        from docx import Document
+
+        doc = Document(io.BytesIO(file_content))
+        md_parts = []
+
+        for para in doc.paragraphs:
+            text = para.text.strip()
+            if not text:
+                continue
+
+            style = para.style.name.lower()
+            if style.startswith("heading 1"):
+                md_parts.append(f"# {text}")
+            elif style.startswith("heading 2"):
+                md_parts.append(f"## {text}")
+            elif style.startswith("heading 3"):
+                md_parts.append(f"### {text}")
+            elif style.startswith("heading"):
+                md_parts.append(f"#### {text}")
+            elif style.startswith("list"):
+                md_parts.append(f"- {text}")
+            else:
+                md_parts.append(text)
+
+        # 处理表格
+        for table in doc.tables:
+            rows = []
+            for row in table.rows:
+                cells = [cell.text.strip() for cell in row.cells]
+                rows.append("| " + " | ".join(cells) + " |")
+            if rows:
+                # 添加表头分隔行
+                if len(rows) > 1:
+                    sep = "| " + " | ".join(["---"] * len(table.columns)) + " |"
+                    rows.insert(1, sep)
+                md_parts.append("\n".join(rows))
+
+        markdown = "\n\n".join(md_parts)
+        return markdown if markdown.strip() else None
+    except Exception as e:
+        logger.warning("python-docx 解析失败: %s", e)
+        return None

backend/app/processors/parsers/markitdown_parser.py

@@ -0,0 +1,30 @@
+"""MarkItDown 解析器（优先）。"""
+
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def parse_with_markitdown(file_content: bytes, filename: str) -> str | None:
+    """用 MarkItDown 解析文档，返回 Markdown 或 None（失败时）。"""
+    try:
+        from markitdown import MarkItDown
+
+        md = MarkItDown()
+        # MarkItDown 需要文件路径或文件对象，用临时文件
+        import tempfile
+        from pathlib import Path
+
+        suffix = Path(filename).suffix.lower()
+        with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
+            tmp.write(file_content)
+            tmp_path = tmp.name
+
+        try:
+            result = md.convert(tmp_path)
+            return result.text_content if result and result.text_content else None
+        finally:
+            Path(tmp_path).unlink(missing_ok=True)
+    except Exception as e:
+        logger.warning("MarkItDown 解析失败 (%s): %s", filename, e)
+        return None

backend/app/processors/parsers/pdf_parser.py

@@ -0,0 +1,47 @@
+"""PDF 解析器（PyMuPDF fallback）。"""
+
+from app.core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+def parse_pdf_with_pymupdf(file_content: bytes) -> str | None:
+    """用 PyMuPDF 解析 PDF，返回 Markdown 或 None（失败时）。"""
+    try:
+        import pymupdf  # PyMuPDF >= 1.24
+
+        doc = pymupdf.open(stream=file_content, filetype="pdf")
+        pages = []
+        for page_num in range(len(doc)):
+            page = doc.load_page(page_num)
+            text = page.get_text()
+            if text.strip():
+                pages.append(text)
+        doc.close()
+
+        if not pages:
+            return None
+
+        # 组合成 Markdown（每页一个段落）
+        markdown = "\n\n".join(pages)
+        return markdown
+    except Exception as e:
+        logger.warning("PyMuPDF 解析失败: %s", e)
+        return None
+
+
+def has_text_layer(file_content: bytes) -> bool:
+    """检查 PDF 是否有文本层。"""
+    try:
+        import pymupdf
+
+        doc = pymupdf.open(stream=file_content, filetype="pdf")
+        for page_num in range(min(3, len(doc))):  # 检查前 3 页
+            page = doc.load_page(page_num)
+            if page.get_text().strip():
+                doc.close()
+                return True
+        doc.close()
+        return False
+    except Exception:
+        return False