diff --git a/UPSTREAM_SYNC.md b/UPSTREAM_SYNC.md new file mode 100644 index 0000000..bea08e2 --- /dev/null +++ b/UPSTREAM_SYNC.md @@ -0,0 +1,86 @@ +# Upstream Sync Record + +This library is a standalone, pip-installable extraction of the `deepdoc/` +module from [infiniflow/ragflow](https://github.com/infiniflow/ragflow), +restructured around explicit configuration injection +(`PdfModelConfig` / `TokenizerConfig`, `DEEPDOC_*` env vars) and +ModelScope-based model distribution instead of ragflow's global settings and +HuggingFace runtime downloads. + +Upstream changes are **ported by behavior, never by wholesale file copy**, +so the library's public API (`from deepdoc import PdfParser, ...`) and +packaging stay stable. + +## Sync history + +### 2026-07-23 — synced to ragflow `18ea0fb7` (2026-07-22) + +Ported fixes (upstream PR references): + +| Area | Fix | Upstream | +|---|---|---| +| vision/operators | `StandardizeImag` → `StandardizeImage` rename; the dynamic `getattr(operators, "StandardizeImage")` dispatch silently skipped image standardization before | [#16785](https://github.com/infiniflow/ragflow/pull/16785) | +| mineru/tcadp | zip extraction hardening: reject symlink / encrypted / absolute-path / `..`-traversal / zip-slip entries; streaming download for TCADP results | [#12527](https://github.com/infiniflow/ragflow/pull/12527) | +| html_parser | parse bodyless HTML fragments (`soup.body or soup`) | [#16423](https://github.com/infiniflow/ragflow/pull/16423) | +| html_parser | switch `html5lib` → `html.parser` (aligns with upstream #14486-era behavior the later fixes assume) | [#14486](https://github.com/infiniflow/ragflow/pull/14486) | +| html_parser | h4 heading mapping `#####` → `####` | [#13833](https://github.com/infiniflow/ragflow/pull/13833) | +| html_parser | oversized-block splitting preserves original text (CJK-aware atom splitting) | [#16052](https://github.com/infiniflow/ragflow/pull/16052) | +| excel_parser | keep `0` / `False` cells; skip fully-empty rows | [#16287](https://github.com/infiniflow/ragflow/pull/16287) | +| excel_parser | no spurious header-only chunk when data rows are an exact multiple of `chunk_rows` | [#15490](https://github.com/infiniflow/ragflow/pull/15490) | +| excel_parser | binary-search actual row count for sheets with abnormal `max_row` | [#13018](https://github.com/infiniflow/ragflow/pull/13018) | +| markdown_parser | full sync to upstream state: protected ranges (fenced code / MD tables / HTML tables) never split by delimiters; GFM separators with 1+ dashes; lone-header attachment | #10896 #11018 #11520 #13892 #15630 #15632 #16109 #16319 | +| ppt_parser | cached, None-safe shape sorting (group shapes with `top=None` no longer crash) | [#13054](https://github.com/infiniflow/ragflow/pull/13054) | +| parser/utils | `get_text`: `if binary:` → `if binary is not None:` | [#13196](https://github.com/infiniflow/ragflow/pull/13196) | +| table_structure_recognizer | English caption patterns (`Figure N` / `Table N` / `Fig. N`) | [#15481](https://github.com/infiniflow/ragflow/pull/15481) | +| vision/ocr | env-configurable ONNX thread counts (`OCR_INTRA_OP_NUM_THREADS` / `OCR_INTER_OP_NUM_THREADS`) and opt-in GPU arena shrinkage (`OCR_GPUMEM_ARENA_SHRINKAGE=1`) | [#12777](https://github.com/infiniflow/ragflow/pull/12777) | +| layout_recognizer | figure/equation placeholder boxes get per-type layoutno namespaces (`figure-N` / `equation-N`), preventing unrelated figure and equation crops from merging on scientific PDFs | [#15873](https://github.com/infiniflow/ragflow/pull/15873) | +| layout_recognizer | re-enable CID-pattern garbage filter for text boxes | [#13404](https://github.com/infiniflow/ragflow/pull/13404) | +| pdf_parser | garbled-text detection → OCR fallback (PUA/CID chars, subset-font encoding garbling, OCR-alphabet coverage check) | [#13404](https://github.com/infiniflow/ragflow/pull/13404) | +| pdf_parser | geometry-based word-space recovery, CJK-aware (`_insert_word_spaces`) | [#16958](https://github.com/infiniflow/ragflow/pull/16958) | +| pdf_parser | remove 299-page hardcoded limit (`MAXIMUM_PAGE_NUMBER = 100000`) | [#14382](https://github.com/infiniflow/ragflow/pull/14382) | +| pdf_parser | chunked `parse_into_bboxes` for large PDFs (`PDF_PARSER_PAGE_BATCH_SIZE`, default 50) with window→global box remapping; lazy `img_np` in `__ocr` | [#14385](https://github.com/infiniflow/ragflow/pull/14385) | +| pdf_parser | out-of-range page-index guards in `cropout` / table-figure insertion | [#12938](https://github.com/infiniflow/ragflow/pull/12938) [#12848](https://github.com/infiniflow/ragflow/pull/12848) | +| pdf_parser (VisionParser) | `pdf_page_num = from_page + idx` (page numbering was wrong when `from_page > 0`); fixed broken relative import of `llm_adapter` | [#12938](https://github.com/infiniflow/ragflow/pull/12938) + fork-local fix | +| parser (new) | EPUB parser (spine-order XHTML extraction, delegates to HtmlParser), exported as `deepdoc.EpubParser` | [#13650](https://github.com/infiniflow/ragflow/pull/13650) | + +Already present before this sync (no action needed): `ast.literal_eval` +security fix (#12236), XLS/calamine loading (#10660), chartsheet guards +(#10819), KMeans multi-column detection (#11415 + #12534 cleanup), +`is_english` regex fix (#11432), OCR GPU memory limit env (#10407), +`vision_figure_parser_docx_wrapper` guard (#12104). + +**Deliberately NOT ported** (decisions): + +- **Go + HTTP microservice pipeline** (#16323, `deepdoc/server/`, `DEEPDOC_URL` + DLA client): conflicts with this library's pure-Python, in-process design. +- **Automatic table orientation detection/rotation** (#12719 feature + + #12981 / #17016 follow-ups): feature not in our baseline; runs 4× OCR per + table; adopt separately if ever needed. +- **`rag.utils.lazy_image` memory refactors** (docx/excel lazy image loading, + #13329 / #13558): optimization only; deferred until vendoring lazy_image is + justified. +- **3-tuple outlines** (`extract_pdf_outlines` returning + `(title, depth, page)`, #10456 TOC feature): our `outlines` stays + `(title, depth)` to avoid breaking downstream consumers. +- **`model_speciess` → `model_species` rename** (#13929): our attribute name + is part of the public surface; renaming would break existing callers. +- **paddleocr / opendataloader / somark parsers**: cloud-API clients, not + currently needed; revisit on demand. +- **MinerU feature evolution** (API-only backends, VLM descriptions, etc.): + only the security fix was taken; feature sync deferred until MinerU usage + is confirmed. +- **Pure refactors** (moving functions between ragflow modules, lefthook + reformatting): not applicable to our layout. + +Regression tests: `tests/test_upstream_sync_regressions.py`. + +## How to run the next sync + +```bash +# in a ragflow clone +git log --reverse --format="%h %ad %s" --date=short 18ea0fb7..origin/main -- deepdoc +``` + +Then for each commit: understand the behavior change, find the corresponding +code in this repo, and re-implement it in this library's style (config +injection, relative imports, vendored `common/` + `depend/`). Update this file. diff --git a/deepdoc/__init__.py b/deepdoc/__init__.py index e68ad56..5904190 100644 --- a/deepdoc/__init__.py +++ b/deepdoc/__init__.py @@ -27,6 +27,7 @@ "PlainParser", "DocxParser", "DoclingParser", + "EpubParser", "ExcelParser", "PptParser", "HtmlParser", diff --git a/deepdoc/parser/__init__.py b/deepdoc/parser/__init__.py index bce30e7..1097b27 100644 --- a/deepdoc/parser/__init__.py +++ b/deepdoc/parser/__init__.py @@ -16,6 +16,7 @@ from .docx_parser import RAGFlowDocxParser as DocxParser from .docling_parser import DoclingParser +from .epub_parser import RAGFlowEpubParser as EpubParser from .excel_parser import RAGFlowExcelParser as ExcelParser from .html_parser import RAGFlowHtmlParser as HtmlParser from .json_parser import RAGFlowJsonParser as JsonParser @@ -31,6 +32,7 @@ "PlainParser", "DocxParser", "DoclingParser", + "EpubParser", "ExcelParser", "PptParser", "HtmlParser", diff --git a/deepdoc/parser/epub_parser.py b/deepdoc/parser/epub_parser.py new file mode 100644 index 0000000..37ab1bc --- /dev/null +++ b/deepdoc/parser/epub_parser.py @@ -0,0 +1,139 @@ +# +# Copyright 2025 The InfiniFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging +import warnings +import zipfile +from io import BytesIO +from xml.etree import ElementTree + +from ..config import TokenizerConfig +from .html_parser import RAGFlowHtmlParser + +# OPF XML namespaces +_OPF_NS = "http://www.idpf.org/2007/opf" +_CONTAINER_NS = "urn:oasis:names:tc:opendocument:xmlns:container" + +# Media types that contain readable XHTML content +_XHTML_MEDIA_TYPES = {"application/xhtml+xml", "text/html", "text/xml"} + +logger = logging.getLogger(__name__) + + +class RAGFlowEpubParser: + """Parse EPUB files by extracting XHTML content in spine (reading) order + and delegating to RAGFlowHtmlParser for chunking.""" + + def __init__(self, tokenizer_cfg: TokenizerConfig | None = None): + self.html_parser = RAGFlowHtmlParser(tokenizer_cfg=tokenizer_cfg) + + def __call__(self, fnm, binary=None, chunk_token_num=512): + if binary is not None: + if not binary: + logger.warning( + "RAGFlowEpubParser received an empty EPUB binary payload for %r", + fnm, + ) + raise ValueError("Empty EPUB binary payload") + zf = zipfile.ZipFile(BytesIO(binary)) + else: + zf = zipfile.ZipFile(fnm) + + try: + content_items = self._get_spine_items(zf) + all_sections = [] + + for item_path in content_items: + try: + html_bytes = zf.read(item_path) + except KeyError: + continue + if not html_bytes: + logger.debug("Skipping empty EPUB content item: %s", item_path) + continue + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=UserWarning) + sections = self.html_parser(item_path, binary=html_bytes, chunk_token_num=chunk_token_num) + all_sections.extend(sections) + + return all_sections + finally: + zf.close() + + @staticmethod + def _get_spine_items(zf): + """Return content file paths in spine (reading) order.""" + # 1. Find the OPF file path from META-INF/container.xml + try: + container_xml = zf.read("META-INF/container.xml") + except KeyError: + return RAGFlowEpubParser._fallback_xhtml_order(zf) + + try: + container_root = ElementTree.fromstring(container_xml) + except ElementTree.ParseError: + logger.warning("Failed to parse META-INF/container.xml; falling back to XHTML order.") + return RAGFlowEpubParser._fallback_xhtml_order(zf) + + rootfile_el = container_root.find(f".//{{{_CONTAINER_NS}}}rootfile") + if rootfile_el is None: + return RAGFlowEpubParser._fallback_xhtml_order(zf) + + opf_path = rootfile_el.get("full-path", "") + if not opf_path: + return RAGFlowEpubParser._fallback_xhtml_order(zf) + + # Base directory of the OPF file (content paths are relative to it) + opf_dir = opf_path.rsplit("/", 1)[0] + "/" if "/" in opf_path else "" + + # 2. Parse the OPF file + try: + opf_xml = zf.read(opf_path) + except KeyError: + return RAGFlowEpubParser._fallback_xhtml_order(zf) + + try: + opf_root = ElementTree.fromstring(opf_xml) + except ElementTree.ParseError: + logger.warning("Failed to parse OPF file '%s'; falling back to XHTML order.", opf_path) + return RAGFlowEpubParser._fallback_xhtml_order(zf) + + # 3. Build id->href+mediatype map from + manifest = {} + for item in opf_root.findall(f".//{{{_OPF_NS}}}item"): + item_id = item.get("id", "") + href = item.get("href", "") + media_type = item.get("media-type", "") + if item_id and href: + manifest[item_id] = (href, media_type) + + # 4. Walk to get reading order + spine_items = [] + for itemref in opf_root.findall(f".//{{{_OPF_NS}}}itemref"): + idref = itemref.get("idref", "") + if idref not in manifest: + continue + href, media_type = manifest[idref] + if media_type not in _XHTML_MEDIA_TYPES: + continue + spine_items.append(opf_dir + href) + + return spine_items if spine_items else RAGFlowEpubParser._fallback_xhtml_order(zf) + + @staticmethod + def _fallback_xhtml_order(zf): + """Fallback: return all .xhtml/.html files sorted alphabetically.""" + return sorted(n for n in zf.namelist() if n.lower().endswith((".xhtml", ".html", ".htm")) and not n.startswith("META-INF/")) diff --git a/deepdoc/parser/excel_parser.py b/deepdoc/parser/excel_parser.py index f4c451e..731ba7e 100644 --- a/deepdoc/parser/excel_parser.py +++ b/deepdoc/parser/excel_parser.py @@ -74,6 +74,14 @@ def clean_string(s): return df.apply(lambda col: col.map(clean_string)) + @staticmethod + def _fill_worksheet_from_dataframe(ws, df: pd.DataFrame): + for col_num, column_name in enumerate(df.columns, 1): + ws.cell(row=1, column=col_num, value=column_name) + for row_num, row in enumerate(df.values, 2): + for col_num, value in enumerate(row, 1): + ws.cell(row=row_num, column=col_num, value=value) + @staticmethod def _dataframe_to_workbook(df): # if contains multiple sheets use _dataframes_to_workbook @@ -84,30 +92,19 @@ def _dataframe_to_workbook(df): wb = Workbook() ws = wb.active ws.title = "Data" - - for col_num, column_name in enumerate(df.columns, 1): - ws.cell(row=1, column=col_num, value=column_name) - - for row_num, row in enumerate(df.values, 2): - for col_num, value in enumerate(row, 1): - ws.cell(row=row_num, column=col_num, value=value) - + RAGFlowExcelParser._fill_worksheet_from_dataframe(ws, df) return wb - + @staticmethod def _dataframes_to_workbook(dfs: dict): wb = Workbook() default_sheet = wb.active wb.remove(default_sheet) - + for sheet_name, df in dfs.items(): df = RAGFlowExcelParser._clean_dataframe(df) ws = wb.create_sheet(title=sheet_name) - for col_num, column_name in enumerate(df.columns, 1): - ws.cell(row=1, column=col_num, value=column_name) - for row_num, row in enumerate(df.values, 2): - for col_num, value in enumerate(row, 1): - ws.cell(row=row_num, column=col_num, value=value) + RAGFlowExcelParser._fill_worksheet_from_dataframe(ws, df) return wb @staticmethod @@ -156,6 +153,55 @@ def _extract_images_from_worksheet(ws, sheetname=None): continue return raw_items + @staticmethod + def _get_actual_row_count(ws): + max_row = ws.max_row + if not max_row: + return 0 + if max_row <= 10000: + return max_row + + max_col = min(ws.max_column or 1, 50) + + def row_has_data(row_idx): + for col_idx in range(1, max_col + 1): + cell = ws.cell(row=row_idx, column=col_idx) + if cell.value is not None and str(cell.value).strip(): + return True + return False + + if not any(row_has_data(i) for i in range(1, min(101, max_row + 1))): + return 0 + + left, right = 1, max_row + last_data_row = 1 + + while left <= right: + mid = (left + right) // 2 + found = False + for r in range(mid, min(mid + 10, max_row + 1)): + if row_has_data(r): + found = True + last_data_row = max(last_data_row, r) + break + if found: + left = mid + 1 + else: + right = mid - 1 + + for r in range(last_data_row, min(last_data_row + 500, max_row + 1)): + if row_has_data(r): + last_data_row = r + + return last_data_row + + @staticmethod + def _get_rows_limited(ws): + actual_rows = RAGFlowExcelParser._get_actual_row_count(ws) + if actual_rows == 0: + return [] + return list(ws.iter_rows(min_row=1, max_row=actual_rows)) + def html(self, fnm, chunk_rows=256): from html import escape @@ -171,7 +217,7 @@ def _fmt(v): for sheetname in wb.sheetnames: ws = wb[sheetname] try: - rows = list(ws.rows) + rows = RAGFlowExcelParser._get_rows_limited(ws) except Exception as e: logging.warning(f"Skip sheet '{sheetname}' due to rows access error: {e}") continue @@ -184,7 +230,12 @@ def _fmt(v): tb_rows_0 += f"{escape(_fmt(t.value))}" tb_rows_0 += "" - for chunk_i in range((len(rows) - 1) // chunk_rows + 1): + # rows[0] is the header; split the remaining data rows into + # ceil(n_data / chunk_rows) chunks. Using +1 here over-counts by one + # when the data-row count is an exact multiple of chunk_rows and emits + # a spurious header-only chunk. + n_data_rows = len(rows) - 1 + for chunk_i in range((n_data_rows + chunk_rows - 1) // chunk_rows): tb = "" tb += f"" tb += tb_rows_0 @@ -223,7 +274,7 @@ def __call__(self, fnm): for sheetname in wb.sheetnames: ws = wb[sheetname] try: - rows = list(ws.rows) + rows = RAGFlowExcelParser._get_rows_limited(ws) except Exception as e: logging.warning(f"Skip sheet '{sheetname}' due to rows access error: {e}") continue @@ -233,11 +284,13 @@ def __call__(self, fnm): for r in list(rows[1:]): fields = [] for i, c in enumerate(r): - if not c.value: + if c.value is None or str(c.value).strip() == "": continue t = str(ti[i].value) if i < len(ti) else "" t += (":" if t else "") + str(c.value) fields.append(t) + if not fields: + continue line = "; ".join(fields) if sheetname.lower().find("sheet") < 0: line += " ——" + sheetname @@ -251,12 +304,12 @@ def row_number(fnm, binary): total = 0 for sheetname in wb.sheetnames: - try: - ws = wb[sheetname] - total += len(list(ws.rows)) - except Exception as e: - logging.warning(f"Skip sheet '{sheetname}' due to rows access error: {e}") - continue + try: + ws = wb[sheetname] + total += RAGFlowExcelParser._get_actual_row_count(ws) + except Exception as e: + logging.warning(f"Skip sheet '{sheetname}' due to rows access error: {e}") + continue return total if fnm.split(".")[-1].lower() in ["csv", "txt"]: diff --git a/deepdoc/parser/html_parser.py b/deepdoc/parser/html_parser.py index 538e37a..67d859c 100644 --- a/deepdoc/parser/html_parser.py +++ b/deepdoc/parser/html_parser.py @@ -16,6 +16,8 @@ # import html +import logging +import re import uuid import chardet @@ -37,7 +39,7 @@ def get_encoding(file): "table", "pre", "code", "blockquote", "figure", "figcaption" ] -TITLE_TAGS = {"h1": "#", "h2": "##", "h3": "###", "h4": "#####", "h5": "#####", "h6": "######"} +TITLE_TAGS = {"h1": "#", "h2": "##", "h3": "###", "h4": "####", "h5": "#####", "h6": "######"} class RAGFlowHtmlParser: @@ -65,7 +67,7 @@ def parser_txt(self, txt, chunk_token_num): raise TypeError("txt type should be string!") temp_sections = [] - soup = BeautifulSoup(txt, "html5lib") + soup = BeautifulSoup(txt, "html.parser") # delete
{sheetname}