#!/usr/bin/env python3 # -*- coding: utf-8 -*- import os import re import docx import numpy as np import requests from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.metrics.pairwise import cosine_similarity from typing import List, Tuple, Dict, Optional from docx.shared import Pt from docx.enum.text import WD_PARAGRAPH_ALIGNMENT from docx.enum.table import WD_TABLE_ALIGNMENT import subprocess import tempfile import json from docx.table import Table, _Cell from docx.text.paragraph import Paragraph from copy import deepcopy from docx.oxml import parse_xml from docx.oxml.ns import nsdecls import logging import base64 class DocCleaner: def __init__(self, ollama_host: str = "http://192.168.1.24:11434"): """ 初始化文档清理器 Args: ollama_host: Ollama服务器地址 """ # 页眉页脚模式 self.header_footer_patterns = [ r'页码\s*\d+-\d+', # 页码格式：页码1-1, 页码2-1等 r'第\s*\d+\s*页\s*共\s*\d+\s*页', # 中文页码（第X页共Y页） r'Page\s*\d+\s*of\s*\d+', # 英文页码 ] # 特殊符号模式 self.special_char_patterns = [ r'©\s*\d{4}.*?版权所有', # 版权信息 r'confidential', # 机密标记 r'draft|草稿', # 草稿标记 r'watermark', # 水印标记 ] # 附录和参考文献标题模式 self.appendix_patterns = [ r'^附录\s*[A-Za-z]?[\s:：]', r'^Appendix\s*[A-Za-z]?[\s:：]', r'^参考文献$', r'^References$', r'^Bibliography$' ] # 初始化TF-IDF向量化器 self.vectorizer = TfidfVectorizer( min_df=1, stop_words='english' ) self.ollama_host = ollama_host self.embedding_model = "bge-m3" # 使用nomic-embed-text模型进行文本嵌入 def _convert_doc_to_docx(self, doc_path: str) -> str: """ 将doc格式转换为docx格式 Args: doc_path: doc文件路径 Returns: str: 转换后的docx文件路径 """ print(f"\n开始转换DOC文件: {doc_path}") # 创建临时文件路径 temp_dir = tempfile.mkdtemp() temp_docx = os.path.join(temp_dir, 'temp.docx') print(f"创建临时目录: {temp_dir}") print(f"目标DOCX文件路径: {temp_docx}") try: # 首先清理可能存在的soffice进程 try: if os.name == 'nt': # Windows os.system('taskkill /f /im soffice.bin /t') os.system('taskkill /f /im soffice.exe /t') else: # Linux/Unix os.system('pkill -9 soffice.bin') os.system('pkill -9 soffice') except Exception as e: print(f"清理已有进程时出错（可以忽略）: {str(e)}") # 检测操作系统类型 if os.name == 'nt': # Windows soffice_cmd = 'soffice' print("检测到Windows系统，使用soffice命令") else: # Linux/Unix # 常见的LibreOffice可执行文件路径 possible_paths = [ 'libreoffice', 'soffice', '/usr/bin/libreoffice', '/usr/bin/soffice', '/usr/lib/libreoffice/program/soffice', '/opt/libreoffice*/program/soffice', ] print("检测到Linux/Unix系统，开始查找LibreOffice...") soffice_cmd = None for path in possible_paths: try: if '*' in path: # 处理通配符路径 import glob matching_paths = glob.glob(path) for match_path in matching_paths: try: print(f"尝试执行: {match_path} --version") subprocess.run([match_path, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=5) soffice_cmd = match_path print(f"找到可用的LibreOffice: {soffice_cmd}") break except Exception as e: print(f"尝试路径失败 {match_path}: {str(e)}") else: print(f"尝试执行: {path} --version") subprocess.run([path, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=5) soffice_cmd = path print(f"找到可用的LibreOffice: {soffice_cmd}") break except Exception as e: print(f"尝试路径失败 {path}: {str(e)}") continue if soffice_cmd is None: # 尝试使用which命令查找 try: print("尝试使用which命令查找LibreOffice...") which_result = subprocess.run(['which', 'libreoffice'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) if which_result.returncode == 0: soffice_cmd = which_result.stdout.strip() print(f"通过which命令找到LibreOffice: {soffice_cmd}") except Exception as e: print(f"which命令查找失败: {str(e)}") if soffice_cmd is None: error_msg = """ 未找到LibreOffice，请按以下步骤安装： 1. 对于Ubuntu/Debian系统： sudo apt-get update sudo apt-get install libreoffice libreoffice-writer 2. 对于CentOS/RHEL系统： sudo yum update sudo yum install libreoffice libreoffice-writer 3. 安装中文字体支持： # Ubuntu/Debian： sudo apt-get install fonts-wqy-zenhei fonts-wqy-microhei # CentOS/RHEL： sudo yum install wqy-zenhei-fonts wqy-microhei-fonts 4. 安装后验证： libreoffice --version 5. 如果仍然失败，请确保： - LibreOffice已正确安装 - 可执行文件在系统PATH中 - 当前用户有执行权限 - 临时目录(/tmp)有足够的权限 """ raise Exception(error_msg) print(f"\n使用命令转换文件: {soffice_cmd}") # 使用soffice（LibreOffice）进行转换 cmd = [ soffice_cmd, '--headless', '--convert-to', 'docx:MS Word 2007 XML', # 指定具体的输出格式 '--outdir', temp_dir, doc_path ] print(f"完整转换命令: {' '.join(cmd)}") # 执行转换命令，设置较长的超时时间 try: process = subprocess.run( cmd, capture_output=True, text=True, timeout=300 # 设置2分钟超时 ) if process.returncode != 0: error_msg = process.stderr or "未知错误" raise Exception(f"转换失败: {error_msg}") print("文件转换成功") except subprocess.TimeoutExpired: # 超时时清理进程 if os.name == 'nt': # Windows os.system('taskkill /f /im soffice.bin /t') os.system('taskkill /f /im soffice.exe /t') else: # Linux/Unix os.system('pkill -9 soffice.bin') os.system('pkill -9 soffice') raise Exception("转换超时（300秒），已终止进程。请检查LibreOffice是否正常运行，或尝试手动转换文件。") # 验证输出文件 if not os.path.exists(temp_docx): raise Exception("转换后的文件未找到") file_size = os.path.getsize(temp_docx) if file_size == 0: raise Exception("转换后的文件大小为0") print(f"转换完成，输出文件大小: {file_size} bytes") return temp_docx except Exception as e: print(f"转换doc文件失败: {str(e)}") # 清理临时文件 try: if os.path.exists(temp_dir): import shutil shutil.rmtree(temp_dir) except: pass # 清理可能残留的进程 try: if os.name == 'nt': # Windows os.system('taskkill /f /im soffice.bin /t') os.system('taskkill /f /im soffice.exe /t') else: # Linux/Unix os.system('pkill -9 soffice.bin') os.system('pkill -9 soffice') except: pass raise Exception(f"转换doc文件失败: {str(e)}") def clean_doc(self, file_path: str) -> Tuple[List[str], List[str], List[Table]]: """ 清理文档并返回处理后的正文、附录和表格 Args: file_path: 文档文件路径 Returns: Tuple[List[str], List[str], List[Table]]: (清理后的正文段落列表, 附录段落列表, 表格列表) """ # 检测文件类型 _, file_extension = os.path.splitext(file_path) file_extension = file_extension.lower() # 如果是doc格式，直接报错 if file_extension == '.doc': raise Exception("不支持doc格式，请先将文件转换为docx格式后再处理") doc = docx.Document(file_path) # 提取所有内容（段落和表格） content = [] tables = [] table_count = 0 try: # 遍历文档体中的所有元素 for element in doc._element.body: if element.tag.endswith('p'): try: paragraph = docx.text.paragraph.Paragraph(element, doc) text = paragraph.text.strip() # 只添加非空段落 if text: # 检查是否是附录标题 is_appendix = any(re.match(pattern, text, re.IGNORECASE) for pattern in self.appendix_patterns) content.append({ 'type': 'paragraph', 'content': text, 'is_appendix_start': is_appendix }) except Exception as e: continue elif element.tag.endswith('tbl'): try: table = docx.table.Table(element, doc) # 验证表格是否有效 if hasattr(table, 'rows') and hasattr(table, 'columns'): tables.append(table) content.append({ 'type': 'table', 'index': table_count }) table_count += 1 except Exception as e: continue except Exception as e: raise Exception(f"解析文档结构失败: {str(e)}") # 分离正文和附录 main_content = [] appendix = [] is_appendix = False for item in content: if item['type'] == 'paragraph': if item['is_appendix_start']: is_appendix = True if is_appendix: appendix.append(item['content']) else: main_content.append(item['content']) elif item['type'] == 'table': table_placeholder = f'TABLE_PLACEHOLDER_{item["index"]}' if is_appendix: appendix.append(table_placeholder) else: main_content.append(table_placeholder) # 清理正文（保留表格标记） cleaned_content = [] for item in main_content: if item.startswith('TABLE_PLACEHOLDER_'): cleaned_content.append(item) else: cleaned_text = self._clean_text([item])[0] if cleaned_text: cleaned_content.append(cleaned_text) return cleaned_content, appendix, tables def _clean_text(self, text: List[str]) -> List[str]: """ 清理文本内容 Args: text: 待清理的文本段落列表 Returns: List[str]: 清理后的文本段落列表 """ cleaned = [] for paragraph in text: # 如果是表格标记，直接保留 if paragraph.startswith('TABLE_PLACEHOLDER_'): cleaned.append(paragraph) continue # 跳过空段落 if not paragraph.strip(): continue # 检查是否是目录项（包含数字序号的行） is_toc_item = bool(re.match(r'^\s*(?:\d+\.)*\d+\s+.*', paragraph)) if not is_toc_item: # 移除页眉页脚 for pattern in self.header_footer_patterns: paragraph = re.sub(pattern, '', paragraph, flags=re.IGNORECASE) # 移除特殊符号 for pattern in self.special_char_patterns: paragraph = re.sub(pattern, '', paragraph, flags=re.IGNORECASE) # 如果段落不为空，添加到结果中 if paragraph.strip(): cleaned.append(paragraph.strip()) return cleaned def _split_content(self, paragraphs: List[str]) -> Tuple[List[str], List[str]]: """ 分离正文与附录/参考文献 Args: paragraphs: 文档段落列表 Returns: Tuple[List[str], List[str]]: (正文段落列表, 附录段落列表) """ main_content = [] appendix = [] is_appendix = False for p in paragraphs: # 检查是否是附录开始 if any(re.match(pattern, p, re.IGNORECASE) for pattern in self.appendix_patterns): is_appendix = True if is_appendix: appendix.append(p) else: main_content.append(p) return main_content, appendix def _get_embeddings(self, texts: List[str]) -> np.ndarray: """ 使用Ollama获取文本嵌入向量 Args: texts: 文本列表 Returns: np.ndarray: 嵌入向量矩阵 """ embeddings = [] for text in texts: try: response = requests.post( f"{self.ollama_host}/api/embeddings", json={ "model": self.embedding_model, "prompt": text } ) response.raise_for_status() embedding = response.json()["embedding"] embeddings.append(embedding) except Exception as e: print(f"获取文本嵌入失败: {str(e)}") # 如果获取嵌入失败，使用零向量 embeddings.append([0.0] * 768) # nomic-embed-text 模型输出维度为768 return np.array(embeddings) def _remove_duplicates(self, paragraphs: List[str], similarity_threshold: float = 0.92) -> List[str]: """ 删除重复段落，保持表格占位符的位置不变 Args: paragraphs: 段落列表 similarity_threshold: 相似度阈值，使用嵌入模型后可以设置更高的阈值 Returns: List[str]: 去重后的段落列表 """ if not paragraphs: return [] # 分离表格占位符和普通段落 table_placeholders = {} text_paragraphs = [] for i, p in enumerate(paragraphs): if p.startswith('TABLE_PLACEHOLDER_'): table_placeholders[i] = p else: text_paragraphs.append((i, p)) try: # 只对非表格段落进行去重 if text_paragraphs: # 获取文本嵌入 text_only = [p[1] for p in text_paragraphs] embeddings = self._get_embeddings(text_only) # 计算余弦相似度矩阵 similarity_matrix = cosine_similarity(embeddings) # 标记要保留的段落 keep_indices = [] for i in range(len(text_paragraphs)): # 如果当前段落没有与之前的段落高度相似，则保留 if not any(similarity_matrix[i][j] > similarity_threshold for j in keep_indices): keep_indices.append(i) # 保留的非表格段落 kept_paragraphs = [(text_paragraphs[i][0], text_only[i]) for i in keep_indices] else: kept_paragraphs = [] # 合并表格占位符和保留的段落，按原始位置排序 all_kept = list(table_placeholders.items()) + kept_paragraphs all_kept.sort(key=lambda x: x[0]) return [p[1] for p in all_kept] except Exception as e: print(f"使用Ollama嵌入模型失败，回退到TF-IDF方法: {str(e)}") # 如果使用Ollama失败，回退到原来的TF-IDF方法 return self._remove_duplicates_tfidf(paragraphs) def _remove_duplicates_tfidf(self, paragraphs: List[str], similarity_threshold: float = 0.85) -> List[str]: """ 使用TF-IDF方法删除重复段落（作为备选方案） Args: paragraphs: 段落列表 similarity_threshold: 相似度阈值 Returns: List[str]: 去重后的段落列表 """ if not paragraphs: return [] # 分离表格占位符和普通段落 table_placeholders = {} text_paragraphs = [] for i, p in enumerate(paragraphs): if p.startswith('TABLE_PLACEHOLDER_'): table_placeholders[i] = p else: text_paragraphs.append((i, p)) if text_paragraphs: # 计算TF-IDF矩阵 text_only = [p[1] for p in text_paragraphs] tfidf_matrix = self.vectorizer.fit_transform(text_only) # 计算余弦相似度矩阵 similarity_matrix = cosine_similarity(tfidf_matrix) # 标记要保留的段落 keep_indices = [] for i in range(len(text_paragraphs)): # 如果当前段落没有与之前的段落高度相似，则保留 if not any(similarity_matrix[i][j] > similarity_threshold for j in keep_indices): keep_indices.append(i) # 保留的非表格段落 kept_paragraphs = [(text_paragraphs[i][0], text_only[i]) for i in keep_indices] else: kept_paragraphs = [] # 合并表格占位符和保留的段落，按原始位置排序 all_kept = list(table_placeholders.items()) + kept_paragraphs all_kept.sort(key=lambda x: x[0]) return [p[1] for p in all_kept] def save_as_docx(self, cleaned_content: List[str], appendix: List[str], tables: List[Table], output_path: str): """ 将清理后的内容保存为docx格式和txt格式 Args: cleaned_content: 清理后的正文段落列表 appendix: 附录段落列表 tables: 表格列表 output_path: 输出文件路径 """ print(f"\n开始保存文档: {output_path}") print(f"- 正文元素数: {len(cleaned_content)}") print(f"- 附录元素数: {len(appendix)}") print(f"- 表格总数: {len(tables)}") # 创建新文档 doc = docx.Document() # 创建文本输出内容列表 text_output = [] # 添加正文内容和表格，保持它们的相对位置 print("\n处理正文内容...") # 创建一个列表来存储所有要插入的元素 elements_to_insert = [] for i, content in enumerate(cleaned_content): try: # 检查是否是表格占位符 table_match = re.match(r'TABLE_PLACEHOLDER_(\d+)', content) if table_match: table_index = int(table_match.group(1)) print(f"正在处理表格占位符: {content} (索引: {table_index})") if table_index < len(tables): table = tables[table_index] try: # 转换表格为文本格式 table_text = self._convert_table_to_text(table) # 添加表格标题 title = doc.add_paragraph(f"表格 {table_index + 1}:") title.alignment = WD_PARAGRAPH_ALIGNMENT.LEFT elements_to_insert.append(('paragraph', title._element)) # 添加表格文本内容，使用等宽字体 p = doc.add_paragraph() run = p.add_run(table_text) run.font.name = 'Courier New' # 使用等宽字体 run.font.size = Pt(10) # 设置字体大小 elements_to_insert.append(('paragraph', p._element)) # 添加空行 elements_to_insert.append(('paragraph', doc.add_paragraph()._element)) # 添加到文本输出 text_output.append(f"表格 {table_index + 1}:") text_output.append(table_text) except Exception as e: print(f"警告：处理表格时出错: {str(e)}") elements_to_insert.append(('paragraph', doc.add_paragraph("【表格处理失败】")._element)) text_output.append("【表格处理失败】") else: # 添加普通段落 p = doc.add_paragraph(content) p.alignment = WD_PARAGRAPH_ALIGNMENT.JUSTIFY elements_to_insert.append(('paragraph', p._element)) # 添加到文本输出 text_output.append(content) except Exception as e: print(f"警告：处理段落或表格时出错: {str(e)}") continue # 按顺序将所有元素插入文档 for element_type, element in elements_to_insert: doc._body._element.append(element) # 如果有附录，添加分隔符和附录内容 if appendix: print("\n处理附录内容...") try: # 添加分页符 doc.add_page_break() # 添加附录标题 title = doc.add_paragraph("附录") title.alignment = WD_PARAGRAPH_ALIGNMENT.CENTER # 添加到文本输出 text_output.append("附录") # 添加附录内容 appendix_elements = [] for content in appendix: # 检查是否是表格占位符 table_match = re.match(r'TABLE_PLACEHOLDER_(\d+)', content) if table_match: table_index = int(table_match.group(1)) print(f"正在处理附录中的表格占位符: {content} (索引: {table_index})") if table_index < len(tables): table = tables[table_index] try: # 转换表格为文本格式 table_text = self._convert_table_to_text(table) # 添加表格标题 title = doc.add_paragraph(f"附录表格 {table_index + 1}:") title.alignment = WD_PARAGRAPH_ALIGNMENT.LEFT appendix_elements.append(('paragraph', title._element)) # 添加表格文本内容，使用等宽字体 p = doc.add_paragraph() run = p.add_run(table_text) run.font.name = 'Courier New' # 使用等宽字体 run.font.size = Pt(10) # 设置字体大小 appendix_elements.append(('paragraph', p._element)) # 添加到文本输出 text_output.append(f"附录表格 {table_index + 1}:") text_output.append(table_text) except Exception as e: print(f"警告：处理附录表格时出错: {str(e)}") appendix_elements.append(('paragraph', doc.add_paragraph("【表格处理失败】")._element)) text_output.append("【表格处理失败】") else: p = doc.add_paragraph(content) p.alignment = WD_PARAGRAPH_ALIGNMENT.JUSTIFY appendix_elements.append(('paragraph', p._element)) # 添加到文本输出 text_output.append(content) # 按顺序将附录元素插入文档 for element_type, element in appendix_elements: doc._body._element.append(element) except Exception as e: print(f"警告：处理附录时出错: {str(e)}") # 保存docx文档 try: doc.save(output_path) print("\nWord文档保存成功!") except Exception as e: print(f"错误：保存Word文档时出错: {str(e)}") raise # 保存文本文件 try: text_file_path = os.path.splitext(output_path)[0] + '.txt' # 移除所有换行符并用空格连接 text_content = ' '.join([t.replace('\n', ' ').strip() for t in text_output if t.strip()]) with open(text_file_path, 'w', encoding='utf-8') as f: f.write(text_content) print(f"文本文件保存成功: {text_file_path}") except Exception as e: print(f"错误：保存文本文件时出错: {str(e)}") raise def _copy_table_fallback(self, doc: docx.Document, table: Table): """ 表格复制的备用方法 Args: doc: 目标文档 table: 源表格 """ # 获取表格的行数和列数 rows = len(table.rows) cols = len(table.columns) # 创建新表格 new_table = doc.add_table(rows=rows, cols=cols) # 复制表格样式 if table.style: new_table.style = table.style # 复制表格属性 new_table._element.tblPr = deepcopy(table._element.tblPr) # 复制网格信息 new_table._element.tblGrid = deepcopy(table._element.tblGrid) # 创建单元格映射以跟踪合并 cell_map = {} # 第一遍：标记合并的单元格 for i in range(rows): for j in range(cols): try: src_cell = table.cell(i, j) # 检查是否是合并单元格的一部分 if src_cell._element.tcPr is not None: # 检查垂直合并 vmerge = src_cell._element.tcPr.xpath('.//w:vMerge') if vmerge: val = vmerge[0].get(qn('w:val'), 'continue') if val == 'restart': # 这是合并的起始单元格 span = self._get_vertical_span(table, i, j) cell_map[(i, j)] = ('vmerge', span) # 检查水平合并 gridspan = src_cell._element.tcPr.xpath('.//w:gridSpan') if gridspan: span = int(gridspan[0].get(qn('w:val'))) if span > 1: cell_map[(i, j)] = ('hmerge', span) except Exception as e: print(f"警告：处理合并单元格时出错 [{i},{j}]: {str(e)}") # 第二遍：复制内容并执行合并 for i in range(rows): for j in range(cols): try: src_cell = table.cell(i, j) dst_cell = new_table.cell(i, j) # 检查是否需要合并 if (i, j) in cell_map: merge_type, span = cell_map[(i, j)] if merge_type == 'vmerge': # 垂直合并 for k in range(1, span): if i + k < rows: dst_cell.merge(new_table.cell(i + k, j)) elif merge_type == 'hmerge': # 水平合并 for k in range(1, span): if j + k < cols: dst_cell.merge(new_table.cell(i, j + k)) # 复制单元格属性 if src_cell._element.tcPr is not None: dst_cell._element.tcPr = deepcopy(src_cell._element.tcPr) # 复制单元格内容 dst_cell.text = "" # 清除默认内容 for src_paragraph in src_cell.paragraphs: dst_paragraph = dst_cell.add_paragraph() # 复制段落属性 if src_paragraph._element.pPr is not None: dst_paragraph._element.pPr = deepcopy(src_paragraph._element.pPr) # 复制文本和格式 for src_run in src_paragraph.runs: dst_run = dst_paragraph.add_run(src_run.text) # 复制运行属性 if src_run._element.rPr is not None: dst_run._element.rPr = deepcopy(src_run._element.rPr) except Exception as e: print(f"警告：复制单元格时出错 [{i},{j}]: {str(e)}") continue def _get_vmerge_value(self, cell_element) -> str: """ 获取单元格的垂直合并属性 Args: cell_element: 单元格元素 Returns: str: 垂直合并属性值 """ vmerge = cell_element.xpath('.//w:vMerge') if vmerge: return vmerge[0].get(qn('w:val'), 'continue') return None def _get_gridspan_value(self, cell_element) -> int: """ 获取单元格的水平合并数量 Args: cell_element: 单元格元素 Returns: int: 水平合并的列数 """ try: gridspan = cell_element.xpath('.//w:gridSpan') if gridspan and gridspan[0].get(qn('w:val')): return int(gridspan[0].get(qn('w:val'))) except (ValueError, TypeError, AttributeError) as e: print(f"警告：获取gridspan值时出错: {str(e)}") return 1 # 默认返回1，表示没有合并 def _get_vertical_span(self, table: Table, start_row: int, col: int) -> int: """ 计算垂直合并的行数 Args: table: 表格对象 start_row: 起始行 col: 列号 Returns: int: 垂直合并的行数 """ span = 1 for i in range(start_row + 1, len(table.rows)): cell = table.cell(i, col) if self._get_vmerge_value(cell._element) == 'continue': span += 1 else: break return span def _convert_table_to_text(self, table: Table) -> str: """ 将表格转换为文本格式，智能处理简单和复杂表格结构 Args: table: docx表格对象 Returns: str: 表格的文本表示 """ try: # 获取表格的行数和列数 rows = len(table.rows) cols = len(table.columns) if rows == 0 or cols == 0: return "【空表格】" # 存储处理后的表格数据 processed_data = [] # 检查是否是复杂表格（具有合并单元格或多级表头） is_complex_table = False max_header_rows = min(3, rows) # 最多检查前3行 # 检查前几行是否存在合并单元格 for i in range(max_header_rows): for j in range(cols): try: cell = table.cell(i, j) if cell._element.tcPr is not None: # 检查垂直合并 vmerge = cell._element.tcPr.xpath('.//w:vMerge') if vmerge: is_complex_table = True break # 检查水平合并 gridspan = cell._element.tcPr.xpath('.//w:gridSpan') if gridspan: is_complex_table = True break except Exception: continue if is_complex_table: break if is_complex_table: # 使用复杂表格处理逻辑 # 第一步：分析表头结构 header_structure = [] # 存储表头的层级结构 # 分析每一列的表头结构 for j in range(cols): column_headers = [] last_header = None for i in range(max_header_rows): try: cell = table.cell(i, j) text = cell.text.strip() # 检查垂直合并 if cell._element.tcPr is not None: vmerge = cell._element.tcPr.xpath('.//w:vMerge') if vmerge: val = vmerge[0].get(qn('w:val'), 'continue') if val == 'continue': # 使用上一个非空表头 if last_header: text = last_header # 检查水平合并 if cell._element.tcPr is not None: gridspan = self._get_gridspan_value(cell._element) if gridspan > 1: # 标记这是一个跨列的表头 text = f"SPAN_{gridspan}_{text}" if text: column_headers.append(text) last_header = text except Exception as e: print(f"警告：分析表头单元格 [{i},{j}] 时出错: {str(e)}") continue header_structure.append(column_headers) # 第二步：构建完整的表头标识符 full_headers = [] for j, headers in enumerate(header_structure): if not headers: full_headers.append(f"列{j + 1}") continue # 处理跨列的表头 header_text = [] current_prefix = "" for h in headers: if h.startswith('SPAN_'): parts = h.split('_', 2) span = int(parts[1]) text = parts[2] # 将跨列的表头添加到后续的列 for k in range(span): if j + k < cols: if k == 0: if text != current_prefix: # 避免重复前缀 header_text.append(text) current_prefix = text else: if text not in header_structure[j + k]: header_structure[j + k].insert(0, text) else: if h != current_prefix: # 避免重复前缀 header_text.append(h) current_prefix = h # 移除重复的表头部分 unique_headers = [] seen = set() for h in header_text: if h not in seen: unique_headers.append(h) seen.add(h) full_headers.append('_'.join(unique_headers)) # 确定实际的表头行数 header_row_count = max(len(headers) for headers in header_structure) if header_row_count == 0: header_row_count = 1 # 处理数据行 for i in range(header_row_count, rows): try: row_data = [] j = 0 while j < cols: try: cell = table.cell(i, j) text = cell.text.strip() # 处理垂直合并 if not text and cell._element.tcPr is not None: vmerge = cell._element.tcPr.xpath('.//w:vMerge') if vmerge and vmerge[0].get(qn('w:val')) == 'continue': # 使用上一行的值 text = table.cell(i - 1, j).text.strip() # 处理水平合并 gridspan = self._get_gridspan_value(cell._element) # 将值复制到所有合并的列 for k in range(gridspan): if j + k < len(full_headers): row_data.append(f"{full_headers[j + k]}:{text}") j += gridspan except Exception as e: print(f"警告：处理数据单元格 [{i},{j}] 时出错: {str(e)}") if j < len(full_headers): row_data.append(f"{full_headers[j]}:") j += 1 # 确保行中至少有一个非空值 if any(data.split(':')[1].strip() for data in row_data): processed_data.append(" ".join(row_data)) except Exception as e: print(f"警告：处理数据行 {i} 时出错: {str(e)}") continue else: # 使用简单表格处理逻辑 # 获取表头 headers = [] for j in range(cols): try: header_text = table.cell(0, j).text.strip() if not header_text: # 如果表头为空，使用默认值 header_text = f"列{j + 1}" headers.append(header_text) except Exception as e: print(f"警告：处理表头单元格 [0,{j}] 时出错: {str(e)}") headers.append(f"列{j + 1}") # 处理数据行 for i in range(1, rows): try: row_data = [] for j in range(cols): try: text = table.cell(i, j).text.strip() row_data.append(f"{headers[j]}:{text}") except Exception as e: print(f"警告：处理数据单元格 [{i},{j}] 时出错: {str(e)}") row_data.append(f"{headers[j]}:") # 确保行中至少有一个非空值 if any(data.split(':')[1].strip() for data in row_data): processed_data.append(" ".join(row_data)) except Exception as e: print(f"警告：处理数据行 {i} 时出错: {str(e)}") continue # 返回处理后的表格文本 if processed_data: return " ".join(processed_data) else: return "【表格无有效数据】" except Exception as e: print(f"警告：处理表格时出错: {str(e)}") return "【表格处理失败】" def _extract_table_text(self, table: Table) -> str: """ 提取表格中的文本内容，现在会返回格式化的文本表示 Args: table: docx表格对象 Returns: str: 表格内容的文本表示 """ return self._convert_table_to_text(table) def process_file(byte_array: bytes, suffix: str = 'docx') -> Tuple[bytes, str]: """ 处理文件的二进制数据 Args: byte_array: 文件的二进制数据 suffix: 文件后缀名（不含点号，例如'doc'或'docx'） Returns: Tuple[bytes, str]: (docx文件字节流, 文本内容) """ try: # 确保后缀名格式正确（添加点号） suffix = suffix.lower().strip() if not suffix.startswith('.'): suffix = '.' + suffix # 创建临时文件 temp_dir = tempfile.mkdtemp() temp_file = os.path.join(temp_dir, f'temp{suffix}') # 保存二进制数据到临时文件 with open(temp_file, 'wb') as f: f.write(byte_array) # 检查文件大小 file_size = len(byte_array) if file_size > 50 * 1024 * 1024: # 50MB raise Exception("文件大小超过50MB限制") # 检查文件格式 if suffix.lower() not in ['.doc', '.docx']: raise Exception("不支持的文件格式，仅支持.doc和.docx格式") # 检查文件头部特征 file_type = None if len(byte_array) >= 8: # DOCX文件特征 (ZIP格式，以PK\x03\x04开头) if byte_array.startswith(b'PK\x03\x04'): file_type = 'docx' logging.info("检测到DOCX文件格式") # DOC文件特征 (复合文件二进制格式，以D0CF11E0开头) elif byte_array.startswith(b'\xD0\xCF\x11\xE0\xA1\xB1\x1A\xE1'): file_type = 'doc' logging.info("检测到DOC文件格式") # 如果无法通过文件头识别，尝试通过内容特征识别 if not file_type and len(byte_array) >= 512: content_start = byte_array[:512] # 检查是否包含Word文档的特征字符串 if (b'Microsoft Word' in content_start or b'word/document.xml' in content_start or b'Word.Document' in content_start): file_type = 'unknown_word' logging.info("通过内容特征检测到Word文档") else: # 尝试读取文件内容 try: with open(temp_file, 'rb') as f: # 尝试以ZIP格式打开（DOCX格式） try: import zipfile with zipfile.ZipFile(f) as zf: if any(name.startswith('word/') for name in zf.namelist()): file_type = 'docx' logging.info("通过ZIP结构检测到DOCX文件") except zipfile.BadZipFile: # 不是有效的ZIP文件，尝试其他检测方法 pass except Exception as e: logging.warning(f"文件内容检测失败: {str(e)}") if not file_type: raise Exception("无法识别的Word文档格式") # 检查文件后缀是否与实际格式匹配 if file_type == 'docx' and suffix.lower() != '.docx': logging.warning("文件实际格式为DOCX，但后缀为%s", suffix) elif file_type == 'doc' and suffix.lower() != '.doc': logging.warning("文件实际格式为DOC，但后缀为%s", suffix) # 如果是doc格式，先转换为docx input_file = temp_file if file_type == 'doc' or (file_type == 'unknown_word' and suffix.lower() == '.doc'): try: input_file = DocCleaner()._convert_doc_to_docx(temp_file) logging.info("DOC文件已成功转换为DOCX格式") except Exception as e: raise Exception(f"转换doc文件失败: {str(e)}") cleaner = DocCleaner() # 清理文档 main_content, appendix, tables = cleaner.clean_doc(input_file) # 创建临时文件用于保存处理结果 output_docx = os.path.join(temp_dir, 'output.docx') # 保存为docx格式 cleaner.save_as_docx(main_content, appendix, tables, output_docx) # 读取docx文件内容 with open(output_docx, 'rb') as f: docx_bytes = f.read() # 读取文本内容 text_file = os.path.splitext(output_docx)[0] + '.txt' with open(text_file, 'r', encoding='utf-8') as f: text_content = f.read() # 清理临时文件 os.remove(temp_file) if input_file != temp_file: try: os.remove(input_file) except: pass os.remove(output_docx) os.remove(text_file) os.rmdir(temp_dir) return docx_bytes, text_content except Exception as e: logging.error(f"处理文件失败: {str(e)}") raise Exception(f"处理文件失败: {str(e)}") def process_directory(input_dir: str, output_dir: str = None): """ 处理指定目录下的所有文档文件 Args: input_dir: 输入目录路径 output_dir: 输出目录路径，如果为None则使用输入目录 """ # 如果未指定输出目录，使用输入目录 if output_dir is None: output_dir = input_dir if not os.path.exists(output_dir): os.makedirs(output_dir) cleaner = DocCleaner() for root, _, files in os.walk(input_dir): for file in files: if file.endswith(('.doc', '.docx')): input_path = os.path.join(root, file) try: # 清理文档 main_content, appendix, tables = cleaner.clean_doc(input_path) # 创建输出文件名（统一使用docx扩展名） base_name = os.path.splitext(file)[0] output_path = os.path.join(output_dir, f"{base_name}_cleaned.docx") # 保存为docx格式 cleaner.save_as_docx(main_content, appendix, tables, output_path) except Exception as e: print(f"处理文件 {file} 时出错: {str(e)}") # 添加更详细的错误信息 if isinstance(e, subprocess.CalledProcessError): print(f"命令执行错误: {e.output}") elif isinstance(e, FileNotFoundError): print("请确保已安装LibreOffice并将其添加到系统PATH中") def qn(tag: str) -> str: """ 将标签转换为带命名空间的格式 Args: tag: 原始标签 Returns: str: 带命名空间的标签 """ prefix = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" return prefix + tag if __name__ == '__main__': import argparse import sys import json import base64 parser = argparse.ArgumentParser(description='文档清理工具') group = parser.add_mutually_exclusive_group(required=True) group.add_argument('--file', help='输入文件路径') group.add_argument('--stdin', action='store_true', help='从标准输入读取Base64编码的文件二进制数据') group.add_argument('--dir', help='输入目录路径') parser.add_argument('--suffix', help='文件后缀名（不含点号，例如doc或docx）', default='docx') parser.add_argument('--output_dir', help='输出目录路径', required=True) args = parser.parse_args() try: # 确保输出目录存在 os.makedirs(args.output_dir, exist_ok=True) result = { 'status': 'success', 'message': '', 'docxPath': '', 'txtPath': '' } if args.stdin: # 从标准输入读取Base64数据 try: # 读取所有输入数据 base64_data = sys.stdin.read().strip() # 解码Base64数据 byte_array = base64.b64decode(base64_data) # 生成输出文件路径 output_docx = os.path.join(args.output_dir, f"output{args.suffix}") output_txt = os.path.join(args.output_dir, "output.txt") # 处理文件 docx_bytes, text_content = process_file(byte_array, args.suffix) # 保存文件 with open(output_docx, 'wb') as f: f.write(docx_bytes) with open(output_txt, 'w', encoding='utf-8') as f: f.write(text_content) result['docxPath'] = output_docx result['txtPath'] = output_txt result['message'] = 'success' logging.info(f"二进制数据处理成功") except Exception as e: result['status'] = 'error' result['message'] = str(e) logging.error(f"处理二进制数据失败: {str(e)}") elif args.file: # 处理单个文件 input_path = args.file try: # 读取文件内容 with open(input_path, 'rb') as f: byte_array = f.read() # 获取文件后缀 _, suffix = os.path.splitext(input_path) # 生成输出文件路径 base_name = os.path.splitext(os.path.basename(input_path))[0] output_docx = os.path.join(args.output_dir, f"{base_name}_cleaned.docx") output_txt = os.path.join(args.output_dir, f"{base_name}_cleaned.txt") # 处理文件 docx_bytes, text_content = process_file(byte_array, suffix) # 保存文件 with open(output_docx, 'wb') as f: f.write(docx_bytes) with open(output_txt, 'w', encoding='utf-8') as f: f.write(text_content) result['docxPath'] = output_docx result['txtPath'] = output_txt result['message'] = 'success' logging.info(f"文件处理成功: {input_path}") except Exception as e: result['status'] = 'error' result['message'] = str(e) logging.error(f"处理文件失败: {str(e)}") else: # 处理目录 try: process_directory(args.dir, args.output_dir) result['message'] = 'success' logging.info(f"目录处理完成: {args.dir} -> {args.output_dir}") except Exception as e: result['status'] = 'error' result['message'] = str(e) logging.error(f"处理目录失败: {str(e)}") # 只输出JSON格式的结果 print(json.dumps(result, ensure_ascii=False)) sys.exit(0 if result['status'] == 'success' else 1) except Exception as e: error_result = { 'status': 'error', 'message': str(e), 'docxPath': '', 'txtPath': '' } logging.error(f"程序执行错误: {str(e)}") print(json.dumps(error_result, ensure_ascii=False)) sys.exit(1)