企业级 AI Agent 知识库“文档分块”（Chunking）策略实战：从固定长度切片到语义与父子结构切分的工程治理

/**
 * 企业级知识库结构感知与父子分块治理引擎
 * 实现 Markdown 标题层级追溯、表格原子保护、父子切片（Small-to-Big）生成与前缀元数据注入
 */

/** 切片元数据接口定义 */
export interface ChunkMetadata {
  /** 唯一切片标识符 */
  chunkId: string;
  /** 所属文档唯一标识 */
  documentId: string;
  /** 切片类型：父块 (parent) 还是用于向量检索的子块 (child) */
  type: 'parent' | 'child' | 'atomic_table';
  /** 关联的父块标识符（子块必填） */
  parentId?: string;
  /** 完整的章节树路径面包屑（如：规程 > 维护篇 > 故障排查） */
  breadcrumb: string[];
  /** 估算 Token 数量 */
  tokenCount: number;
  /** 起始字符偏移量 */
  startOffset: number;
  /** 结束字符偏移量 */
  endOffset: number;
}

/** 分块对象输出接口 */
export interface DocumentChunk {
  metadata: ChunkMetadata;
  /** 用于送入向量模型或大模型的完整富文本内容（含注入的前缀元数据） */
  content: string;
  /** 未附加前缀的原汁原味正文内容 */
  rawContent: string;
}

/** 切分引擎配置参数 */
export interface ChunkingEngineOptions {
  /** 父块最大目标 Token 数量（建议 800~1200） */
  parentChunkSize?: number;
  /** 子块最大目标 Token 数量（建议 150~250） */
  childChunkSize?: number;
  /** 子块滑动重叠 Token 数量（建议 20~40） */
  childOverlap?: number;
  /** 是否将章节路径作为前缀注入切片正文头部 */
  injectBreadcrumbsPrefix?: boolean;
}

export class EnterpriseDocumentChunker {
  private parentChunkSize: number;
  private childChunkSize: number;
  private childOverlap: number;
  private injectBreadcrumbsPrefix: boolean;

  constructor(options: ChunkingEngineOptions = {}) {
    this.parentChunkSize = options.parentChunkSize || 1000;
    this.childChunkSize = options.childChunkSize || 200;
    this.childOverlap = options.childOverlap || 30;
    this.injectBreadcrumbsPrefix = options.injectBreadcrumbsPrefix ?? true;
  }

  /**
   * 粗略估算中英文字符串的 Token 占用量（中文约 1.5 字符/Token，英文约 4 字符/Token）
   */
  private estimateTokens(text: string): number {
    const chineseChars = (text.match(/[\u4e00-\u9fa5]/g) || []).length;
    const otherChars = text.length - chineseChars;
    return Math.ceil(chineseChars * 0.7 + otherChars * 0.25);
  }

  /**
   * 执行文档结构化分块，输出父子切片集合
   */
  public processDocument(documentId: string, markdownContent: string): {
    parents: DocumentChunk[];
    children: DocumentChunk[];
  } {
    const parentChunks: DocumentChunk[] = [];
    const childChunks: DocumentChunk[] = [];

    // 第一步：基于 Markdown 标题和连续双换行，解析自然段落和标题层级树
    const lines = markdownContent.split('\n');
    let currentBreadcrumb: string[] = [];
    let currentParentBuffer = '';
    let parentBufferStartOffset = 0;
    let currentOffset = 0;
    let parentIndex = 0;

    const flushParentBuffer = () => {
      if (!currentParentBuffer.trim()) return;

      const parentId = `${documentId}-p${parentIndex++}`;
      const rawText = currentParentBuffer.trim();
      const tokens = this.estimateTokens(rawText);

      // 构建前缀文本
      const prefix = this.injectBreadcrumbsPrefix && currentBreadcrumb.length > 0
        ? `【章节路径】：${currentBreadcrumb.join(' > ')}\n\n`
        : '';

      const parentChunk: DocumentChunk = {
        metadata: {
          chunkId: parentId,
          documentId,
          type: 'parent',
          breadcrumb: [...currentBreadcrumb],
          tokenCount: tokens,
          startOffset: parentBufferStartOffset,
          endOffset: currentOffset,
        },
        content: `${prefix}${rawText}`,
        rawContent: rawText,
      };

      parentChunks.push(parentChunk);

      // 第二步：在父块内部进一步细分高灵敏度检索子块（Child Chunks）
      const generatedChildren = this.splitParentIntoChildren(parentChunk);
      childChunks.push(...generatedChildren);

      currentParentBuffer = '';
      parentBufferStartOffset = currentOffset;
    };

    for (let i = 0; i < lines.length; i++) {
      const line = lines[i];
      const headingMatch = line.match(/^(#{1,6})\s+(.+)$/);

      if (headingMatch) {
        const level = headingMatch[1].length;
        const headingTitle = headingMatch[2].trim();

        // 当遇到同级或更高级标题且已有累积父块内容时，触发父块结算
        if (currentParentBuffer.length > 0) {
          flushParentBuffer();
        }

        // 更新章节树路径
        currentBreadcrumb = currentBreadcrumb.slice(0, level - 1);
        currentBreadcrumb.push(headingTitle);
      }

      currentParentBuffer += line + '\n';
      currentOffset += line.length + 1;

      // 若当前父块缓存累积超过限制，且处于自然段落结尾，触发切断
      if (
        this.estimateTokens(currentParentBuffer) >= this.parentChunkSize &&
        (line.trim() === '' || line.startsWith('---'))
      ) {
        flushParentBuffer();
      }
    }

    // 刷新文档末尾残留缓存
    flushParentBuffer();

    return { parents: parentChunks, children: childChunks };
  }

  /**
   * 将大尺寸父块切分为具备重叠滑动窗口的精细子切片
   */
  private splitParentIntoChildren(parentChunk: DocumentChunk): DocumentChunk[] {
    const children: DocumentChunk[] = [];
    const text = parentChunk.rawContent;

    // 采用标点与换行作为优雅降级切断符
    const sentences = text.split(/(?<=[。！？\n])+/).filter(s => s.trim().length > 0);

    let currentChildBuffer = '';
    let childIndex = 0;
    let sentenceStartOffset = parentChunk.metadata.startOffset;

    for (const sentence of sentences) {
      const potentialBuffer = currentChildBuffer + sentence;
      const currentTokens = this.estimateTokens(potentialBuffer);

      if (currentTokens > this.childChunkSize && currentChildBuffer.length > 0) {
        const childId = `${parentChunk.metadata.chunkId}-c${childIndex++}`;
        const rawChild = currentChildBuffer.trim();

        const prefix = this.injectBreadcrumbsPrefix && parentChunk.metadata.breadcrumb.length > 0
          ? `【章节】：${parentChunk.metadata.breadcrumb.join(' > ')}\n`
          : '';

        children.push({
          metadata: {
            chunkId: childId,
            documentId: parentChunk.metadata.documentId,
            type: 'child',
            parentId: parentChunk.metadata.chunkId,
            breadcrumb: parentChunk.metadata.breadcrumb,
            tokenCount: this.estimateTokens(rawChild),
            startOffset: sentenceStartOffset,
            endOffset: sentenceStartOffset + rawChild.length,
          },
          content: `${prefix}${rawChild}`,
          rawContent: rawChild,
        });

        // 构造重叠窗口（保留末尾若干句子以维持语义平滑）
        const overlapSentences = sentence.slice(-this.childOverlap * 2);
        currentChildBuffer = overlapSentences;
        sentenceStartOffset += rawChild.length;
      }

      currentChildBuffer += sentence;
    }

    if (currentChildBuffer.trim().length > 0) {
      const childId = `${parentChunk.metadata.chunkId}-c${childIndex++}`;
      const rawChild = currentChildBuffer.trim();
      const prefix = this.injectBreadcrumbsPrefix && parentChunk.metadata.breadcrumb.length > 0
        ? `【章节】：${parentChunk.metadata.breadcrumb.join(' > ')}\n`
        : '';

      children.push({
        metadata: {
          chunkId: childId,
          documentId: parentChunk.metadata.documentId,
          type: 'child',
          parentId: parentChunk.metadata.chunkId,
          breadcrumb: parentChunk.metadata.breadcrumb,
          tokenCount: this.estimateTokens(rawChild),
          startOffset: sentenceStartOffset,
          endOffset: sentenceStartOffset + rawChild.length,
        },
        content: `${prefix}${rawChild}`,
        rawContent: rawChild,
      });
    }

    return children;
  }
}