企业级 AI Agent 流量录制回放与离线确定性回归测试实战：面向模型切换与 Prompt 演进的无损仿真验证架构

/**
 * 企业级 AI Agent 流量录制回放与离线确定性回归测试核心引擎
 * 职责：拦截并录制生产多轮交互轨迹、虚拟化外部依赖打桩、执行确定性离线回放与语义模糊断言
 */

import { createHash } from 'node:crypto';

// 工具调用记录契约定义
export interface RecordedToolCall {
  stepIndex: number;
  toolName: string;
  arguments: Record<string, any>;
  response: any;
  isMutating: boolean; // 是否为非幂等写操作
  timestampMs: number;
}

// 生产流量全息录制档案格式
export interface AgentTrafficRecord {
  recordId: string;
  tenantId: string;
  sessionId: string;
  recordedAt: string;
  initialUserInput: string;
  systemPromptVersion: string;
  modelIdentifier: string;
  toolCalls: RecordedToolCall[];
  finalOutputText: string;
  expectedFinalState: string;
}

// 回放断言评估结果结构
export interface ReplayAssertionResult {
  passed: boolean;
  score: number; // 综合回归吻合度评分 (0.0 ~ 1.0)
  discrepancies: Array<{
    severity: 'CRITICAL' | 'WARNING';
    stepIndex: number;
    rule: string;
    message: string;
  }>;
}

/**
 * 虚拟工具打桩仿真器（Virtual Tool Mock Harness）
 * 在离线环境下安全拦截外部网络请求，基于多维度匹配算法返回生产历史仿真报文
 */
export class VirtualToolHarness {
  private recordedMocks: Map<string, RecordedToolCall[]> = new Map();
  private invocationHistory: RecordedToolCall[] = [];

  constructor(records: RecordedToolCall[]) {
    // 按工具名称对生产调用轨迹建立索引
    for (const record of records) {
      const list = this.recordedMocks.get(record.toolName) || [];
      list.push(record);
      this.recordedMocks.set(record.toolName, list);
    }
  }

  /**
   * 工具调用拦截打桩入口
   * 采用两阶段匹配：精确匹配 -> 语义容差匹配
   */
  public async executeMockTool(
    toolName: string,
    args: Record<string, any>,
    stepIndex: number
  ): Promise<any> {
    const candidates = this.recordedMocks.get(toolName);
    if (!candidates || candidates.length === 0) {
      throw new Error(`[虚拟打桩阻断] 录制档案中未包含工具 [${toolName}] 的生产调用样本，禁止穿透外部真实网络！`);
    }

    // 优先：按步长索引或完全一致的参数特征匹配
    let matched = candidates.find(c => this.isExactMatch(c.arguments, args));

    // 次选：当 Prompt 演进引起非关键字段轻微变化时的语义容差模糊匹配
    if (!matched) {
      matched = candidates.find(c => this.isSemanticCompatible(c.arguments, args));
    }

    if (!matched) {
      throw new Error(`[虚拟打桩异常] 工具 [${toolName}] 入参无法在历史录制中找到兼容桩数据: ${JSON.stringify(args)}`);
    }

    // 记录回放期间实际发生的工具调用行为
    this.invocationHistory.push({
      stepIndex,
      toolName,
      arguments: args,
      response: matched.response,
      isMutating: matched.isMutating,
      timestampMs: Date.now()
    });

    return matched.response;
  }

  public getInvocations(): RecordedToolCall[] {
    return this.invocationHistory;
  }

  // 严格参数哈希对比
  private isExactMatch(a: Record<string, any>, b: Record<string, any>): boolean {
    return this.hashObject(a) === this.hashObject(b);
  }

  // 语义兼容性比对：核心业务主键严格相等，允许非关键展示字段冗余
  private isSemanticCompatible(baseline: Record<string, any>, current: Record<string, any>): boolean {
    const criticalKeys = ['id', 'orderId', 'userId', 'amount', 'status', 'voucherNo', 'itemId'];
    for (const key of criticalKeys) {
      if (baseline[key] !== undefined || current[key] !== undefined) {
        if (String(baseline[key]) !== String(current[key])) {
          return false;
        }
      }
    }
    return true;
  }

  private hashObject(obj: any): string {
    const sortedStr = JSON.stringify(obj, Object.keys(obj || {}).sort());
    return createHash('sha256').update(sortedStr).digest('hex');
  }
}

/**
 * 确定性回归断言评估器（Deterministic Regression Evaluator）
 * 对比录制基准轨迹与当前回放轨迹，生成细粒度的质量准入报告
 */
export class DeterministicRegressionEvaluator {
  /**
   * 执行综合三层断言比对
   */
  public evaluate(
    baseline: AgentTrafficRecord,
    replayedInvocations: RecordedToolCall[],
    replayedFinalState: string
  ): ReplayAssertionResult {
    const discrepancies: ReplayAssertionResult['discrepancies'] = [];
    let criticalErrors = 0;

    // 断言第 1 层：工具调用序列与意图同构性比对
    if (baseline.toolCalls.length !== replayedInvocations.length) {
      discrepancies.push({
        severity: 'WARNING',
        stepIndex: -1,
        rule: 'STEP_COUNT_DRIFT',
        message: `推理步数发生漂移：基准步数 ${baseline.toolCalls.length}，当前回放步数 ${replayedInvocations.length}`
      });
    }

    // 断言第 2 层：非幂等写操作因果一致性与参数强约束校验
    const baselineWrites = baseline.toolCalls.filter(c => c.isMutating);
    const replayedWrites = replayedInvocations.filter(c => c.isMutating);

    if (baselineWrites.length !== replayedWrites.length) {
      discrepancies.push({
        severity: 'CRITICAL',
        stepIndex: -1,
        rule: 'MUTATING_ACTION_MISMATCH',
        message: `写操作工具数量不一致！基准调用 ${baselineWrites.length} 次，回放调用 ${replayedWrites.length} 次，存在状态破坏风险！`
      });
      criticalErrors++;
    }

    // 逐项校验对应步长的关键工具参数一致性
    const minLength = Math.min(baseline.toolCalls.length, replayedInvocations.length);
    for (let i = 0; i < minLength; i++) {
      const bCall = baseline.toolCalls[i];
      const rCall = replayedInvocations[i];

      if (bCall.toolName !== rCall.toolName) {
        discrepancies.push({
          severity: 'CRITICAL',
          stepIndex: i,
          rule: 'TOOL_IDENTITY_DIVERGENCE',
          message: `第 ${i + 1} 步工具选择发生分叉：预期调用 [${bCall.toolName}]，实际调用 [${rCall.toolName}]`
        });
        criticalErrors++;
        continue;
      }

      // 关键业务主键强断言校验
      for (const [key, val] of Object.entries(bCall.arguments)) {
        if (['orderId', 'amount', 'tenantId'].includes(key) && rCall.arguments[key] !== val) {
          discrepancies.push({
            severity: 'CRITICAL',
            stepIndex: i,
            rule: 'CRITICAL_PARAMETER_MUTATION',
            message: `关键字段 [${key}] 发生破损：基准值 ${val}，回放值 ${rCall.arguments[key]}`
          });
          criticalErrors++;
        }
      }
    }

    // 断言第 3 层：最终业务状态机终态收敛校验
    if (baseline.expectedFinalState !== replayedFinalState) {
      discrepancies.push({
        severity: 'CRITICAL',
        stepIndex: -1,
        rule: 'FINAL_STATE_DIVERGENCE',
        message: `业务终态未收敛：预期状态 [${baseline.expectedFinalState}]，实际终态 [${replayedFinalState}]`
      });
      criticalErrors++;
    }

    // 计算综合吻合度得分 (0.0 ~ 1.0)
    const baseScore = 1.0 - (criticalErrors * 0.35) - (discrepancies.length * 0.05);
    const score = Math.max(0.0, Math.min(1.0, Number(baseScore.toFixed(2))));
    const passed = criticalErrors === 0 && score >= 0.85;

    return { passed, score, discrepancies };
  }
}