企业级 AI Agent 黄金测试集构建与 CI/CD 评测门禁实战：破解长尾样本匮乏、静默能力退化与自动化验收阻断难题

百信企业智能体黄金测试集四层金字塔架构

1. 第一层：核心确定性主干用例（Deterministic Happy Paths, 占比 35%）：涵盖企业核心高频业务流程的标准正向用例（如标准物料库存查询、规范采购申请提交、合规财务报表生成）。该层用例明确规定了意图分类标签、必需调用的工具链及精确参数结构，是验证智能体基本业务能力的不可逾越底线；
2. 第二层：真实生产镜像采样与脱敏提纯（Production Replay & PII Sanitization, 占比 35%）：直接来源于生产网关的真实用户交互轨迹。通过自动化 PII 数据脱敏管线（抹除真实姓名、手机号、企业税号与内部密钥），提纯出带有真实口语化表达、语法残缺、错别字及行话缩写的真实交互，确保评测环境高度贴近生产现场；
3. 第三层：复杂长尾边界与脏数据用例（Boundary & Dirty Data Cases, 占比 20%）：专门构造参数缺失、极值溢出、多意图交织、反向撤回修改以及外部依赖超时的异常场景。验证智能体在面临业务冲突时是否具备主动澄清（Active Clarification）、事务回滚（Transaction Compensation）及优雅降级能力；
4. 第四层：红蓝攻防对抗与合规守卫用例（Adversarial Injection & Security Guardrails, 占比 10%）：针对恶意用户的对抗样本库，包含越狱提示词（Jailbreak Prompts）、间接提示词注入（Indirect Injection）、越权访问（IDOR）诱导以及敏感商业秘密嗅探。验证底层安全沙箱与输出内容审查是否能实现 100% 拦截。

import { existsSync, readFileSync, writeFileSync } from 'node:fs';
import { resolve } from 'node:path';

/**
 * 测试用例严重级别
 */
export type TestCaseSeverity = 'critical' | 'high' | 'medium' | 'low';

/**
 * 期望工具调用结构
 */
export interface ExpectedToolCall {
  toolName: string;
  expectedArgs: Record<string, unknown>;
  isOrderStrict?: boolean;
}

/**
 * 实际工具调用记录
 */
export interface ActualToolCall {
  toolName: string;
  args: Record<string, unknown>;
  timestamp: number;
}

/**
 * 黄金测试用例定义
 */
export interface GoldenTestCase {
  id: string;
  name: string;
  category: 'deterministic' | 'production_replay' | 'boundary' | 'adversarial';
  severity: TestCaseSeverity;
  userPrompt: string;
  contextMock?: Record<string, unknown>;
  expectedToolCalls: ExpectedToolCall[];
  expectedStateMutations?: Record<string, unknown>;
  forbiddenPatterns?: string[];
  referenceOutput?: string;
}

/**
 * 单项断言结果
 */
export interface AssertionResult {
  assertionName: string;
  passed: boolean;
  score: number; // 0.0 ~ 1.0
  message: string;
}

/**
 * 单个用例执行结果
 */
export interface CaseExecutionResult {
  caseId: string;
  severity: TestCaseSeverity;
  passed: boolean;
  overallScore: number;
  durationMs: number;
  tokenUsage: number;
  assertions: AssertionResult[];
}

/**
 * 评测门禁报告结构
 */
export interface GatekeeperSummaryReport {
  totalCases: number;
  passedCases: number;
  failedCases: number;
  overallPassRate: number;
  wilsonConfidenceLowerBound: number;
  criticalViolations: number;
  regressionDetected: boolean;
  gatePassed: boolean;
  gateReason: string;
}

/**
 * 混合断言执行器
 */
export class AssertionEngine {
  /**
   * 执行工具调用轨迹与参数匹配校验
   */
  public static verifyToolTrajectory(
    expectedCalls: ExpectedToolCall[],
    actualCalls: ActualToolCall[]
  ): AssertionResult {
    if (expectedCalls.length === 0 && actualCalls.length === 0) {
      return { assertionName: 'ToolTrajectory', passed: true, score: 1.0, message: '无需工具调用且实际未调用' };
    }

    // 1. 数量与工具名称严格比对
    if (expectedCalls.length !== actualCalls.length) {
      return {
        assertionName: 'ToolTrajectory',
        passed: false,
        score: 0.0,
        message: `期望调用 ${expectedCalls.length} 个工具，实际调用 ${actualCalls.length} 个工具`
      };
    }

    // 2. 逐项核验名称与入参严格子集
    for (let i = 0; i < expectedCalls.length; i++) {
      const exp = expectedCalls[i];
      const act = actualCalls[i];

      if (exp.toolName !== act.toolName) {
        return {
          assertionName: 'ToolTrajectory',
          passed: false,
          score: 0.0,
          message: `第 ${i + 1} 步工具不匹配：期望 ${exp.toolName}，实际 ${act.toolName}`
        };
      }

      // 参数键值严格校验
      for (const [key, val] of Object.entries(exp.expectedArgs)) {
        if (JSON.stringify(act.args[key]) !== JSON.stringify(val)) {
          return {
            assertionName: 'ToolTrajectory',
            passed: false,
            score: 0.0,
            message: `工具 ${exp.toolName} 参数 ${key} 不匹配：期望 ${JSON.stringify(val)}，实际 ${JSON.stringify(act.args[key])}`
          };
        }
      }
    }

    return { assertionName: 'ToolTrajectory', passed: true, score: 1.0, message: '工具调用轨迹与参数校验完全吻合' };
  }

  /**
   * 违规敏感词与反向模式扫描
   */
  public static verifyForbiddenPatterns(
    outputText: string,
    patterns: string[] = []
  ): AssertionResult {
    for (const pattern of patterns) {
      const regex = new RegExp(pattern, 'i');
      if (regex.test(outputText)) {
        return {
          assertionName: 'ForbiddenPatternsGuard',
          passed: false,
          score: 0.0,
          message: `命中禁止输出安全模式/敏感词：${pattern}`
        };
      }
    }
    return { assertionName: 'ForbiddenPatternsGuard', passed: true, score: 1.0, message: '安全模式合规扫描通过' };
  }
}

/**
 * 生产级 CI/CD 评测门禁执行器
 */
export class EvaluationGatekeeper {
  private readonly dataset: GoldenTestCase[];
  private readonly baselinePath?: string;

  constructor(datasetPath: string, baselinePath?: string) {
    if (!existsSync(datasetPath)) {
      throw new Error(`黄金测试集文件不存在：${datasetPath}`);
    }
    this.dataset = JSON.parse(readFileSync(datasetPath, 'utf-8'));
    this.baselinePath = baselinePath;
  }

  /**
   * 计算威尔逊得分区间置信下界 (Wilson Score Interval 95% Confidence)
   */
  public static calculateWilsonLowerBound(successes: number, total: number): number {
    if (total === 0) return 0;
    const z = 1.95996; // 95% 置信水平的 z 分数
    const p = successes / total;
    const z2 = z * z;
    const denominator = 1 + z2 / total;
    const numerator = p + z2 / (2 * total) - z * Math.sqrt((p * (1 - p) + z2 / (4 * total)) / total);
    return Math.max(0, numerator / denominator);
  }

  /**
   * 模拟执行单个智能体推演并运行断言（实际场景对接智能体推演运行时）
   */
  public async executeCase(testCase: GoldenTestCase): Promise<CaseExecutionResult> {
    const startTime = Date.now();

    // 模拟智能体真实调用产物（工程中替换为对 Agent RPC/HTTP 的调用）
    const simulatedToolCalls: ActualToolCall[] = testCase.expectedToolCalls.map(exp => ({
      toolName: exp.toolName,
      args: { ...exp.expectedArgs },
      timestamp: Date.now()
    }));
    const simulatedOutput = `智能体对用例 [ ${testCase.name} ] 的合法响应处理完成。`;

    // 运行混合断言集
    const assertions: AssertionResult[] = [
      AssertionEngine.verifyToolTrajectory(testCase.expectedToolCalls, simulatedToolCalls),
      AssertionEngine.verifyForbiddenPatterns(simulatedOutput, testCase.forbiddenPatterns)
    ];

    const allPassed = assertions.every(a => a.passed);
    const avgScore = assertions.reduce((sum, a) => sum + a.score, 0) / Math.max(1, assertions.length);

    return {
      caseId: testCase.id,
      severity: testCase.severity,
      passed: allPassed,
      overallScore: avgScore,
      durationMs: Date.now() - startTime,
      tokenUsage: 450, // 记录实际耗费 Token
      assertions
    };
  }

  /**
   * 执行全量评测并判定门禁准入结果
   */
  public async runGatekeeperEvaluation(reportOutputPath?: string): Promise<GatekeeperSummaryReport> {
    console.log(`[CI/CD Gatekeeper] 开始执行黄金测试集自动化评测，用例总数：${this.dataset.length} ...`);

    const results: CaseExecutionResult[] = [];
    let criticalViolations = 0;

    for (const testCase of this.dataset) {
      const res = await this.executeCase(testCase);
      results.push(res);
      if (!res.passed && testCase.severity === 'critical') {
        criticalViolations++;
      }
    }

    const passedCount = results.filter(r => r.passed).length;
    const passRate = passedCount / Math.max(1, this.dataset.length);
    const wilsonLower = EvaluationGatekeeper.calculateWilsonLowerBound(passedCount, this.dataset.length);

    // 门禁准入判定准则
    let gatePassed = true;
    let gateReason = '通过所有质量门禁检查，准予上线发版';

    if (criticalViolations > 0) {
      gatePassed = false;
      gateReason = `发现 ${criticalViolations} 个致命级别（Critical）用例违背断言，触发一票否决阻断！`;
    } else if (wilsonLower < 0.90) {
      gatePassed = false;
      gateReason = `测试集通过率 95% 置信区间下界 (${(wilsonLower * 100).toFixed(2)}%) 低于准入标准 (90.00%)，阻断发布！`;
    }

    const report: GatekeeperSummaryReport = {
      totalCases: this.dataset.length,
      passedCases: passedCount,
      failedCases: this.dataset.length - passedCount,
      overallPassRate: Number(passRate.toFixed(4)),
      wilsonConfidenceLowerBound: Number(wilsonLower.toFixed(4)),
      criticalViolations,
      regressionDetected: false,
      gatePassed,
      gateReason
    };

    if (reportOutputPath) {
      writeFileSync(reportOutputPath, JSON.stringify({ report, details: results }, null, 2));
      console.log(`[CI/CD Gatekeeper] 机器可读详细评测报告已保存至：${reportOutputPath}`);
    }

    return report;
  }
}

// CLI 入口支持：通过非零退出码阻断流水线
if (process.argv[1] && process.argv[1].endsWith('gatekeeper.ts')) {
  const inputDataset = process.argv[2] || './golden-dataset.json';
  const outReport = process.argv[3] || './gatekeeper-report.json';
  const gatekeeper = new EvaluationGatekeeper(inputDataset);

  gatekeeper.runGatekeeperEvaluation(outReport).then(report => {
    console.log('\n================ CI/CD 评测门禁报告摘要 ================');
    console.log(`总用例数: ${report.totalCases} | 通过数: ${report.passedCases} | 失败数: ${report.failedCases}`);
    console.log(`实际通过率: ${(report.overallPassRate * 100).toFixed(2)}% | 威尔逊置信下界: ${(report.wilsonConfidenceLowerBound * 100).toFixed(2)}%`);
    console.log(`致命级别违背: ${report.criticalViolations}`);
    console.log(`门禁裁决结论: ${report.gatePassed ? '✅ APPROVED 允许发布' : '❌ BLOCKED 阻断发布'}`);
    console.log(`原因说明: ${report.gateReason}`);
    console.log('=========================================================\n');

    if (!report.gatePassed) {
      process.exit(1); // 非零退出码，触发 CI/CD 构建失败熔断
    }
  }).catch(err => {
    console.error('[CI/CD Gatekeeper] 评测引擎发生致命未捕获异常：', err);
    process.exit(2);
  });
}

CI/CD 流水线（GitLab CI / CNB）阶段性分级执行配置

1. 提交触发阶段（Commit / MR Stage）：运行精简冒烟测试集（Smoke Golden Set, 约 50 个高优先级核心用例），评测执行时间严格控制在 3 分钟以内，快速反馈开发者的单次修改是否存在严重破坏性退化；
2. 夜间全量回归阶段（Nightly Regression Stage）：在每日凌晨的定时流水线中，调度运行全量黄金测试集（Full Golden Set, 500~1000+ 用例），开启 Pass@3 多次采样，生成全景能力雷达图与趋势追踪曲线；
3. 发版准入阶段（Release Gatekeeper Stage）：合并主干或打 Tag 发版时触发，严格执行一票否决与威尔逊置信下界校验，评测报告自动签名并作为源码交付物清册的一部分归档留存。