STT 集成与语音分析

// audio/types.ts

export interface TranscriptSegment {
  start: number;           // 开始时间（毫秒）
  end: number;             // 结束时间（毫秒）
  text: string;            // 转写文本
  confidence?: number;     // 置信度 0-1
  speaker?: string;        // 说话人标签
}

export interface TranscriptResult {
  fullText: string;                    // 完整转写文本
  segments: TranscriptSegment[];       // 带时间戳的逐句转写
  language: string;                    // 检测到的语言
  duration: number;                    // 总时长（毫秒）
}

export interface AudioMetadata {
  path: string;
  format: string;          // wav, mp3, m4a, ...
  sampleRate: number;
  channels: number;
  duration: number;        // 秒
  fileSize: number;        // bytes
}

export interface SpeakerSegment {
  speaker: 'interviewer' | 'candidate' | 'unknown';
  start: number;
  end: number;
  text: string;
}

export type STTProvider = 'whisper' | 'funasr';

// audio/preprocessor.ts

import { execSync } from 'child_process';
import { statSync } from 'fs';
import { AudioMetadata } from './types';

export class AudioPreprocessor {
  /**
   * 探测音频元信息
   */
  probe(filePath: string): AudioMetadata {
    const output = execSync(
      `ffprobe -v quiet -print_format json -show_format -show_streams "${filePath}"`,
      { encoding: 'utf-8' }
    );
    const info = JSON.parse(output);
    const audioStream = info.streams.find((s: any) => s.codec_type === 'audio');

    return {
      path: filePath,
      format: info.format.format_name,
      sampleRate: parseInt(audioStream?.sample_rate ?? '16000'),
      channels: audioStream?.channels ?? 1,
      duration: parseFloat(info.format.duration),
      fileSize: statSync(filePath).size,
    };
  }

  /**
   * 转换为 Whisper 友好格式（16kHz mono WAV）
   */
  normalize(inputPath: string, outputPath: string): string {
    execSync(
      `ffmpeg -y -i "${inputPath}" -ar 16000 -ac 1 -f wav "${outputPath}"`,
      { stdio: 'pipe' }
    );
    return outputPath;
  }

  /**
   * 按时间分段（长录音切割，Whisper 有 25MB 限制）
   */
  split(inputPath: string, outputDir: string, segmentSeconds = 600): string[] {
    const metadata = this.probe(inputPath);
    const totalSeconds = metadata.duration;

    if (totalSeconds <= segmentSeconds) {
      return [inputPath];
    }

    const segments: string[] = [];
    for (let start = 0; start < totalSeconds; start += segmentSeconds) {
      const outputPath = `${outputDir}/segment_${start}.wav`;
      execSync(
        `ffmpeg -y -i "${inputPath}" -ss ${start} -t ${segmentSeconds} -ar 16000 -ac 1 "${outputPath}"`,
        { stdio: 'pipe' }
      );
      segments.push(outputPath);
    }

    return segments;
  }

  /**
   * 简单降噪（可选，对低质量录音有帮助）
   */
  denoise(inputPath: string, outputPath: string): string {
    // 使用 ffmpeg 的 anlmdn 滤镜做轻量降噪
    execSync(
      `ffmpeg -y -i "${inputPath}" -af "anlmdn=s=7:p=0.002:r=0.002" "${outputPath}"`,
      { stdio: 'pipe' }
    );
    return outputPath;
  }
}

// audio/whisper.ts

import OpenAI from 'openai';
import { createReadStream } from 'fs';
import { TranscriptResult, TranscriptSegment } from './types';

export class WhisperAdapter {
  private client: OpenAI;

  constructor(apiKey: string) {
    this.client = new OpenAI({ apiKey });
  }

  async transcribe(filePath: string, language = 'zh'): Promise<TranscriptResult> {
    const response = await this.client.audio.transcriptions.create({
      file: createReadStream(filePath),
      model: 'whisper-1',
      language,
      response_format: 'verbose_json',
      timestamp_granularities: ['segment'],
    });

    const segments: TranscriptSegment[] = (response as any).segments?.map((seg: any) => ({
      start: Math.round(seg.start * 1000),
      end: Math.round(seg.end * 1000),
      text: seg.text.trim(),
      confidence: seg.avg_logprob ? Math.exp(seg.avg_logprob) : undefined,
    })) ?? [];

    return {
      fullText: response.text,
      segments,
      language: (response as any).language ?? language,
      duration: segments.length > 0 ? segments[segments.length - 1].end : 0,
    };
  }

  /**
   * 处理长音频（分段转写 + 合并）
   */
  async transcribeLong(
    segmentPaths: string[],
    language = 'zh',
    onProgress?: (done: number, total: number) => void,
  ): Promise<TranscriptResult> {
    const allSegments: TranscriptSegment[] = [];
    let fullText = '';
    let timeOffset = 0;

    for (let i = 0; i < segmentPaths.length; i++) {
      const result = await this.transcribe(segmentPaths[i], language);

      // 偏移时间戳
      const offsetSegments = result.segments.map(seg => ({
        ...seg,
        start: seg.start + timeOffset,
        end: seg.end + timeOffset,
      }));

      allSegments.push(...offsetSegments);
      fullText += result.fullText + '\n';
      timeOffset += result.duration;

      onProgress?.(i + 1, segmentPaths.length);
    }

    return {
      fullText: fullText.trim(),
      segments: allSegments,
      language,
      duration: timeOffset,
    };
  }
}

// audio/funasr.ts

import { execSync } from 'child_process';
import { readFileSync } from 'fs';
import { TranscriptResult, TranscriptSegment } from './types';

export class FunASRAdapter {
  private modelPath: string;
  private pythonBin: string;

  constructor(opts: { modelPath?: string; pythonBin?: string } = {}) {
    this.modelPath = opts.modelPath ?? '~/.cache/funasr/models';
    this.pythonBin = opts.pythonBin ?? 'python3';
  }

  async transcribe(filePath: string): Promise<TranscriptResult> {
    // 调用 Python 脚本执行 FunASR 推理
    const scriptPath = `${__dirname}/funasr_infer.py`;
    const outputPath = `/tmp/funasr_result_${Date.now()}.json`;

    execSync(
      `${this.pythonBin} "${scriptPath}" --input "${filePath}" --output "${outputPath}" --model-dir "${this.modelPath}"`,
      { timeout: 120000, stdio: 'pipe' }
    );

    const result = JSON.parse(readFileSync(outputPath, 'utf-8'));

    const segments: TranscriptSegment[] = result.sentences.map((s: any) => ({
      start: s.start,
      end: s.end,
      text: s.text,
      confidence: s.confidence,
    }));

    return {
      fullText: segments.map(s => s.text).join(''),
      segments,
      language: 'zh',
      duration: segments.length > 0 ? segments[segments.length - 1].end : 0,
    };
  }

  /**
   * 检查 FunASR 是否可用
   */
  isAvailable(): boolean {
    try {
      execSync(`${this.pythonBin} -c "import funasr"`, { stdio: 'pipe' });
      return true;
    } catch {
      return false;
    }
  }
}

// tools/transcribe.ts

import { Tool, ToolResult, ToolContext } from './types';
import { AudioPreprocessor } from '../audio/preprocessor';
import { WhisperAdapter } from '../audio/whisper';
import { FunASRAdapter } from '../audio/funasr';
import { TranscriptResult, STTProvider } from '../audio/types';

interface TranscribeInput {
  audioFilePath: string;
  language?: string;
  provider?: STTProvider;
  denoise?: boolean;
}

export const transcribeTool: Tool<TranscribeInput, TranscriptResult> = {
  name: 'transcribe_audio',
  description: '将面试录音转为带时间戳的文字稿。支持 Whisper API（在线）和 FunASR（离线）两种模式。',
  parameters: {
    type: 'object',
    properties: {
      audioFilePath: { type: 'string', description: '音频文件路径' },
      language: { type: 'string', description: '语言（默认 zh）', enum: ['zh', 'en'] },
      provider: { type: 'string', description: 'STT 引擎', enum: ['whisper', 'funasr'] },
      denoise: { type: 'boolean', description: '是否预降噪（默认 false）' },
    },
    required: ['audioFilePath'],
  },

  async execute(input, ctx): Promise<ToolResult<TranscriptResult>> {
    const { audioFilePath, language = 'zh', provider = 'whisper', denoise = false } = input;
    const preprocessor = new AudioPreprocessor();

    // 1. 探测音频信息
    let metadata;
    try {
      metadata = preprocessor.probe(audioFilePath);
    } catch {
      return { success: false, error: { code: 'input_error', message: `无法读取音频文件: ${audioFilePath}` } };
    }

    // 2. 预处理
    const normalizedPath = `/tmp/normalized_${Date.now()}.wav`;
    let processedPath = preprocessor.normalize(audioFilePath, normalizedPath);

    if (denoise) {
      const denoisedPath = `/tmp/denoised_${Date.now()}.wav`;
      processedPath = preprocessor.denoise(processedPath, denoisedPath);
    }

    // 3. 分段（如果超过 10 分钟）
    const segments = preprocessor.split(processedPath, '/tmp', 600);

    // 4. 转写
    let result: TranscriptResult;
    try {
      if (provider === 'whisper') {
        const whisper = new WhisperAdapter(process.env.OPENAI_API_KEY!);
        result = segments.length === 1
          ? await whisper.transcribe(segments[0], language)
          : await whisper.transcribeLong(segments, language);
      } else {
        const funasr = new FunASRAdapter();
        if (!funasr.isAvailable()) {
          return { success: false, error: { code: 'service_error', message: 'FunASR 未安装。请运行: pip install funasr' } };
        }
        result = await funasr.transcribe(processedPath);
      }
    } catch (err) {
      return { success: false, error: { code: 'service_error', message: `转写失败: ${err}` } };
    }

    return { success: true, data: result };
  },
};

// audio/speaker-detect.ts

import { TranscriptSegment, SpeakerSegment } from './types';

/**
 * 启发式说话人分离
 * 逻辑：面试中通常面试官说话短（提问），候选人说话长（回答）
 */
export function detectSpeakersHeuristic(segments: TranscriptSegment[]): SpeakerSegment[] {
  // Step 1: 按停顿分组（>2s 的间隔认为是换人）
  const groups = groupByPause(segments, 2000);

  // Step 2: 计算每组的时长
  const durations = groups.map(g => ({
    segments: g,
    totalDuration: g.reduce((sum, s) => sum + (s.end - s.start), 0),
    text: g.map(s => s.text).join(''),
  }));

  // Step 3: 启发式分类
  // - 短发言 + 问号结尾 → 面试官
  // - 长发言 → 候选人
  const avgDuration = durations.reduce((s, d) => s + d.totalDuration, 0) / durations.length;

  return durations.map(d => {
    const isQuestion = d.text.includes('？') || d.text.includes('?') || d.text.endsWith('吗');
    const isShort = d.totalDuration < avgDuration * 0.5;

    const speaker = (isQuestion || isShort) ? 'interviewer' : 'candidate';

    return {
      speaker,
      start: d.segments[0].start,
      end: d.segments[d.segments.length - 1].end,
      text: d.text,
    } as SpeakerSegment;
  });
}

/**
 * LLM 辅助说话人分离（精度更高）
 */
export async function detectSpeakersWithLLM(
  segments: TranscriptSegment[],
  queryEngine: QueryEngine,
): Promise<SpeakerSegment[]> {
  // 先用启发式做初步分类
  const heuristic = detectSpeakersHeuristic(segments);

  // 对不确定的段落用 LLM 辅助判断
  const uncertain = heuristic.filter(s => s.speaker === 'unknown');
  if (uncertain.length === 0) return heuristic;

  const response = await queryEngine.query({
    task: 'split_qa',
    messages: [{
      role: 'user',
      content: `以下是面试录音的转写片段，请判断每段是面试官还是候选人在说话。输出JSON数组。

${uncertain.map((s, i) => `[${i}] "${s.text.slice(0, 100)}"`).join('\n')}`,
    }],
    systemPrompt: '你是面试录音的说话人分类器。面试官通常提问、追问，候选人通常回答、解释。只输出JSON。',
  });

  try {
    const labels = JSON.parse(response.content ?? '[]');
    for (let i = 0; i < uncertain.length && i < labels.length; i++) {
      uncertain[i].speaker = labels[i].speaker ?? 'candidate';
    }
  } catch {
    // LLM 输出解析失败，保持启发式结果
  }

  return heuristic;
}

function groupByPause(segments: TranscriptSegment[], pauseThreshold: number): TranscriptSegment[][] {
  const groups: TranscriptSegment[][] = [];
  let current: TranscriptSegment[] = [];

  for (let i = 0; i < segments.length; i++) {
    current.push(segments[i]);

    if (i < segments.length - 1) {
      const gap = segments[i + 1].start - segments[i].end;
      if (gap > pauseThreshold) {
        groups.push(current);
        current = [];
      }
    }
  }

  if (current.length > 0) groups.push(current);
  return groups;
}

// tools/detect-speakers.ts

import { Tool, ToolResult } from './types';
import { SpeakerSegment, TranscriptSegment } from '../audio/types';
import { detectSpeakersHeuristic, detectSpeakersWithLLM } from '../audio/speaker-detect';

interface DetectSpeakersInput {
  segments: TranscriptSegment[];
  method?: 'heuristic' | 'llm_assisted';
}

interface DetectSpeakersOutput {
  speakers: SpeakerSegment[];
  labeledTranscript: string;
  interviewerSegments: number;
  candidateSegments: number;
}

export const detectSpeakersTool: Tool<DetectSpeakersInput, DetectSpeakersOutput> = {
  name: 'detect_speakers',
  description: '对转写结果进行说话人分离，区分面试官和候选人。',
  parameters: {
    type: 'object',
    properties: {
      segments: { type: 'array', description: '带时间戳的转写分段' },
      method: { type: 'string', enum: ['heuristic', 'llm_assisted'], description: '分离方法' },
    },
    required: ['segments'],
  },

  async execute(input, ctx): Promise<ToolResult<DetectSpeakersOutput>> {
    const { segments, method = 'heuristic' } = input;

    let speakers: SpeakerSegment[];
    if (method === 'llm_assisted' && ctx.queryEngine) {
      speakers = await detectSpeakersWithLLM(segments, ctx.queryEngine);
    } else {
      speakers = detectSpeakersHeuristic(segments);
    }

    // 生成带标签的完整文本
    const labeledTranscript = speakers.map(s => {
      const label = s.speaker === 'interviewer' ? '面试官' : '候选人';
      return `${label}：${s.text}`;
    }).join('\n\n');

    return {
      success: true,
      data: {
        speakers,
        labeledTranscript,
        interviewerSegments: speakers.filter(s => s.speaker === 'interviewer').length,
        candidateSegments: speakers.filter(s => s.speaker === 'candidate').length,
      },
    };
  },
};

// 在 SessionConfig 中新增音频相关配置

interface AudioConfig {
  sttProvider: 'whisper' | 'funasr';       // STT 引擎选择
  language: 'zh' | 'en';                    // 语言
  denoise: boolean;                         // 是否降噪
  speakerDetection: 'heuristic' | 'llm_assisted';  // 说话人分离方法
  fillerWords: string[];                    // 填充词列表（可自定义）
  longPauseThreshold: number;               // 长停顿阈值（ms）
  idealWpm: { min: number; max: number };   // 理想语速范围
}

const DEFAULT_AUDIO_CONFIG: AudioConfig = {
  sttProvider: 'whisper',
  language: 'zh',
  denoise: false,
  speakerDetection: 'heuristic',
  fillerWords: ['嗯', '那个', '就是', '然后', '这个', '额', '对对对'],
  longPauseThreshold: 2000,
  idealWpm: { min: 200, max: 280 },
};