fls-asp-hm-intelligent-tutoring/src/views/demo/AsrDemo.vue

<template>
  <div class="no-select">
    <!-- <button @mousedown="recStart('mp3')" @mouseup="recStop">
      mp3按住录音并实时asr
    </button> -->
    <van-button
      style="margin: 20px 0"
      type="primary"
      size="large"
      @touchstart="recStart('wav')"
      @touchend="recStop"
      >手机端长按</van-button
    >
    <van-button
      type="primary"
      size="large"
      @mousedown="recStart('wav')"
      @mouseup="recStop"
      >电脑端鼠标长按</van-button
    >
    <!-- <button @mousedown="recStart('pcm')" @mouseup="recStop">
      pcm按住录音并实时asr(不支持播放)
    </button> -->
    <div>{{ userTalk }}</div>
    <div>{{ text }}</div>
    <!-- <div class="audioPlay"></div>
    <div class="progress"></div> -->
  </div>
</template>

<script setup>
import { ref, onMounted, onBeforeUnmount } from "vue";
import axios from "axios";
import Recorder from "recorder-core";
import "recorder-core/src/engine/mp3";
import "recorder-core/src/engine/mp3-engine";
import "recorder-core/src/engine/wav";
import "recorder-core/src/engine/pcm";
//可选的插件支持项，把需要的插件按需引入进来即可
//import 'recorder-core/src/extensions/waveview'
import "recorder-core/src/extensions/frequency.histogram.view";
import "recorder-core/src/extensions/lib.fft";

const userTalk = ref("");

let rec;
let testOutputWavLog = ref(false); // 用于调试是否输出wav格式
let testSampleRate = ref(16000);
let testBitRate = ref(16);
let SendInterval = 300; // 控制实时传输间隔

let realTimeSendTryType,
  realTimeSendTryEncBusy,
  realTimeSendTryTime = 0,
  realTimeSendTryNumber,
  transferUploadNumberMax,
  realTimeSendTryChunk,
  voicePkgSeq = 0;

const RealTimeSendTryReset = (type) => {
  realTimeSendTryType = type;
  realTimeSendTryTime = 0;
  realTimeSendTryEncBusy = 0;
  realTimeSendTryNumber = 0;
  transferUploadNumberMax = 0;
  realTimeSendTryChunk = null;
};

const RealTimeSendTry = (buffers, bufferSampleRate, isClose) => {
  const t1 = Date.now();
  if (realTimeSendTryTime === 0) {
    realTimeSendTryTime = t1;
    realTimeSendTryEncBusy = 0;
    realTimeSendTryNumber = 0;
    transferUploadNumberMax = 0;
    realTimeSendTryChunk = null;
  }

  if (!isClose && t1 - realTimeSendTryTime < SendInterval) {
    return; // 控制缓冲达到指定间隔才进行传输
  }

  realTimeSendTryTime = t1;
  const number = ++realTimeSendTryNumber;

  let pcm = [],
    pcmSampleRate = 0;
  if (buffers.length > 0) {
    // 借用SampleData函数进行数据的连续处理，采样率转换是顺带的，得到新的pcm数据
    const chunk = Recorder.SampleData(
      buffers,
      bufferSampleRate,
      testSampleRate.value,
      realTimeSendTryChunk,
      { frameType: isClose ? "" : realTimeSendTryType }
    );

    for (
      let i = realTimeSendTryChunk ? realTimeSendTryChunk.index : 0;
      i < chunk.index;
      i++
    ) {
      buffers[i] = null; // 清理已处理完的缓冲数据
    }

    realTimeSendTryChunk = chunk;
    pcm = chunk.data;
    pcmSampleRate = chunk.sampleRate;
  }

  // 如果没有新数据，或者结束时数据量太小，不能进行转码
  if (pcm.length === 0 || (isClose && pcm.length < 2000)) {
    TransferUpload(number, null, 0, null, isClose);
    return;
  }

  // 实时编码队列阻塞处理
  if (!isClose) {
    if (realTimeSendTryEncBusy >= 2) {
      console.log("编码队列阻塞，已丢弃一帧", 1);
      return;
    }
  }
  realTimeSendTryEncBusy++;

  const encStartTime = Date.now();
  const recMock = Recorder({
    type: realTimeSendTryType,
    sampleRate: testSampleRate.value, // 采样率
    bitRate: testBitRate.value, // 比特率
  });

  recMock.mock(pcm, pcmSampleRate);
  recMock.stop(
    (blob, duration) => {
      if (realTimeSendTryEncBusy) realTimeSendTryEncBusy--;
      blob.encTime = Date.now() - encStartTime;
      TransferUpload(number, blob, duration, recMock, isClose);
    },
    (msg) => {
      if (realTimeSendTryEncBusy) realTimeSendTryEncBusy--;
      console.log("不应该出现的错误:" + msg, 1);
    }
  );
};

const TransferUpload = (number, blobOrNull, duration, blobRec, isClose) => {
  transferUploadNumberMax = Math.max(transferUploadNumberMax, number);
  if (blobOrNull) {
    let blob = blobOrNull;
    let encTime = blob.encTime;
    // 发送数据的方式一：Base64文本发送
    let reader = new FileReader();
    reader.onloadend = () => {
      let base64 = (/.+;\s*base64\s*,\s*(.+)$/i.exec(reader.result) || [])[1];
      // 可以实现 WebSocket send(base64), WebRTC send(base64), XMLHttpRequest send(base64)
      asrPost("SESSION_IN", base64);
    };
    reader.readAsDataURL(blob);

    // 发送数据的方式二：Blob二进制发送
    // 可以实现 WebSocket send(blob), WebRTC send(blob), XMLHttpRequest send(blob)

    const numberFail =
      number < transferUploadNumberMax
        ? '<span style="color:red">顺序错乱的数据，如果要求不高可以直接丢弃，或者调大SendInterval试试</span>'
        : "";
    const logMsg =
      "No." +
      (number < 100 ? ("000" + number).substr(-3) : number) +
      numberFail;

    console.log(
      blob,
      duration,
      blobRec,
      logMsg + "花" + ("___" + encTime).substr(-3) + "ms"
    );

    // 插入html
    const childDiv = document.createElement("div");
    const button = document.createElement("button");

    childDiv.textContent = `${logMsg + ("___" + encTime).substr(-3) + "ms"}`;
    button.textContent = "播放";
    button.addEventListener("click", () => {
      recPlay(blob);
    });

    childDiv.appendChild(button);
    //document.querySelector(".progress").appendChild(childDiv);
  }

  if (isClose) {
    console.log(
      "No." +
        (number < 100 ? ("000" + number).substr(-3) : number) +
        ":已停止传输"
    );
  }
};

const recStart = async (type) => {
  userTalk.value = "";
  text.value = "";
  lastStrIndex = 0;
  await asrPost("SESSION_BEGIN");
  rec.start();
  RealTimeSendTryReset(type);
};

const recStop = async () => {
  rec.stop();
  RealTimeSendTry([], 0, true); // 最后一次发送
  await asrPost("SESSION_END");
  voicePkgSeq = 0;
  chatGpt();
};

const recPlay = (blob) => {
  const audioPlayElement = document.querySelector(".audioPlay");
  audioPlayElement.innerHTML = "";
  const audio = document.createElement("audio");
  audio.controls = true;
  audioPlayElement.appendChild(audio);
  audio.src = (window.URL || webkitURL).createObjectURL(blob);
  audio.play();

  setTimeout(() => {
    (window.URL || webkitURL).revokeObjectURL(audio.src);
  }, 5000);
};

const asrPost = async (eventCodeType = "SESSION_IN", base64) => {
  const wgToken = `C1qziFGlIv3tnCQxcFaStrLuZOO2ZZXjN7FB_G0WlrOLjclfObbSaXAKzl4RWwQBf_0Zhsm0CoVvdVsYMD18iM_LJrxtn7LHJJQuF9UoUuF3fvqOwrG4EF6Z4GahtxtQ2oeaPQBBNKlgVW1xUW7tkhEdXWqzDHPA_I_91Lczk0PI4guhx1c88Hst4-HI8pdMbiUdEJzj3d3a2W06Fa0XA9Q0taAwaRd1k9jUrDVyj9GfS84_SIgJF4SPjWVfsraV79ieb_StgRcUwZjbscGPMlifnJD6F00wwNbxG7AuCHbl3EtMfSed1vuVx3AsizIckwzIVSVRpOGw72cdAMui-I6es9Ozj2ITzSa5KgyXEpX4qCHF1VcCM1wlHLQ_5hLnJIi4r8NsnJPsxMYrTw`;
  const res = await axios.post(
    `https://fls-ai-stg-sit.pingan.com.cn/openapi/ai/voice/asr/v1?channelId=ASP-TEST&sceneId=ASP-TEST&token=${wgToken}`,
    {
      sessionId: "N7FB_G0WlrOLjc",
      eventCodeType,
      voicePkgSeq: ++voicePkgSeq,
      audio: base64,
      format: 16000,
      encoding: "WAV",
    },
    {
      headers: {
        "X-Ai-Asr-Appid": "2b1317fb5b284b308dc90a6fdeae6c4e",
      },
    }
  );

  console.error(res.data.body.asrResultText);

  userTalk.value = userTalk.value + res.data.body.asrResultText;
};

onMounted(() => {
  if (rec) {
    rec.close();
  }

  rec = Recorder({
    type: "unknown",
    onProcess: (buffers, powerLevel, bufferDuration, bufferSampleRate) => {
      RealTimeSendTry(buffers, bufferSampleRate, false);
    },
  });

  const t = setTimeout(() => {
    console.log("无法录音：权限请求被忽略（超时假装手动点击了确认对话框）", 1);
  }, 8000);

  rec.open(
    () => {
      clearTimeout(t);
    },
    (msg, isUserNotAllow) => {
      clearTimeout(t);
      console.log(
        (isUserNotAllow ? "UserNotAllow，" : "") + "无法录音:" + msg,
        1
      );
    }
  );
});

onBeforeUnmount(() => {
  if (rec) rec.close();
});

////////

import PQueue from "p-queue";

const text = ref("");

// 创建一个队列实例，设置并发数为 1
const queue = new PQueue({ concurrency: 1 });
const queue1 = new PQueue({ concurrency: 1 });

// 需要tts的文本
const messageQueue = [];

// lastStrIndex 用于记录上一个字符串的结束位置
let lastStrIndex = 0;

// 将字符串根据标点符号断句分割，并添加到messageQueue中
const splitMessage = async (str) => {
  const punctuation = ["。", "！", "？", "；", "：", "，"];

  for (let i = lastStrIndex; i < str.length; i++) {
    if (punctuation.includes(str[i])) {
      const message = str.slice(lastStrIndex, i + 1);
      console.log(message, "==========");
      await playTTS(message);
      lastStrIndex = i + 1; // 更新上一个字符串的结束位置
    }
  }
};

const playTTS = async (ttsMessage) => {
  //const ttsMessage =
  //  "这个错误通常表示浏览器无法识别音频数据的格式或编码，导致无法加载音频源。为了解决这个问题，你可以尝试使用静态的 WAV 格式音频文件以确保能够正常播放，并且避免直接处理原始的音频数据。以下是一个示例代码来加载外部的 WAV 格式音频文件："; // 替换为你想要播报的文本

  try {
    const wgToken = `C1qziFGlIv3tnCQxcFaStrLuZOO2ZZXjN7FB_G0WlrOLjclfObbSaXAKzl4RWwQBf_0Zhsm0CoVvdVsYMD18iM_LJrxtn7LHJJQuF9UoUuF3fvqOwrG4EF6Z4GahtxtQ2oeaPQBBNKlgVW1xUW7tkhEdXWqzDHPA_I_91Lczk0PI4guhx1c88Hst4-HI8pdMbiUdEJzj3d3a2W06Fa0XA9Q0taAwaRd1k9jUrDVyj9GfS84_SIgJF4SPjWVfsraV79ieb_StgRcUwZjbscGPMlifnJD6F00wwNbxG7AuCHbl3EtMfSed1vuVx3AsizIckwzIVSVRpOGw72cdAMui-I6es9Ozj2ITzSa5KgyXEpX4qCHF1VcCM1wlHLQ_5hLnJIi4r8NsnJPsxMYrTw`;
    const res = await axios.post(
      `https://fls-ai-stg-sit.pingan.com.cn/openapi/ai/voice/tts/v2?channelId=ASP-TEST&sceneId=ASP-TEST&token=${wgToken}`,
      { sessionId: "N7FB_G0WlrOLjc", text: ttsMessage },
      {
        responseType: "arraybuffer",
        headers: {
          "X-Ai-TTS-Appid": "2b1317fb5b284b308dc90a6fdeae6c4e",
        },
      }
    );
    console.log(res.data);

    queue1.add(async () => {
      // 播放获取到的音频
      await playAudio(res.data);
    });
  } catch (error) {
    console.error("Error calling TTS API:", error);
  }
};

// const playAudio = async (audioData) => {
//   const blob = new Blob([audioData], { type: "audio/wav" });
//   const url = URL.createObjectURL(blob);
//   const audio = new Audio(url);
//   audio.onended = () => URL.revokeObjectURL(url);
//   audio.onerror = (error) => console.error("Audio playback error:", error);
//   audio
//     .play()
//     .then(() => console.log("Audio playing"))
//     .catch((error) => console.error("Error playing audio:", error));
// };

const playAudio = (audioData, options = {}) => {
  return new Promise((resolve, reject) => {
    const blob = new Blob([audioData], { type: "audio/wav" });
    const url = URL.createObjectURL(blob);
    const audio = new Audio(url);

    if (options.volume) {
      audio.volume = options.volume; // 设置音量
    }

    audio.onended = () => {
      URL.revokeObjectURL(url);
      resolve();
    };

    audio.onerror = (error) => {
      URL.revokeObjectURL(url);
      reject(error);
    };

    audio
      .play()
      .then(() => console.log("Audio playing"))
      .catch(reject);
  });
};

const chatGpt = async () => {
  const downloadUrl =
    "https://fls-ai.pingan.com.cn/openapi/ai/llm/forward/api/ai/nlp/dialogue?token=mVcexzY_mjtGAL5_exPlmAyfOJxuuEthWY1mk9tUFC_HwceY58uRZ2WDhz7-ttexCdUtFN8C7V636_jIq6fzaSfqIj8OQyhUPKPMa2eZjLlblT77ySqBt_lYM6iEAhrj7-raGmySMmkLS4Rqh651Ak2tqmUbjS64cqv5ofMsuadOCg1J-CtLFt7NeSoU4N3Kpm5MJ_4sOFBhQGfBym88dcwxosFl9LbvhpyleXFf6fOZkkOj0l2X8Nr2pfNjYs3_VOmCQxrxXh1XZ_a1v9qj5_rA9k9wGNNQfmr2JwJTUT4V9NwtNq94gNFt8C0J6MWKVE2eyr25Rke8tkKu3CGNNmspmEFpr6LavPlaWnWOIh9CRJ1cIDB70pg_JD2l0nPTkPbtaTQaIGTz"; // 替换为实际的下载URL

  try {
    const response = await axios.post(
      downloadUrl,
      {
        conversationId: "1976cfe3a5174f9ba768677f789cad7e",
        content: `${userTalk.value} 请输出纯文本的回答，不要使用markdown输出`,
        messageId: Date.now(),
        applicationId: "",
      },
      {
        headers: {
          "Team-Id": "123456",
          Authorization: "9833ae306bde47f8b00b20c18ec809ae",
          "Content-Type": "application/json",
        },
        onDownloadProgress: ({ event }) => {
          //console.log(event);
          const xhr = event.target;
          const { responseText } = xhr;
          //   // Always process the final line
          //   const lastIndex = responseText.lastIndexOf(
          //     "\n",
          //     responseText.length - 2
          //   );
          //   let chunk = responseText;

          //   if (lastIndex !== -1) chunk = responseText.substring(lastIndex);
          //console.log(responseText, "chunk====");
          text.value = responseText;
          queue.add(async () => {
            await splitMessage(responseText);
          });
        },
      }
    );
    console.log(response);
    console.log(messageQueue, "messageQueue=======0000");
  } catch (error) {
    console.error("下载文件时出错:", error);
  }
};
</script>

<style scoped>
.no-select {
  -webkit-user-select: none;
  -moz-user-select: none;
  -ms-user-select: none;
  user-select: none;
}
* {
  -webkit-touch-callout: none;

  -webkit-user-select: none;

  -khtml-user-select: none;

  -moz-user-select: none;

  -ms-user-select: none;

  user-select: none;

  -o-user-select: none;
}
</style>