|
|
@@ -0,0 +1,318 @@
|
|
|
+import AvatarllmService from '../../util/ant-avatar-tinyapp-llm-service.min.js';
|
|
|
+
|
|
|
+Page({
|
|
|
+ data: {
|
|
|
+ // rtc参数
|
|
|
+ elementId:'myRtcroom',
|
|
|
+ "webrtcInfo": {
|
|
|
+ "roomId": "",
|
|
|
+ "rtoken": "",
|
|
|
+ "uid": "",
|
|
|
+ "signature": ""
|
|
|
+ },
|
|
|
+
|
|
|
+ // 数字人参数
|
|
|
+ avartarParam:{
|
|
|
+ // tenantCode:"NpZ_5pv1", // 请联系蚂蚁数字人团队获取
|
|
|
+ // interactCode:"8f94744e717347bbaa0e4a7cc0389e68", // 请联系蚂蚁数字人团队获取
|
|
|
+ tenantCode:"hanghui", // 请联系蚂蚁数字人团队获取
|
|
|
+ interactCode:"177d745998de4c1f90159a5a3de8e1d8", // 请联系蚂蚁数字人团队获取
|
|
|
+ interactiveConfig:{},
|
|
|
+ streamId:'',
|
|
|
+ },
|
|
|
+ llmService:null,
|
|
|
+ chatList: [],
|
|
|
+ scrollTop: 0
|
|
|
+ },
|
|
|
+
|
|
|
+ onLoad() {
|
|
|
+ // 初始化数字人服务
|
|
|
+ this.initLLMService();
|
|
|
+ },
|
|
|
+
|
|
|
+ onUnload(){
|
|
|
+ if (this.data.avartarParam.streamId) {
|
|
|
+ this.stopLive();
|
|
|
+ }
|
|
|
+ },
|
|
|
+
|
|
|
+ /*************************** Avatar ******************************/
|
|
|
+ initLLMService() {
|
|
|
+ // 新建 llmService
|
|
|
+ this.data.llmService = new AvatarllmService(
|
|
|
+
|
|
|
+ this.data.avartarParam.tenantCode,
|
|
|
+ this.data.avartarParam.interactCode,
|
|
|
+ this.data.elementId,
|
|
|
+ this,
|
|
|
+ null,
|
|
|
+ 'cn',
|
|
|
+ );
|
|
|
+
|
|
|
+ // 获取交互配置
|
|
|
+ this.data.llmService.init((result) => {
|
|
|
+ console.log(result)
|
|
|
+ if (result && result.interactiveConfigResult) {
|
|
|
+ const interactiveConfig = result.interactiveConfigResult
|
|
|
+ this.data.avartarParam.interactiveConfig = interactiveConfig
|
|
|
+
|
|
|
+ // 开播
|
|
|
+ this.startLive();
|
|
|
+ }
|
|
|
+ });
|
|
|
+
|
|
|
+ // ----------------------- Event ----------------------------
|
|
|
+ this.data.llmService.onInitSuccess = (streamId) => {
|
|
|
+ console.log('【Lynn】init success, avatar ready, streamId【流服务初始化成功的回调，此时已拉流成功。建议此回调之后展示数字人对话的播放器】: ', streamId);
|
|
|
+ }
|
|
|
+
|
|
|
+ this.data.llmService.onASRTextFinish = (text) => {
|
|
|
+ // 停止录音
|
|
|
+ if (!text) return;
|
|
|
+ let lastQ = ( this.data.chatList || []).at(-1)
|
|
|
+ if (lastQ && lastQ.type == 1 && !lastQ.isEnd) {
|
|
|
+ let newArr = this.data.chatList
|
|
|
+ let lastIndex = this.data.chatList.length - 1
|
|
|
+ newArr[lastIndex] = {
|
|
|
+ ...newArr[lastIndex],
|
|
|
+ text: text,
|
|
|
+ isEnd: true
|
|
|
+ }
|
|
|
+ this.updateChat(newArr)
|
|
|
+ }
|
|
|
+ console.log('【Lynn】onASRTextFinish【语音识别完成，识别到的最终文本】: ', text);
|
|
|
+
|
|
|
+ this.stopAsr();
|
|
|
+ // this.interrupt()
|
|
|
+ this.sendText(text);
|
|
|
+ };
|
|
|
+
|
|
|
+ this.data.llmService.onASRTextChange = (text) => {
|
|
|
+ let lastQ =( this.data.chatList || []).at(-1) || {}
|
|
|
+ if (lastQ && lastQ.type == 1 && !lastQ.isEnd) {
|
|
|
+ let newArr = this.data.chatList
|
|
|
+ let lastIndex = this.data.chatList.length - 1
|
|
|
+ newArr[lastIndex] = {
|
|
|
+ ...newArr[lastIndex],
|
|
|
+ text
|
|
|
+ }
|
|
|
+ this.updateChat(newArr)
|
|
|
+ } else {
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 1,
|
|
|
+ text,
|
|
|
+ isEnd: false
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ }
|
|
|
+ console.log('【Lynn】onASRTextChange【语音识别过程中，识别到的中间文本】: ', text);
|
|
|
+ };
|
|
|
+
|
|
|
+ this.data.llmService.onRecordingStart = () => {
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 0,
|
|
|
+ text: '....开始录音的回调，麦克风已开启成功....',
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ console.log('【Lynn】onRecordingStart【开始录音的回调，麦克风已开启成功】');
|
|
|
+ };
|
|
|
+
|
|
|
+ this.data.llmService.onRecordingStop = (
|
|
|
+
|
|
|
+ ) => {
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 0,
|
|
|
+ text: '....停止录音的回调，麦克风已关闭....',
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ console.log('【Lynn】onRecordingStop【停止录音的回调，麦克风已关闭】');
|
|
|
+ };
|
|
|
+
|
|
|
+ // 收到数字人对话的结果
|
|
|
+ this.data.llmService.onAvatarMessage = (result) => {
|
|
|
+ console.log('【onAvatarMessage】',result)
|
|
|
+ const text = result.text;
|
|
|
+ let lastA =( this.data.chatList || []).at(-1)
|
|
|
+ if (lastA && lastA.type == 2 && !lastA.isEnd) {
|
|
|
+ let newArr = this.data.chatList
|
|
|
+ let lastIndex = this.data.chatList.length - 1
|
|
|
+ newArr[lastIndex] = {
|
|
|
+ ...newArr[lastIndex],
|
|
|
+ text: (newArr[lastIndex].text || '') + text
|
|
|
+ }
|
|
|
+ this.updateChat(newArr)
|
|
|
+ } else {
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 2,
|
|
|
+ text,
|
|
|
+ isEnd: false
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ }
|
|
|
+ let output = text;
|
|
|
+ console.log('【数字人返回的对话结果，分段返回，会多次调用。--------------开始--------------】：', text)
|
|
|
+ if (text.includes('$$')) {
|
|
|
+ output = text.replace('$$', '');
|
|
|
+ }
|
|
|
+ console.log('【Lynn】output:', output);
|
|
|
+ console.log('【数字人返回的对话结果，分段返回，会多次调用。--------------结束--------------】')
|
|
|
+ };
|
|
|
+
|
|
|
+ this.data.llmService.onAvatarPlayBegin = () => {
|
|
|
+ console.log('【Lynn】onAvatarPlayBegin 【数字⼈播报开始】');
|
|
|
+ };
|
|
|
+
|
|
|
+ this.data.llmService.onAvatarPlayEnd = () => {
|
|
|
+ let newArr = this.data.chatList
|
|
|
+ let lastIndex = this.data.chatList.length - 1
|
|
|
+ newArr[lastIndex] = {
|
|
|
+ ...newArr[lastIndex],
|
|
|
+ isEnd: true
|
|
|
+ }
|
|
|
+ this.updateChat(newArr)
|
|
|
+ console.log('【Lynn】onAvatarPlayEnd 【数字⼈播报结束】');
|
|
|
+ };
|
|
|
+
|
|
|
+ // 设置静默超时的时间，单位是 秒
|
|
|
+ // this.data.llmService.overtimeInterval = 30
|
|
|
+ this.data.llmService.onOverTimeTrigger = (streamId) => {
|
|
|
+ console.log('【Lynn】onStopLiveSuccess 【静默超时的回调】');
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 0,
|
|
|
+ text: '....静默超时....',
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ this.stopLive()
|
|
|
+ };
|
|
|
+ },
|
|
|
+
|
|
|
+ // 开启流服务
|
|
|
+ startLive(){
|
|
|
+ this.data.llmService.startLive((res)=>{
|
|
|
+ if(res){
|
|
|
+ // 保存下streamID
|
|
|
+ if(res.streamId){
|
|
|
+ this.data.avartarParam.streamId = res.streamId
|
|
|
+ }
|
|
|
+
|
|
|
+ // 设置下webrtcInfo参数
|
|
|
+ if (res.webrtcInfo) {
|
|
|
+ this.setData({
|
|
|
+ webrtcInfo: res.webrtcInfo,
|
|
|
+ });
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 0,
|
|
|
+ text: '....开启流服务....',
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ console.log("【Lynn】【开启流服务】", this.data.webrtcInfo)
|
|
|
+ }
|
|
|
+ }
|
|
|
+ })
|
|
|
+ },
|
|
|
+
|
|
|
+ // 开始录音
|
|
|
+ startAsr(){
|
|
|
+ this.interrupt()
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 0,
|
|
|
+ text: '....开始录音....',
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ this.data.llmService.startAsr();
|
|
|
+ },
|
|
|
+
|
|
|
+ // 停止录音
|
|
|
+ stopAsr(){
|
|
|
+ console.log('【停止录音。。。。。。】')
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 0,
|
|
|
+ text: '....停止录音....',
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ this.data.llmService.stopAsr();
|
|
|
+ },
|
|
|
+
|
|
|
+ // 发送消息
|
|
|
+ sendText(text){
|
|
|
+ this.data.llmService.sendText(text, this.data.avartarParam.streamId);
|
|
|
+ },
|
|
|
+
|
|
|
+ // 打断消息
|
|
|
+ interrupt() {
|
|
|
+ if (this.data.llmService) {
|
|
|
+ console.log('【打断消息。。。。。】')
|
|
|
+ this.setData({
|
|
|
+ chatList: this.data.chatList.concat([{
|
|
|
+ type: 0,
|
|
|
+ text: '....打断消息....',
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ })
|
|
|
+ this.data.llmService.interruptBoardcast()
|
|
|
+ }
|
|
|
+ },
|
|
|
+
|
|
|
+ // 关闭流服务
|
|
|
+ stopLive(){
|
|
|
+ this.data.llmService.stopLive(this.data.avartarParam.streamId).then((res) => {
|
|
|
+
|
|
|
+ });
|
|
|
+ },
|
|
|
+
|
|
|
+ query1(){
|
|
|
+ this.interrupt()
|
|
|
+ let text = '今天天气怎么样'
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 1,
|
|
|
+ text,
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ this.sendText(text);
|
|
|
+ },
|
|
|
+
|
|
|
+ query2(){
|
|
|
+ this.interrupt()
|
|
|
+ let text = '你是谁'
|
|
|
+ let list = this.data.chatList.concat([{
|
|
|
+ type: 1,
|
|
|
+ text,
|
|
|
+ isEnd: true
|
|
|
+ }])
|
|
|
+ this.updateChat(list)
|
|
|
+ this.sendText(text);
|
|
|
+ },
|
|
|
+ onClear() {
|
|
|
+ this.setData({
|
|
|
+ chatList: []
|
|
|
+ })
|
|
|
+ },
|
|
|
+ updateChat(list){
|
|
|
+ this.setData({
|
|
|
+ chatList: list
|
|
|
+ })
|
|
|
+ console.log('updateChat', list)
|
|
|
+ const _this = this
|
|
|
+ // 获取 scroll-view 的节点
|
|
|
+ const selectorQuery = my.createSelectorQuery();
|
|
|
+ // 滚动到最后一条消息
|
|
|
+ selectorQuery
|
|
|
+ .select(`#message-${_this.data.chatList.length - 1}`)
|
|
|
+ .boundingClientRect()
|
|
|
+ .exec(messageRect => {
|
|
|
+ console.log(messageRect)
|
|
|
+ if(messageRect[0]){
|
|
|
+ _this.setData({
|
|
|
+ scrollTop: (messageRect[0].top || 0) + (messageRect[0].bottom || 0) + (_this.data.chatList.length * 30) // 假设容器足够高，设置一个大的 scrollTop 值
|
|
|
+ });
|
|
|
+ }
|
|
|
+ })
|
|
|
+ }
|
|
|
+ });
|
