|
|
|
@ -32,31 +32,20 @@ import androidx.annotation.NonNull; |
|
|
|
import androidx.databinding.DataBindingUtil; |
|
|
|
import androidx.recyclerview.widget.GridLayoutManager; |
|
|
|
|
|
|
|
import com.aispeech.AIError; |
|
|
|
import com.aispeech.AIResult; |
|
|
|
import com.aispeech.common.AIConstant; |
|
|
|
import com.aispeech.common.JSONResultParser; |
|
|
|
import com.aispeech.export.config.AICloudASRConfig; |
|
|
|
import com.aispeech.export.engines2.AICloudASREngine; |
|
|
|
import com.aispeech.export.intent.AICloudASRIntent; |
|
|
|
import com.aispeech.export.listeners.AIASRListener; |
|
|
|
import com.alibaba.android.arouter.facade.annotation.Route; |
|
|
|
import com.alibaba.android.arouter.launcher.ARouter; |
|
|
|
import com.google.gson.Gson; |
|
|
|
import com.iflytek.sparkchain.core.asr.ASR; |
|
|
|
import com.iflytek.sparkchain.core.asr.AsrCallbacks; |
|
|
|
import com.iflytek.sparkchain.core.asr.Segment; |
|
|
|
import com.iflytek.sparkchain.core.asr.Transcription; |
|
|
|
import com.iflytek.sparkchain.core.asr.Vad; |
|
|
|
import com.shockman.sm.vendor.IHttp; |
|
|
|
import com.shockman.sm.vendor.SmUtils; |
|
|
|
import com.shockman.sm.vendor.TargetVo; |
|
|
|
|
|
|
|
import org.greenrobot.eventbus.EventBus; |
|
|
|
import org.java_websocket.drafts.Draft_6455; |
|
|
|
import org.java_websocket.enums.ReadyState; |
|
|
|
|
|
|
|
import java.io.File; |
|
|
|
import java.net.URI; |
|
|
|
import java.text.SimpleDateFormat; |
|
|
|
import java.util.HashMap; |
|
|
|
import java.util.List; |
|
|
|
import java.util.Scanner; |
|
|
|
import java.util.concurrent.Executors; |
|
|
|
import java.util.regex.Matcher; |
|
|
|
import java.util.regex.Pattern; |
|
|
|
|
|
|
|
import qianmu.container.R; |
|
|
|
import qianmu.container.activity.BaseActivity; |
|
|
|
@ -64,24 +53,12 @@ import qianmu.container.adapter.KeyAdapter; |
|
|
|
import qianmu.container.app.Constant; |
|
|
|
import qianmu.container.app.MyApplication; |
|
|
|
import qianmu.container.data.AppData; |
|
|
|
import qianmu.container.data.DeviceData; |
|
|
|
import qianmu.container.data.ScreenSaverData; |
|
|
|
import qianmu.container.databinding.ActivityWebviewBinding; |
|
|
|
import qianmu.container.entity.AppVersion; |
|
|
|
import qianmu.container.entity.KdxfBean; |
|
|
|
import qianmu.container.entity.MessageEvent; |
|
|
|
import qianmu.container.entity.ResourceFileBean; |
|
|
|
import qianmu.container.entity.ScreenSaver; |
|
|
|
import qianmu.container.entity.TTSMessage; |
|
|
|
import qianmu.container.entity.Vocbean; |
|
|
|
import qianmu.container.listener.QueryFinishListener; |
|
|
|
import qianmu.container.socket.SocketClient; |
|
|
|
import qianmu.container.util.AppUtil; |
|
|
|
import qianmu.container.util.AudioPlay; |
|
|
|
import qianmu.container.util.CalendarUtils; |
|
|
|
import qianmu.container.util.DeviceUtil; |
|
|
|
import qianmu.container.util.FileUtil; |
|
|
|
import qianmu.container.util.GsonUtil; |
|
|
|
import qianmu.container.util.KeyboardUtil; |
|
|
|
import qianmu.container.util.LoggerUtil; |
|
|
|
import qianmu.container.util.StringUtil; |
|
|
|
@ -103,13 +80,10 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
//static String HtmlUrl = "http://192.168.1.196:5173/";
|
|
|
|
int time = 0; |
|
|
|
private VideoView currentVideo; |
|
|
|
SocketClient localSocketClient; |
|
|
|
|
|
|
|
// 语音听写对象
|
|
|
|
private ASR mAsr; |
|
|
|
private boolean isrun = false; // 是否正在听写
|
|
|
|
private boolean isdws = false; // 是否可以修正
|
|
|
|
private String language = "zh_cn"; |
|
|
|
private AICloudASREngine mEngine; |
|
|
|
private AICloudASRIntent aiCloudASRIntent; |
|
|
|
private TTSUtil ttsUtil; |
|
|
|
|
|
|
|
Handler handler = new Handler() { |
|
|
|
@ -176,8 +150,6 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
//设置密码
|
|
|
|
initPass(); |
|
|
|
if("sbc".equals(Constant.TTSFac)){ // 思必驰
|
|
|
|
connectLocalSocket(); |
|
|
|
}else{ //kdxf
|
|
|
|
initASR(); |
|
|
|
ttsUtil = new TTSUtil(); |
|
|
|
ttsUtil.initTts(); |
|
|
|
@ -422,7 +394,6 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
@Override |
|
|
|
protected void onDestroy() { |
|
|
|
super.onDestroy(); |
|
|
|
destroyLocalSocketClient(); |
|
|
|
LoggerUtil.e(TAG, "onDestroy()"); |
|
|
|
} |
|
|
|
|
|
|
|
@ -447,7 +418,6 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
}); |
|
|
|
binding.web.addJavascriptInterface(new AndroidtoJs(), "android"); |
|
|
|
} |
|
|
|
int count = 0; |
|
|
|
// JS 调用 android
|
|
|
|
public class AndroidtoJs extends Object { |
|
|
|
// 回到屏保页面
|
|
|
|
@ -484,25 +454,12 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
public void startRecord() { |
|
|
|
LoggerUtil.e(TAG,"JS调用了Android的方法:startRecord()"); |
|
|
|
if("sbc".equals(Constant.TTSFac)) { |
|
|
|
if (localSocketClient == null) initLocalSocketClient(); //-------------------------------
|
|
|
|
localSocketClient.send(StringUtil.strSplice("{\"method\":\"/tts/stop\"}")); |
|
|
|
localSocketClient.send(StringUtil.strSplice("{\"method\":\"/asr/start\"}")); |
|
|
|
}else{ |
|
|
|
if(isrun){ |
|
|
|
return; |
|
|
|
} |
|
|
|
ttsUtil.stopTTs(); |
|
|
|
if(mAsr == null){ |
|
|
|
if(mEngine == null){ |
|
|
|
LoggerUtil.d(TAG, "未初始化"); |
|
|
|
initASR(); |
|
|
|
} |
|
|
|
count++; |
|
|
|
int ret = mAsr.startListener(count+""); |
|
|
|
if(ret != 0){ |
|
|
|
LoggerUtil.d(TAG,"识别开启失败，错误码:"+ret+"\n"); |
|
|
|
}else{ |
|
|
|
isrun = true; |
|
|
|
} |
|
|
|
mEngine.start(aiCloudASRIntent); |
|
|
|
} |
|
|
|
runOnUiThread(new Runnable() { |
|
|
|
@Override |
|
|
|
@ -516,9 +473,6 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
public void stopRecord() { |
|
|
|
LoggerUtil.e(TAG,"JS调用了Android的方法:stopRecord()"); |
|
|
|
if("sbc".equals(Constant.TTSFac)) { |
|
|
|
if (localSocketClient == null) initLocalSocketClient(); //-------------------------------
|
|
|
|
localSocketClient.send(StringUtil.strSplice("{\"method\":\"/asr/stop\"}")); |
|
|
|
}else{ |
|
|
|
stopAsr(); |
|
|
|
} |
|
|
|
} |
|
|
|
@ -527,9 +481,6 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
public void startTTS(String txt) { |
|
|
|
LoggerUtil.e(TAG,"JS调用了Android的方法:startTTS()"); |
|
|
|
if("sbc".equals(Constant.TTSFac)) { |
|
|
|
if (localSocketClient == null) initLocalSocketClient(); //-------------------------------
|
|
|
|
localSocketClient.send(StringUtil.strSplice("{\"method\": \"/tts/start\",\"params\": {\"text\":\"", txt, "\", \"mode\":\"autoPlay\"}}")); |
|
|
|
}else{ |
|
|
|
ttsUtil.startTTs(txt); |
|
|
|
} |
|
|
|
} |
|
|
|
@ -538,9 +489,6 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
public void stopTTS() { |
|
|
|
LoggerUtil.e(TAG,"JS调用了Android的方法:stopTTS()"); |
|
|
|
if("sbc".equals(Constant.TTSFac)){ |
|
|
|
if (localSocketClient == null) initLocalSocketClient(); //-------------------------------
|
|
|
|
localSocketClient.send(StringUtil.strSplice("{\"method\":\"/tts/stop\"}")); |
|
|
|
}else{ |
|
|
|
ttsUtil.stopTTs(); |
|
|
|
} |
|
|
|
runOnUiThread(new Runnable() { |
|
|
|
@ -611,37 +559,26 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
|
|
|
|
//-------- 科大讯飞
|
|
|
|
private void initASR(){ |
|
|
|
if(mAsr == null){ |
|
|
|
mAsr = new ASR(); |
|
|
|
mAsr.registerCallbacks(mAsrCallbacks); |
|
|
|
mAsr.language(language);//语种，zh_cn:中文，en_us:英文。其他语种参见集成文档
|
|
|
|
mAsr.domain("iat");//应用领域,iat:日常用语。其他领域参见集成文档
|
|
|
|
mAsr.accent("mandarin");//方言，mandarin:普通话。方言仅当language为中文时才会生效。其他方言参见集成文档。
|
|
|
|
mAsr.vinfo(true);//返回子句结果对应的起始和结束的端点帧偏移值。
|
|
|
|
if("zh_cn".equals(language)){ |
|
|
|
mAsr.dwa("wpgs");//动态修正
|
|
|
|
} |
|
|
|
if(mEngine == null){ |
|
|
|
AICloudASRConfig config = new AICloudASRConfig(); |
|
|
|
config.setLocalVadEnable(true); |
|
|
|
config.setVadResource("vad_aihome_v0.11.bin"); |
|
|
|
mEngine = AICloudASREngine.createInstance(); |
|
|
|
aiCloudASRIntent = new AICloudASRIntent(); |
|
|
|
aiCloudASRIntent.setRealback(true);//打开实时反馈功能
|
|
|
|
aiCloudASRIntent.setResourceType("aihome"); |
|
|
|
aiCloudASRIntent.setEnableNumberConvert(true);//设置启用识别结果汉字数字转阿拉伯数字功能
|
|
|
|
aiCloudASRIntent.setWaitingTimeout(5000);//设置等待识别结果超时时长，默认5000ms
|
|
|
|
aiCloudASRIntent.setNoSpeechTimeOut(2000); |
|
|
|
aiCloudASRIntent.setPauseTime(1000); |
|
|
|
mEngine.init(config, new AIASRListenerImpl()); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
AsrCallbacks mAsrCallbacks = new AsrCallbacks() { |
|
|
|
@Override |
|
|
|
public void onResult(ASR.ASRResult asrResult, Object o) { |
|
|
|
//以下信息需要开发者根据自身需求，如无必要，可不需要解析执行。
|
|
|
|
int status = asrResult.getStatus(); //结果数据状态，0：识别的第一块结果,1：识别中间结果,2：识别最后一块结果
|
|
|
|
String result = asrResult.getBestMatchText(); //识别结果
|
|
|
|
if(status == 0){ |
|
|
|
voiceContent(result); |
|
|
|
}else if(status == 2){ |
|
|
|
voiceContent(result); |
|
|
|
stopAsr(); |
|
|
|
}else{ |
|
|
|
voiceContent(result); |
|
|
|
} |
|
|
|
} |
|
|
|
private class AIASRListenerImpl implements AIASRListener { |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onError(ASR.ASRError asrError, Object o) { |
|
|
|
public void onError(AIError error) { |
|
|
|
LoggerUtil.e("ASRInit: ",error.getMessage()); |
|
|
|
// 在非主线程中需要调用 WebView 方法时
|
|
|
|
runOnUiThread(new Runnable() { |
|
|
|
@Override |
|
|
|
@ -652,28 +589,82 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
stopAsr(); |
|
|
|
} |
|
|
|
|
|
|
|
public void onResults(AIResult results) { |
|
|
|
if (results.getResultType() == AIConstant.AIENGINE_MESSAGE_TYPE_JSON) { |
|
|
|
JSONResultParser parser = new JSONResultParser((String) results.getResultObject()); |
|
|
|
if("".equals(parser.getAllText())) { |
|
|
|
voiceContent(parser.getVar()); |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onInit(int status) { |
|
|
|
if (status == AIConstant.OPT_SUCCESS) { |
|
|
|
LoggerUtil.e("ASRInit: ","初始化成功!"); |
|
|
|
} else { |
|
|
|
LoggerUtil.e("ASRInit: ","初始化失败!code:" + status); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onBeginOfSpeech() { |
|
|
|
public void onBeginningOfSpeech() { |
|
|
|
//本地vad打开时，才会执行
|
|
|
|
LoggerUtil.e("ASRInit: ","onBeginningOfSpeech"); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onNotOneShot() { |
|
|
|
//do nothing
|
|
|
|
LoggerUtil.e("ASRInit: ","onNotOneShot"); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onEndOfSpeech() { |
|
|
|
Log.d(TAG, "结束说话"); |
|
|
|
//本地vad打开时，才会执行
|
|
|
|
runOnUiThread(new Runnable() { |
|
|
|
@Override |
|
|
|
public void run() { |
|
|
|
binding.web.evaluateJavascript("javascript: window.youAskOver('');", null); |
|
|
|
} |
|
|
|
}); |
|
|
|
LoggerUtil.e("ASRInit: ","onEndOfSpeech"); |
|
|
|
} |
|
|
|
}; |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onReadyForSpeech() { |
|
|
|
LoggerUtil.e("ASRInit: ","onReadyForSpeech"); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onResultDataReceived(byte[] buffer, int size, int wakeupType) { |
|
|
|
LoggerUtil.e("ASRInit: ","onReadyForSpeech "+String.valueOf(size)); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onRmsChanged(float rmsdB) { |
|
|
|
Log.d(TAG, "" + rmsdB); |
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onResultDataReceived(byte[] buffer, int size) { |
|
|
|
//do nothing
|
|
|
|
} |
|
|
|
|
|
|
|
@Override |
|
|
|
public void onRawDataReceived(byte[] buffer, int size) { |
|
|
|
//do nothing
|
|
|
|
LoggerUtil.e("ASRInit: ","onRawDataReceived "+String.valueOf(size)); |
|
|
|
} |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
/** |
|
|
|
* android调用js：传送识别语音内容 |
|
|
|
*/ |
|
|
|
public void voiceContent(String msg) { |
|
|
|
if (binding.web != null) { |
|
|
|
LoggerUtil.e(TAG, "android调用js方法:voiceContent(),语音内容：" + msg); |
|
|
|
LoggerUtil.e(TAG, "android调用js方法:giveAskText(),语音内容：" + msg); |
|
|
|
runOnUiThread(new Runnable() { |
|
|
|
@Override |
|
|
|
public void run() { |
|
|
|
@ -683,110 +674,9 @@ public class WebViewActivity extends BaseActivity { |
|
|
|
} |
|
|
|
} |
|
|
|
private void stopAsr(){ |
|
|
|
if(isrun){ |
|
|
|
if(mAsr!=null){ |
|
|
|
mAsr.stopListener(false); |
|
|
|
} |
|
|
|
isrun = false; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
// 思必驰----------------
|
|
|
|
private void initLocalSocketClient() { |
|
|
|
try { |
|
|
|
URI localUri = new URI("ws://127.0.0.1:50002"); |
|
|
|
LoggerUtil.e("TTSSocketUri", localUri.toString()); |
|
|
|
localSocketClient = new SocketClient(localUri, new Draft_6455()); |
|
|
|
setLocalSocketListener(); |
|
|
|
} catch (Throwable t) { |
|
|
|
LoggerUtil.e("initSocketClient", StringUtil.getThrowableStr(t)); |
|
|
|
} |
|
|
|
} |
|
|
|
public void connectLocalSocket() { |
|
|
|
try { |
|
|
|
//客户端不存在时 创建客户端设置监听事件
|
|
|
|
if (localSocketClient == null) initLocalSocketClient(); |
|
|
|
//获取客户端当前的连接状态
|
|
|
|
ReadyState state = localSocketClient.getReadyState(); |
|
|
|
LoggerUtil.e("connectLocalSocket", "TTS客户端连接状态："+ GsonUtil.getGson().toJson(state)); |
|
|
|
boolean open = localSocketClient.isOpen(); |
|
|
|
LoggerUtil.e("connectLocalSocket", "TTS客户端连接状态：open="+open); |
|
|
|
if (open) return; |
|
|
|
//获取客户端当前的连接状态
|
|
|
|
if (localSocketClient.isOpen()) return; |
|
|
|
//未连接状态时 连接服务器
|
|
|
|
if (state.equals(ReadyState.NOT_YET_CONNECTED)) localSocketClient.connect(); |
|
|
|
else if (state.equals(ReadyState.CLOSING) || state.equals(ReadyState.CLOSED)) { |
|
|
|
//正在关闭或者关闭状态时 重新连接服务器
|
|
|
|
localSocketClient.reconnect(); |
|
|
|
} |
|
|
|
} catch (Throwable t) { |
|
|
|
LoggerUtil.e("connectLocalSocket", StringUtil.getThrowableStr(t)); |
|
|
|
destroyLocalSocketClient(); |
|
|
|
if(mEngine!=null){ |
|
|
|
mEngine.cancel(); |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
//语音交互
|
|
|
|
private void setLocalSocketListener() { |
|
|
|
localSocketClient.setOnOpenListener((handshakeData) -> { |
|
|
|
LoggerUtil.e("connectLocalSocket", "TTS客户端连接成功"); |
|
|
|
localSocketClient.send(StringUtil.strSplice("{\"method\":\"/asr/stop\"}")); |
|
|
|
}); |
|
|
|
|
|
|
|
localSocketClient.setOnMessageListener((conn, message) -> { |
|
|
|
try { |
|
|
|
TTSMessage messageBean = new Gson().fromJson(message, TTSMessage.class); |
|
|
|
TTSMessage.Params content = messageBean.getParams(); |
|
|
|
if (content == null) return; |
|
|
|
String method = messageBean.getMethod(); |
|
|
|
if("asr.text".equals(method)){ |
|
|
|
LoggerUtil.e("ttsSocket:",content.getText()); |
|
|
|
runOnUiThread(new Runnable() { |
|
|
|
@Override |
|
|
|
public void run() { |
|
|
|
binding.web.loadUrl(StringUtil.strSplice("javascript:window.giveAskText('", content.getText(), "');")); |
|
|
|
} |
|
|
|
}); |
|
|
|
}else if("asr.result".equals(method)){ |
|
|
|
LoggerUtil.e("ttsSocket:",content.getText()); |
|
|
|
runOnUiThread(new Runnable() { |
|
|
|
@Override |
|
|
|
public void run() { |
|
|
|
localSocketClient.send(StringUtil.strSplice("{\"method\":\"/asr/stop\"}")); |
|
|
|
binding.web.loadUrl(StringUtil.strSplice("javascript:window.youAskOver('", content.getText(), "');")); |
|
|
|
} |
|
|
|
}); |
|
|
|
}else if("tts.result".equals(method)){ |
|
|
|
LoggerUtil.e("ttsSocket:", messageBean.getParams().getStatus().trim()); |
|
|
|
if("end".equals(messageBean.getParams().getStatus().trim())) { |
|
|
|
runOnUiThread(new Runnable() { |
|
|
|
@Override |
|
|
|
public void run() { |
|
|
|
changeVideo("bg"); |
|
|
|
} |
|
|
|
}); |
|
|
|
} |
|
|
|
} |
|
|
|
} catch (Throwable t) { |
|
|
|
LoggerUtil.e("setOnMessageListener", StringUtil.getThrowableStr(t)); |
|
|
|
} |
|
|
|
}); |
|
|
|
|
|
|
|
localSocketClient.setOnCloseListener((code, reason, remote) -> { |
|
|
|
LoggerUtil.e("LocSocCliManager", "onClose:code="+code); |
|
|
|
if (localSocketClient == null){ |
|
|
|
initLocalSocketClient(); |
|
|
|
}else { |
|
|
|
localSocketClient.reconnect(); |
|
|
|
} |
|
|
|
|
|
|
|
}); |
|
|
|
localSocketClient.setOnErrorListener((ex) -> LoggerUtil.e("LocSocCliManager", "onError")); |
|
|
|
} |
|
|
|
//销毁当前的客户端
|
|
|
|
public void destroyLocalSocketClient() { |
|
|
|
if (localSocketClient == null) return; |
|
|
|
localSocketClient.close(); |
|
|
|
localSocketClient = null; |
|
|
|
} |
|
|
|
} |
|
|
|
|
