diff --git a/app/src/main/java/qianmu/container/activity/H5/WebViewActivity.java b/app/src/main/java/qianmu/container/activity/H5/WebViewActivity.java index 381507c..87d872f 100644 --- a/app/src/main/java/qianmu/container/activity/H5/WebViewActivity.java +++ b/app/src/main/java/qianmu/container/activity/H5/WebViewActivity.java @@ -39,9 +39,12 @@ import com.aispeech.AIResult; import com.aispeech.common.AIConstant; import com.aispeech.common.JSONResultParser; import com.aispeech.export.config.AICloudASRConfig; +import com.aispeech.export.config.AILocalSignalAndWakeupConfig; import com.aispeech.export.engines2.AICloudASREngine; +import com.aispeech.export.engines2.AILocalSignalAndWakeupEngine; import com.aispeech.export.intent.AICloudASRIntent; import com.aispeech.export.listeners.AIASRListener; +import com.aispeech.export.listeners.AILocalSignalAndWakeupListener; import com.alibaba.android.arouter.facade.annotation.Route; import com.alibaba.android.arouter.launcher.ARouter; @@ -61,6 +64,8 @@ import qianmu.container.data.AppData; import qianmu.container.data.ScreenSaverData; import qianmu.container.databinding.ActivityWebviewBinding; import qianmu.container.entity.MessageEvent; +import qianmu.container.recorder.AudioRecorder; +import qianmu.container.recorder.IRecord; import qianmu.container.socket.SocketClient; import qianmu.container.util.CalendarUtils; import qianmu.container.util.DeviceUtil; @@ -88,6 +93,7 @@ public class WebViewActivity extends BaseActivity { private SoundPool soundPool; private int soundId; // 语音听写对象 + private AILocalSignalAndWakeupEngine mLocalEngine; private AICloudASREngine mEngine; private AICloudASRIntent aiCloudASRIntent; private TTSUtil ttsUtil; @@ -927,10 +933,16 @@ public class WebViewActivity extends BaseActivity { //-------- private void initASR(){ + + AILocalSignalAndWakeupConfig config = new AILocalSignalAndWakeupConfig(); + config.setSspeResource("sspe_aec-bf-bss-wkp_ch10-mic8-ref2_outgain4_v2.0.0.165_20251127_v1.bin");//设置SSPE资源 + mLocalEngine = AILocalSignalAndWakeupEngine.createInstance(); + mLocalEngine.init(config, new SSPEListener()); + if(mEngine == null){ - AICloudASRConfig config = new AICloudASRConfig(); - config.setLocalVadEnable(true); - config.setVadResource("vad_aihome_v0.12c_noDither.bin"); + AICloudASRConfig asrconfig = new AICloudASRConfig(); + asrconfig.setLocalVadEnable(true); + asrconfig.setVadResource("vad_aihome_v0.12c_noDither.bin"); mEngine = AICloudASREngine.createInstance(); aiCloudASRIntent = new AICloudASRIntent(); aiCloudASRIntent.setEnablePunctuation(true); @@ -939,10 +951,134 @@ public class WebViewActivity extends BaseActivity { aiCloudASRIntent.setResourceType(Constant.asrModel); aiCloudASRIntent.setEnableNumberConvert(true);//设置启用识别结果汉字数字转阿拉伯数字功能 aiCloudASRIntent.setWaitingTimeout(30000); //设置等待识别结果超时时长，默认5000ms - aiCloudASRIntent.setNoSpeechTimeOut(10000); + aiCloudASRIntent.setNoSpeechTimeOut(6000); aiCloudASRIntent.setPauseTime(2000); //aiCloudASRIntent.setServer("wss://asr.dui.ai/runtime/v2/recognize"); - mEngine.init(config, new AIASRListenerImpl()); + mEngine.init(asrconfig, new AIASRListenerImpl()); + } + + initRecorder(); + } + + private void initRecorder() { + AudioRecorder.getInstance().registerListener(new RecorderListener()); + IRecord.RecordConfig config = new IRecord.RecordConfig(); + config.recorderType = IRecord.TYPE_JNI; + config.audioChannel = 10; +// config.filter = new int[]{0, 1, 2, 3, 6, 7}; + config.dump = true; // 不保存外部录音机收到的音频 + AudioRecorder.getInstance().init(getApplicationContext(), config); + } + + public class RecorderListener implements AudioRecorder.RecorderListener { + + @Override + public void onAudioBuffer(byte[] data, int size) { + feedEngine(data, size); + } + } + + protected void feedEngine(byte[] data, int length) { + if (mLocalEngine != null) { + mLocalEngine.feedData(data, length); + } + } + + private class SSPEListener implements AILocalSignalAndWakeupListener { + @Override + public void onInit(int resultCode) { + if (resultCode == AIConstant.OPT_SUCCESS) { + LoggerUtil.e("setSspeResource:","sspe init success"); + } else { + LoggerUtil.e("setSspeResource:","sspe init fail: " + resultCode); + } + } + + @Override + public void onError(AIError aiError) { + + } + + @Override + public void onWakeup(double v, String s) { + + } + + @Override + public void onWakeup(String s) { + + } + + @Override + public void onNearInformation(String s) { + + } + + @Override + public void onDoaResult(int i) { + + } + + @Override + public void onReadyForSpeech() { + + } + + @Override + public void onRawDataReceived(byte[] bytes, int i) { + + } + + @Override + public void onResultDataReceived(byte[] bytes, int i, int i1) { + if (mEngine != null) { + mEngine.feedData(bytes, i); + } + } + + @Override + public void onVprintCutDataReceived(int i, byte[] bytes, int i1) { + + } + + @Override + public void onAgcDataReceived(byte[] bytes, int i) { + + } + + @Override + public void onInputDataReceived(byte[] bytes, int i) { + + } + + @Override + public void onOutputDataReceived(byte[] bytes, int i) { + + } + + @Override + public void onEchoDataReceived(byte[] bytes, int i) { + + } + + @Override + public void onSevcDoaResult(int i) { + + } + + @Override + public void onSevcNoiseResult(String s) { + + } + + @Override + public void onMultibfDataReceived(byte[] bytes, int i, int i1) { + + } + + @Override + public void onEchoVoipDataReceived(byte[] bytes, int i) { + } } diff --git a/app/src/main/java/qianmu/container/recorder/AudioRecorder.java b/app/src/main/java/qianmu/container/recorder/AudioRecorder.java new file mode 100644 index 0000000..b81057d --- /dev/null +++ b/app/src/main/java/qianmu/container/recorder/AudioRecorder.java @@ -0,0 +1,225 @@ +package qianmu.container.recorder; + + +import static qianmu.container.recorder.IRecord.TYPE_HOTWORD; +import static qianmu.container.recorder.IRecord.TYPE_JNI; +import static qianmu.container.recorder.IRecord.TYPE_NATIVE; +import android.content.Context; +import android.util.Log; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * Created by Android Studio. + * User: gaozhilong + * Date: 2025/12/19 + * Time: 11:23 + */ +public class AudioRecorder { + + private static final String TAG = "AudioRecorder"; + + private State mCurrentState = State.NO_READY; + private IRecord.RecordConfig mConfig; + private List mListeners = new ArrayList<>(); + private IRecord mRecorder; + private FileOutputStream mDumpFileStream; + private Context mContext; + + private AudioRecorder() { + } + + private static class AudioRecorderHolder { + private static final AudioRecorder INSTANCE = new AudioRecorder(); + } + + public static AudioRecorder getInstance() { + return AudioRecorderHolder.INSTANCE; + } + + public synchronized void init(Context context, IRecord.RecordConfig cfg) { + Log.i(TAG, "record cfg=" + cfg.toString()); + mConfig = cfg; + mContext = context; + if (mConfig.recorderType == TYPE_NATIVE) { + mRecorder = new NativeRecorder(); + } else if (mConfig.recorderType == TYPE_JNI) { + mRecorder = new JniRecorder(); + } else if (mConfig.recorderType == TYPE_HOTWORD) { + + } + mRecorder.init(context, mConfig); + mCurrentState = State.READY; + openDump(); + } + + public int getAudioRecordSessionId() { + if (mRecorder instanceof NativeRecorder) + return ((NativeRecorder) mRecorder).getAudioRecordSessionId(); + return 0; + } + + public synchronized void startRecord() { + Log.i(TAG, "startRecord"); + if (mCurrentState == State.NO_READY) { + Log.e(TAG, "record not initialized"); + return; + } + if (mCurrentState == State.START) { + Log.w(TAG, "already recording...ignore"); + return; + } + Log.d(TAG, "startRecord: ============startRecord begin============"); + mRecorder.start(); + new Thread(this::recordRead).start(); + Log.d(TAG, "startRecord: ============startRecord end============"); + } + + public synchronized void stopRecord() { + Log.d(TAG, "stopRecord: ============stopRecord begin============"); + if (mCurrentState == State.NO_READY || mCurrentState == State.READY) { + Log.e(TAG, "record not started yet"); + } else { + mRecorder.stop(); + mCurrentState = State.STOP; + } + Log.d(TAG, "stopRecord: ============stopRecord end============"); + } + + public void release() { + Log.d(TAG, "release: ============release begin============"); + if (mRecorder != null) { + mRecorder.release(); + } + mCurrentState = State.NO_READY; + Log.d(TAG, "release: ============release end============"); + } + + private void recordRead() { + byte[] audioData = new byte[mRecorder.getBufferSize()]; + mCurrentState = State.START; + while (mCurrentState == State.START) { + int size = mRecorder.read(audioData, 0, mRecorder.getBufferSize()); + + byte[] publishBuffer; + if (mConfig.filter != null) { + publishBuffer = new byte[audioData.length / mConfig.audioChannel * mConfig.filter.length]; + splitAudio(audioData, mConfig.audioChannel, publishBuffer, mConfig.filter); + } else { + if (size > 0) { + publishBuffer = Arrays.copyOfRange(audioData, 0, size); + } else { + Log.w(TAG, "read size <= 0"); + publishBuffer = new byte[0]; + } + } + + if (size > 0) { + if (mListeners != null) { + for (RecorderListener listener : mListeners) { + listener.onAudioBuffer(publishBuffer, publishBuffer.length); + } + } + + dump(publishBuffer); + } + } + closeDump(); + } + + public void registerListener(RecorderListener cb) { + if (cb != null && !mListeners.contains(cb)) { + Log.i(TAG, "registerListener " + cb.toString()); + mListeners.add(cb); + } + } + + private void splitAudio(byte[] src, int channelNum, byte[] raw, int[] mics) { + int frameSize = src.length / channelNum / 2; + int rawCh = mics.length; + + for (int i = 0; i < frameSize; i++) { + int rawNum = 0; + for (int j = 0; j < channelNum; j++) { + if (j != mics[rawNum]) { + continue; + } + + int rawIdx = (i * rawCh + rawNum) * 2; + int srcIdx = (i * channelNum + j) * 2; + raw[rawIdx] = src[srcIdx]; + raw[rawIdx + 1] = src[srcIdx + 1]; + + rawNum++; + if (rawNum == rawCh) { + break; + } + } + } + } + + public void unregisterListener(RecorderListener cb) { + if (cb != null && mListeners.contains(cb)) { + Log.i(TAG, "unregisterListener " + cb.toString()); + mListeners.remove(cb); + } + if (mListeners.isEmpty()) { + stopRecord(); + } + } + + private void openDump() { + if (mConfig.dump) return; + + try { + String dumpPath = mContext.getExternalFilesDir("record") + File.separator + System.currentTimeMillis() + ".pcm"; + File file = new File(dumpPath); + if (!Objects.requireNonNull(file.getParentFile()).exists()) { + if (file.getParentFile().mkdirs()) { + Log.i(TAG, "successfully created " + file.getParentFile().getPath()); + } + } + mDumpFileStream = new FileOutputStream(file); + } catch (IllegalStateException | FileNotFoundException e) { + Log.e(TAG, "create audio data file error: " + e.getMessage()); + } + } + + private void dump(byte[] data) { + if (mDumpFileStream == null) return; + try { + mDumpFileStream.write(data); + } catch (IOException e) { + Log.e(TAG, "write audio data error: " + e.getMessage()); + } + } + + private void closeDump() { + if (mDumpFileStream == null) return; + try { + mDumpFileStream.flush(); + mDumpFileStream.close(); + } catch (IOException e) { + Log.e(TAG, "close audio data file error: " + e.getMessage()); + } + } + + public interface RecorderListener { + void onAudioBuffer(byte[] data, int volume); + } + + public enum State { + NO_READY, + READY, + START, + STOP + } +} + diff --git a/app/src/main/java/qianmu/container/recorder/IRecord.java b/app/src/main/java/qianmu/container/recorder/IRecord.java new file mode 100644 index 0000000..81525c0 --- /dev/null +++ b/app/src/main/java/qianmu/container/recorder/IRecord.java @@ -0,0 +1,44 @@ +package qianmu.container.recorder; + +import android.content.Context; +import android.media.AudioFormat; +import android.media.MediaRecorder; +/** + * Created by Android Studio. + * User: gaozhilong + * Date: 2025/12/19 + * Time: 11:24 + */ +public interface IRecord { + + int TYPE_NATIVE = 0; + int TYPE_JNI = 1; + int TYPE_HOTWORD = 2; + + void init(Context context, RecordConfig config); + + void start(); + + int read(byte[] buffer, int offset, int length); + + void stop(); + + void release(); + + int getBufferSize(); + + class RecordConfig { + public int recorderType = 0; // 0: 普通录音, 1: JNI录音机 + public int audioSource = MediaRecorder.AudioSource.VOICE_RECOGNITION; + public int sampleRate = 16000; + public int audioChannel = AudioFormat.CHANNEL_IN_MONO; + public int audioEncoding = AudioFormat.ENCODING_PCM_16BIT; + public int captureAudioMs = 2500; + public int audioFormat = AudioFormat.ENCODING_PCM_16BIT; + public int captureSession = 0; + public boolean dump = false; + public int[] filter; + public int recordInterval = 0; + } +} + diff --git a/app/src/main/java/qianmu/container/recorder/JniRecorder.java b/app/src/main/java/qianmu/container/recorder/JniRecorder.java new file mode 100644 index 0000000..3e3038e --- /dev/null +++ b/app/src/main/java/qianmu/container/recorder/JniRecorder.java @@ -0,0 +1,49 @@ +package qianmu.container.recorder; + +import android.content.Context; + +import com.aispeech.AIAudioRecord; + +/** + * Created by Android Studio. + * User: gaozhilong + * Date: 2025/12/19 + * Time: 11:24 + */ +public class JniRecorder implements IRecord { + private int mBufferSize; + AIAudioRecord mJniRecorder; + + @Override + public void init(Context context, RecordConfig cfg) { + mBufferSize = cfg.sampleRate * 100 * 2 * cfg.audioChannel / 1000; + mJniRecorder = new AIAudioRecord(); + mJniRecorder._native_setup(cfg.audioSource, cfg.sampleRate, cfg.audioChannel); + } + + @Override + public void start() { + mJniRecorder._native_start(); + } + + @Override + public int read(byte[] buffer, int offset, int length) { + return mJniRecorder._native_read_in_byte_array(buffer, 0, length); + } + + @Override + public void stop() { + mJniRecorder._native_stop(); + } + + @Override + public void release() { + + } + + @Override + public int getBufferSize() { + return mBufferSize; + } +} + diff --git a/app/src/main/java/qianmu/container/recorder/NativeRecorder.java b/app/src/main/java/qianmu/container/recorder/NativeRecorder.java new file mode 100644 index 0000000..30500a6 --- /dev/null +++ b/app/src/main/java/qianmu/container/recorder/NativeRecorder.java @@ -0,0 +1,99 @@ +package qianmu.container.recorder; + +import static android.Manifest.permission.RECORD_AUDIO; +import android.content.Context; +import android.content.pm.PackageManager; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.util.Log; + +import androidx.core.content.ContextCompat; + +/** + * Created by Android Studio. + * User: gaozhilong + * Date: 2025/12/19 + * Time: 11:25 + */ +public class NativeRecorder implements IRecord { + public static final String TAG = "NativeRecorder"; + private int mBufferSize; + private AudioRecord mAudioRecord; + + @Override + public void init(Context context, RecordConfig config) { + if (config.recordInterval > 0) { + int channelCount = (config.audioChannel == AudioFormat.CHANNEL_IN_MONO) ? 1 : 2; + int frameSize = config.sampleRate * 2 / 1000; + mBufferSize = frameSize * config.recordInterval * channelCount; + } else { + mBufferSize = AudioRecord.getMinBufferSize(config.sampleRate, config.audioChannel, config.audioEncoding); + } + Log.d(TAG, "mBufferSize:" + mBufferSize); + if (ContextCompat.checkSelfPermission(context, RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { + mAudioRecord = new AudioRecord(config.audioSource, config.sampleRate, config.audioChannel, config.audioFormat, mBufferSize); + } else { + throw new RuntimeException("no permission to record!"); + } + } + + @Override + public void start() { + if (mAudioRecord != null) { + mAudioRecord.startRecording(); + } + } + + @Override + public int read(byte[] buffer, int offset, int length) { + if (mAudioRecord != null) { + return mAudioRecord.read(buffer, 0, length); + } else { + return 0; + } + } + + @Override + public void stop() { + if (mAudioRecord != null) { + mAudioRecord.stop(); + } + } + + @Override + public void release() { + if (mAudioRecord != null) { + mAudioRecord.release(); + mAudioRecord = null; + } + } + + @Override + public int getBufferSize() { + return mBufferSize; + } + + /** + * Check whether recorder is recording. + * + * @return true if audio record is in recording state, otherwise false. + */ + public boolean isRecording() { + boolean bRecording = false; + + if ((mAudioRecord != null) && (mAudioRecord.getState() == AudioRecord.STATE_INITIALIZED)) { + bRecording = (mAudioRecord.getRecordingState() == AudioRecord.RECORDSTATE_RECORDING); + } + + return bRecording; + } + + /** + * 获取正在录音的会话ID + * + * @return + */ + public int getAudioRecordSessionId() { + return isRecording() ? mAudioRecord.getAudioSessionId() : 0; + } +}