diff --git a/build.gradle b/build.gradle index 7ada6530..8b6ac8ff 100644 --- a/build.gradle +++ b/build.gradle @@ -10,6 +10,7 @@ buildscript { dependencies { classpath 'com.android.tools.build:gradle:3.6.3' classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version" + classpath 'com.github.kezong:fat-aar:1.3.8' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files } diff --git a/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/call/RecordVideoActivity.java b/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/call/RecordVideoActivity.java index eabe0c84..55f4d368 100644 --- a/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/call/RecordVideoActivity.java +++ b/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/call/RecordVideoActivity.java @@ -36,7 +36,6 @@ import androidx.localbroadcastmanager.content.LocalBroadcastManager; import com.alibaba.fastjson.JSON; -//import com.iot.gvoice.interfaces.GvoiceJNIBridge; import com.tencent.iot.explorer.device.android.app.R; import com.tencent.iot.explorer.device.android.app.utils.CommonUtils; import com.tencent.iot.explorer.device.common.stateflow.entity.CallingType; @@ -49,7 +48,6 @@ import com.tencent.iot.explorer.device.video.recorder.encoder.AudioEncoder; import com.tencent.iot.explorer.device.video.recorder.encoder.VideoEncoder; import com.tencent.iot.explorer.device.video.recorder.listener.OnEncodeListener; -import com.tencent.iot.explorer.device.video.recorder.listener.OnReadAECProcessedPcmListener; import com.tencent.iot.explorer.device.video.recorder.param.AudioEncodeParam; import com.tencent.iot.explorer.device.video.recorder.param.MicParam; import com.tencent.iot.explorer.device.video.recorder.param.VideoEncodeParam; @@ -79,7 +77,8 @@ import static com.tencent.iot.explorer.device.android.utils.ConvertUtils.byte2HexOnlyLatest8; import static com.tencent.iot.explorer.device.video.recorder.consts.LogConst.RTC_TAG; -public class RecordVideoActivity extends AppCompatActivity implements TextureView.SurfaceTextureListener, OnEncodeListener, SurfaceHolder.Callback { +public class RecordVideoActivity extends AppCompatActivity implements TextureView.SurfaceTextureListener, OnEncodeListener, SurfaceHolder.Callback, + IMediaPlayer.OnInfoListener { private static Timer bitRateTimer; private String TAG = RecordVideoActivity.class.getSimpleName(); @@ -117,14 +116,6 @@ public class RecordVideoActivity extends AppCompatActivity implements TextureVie private int vh = 240; private int frameRate = 15; - private LinkedBlockingDeque playPcmData = new LinkedBlockingDeque<>(); // 内存队列，用于缓存获取到的播放器音频pcm - private String audioCacheFilePath = Environment.getExternalStorageDirectory().getAbsolutePath()+ "/audio1_cache_file"; - private Handler writeHandler; - private final int MESSAGE_AUDIO_ENCODE_FROM_BYTE = 3000; - private FileOutputStream fos1; - private FileOutputStream fos2; - private FileOutputStream fos3; - @Override protected void onCreate(@Nullable Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -187,97 +178,33 @@ private void initAudioEncoder() { .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT) // PCM .build(); AudioEncodeParam audioEncodeParam = new AudioEncodeParam.Builder().build(); - audioEncoder = new AudioEncoder(micParam, audioEncodeParam, processedPcmListener); + audioEncoder = new AudioEncoder(this, micParam, audioEncodeParam, true, true); audioEncoder.setOnEncodeListener(this); } - private OnReadAECProcessedPcmListener processedPcmListener = new OnReadAECProcessedPcmListener() { - @Override - public byte[] onReadAECProcessedPcmListener(byte[] micPcmBytes) { -// byte[] playerPcmBytes = onReadPlayerPlayPcm(micPcmBytes.length); -// byte[] cancell = new byte[micPcmBytes.length]; -// for (int i = 0; i<4; i++) { -// byte[] tempAudioR = new byte[640]; -// byte[] tempPlayer = new byte[640]; -// System.arraycopy(micPcmBytes, i*640, tempAudioR, 0, 640); -// System.arraycopy(playerPcmBytes, i*640, tempPlayer, 0, 640); -// byte[] tempC = GvoiceJNIBridge.cancellation(tempAudioR, tempPlayer); -// System.arraycopy(tempC, 0, cancell, i*640, 640); -// } -// -// if (writeHandler != null) { -// JSONObject jsonObject = new JSONObject(); -// try { -// jsonObject.put("micBytesData", micPcmBytes); -// jsonObject.put("playerBytesData", playerPcmBytes); -// jsonObject.put("cancellBytesData", cancell); -// } catch (JSONException e) { -// e.printStackTrace(); -// } -// Message message = writeHandler.obtainMessage(MESSAGE_AUDIO_ENCODE_FROM_BYTE, jsonObject); -// message.arg1 = micPcmBytes.length; -// writeHandler.sendMessage(message); -// } -// return cancell; - return micPcmBytes; - } - - @Override - public void audioCodecRelease() { -// GvoiceJNIBridge.destory(); -// playPcmData.clear(); - } - }; - - private byte[] onReadPlayerPlayPcm(int length) { - if (player != null && player.isPlaying()) { - byte[] data = new byte[204800]; - int len = player._getPcmData(data); - if (playPcmData.size() > 8*length) { - if (len > 6*length) { - len = 6*length; - } else if (len == 0) { - } else { - int temp = playPcmData.size() - (6*length - len); - for (int i = 0 ; i < temp ; i++) { - playPcmData.remove(); - } - } - } else if (len > 8*length) { - len = 6*length; - } - if (len > 0) { - byte[] playerBytes = new byte[len]; - System.arraycopy(data, 0, playerBytes, 0, len); - List tmpList = new ArrayList<>(); - for (byte b : playerBytes){ - tmpList.add(b); - } - playPcmData.addAll(tmpList); - } - if (playPcmData.size() > length) { - byte[] res = new byte[length]; - try { - for (int i = 0 ; i < length ; i++) { - res[i] = playPcmData.take(); - } - } catch (InterruptedException e) { - e.printStackTrace(); - } - return res; - } else { - return new byte[length]; - } - } - return new byte[length]; - } - private void initVideoEncoder() { VideoEncodeParam videoEncodeParam = new VideoEncodeParam.Builder().setSize(vw, vh).setFrameRate(frameRate).setBitRate(vw*vh).build(); videoEncoder = new VideoEncoder(videoEncodeParam); videoEncoder.setEncoderListener(this); } + @Override + public boolean onInfo(IMediaPlayer mp, int what, int extra) { + return false; + } + + @Override + public boolean onInfoSEI(IMediaPlayer mp, int what, int extra, String sei_content) { + return false; + } + + @Override + public void onInfoAudioPcmData(IMediaPlayer mp, byte[] arrPcm, int length) { + if (audioEncoder != null && length > 0) { + audioEncoder.setPlayerPcmData(arrPcm); + } + } + public class AdapterBitRateTask extends TimerTask { @Override @@ -335,11 +262,7 @@ private void startRecord() { if (phoneInfo.getCallType() == CallingType.TYPE_VIDEO_CALL) { startEncodeVideo = true; } - if (!TextUtils.isEmpty(audioCacheFilePath)) { - new WriteThread().start(); - } audioEncoder.start(); -// GvoiceJNIBridge.init(); startBitRateAdapter(); } @@ -400,7 +323,6 @@ private void releasePlayer() { if (player.isPlaying()) { player.stop(); } - player._setApmStatus(false); player.release(); } } @@ -443,6 +365,7 @@ private void play() { } player = new IjkMediaPlayer(); player.reset(); + player.setOnInfoListener(this); mHandler.sendEmptyMessageDelayed(MSG_UPDATE_HUD, 500); /* * probesize & analyzeduration 可通过这两个参数进行首开延时优化 @@ -470,7 +393,6 @@ private void play() { player.setMaxPacketNum(2); player.setSurface(this.surface); player.setAndroidIOCallback(ReadByteIO.Companion.getInstance()); - player._setApmStatus(true); player.setOnErrorListener(new IMediaPlayer.OnErrorListener() { @Override public boolean onError(IMediaPlayer mp, int what, int extra) { @@ -725,70 +647,4 @@ private String getSendStreamStatus() { return "Unkown"; } - class WriteThread extends Thread { - @SuppressLint("HandlerLeak") - @Override - public void run() { - super.run(); - Looper.prepare(); - File file1 = new File(audioCacheFilePath+"_file1.pcm"); - File file2 = new File(audioCacheFilePath+"_file2.pcm"); - File file3 = new File(audioCacheFilePath+"_file3.pcm"); - Log.i(TAG, "audio cache pcm file path:" + audioCacheFilePath); - if (file1.exists()) { - file1.delete(); - } - if (file2.exists()) { - file2.delete(); - } - if (file3.exists()) { - file3.delete(); - } - try { - file1.createNewFile(); - file2.createNewFile(); - file3.createNewFile(); - } catch (IOException e) { - e.printStackTrace(); - } - try { - fos1 = new FileOutputStream(file1); - fos2 = new FileOutputStream(file2); - fos3 = new FileOutputStream(file3); - } catch (FileNotFoundException e) { - e.printStackTrace(); - Log.e(TAG, "临时缓存文件未找到"); - } - writeHandler = new Handler() { - @Override - public void handleMessage(Message msg) { - super.handleMessage(msg); - if (msg.what == MESSAGE_AUDIO_ENCODE_FROM_BYTE) { - JSONObject jsonObject = (JSONObject) msg.obj; - if (AudioRecord.ERROR_INVALID_OPERATION != msg.arg1) { - if (fos1 != null && fos2 != null && fos3 != null) { - try { - byte[] micBytesData = (byte[]) jsonObject.get("micBytesData"); - byte[] playerBytesData = (byte[]) jsonObject.get("playerBytesData"); - byte[] cancellBytesData = (byte[]) jsonObject.get("cancellBytesData"); - fos1.write(micBytesData); - fos1.flush(); - fos2.write(playerBytesData); - fos2.flush(); - fos3.write(cancellBytesData); - fos3.flush(); - } catch (IOException e) { - e.printStackTrace(); - } catch (JSONException e) { - e.printStackTrace(); - } - } - } - } - } - }; - Looper.loop(); - } - } - } \ No newline at end of file diff --git a/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/device_and_device/RecordVideoActivity3.java b/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/device_and_device/RecordVideoActivity3.java index f7a9e0e9..187ccbf9 100644 --- a/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/device_and_device/RecordVideoActivity3.java +++ b/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/device_and_device/RecordVideoActivity3.java @@ -143,7 +143,7 @@ private void initAudioEncoder() { .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT) // PCM .build(); AudioEncodeParam audioEncodeParam = new AudioEncodeParam.Builder().build(); - audioEncoder = new AudioEncoder(micParam, audioEncodeParam); + audioEncoder = new AudioEncoder(this, micParam, audioEncodeParam); audioEncoder.setOnEncodeListener(this); } diff --git a/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/device_and_device/RecordVideoActivity4.java b/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/device_and_device/RecordVideoActivity4.java index 3187e3c0..f1b468ed 100644 --- a/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/device_and_device/RecordVideoActivity4.java +++ b/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/device_and_device/RecordVideoActivity4.java @@ -237,7 +237,7 @@ private void initAudioEncoder() { .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT) // PCM .build(); AudioEncodeParam audioEncodeParam = new AudioEncodeParam.Builder().build(); - audioEncoder = new AudioEncoder(micParam, audioEncodeParam); + audioEncoder = new AudioEncoder(this, micParam, audioEncodeParam); audioEncoder.setOnEncodeListener(this); } diff --git a/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/push_stream/RecordVideoActivity2.java b/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/push_stream/RecordVideoActivity2.java index ae49ff38..8620f3e5 100644 --- a/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/push_stream/RecordVideoActivity2.java +++ b/explorer/device-android-demo/src/main/java/com/tencent/iot/explorer/device/video/push_stream/RecordVideoActivity2.java @@ -51,9 +51,10 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import tv.danmaku.ijk.media.player.IMediaPlayer; import tv.danmaku.ijk.media.player.IjkMediaPlayer; -public class RecordVideoActivity2 extends AppCompatActivity implements TextureView.SurfaceTextureListener, OnEncodeListener, SurfaceHolder.Callback { +public class RecordVideoActivity2 extends AppCompatActivity implements TextureView.SurfaceTextureListener, OnEncodeListener, SurfaceHolder.Callback, IMediaPlayer.OnInfoListener { private static final String TAG = RecordVideoActivity2.class.getSimpleName(); private static final int MAX_CONNECT_NUM = 4; @@ -131,20 +132,21 @@ protected void onCreate(@Nullable Bundle savedInstanceState) { } initAudioEncoder(); initVideoEncoder(); - VideoFormat format = new VideoFormat.Builder().setVideoWidth(vw).setVideoHeight(vh).build(); + VideoFormat format = new VideoFormat.Builder().setVideoWidth(vw).setVideoHeight(vh).setAudioSampleRate(16000).build(); VideoNativeInteface.getInstance().initVideoFormat(format); } private void initAudioEncoder() { MicParam micParam = new MicParam.Builder() .setAudioSource(MediaRecorder.AudioSource.MIC) - .setSampleRateInHz(8000) // 采样率 + .setSampleRateInHz(16000) // 采样率 .setChannelConfig(AudioFormat.CHANNEL_IN_MONO) .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT) // PCM .build(); AudioEncodeParam audioEncodeParam = new AudioEncodeParam.Builder().build(); - audioEncoder = new AudioEncoder(micParam, audioEncodeParam); + audioEncoder = new AudioEncoder(this, micParam, audioEncodeParam, true, true); audioEncoder.setOnEncodeListener(this); + audioEncoder.recordPcmFile(true); } private void initVideoEncoder() { @@ -253,6 +255,7 @@ public void onSurfaceTextureUpdated(SurfaceTexture surface) { } private void play() { player = new IjkMediaPlayer(); player.reset(); + player.setOnInfoListener(this); if (phoneInfo.getCallType() == CallingType.TYPE_AUDIO_CALL) { player.setOption(IjkMediaPlayer.OPT_CATEGORY_FORMAT, "analyzeduration", 1000); player.setOption(IjkMediaPlayer.OPT_CATEGORY_FORMAT, "probesize", 64); @@ -419,5 +422,22 @@ public void surfaceChanged(SurfaceHolder surfaceHolder, int i, int i1, int i2) { public void surfaceDestroyed(SurfaceHolder surfaceHolder) { Log.d(TAG, "surface destroyed."); } + + @Override + public boolean onInfo(IMediaPlayer mp, int what, int extra) { + return false; + } + + @Override + public boolean onInfoSEI(IMediaPlayer mp, int what, int extra, String sei_content) { + return false; + } + + @Override + public void onInfoAudioPcmData(IMediaPlayer mp, byte[] arrPcm, int length) { + if (audioEncoder != null && length > 0) { + audioEncoder.setPlayerPcmData(arrPcm); + } + } } diff --git a/explorer/explorer-device-video/build.gradle b/explorer/explorer-device-video/build.gradle index 5f23fc12..ac829c07 100644 --- a/explorer/explorer-device-video/build.gradle +++ b/explorer/explorer-device-video/build.gradle @@ -4,6 +4,7 @@ apply plugin: 'signing' apply plugin: 'kotlin-android' apply plugin: 'kotlin-android-extensions' +apply plugin: 'com.kezong.fat-aar' android { compileSdkVersion 29 @@ -39,6 +40,9 @@ android { lintOptions { abortOnError false } + repositories { + flatDir { dirs 'libs' } + } } repositories { @@ -46,16 +50,16 @@ repositories { } dependencies { - api fileTree(include: ['*.jar', '*.aar'], dir: 'libs') + embed (name:'android_gvoice-release',ext:'aar') testImplementation 'junit:junit:4.+' androidTestImplementation 'androidx.test.ext:junit:1.1.3' androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0' implementation 'com.alibaba:fastjson:1.2.73' api 'org.jetbrains.kotlinx:kotlinx-coroutines-android:1.3.4' api 'com.tencent.iot.thirdparty.android:xp2p-sdk:2.4.35' - api 'com.tencent.iot.thirdparty.android:ijkplayer-java:2.0.7' - api 'com.tencent.iot.thirdparty.android:ijkplayer-armv7a:2.0.7' - api 'com.tencent.iot.thirdparty.android:ijkplayer-arm64:2.0.7' + api 'com.tencent.iot.thirdparty.android:ijkplayer-java:2.0.14' + api 'com.tencent.iot.thirdparty.android:ijkplayer-armv7a:2.0.14' + api 'com.tencent.iot.thirdparty.android:ijkplayer-arm64:2.0.14' api project(':explorer:explorer-media-common') //xxxapi project(':explorer:explorer//xxx-media-common') if (findProject(':explorer:explorer-device-android') != null) { diff --git a/explorer/explorer-device-video/libs/android_gvoice-release.aar b/explorer/explorer-device-video/libs/android_gvoice-release.aar new file mode 100644 index 00000000..526a5b48 Binary files /dev/null and b/explorer/explorer-device-video/libs/android_gvoice-release.aar differ diff --git a/explorer/explorer-device-video/src/main/java/com/tencent/iot/explorer/device/video/recorder/encoder/AudioEncoder.java b/explorer/explorer-device-video/src/main/java/com/tencent/iot/explorer/device/video/recorder/encoder/AudioEncoder.java index 25fc1931..6853dc31 100644 --- a/explorer/explorer-device-video/src/main/java/com/tencent/iot/explorer/device/video/recorder/encoder/AudioEncoder.java +++ b/explorer/explorer-device-video/src/main/java/com/tencent/iot/explorer/device/video/recorder/encoder/AudioEncoder.java @@ -1,5 +1,6 @@ package com.tencent.iot.explorer.device.video.recorder.encoder; +import android.content.Context; import android.media.AudioFormat; import android.media.AudioRecord; import android.media.MediaCodec; @@ -7,23 +8,39 @@ import android.media.MediaFormat; import android.media.audiofx.AcousticEchoCanceler; import android.media.audiofx.AutomaticGainControl; +import android.os.Handler; +import android.os.HandlerThread; import android.os.Looper; +import android.os.Message; +import android.text.TextUtils; import android.util.Log; +import com.iot.gvoice.interfaces.GvoiceJNIBridge; import com.tencent.iot.explorer.device.video.recorder.listener.OnEncodeListener; import com.tencent.iot.explorer.device.video.recorder.listener.OnReadAECProcessedPcmListener; import com.tencent.iot.explorer.device.video.recorder.param.AudioEncodeParam; import com.tencent.iot.explorer.device.video.recorder.param.MicParam; +import org.jetbrains.annotations.NotNull; +import org.json.JSONException; +import org.json.JSONObject; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.LinkedBlockingDeque; import static com.tencent.iot.explorer.device.android.utils.ConvertUtils.byte2HexOnlyLatest8; import static com.tencent.iot.explorer.device.video.recorder.consts.LogConst.RTC_TAG; -public class AudioEncoder { +public class AudioEncoder implements Handler.Callback { /** * 采样频率对照表 @@ -46,6 +63,11 @@ public class AudioEncoder { } private final String TAG = AudioEncoder.class.getSimpleName(); + private static final int MSG_START = 1; + private static final int MSG_STOP = 2; + private static final int MSG_REC_PLAY_PCM = 3; + private static final int MSG_RELEASE = 4; + private boolean isRecord = false; private MediaCodec audioCodec; private AudioRecord audioRecord; private AcousticEchoCanceler canceler; @@ -54,40 +76,110 @@ public class AudioEncoder { private final MicParam micParam; private final AudioEncodeParam audioEncodeParam; private OnEncodeListener encodeListener; + private OnReadAECProcessedPcmListener micPcmListener; private volatile boolean stopEncode = false; private long seq = 0L; private long beforSeq = 0L; private int bufferSizeInBytes; - private OnReadAECProcessedPcmListener mAECProcessedPcmListener; + private final HandlerThread readThread; + private final ReadHandler mReadHandler; + private volatile boolean recorderState = true; //录制状态 + + private boolean enableAEC = false; + + private FileOutputStream fos1; + private FileOutputStream fos2; + private FileOutputStream fos3; + private String speakPcmFilePath = "/storage/emulated/0/speak_pcm_"; + + private static final int SAVE_PCM_DATA = 1; + private LinkedBlockingDeque playPcmData = new LinkedBlockingDeque<>(); // 内存队列，用于缓存获取到的播放器音频pcm + + @Override + public boolean handleMessage(@NotNull Message msg) { + switch (msg.what) { + case MSG_START: + startInternal(); + break; + case MSG_STOP: + stopInternal(); + break; + case MSG_REC_PLAY_PCM: + recPlayPcmInternal((byte[]) msg.obj); + break; + case MSG_RELEASE: + releaseInternal(); + break; + } + return false; + } + + private class MyHandler extends Handler { + + public MyHandler() { + } + + @Override + public void handleMessage(Message msg) { + super.handleMessage(msg); + + try { + if (msg.what == SAVE_PCM_DATA && fos1 != null && fos2 != null && fos3 != null) { + JSONObject jsonObject = (JSONObject) msg.obj; + byte[] nearBytesData = (byte[]) jsonObject.get("nearPcmBytes"); + fos1.write(nearBytesData); + fos1.flush(); + byte[] playerPcmBytes = (byte[]) jsonObject.get("playerPcmBytes"); + fos2.write(playerPcmBytes); + fos2.flush(); + byte[] aecPcmBytes = (byte[]) jsonObject.get("aecPcmBytes"); + fos3.write(aecPcmBytes); + fos3.flush(); + } - public AudioEncoder(MicParam micParam, AudioEncodeParam audioEncodeParam) { - this(micParam, audioEncodeParam, false, false); + } catch (IOException e) { + Log.e(TAG, "*======== IOException: " + e); + e.printStackTrace(); + } catch (JSONException e) { + Log.e(TAG, "*======== JSONException: " + e); + e.printStackTrace(); + } + } } + private final Handler mHandler = new MyHandler(); - public AudioEncoder(MicParam micParam, AudioEncodeParam audioEncodeParam, OnReadAECProcessedPcmListener listener) { - this(micParam, audioEncodeParam, false, false); - this.mAECProcessedPcmListener = listener; + public AudioEncoder(Context context, MicParam micParam, AudioEncodeParam audioEncodeParam) { + this(context, micParam, audioEncodeParam, false, false); } - public AudioEncoder(MicParam micParam, AudioEncodeParam audioEncodeParam, boolean enableAEC, boolean enableAGC) { + public AudioEncoder(Context context, MicParam micParam, AudioEncodeParam audioEncodeParam, boolean enableAEC, boolean enableAGC) { this.micParam = micParam; this.audioEncodeParam = audioEncodeParam; initAudio(); int audioSessionId = audioRecord.getAudioSessionId(); + this.enableAEC = enableAEC; if (enableAEC && audioSessionId != 0) { Log.e(TAG, "=====initAEC result: " + initAEC(audioSessionId)); } if (enableAGC && audioSessionId != 0) { Log.e(TAG, "=====initAGC result: " + initAGC(audioSessionId)); } + readThread = new HandlerThread(TAG); + readThread.start(); + mReadHandler = new ReadHandler(readThread.getLooper(), this); + GvoiceJNIBridge.init(context); } public void setOnEncodeListener(OnEncodeListener listener) { this.encodeListener = listener; } + public void setOnReadAECProcessedPcmListener(OnReadAECProcessedPcmListener listener) { + this.micPcmListener = listener; + } + private void initAudio() { bufferSizeInBytes = AudioRecord.getMinBufferSize(micParam.getSampleRateInHz(), micParam.getChannelConfig(), micParam.getAudioFormat()); Log.e(TAG, "=====bufferSizeInBytes: " + bufferSizeInBytes); @@ -108,19 +200,76 @@ private void initAudio() { } public void start() { + Log.i(TAG, "start"); + mReadHandler.obtainMessage(MSG_START).sendToTarget(); + } + + private void startInternal() { + if (isRecord) { + fos1 = createFiles("near"); + fos2 = createFiles("far"); + fos3 = createFiles("aec"); + } + if (!playPcmData.isEmpty()) { + playPcmData.clear(); + } new CodecThread().start(); } + private FileOutputStream createFiles(String format) { + + if (!TextUtils.isEmpty(speakPcmFilePath)) { + File file1 = new File(speakPcmFilePath+format+".pcm"); + Log.i(TAG, "speak cache pcm file path:" + speakPcmFilePath); + if (file1.exists()) { + file1.delete(); + } + try { + file1.createNewFile(); + } catch (IOException e) { + e.printStackTrace(); + return null; + } + try { + FileOutputStream fos = new FileOutputStream(file1); + return fos; + } catch (FileNotFoundException e) { + e.printStackTrace(); + Log.e(TAG, "临时缓存文件未找到"); + return null; + } + } + return null; + } + public void stop() { + Log.i(TAG, "stop"); + mReadHandler.obtainMessage(MSG_STOP).sendToTarget(); + } + + private void stopInternal() { stopEncode = true; } + public void setPlayerPcmData(byte[] pcmData) { + mReadHandler.obtainMessage(MSG_REC_PLAY_PCM, pcmData).sendToTarget(); + } + + private void recPlayPcmInternal(byte [] pcmData) { + if (pcmData != null && pcmData.length > 0 && recorderState) { + List tmpList = new ArrayList<>(); + for (byte b : pcmData) { + tmpList.add(b); + } + playPcmData.addAll(tmpList); + } + } + public boolean isDevicesSupportAEC() { return AcousticEchoCanceler.isAvailable(); } private boolean initAEC(int audioSession) { - boolean isDevicesSupportAEC = isDevicesSupportAEC(); Log.e(TAG, "isDevicesSupportAEC: "+isDevicesSupportAEC); if (!isDevicesSupportAEC) { @@ -130,8 +279,12 @@ private boolean initAEC(int audioSession) { return false; } canceler = AcousticEchoCanceler.create(audioSession); - canceler.setEnabled(true); - return canceler.getEnabled(); + if (canceler != null) { + canceler.setEnabled(true); + return canceler.getEnabled(); + } else { + return false; + } } public boolean isDevicesSupportAGC() { @@ -139,7 +292,6 @@ public boolean isDevicesSupportAGC() { } private boolean initAGC(int audioSession) { - boolean isDevicesSupportAGC = isDevicesSupportAGC(); Log.e(TAG, "isDevicesSupportAGC: "+isDevicesSupportAGC); if (!isDevicesSupportAGC) { @@ -149,11 +301,20 @@ private boolean initAGC(int audioSession) { return false; } control = AutomaticGainControl.create(audioSession); - control.setEnabled(true); - return control.getEnabled(); + if (control != null) { + control.setEnabled(true); + return control.getEnabled(); + } else { + return false; + } } private void release() { + Log.i(TAG, "release"); + mReadHandler.obtainMessage(MSG_RELEASE).sendToTarget(); + } + + private void releaseInternal() { if (audioRecord != null) { audioRecord.stop(); audioRecord.release(); @@ -177,9 +338,10 @@ private void release() { control.release(); control = null; } - if (mAECProcessedPcmListener != null) { - mAECProcessedPcmListener.audioCodecRelease(); - } + } + + public void recordPcmFile(boolean isRecord) { + this.isRecord = isRecord; } private void addADTStoPacket(ByteBuffer outputBuffer) { @@ -248,19 +410,27 @@ public void run() { inputBuffer = audioCodec.getInputBuffers()[audioInputBufferId]; } int readSize = -1; - int size = mAECProcessedPcmListener != null ? 2560 : bufferSizeInBytes; + int size = enableAEC ? 640 : bufferSizeInBytes; byte[] audioRecordData = new byte[size]; - if (inputBuffer != null) { + if (inputBuffer != null && micPcmListener == null) { readSize = audioRecord.read(audioRecordData, 0, size); + } else if (inputBuffer == null && micPcmListener != null) { + readSize = size; } if (readSize >= 0) { inputBuffer.clear(); - if (mAECProcessedPcmListener != null) { - Log.i(RTC_TAG, String.format("audioRecord read capture original frame data before other process:%s, seq:%d", byte2HexOnlyLatest8(audioRecordData), beforSeq)); - byte[] cancell = mAECProcessedPcmListener.onReadAECProcessedPcmListener(audioRecordData); - Log.i(RTC_TAG, String.format("audioRecord read capture original frame data after other process:%s, seq:%d", byte2HexOnlyLatest8(audioRecordData), beforSeq)); + if (enableAEC){ + byte [] playerPcmBytes = onReadPlayerPlayPcm(size); + if (micPcmListener != null) { + audioRecordData = micPcmListener.onReadAECProcessedPcmListener(size); + } + + byte[] aecPcmBytes = GvoiceJNIBridge.cancellation(audioRecordData, playerPcmBytes); + if (isRecord) { + writePcmBytesToFile(audioRecordData, playerPcmBytes, aecPcmBytes); + } beforSeq++; - inputBuffer.put(cancell); + inputBuffer.put(aecPcmBytes); } else { inputBuffer.put(audioRecordData); Log.i(RTC_TAG, String.format("audioRecord read capture origina l frame data:%s", byte2HexOnlyLatest8(audioRecordData))); @@ -292,4 +462,61 @@ public void run() { Looper.loop(); } } + + private byte[] onReadPlayerPlayPcm(int length) { + if (playPcmData.size() > length) { + byte[] res = new byte[length]; + try { + for (int i = 0 ; i < length ; i++) { + res[i] = playPcmData.take(); + } + } catch (InterruptedException e) { + e.printStackTrace(); + } + Log.e(TAG, "onReadPlayerPlayPcm playPcmData.length： " + playPcmData.size()); + if (playPcmData.size()>20000) { + playPcmData.clear(); + } + return res; + } else { + return new byte[length]; + } + } + + + private void writePcmBytesToFile(byte[] nearPcmBytes, byte[] playerPcmBytes, byte[] aecPcmBytes) { + if (mHandler != null) { + JSONObject jsonObject = new JSONObject(); + try { + jsonObject.put("nearPcmBytes", nearPcmBytes); + jsonObject.put("playerPcmBytes", playerPcmBytes); + jsonObject.put("aecPcmBytes", aecPcmBytes); + } catch (JSONException e) { + e.printStackTrace(); + } + Message message = mHandler.obtainMessage(SAVE_PCM_DATA, jsonObject); + mHandler.sendMessage(message); + } + } + + public static class ReadHandler extends Handler { + + public ReadHandler(Looper looper, Callback callback) { + super(looper, callback); + } + + public void runAndWaitDone(final Runnable runnable) { + final CountDownLatch countDownLatch = new CountDownLatch(1); + post(() -> { + runnable.run(); + countDownLatch.countDown(); + }); + + try { + countDownLatch.await(); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } } diff --git a/explorer/explorer-device-video/src/main/java/com/tencent/iot/explorer/device/video/recorder/listener/OnReadAECProcessedPcmListener.java b/explorer/explorer-device-video/src/main/java/com/tencent/iot/explorer/device/video/recorder/listener/OnReadAECProcessedPcmListener.java index b6fac214..05339203 100644 --- a/explorer/explorer-device-video/src/main/java/com/tencent/iot/explorer/device/video/recorder/listener/OnReadAECProcessedPcmListener.java +++ b/explorer/explorer-device-video/src/main/java/com/tencent/iot/explorer/device/video/recorder/listener/OnReadAECProcessedPcmListener.java @@ -1,6 +1,5 @@ package com.tencent.iot.explorer.device.video.recorder.listener; public interface OnReadAECProcessedPcmListener { - byte[] onReadAECProcessedPcmListener(byte[] micPcmBytes); - void audioCodecRelease(); + byte[] onReadAECProcessedPcmListener(int micPcmBytesSize); }