Android中音視頻合成的幾種方案詳析

發布時間：2020-08-25 05:23:08 來源：腳本之家閱讀：415 作者：AnswerZhao 欄目：移動開發

前言

最近工作中遇到了音視頻處理的需求，Android下音視頻合成，在當前調研方案中主要有三大類方法：MediaMux硬解碼，mp4parser，FFmepg。三種方法均可實現，但是也有不同的局限和問題，先將實現和問題記錄于此，便于之后的總結學習。下面話不多說了，來一起看看詳細的介紹吧。

方法一（Fail）

利用MediaMux實現音視頻的合成。

效果：可以實現音視頻的合并，利用Android原生的VideoView和SurfaceView播放正常，大部分的播放器也播放正常，但是，但是，在上傳Youtube就會出現問題：音頻不連續，分析主要是上傳Youtube時會被再次的壓縮，可能在壓縮的過程中出現音頻的幀率出現問題。

分析：在MediaCodec.BufferInfo的處理中，時間戳presentationTimeUs出現問題，導致Youtube的壓縮造成音頻的紊亂。

public static void muxVideoAndAudio(String videoPath, String audioPath, String muxPath) {
 try {
  MediaExtractor videoExtractor = new MediaExtractor();
  videoExtractor.setDataSource(videoPath);
  MediaFormat videoFormat = null;
  int videoTrackIndex = -1;
  int videoTrackCount = videoExtractor.getTrackCount();
  for (int i = 0; i < videoTrackCount; i++) {
  videoFormat = videoExtractor.getTrackFormat(i);
  String mimeType = videoFormat.getString(MediaFormat.KEY_MIME);
  if (mimeType.startsWith("video/")) {
   videoTrackIndex = i;
   break;
  }
  }
  MediaExtractor audioExtractor = new MediaExtractor();
  audioExtractor.setDataSource(audioPath);
  MediaFormat audioFormat = null;
  int audioTrackIndex = -1;
  int audioTrackCount = audioExtractor.getTrackCount();
  for (int i = 0; i < audioTrackCount; i++) {
  audioFormat = audioExtractor.getTrackFormat(i);
  String mimeType = audioFormat.getString(MediaFormat.KEY_MIME);
  if (mimeType.startsWith("audio/")) {
   audioTrackIndex = i;
   break;
  }
  }
  videoExtractor.selectTrack(videoTrackIndex);
  audioExtractor.selectTrack(audioTrackIndex);
  MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();
  MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();
  MediaMuxer mediaMuxer = new MediaMuxer(muxPath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
  int writeVideoTrackIndex = mediaMuxer.addTrack(videoFormat);
  int writeAudioTrackIndex = mediaMuxer.addTrack(audioFormat);
  mediaMuxer.start();
  ByteBuffer byteBuffer = ByteBuffer.allocate(500 * 1024);
  long sampleTime = 0;
  {
  videoExtractor.readSampleData(byteBuffer, 0);
  if (videoExtractor.getSampleFlags() == MediaExtractor.SAMPLE_FLAG_SYNC) {
   videoExtractor.advance();
  }
  videoExtractor.readSampleData(byteBuffer, 0);
  long secondTime = videoExtractor.getSampleTime();
  videoExtractor.advance();
  long thirdTime = videoExtractor.getSampleTime();
  sampleTime = Math.abs(thirdTime - secondTime);
  }
  videoExtractor.unselectTrack(videoTrackIndex);
  videoExtractor.selectTrack(videoTrackIndex);
  while (true) {
  int readVideoSampleSize = videoExtractor.readSampleData(byteBuffer, 0);
  if (readVideoSampleSize < 0) {
   break;
  }
  videoBufferInfo.size = readVideoSampleSize;
  videoBufferInfo.presentationTimeUs += sampleTime;
  videoBufferInfo.offset = 0;
  //noinspection WrongConstant
  videoBufferInfo.flags = MediaCodec.BUFFER_FLAG_SYNC_FRAME;//videoExtractor.getSampleFlags()
  mediaMuxer.writeSampleData(writeVideoTrackIndex, byteBuffer, videoBufferInfo);
  videoExtractor.advance();
  }
  while (true) {
  int readAudioSampleSize = audioExtractor.readSampleData(byteBuffer, 0);
  if (readAudioSampleSize < 0) {
   break;
  }
  audioBufferInfo.size = readAudioSampleSize;
  audioBufferInfo.presentationTimeUs += sampleTime;
  audioBufferInfo.offset = 0;
  //noinspection WrongConstant
  audioBufferInfo.flags = MediaCodec.BUFFER_FLAG_SYNC_FRAME;// videoExtractor.getSampleFlags()
  mediaMuxer.writeSampleData(writeAudioTrackIndex, byteBuffer, audioBufferInfo);
  audioExtractor.advance();
  }
  mediaMuxer.stop();
  mediaMuxer.release();
  videoExtractor.release();
  audioExtractor.release();
 } catch (IOException e) {
  e.printStackTrace();
 }
 }

方法二（Success）

public static void muxVideoAudio(String videoFilePath, String audioFilePath, String outputFile) {
 try {
  MediaExtractor videoExtractor = new MediaExtractor();
  videoExtractor.setDataSource(videoFilePath);
  MediaExtractor audioExtractor = new MediaExtractor();
  audioExtractor.setDataSource(audioFilePath);
  MediaMuxer muxer = new MediaMuxer(outputFile, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
  videoExtractor.selectTrack(0);
  MediaFormat videoFormat = videoExtractor.getTrackFormat(0);
  int videoTrack = muxer.addTrack(videoFormat);
  audioExtractor.selectTrack(0);
  MediaFormat audioFormat = audioExtractor.getTrackFormat(0);
  int audioTrack = muxer.addTrack(audioFormat);
  LogUtil.d(TAG, "Video Format " + videoFormat.toString());
  LogUtil.d(TAG, "Audio Format " + audioFormat.toString());
  boolean sawEOS = false;
  int frameCount = 0;
  int offset = 100;
  int sampleSize = 256 * 1024;
  ByteBuffer videoBuf = ByteBuffer.allocate(sampleSize);
  ByteBuffer audioBuf = ByteBuffer.allocate(sampleSize);
  MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo();
  MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo();
  videoExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
  audioExtractor.seekTo(0, MediaExtractor.SEEK_TO_CLOSEST_SYNC);
  muxer.start();
  while (!sawEOS) {
  videoBufferInfo.offset = offset;
  videoBufferInfo.size = videoExtractor.readSampleData(videoBuf, offset);
  if (videoBufferInfo.size < 0 || audioBufferInfo.size < 0) {
   sawEOS = true;
   videoBufferInfo.size = 0;
  } else {
   videoBufferInfo.presentationTimeUs = videoExtractor.getSampleTime();
   //noinspection WrongConstant
   videoBufferInfo.flags = videoExtractor.getSampleFlags();
   muxer.writeSampleData(videoTrack, videoBuf, videoBufferInfo);
   videoExtractor.advance();
   frameCount++;
  }
  }
  
  boolean sawEOS2 = false;
  int frameCount2 = 0;
  while (!sawEOS2) {
  frameCount2++;
  audioBufferInfo.offset = offset;
  audioBufferInfo.size = audioExtractor.readSampleData(audioBuf, offset);
  if (videoBufferInfo.size < 0 || audioBufferInfo.size < 0) {
   sawEOS2 = true;
   audioBufferInfo.size = 0;
  } else {
   audioBufferInfo.presentationTimeUs = audioExtractor.getSampleTime();
   //noinspection WrongConstant
   audioBufferInfo.flags = audioExtractor.getSampleFlags();
   muxer.writeSampleData(audioTrack, audioBuf, audioBufferInfo);
   audioExtractor.advance();
  }
  }
  muxer.stop();
  muxer.release();
  LogUtil.d(TAG,"Output: "+outputFile);
 } catch (IOException e) {
  LogUtil.d(TAG, "Mixer Error 1 " + e.getMessage());
 } catch (Exception e) {
  LogUtil.d(TAG, "Mixer Error 2 " + e.getMessage());
 }
 }

方法三

利用mp4parser實現

mp4parser是一個視頻處理的開源工具箱，由于mp4parser里的方法都依靠工具箱里的一些內容，所以需要將這些內容打包成jar包，放到自己的工程里，才能對mp4parser的方法進行調用。

compile “com.googlecode.mp4parser:isoparser:1.1.21”

問題：上傳Youtube壓縮后，視頻數據丟失嚴重，大部分就只剩下一秒鐘的時長，相當于把視頻變成圖片了，囧

 public boolean mux(String videoFile, String audioFile, final String outputFile) {
 if (isStopMux) {
  return false;
 }
 Movie video;
 try {
  video = MovieCreator.build(videoFile);
 } catch (RuntimeException e) {
  e.printStackTrace();
  return false;
 } catch (IOException e) {
  e.printStackTrace();
  return false;
 }
 Movie audio;
 try {
  audio = MovieCreator.build(audioFile);
 } catch (IOException e) {
  e.printStackTrace();
  return false;
 } catch (NullPointerException e) {
  e.printStackTrace();
  return false;
 }
 Track audioTrack = audio.getTracks().get(0);
 video.addTrack(audioTrack);
 Container out = new DefaultMp4Builder().build(video);
 FileOutputStream fos;
 try {
  fos = new FileOutputStream(outputFile);
 } catch (FileNotFoundException e) {
  e.printStackTrace();
  return false;
 }
 BufferedWritableFileByteChannel byteBufferByteChannel = new
  BufferedWritableFileByteChannel(fos);
 try {
  out.writeContainer(byteBufferByteChannel);
  byteBufferByteChannel.close();
  fos.close();
  if (isStopMux) {
  return false;
  }
  runOnUiThread(new Runnable() {
  @Override
  public void run() {
   mCustomeProgressDialog.setProgress(100);
   goShareActivity(outputFile);
//   FileUtils.insertMediaDB(AddAudiosActivity.this,outputFile);//
  }
  });
 } catch (IOException e) {
  e.printStackTrace();
  if (mCustomeProgressDialog.isShowing()) {
  mCustomeProgressDialog.dismiss();
  }
  ToastUtil.showShort(getString(R.string.process_failed));
  return false;
 }
 return true;
 }
 private static class BufferedWritableFileByteChannel implements WritableByteChannel {
 private static final int BUFFER_CAPACITY = 2000000;
 private boolean isOpen = true;
 private final OutputStream outputStream;
 private final ByteBuffer byteBuffer;
 private final byte[] rawBuffer = new byte[BUFFER_CAPACITY];
 private BufferedWritableFileByteChannel(OutputStream outputStream) {
  this.outputStream = outputStream;
  this.byteBuffer = ByteBuffer.wrap(rawBuffer);
 }
 @Override
 public int write(ByteBuffer inputBuffer) throws IOException {
  int inputBytes = inputBuffer.remaining();
  if (inputBytes > byteBuffer.remaining()) {
  dumpToFile();
  byteBuffer.clear();
  if (inputBytes > byteBuffer.remaining()) {
   throw new BufferOverflowException();
  }
  }
  byteBuffer.put(inputBuffer);
  return inputBytes;
 }
 @Override
 public boolean isOpen() {
  return isOpen;
 }
 @Override
 public void close() throws IOException {
  dumpToFile();
  isOpen = false;
 }
 private void dumpToFile() {
  try {
  outputStream.write(rawBuffer, 0, byteBuffer.position());
  } catch (IOException e) {
  throw new RuntimeException(e);
  }
 }
 }

方法四

利用FFmpeg大法

FFmpeg 由于其豐富的 codec 插件，詳細的文檔說明，并且與其調試復雜量大的編解碼代碼(是的，用 MediaCodec 實現起來十分啰嗦和繁瑣)還是不如調試一行 ffmpeg 命令來的簡單。

Merge Video /Audio and retain both audios

可以實現，兼容性強，但由于是軟解碼，合并速度很慢，忍受不了，而相應的FFmpeg優化還不太了解，囧…….

總結

以上就是這篇文章的全部內容了，希望本文的內容對大家的學習或者工作具有一定的參考學習價值，如果有疑問大家可以留言交流，謝謝大家對億速云的支持。

向AI問一下細節

亚洲激情专区-91九色丨porny丨老师-久久久久久久女国产乱让韩-国产精品午夜小视频观看

Android中音視頻合成的幾種方案詳析

猜你喜歡

亚洲激情专区-91九色丨porny丨老师-久久久久久久女国产乱让韩-国产精品午夜小视频观看

Android中音視頻合成的幾種方案詳析

猜你喜歡

最新資訊

相關推薦

相關標簽