Android中音视频合成的几种方案详析

前言

最近工作中遇到了音视频处理的需求，android下音视频合成，在当前调研方案中主要有三大类方法：mediamux硬解码，mp4parser，ffmepg。三种方法均可实现，但是也有不同的局限和问题，先将实现和问题记录于此，便于之后的总结学习。下面话不多说了，来一起看看详细的介绍吧。

方法一（fail）

利用mediamux实现音视频的合成。

效果：可以实现音视频的合并，利用android原生的videoview和surfaceview播放正常，大部分的播放器也播放正常，但是，但是，在上传youtube就会出现问题：音频不连续，分析主要是上传youtube时会被再次的压缩，可能在压缩的过程中出现音频的帧率出现问题。

分析：在mediacodec.bufferinfo的处理中，时间戳presentationtimeus出现问题，导致youtube的压缩造成音频的紊乱。

public static void muxvideoandaudio(string videopath, string audiopath, string muxpath) {
 try {
  mediaextractor videoextractor = new mediaextractor();
  videoextractor.setdatasource(videopath);
  mediaformat videoformat = null;
  int videotrackindex = -1;
  int videotrackcount = videoextractor.gettrackcount();
  for (int i = 0; i < videotrackcount; i++) {
  videoformat = videoextractor.gettrackformat(i);
  string mimetype = videoformat.getstring(mediaformat.key_mime);
  if (mimetype.startswith("video/")) {
   videotrackindex = i;
   break;
  }
  }
  mediaextractor audioextractor = new mediaextractor();
  audioextractor.setdatasource(audiopath);
  mediaformat audioformat = null;
  int audiotrackindex = -1;
  int audiotrackcount = audioextractor.gettrackcount();
  for (int i = 0; i < audiotrackcount; i++) {
  audioformat = audioextractor.gettrackformat(i);
  string mimetype = audioformat.getstring(mediaformat.key_mime);
  if (mimetype.startswith("audio/")) {
   audiotrackindex = i;
   break;
  }
  }
  videoextractor.selecttrack(videotrackindex);
  audioextractor.selecttrack(audiotrackindex);
  mediacodec.bufferinfo videobufferinfo = new mediacodec.bufferinfo();
  mediacodec.bufferinfo audiobufferinfo = new mediacodec.bufferinfo();
  mediamuxer mediamuxer = new mediamuxer(muxpath, mediamuxer.outputformat.muxer_output_mpeg_4);
  int writevideotrackindex = mediamuxer.addtrack(videoformat);
  int writeaudiotrackindex = mediamuxer.addtrack(audioformat);
  mediamuxer.start();
  bytebuffer bytebuffer = bytebuffer.allocate(500 * 1024);
  long sampletime = 0;
  {
  videoextractor.readsampledata(bytebuffer, 0);
  if (videoextractor.getsampleflags() == mediaextractor.sample_flag_sync) {
   videoextractor.advance();
  }
  videoextractor.readsampledata(bytebuffer, 0);
  long secondtime = videoextractor.getsampletime();
  videoextractor.advance();
  long thirdtime = videoextractor.getsampletime();
  sampletime = math.abs(thirdtime - secondtime);
  }
  videoextractor.unselecttrack(videotrackindex);
  videoextractor.selecttrack(videotrackindex);
  while (true) {
  int readvideosamplesize = videoextractor.readsampledata(bytebuffer, 0);
  if (readvideosamplesize < 0) {
   break;
  }
  videobufferinfo.size = readvideosamplesize;
  videobufferinfo.presentationtimeus += sampletime;
  videobufferinfo.offset = 0;
  //noinspection wrongconstant
  videobufferinfo.flags = mediacodec.buffer_flag_sync_frame;//videoextractor.getsampleflags()
  mediamuxer.writesampledata(writevideotrackindex, bytebuffer, videobufferinfo);
  videoextractor.advance();
  }
  while (true) {
  int readaudiosamplesize = audioextractor.readsampledata(bytebuffer, 0);
  if (readaudiosamplesize < 0) {
   break;
  }
  audiobufferinfo.size = readaudiosamplesize;
  audiobufferinfo.presentationtimeus += sampletime;
  audiobufferinfo.offset = 0;
  //noinspection wrongconstant
  audiobufferinfo.flags = mediacodec.buffer_flag_sync_frame;// videoextractor.getsampleflags()
  mediamuxer.writesampledata(writeaudiotrackindex, bytebuffer, audiobufferinfo);
  audioextractor.advance();
  }
  mediamuxer.stop();
  mediamuxer.release();
  videoextractor.release();
  audioextractor.release();
 } catch (ioexception e) {
  e.printstacktrace();
 }
 }

方法二（success）

public static void muxvideoaudio(string videofilepath, string audiofilepath, string outputfile) {
 try {
  mediaextractor videoextractor = new mediaextractor();
  videoextractor.setdatasource(videofilepath);
  mediaextractor audioextractor = new mediaextractor();
  audioextractor.setdatasource(audiofilepath);
  mediamuxer muxer = new mediamuxer(outputfile, mediamuxer.outputformat.muxer_output_mpeg_4);
  videoextractor.selecttrack(0);
  mediaformat videoformat = videoextractor.gettrackformat(0);
  int videotrack = muxer.addtrack(videoformat);
  audioextractor.selecttrack(0);
  mediaformat audioformat = audioextractor.gettrackformat(0);
  int audiotrack = muxer.addtrack(audioformat);
  logutil.d(tag, "video format " + videoformat.tostring());
  logutil.d(tag, "audio format " + audioformat.tostring());
  boolean saweos = false;
  int framecount = 0;
  int offset = 100;
  int samplesize = 256 * 1024;
  bytebuffer videobuf = bytebuffer.allocate(samplesize);
  bytebuffer audiobuf = bytebuffer.allocate(samplesize);
  mediacodec.bufferinfo videobufferinfo = new mediacodec.bufferinfo();
  mediacodec.bufferinfo audiobufferinfo = new mediacodec.bufferinfo();
  videoextractor.seekto(0, mediaextractor.seek_to_closest_sync);
  audioextractor.seekto(0, mediaextractor.seek_to_closest_sync);
  muxer.start();
  while (!saweos) {
  videobufferinfo.offset = offset;
  videobufferinfo.size = videoextractor.readsampledata(videobuf, offset);
  if (videobufferinfo.size < 0 || audiobufferinfo.size < 0) {
   saweos = true;
   videobufferinfo.size = 0;
  } else {
   videobufferinfo.presentationtimeus = videoextractor.getsampletime();
   //noinspection wrongconstant
   videobufferinfo.flags = videoextractor.getsampleflags();
   muxer.writesampledata(videotrack, videobuf, videobufferinfo);
   videoextractor.advance();
   framecount++;
  }
  }
  
  boolean saweos2 = false;
  int framecount2 = 0;
  while (!saweos2) {
  framecount2++;
  audiobufferinfo.offset = offset;
  audiobufferinfo.size = audioextractor.readsampledata(audiobuf, offset);
  if (videobufferinfo.size < 0 || audiobufferinfo.size < 0) {
   saweos2 = true;
   audiobufferinfo.size = 0;
  } else {
   audiobufferinfo.presentationtimeus = audioextractor.getsampletime();
   //noinspection wrongconstant
   audiobufferinfo.flags = audioextractor.getsampleflags();
   muxer.writesampledata(audiotrack, audiobuf, audiobufferinfo);
   audioextractor.advance();
  }
  }
  muxer.stop();
  muxer.release();
  logutil.d(tag,"output: "+outputfile);
 } catch (ioexception e) {
  logutil.d(tag, "mixer error 1 " + e.getmessage());
 } catch (exception e) {
  logutil.d(tag, "mixer error 2 " + e.getmessage());
 }
 }

方法三

利用mp4parser实现

mp4parser是一个视频处理的开源工具箱，由于mp4parser里的方法都依靠工具箱里的一些内容，所以需要将这些内容打包成jar包，放到自己的工程里，才能对mp4parser的方法进行调用。

compile “com.googlecode.mp4parser:isoparser:1.1.21”

问题：上传youtube压缩后，视频数据丢失严重，大部分就只剩下一秒钟的时长，相当于把视频变成图片了，囧

 public boolean mux(string videofile, string audiofile, final string outputfile) {
 if (isstopmux) {
  return false;
 }
 movie video;
 try {
  video = moviecreator.build(videofile);
 } catch (runtimeexception e) {
  e.printstacktrace();
  return false;
 } catch (ioexception e) {
  e.printstacktrace();
  return false;
 }
 movie audio;
 try {
  audio = moviecreator.build(audiofile);
 } catch (ioexception e) {
  e.printstacktrace();
  return false;
 } catch (nullpointerexception e) {
  e.printstacktrace();
  return false;
 }
 track audiotrack = audio.gettracks().get(0);
 video.addtrack(audiotrack);
 container out = new defaultmp4builder().build(video);
 fileoutputstream fos;
 try {
  fos = new fileoutputstream(outputfile);
 } catch (filenotfoundexception e) {
  e.printstacktrace();
  return false;
 }
 bufferedwritablefilebytechannel bytebufferbytechannel = new
  bufferedwritablefilebytechannel(fos);
 try {
  out.writecontainer(bytebufferbytechannel);
  bytebufferbytechannel.close();
  fos.close();
  if (isstopmux) {
  return false;
  }
  runonuithread(new runnable() {
  @override
  public void run() {
   mcustomeprogressdialog.setprogress(100);
   goshareactivity(outputfile);
//   fileutils.insertmediadb(addaudiosactivity.this,outputfile);//
  }
  });
 } catch (ioexception e) {
  e.printstacktrace();
  if (mcustomeprogressdialog.isshowing()) {
  mcustomeprogressdialog.dismiss();
  }
  toastutil.showshort(getstring(r.string.process_failed));
  return false;
 }
 return true;
 }
 private static class bufferedwritablefilebytechannel implements writablebytechannel {
 private static final int buffer_capacity = 2000000;
 private boolean isopen = true;
 private final outputstream outputstream;
 private final bytebuffer bytebuffer;
 private final byte[] rawbuffer = new byte[buffer_capacity];
 private bufferedwritablefilebytechannel(outputstream outputstream) {
  this.outputstream = outputstream;
  this.bytebuffer = bytebuffer.wrap(rawbuffer);
 }
 @override
 public int write(bytebuffer inputbuffer) throws ioexception {
  int inputbytes = inputbuffer.remaining();
  if (inputbytes > bytebuffer.remaining()) {
  dumptofile();
  bytebuffer.clear();
  if (inputbytes > bytebuffer.remaining()) {
   throw new bufferoverflowexception();
  }
  }
  bytebuffer.put(inputbuffer);
  return inputbytes;
 }
 @override
 public boolean isopen() {
  return isopen;
 }
 @override
 public void close() throws ioexception {
  dumptofile();
  isopen = false;
 }
 private void dumptofile() {
  try {
  outputstream.write(rawbuffer, 0, bytebuffer.position());
  } catch (ioexception e) {
  throw new runtimeexception(e);
  }
 }
 }

方法四

利用ffmpeg大法

ffmpeg 由于其丰富的 codec 插件，详细的文档说明，并且与其调试复杂量大的编解码代码(是的，用 mediacodec 实现起来十分啰嗦和繁琐)还是不如调试一行 ffmpeg 命令来的简单。

merge video /audio and retain both audios

可以实现，兼容性强，但由于是软解码，合并速度很慢，忍受不了，而相应的ffmpeg优化还不太了解，囧…….

总结

以上就是这篇文章的全部内容了，希望本文的内容对大家的学习或者工作具有一定的参考学习价值，如果有疑问大家可以留言交流，谢谢大家对的支持。