前言
最近工作中遇到了音视频处理的需求,Android下音视频合成,在当前调研方案中主要有三大类方法:MediaMux硬解码,mp4parser,FFmepg。三种方法均可实现,但是也有不同的局限和问题,先将实现和问题记录于此,便于之后的总结学习。下面话不多说了,来一起看看详细的介绍吧。
方法一(Fail)
利用MediaMux实现音视频的合成。
效果:可以实现音视频的合并,利用Android原生的VideoView和SurfaceView播放正常,大部分的播放器也播放正常,但是,但是,在上传Youtube就会出现问题:音频不连续,分析主要是上传Youtube时会被再次的压缩,可能在压缩的过程中出现音频的帧率出现问题。
分析:在MediaCodec.BufferInfo的处理中,时间戳presentationTimeUs出现问题,导致Youtube的压缩造成音频的紊乱。
public static void muxVideoAndAudio(String videoPath,String audioPath,String muxPath) { try { MediaExtractor videoExtractor = new MediaExtractor(); videoExtractor.setDataSource(videoPath); MediaFormat videoFormat = null; int videoTrackIndex = -1; int videoTrackCount = videoExtractor.getTrackCount(); for (int i = 0; i < videoTrackCount; i++) { videoFormat = videoExtractor.getTrackFormat(i); String mimeType = videoFormat.getString(MediaFormat.KEY_MIME); if (mimeType.startsWith("video/")) { videoTrackIndex = i; break; } } MediaExtractor audioExtractor = new MediaExtractor(); audioExtractor.setDataSource(audioPath); MediaFormat audioFormat = null; int audioTrackIndex = -1; int audioTrackCount = audioExtractor.getTrackCount(); for (int i = 0; i < audioTrackCount; i++) { audioFormat = audioExtractor.getTrackFormat(i); String mimeType = audioFormat.getString(MediaFormat.KEY_MIME); if (mimeType.startsWith("audio/")) { audioTrackIndex = i; break; } } videoExtractor.selectTrack(videoTrackIndex); audioExtractor.selectTrack(audioTrackIndex); MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo(); MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo(); Mediamuxer mediamuxer = new Mediamuxer(muxPath,Mediamuxer.OutputFormat.muxer_OUTPUT_MPEG_4); int writeVideoTrackIndex = mediamuxer.addTrack(videoFormat); int writeAudioTrackIndex = mediamuxer.addTrack(audioFormat); mediamuxer.start(); ByteBuffer byteBuffer = ByteBuffer.allocate(500 * 1024); long sampleTime = 0; { videoExtractor.readSampleData(byteBuffer,0); if (videoExtractor.getSampleFlags() == MediaExtractor.SAMPLE_FLAG_SYNC) { videoExtractor.advance(); } videoExtractor.readSampleData(byteBuffer,0); long secondTime = videoExtractor.getSampleTime(); videoExtractor.advance(); long thirdTime = videoExtractor.getSampleTime(); sampleTime = Math.abs(thirdTime - secondTime); } videoExtractor.unselectTrack(videoTrackIndex); videoExtractor.selectTrack(videoTrackIndex); while (true) { int readVideoSampleSize = videoExtractor.readSampleData(byteBuffer,0); if (readVideoSampleSize < 0) { break; } videoBufferInfo.size = readVideoSampleSize; videoBufferInfo.presentationTimeUs += sampleTime; videoBufferInfo.offset = 0; //noinspection WrongConstant videoBufferInfo.flags = MediaCodec.BUFFER_FLAG_SYNC_FRAME;//videoExtractor.getSampleFlags() mediamuxer.writeSampleData(writeVideoTrackIndex,byteBuffer,videoBufferInfo); videoExtractor.advance(); } while (true) { int readAudioSampleSize = audioExtractor.readSampleData(byteBuffer,0); if (readAudioSampleSize < 0) { break; } audioBufferInfo.size = readAudioSampleSize; audioBufferInfo.presentationTimeUs += sampleTime; audioBufferInfo.offset = 0; //noinspection WrongConstant audioBufferInfo.flags = MediaCodec.BUFFER_FLAG_SYNC_FRAME;// videoExtractor.getSampleFlags() mediamuxer.writeSampleData(writeAudioTrackIndex,audioBufferInfo); audioExtractor.advance(); } mediamuxer.stop(); mediamuxer.release(); videoExtractor.release(); audioExtractor.release(); } catch (IOException e) { e.printstacktrace(); } }
方法二(Success)
public static void muxVideoAudio(String videoFilePath,String audioFilePath,String outputFile) { try { MediaExtractor videoExtractor = new MediaExtractor(); videoExtractor.setDataSource(videoFilePath); MediaExtractor audioExtractor = new MediaExtractor(); audioExtractor.setDataSource(audioFilePath); Mediamuxer muxer = new Mediamuxer(outputFile,Mediamuxer.OutputFormat.muxer_OUTPUT_MPEG_4); videoExtractor.selectTrack(0); MediaFormat videoFormat = videoExtractor.getTrackFormat(0); int videoTrack = muxer.addTrack(videoFormat); audioExtractor.selectTrack(0); MediaFormat audioFormat = audioExtractor.getTrackFormat(0); int audioTrack = muxer.addTrack(audioFormat); LogUtil.d(TAG,"Video Format " + videoFormat.toString()); LogUtil.d(TAG,"Audio Format " + audioFormat.toString()); boolean sawEOS = false; int frameCount = 0; int offset = 100; int sampleSize = 256 * 1024; ByteBuffer videoBuf = ByteBuffer.allocate(sampleSize); ByteBuffer audioBuf = ByteBuffer.allocate(sampleSize); MediaCodec.BufferInfo videoBufferInfo = new MediaCodec.BufferInfo(); MediaCodec.BufferInfo audioBufferInfo = new MediaCodec.BufferInfo(); videoExtractor.seekTo(0,MediaExtractor.SEEK_TO_CLOSEST_SYNC); audioExtractor.seekTo(0,MediaExtractor.SEEK_TO_CLOSEST_SYNC); muxer.start(); while (!sawEOS) { videoBufferInfo.offset = offset; videoBufferInfo.size = videoExtractor.readSampleData(videoBuf,offset); if (videoBufferInfo.size < 0 || audioBufferInfo.size < 0) { sawEOS = true; videoBufferInfo.size = 0; } else { videoBufferInfo.presentationTimeUs = videoExtractor.getSampleTime(); //noinspection WrongConstant videoBufferInfo.flags = videoExtractor.getSampleFlags(); muxer.writeSampleData(videoTrack,videoBuf,videoBufferInfo); videoExtractor.advance(); frameCount++; } } boolean sawEOS2 = false; int frameCount2 = 0; while (!sawEOS2) { frameCount2++; audioBufferInfo.offset = offset; audioBufferInfo.size = audioExtractor.readSampleData(audioBuf,offset); if (videoBufferInfo.size < 0 || audioBufferInfo.size < 0) { sawEOS2 = true; audioBufferInfo.size = 0; } else { audioBufferInfo.presentationTimeUs = audioExtractor.getSampleTime(); //noinspection WrongConstant audioBufferInfo.flags = audioExtractor.getSampleFlags(); muxer.writeSampleData(audioTrack,audioBuf,audioBufferInfo); audioExtractor.advance(); } } muxer.stop(); muxer.release(); LogUtil.d(TAG,"Output: "+outputFile); } catch (IOException e) { LogUtil.d(TAG,"mixer Error 1 " + e.getMessage()); } catch (Exception e) { LogUtil.d(TAG,"mixer Error 2 " + e.getMessage()); } }
方法三
利用mp4parser实现
mp4parser是一个视频处理的开源工具箱,由于mp4parser里的方法都依靠工具箱里的一些内容,所以需要将这些内容打包成jar包,放到自己的工程里,才能对mp4parser的方法进行调用。
compile “com.googlecode.mp4parser:isoparser:1.1.21”
问题:上传Youtube压缩后,视频数据丢失严重,大部分就只剩下一秒钟的时长,相当于把视频变成图片了,
public boolean mux(String videoFile,String audioFile,final String outputFile) { if (isstopMux) { return false; } Movie video; try { video = MovieCreator.build(videoFile); } catch (RuntimeException e) { e.printstacktrace(); return false; } catch (IOException e) { e.printstacktrace(); return false; } Movie audio; try { audio = MovieCreator.build(audioFile); } catch (IOException e) { e.printstacktrace(); return false; } catch (NullPointerException e) { e.printstacktrace(); return false; } Track audioTrack = audio.getTracks().get(0); video.addTrack(audioTrack); Container out = new DefaultMp4Builder().build(video); FileOutputStream fos; try { fos = new FileOutputStream(outputFile); } catch (FileNotFoundException e) { e.printstacktrace(); return false; } BufferedWritableFileByteChannel byteBufferByteChannel = new BufferedWritableFileByteChannel(fos); try { out.writeContainer(byteBufferByteChannel); byteBufferByteChannel.close(); fos.close(); if (isstopMux) { return false; } runOnUiThread(new Runnable() { @Override public void run() { mCustomeProgressDialog.setProgress(100); goShareActivity(outputFile); // FileUtils.insertMediaDB(AddAudiosActivity.this,outputFile);// } }); } catch (IOException e) { e.printstacktrace(); if (mCustomeProgressDialog.isShowing()) { mCustomeProgressDialog.dismiss(); } ToastUtil.showShort(getString(R.string.process_Failed)); return false; } return true; } private static class BufferedWritableFileByteChannel implements WritableByteChannel { private static final int BUFFER_CAPACITY = 2000000; private boolean isOpen = true; private final OutputStream outputStream; private final ByteBuffer byteBuffer; private final byte[] rawBuffer = new byte[BUFFER_CAPACITY]; private BufferedWritableFileByteChannel(OutputStream outputStream) { this.outputStream = outputStream; this.byteBuffer = ByteBuffer.wrap(rawBuffer); } @Override public int write(ByteBuffer inputBuffer) throws IOException { int inputBytes = inputBuffer.remaining(); if (inputBytes > byteBuffer.remaining()) { dumpToFile(); byteBuffer.clear(); if (inputBytes > byteBuffer.remaining()) { throw new BufferOverflowException(); } } byteBuffer.put(inputBuffer); return inputBytes; } @Override public boolean isopen() { return isOpen; } @Override public void close() throws IOException { dumpToFile(); isOpen = false; } private void dumpToFile() { try { outputStream.write(rawBuffer,byteBuffer.position()); } catch (IOException e) { throw new RuntimeException(e); } } }
方法四
利用FFmpeg大法
FFmpeg 由于其丰富的 codec 插件,详细的文档说明,并且与其调试复杂量大的编解码代码(是的,用 MediaCodec 实现起来十分潞头彼)还是不如调试一行 ffmpeg 命令来的简单。
Merge Video /Audio and retain both audios
可以实现,兼容性强,但由于是软解码,合并速度很慢,忍受不了,而相应的FFmpeg优化还不太了解,濉.
总结
以上就是这篇文章的全部内容了,希望本文的内容对大家的学习或者工作具有一定的参考学习价值,如果有疑问大家可以留言交流,谢谢大家对编程小技巧的支持。
版权声明:本文内容由互联网用户自发贡献,该文观点与技术仅代表作者本人。本站仅提供信息存储空间服务,不拥有所有权,不承担相关法律责任。如发现本站有涉嫌侵权/违法违规的内容, 请发送邮件至 dio@foxmail.com 举报,一经查实,本站将立刻删除。