Android 视频通信，低延时解决方案
程序员文章站
2022-09-04 15:29:48
背景： 由于，项目需要，需要进行视频通信，把a的画面，转给b。 运维部署： APP1：编码摄像头采集的数据，并且发送数据到服务端 APP2：从服务端，拉取数据，并且进行解码显示 服务端：接收APP1提交的数据，发送APP1提交数据到APP2 应用说明： APP1：camera = Camera.op ......
背景：
由于，项目需要，需要进行视频通信，把a的画面，转给b。
运维部署：
app1：编码摄像头采集的数据，并且发送数据到服务端
app2：从服务端，拉取数据，并且进行解码显示
服务端：接收app1提交的数据，发送app1提交数据到app2
应用说明：
app1：camera = camera.open(camera.camerainfo.camera_facing_front);
camera.parameters parameters = camera.getparameters();
parameters.setpreviewformat(imageformat.nv21);
parameters.setpreviewsize(width, height);
// 设置屏幕亮度
parameters.setexposurecompensation(parameters.getmaxexposurecompensation() / 2);
camera.setparameters(parameters);
camera.setdisplayorientation(90);
camera.setpreviewcallback(new camera.previewcallback() {
@override
public void onpreviewframe(byte[] data, camera camera) {
// 采集视频数据，同时记录采集视频的时间点，解码需要（保证视频连续，流畅，且不花屏需要）
stamptime = system.nanotime();
yuv_data = data;
}
});
1 public class avckeyframeencoder {
2 private final static string tag = "meidacodec";
3 private int timeout_usec = 12000;
4
5 private mediacodec mediacodec;
6 int m_width;
7 int m_height;
8 int m_framerate;
9
10 public byte[] configbyte;
11
12 //待解码视频缓冲队列，静态成员！
13 public byte[] yuv_data = null;
14 public long stamptime = 0;
15
16 public avckeyframeencoder(int width, int height, int framerate) {
17 m_width = width;
18 m_height = height;
19 m_framerate = framerate;
20
21 //正常的编码出来是横屏的。因为手机本身采集的数据默认就是横屏的
22 // mediaformat mediaformat = mediaformat.createvideoformat(mime, width, height);
23 //如果你需要旋转90度或者270度，那么需要把宽和高对调。否则会花屏。因为比如你320 x 240，图像旋转90°之后宽高变成了240 x 320。
24 mediaformat mediaformat = mediaformat.createvideoformat("video/avc", width, height);
25 mediaformat.setinteger(mediaformat.key_color_format, mediacodecinfo.codeccapabilities.color_formatyuv420semiplanar);
26 mediaformat.setinteger(mediaformat.key_bit_rate, 125000);
27 mediaformat.setinteger(mediaformat.key_frame_rate, framerate); // 30
28 mediaformat.setinteger(mediaformat.key_i_frame_interval, 1);
29 try {
30 mediacodec = mediacodec.createencoderbytype("video/avc");
31 } catch (ioexception e) {
32 e.printstacktrace();
33 }
34
35 //配置编码器参数
36 mediacodec.configure(mediaformat, null, null, mediacodec.configure_flag_encode);
37
38 //启动编码器
39 mediacodec.start();
40 }
41
42 public void stopencoder() {
43 try {
44 mediacodec.stop();
45 mediacodec.release();
46 } catch (exception e) {
47 e.printstacktrace();
48 }
49 }
50
51 public boolean isruning = false;
52
53 public void startencoderthread(final isavevideo savevideo, final icall callback) {
54 isruning = true;
55 new thread(new runnable() {
56 @override
57 public void run() {
58 byte[] input = null;
59 long pts = 0;
60 while (isruning) {
61 // 访问mainactivity用来缓冲待解码数据的队列
62 if(yuv_data == null){
63 continue;
64 }
65
66 if (yuv_data != null) {
67 //从缓冲队列中取出一帧
68 input = yuv_data;
69 pts = stamptime;
70 yuv_data = null;
71 byte[] yuv420sp = new byte[m_width * m_height * 3 / 2];
72
73 nv21tonv12(input, yuv420sp, m_width, m_height);
74 input = yuv420sp;
75 }
76
77 if (input != null) {
78 try {
79 //编码器输入缓冲区
80 bytebuffer[] inputbuffers = mediacodec.getinputbuffers();
81
82 //编码器输出缓冲区
83 bytebuffer[] outputbuffers = mediacodec.getoutputbuffers();
84 int inputbufferindex = mediacodec.dequeueinputbuffer(-1);
85 if (inputbufferindex >= 0) {
86 bytebuffer inputbuffer = inputbuffers[inputbufferindex];
87 inputbuffer.clear();
88 //把转换后的yuv420格式的视频帧放到编码器输入缓冲区中
89 inputbuffer.put(input);
90 mediacodec.queueinputbuffer(inputbufferindex, 0, input.length, pts, 0);
91 }
92
93 mediacodec.bufferinfo bufferinfo = new mediacodec.bufferinfo();
94 int outputbufferindex = mediacodec.dequeueoutputbuffer(bufferinfo, timeout_usec);
95 while (outputbufferindex >= 0) {
96 //log.i("avcencoder", "get h264 buffer success! flag = "+bufferinfo.flags+",pts = "+bufferinfo.presentationtimeus+"");
97 bytebuffer outputbuffer = outputbuffers[outputbufferindex];
98 byte[] outdata = new byte[bufferinfo.size];
99 outputbuffer.get(outdata);
100 if (bufferinfo.flags == buffer_flag_codec_config) {
101 configbyte = new byte[bufferinfo.size];
102 configbyte = outdata;
103 } else if (bufferinfo.flags == buffer_flag_key_frame) {
104 byte[] keyframe = new byte[bufferinfo.size + configbyte.length];
105 system.arraycopy(configbyte, 0, keyframe, 0, configbyte.length);
106 //把编码后的视频帧从编码器输出缓冲区中拷贝出来
107 system.arraycopy(outdata, 0, keyframe, configbyte.length, outdata.length);
108
109 logs.i("上传i帧 " + keyframe.length);
110 byte[] send_data = new byte[13 + keyframe.length];
111 system.arraycopy(new byte[]{0x01}, 0, send_data, 0, 1);
112 system.arraycopy(intbytes.longtobytes(pts), 0, send_data, 1, 8);
113 system.arraycopy(intbytes.inttobytearray(keyframe.length), 0, send_data, 9, 4);
114 system.arraycopy(keyframe, 0, send_data, 13, keyframe.length);
115 if(savevideo != null){
116 savevideo.savevideodata(send_data);
117 }
118
119 if(callback != null){
120 callback.callback(keyframe, pts);
121 }
122 } else {
123 byte[] send_data = new byte[13 + outdata.length];
124 system.arraycopy(new byte[]{0x02}, 0, send_data, 0, 1);
125 system.arraycopy(intbytes.longtobytes(pts), 0, send_data, 1, 8);
126 system.arraycopy(intbytes.inttobytearray(outdata.length), 0, send_data, 9, 4);
127 system.arraycopy(outdata, 0, send_data, 13, outdata.length);
128 if(savevideo != null){
129 savevideo.savevideodata(send_data);
130 }
131
132 if(callback != null){
133 callback.callback(outdata, pts);
134 }
135 }
136
137 mediacodec.releaseoutputbuffer(outputbufferindex, false);
138 outputbufferindex = mediacodec.dequeueoutputbuffer(bufferinfo, timeout_usec);
139 }
140
141 } catch (throwable t) {
142 t.printstacktrace();
143 break;
144 }
145 }
146 }
147 }
148 }).start();
149 }
150
151 private void nv21tonv12(byte[] nv21, byte[] nv12, int width, int height) {
152 if (nv21 == null || nv12 == null) return;
153 int framesize = width * height;
154 int i = 0, j = 0;
155 system.arraycopy(nv21, 0, nv12, 0, framesize);
156 for (i = 0; i < framesize; i++) {
157 nv12[i] = nv21[i];
158 }
159
160 for (j = 0; j < framesize / 2; j += 2) {
161 nv12[framesize + j - 1] = nv21[j + framesize];
162 }
163
164 for (j = 0; j < framesize / 2; j += 2) {
165 nv12[framesize + j] = nv21[j + framesize - 1];
166 }
167 }
168 }
其中使用到了，接口用于，把采集和编码后的数据，往外部传递，通过线程提交到服务端。或者通过本地解码显示，查看，编码解码时间差。
通过使用 arrayblockingqueue<byte[]> h264queue = new arrayblockingqueue<byte[]>(10); 队列，对接口提交数据，进行暂时保存，在后台对数据，进行解码或提交到服务端。
app2：接入服务端，然后从i帧数据开始拿数据，（且数据是最新的i帧开始保存的数据）。同时需要把，之前采集得到的时间点传给：
mediacodec 对象的 queueinputbuffer 方法的时间戳参数（第四个）。
服务端：一帧一帧接收app1传入数据，对i帧开始的数据进行记录，同时对非i帧开始的数据，进行丢弃。一次只保存一帧内容。读取数据，并且移除已经添加数据，循环发送给app2
public class videodecoder {
private thread mdecodethread;
private mediacodec mcodec;
private boolean mstopflag = false;
private int video_width = 640;
private int video_height = 480;
private int framerate = 25;
private boolean isuseppsandsps = false;
private receivevideothread runthread = null;
public videodecoder(string ip, int port, byte type, int roomid){
runthread = new receivevideothread(ip, port, type, roomid);
new thread(runthread).start();
}
public void initreaddata(surface surface){
try {
//通过多媒体格式名创建一个可用的解码器
mcodec = mediacodec.createdecoderbytype("video/avc");
} catch (ioexception e) {
e.printstacktrace();
}
//初始化编码器
final mediaformat mediaformat = mediaformat.createvideoformat("video/avc", video_width, video_height);
//设置帧率
mediaformat.setinteger(mediaformat.key_frame_rate, framerate);
//https://developer.android.com/reference/android/media/mediaformat.html#key_max_input_size
//设置配置参数，参数介绍 ：
// format 如果为解码器，此处表示输入数据的格式；如果为编码器，此处表示输出数据的格式。
//surface 指定一个surface，可用作decode的输出渲染。
//crypto 如果需要给媒体数据加密，此处指定一个crypto类.
// flags 如果正在配置的对象是用作编码器，此处加上configure_flag_encode 标签。
mcodec.configure(mediaformat, surface, null, 0);
startdecodingthread();
}
private void startdecodingthread() {
mcodec.start();
mdecodethread = new thread(new decodeh264thread());
mdecodethread.start();
}
@requiresapi(api = build.version_codes.lollipop)
private class decodeh264thread implements runnable {
@override
public void run() {
try {
// savedataloop();
decodeloop_new();
} catch (exception e) {
e.printstacktrace();
}
}
private void decodeloop_new() {
// 存放目标文件的数据
bytebuffer[] inputbuffers = mcodec.getinputbuffers();
// 解码后的数据，包含每一个buffer的元数据信息，例如偏差，在相关解码器中有效的数据大小
mediacodec.bufferinfo info = new mediacodec.bufferinfo();
long timeoutus = 1000;
byte[] marker0 = new byte[]{0, 0, 0, 1};
byte[] dummyframe = new byte[]{0x00, 0x00, 0x01, 0x20};
byte[] streambuffer = null;
while (true) {
if(runthread.h264queue.size() > 0){
streambuffer = runthread.h264queue.poll();
}else{
try {
thread.sleep(20);
}catch (exception ex){
}
continue;
}
byte[] time_data = new byte[8];
system.arraycopy(streambuffer, 0, time_data, 0, 8);
long pts = intbytes.bytestolong(time_data);
byte[] video_data = new byte[streambuffer.length - 8];
system.arraycopy(streambuffer, 8, video_data, 0, video_data.length);
streambuffer = video_data;
logs.i("得到 streambuffer " + streambuffer.length + " pts " + pts);
int bytes_cnt = 0;
mstopflag = false;
while (mstopflag == false) {
bytes_cnt = streambuffer.length;
if (bytes_cnt == 0) {
streambuffer = dummyframe;
}
int startindex = 0;
int remaining = bytes_cnt;
while (true) {
if (remaining == 0 || startindex >= remaining) {
break;
}
int nextframestart = kmpmatch(marker0, streambuffer, startindex + 2, remaining);
if (nextframestart == -1) {
nextframestart = remaining;
} else {
}
int inindex = mcodec.dequeueinputbuffer(timeoutus);
if (inindex >= 0) {
bytebuffer bytebuffer = inputbuffers[inindex];
bytebuffer.clear();
bytebuffer.put(streambuffer, startindex, nextframestart - startindex);
//在给指定index的inputbuffer[]填充数据后，调用这个函数把数据传给解码器
mcodec.queueinputbuffer(inindex, 0, nextframestart - startindex, pts, 0);
startindex = nextframestart;
} else {
continue;
}
int outindex = mcodec.dequeueoutputbuffer(info, timeoutus);
if (outindex >= 0) {
//帧控制是不在这种情况下工作，因为没有pts h264是可用的
/*
while (info.presentationtimeus / 1000 > system.currenttimemillis() - startms) {
try {
thread.sleep(100);
} catch (interruptedexception e) {
e.printstacktrace();
}
}
*/
boolean dorender = (info.size != 0);
//对outputbuffer的处理完后，调用这个函数把buffer重新返回给codec类。
// todo:添加处理，保存原始帧数据
if (dorender) {
image image = mcodec.getoutputimage(outindex);
if (image != null) {
// 通过反射
// 发送数据到指定接口
byte[] data = getdatafromimage(image, color_formatnv21);
}
}
mcodec.releaseoutputbuffer(outindex, dorender);
} else {
// log.e(tag, "bbbb");
}
}
mstopflag = true;
}
// logs.i("处理单帧视频耗时：" + (system.currenttimemillis() - c_start));
}
}
}
private static final boolean verbose = false;
private static final long default_timeout_us = 10000;
private static final int color_formati420 = 1;
private static final int color_formatnv21 = 2;
private static boolean isimageformatsupported(image image) {
int format = image.getformat();
switch (format) {
case imageformat.yuv_420_888:
case imageformat.nv21:
case imageformat.yv12:
return true;
}
return false;
}
@requiresapi(api = build.version_codes.lollipop)
private static byte[] getdatafromimage(image image, int colorformat) {
if (colorformat != color_formati420 && colorformat != color_formatnv21) {
throw new illegalargumentexception("only support color_formati420 " + "and color_formatnv21");
}
if (!isimageformatsupported(image)) {
throw new runtimeexception("can't convert image to byte array, format " + image.getformat());
}
rect crop = image.getcroprect();
int format = image.getformat();
int width = crop.width();
int height = crop.height();
image.plane[] planes = image.getplanes();
byte[] data = new byte[width * height * imageformat.getbitsperpixel(format) / 8];
byte[] rowdata = new byte[planes[0].getrowstride()];
if (verbose) logs.i("get data from " + planes.length + " planes");
int channeloffset = 0;
int outputstride = 1;
for (int i = 0; i < planes.length; i++) {
switch (i) {
case 0:
channeloffset = 0;
outputstride = 1;
break;
case 1:
if (colorformat == color_formati420) {
channeloffset = width * height;
outputstride = 1;
} else if (colorformat == color_formatnv21) {
channeloffset = width * height + 1;
outputstride = 2;
}
break;
case 2:
if (colorformat == color_formati420) {
channeloffset = (int) (width * height * 1.25);
outputstride = 1;
} else if (colorformat == color_formatnv21) {
channeloffset = width * height;
outputstride = 2;
}
break;
}
bytebuffer buffer = planes[i].getbuffer();
int rowstride = planes[i].getrowstride();
int pixelstride = planes[i].getpixelstride();
if (verbose) {
logs.i("pixelstride " + pixelstride);
logs.i("rowstride " + rowstride);
logs.i("width " + width);
logs.i("height " + height);
logs.i("buffer size " + buffer.remaining());
}
int shift = (i == 0) ? 0 : 1;
int w = width >> shift;
int h = height >> shift;
buffer.position(rowstride * (crop.top >> shift) + pixelstride * (crop.left >> shift));
for (int row = 0; row < h; row++) {
int length;
if (pixelstride == 1 && outputstride == 1) {
length = w;
buffer.get(data, channeloffset, length);
channeloffset += length;
} else {
length = (w - 1) * pixelstride + 1;
buffer.get(rowdata, 0, length);
for (int col = 0; col < w; col++) {
data[channeloffset] = rowdata[col * pixelstride];
channeloffset += outputstride;
}
}
if (row < h - 1) {
buffer.position(buffer.position() + rowstride - length);
}
}
if (verbose) logs.i("finished reading data from plane " + i);
}
return data;
}
private int kmpmatch(byte[] pattern, byte[] bytes, int start, int remain) {
try {
thread.sleep(30);
} catch (interruptedexception e) {
e.printstacktrace();
}
int[] lsp = computelsptable(pattern);
int j = 0; // number of chars matched in pattern
for (int i = start; i < remain; i++) {
while (j > 0 && bytes[i] != pattern[j]) {
// fall back in the pattern
j = lsp[j - 1]; // strictly decreasing
}
if (bytes[i] == pattern[j]) {
// next char matched, increment position
j++;
if (j == pattern.length)
return i - (j - 1);
}
}
return -1; // not found
}
private int[] computelsptable(byte[] pattern) {
int[] lsp = new int[pattern.length];
lsp[0] = 0; // base case
for (int i = 1; i < pattern.length; i++) {
// start by assuming we're extending the previous lsp
int j = lsp[i - 1];
while (j > 0 && pattern[i] != pattern[j])
j = lsp[j - 1];
if (pattern[i] == pattern[j])
j++;
lsp[i] = j;
}
return lsp;
}
public void stopdecode() {
if(runthread != null){
runthread.stopreceive();
}
}
}
总结：
通过对视频的处理，学习到了，一些处理视频的细节点。同时加深了，依赖导致在实际项目中的使用。to android.