android使用RTP传输Camera数据
程序员文章站
2022-04-24 22:32:41
...
最近公司要做一个一对一的视频传输的项目，要求使用Camera2API传输H264格式的数据。我们这篇文章主要是说怎么实现的，原理部分我会附上其他链接。
那么来看下发送端的主要逻辑。
1.获取Camera数据
2.将获取到的Camera数据转换成YUV420SP格式
3.使用MediaCodec硬编码成H264数据
4.使用RTP协议跟Socket发送Camera数据
1.获取Camera数据
原理理解的部分可参考：ImageReader获得预览数据
下面就是我所实现的，直接贴代码了。
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public class Camera2Helper {
private static final String TAG = "Camera2Helper";
private Context mContext;
private ImageReader mImageReader;
private HandlerThread mBackgroundThread;
private Handler mBackgroundHandler;
private CameraDevice mCameraDevice;
private CaptureRequest.Builder mPreviewRequestBuilder;
private CameraCaptureSession mCaptureSession;
private ImageDataListener mImageDataListener;
private String mCameraId = "0";
private Semaphore mCameraOpenCloseLock = new Semaphore(1);
private final CameraDevice.StateCallback mStateCallback = new CameraDevice.StateCallback(){
@Override
public void onOpened(@NonNull CameraDevice camera) {
Log.i(TAG, "onOpened");
mCameraOpenCloseLock.release();
mCameraDevice = camera;
createCameraPreviewSession();
}
@Override
public void onDisconnected(@NonNull CameraDevice camera) {
Log.i(TAG, "onDisconnected");
mCameraOpenCloseLock.release();
camera.close();
mCameraDevice = null;
}
@Override
public void onError(@NonNull CameraDevice camera, int error) {
Log.e(TAG, "onError openDevice error:" + error);
mCameraOpenCloseLock.release();
camera.close();
mCameraDevice = null;
}
};
public Camera2Helper(Context context) {
this.mContext = context;
}
@SuppressLint("MissingPermission")
@RequiresApi(api = Build.VERSION_CODES.LOLLIPOP)
public void startCamera(int width, int height) {
Log.i(TAG, "start Camera.");
startBackgroundThread();
setUpCameraOutputs(width, height);
CameraManager cameraManager = (CameraManager) mContext.getSystemService(Context.CAMERA_SERVICE);
try {
cameraManager.openCamera(mCameraId, mStateCallback, mBackgroundHandler);
} catch (CameraAccessException e) {
Log.e(TAG, "startCamera error: " + e.getMessage());
}
}
public void playCamera() {
Log.i(TAG, "pauseCamera");
try {
mCaptureSession.setRepeatingRequest(mPreviewRequestBuilder.build(), null, mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
public void pauseCamera() {
Log.i(TAG, "pauseCamera");
try {
mCaptureSession.stopRepeating();
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
public void closeCamera() {
Log.i(TAG, "closeCamera");
try {
mCameraOpenCloseLock.acquire();
if (mCaptureSession != null) {
mCaptureSession.close();
mCaptureSession = null;
}
if (mCameraDevice != null) {
mCameraDevice.close();
mCameraDevice = null;
}
if (mImageReader != null) {
mImageReader.close();
mImageReader = null;
}
} catch (InterruptedException e) {
throw new RuntimeException("Interrupted while trying to lock camera closing.", e);
} finally {
mCameraOpenCloseLock.release();
}
}
/**
* Creates a new {@link CameraCaptureSession} for camera preview.
*/
private void createCameraPreviewSession() {
try {
Surface imageSurface = mImageReader.getSurface();
// We set up a CaptureRequest.Builder with the output Surface.
mPreviewRequestBuilder = mCameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
mPreviewRequestBuilder.addTarget(imageSurface);
mCameraDevice.createCaptureSession(Arrays.asList(imageSurface), new CameraCaptureSession.StateCallback() {
@Override
public void onConfigured(@NonNull CameraCaptureSession session) {
Log.i(TAG, "onConfigured");
// The camera is already closed
if (null == mCameraDevice) {
return;
}
mCaptureSession = session;
mPreviewRequestBuilder.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
try {
mCaptureSession.setRepeatingRequest(mPreviewRequestBuilder.build(), new CameraCaptureSession.CaptureCallback() {
@Override
public void onCaptureStarted(@NonNull CameraCaptureSession session, @NonNull CaptureRequest request, long timestamp, long frameNumber) {
super.onCaptureStarted(session, request, timestamp, frameNumber);
}
}, mBackgroundHandler);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
@Override
public void onConfigureFailed(@NonNull CameraCaptureSession session) {
Log.i(TAG, "onConfigureFailed");
}
},null);
} catch (CameraAccessException e) {
e.printStackTrace();
}
}
private void setUpCameraOutputs(int width, int height) {
Log.i(TAG, "setUpCameraOutputs start");
mImageReader = ImageReader.newInstance(width, height, ImageFormat.YUV_420_888, /*maxImages*/2);
mImageReader.setOnImageAvailableListener(new RTPOnImageAvailableListener(), mBackgroundHandler);
return;
}
/**
* Starts a background thread and its {@link Handler}.
*/
private void startBackgroundThread() {
mBackgroundThread = new HandlerThread("CameraBackground");
mBackgroundThread.start();
mBackgroundHandler = new Handler(mBackgroundThread.getLooper());
}
private class RTPOnImageAvailableListener implements ImageReader.OnImageAvailableListener{
@Override
public void onImageAvailable(ImageReader reader) {
Log.i(TAG, "onImageAvailable");
Image readImage = reader.acquireNextImage();
byte[] data = ImageUtil.getBytesFromImageAsType(readImage, 1);
//byte[] data = ImageUtil.getBytesFromImageAsType(readImage);
//byte[] rotateData = ImageUtil.rotateYUVDegree90(data, readImage.getWidth(), readImage.getHeight());
readImage.close();
/**if (rotateData == null) {
Log.e(TAG, "The rotated data is null.");
}*/
mImageDataListener.OnImageDataListener(data);
}
}
public void setImageDataListener(ImageDataListener listener) {
this.mImageDataListener = listener;
}
public static interface ImageDataListener{
public void OnImageDataListener(byte[] reader);
}
}
2.将获取到的Camera数据转换成YUV420SP格式
原理部分参考：Android: Image类浅析(结合YUV_420_888)
代码：
public static byte[] getBytesFromImageAsType(Image image, int type) {
try {
//Get the source data, if it is YUV format data planes.length = 3
final Image.Plane[] planes = image.getPlanes();
//Data effective width, in general, image width <= rowStride, which is also the reason for byte []. Length <= capacity
// So we only take the width part
int width = image.getWidth();
int height = image.getHeight();
Log.i(TAG, "image width = " + image.getWidth() + "; image height = " + image.getHeight());
//This is used to fill the final YUV data, which requires 1.5 times the picture size, because the YUV ratio is 4: 1: 1
byte[] yuvBytes = new byte[width * height * ImageFormat.getBitsPerPixel(ImageFormat.YUV_420_888) / 8];
//The position to which the target array is filled
int dstIndex = 0;
//Temporary storage of uv data
byte uBytes[] = new byte[width * height / 4];
byte vBytes[] = new byte[width * height / 4];
int uIndex = 0;
int vIndex = 0;
int pixelsStride, rowStride;
for (int i = 0; i < planes.length; i++) {
pixelsStride = planes[i].getPixelStride();
rowStride = planes[i].getRowStride();
ByteBuffer buffer = planes[i].getBuffer();
//The index of the source data. The data of y is continuous in byte. The data of u is shifted to the left. It is assumed that both are even-numbered bits.
byte[] bytes = new byte[buffer.capacity()];
buffer.get(bytes);
int srcIndex = 0;
if (i == 0) {
//Take out all the valid areas of Y directly, or store them as a temporary byte, and then copy it to the next step.
for (int j = 0; j < height; j++) {
System.arraycopy(bytes, srcIndex, yuvBytes, dstIndex, width);
srcIndex += rowStride;
dstIndex += width;
}
} else if (i == 1) {
//Take corresponding data according to pixelsStride
for (int j = 0; j < height / 2; j++) {
for (int k = 0; k < width / 2; k++) {
uBytes[uIndex++] = bytes[srcIndex];
srcIndex += pixelsStride;
}
if (pixelsStride == 2) {
srcIndex += rowStride - width;
} else if (pixelsStride == 1) {
srcIndex += rowStride - width / 2;
}
}
} else if (i == 2) {
//Take corresponding data according to pixelsStride
for (int j = 0; j < height / 2; j++) {
for (int k = 0; k < width / 2; k++) {
vBytes[vIndex++] = bytes[srcIndex];
srcIndex += pixelsStride;
}
if (pixelsStride == 2) {
srcIndex += rowStride - width;
} else if (pixelsStride == 1) {
srcIndex += rowStride - width / 2;
}
}
}
}
//Fill based on required result type
switch (type) {
case YUV420P:
System.arraycopy(uBytes, 0, yuvBytes, dstIndex, uBytes.length);
System.arraycopy(vBytes, 0, yuvBytes, dstIndex + uBytes.length, vBytes.length);
break;
case YUV420SP:
for (int i = 0; i < vBytes.length; i++) {
yuvBytes[dstIndex++] = uBytes[i];
yuvBytes[dstIndex++] = vBytes[i];
}
break;
case NV21:
for (int i = 0; i < vBytes.length; i++) {
yuvBytes[dstIndex++] = vBytes[i];
yuvBytes[dstIndex++] = uBytes[i];
}
break;
}
return yuvBytes;
} catch (final Exception e) {
if (image != null) {
image.close();
}
Log.e(TAG, e.toString());
}
return null;
}
3.使用MediaCodec硬编码成H264数据
原理部分参考：Android MediaCodec 使用说明
代码：
public class AvcEncoder {
private static final String TAG = "AvcEncoder";
private static final String MIME_TYPE = "video/avc";
private MediaCodec mMediaCodec;
private int mWidth;
private int mHeight;
private byte[] mInfo = null;
@SuppressLint("NewApi")
public AvcEncoder(int width, int height, int framerate, int bitrate) {
mWidth = width;
mHeight = height;
Log.i(TAG, "AvcEncoder:" + mWidth + "+" + mHeight);
try {
mMediaCodec = MediaCodec.createEncoderByType(MIME_TYPE);
MediaFormat mediaFormat = MediaFormat.createVideoFormat(MIME_TYPE, width, height);
mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitrate);
mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, framerate);
mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar);
mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
mMediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
mMediaCodec.start();
} catch (IOException e) {
e.printStackTrace();
}
}
@SuppressLint("NewApi")
public int offerEncoder(byte[] input, byte[] output) {
Log.i(TAG, "offerEncoder:"+input.length+"+"+output.length);
int pos = 0;
try {
ByteBuffer[] inputBuffers = mMediaCodec.getInputBuffers();
ByteBuffer[] outputBuffers = mMediaCodec.getOutputBuffers();
int inputBufferIndex = mMediaCodec.dequeueInputBuffer(-1);
if (inputBufferIndex >= 0) {
ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
inputBuffer.clear();
inputBuffer.put(input);
mMediaCodec.queueInputBuffer(inputBufferIndex, 0, input.length, 0, 0);
}
MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
int outputBufferIndex = mMediaCodec.dequeueOutputBuffer(bufferInfo,0);
while (outputBufferIndex >= 0) {
ByteBuffer outputBuffer = outputBuffers[outputBufferIndex];
byte[] outData = new byte[bufferInfo.size];
outputBuffer.get(outData);
if(mInfo != null){
System.arraycopy(outData, 0, output, pos, outData.length);
pos += outData.length;
}else{ //Save pps sps only in the first frame, save it for later use
ByteBuffer spsPpsBuffer = ByteBuffer.wrap(outData);
if (spsPpsBuffer.getInt() == 0x00000001) {
mInfo = new byte[outData.length];
System.arraycopy(outData, 0, mInfo, 0, outData.length);
}else {
return -1;
}
}
if(output[4] == 0x65) { //key frame When the encoder generates the key frame, there is only 00 00 00 01 65 without pps sps.
System.arraycopy(mInfo, 0, output, 0, mInfo.length);
System.arraycopy(outData, 0, output, mInfo.length, outData.length);
}
mMediaCodec.releaseOutputBuffer(outputBufferIndex, false);
outputBufferIndex = mMediaCodec.dequeueOutputBuffer(bufferInfo, 0);
}
} catch (Throwable t) {
t.printStackTrace();
}
Log.i(TAG, "offerEncoder+pos:" + pos);
return pos;
}
@SuppressLint("NewApi")
public void close() {
try {
mMediaCodec.stop();
mMediaCodec.release();
} catch (Exception e){
e.printStackTrace();
}
}
}
4.使用RTP协议跟Socket发送Camera数据
RTP协议相关原理部分，可参考下面:RTP相关
代码：
/**
RTP packet header
Bit offset[b] 0-1 2 3 4-7 8 9-15 16-31
0 Version P X CC M PT Sequence Number 31
32 Timestamp 63
64 SSRC identifier 95
*/
public void addPacket(byte[] prefixData, byte[] data, int offset, int size, long timeUs) throws IOException{
ByteBuffer buffer = ByteBuffer.allocate(500000);
buffer.put((byte)(2 << 6));
buffer.put((byte)(payloadType));
buffer.putShort(sequenceNumber++);
buffer.putInt((int)(timeUs));
buffer.putInt(12345678);
buffer.putInt(size);
if (prefixData != null) {
buffer.put(prefixData);
}
buffer.put(data, offset, size);
sendPacket(buffer, buffer.position());
}
protected void sendPacket(ByteBuffer buffer, int size) throws IOException{
socket.sendPacket(buffer.array(), 0, size);
buffer.clear();
}
注意：
1.这个demo没有动态申请权限，运行的时候，需要手动从设置中打开。
2.IP地址也是写死的，需要你动态获取到对方的IP。
3.我在网上看的时候，发现很多RTP协议封装数据的时候，会将一帧拆成好几个包发送，但这个demo是直接将一帧封装成包发过去的。应该是底层协议会自动拆包组包。
参考博客：android硬编码h264数据，并使用rtp推送数据流，实现一个简单的直播-MediaCodec（一）
我的项目地址：gitHub
上一篇： Django动态展示Pyecharts图表数据的几种方法
下一篇： php二维数组根据某个字段去重
推荐阅读
-
Android数据共享 sharedPreferences 的使用方法
-
在Android系统中使用gzip进行数据传递实例代码
-
详解Android数据存储—使用SQLite数据库
-
Android 使用Intent传递数据的实现思路与代码
-
Android 使用ContentObserver监听数据库内容是否更改
-
使用post方法实现json往返传输数据的方法
-
Android 数据存储之 FileInputStream 工具类及FileInputStream类的使用
-
Android变形(Transform)之Camera使用介绍
-
Android使用文件进行数据存储的方法
-
解析Android中string-array数据源的简单使用