现在我们需要实现一个程序，该程序调用摄像头，并将图像显示到屏幕（安卓APP）。
# 一般开发工作
program: camera -> process -> display
# 我们的思路
program_1(linux_system): camera(v4l2) -> process -> rtsp_send
program_2(user_android): rtsp_receive -> display安卓运行C/C++程序一般有如下几种方法：JNI、QT、以及直接编译C/C++程序。我选择第三种方式来实现这个程序。为此，我们需要进行如下工作：
- 下载Mediamtx推流。
- 配置交叉编译工具链。
- 移植FFmpeg。
- 编写C/C++应用程序。
关键词： Android、Linux like、C/C++、V4L2、FFmpeg
注意： 本章节的FFmpeg库使用静态链接。
一、交叉编译工具链
根据自己的架构下载对应的工具链：developer.arm.com。我目标机器的CPU架构是ARM64-V8，Host编译环境是X86_64，所以我使用的编译器是gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu。
我们将其放到Host上解压即可，是否需要配置环境变量看个人习惯即可。
二、编译FFmpeg
自行在FFmpeg.org或者git上下载FFmpeg，我使用的版本是4.3.1。
进入FFmpeg目录，创建一个脚本用来控制配置configure文件。
#!/bin/bash
export TOOLS=/home/xjt/Gogs/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu
export SYSROOT=${TOOLS}/aarch64-none-linux-gnu/libc
# export PREFIX=./linux/arm64
export LD=${TOOLS}/bin/aarch64-none-linux-gnu-ld
export AR=${TOOLS}/bin/aarch64-none-linux-gnu-ar
export RANLIB=${TOOLS}/bin/aarch64-none-linux-gnu-ranlib
function build_lib
{
./configure \
--disable-shared \
--enable-static \
--prefix=$PREFIX \
--cross-prefix=${TOOLS}/bin/aarch64-linux-gnu- \
--cc=${TOOLS}/bin/aarch64-none-linux-gnu-gcc \
--nm=${TOOLS}/bin/aarch64-none-linux-gnu-g++ \
--ld=${LD} \
--ar=${AR} \
--ranlib=${RANLIB} \
--target-os=linux \
--arch=arm64 \
--sysroot=$SYSROOT \
--enable-runtime-cpudetect \
--enable-cross-compile \
--enable-pic \
--enable-gpl \
--enable-nonfree \
--enable-yasm \
--enable-muxer=mpeg4 \
--enable-muxer=rtsp \
--enable-encoder=mpeg4 \
--enable-decoder=mpeg4 \
--enable-network \
--enable-protocol=tcp \
--enable-pthreads \
--disable-ffmpeg \
--disable-ffplay \
--disable-ffprobe \
--disable-avdevice \
--disable-doc \
--extra-ldflags="-L${SYSROOT}/libc/lib -L/home/xjt/Gogs/x264/install/lib -lc" \
--extra-cflags="-I${SYSROOT}/libc/usr/include -I/home/xjt/Gogs/x264/install/include -Wfatal-errors -Wno-deprecated"
# --enable-libx264 \
# --extra-libs=-ldl
}
build_lib注意：
- 这里如果需要静态链接
x264，则需要在后面加上--extra-libs=-ldl。 - RTSP需要
--enable-network、--enable-protocol=tcp、--enable-muxer=rtsp。
接下来配置configure、make、make install即可。
./build.sh
make -j8
make install之后在PREFIX目录下找到输出，一般有include、lib和share。
三、应用程序
应用程序目录如下：
.
├── build
│ ├── bin
│ ├── CMakeCache.txt
│ ├── CMakeFiles
│ ├── cmake_install.cmake
│ └── Makefile
├── CMakeLists.txt
├── lib
│ ├── ffmpeg
│ └── x264
└── src
├── CMakeLists.txt
├── main.cpp
├── v4l2_stream.c
└── v4l2_stream.h3.1 ./CMakeLists.txt
PROJECT(RK3568_APP)
CMAKE_MINIMUM_REQUIRED(VERSION 3.5)
SET(COMPILER_PATH "/home/xjt/Gogs/gcc-arm-10.3-2021.07-x86_64-aarch64-none-linux-gnu")
SET(CMAKE_C_COMPILER ${COMPILER_PATH}/bin/aarch64-none-linux-gnu-gcc)
SET(CMAKE_CXX_COMPILER ${COMPILER_PATH}/bin/aarch64-none-linux-gnu-g++)
SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -s -O3 -lc -ldl -lrt -lm")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -s -O3 -lstdc++ -lc -ldl -lrt -lm")
# INCLUDE_DIRECTORIES(./lib/x264/include)
# LINK_DIRECTORIES(./lib/x264/lib)
INCLUDE_DIRECTORIES(./lib/ffmpeg/include)
LINK_DIRECTORIES(./lib/ffmpeg/lib)
SET(CMAKE_EXE_LINKER_FLAGS "-static")
ADD_SUBDIRECTORY(src bin)3.2 ./src/CMakeLists.txt
FILE(
GLOB_RECURSE SRC_LIST
./*.c
./*.cpp
)
# Exe output path
SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR/bin})
ADD_EXECUTABLE(demo ${SRC_LIST})
TARGET_LINK_LIBRARIES(
demo
# x264;
avformat;
swscale;
avcodec;
avutil;
swresample;
pthread;
)3.3 ./src/main.cpp
#include <chrono>
#include <thread>
#include <iostream>
#include <cstdio>
#ifdef __cplusplus
extern "C" {
#endif
#include <stdio.h>
#include <libavcodec/avcodec.h>
#include <libavformat/avformat.h>
#include <libavutil/opt.h>
#include <libavutil/time.h>
#include <libswscale/swscale.h>
#include <libavutil/avutil.h>
#include <libavutil/imgutils.h>
#include <libswresample/swresample.h>
#include <pthread.h>
#include "v4l2_stream.h"
#ifdef __cplusplus
}
#endif
// using namespace std;
static int video_is_eof;
#define STREAM_FRAME_RATE 60
#define STREAM_PIX_FMT AV_PIX_FMT_YUV420P /* default pix_fmt */
#define VIDEO_CODEC_ID AV_CODEC_ID_MPEG4
/* video output */
static AVFrame *frame;
static AVPicture src_picture, dst_picture;
uint8_t input[640 * 480 * 2];
uint8_t output[640 * 480 * 2];
uint8_t output_128[640 * 480];
/* Add an output stream. */
static AVStream *add_stream(AVFormatContext *oc, AVCodec **codec, enum AVCodecID codec_id)
{
AVCodecContext *c;
AVStream *st;
/* find the encoder */
*codec = avcodec_find_encoder(codec_id);
if (!(*codec)) {
av_log(NULL, AV_LOG_ERROR, "Could not find encoder for '%s'.\n", avcodec_get_name(codec_id));
}
else {
st = avformat_new_stream(oc, *codec);
if (!st) {
av_log(NULL, AV_LOG_ERROR, "Could not allocate stream.\n");
}
else {
st->id = oc->nb_streams - 1;
st->time_base.den = STREAM_FRAME_RATE;
st->time_base.num = 1;
c = st->codec;
c->codec_id = codec_id;
c->bit_rate = 16000000;
c->width = 640;
c->height = 480;
c->time_base.den = STREAM_FRAME_RATE;
c->time_base.num = 1;
c->gop_size = 15;
c->pix_fmt = STREAM_PIX_FMT;
}
}
return st;
}
static int open_video(AVFormatContext *oc, AVCodec *codec, AVStream *st)
{
int ret;
AVCodecContext *c = st->codec;
// AVCodecContext *c = avcodec_alloc_context3(codec);
/* open the codec */
ret = avcodec_open2(c, codec, NULL);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Could not open video codec.\n", avcodec_get_name(c->codec_id));
}
else {
/* allocate and init a re-usable frame */
frame = av_frame_alloc();
if (!frame) {
av_log(NULL, AV_LOG_ERROR, "Could not allocate video frame.\n");
ret = -1;
}
else {
frame->format = c->pix_fmt;
frame->width = c->width;
frame->height = c->height;
/* Allocate the encoded raw picture. */
ret = avpicture_alloc(&dst_picture, c->pix_fmt, c->width, c->height);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Could not allocate picture.\n");
}
else {
/* copy data and linesize picture pointers to frame */
*((AVPicture *)frame) = dst_picture;
}
}
}
return ret;
}
int VIDEOWRAP_YUYV422_to_YUV420P(int imgWidth, int imgHeight, uint8_t* inputData, uint8_t* outputData, int inputLength)
{
struct SwsContext* ctx = sws_getContext(imgWidth, imgHeight, AV_PIX_FMT_YUYV422,
imgWidth, imgHeight, AV_PIX_FMT_YUV420P,
SWS_BICUBIC, NULL, NULL, NULL);
if (!ctx) {
fprintf(stderr, "Failed to allocate SwsContext.\n");
return -1; // Error handling
}
struct AVFrame* input_pFrame = av_frame_alloc();
struct AVFrame* output_pFrame = av_frame_alloc();
if (!input_pFrame || !output_pFrame) {
fprintf(stderr, "Failed to allocate AVFrame.\n");
if (input_pFrame) av_frame_free(&input_pFrame);
if (output_pFrame) av_frame_free(&output_pFrame);
sws_freeContext(ctx);
return -1; // Error handling
}
av_image_fill_arrays(input_pFrame->data, input_pFrame->linesize, inputData, AV_PIX_FMT_YUYV422, imgWidth, imgHeight, 1);
av_image_fill_arrays(output_pFrame->data, output_pFrame->linesize, outputData, AV_PIX_FMT_YUV420P, imgWidth, imgHeight, 1);
sws_scale(ctx,
(const uint8_t**)input_pFrame->data, input_pFrame->linesize, 0, imgHeight,
output_pFrame->data, output_pFrame->linesize);
#if 0
FILE *fl;
fl = fopen("img.yuv", "w");
if (NULL == fl)
{
fprintf(stderr, "open write file failed.");
}
fwrite(outputData, inputLength * 1.5, 1, fl);
fclose(fl);
#endif
// Release allocated resources
av_frame_free(&input_pFrame);
av_frame_free(&output_pFrame);
sws_freeContext(ctx);
return 0;
}
/* Prepare a dummy image. */
static void fill_yuv_image(AVPicture *pict, int frame_index, int width, int height)
{
int x, y, i;
i = frame_index;
memcpy(input, v4l2_sp_globalBuffer[v4l2_sp_globalBufferIndex], v4l2_sp_globalBufferLength);
VIDEOWRAP_YUYV422_to_YUV420P(
640, 480,
input,
output,
v4l2_sp_globalBufferLength
);
memcpy(pict->data[0], output, 640 * 480);
memcpy(pict->data[1], output_128, 640 * 480 * 0.5);
memcpy(pict->data[2], output_128, 640 * 480 * 0.5);
}
static int write_video_frame(AVFormatContext *oc, AVStream *st, int64_t frameCount)
{
int ret = 0;
AVCodecContext *c = st->codec;
// AVCodecContext *c = avcodec_alloc_context3(st->codecpar);
fill_yuv_image(&dst_picture, frameCount, c->width, c->height);
AVPacket pkt = { 0 };
int got_packet;
av_init_packet(&pkt);
/* encode the image */
frame->pts = frameCount;
ret = avcodec_encode_video2(c, &pkt, frame, &got_packet);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error encoding video frame.\n");
}
else {
if (got_packet) {
pkt.stream_index = st->index;
pkt.pts = av_rescale_q_rnd(pkt.pts, c->time_base, st->time_base, AVRounding(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
ret = av_write_frame(oc, &pkt);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Error while writing video frame.\n");
}
}
}
av_packet_unref(&pkt);
return ret;
}
int main(int argc, char* argv[])
{
printf("starting...\n");
pthread_t v4l2_thread;
pthread_create(&v4l2_thread, NULL, ThreadEntryPoint, NULL);
pthread_detach(v4l2_thread);
memset(output_128, 128, sizeof(output_128));
const char *url = "rtsp://192.168.50.84:8554/stream";
AVFormatContext *outContext;
AVStream *video_st;
AVCodec *video_codec;
int ret = 0;
int64_t frameCount = 0;
av_log_set_level(AV_LOG_DEBUG);
av_register_all();
avformat_network_init();
avformat_alloc_output_context2(&outContext, NULL, "rtsp", url);
if (!outContext) {
av_log(NULL, AV_LOG_FATAL, "Could not allocate an output context for '%s'.\n", url);
return 0;
}
if (!outContext->oformat) {
av_log(NULL, AV_LOG_FATAL, "Could not create the output format for '%s'.\n", url);
return 0;
}
video_st = add_stream(outContext, &video_codec, VIDEO_CODEC_ID);
/* Now that all the parameters are set, we can open the video codec and allocate the necessary encode buffers. */
if (video_st) {
av_log(NULL, AV_LOG_DEBUG, "Video stream codec %s.\n ", avcodec_get_name(video_st->codec->codec_id));
ret = open_video(outContext, video_codec, video_st);
if (ret < 0) {
av_log(NULL, AV_LOG_FATAL, "Open video stream failed.\n");
return 0;
}
}
else {
av_log(NULL, AV_LOG_FATAL, "Add video stream for the codec '%s' failed.\n", avcodec_get_name(VIDEO_CODEC_ID));
return 0;
}
av_dump_format(outContext, 0, url, 1);
ret = avformat_write_header(outContext, NULL);
if (ret != 0) {
av_log(NULL, AV_LOG_ERROR, "Failed to connect to RTSP server for '%s'.\n", url);
return 0;
}
while (video_st) {
frameCount++;
ret = write_video_frame(outContext, video_st, frameCount);
if (ret < 0) {
av_log(NULL, AV_LOG_ERROR, "Write video frame failed.\n", url);
return 0;
}
}
avcodec_close(video_st->codec);
av_free(src_picture.data[0]);
av_free(dst_picture.data[0]);
av_frame_free(&frame);
avformat_free_context(outContext);
printf("finished.\n");
return 0;
}3.4 ./src/v4l2_stream.h
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include <fcntl.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/ioctl.h>
#include <asm/types.h>
#include <sys/mman.h>
#include <linux/videodev2.h>
#include <errno.h>
#include <stdint.h>
#include <stdbool.h>
#ifdef __cplusplus
extern "C" {
#endif
#define _Video_Device_SP_ "/dev/video9"
#define _V4L2_SP_PIX_FMT_ V4L2_PIX_FMT_YUYV
#define _V4L2_SP_BUF_TYPE_ V4L2_BUF_TYPE_VIDEO_CAPTURE
#define _V4L2_SP_BUF_REQ_COUNT_ 60
extern const int v4l2_sp_capture_width;
extern const int v4l2_sp_capture_height;
// device number for v4l2 single-planar
extern int v4l2_sp_fd;
// global memory for mmap v4l2 query buffer, 5 block
extern uint8_t* v4l2_sp_globalBuffer[_V4L2_SP_BUF_REQ_COUNT_];
// global memory buffer length
extern int v4l2_sp_globalBufferLength;
// to specify which memory is currently available
extern int v4l2_sp_globalBufferIndex;
// to specify globalBuffer length
extern int v4l2_sp_globalBufferLength;
// stop capture or not
extern bool v4l2_sp_captureLoopFlag;
int V4L2_SP_Streaming();
void* ThreadEntryPoint(void* arg);
#ifdef __cplusplus
}
#endif3.5 ./src/v4l2_stream.c
#include "v4l2_stream.h"
const int v4l2_sp_capture_width = 640;
const int v4l2_sp_capture_height = 480;
int v4l2_sp_fd;
uint8_t* v4l2_sp_globalBuffer[_V4L2_SP_BUF_REQ_COUNT_];
int v4l2_sp_globalBufferLength;
int v4l2_sp_globalBufferIndex;
bool v4l2_sp_captureLoopFlag = true;
int V4L2_SP_Streaming()
{
/* Step 0 : Pre-Define */
int retval = 0; // V4L2_SP_Streaming return value
int ret = 0; // ioctl check
#if Print_Debug_Info
printf("V4L2_SP_Streaming(): Step 1 Open Device\n");
#endif
/* Step 1 : Open Device */
v4l2_sp_fd = open(_Video_Device_SP_, O_RDWR);
if (v4l2_sp_fd == -1)
{
perror("V4L2_SP_Streaming() open device");
retval = 1;
goto out_return;
}
#if Print_Debug_Info
printf("V4L2_SP_Streaming(): Step 2 Setting Capture Format\n");
#endif
/* Step 2: Setting Capture Format */
struct v4l2_format format;
format.type = _V4L2_SP_BUF_TYPE_;
format.fmt.pix.width = v4l2_sp_capture_width;
format.fmt.pix.height = v4l2_sp_capture_height;
format.fmt.pix.pixelformat = _V4L2_SP_PIX_FMT_;
ret = ioctl(v4l2_sp_fd, VIDIOC_S_FMT, &format); // set FMT
if (ret == -1)
{
perror("V4L2_SP_Streaming() setting format");
retval = 2;
goto error_close;
}
#if Print_Debug_Info
printf("V4L2_SP_Streaming(): Step 3 Checking FMT Setting\n");
#endif
/* Step 3: Checking FMT Setting */
ret = ioctl(v4l2_sp_fd, VIDIOC_G_FMT, &format); // get FMT
if (ret == -1)
{
perror("V4L2_SP_Streaming() checking format");
retval = 3;
goto error_close;
}
if (format.fmt.pix.pixelformat == _V4L2_SP_PIX_FMT_) // set FMT == get FMT ?
{
printf("ioctl VIDIOC_S_FMT sucessful\n");
}
else
{
printf("ioctl VIDIOC_S_FMT failed\n");
retval = 4;
goto error_close;
}
#if Print_Debug_Info
printf("V4L2_SP_Streaming(): Step 4 Request Buffer\n");
#endif
/* Step 4: Request Buffer */
struct v4l2_requestbuffers reqbuf;
reqbuf.count = _V4L2_SP_BUF_REQ_COUNT_;
reqbuf.type = _V4L2_SP_BUF_TYPE_;
reqbuf.memory = V4L2_MEMORY_MMAP;
ret = ioctl(v4l2_sp_fd, VIDIOC_REQBUFS, &reqbuf);
if (-1 == ret)
{
perror("V4L2_SP_Streaming() reqeuset buff");
retval = 1;
goto error_close;
}
#if Print_Debug_Info
printf("V4L2_SP_Streaming(): Step 5 mmap Buffer\n");
#endif
/* Step 5: mmap Buffer */
struct v4l2_buffer buff;
buff.type = _V4L2_SP_BUF_TYPE_;
buff.memory = V4L2_MEMORY_MMAP;
for (int i = 0; i < _V4L2_SP_BUF_REQ_COUNT_; ++i)
{
// query buf
buff.index = i;
ret = ioctl(v4l2_sp_fd, VIDIOC_QUERYBUF, &buff);
if (ret == -1)
{
perror("V4L2_SP_Streaming() query");
retval = 2;
goto error_close;
}
printf("buf[%d]: len = %d offset: %d\n", i, buff.length, buff.m.offset);
// mmap
v4l2_sp_globalBuffer[i] = mmap(NULL, buff.length, PROT_READ, MAP_SHARED, v4l2_sp_fd, buff.m.offset);
v4l2_sp_globalBufferLength = buff.length;
if (MAP_FAILED == v4l2_sp_globalBuffer[i])
{
perror("V4L2_SP_Streaming() mmap");
retval = 3;
goto error_munmap;
}
// queue
ret = ioctl(v4l2_sp_fd, VIDIOC_QBUF, &buff);
if (-1 == ret)
{
perror("V4L2_SP_Streaming() queue");
retval = 4;
goto error_munmap;
}
}
#if Print_Debug_Info
printf("V4L2_SP_Streaming(): Step 6 Start Streaming\n");
#endif
/* Step 6: Start Streaming */
int on = _V4L2_SP_BUF_TYPE_;
ret = ioctl(v4l2_sp_fd, VIDIOC_STREAMON, &on);
if (-1 == ret)
{
perror("V4L2_SP_Streaming() VIDIOC_STREAMON");
retval = 5;
goto error_munmap;
}
#if Print_Debug_Info
printf("V4L2_SP_Streaming(): Step 7 Capture\n");
#endif
/* Step 7: Capture */
while (v4l2_sp_captureLoopFlag)
{
// deque
ret = ioctl(v4l2_sp_fd, VIDIOC_DQBUF, &buff);
if (ret != -1)
{
// updata global var
v4l2_sp_globalBufferIndex = buff.index;
v4l2_sp_globalBufferLength = buff.bytesused;
// queue
ret = ioctl(v4l2_sp_fd, VIDIOC_QBUF, &buff);
}
}
#if Print_Debug_Info
printf("V4L2_SP_Streaming(): Exit\n");
#endif
error_munmap:
for (int i = 0; i < _V4L2_SP_BUF_REQ_COUNT_; ++i)
{
if (v4l2_sp_globalBuffer[i] != NULL)
munmap(v4l2_sp_globalBuffer[i], v4l2_sp_globalBufferLength);
}
error_close:
close(v4l2_sp_fd);
out_return:
return retval;
}
void* ThreadEntryPoint(void* arg) {
int result = V4L2_SP_Streaming();
// 使用 pthread_exit() 返回 int 类型的结果
pthread_exit((void*)result);
return NULL;
}3.6 注意事项
其中fill_yuv_image()和v4l2采集并没有上锁，需要注意是否存在IO冲突。
四、预览和优化
4.1 预览
# on win10
.\ffplay.exe rtsp://192.168.50.84:8554/stream 对于可能存在的延迟累积我使用-flags low_delay -vf setpts=0有效，其他情况可以参考：stackoverflow。
4.2 发送端问题
发送端有如下两个问题：
- 上述3.6节中提到的互斥问题。
- CPU调度可能引起v4l2采集队列的卡顿问题。
对于问题1可以通过修改应用程序代码实现，我使用60片缓存“避免”了该问题，如果需要完全避免，则需要互斥量和广播。
对于问题2则使用taskset来指定进程的affinity mas。
taskset -a E *
taskset -a 1 demo上述命令的demo指定到CPU0上运行，其余进程只能在CPU1~3上运行。
Comments | NOTHING