实时使用摄像头实现物体设别
准备环境
硬件环境： Neardi LPA3588开发板、USB camera（或其它camera)
软件环境： RK3588 SDK 、 Neardi LPA3588 Ubuntu image
下载源码
在LPA3588开发板下载rknpu2
git clone https://github.com/rockchip-linux/rknpu2
安装OpenCV
sudo apt install libopencv-dev libopencv-videoio-dev libopencv-video-dev libopencv-imgproc-dev libopencv-highgui-dev
编译
进入rknn_ssd_demo目录，修改CMakeLists.txt sample code使用的是OpenCV3的库，在此我们注释了以便使用我们安装的OpenCV4的库.
neardi@LPA3588:~/rknn/rknpu2/examples/rknn_ssd_demo$ git diff .
diff --git a/examples/rknn_ssd_demo/CMakeLists.txt b/examples/rknn_ssd_demo/CMakeLists.txt
index bd236f6..40537c1 100644
--- a/examples/rknn_ssd_demo/CMakeLists.txt
+++ b/examples/rknn_ssd_demo/CMakeLists.txt
@@ -33,7 +33,7 @@ else()
if(LIB_ARCH STREQUAL "armhf")
set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/../3rdparty/opencv/opencv-linux-armhf/share/OpenCV)
else()
- set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/../3rdparty/opencv/opencv-linux-aarch64/share/OpenCV)
+ #set(OpenCV_DIR ${CMAKE_SOURCE_DIR}/../3rdparty/opencv/opencv-linux-aarch64/share/OpenCV)
endif()
endif()
find_package(OpenCV REQUIRED)
Rockchip原始的rknn_ssd_demo是对图片进行分析， 因此使用单线程； 在此是从摄像头获取数据进行物体识别， 单线程有点卡， 因此使用多线程来实现。 首先， 创建queue数据结构， 代码如附件queue.hpp。
添加新的头文件， 如下：
neardi@LPA3588:~/rknn/rknpu2/examples/rknn_ssd_demo$ git diff .
--- a/examples/rknn_ssd_demo/src/main.cc
+++ b/examples/rknn_ssd_demo/src/main.cc
@@ -29,6 +29,19 @@
#include <fstream>
#include <iostream>
+#include "opencv2/core/core.hpp"
+#include "opencv2/imgcodecs.hpp"
+#include "opencv2/imgproc.hpp"
+#include "opencv2/highgui.hpp"
+
+#include <opencv2/videoio/videoio.hpp>
+#include <opencv2/video.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+#include "rknn_api.h"
+#include "ssd.h"
+#include "queue.hpp"
+
using namespace std;
using namespace cv;
创建2个全局变量，_idleimgbuf & _imgdata， 用于保存从camera获取的image, 并且定义最大长度是300
+/**
+ * create memory pool for images gotten from camera.
+ */
+Queue<cv::Mat*> _idleimgbuf;
+Queue<cv::Mat*> _imgdata;
+
添加线程函数， 从camera读取数据image, 如下：
bool g_exit = false;
static int camera_thread(int index)
{
/**
* load image from camera
*/
cv::VideoCapture cap;
cap.open(index);
if (!cap.isOpened()) {
return -1;
}
cv::Mat first;
cap >> first;
for(int i = 0; i < max_img_count; i++) {
_idleimgbuf.push(new cv::Mat(first));
}
while (!g_exit)
{
cv::Mat* pimg = NULL;
_idleimgbuf.pop(pimg);
if (NULL != pimg) {
cap >> *pimg;
if (pimg->empty()) {
_idleimgbuf.push(pimg);
} else {
_idleimgbuf.push(pimg);
}
} else {
/**
* wait a moment to avoid consume high CPU performance.
*/
usleep(100);
continue;
}
}
cap.release();
return 0;
}
这里我们读取camera第一帧数据，以便知道camera frame的大小。
更改输入参数，并启动线程代码
/*-------------------------------------------
Main Function
-------------------------------------------*/
int main(int argc, char** argv)
{
const int img_width = 300;
const int img_height = 300;
const int img_channels = 3;
int ret = 0;
int model_len = 0;
unsigned char* model = nullptr;
rknn_context ctx = 0;
const char* model_path = argv[1];
if (argc != 3) {
printf("Usage:%s model camera\n", argv[0]);
return -1;
}
int index = std::stoi(argv[2]);
std::thread camthread(&camera_thread, index);
// Load RKNN Model
printf("Loading model ...\n");
model = load_model(model_path, &model_len);
...
return 0;
主函数更改读取数据部分
namedWindow("Video", 1);
do {
cv::Mat* pimg = nullptr;
_idleimgbuf.pop(pimg);
if (nullptr == pimg) {
usleep(100);
if (waitKey(30) >= 0) {
g_exit = true;
break;
}
continue;
}
// if origin model is from Caffe, you maybe not need do BGR2RGB.
cv::Mat orig_img = pimg->clone();
_idleimgbuf.push(pimg);
cv::Mat orig_img_rgb;
cv::cvtColor(orig_img, orig_img_rgb, cv::COLOR_BGR2RGB);
cv::Mat img = orig_img_rgb.clone();
if (orig_img_rgb.cols != img_width || orig_img_rgb.rows != img_height) {
printf("resize %d %d to %d %d\n", orig_img_rgb.cols, orig_img_rgb.rows, img_width, img_height);
cv::resize(orig_img_rgb, img, cv::Size(img_width, img_height), 0, 0, cv::INTER_LINEAR);
}
释放资源
/**
* free the queue buffer.
*/
while(_idleimgbuf.size() > 0)
{
cv::Mat* pmat = NULL;
_idleimgbuf.pop(pmat);
if (NULL != pmat) {
delete pmat;
}
}
while(_imgdata.size() > 0)
{
cv::Mat* pmat = NULL;
_imgdata.pop(pmat);
if (NULL != pmat) {
delete pmat;
}
}
编译及运行rknn_ssd_demo
./rknn_ssd_demo ./model/RK3588/ssd_inception_v2.rknn 41
这里的41是/dev/video41设备的索引，需要根据实际camera输入，设别结果如下：
