diff --git a/tutorials/mxBaseVideoSample/BlockingQueue/BlockingQueue.h b/tutorials/mxBaseVideoSample/BlockingQueue/BlockingQueue.h new file mode 100644 index 0000000000000000000000000000000000000000..9a42e9d5494c9e15b190e4d297764ff2346eb98e --- /dev/null +++ b/tutorials/mxBaseVideoSample/BlockingQueue/BlockingQueue.h @@ -0,0 +1,209 @@ +/* + * Copyright(C) 2020. Huawei Technologies Co.,Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef BLOCKING_QUEUE_H +#define BLOCKING_QUEUE_H + +#include +#include +#include +#include + +#include "MxBase/ErrorCode/ErrorCodes.h" + +static const int DEFAULT_MAX_QUEUE_SIZE = 256; + +template class BlockingQueue { +public: + BlockingQueue(uint32_t maxSize = DEFAULT_MAX_QUEUE_SIZE) : max_size_(maxSize), is_stoped_(false) {} + + ~BlockingQueue() {} + + APP_ERROR Pop(T &item) + { + std::unique_lock lock(mutex_); + + while (queue_.empty() && !is_stoped_) { + empty_cond_.wait(lock); + } + + if (is_stoped_) { + return APP_ERR_QUEUE_STOPED; + } + + if (queue_.empty()) { + return APP_ERR_QUEUE_EMPTY; + } else { + item = queue_.front(); + queue_.pop_front(); + } + + full_cond_.notify_one(); + + return APP_ERR_OK; + } + + APP_ERROR Pop(T& item, unsigned int timeOutMs) + { + std::unique_lock lock(mutex_); + auto realTime = std::chrono::milliseconds(timeOutMs); + + while (queue_.empty() && !is_stoped_) { + empty_cond_.wait_for(lock, realTime); + } + + if (is_stoped_) { + return APP_ERR_QUEUE_STOPED; + } + + if (queue_.empty()) { + return APP_ERR_QUEUE_EMPTY; + } else { + item = queue_.front(); + queue_.pop_front(); + } + + full_cond_.notify_one(); + + return APP_ERR_OK; + } + + APP_ERROR Push(const T& item, bool isWait = false) + { + std::unique_lock lock(mutex_); + + while (queue_.size() >= max_size_ && isWait && !is_stoped_) { + full_cond_.wait(lock); + } + + if (is_stoped_) { + return APP_ERR_QUEUE_STOPED; + } + + if (queue_.size() >= max_size_) { + return APP_ERR_QUEUE_FULL; + } + queue_.push_back(item); + + empty_cond_.notify_one(); + + return APP_ERR_OK; + } + + APP_ERROR Push_Front(const T &item, bool isWait = false) + { + std::unique_lock lock(mutex_); + + while (queue_.size() >= max_size_ && isWait && !is_stoped_) { + full_cond_.wait(lock); + } + + if (is_stoped_) { + return APP_ERR_QUEUE_STOPED; + } + + if (queue_.size() >= max_size_) { + return APP_ERR_QUEUE_FULL; + } + + queue_.push_front(item); + + empty_cond_.notify_one(); + + return APP_ERR_OK; + } + + void Stop() + { + { + std::unique_lock lock(mutex_); + is_stoped_ = true; + } + + full_cond_.notify_all(); + empty_cond_.notify_all(); + } + + void Restart() + { + { + std::unique_lock lock(mutex_); + is_stoped_ = false; + } + } + + // if the queue is stoped ,need call this function to release the unprocessed items + std::list GetRemainItems() + { + std::unique_lock lock(mutex_); + + if (!is_stoped_) { + return std::list(); + } + + return queue_; + } + + APP_ERROR GetBackItem(T &item) + { + if (is_stoped_) { + return APP_ERR_QUEUE_STOPED; + } + + if (queue_.empty()) { + return APP_ERR_QUEUE_EMPTY; + } + + item = queue_.back(); + return APP_ERR_OK; + } + + std::mutex *GetLock() + { + return &mutex_; + } + + APP_ERROR IsFull() + { + std::unique_lock lock(mutex_); + return queue_.size() >= max_size_; + } + + int GetSize() + { + return queue_.size(); + } + + APP_ERROR IsEmpty() + { + return queue_.empty(); + } + + void Clear() + { + std::unique_lock lock(mutex_); + queue_.clear(); + } + +private: + std::list queue_; + std::mutex mutex_; + std::condition_variable empty_cond_; + std::condition_variable full_cond_; + uint32_t max_size_; + + bool is_stoped_; +}; +#endif // __INC_BLOCKING_QUEUE_H__ diff --git a/tutorials/mxBaseVideoSample/CMakeLists.txt b/tutorials/mxBaseVideoSample/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..f5526f8a550197965b63be26041517503f316eca --- /dev/null +++ b/tutorials/mxBaseVideoSample/CMakeLists.txt @@ -0,0 +1,47 @@ +cmake_minimum_required(VERSION 3.10) +project(stream_pull_sample) + +add_compile_options(-std=c++11 -fPIC -fstack-protector-all -g -Wl,-z,relro,-z,now,-z -pie -Wall) +add_definitions(-D_GLIBCXX_USE_CXX11_ABI=0 -Dgoogle=mindxsdk_private) + +set(OUTPUT_NAME "stream_pull_test") +set(CMAKE_RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}") + +set(MX_SDK_HOME {SDK实际安装路径}) +set(FFMPEG_PATH {ffmpeg安装路径}) + +include_directories( + ${MX_SDK_HOME}/include + ${MX_SDK_HOME}/opensource/include + ${MX_SDK_HOME}/opensource/include/opencv4 + ${MX_SDK_HOME}/include/MxBase/postprocess/include + ${FFMPEG_PATH}/include +) + +link_directories( + ${MX_SDK_HOME}/lib + ${MX_SDK_HOME}/opensource/lib + ${MX_SDK_HOME}/lib/modelpostprocessors + ${FFMPEG_PATH}/lib + /usr/local/Ascend/ascend-toolkit/latest/acllib/lib64 +) + +add_executable(${OUTPUT_NAME} main.cpp VideoProcess/VideoProcess.cpp VideoProcess/VideoProcess.h + Yolov3Detection/Yolov3Detection.cpp Yolov3Detection/Yolov3Detection.h) +target_link_libraries(${OUTPUT_NAME} + avcodec + avdevice + avfilter + avformat + avcodec + avutil + swscale + swresample + glog + mxbase + cpprest + opencv_world + pthread m + yolov3postprocess + ) + diff --git a/tutorials/mxBaseVideoSample/README.md b/tutorials/mxBaseVideoSample/README.md new file mode 100644 index 0000000000000000000000000000000000000000..36165275ad30983db7038c288a7ae2b7e969ecbe --- /dev/null +++ b/tutorials/mxBaseVideoSample/README.md @@ -0,0 +1,126 @@ +# 基于MxBase 的yolov3视频流推理样例 + +## 介绍 + +本开发样例是基于mxBase开发的端到端推理的C++应用程序,可在昇腾芯片上实现视频流的目标检测,并把可视化结果保存到本地。 + +### 准备工作 + +> 模型转换 + +步骤1 在 + +**步骤1** 在ModelZoo上下载YOLOv3模型 ,选择“历史版本”中版本1.1下载。[下载地址](https://www.hiascend.com/zh/software/modelzoo/detail/C/210261e64adc42d2b3d84c447844e4c7) + +**步骤2** 将获取到的YOLOv3模型pb文件存放至:"样例项目所在目录/model/"。 + +**步骤3** 模型转换 + +在pb文件所在目录下执行以下命令 + +``` +# 设置环境变量(请确认install_path路径是否正确) +# Set environment PATH (Please confirm that the install_path is correct). + +export install_path=/usr/local/Ascend/ascend-toolkit/latest +export PATH=/usr/local/python3.7.5/bin:${install_path}/atc/ccec_compiler/bin:${install_path}/atc/bin:$PATH +export PYTHONPATH=${install_path}/atc/python/site-packages:${install_path}/atc/python/site-packages/auto_tune.egg/auto_tune:${install_path}/atc/python/site-packages/schedule_search.egg +export LD_LIBRARY_PATH=${install_path}/atc/lib64:$LD_LIBRARY_PATH +export ASCEND_OPP_PATH=${install_path}/opp + +# 执行,转换YOLOv3模型 +# Execute, transform YOLOv3 model. + +atc --model=./yolov3_tf.pb --framework=3 --output=./yolov3_tf_bs1_fp16 --soc_version=Ascend310 --insert_op_conf=./aipp_yolov3_416_416.aippconfig --input_shape="input/input_data:1,416,416,3" --out_nodes="conv_lbbox/BiasAdd:0;conv_mbbox/BiasAdd:0;conv_sbbox/BiasAdd:0" +# 说明:out_nodes制定了输出节点的顺序,需要与模型后处理适配。 +``` + +执行完模型转换脚本后,会生成相应的.om模型文件。 执行完模型转换脚本后,会生成相应的.om模型文件。 + +模型转换使用了ATC工具,如需更多信息请参考: + + https://support.huaweicloud.com/tg-cannApplicationDev330/atlasatc_16_0005.html + +> 相关参数修改 + +main.cpp文件中,添加模型路径与 rtsp 流源地址(需要自行准备可用的视频流,视频流格式为H264) + +[Live555拉流教程](../../docs/参考资料/Live555离线视频转RTSP说明文档) + +``` +... +initParam.modelPath = "{yolov3模型路径}"; +... + std::string streamName = "rtsp_Url"; +``` + +VideoProcess.cpp文件中,设置视频的宽高值 + +``` +const uint32_t VIDEO_WIDTH = {视频宽度}; +const uint32_t VIDEO_HEIGHT = {视频高度}; +``` + +### 配置环境变量 + +``` +# 执行如下命令,打开.bashrc文件 +vi .bashrc +# 在.bashrc文件中添加以下环境变量 +MX_SDK_HOME=${SDK安装路径} +FFMPEG_PATH=${FFMPEG安装路径} +# 若环境中没有安装ffmpeg,请联系支撑人员 + +LD_LIBRARY_PATH=${MX_SDK_HOME}/lib:${MX_SDK_HOME}/opensource/lib:/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64:/usr/local/Ascend/ascend-toolkit/:/usr/local/python3.7.5/lib:${FFMPEG_PATH}/lib +# 保存退出.bashrc文件 +# 执行如下命令使环境变量生效 +source ~/.bashrc + +#查看环境变量 +env +``` + +### 配置CMakeLists + +配置CMakeLists.txt文件中的`MX_SDK_HOME`与`FFMPEG_PATH`环境变量 + +``` +set(MX_SDK_HOME {SDK实际安装路径}) +set(FFMPEG_PATH {ffmpeg安装路径}) +``` + +### 编译项目文件 + +新建立build目录,进入build执行cmake ..(..代表包含CMakeLists.txt的源文件父目录),在build目录下生成了编译需要的Makefile和中间文件。执行make构建工程,构建成功后就会生成可执行文件。 + +``` +mkdir build + +cd build + +cmake .. + +make -j +Scanning dependencies of target stream_pull_test +[ 25%] Building CXX object CMakeFiles/stream_pull_test.dir/main.cpp.o +[ 50%] Building CXX object CMakeFiles/stream_pull_test.dir/VideoProcess/VideoProcess.cpp.o +[ 75%] Building CXX object CMakeFiles/stream_pull_test.dir/Yolov3Detection/Yolov3Detection.cpp.o +[100%] Linking CXX executable ../stream_pull_test +[100%] Built target stream_pull_test + +# stream_pull_test就是CMakeLists文件中指定生成的可执行文件。 +``` + +### 执行脚本 + +执行run.sh脚本前请先确认可执行文件stream_pull_test已生成。 + +``` +chmod +x run.sh +bash run.sh +``` + +### 查看结果 + +执行run.sh完毕后,可视化结果会被保存在工程目录下result文件夹中。 + diff --git a/tutorials/mxBaseVideoSample/VideoProcess/VideoProcess.cpp b/tutorials/mxBaseVideoSample/VideoProcess/VideoProcess.cpp new file mode 100644 index 0000000000000000000000000000000000000000..343d2d67e982363ca85101850a67053f0a8be9d3 --- /dev/null +++ b/tutorials/mxBaseVideoSample/VideoProcess/VideoProcess.cpp @@ -0,0 +1,299 @@ +/* + * Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "MxBase/ErrorCode/ErrorCodes.h" +#include "MxBase/Log/Log.h" +#include "opencv2/opencv.hpp" +#include "VideoProcess.h" + +namespace { + static AVFormatContext *formatContext = nullptr; + const uint32_t VIDEO_WIDTH = {视频宽度}; + const uint32_t VIDEO_HEIGHT = {视频高度}; + const uint32_t MAX_QUEUE_LENGHT = 1000; + const uint32_t QUEUE_POP_WAIT_TIME = 10; + const uint32_t YUV_BYTE_NU = 3; + const uint32_t YUV_BYTE_DE = 2; +} + +APP_ERROR VideoProcess::StreamInit(const std::string &rtspUrl) +{ + avformat_network_init(); + + AVDictionary *options = nullptr; + av_dict_set(&options, "rtsp_transport", "tcp", 0); + av_dict_set(&options, "stimeout", "3000000", 0); + APP_ERROR ret = avformat_open_input(&formatContext, rtspUrl.c_str(), nullptr, &options); + if (options != nullptr) { + av_dict_free(&options); + } + if(ret != APP_ERR_OK){ + LogError << "Couldn't open input stream " << rtspUrl.c_str() << " ret = " << ret; + return APP_ERR_STREAM_NOT_EXIST; + } + + ret = avformat_find_stream_info(formatContext, nullptr); + if(ret != APP_ERR_OK){ + LogError << "Couldn't find stream information"; + return APP_ERR_STREAM_NOT_EXIST; + } + av_dump_format(formatContext, 0, rtspUrl.c_str(), 0); + return APP_ERR_OK; +} + +APP_ERROR VideoProcess::StreamDeInit() +{ + avformat_close_input(&formatContext); + return APP_ERR_OK; +} + +APP_ERROR VideoProcess::VideoDecodeCallback(std::shared_ptr buffer, MxBase::DvppDataInfo &inputDataInfo, + void *userData) +{ + auto deleter = [] (MxBase::MemoryData *mempryData) { + if (mempryData == nullptr) { + LogError << "MxbsFree failed"; + return; + } + APP_ERROR ret = MxBase::MemoryHelper::MxbsFree(*mempryData); + delete mempryData; + if (ret != APP_ERR_OK) { + LogError << GetError(ret) << " MxbsFree failed"; + return; + } + LogInfo << "MxbsFree successfully"; + }; + auto output = std::shared_ptr(new MxBase::MemoryData(buffer.get(), + (size_t)inputDataInfo.dataSize, MxBase::MemoryData::MEMORY_DVPP, inputDataInfo.frameId), deleter); + + if (userData == nullptr) { + LogError << "userData is nullptr"; + return APP_ERR_COMM_INVALID_POINTER; + } + auto *queue = (BlockingQueue>*)userData; + queue->Push(output); + return APP_ERR_OK; +} + +APP_ERROR VideoProcess::VideoDecodeInit() +{ + MxBase::VdecConfig vdecConfig; + vdecConfig.inputVideoFormat = MxBase::MXBASE_STREAM_FORMAT_H264_MAIN_LEVEL; + vdecConfig.outputImageFormat = MxBase::MXBASE_PIXEL_FORMAT_YUV_SEMIPLANAR_420; + vdecConfig.deviceId = DEVICE_ID; + vdecConfig.channelId = CHANNEL_ID; + vdecConfig.callbackFunc = VideoDecodeCallback; + vdecConfig.outMode = 1; + + vDvppWrapper = std::make_shared(); + if (vDvppWrapper == nullptr) { + LogError << "Failed to create dvppWrapper"; + return APP_ERR_COMM_INIT_FAIL; + } + APP_ERROR ret = vDvppWrapper->InitVdec(vdecConfig); + if (ret != APP_ERR_OK) { + LogError << "Failed to initialize dvppWrapper"; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR VideoProcess::VideoDecodeDeInit() +{ + APP_ERROR ret = vDvppWrapper->DeInitVdec(); + if (ret != APP_ERR_OK) { + LogError << "Failed to deinitialize dvppWrapper"; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR VideoProcess::VideoDecode(MxBase::MemoryData &streamData, const uint32_t &height, + const uint32_t &width, void *userData) +{ + static uint32_t frameId = 0; + MxBase::MemoryData dvppMemory((size_t)streamData.size, + MxBase::MemoryData::MEMORY_DVPP, DEVICE_ID); + APP_ERROR ret = MxBase::MemoryHelper::MxbsMallocAndCopy(dvppMemory, streamData); + if (ret != APP_ERR_OK) { + LogError << "Failed to MxbsMallocAndCopy"; + return ret; + } + MxBase::DvppDataInfo inputDataInfo; + inputDataInfo.dataSize = dvppMemory.size; + inputDataInfo.data = (uint8_t *)dvppMemory.ptrData; + inputDataInfo.height = VIDEO_HEIGHT; + inputDataInfo.width = VIDEO_WIDTH; + inputDataInfo.channelId = CHANNEL_ID; + inputDataInfo.frameId = frameId; + ret = vDvppWrapper->DvppVdec(inputDataInfo, userData); + + if (ret != APP_ERR_OK) { + LogError << "DvppVdec Failed"; + MxBase::MemoryHelper::MxbsFree(dvppMemory); + return ret; + } + frameId++; + return APP_ERR_OK; +} + +void VideoProcess::GetFrames(std::shared_ptr>> blockingQueue, + std::shared_ptr videoProcess) +{ + MxBase::DeviceContext device; + device.devId = DEVICE_ID; + APP_ERROR ret = MxBase::DeviceManager::GetInstance()->SetDevice(device); + if (ret != APP_ERR_OK) { + LogError << "SetDevice failed"; + return; + } + + AVPacket pkt; + while(!stopFlag){ + av_init_packet(&pkt); + APP_ERROR ret = av_read_frame(formatContext, &pkt); + if(ret != APP_ERR_OK){ + LogError << "Read frame failed, continue"; + if(ret == AVERROR_EOF){ + LogError << "StreamPuller is EOF, over!"; + break; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + + MxBase::MemoryData streamData((void *)pkt.data, (size_t)pkt.size, + MxBase::MemoryData::MEMORY_HOST_NEW, DEVICE_ID); + ret = videoProcess->VideoDecode(streamData, VIDEO_HEIGHT, VIDEO_WIDTH, (void*)blockingQueue.get()); + if (ret != APP_ERR_OK) { + LogError << "VideoDecode failed"; + return; + } + av_packet_unref(&pkt); + } + av_packet_unref(&pkt); +} + +APP_ERROR VideoProcess::SaveResult(std::shared_ptr resultInfo, const uint32_t frameId, + const std::vector> objInfos) +{ + MxBase::MemoryData memoryDst(resultInfo->size,MxBase::MemoryData::MEMORY_HOST_NEW); + APP_ERROR ret = MxBase::MemoryHelper::MxbsMallocAndCopy(memoryDst, *resultInfo); + if(ret != APP_ERR_OK){ + LogError << "Fail to malloc and copy host memory."; + return ret; + } + cv::Mat imgYuv = cv::Mat(VIDEO_HEIGHT* YUV_BYTE_NU / YUV_BYTE_DE, VIDEO_WIDTH, CV_8UC1, memoryDst.ptrData); + cv::Mat imgBgr = cv::Mat(VIDEO_HEIGHT, VIDEO_WIDTH, CV_8UC3); + cv::cvtColor(imgYuv, imgBgr, cv::COLOR_YUV2BGR_NV12); + + std::vector info; + for (uint32_t i = 0; i < objInfos.size(); i++) { + float maxConfidence = 0; + uint32_t index; + for (uint32_t j = 0; j < objInfos[i].size(); j++) { + if (objInfos[i][j].confidence > maxConfidence) { + maxConfidence = objInfos[i][j].confidence; + index = j; + } + } + info.push_back(objInfos[i][index]); + LogInfo << "id: " << info[i].classId << "; lable: " << info[i].className + << "; confidence: " << info[i].confidence + << "; box: [ (" << info[i].x0 << "," << info[i].y0 << ") " + << "(" << info[i].x1 << "," << info[i].y1 << ") ]"; + + const cv::Scalar green = cv::Scalar(0, 255, 0); + const uint32_t thickness = 4; + const uint32_t xOffset = 10; + const uint32_t yOffset = 10; + const uint32_t lineType = 8; + const float fontScale = 1.0; + + cv::putText(imgBgr, info[i].className, cv::Point(info[i].x0 + xOffset, info[i].y0 + yOffset), + cv::FONT_HERSHEY_SIMPLEX, fontScale, green, thickness, lineType); + cv::rectangle(imgBgr,cv::Rect(info[i].x0, info[i].y0, + info[i].x1 - info[i].x0, info[i].y1 - info[i].y0), + green, thickness); + std::string fileName = "./result/result" + std::to_string(frameId+1) + ".jpg"; + cv::imwrite(fileName, imgBgr); + } + + ret = MxBase::MemoryHelper::MxbsFree(memoryDst); + if(ret != APP_ERR_OK){ + LogError << "Fail to MxbsFree memory."; + return ret; + } + return APP_ERR_OK; +} + +void VideoProcess::GetResults(std::shared_ptr>> blockingQueue, + std::shared_ptr yolov3Detection, + std::shared_ptr videoProcess) +{ + uint32_t frameId = 0; + MxBase::DeviceContext device; + device.devId = DEVICE_ID; + APP_ERROR ret = MxBase::DeviceManager::GetInstance()->SetDevice(device); + if (ret != APP_ERR_OK) { + LogError << "SetDevice failed"; + return; + } + while (!stopFlag) { + std::shared_ptr data = nullptr; + APP_ERROR ret = blockingQueue->Pop(data, QUEUE_POP_WAIT_TIME); + if (ret != APP_ERR_OK) { + LogError << "Pop failed"; + return; + } + LogInfo << "get result"; + + MxBase::TensorBase resizeFrame; + auto result = std::make_shared(); + result = std::static_pointer_cast(data); + + ret = yolov3Detection->ResizeFrame(result, VIDEO_HEIGHT, VIDEO_WIDTH,resizeFrame); + if (ret != APP_ERR_OK) { + LogError << "Resize failed"; + return; + } + + std::vector inputs = {}; + std::vector outputs = {}; + inputs.push_back(resizeFrame); + ret = yolov3Detection->Inference(inputs, outputs); + if (ret != APP_ERR_OK) { + LogError << "Inference failed, ret=" << ret << "."; + return; + } + + std::vector> objInfos; + ret = yolov3Detection->PostProcess(outputs, VIDEO_HEIGHT, VIDEO_WIDTH, objInfos); + if (ret != APP_ERR_OK) { + LogError << "PostProcess failed, ret=" << ret << "."; + return; + } + + ret = videoProcess->SaveResult(result, frameId, objInfos); + if (ret != APP_ERR_OK) { + LogError << "Save result failed, ret=" << ret << "."; + return; + } + frameId++; + + } +} \ No newline at end of file diff --git a/tutorials/mxBaseVideoSample/VideoProcess/VideoProcess.h b/tutorials/mxBaseVideoSample/VideoProcess/VideoProcess.h new file mode 100644 index 0000000000000000000000000000000000000000..791c46a940573c09d749b1894052db9f4a6d4177 --- /dev/null +++ b/tutorials/mxBaseVideoSample/VideoProcess/VideoProcess.h @@ -0,0 +1,64 @@ +/* + * Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef STREAM_PULL_SAMPLE_VIDEOPROCESS_H +#define STREAM_PULL_SAMPLE_VIDEOPROCESS_H + +#include "MxBase/ErrorCode/ErrorCodes.h" +#include "MxBase/DvppWrapper/DvppWrapper.h" +#include "MxBase/MemoryHelper/MemoryHelper.h" +#include "MxBase/DeviceManager/DeviceManager.h" +#include "MxBase/Tensor/TensorBase/TensorBase.h" +#include "ObjectPostProcessors/Yolov3PostProcess.h" +#include "../BlockingQueue/BlockingQueue.h" +#include "../Yolov3Detection/Yolov3Detection.h" + +extern "C"{ +#include "libavformat/avformat.h" +#include "libavcodec/avcodec.h" +#include "libavutil/avutil.h" +#include "libswscale/swscale.h" +} + + +class VideoProcess { +private: + static APP_ERROR VideoDecodeCallback(std::shared_ptr buffer, + MxBase::DvppDataInfo &inputDataInfo, void *userData); + APP_ERROR VideoDecode(MxBase::MemoryData &streamData, const uint32_t &height, + const uint32_t &width, void *userData); + APP_ERROR SaveResult(const std::shared_ptr resulInfo, const uint32_t frameId, + const std::vector> objInfos); +public: + APP_ERROR StreamInit(const std::string &rtspUrl); + APP_ERROR StreamDeInit(); + APP_ERROR VideoDecodeInit(); + APP_ERROR VideoDecodeDeInit(); + static void GetFrames(std::shared_ptr>> blockingQueue, + std::shared_ptr videoProcess); + static void GetResults(std::shared_ptr>> blockingQueue, + std::shared_ptr yolov3Detection, + std::shared_ptr videoProcess); +private: + std::shared_ptr vDvppWrapper; + const uint32_t CHANNEL_ID = 0; +public: + static bool stopFlag; + static const uint32_t DEVICE_ID = 0; + +}; + +#endif //STREAM_PULL_SAMPLE_VIDEOPROCESS_H \ No newline at end of file diff --git a/tutorials/mxBaseVideoSample/Yolov3Detection/Yolov3Detection.cpp b/tutorials/mxBaseVideoSample/Yolov3Detection/Yolov3Detection.cpp new file mode 100644 index 0000000000000000000000000000000000000000..de889fa95af0122f53b0224fede156c3f340b243 --- /dev/null +++ b/tutorials/mxBaseVideoSample/Yolov3Detection/Yolov3Detection.cpp @@ -0,0 +1,219 @@ +/* + * Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "MxBase/ErrorCode/ErrorCodes.h" +#include "MxBase/Log/Log.h" +#include "Yolov3Detection.h" + +namespace { + const uint32_t YUV_BYTE_NU = 3; + const uint32_t YUV_BYTE_DE = 2; + const uint32_t VPC_H_ALIGN = 2; +} + +APP_ERROR Yolov3Detection::LoadLabels(const std::string &labelPath, std::map &labelMap) +{ + std::ifstream infile; + // open label file + infile.open(labelPath, std::ios_base::in); + std::string s; + // check label file validity + if (infile.fail()) { + LogError << "Failed to open label file: " << labelPath << "."; + return APP_ERR_COMM_OPEN_FAIL; + } + labelMap.clear(); + // construct label map + int count = 0; + while (std::getline(infile, s)) { + if (s.find('#') <= 1) { + continue; + } + size_t eraseIndex = s.find_last_not_of("\r\n\t"); + if (eraseIndex != std::string::npos) { + s.erase(eraseIndex + 1, s.size() - eraseIndex); + } + labelMap.insert(std::pair(count, s)); + count++; + } + infile.close(); + return APP_ERR_OK; +} + +void Yolov3Detection::SetYolov3PostProcessConfig(const InitParam &initParam, std::map> &config) +{ + MxBase::ConfigData configData; + const std::string checkTensor = initParam.checkTensor ? "true" : "false"; + configData.SetJsonValue("CLASS_NUM", std::to_string(initParam.classNum)); + configData.SetJsonValue("BIASES_NUM", std::to_string(initParam.biasesNum)); + configData.SetJsonValue("BIASES", initParam.biases); + configData.SetJsonValue("OBJECTNESS_THRESH", initParam.objectnessThresh); + configData.SetJsonValue("IOU_THRESH", initParam.iouThresh); + configData.SetJsonValue("SCORE_THRESH", initParam.scoreThresh); + configData.SetJsonValue("YOLO_TYPE", std::to_string(initParam.yoloType)); + configData.SetJsonValue("MODEL_TYPE", std::to_string(initParam.modelType)); + configData.SetJsonValue("INPUT_TYPE", std::to_string(initParam.inputType)); + configData.SetJsonValue("ANCHOR_DIM", std::to_string(initParam.anchorDim)); + configData.SetJsonValue("CHECK_MODEL", checkTensor); + + auto jsonStr = configData.GetCfgJson().serialize(); + config["postProcessConfigContent"] = std::make_shared(jsonStr); + config["labelPath"] = std::make_shared(initParam.labelPath); +} + +APP_ERROR Yolov3Detection::FrameInit(const InitParam &initParam) +{ + deviceId = initParam.deviceId; + APP_ERROR ret = MxBase::DeviceManager::GetInstance()->InitDevices(); + if (ret != APP_ERR_OK) { + LogError << "Init devices failed, ret=" << ret << "."; + return ret; + } + ret = MxBase::TensorContext::GetInstance()->SetContext(initParam.deviceId); + if (ret != APP_ERR_OK) { + LogError << "Set context failed, ret=" << ret << "."; + return ret; + } + yDvppWrapper = std::make_shared(); + ret = yDvppWrapper->Init(); + if (ret != APP_ERR_OK) { + LogError << "DvppWrapper init failed, ret=" << ret << "."; + return ret; + } + model = std::make_shared(); + LogInfo << "model path: " << initParam.modelPath; + ret = model->Init(initParam.modelPath, modelDesc); + if (ret != APP_ERR_OK) { + LogError << "ModelInferenceProcessor init failed, ret=" << ret << "."; + return ret; + } + + std::map> config; + SetYolov3PostProcessConfig(initParam, config); + post = std::make_shared(); + ret = post->Init(config); + if (ret != APP_ERR_OK) { + LogError << "Yolov3PostProcess init failed, ret=" << ret << "."; + return ret; + } + // load labels from file + ret = LoadLabels(initParam.labelPath, labelMap); + if (ret != APP_ERR_OK) { + LogError << "Failed to load labels, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR Yolov3Detection::FrameDeInit() +{ + yDvppWrapper->DeInit(); + model->DeInit(); + post->DeInit(); + MxBase::DeviceManager::GetInstance()->DestroyDevices(); + return APP_ERR_OK; +} + +APP_ERROR Yolov3Detection::ResizeFrame(const std::shared_ptr frameInfo, const uint32_t &height, + const uint32_t &width, MxBase::TensorBase &tensor) +{ + MxBase::DvppDataInfo input = {}; + input.height = height; + input.width = width; + input.heightStride = height; + input.widthStride = width; + input.dataSize = frameInfo->size; + input.data = (uint8_t*)frameInfo->ptrData; + + const uint32_t resizeHeight = 416; + const uint32_t resizeWidth = 416; + MxBase::ResizeConfig resize = {}; + resize.height = resizeHeight; + resize.width = resizeWidth; + + MxBase::DvppDataInfo output = {}; + APP_ERROR ret = yDvppWrapper->VpcResize(input, output, resize); + if(ret != APP_ERR_OK){ + LogError << GetError(ret) << "VpcResize failed."; + return ret; + } + + MxBase::MemoryData outMemoryData((void*)output.data, output.dataSize, MxBase::MemoryData::MEMORY_DVPP, deviceId); + if (output.heightStride % VPC_H_ALIGN != 0) { + LogError << "Output data height(" << output.heightStride << ") can't be divided by " << VPC_H_ALIGN << "."; + MxBase::MemoryHelper::MxbsFree(outMemoryData); + return APP_ERR_COMM_INVALID_PARAM; + } + std::vector shape = {output.heightStride * YUV_BYTE_NU / YUV_BYTE_DE, output.widthStride}; + tensor = MxBase::TensorBase(outMemoryData, false, shape, MxBase::TENSOR_DTYPE_UINT8); + + return APP_ERR_OK; +} + +APP_ERROR Yolov3Detection::Inference(const std::vector &inputs, + std::vector &outputs) +{ + auto dtypes = model->GetOutputDataType(); + for (size_t i = 0; i < modelDesc.outputTensors.size(); ++i) { + std::vector shape = {}; + for (size_t j = 0; j < modelDesc.outputTensors[i].tensorDims.size(); ++j) { + shape.push_back((uint32_t)modelDesc.outputTensors[i].tensorDims[j]); + } + MxBase::TensorBase tensor(shape, dtypes[i], MxBase::MemoryData::MemoryType::MEMORY_DVPP, deviceId); + APP_ERROR ret = MxBase::TensorBase::TensorBaseMalloc(tensor); + if (ret != APP_ERR_OK) { + LogError << "TensorBaseMalloc failed, ret=" << ret << "."; + return ret; + } + outputs.push_back(tensor); + } + + MxBase::DynamicInfo dynamicInfo = {}; + dynamicInfo.dynamicType = MxBase::DynamicType::STATIC_BATCH; + auto startTime = std::chrono::high_resolution_clock::now(); + if(inputs[0].GetBuffer() == nullptr){ + LogError << "input is null"; + return APP_ERR_FAILURE; + } + APP_ERROR ret = model->ModelInference(inputs, outputs, dynamicInfo); + auto endTime = std::chrono::high_resolution_clock::now(); + double costMs = std::chrono::duration(endTime - startTime).count(); + g_inferCost.push_back(costMs); + if (ret != APP_ERR_OK) { + LogError << "ModelInference failed, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} + +APP_ERROR Yolov3Detection::PostProcess(const std::vector &outputs,const uint32_t &height, + const uint32_t &width, std::vector> &objInfos) +{ + MxBase::ResizedImageInfo imgInfo; + imgInfo.widthOriginal = width; + imgInfo.heightOriginal = height; + imgInfo.widthResize = 416; + imgInfo.heightResize = 416; + imgInfo.resizeType = MxBase::RESIZER_STRETCHING; + std::vector imageInfoVec = {}; + imageInfoVec.push_back(imgInfo); + APP_ERROR ret = post->Process(outputs, objInfos, imageInfoVec); + if (ret != APP_ERR_OK) { + LogError << "Process failed, ret=" << ret << "."; + return ret; + } + return APP_ERR_OK; +} \ No newline at end of file diff --git a/tutorials/mxBaseVideoSample/Yolov3Detection/Yolov3Detection.h b/tutorials/mxBaseVideoSample/Yolov3Detection/Yolov3Detection.h new file mode 100644 index 0000000000000000000000000000000000000000..ea07d02fa6cc01ae13044cec4d3abf4c7e544702 --- /dev/null +++ b/tutorials/mxBaseVideoSample/Yolov3Detection/Yolov3Detection.h @@ -0,0 +1,67 @@ +/* + * Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef STREAM_PULL_SAMPLE_YOLOV3DETECTION_H +#define STREAM_PULL_SAMPLE_YOLOV3DETECTION_H + +#include "MxBase/DvppWrapper/DvppWrapper.h" +#include "MxBase/MemoryHelper/MemoryHelper.h" +#include "MxBase/DeviceManager/DeviceManager.h" +#include "MxBase/Tensor/TensorContext/TensorContext.h" +#include "MxBase/ModelInfer/ModelInferenceProcessor.h" +#include "ObjectPostProcessors/Yolov3PostProcess.h" +#include "opencv2/opencv.hpp" + +extern std::vector g_inferCost; + +struct InitParam { + uint32_t deviceId; + std::string labelPath; + bool checkTensor; + std::string modelPath; + uint32_t classNum; + uint32_t biasesNum; + std::string biases; + std::string objectnessThresh; + std::string iouThresh; + std::string scoreThresh; + uint32_t yoloType; + uint32_t modelType; + uint32_t inputType; + uint32_t anchorDim; +}; + +class Yolov3Detection { +protected: + APP_ERROR LoadLabels(const std::string &labelPath, std::map &labelMap); + void SetYolov3PostProcessConfig(const InitParam &initParam, std::map> &config); +public: + APP_ERROR FrameInit(const InitParam &initParam); + APP_ERROR FrameDeInit(); + APP_ERROR ResizeFrame(const std::shared_ptr frameInfo, const uint32_t &height, + const uint32_t &width, MxBase::TensorBase &tensor); + APP_ERROR Inference(const std::vector &inputs, std::vector &outputs); + APP_ERROR PostProcess(const std::vector &outputs,const uint32_t &height, + const uint32_t &width, std::vector> &objInfos); +private: + std::shared_ptr yDvppWrapper; + std::shared_ptr model; + std::shared_ptr post; + MxBase::ModelDesc modelDesc = {}; + std::map labelMap = {}; + uint32_t deviceId = 0; +}; +#endif //STREAM_PULL_SAMPLE_YOLOV3DETECTION_H \ No newline at end of file diff --git a/tutorials/mxBaseVideoSample/main.cpp b/tutorials/mxBaseVideoSample/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..50b09bc76c52124d202d85bc87da8360422cfdb3 --- /dev/null +++ b/tutorials/mxBaseVideoSample/main.cpp @@ -0,0 +1,135 @@ +/* + * Copyright(C) 2021. Huawei Technologies Co.,Ltd. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include +#include +#include "MxBase/ErrorCode/ErrorCodes.h" +#include "MxBase/Log/Log.h" +#include "MxBase/DvppWrapper/DvppWrapper.h" +#include "MxBase/MemoryHelper/MemoryHelper.h" +#include "MxBase/DeviceManager/DeviceManager.h" +#include "BlockingQueue/BlockingQueue.h" +#include "VideoProcess/VideoProcess.h" +#include "Yolov3Detection/Yolov3Detection.h" + +bool VideoProcess::stopFlag = false; +std::vector g_inferCost; +namespace { + const uint32_t MAX_QUEUE_LENGHT = 1000; +} + +static void SigHandler(int signal) +{ + if (signal == SIGINT) { + VideoProcess::stopFlag = true; + } +} + +void InitYolov3Param(InitParam &initParam, const uint32_t deviceID) +{ + initParam.deviceId = deviceID; + initParam.labelPath = "./model/coco.names"; + initParam.checkTensor = true; + initParam.modelPath = "{yolov3模型路径}"; + initParam.classNum = 80; + initParam.biasesNum = 18; + initParam.biases = "10,13,16,30,33,23,30,61,62,45,59,119,116,90,156,198,373,326"; + initParam.objectnessThresh = "0.001"; + initParam.iouThresh = "0.5"; + initParam.scoreThresh = "0.001"; + initParam.yoloType = 3; + initParam.modelType = 0; + initParam.inputType = 0; + initParam.anchorDim = 3; +} + +int main() { + std::string streamName = "rtsp_Url"; + APP_ERROR ret = MxBase::DeviceManager::GetInstance()->InitDevices(); + if (ret != APP_ERR_OK) { + LogError << "InitDevices failed"; + return ret; + } + + auto videoProcess = std::make_shared(); + auto yolov3 = std::make_shared(); + + InitParam initParam; + InitYolov3Param(initParam, videoProcess->DEVICE_ID); + yolov3->FrameInit(initParam); + MxBase::DeviceContext device; + device.devId = videoProcess->DEVICE_ID; + ret = MxBase::DeviceManager::GetInstance()->SetDevice(device); + if (ret != APP_ERR_OK) { + LogError << "SetDevice failed"; + return ret; + } + ret = videoProcess->StreamInit(streamName); + if (ret != APP_ERR_OK) { + LogError << "StreamInit failed"; + return ret; + } + ret = videoProcess->VideoDecodeInit(); + if (ret != APP_ERR_OK) { + LogError << "VideoDecodeInit failed"; + MxBase::DeviceManager::GetInstance()->DestroyDevices(); + return ret; + } + + auto blockingQueue = std::make_shared>>(MAX_QUEUE_LENGHT); + std::thread getFrame(videoProcess->GetFrames, blockingQueue, videoProcess); + std::thread getResult(videoProcess->GetResults, blockingQueue, yolov3, videoProcess); + + if (signal(SIGINT, SigHandler) == SIG_ERR) { + LogError << "can not catch SIGINT"; + return APP_ERR_COMM_FAILURE; + } + + while (!videoProcess->stopFlag) { + sleep(10); + } + getFrame.join(); + getResult.join(); + + blockingQueue->Stop(); + blockingQueue->Clear(); + + ret = yolov3->FrameDeInit(); + if (ret != APP_ERR_OK) { + LogError << "FrameInit failed"; + return ret; + } + ret = videoProcess->StreamDeInit(); + if (ret != APP_ERR_OK) { + LogError << "StreamDeInit failed"; + return ret; + } + ret = videoProcess->VideoDecodeDeInit(); + if (ret != APP_ERR_OK) { + LogError << "VideoDecodeDeInit failed"; + return ret; + } + ret = MxBase::DeviceManager::GetInstance()->DestroyDevices(); + if (ret != APP_ERR_OK) { + LogError << "DestroyDevices failed"; + return ret; + } + return 0; +} \ No newline at end of file diff --git a/tutorials/mxBaseVideoSample/model/coco.names b/tutorials/mxBaseVideoSample/model/coco.names new file mode 100644 index 0000000000000000000000000000000000000000..1db41f581a4b1b54086cfc44f81a192b262ad63c --- /dev/null +++ b/tutorials/mxBaseVideoSample/model/coco.names @@ -0,0 +1,81 @@ +# This file is originally from https://github.com/pjreddie/darknet/blob/master/data/coco.names +person +bicycle +car +motorbike +aeroplane +bus +train +truck +boat +traffic light +fire hydrant +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +backpack +umbrella +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +sofa +pottedplant +bed +diningtable +toilet +tvmonitor +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/tutorials/mxBaseVideoSample/model/yolov3_tf_bs1_fp16.cfg b/tutorials/mxBaseVideoSample/model/yolov3_tf_bs1_fp16.cfg new file mode 100644 index 0000000000000000000000000000000000000000..e7da9ec45dc4043956e8789e6276719a8d40af59 --- /dev/null +++ b/tutorials/mxBaseVideoSample/model/yolov3_tf_bs1_fp16.cfg @@ -0,0 +1,10 @@ +CLASS_NUM=80 +BIASES_NUM=18 +BIASES=10,13,16,30,33,23,30,61,62,45,59,119,116,90,156,198,373,326 +SCORE_THRESH=0.3 +OBJECTNESS_THRESH=0.3 +IOU_THRESH=0.45 +YOLO_TYPE=3 +ANCHOR_DIM=3 +MODEL_TYPE=0 +RESIZE_FLAG=0 diff --git a/tutorials/mxBaseVideoSample/run.sh b/tutorials/mxBaseVideoSample/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..5e3d6e2fe07c1e6edcfaef9dbc017cdee607ff6f --- /dev/null +++ b/tutorials/mxBaseVideoSample/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Copyright(C) Huawei Technologies Co.,Ltd. 2012-2021 All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CUR_PATH=$(cd "$(dirname "$0")"; pwd) + +echo $CUR_PATH + +export LD_LIBRARY_PATH=${MX_SDK_HOME}/lib:${MX_SDK_HOME}/opensource/lib:${MX_SDK_HOME}/opensource/lib64:/usr/local/Ascend/ascend-toolkit/latest/acllib/lib64:/usr/local/Ascend/driver/lib64/:/usr/local/python3.7.5/lib:/usr/local/ffmpeg/lib:${LD_LIBRARY_PATH} + +./stream_pull_test \ No newline at end of file