diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index 1c964a2b7..45d9adc71 100755 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -15,6 +15,7 @@ IF(ENABLE_LINUX) ENDIF(ENABLE_LINUX) IF(ENABLE_WINDOWS) + set(VCPKG_PLATFORM_TOOLSET v143) add_compile_definitions(WINDOWS) set(VCPKG_TARGET_TRIPLET "x64-windows" CACHE STRING "x64-windows") set(VCPKG_PLATFORM_TOOLSET "v143" CACHE STRING "v143" FORCE) @@ -199,6 +200,13 @@ SET(CORE_FILES src/SimpleControlModule.cpp src/APErrorObject.cpp src/APHealthObject.cpp + .././samples/face_detection_cpu/face_detection_cpu.cpp + .././samples/create_thumbnail_from_mp4_video/generate_thumbnail_from_mp4_video.cpp + .././samples/timelapse-sample/timelapse_summary.h + .././samples/timelapse-sample/timelapse_summary.cpp + .././samples/relay-sample/relay_sample.cpp + .././samples/relay-sample/relay_sample.cpp + .././samples/timelapse-sample/timelapse_summary.h ) SET(CORE_FILES_H @@ -260,11 +268,17 @@ SET(CORE_FILES_H include/OverlayModule.h include/OrderedCacheOfFiles.h include/TestSignalGeneratorSrc.h + .././samples/timelapse-sample/timelapse_summary.cpp include/AbsControlModule.h include/SimpleControlModule.h include/APErrorObject.h include/APCallback.h include/APHealthObject.h + .././samples/face_detection_cpu/face_detection_cpu.h + .././samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.h + .././samples/timelapse-sample/timelapse_summary.h + .././samples/relay-sample/relay_sample.h + .././samples/relay-sample/relay_sample.h ) IF(ENABLE_WINDOWS) @@ -525,7 +539,6 @@ target_include_directories ( aprapipes PRIVATE ${NVCODEC_INCLUDE_DIR} ) - # aprapipes Unit Tests IF (ENABLE_ARM64) @@ -573,6 +586,35 @@ IF (ENABLE_CUDA) ENDIF(NOT ENABLE_ARM64) ENDIF(ENABLE_CUDA) +SET(SAMPLE_UT_FILES + test/utmain.cpp + test/test_utils.cpp + test/test_utils.h + .././samples/timelapse-sample/timelapse_summary_test.cpp + .././samples/create_thumbnail_from_mp4_video/test_generate_thumbnail_from_mp4_video.cpp + .././samples/face_detection_cpu/test_face_detection_cpu.cpp + .././samples/play_mp4_from_beginning/test_play_mp4_video_from_beginning.cpp + .././samples/relay-sample/relay_sample_test.cpp +) + +SET(SAMPLE_CORE_FILES + .././samples/timelapse-sample/timelapse_summary.h + .././samples/timelapse-sample/timelapse_summary.cpp + .././samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.cpp + .././samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.h + .././samples/face_detection_cpu/face_detection_cpu.h + .././samples/face_detection_cpu/face_detection_cpu.cpp + .././samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.h + .././samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.cpp + .././samples/relay-sample/relay_sample.h + .././samples/relay-sample/relay_sample.cpp +) + +SET(SAMPLE_SOURCE + ${SAMPLE_CORE_FILES} + ${SAMPLE_UT_FILES} +) + SET(UT_FILES test/utmain.cpp test/unit_tests.cpp @@ -629,10 +671,32 @@ SET(UT_FILES test/testSignalGeneratorSrc_tests.cpp test/audioToTextXform_tests.cpp test/simpleControlModuleTests.cpp + test/timelapse.cpp ${ARM64_UT_FILES} ${CUDA_UT_FILES} ) +SET(SAMPLE_UT_FILES + test/utmain.cpp + test/test_utils.cpp + test/test_utils.h + ../samples/face_detection_cpu/test_face_detection_cpu.cpp + ../samples/relay-sample/relay_sample_test.cpp + .././samples/timelapse-sample/timelapse_summary_test.cpp +) + +SET(SAMPLE_CORE_FILES + ../samples/relay-sample/relay_sample.cpp + ../samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.cpp + ../samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.cpp + ../samples/face_detection_cpu/face_detection_cpu.cpp + ../samples/timelapse-sample/timelapse_summary.cpp) + +SET(SAMPLE_SOURCE + ${SAMPLE_CORE_FILES} + ${SAMPLE_UT_FILES} +) + IF(ENABLE_LINUX) list(APPEND UT_FILES test/gtkglrenderer_tests.cpp @@ -646,6 +710,43 @@ IF(ENABLE_LINUX) ENDIF(ENABLE_LINUX) add_executable(aprapipesut ${UT_FILES}) +add_executable(aprapipessampleut ${SAMPLE_SOURCE}) + +target_include_directories ( aprapipessampleut PRIVATE + ${JETSON_MULTIMEDIA_LIB_INCLUDE} + ${FFMPEG_INCLUDE_DIRS} + ${OpenCV_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} + ${LIBMP4_INC_DIR} + ${BARESIP_INC_DIR} + ${LIBRE_INC_DIR} + ${NVCODEC_INCLUDE_DIR} + test +) + +IF(ENABLE_ARM64) + target_include_directories(aprapipesut PRIVATE ${VCPKG_GTK_INCLUDE_DIRS}) +ENDIF(ENABLE_ARM64) + +IF(ENABLE_LINUX) + target_include_directories(aprapipesut PRIVATE ${GTK3_INCLUDE_DIRS}) + target_link_libraries(aprapipesut + ${GDK3_LIBRARIES} + ${GTK3_LIBRARIES} + ) +ENDIF(ENABLE_LINUX) + +target_include_directories ( aprapipessampleut PRIVATE +${JETSON_MULTIMEDIA_LIB_INCLUDE} +${FFMPEG_INCLUDE_DIRS} +${OpenCV_INCLUDE_DIRS} +${Boost_INCLUDE_DIRS} +${LIBMP4_INC_DIR} +${BARESIP_INC_DIR} +${LIBRE_INC_DIR} +${NVCODEC_INCLUDE_DIR} +test +) IF(ENABLE_ARM64) target_include_directories ( aprapipesut PRIVATE ${JETSON_MULTIMEDIA_LIB_INCLUDE} ${FFMPEG_ROOT} ${JPEG_INCLUDE_DIR}) @@ -658,6 +759,8 @@ ENDIF (ENABLE_CUDA) find_library(OPENH264_LIB NAMES openh264.lib libopenh264.a REQUIRED) find_library(LIBMP4_LIB NAMES mp4lib.lib libmp4lib.a REQUIRED) + + IF(ENABLE_ARM64) target_include_directories(aprapipesut PRIVATE ${VCPKG_GTK_INCLUDE_DIRS}) ENDIF(ENABLE_ARM64) @@ -670,6 +773,41 @@ IF(ENABLE_LINUX) ) ENDIF(ENABLE_LINUX) +target_include_directories ( aprapipessampleut PRIVATE +${JETSON_MULTIMEDIA_LIB_INCLUDE} +${FFMPEG_INCLUDE_DIRS} +${OpenCV_INCLUDE_DIRS} +${Boost_INCLUDE_DIRS} +${LIBMP4_INC_DIR} +${BARESIP_INC_DIR} +${LIBRE_INC_DIR} +${NVCODEC_INCLUDE_DIR} +test +) + +target_link_libraries(aprapipessampleut + aprapipes + ${JPEG_LIBRARIES} + ${LIBMP4_LIB} + ${OPENH264_LIB} + ${Boost_LIBRARIES} + ${FFMPEG_LIBRARIES} + ${OpenCV_LIBRARIES} + ${JETSON_LIBS} + ${NVCUDAToolkit_LIBS} + ${NVCODEC_LIB} + ${NVJPEGLIB_L4T} + ${CURSES_LIBRARIES} + ZXing::Core + ZXing::ZXing + BZip2::BZip2 + ZLIB::ZLIB + LibLZMA::LibLZMA + bigint::bigint + sfml-audio + whisper::whisper + ) + target_link_libraries(aprapipesut aprapipes ${GLEW_LIBRARIES} @@ -679,6 +817,28 @@ target_link_libraries(aprapipesut ${Boost_LIBRARIES} ${FFMPEG_LIBRARIES} ${OpenCV_LIBRARIES} + ${NVCUDAToolkit_LIBS} + ${NVCODEC_LIB} + ${NVJPEGLIB_L4T} + ${CURSES_LIBRARIES} + ZXing::Core + ZXing::ZXing + BZip2::BZip2 + ZLIB::ZLIB + LibLZMA::LibLZMA + bigint::bigint + sfml-audio + whisper::whisper + ) + + target_link_libraries(aprapipesut + aprapipes + ${JPEG_LIBRARIES} + ${LIBMP4_LIB} + ${OPENH264_LIB} + ${Boost_LIBRARIES} + ${FFMPEG_LIBRARIES} + ${OpenCV_LIBRARIES} ${JETSON_LIBS} ${NVCUDAToolkit_LIBS} ${NVCODEC_LIB} @@ -702,6 +862,7 @@ IF(ENABLE_WINDOWS) ENDIF(GHA) ENDIF(ENABLE_WINDOWS) + include(GNUInstallDirs) # BUILD_INTERFACE specifies where to find includes during build time @@ -736,3 +897,5 @@ install( install(DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/include DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/aprapipes) + +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/../samples ${CMAKE_CURRENT_BINARY_DIR}/samples) \ No newline at end of file diff --git a/base/include/FaceDetectorXform.h b/base/include/FaceDetectorXform.h index bbbca3ebc..9ccf07e46 100644 --- a/base/include/FaceDetectorXform.h +++ b/base/include/FaceDetectorXform.h @@ -6,11 +6,14 @@ class FaceDetectorXformProps : public ModuleProps { public: - FaceDetectorXformProps(double _scaleFactor = 1.0, float _confidenceThreshold = 0.5) : scaleFactor(_scaleFactor), confidenceThreshold(_confidenceThreshold) + FaceDetectorXformProps(double _scaleFactor = 1.0, float _confidenceThreshold = 0.5, std::string _Face_Detection_Configuration= "./data/assets/deploy.prototxt", std::string _Face_Detection_Weights= "./data/assets/res10_300x300_ssd_iter_140000_fp16.caffemodel") + : scaleFactor(_scaleFactor), confidenceThreshold(_confidenceThreshold), FACE_DETECTION_CONFIGURATION(_Face_Detection_Configuration), FACE_DETECTION_WEIGHTS(_Face_Detection_Weights) { } double scaleFactor; float confidenceThreshold; + std::string FACE_DETECTION_CONFIGURATION; + std::string FACE_DETECTION_WEIGHTS; size_t getSerializeSize() { diff --git a/base/include/FacialLandmarksCV.h b/base/include/FacialLandmarksCV.h index ef52acbaa..3f94b1a0c 100644 --- a/base/include/FacialLandmarksCV.h +++ b/base/include/FacialLandmarksCV.h @@ -81,4 +81,5 @@ class FacialLandmarkCV : public Module FacialLandmarkCVProps mProp; bool handlePropsChange(frame_sp& frame); std::string mOutputPinId1; + std::string rawFramePinId; }; \ No newline at end of file diff --git a/base/include/MotionVectorExtractor.h b/base/include/MotionVectorExtractor.h index a50cd1b55..6aa39f3d7 100644 --- a/base/include/MotionVectorExtractor.h +++ b/base/include/MotionVectorExtractor.h @@ -6,18 +6,15 @@ class MvExtractDetailAbs; class DetailFfmpeg; class DetailOpenH264; -class MotionVectorExtractorProps : public ModuleProps -{ +class MotionVectorExtractorProps : public ModuleProps { public: - enum MVExtractMethod - { - FFMPEG, - OPENH264 - }; + enum MVExtractMethod { FFMPEG, OPENH264 }; - MotionVectorExtractorProps(MVExtractMethod _MVExtractMethod = MVExtractMethod::FFMPEG, bool _sendDecodedFrame = false, int _motionVectorThreshold = 2) : MVExtract(_MVExtractMethod), sendDecodedFrame(_sendDecodedFrame), motionVectorThreshold(_motionVectorThreshold) - { - } + MotionVectorExtractorProps( + MVExtractMethod _MVExtractMethod = MVExtractMethod::FFMPEG, + bool _sendDecodedFrame = false, int _motionVectorThreshold = 2) + : MVExtract(_MVExtractMethod), sendDecodedFrame(_sendDecodedFrame), + motionVectorThreshold(_motionVectorThreshold) {} size_t getSerializeSize() { @@ -26,26 +23,26 @@ class MotionVectorExtractorProps : public ModuleProps bool sendDecodedFrame = false; int motionVectorThreshold; MVExtractMethod MVExtract = MVExtractMethod::FFMPEG; + private: friend class boost::serialization::access; template - void serialize(Archive& ar, const unsigned int version) - { + void serialize(Archive& ar, const unsigned int version) { ar& boost::serialization::base_object(*this); ar& sendDecodedFrame; ar& motionVectorThreshold; } }; -class MotionVectorExtractor : public Module -{ +class MotionVectorExtractor : public Module { public: MotionVectorExtractor(MotionVectorExtractorProps _props); virtual ~MotionVectorExtractor() {}; bool init(); bool term(); void setProps(MotionVectorExtractorProps& props); + protected: bool process(frame_container& frame); bool validateInputPins(); @@ -54,8 +51,9 @@ class MotionVectorExtractor : public Module bool processSOS(frame_sp& frame); void setMetadata(frame_sp metadata); bool handlePropsChange(frame_sp& frame); + private: boost::shared_ptr mDetail; framemetadata_sp rawOutputMetadata; bool mShouldTriggerSOS = true; -}; +}; \ No newline at end of file diff --git a/base/src/FaceDetectorXform.cpp b/base/src/FaceDetectorXform.cpp index 9fc912cfb..84f2836b2 100644 --- a/base/src/FaceDetectorXform.cpp +++ b/base/src/FaceDetectorXform.cpp @@ -11,6 +11,7 @@ #include "AIPExceptions.h" #include "ApraFaceInfo.h" #include "FaceDetectsInfo.h" +#include "Overlay.h" class FaceDetectorXform::Detail { @@ -34,13 +35,12 @@ class FaceDetectorXform::Detail framemetadata_sp mOutputMetadata; FaceDetectorXformProps mProps; std::string mOutputPinId; + std::string rawFramePinId; cv::Mat mInputImg; int mFrameType; cv::dnn::Net network; cv::Mat inputBlob; cv::Mat detection; - const std::string FACE_DETECTION_CONFIGURATION = "./data/assets/deploy.prototxt"; - const std::string FACE_DETECTION_WEIGHTS = "./data/assets/res10_300x300_ssd_iter_140000_fp16.caffemodel"; // scalar with mean values which are subtracted from channels. // Values are intended to be in (mean-R, mean-G, mean-B) order if image has BGR ordering and swapRB is true. const cv::Scalar meanValuesRGB = cv::Scalar({104., 177.0, 123.0}); @@ -75,7 +75,7 @@ bool FaceDetectorXform::validateInputPins() bool FaceDetectorXform::validateOutputPins() { - if (getNumberOfOutputPins() != 1) + if (getNumberOfOutputPins() > 2) { LOG_ERROR << "<" << getId() << ">::validateOutputPins size is expected to be 1. Actual<" << getNumberOfOutputPins() << ">"; return false; @@ -88,11 +88,12 @@ void FaceDetectorXform::addInputPin(framemetadata_sp &metadata, string &pinId) Module::addInputPin(metadata, pinId); mDetail->mOutputMetadata = framemetadata_sp(new FrameMetadata(FrameMetadata::FACEDETECTS_INFO)); mDetail->mOutputPinId = addOutputPin(mDetail->mOutputMetadata); + mDetail->rawFramePinId = addOutputPin(metadata); } bool FaceDetectorXform::init() { - mDetail->network = cv::dnn::readNetFromCaffe(mDetail->FACE_DETECTION_CONFIGURATION, mDetail->FACE_DETECTION_WEIGHTS); + mDetail->network = cv::dnn::readNetFromCaffe(mDetail->mProps.FACE_DETECTION_CONFIGURATION, mDetail->mProps.FACE_DETECTION_WEIGHTS); if (mDetail->network.empty()) { LOG_ERROR << "Failed to load network with the given settings. Please check the loaded parameters."; @@ -121,11 +122,14 @@ bool FaceDetectorXform::process(frame_container &frames) mDetail->inputBlob = cv::dnn::blobFromImage(mDetail->mInputImg, mDetail->mProps.scaleFactor, cv::Size(mDetail->mInputImg.cols, mDetail->mInputImg.rows), mDetail->meanValuesRGB, false, false); mDetail->network.setInput(mDetail->inputBlob, "data"); - + mDetail->detection = mDetail->network.forward("detection_out"); cv::Mat detectionMatrix(mDetail->detection.size[2], mDetail->detection.size[3], CV_32F, mDetail->detection.ptr()); + std::vector rectangleOverlays; + CompositeOverlay compositeOverlay; + for (int i = 0; i < detectionMatrix.rows; i++) { float confidence = detectionMatrix.at(i, 2); @@ -136,20 +140,42 @@ bool FaceDetectorXform::process(frame_container &frames) } mDetail->faceInfo.x1 = detectionMatrix.at(i, 3) * mDetail->mInputImg.cols; - mDetail->faceInfo.y2 = detectionMatrix.at(i, 4) * mDetail->mInputImg.rows; + mDetail->faceInfo.y1 = detectionMatrix.at(i, 4) * mDetail->mInputImg.rows; mDetail->faceInfo.x2 = detectionMatrix.at(i, 5) * mDetail->mInputImg.cols; - mDetail->faceInfo.y1 = detectionMatrix.at(i, 6) * mDetail->mInputImg.rows; + mDetail->faceInfo.y2 = detectionMatrix.at(i, 6) * mDetail->mInputImg.rows; mDetail->faceInfo.score = confidence; mDetail->faces.emplace_back(mDetail->faceInfo); + + RectangleOverlay rectangleOverlay; + rectangleOverlay.x1 = mDetail->faceInfo.x1; + rectangleOverlay.y1 = mDetail->faceInfo.y1; + rectangleOverlay.x2 = mDetail->faceInfo.x2; + rectangleOverlay.y2 = mDetail->faceInfo.y2; + rectangleOverlays.push_back(rectangleOverlay); } + for (auto &rectangleOverlay : rectangleOverlays) { + compositeOverlay.add(&rectangleOverlay); + } + + auto rawFrame = frames.cbegin()->second; + frames.insert(make_pair(mDetail->rawFramePinId, rawFrame)); + mDetail->faceDetectsInfo.faces = mDetail->faces; - auto outFrame = makeFrame(mDetail->faceDetectsInfo.getSerializeSize()); - mDetail->faceDetectsInfo.serialize(outFrame->data(), mDetail->faceDetectsInfo.getSerializeSize()); - frames.insert(make_pair(mDetail->mOutputPinId, outFrame)); + + if (rectangleOverlays.size() > 0) { + DrawingOverlay drawingOverlay; + drawingOverlay.add(&compositeOverlay); + auto mvSize = drawingOverlay.mGetSerializeSize(); + auto outFrame = makeFrame(mvSize, mDetail->mOutputPinId); + drawingOverlay.serialize(outFrame); + frames.insert(make_pair(mDetail->mOutputPinId, outFrame)); + } + mDetail->faces.clear(); mDetail->faceDetectsInfo.faces.clear(); + send(frames); return true; } diff --git a/base/src/FacialLandmarksCV.cpp b/base/src/FacialLandmarksCV.cpp index a9ef0ce5a..02b55b1cd 100644 --- a/base/src/FacialLandmarksCV.cpp +++ b/base/src/FacialLandmarksCV.cpp @@ -12,6 +12,7 @@ #include "Logger.h" #include "Utils.h" #include "AIPExceptions.h" +#include "Overlay.h" class Detail { @@ -87,6 +88,7 @@ class Detail FacialLandmarkCVProps props; cv::Mat iImg; vector>landmarks; + vector faces; protected: framemetadata_sp mInputMetadata; @@ -113,6 +115,7 @@ class DetailSSD : public Detail { //input must be 3 channel image(RGB) // Create a 4-dimensional blob from the image. Optionally resizes and crops image from center, subtract mean values, scales values by scalefactor, swap Blue and Red channels. + iImg.data = static_cast(buffer->data()); cv::Mat inputBlob = cv::dnn::blobFromImage(iImg, 1.0, cv::Size(300, 300), cv::Scalar(104, 177, 123), false, false); // Set the input blob as input to the face detector network @@ -123,8 +126,6 @@ class DetailSSD : public Detail cv::Mat detectionMatrix(detection.size[2], detection.size[3], CV_32F, detection.ptr()); - vector faces; - for (int i = 0; i < detectionMatrix.rows; i++) { float confidence = detectionMatrix.at(i, 2); @@ -139,13 +140,16 @@ class DetailSSD : public Detail cv::Rect faceRect(x1, y1, x2 - x1, y2 - y1); faces.push_back(faceRect); - cv::rectangle(iImg, faceRect, cv::Scalar(0, 255, 0), 2); } } + if (faces.size() == 0) { + return false; + } + bool success = facemark->fit(iImg, faces, landmarks); - return true; + return success; } private: @@ -164,17 +168,15 @@ class DetailHCASCADE : public Detail bool compute(frame_sp buffer) { - vector faces; faceDetector.detectMultiScale(iImg, faces); - for (int i = 0; i < faces.size(); i++) - { - rectangle(iImg, faces[i], cv::Scalar(0, 255, 0), 2); + if (faces.size() == 0) { + return false; } - bool success = facemark->fit(iImg, faces, landmarks); - - return true; + bool success = facemark->fit(iImg, faces, landmarks); + + return success; } private: @@ -243,6 +245,7 @@ void FacialLandmarkCV::addInputPin(framemetadata_sp &metadata, string &pinId) Module::addInputPin(metadata, pinId); auto landmarksOutputMetadata = framemetadata_sp(new FrameMetadata(FrameMetadata::FACE_LANDMARKS_INFO)); mOutputPinId1 = addOutputPin(landmarksOutputMetadata); + rawFramePinId = addOutputPin(metadata); } bool FacialLandmarkCV::init() @@ -278,8 +281,26 @@ bool FacialLandmarkCV::term() bool FacialLandmarkCV::process(frame_container& frames) { auto frame = frames.cbegin()->second; + bool computeValue = mDetail->compute(frame); + + if (computeValue == false) { + send(frames); + return true; + } + + std::vector rectangleOverlays; + + for (const auto& face :mDetail->faces) { + RectangleOverlay rectangleOverlay; + rectangleOverlay.x1 = face.x; + rectangleOverlay.y1 = face.y; + rectangleOverlay.x2 = face.x + face.width; + rectangleOverlay.y2 = face.y + face.height; - mDetail->compute(frame); + rectangleOverlays.push_back(rectangleOverlay); + } + + std::vector circleOverlays; // Convert the landmarks from cv::Point2f to ApraPoint2f vector> apralandmarks; @@ -287,6 +308,13 @@ bool FacialLandmarkCV::process(frame_container& frames) vector apralandmark; for (const auto& point : landmark) { apralandmark.emplace_back(ApraPoint2f(point)); + + CircleOverlay circleOverlay; + circleOverlay.x1 = point.x; + circleOverlay.y1 = point.y; + circleOverlay.radius = 1; + + circleOverlays.push_back(circleOverlay); } apralandmarks.emplace_back(std::move(apralandmark)); } @@ -297,11 +325,31 @@ bool FacialLandmarkCV::process(frame_container& frames) bufferSize += sizeof(apralandmarks[i]) + (sizeof(ApraPoint2f) + 2 * sizeof(int)) * apralandmarks[i].size(); } - auto landmarksFrame = makeFrame(bufferSize); + CompositeOverlay compositeOverlay; - Utils::serialize>>(apralandmarks, landmarksFrame->data(), bufferSize); + for (auto &rectangleOverlay : rectangleOverlays) { + compositeOverlay.add(&rectangleOverlay); + } + + for (auto &circleOverlay : circleOverlays) { + compositeOverlay.add(&circleOverlay); + } + + auto rawFrame = frames.cbegin()->second; + + frames.insert(make_pair(rawFramePinId, rawFrame)); + + if (rectangleOverlays.size() > 0 || circleOverlays.size() > 0) { + DrawingOverlay drawingOverlay; + drawingOverlay.add(&compositeOverlay); + auto mvSize = drawingOverlay.mGetSerializeSize(); + auto landmarksFrame = makeFrame(mvSize, mOutputPinId1); + drawingOverlay.serialize(landmarksFrame); + frames.insert(make_pair(mOutputPinId1, landmarksFrame)); + } - frames.insert(make_pair(mOutputPinId1, landmarksFrame)); + mDetail->faces.clear(); + mDetail->landmarks.clear(); send(frames); diff --git a/base/src/MotionVectorExtractor.cpp b/base/src/MotionVectorExtractor.cpp index e274351b7..fdfdf29ac 100644 --- a/base/src/MotionVectorExtractor.cpp +++ b/base/src/MotionVectorExtractor.cpp @@ -18,44 +18,53 @@ extern "C" class MvExtractDetailAbs { public: - MvExtractDetailAbs(MotionVectorExtractorProps props, std::function _makeFrameWithPinId, std::function _makeframe) - { + MvExtractDetailAbs( + MotionVectorExtractorProps props, + std::function _makeFrameWithPinId, + std::function + _makeframe) { makeFrameWithPinId = _makeFrameWithPinId; makeframe = _makeframe; sendDecodedFrame = props.sendDecodedFrame; threshold = props.motionVectorThreshold; }; - ~MvExtractDetailAbs() - { - } - virtual void setProps(MotionVectorExtractorProps props) - { + ~MvExtractDetailAbs() {} + virtual void setProps(MotionVectorExtractorProps props) { sendDecodedFrame = props.sendDecodedFrame; } - virtual void getMotionVectors(frame_container& frames, frame_sp& outFrame, frame_sp& decodedFrame) = 0; + virtual void getMotionVectors(frame_container& frames, frame_sp& outFrame, + frame_sp& decodedFrame) = 0; virtual void initDecoder() = 0; + public: int mWidth = 0; int mHeight = 0; std::string rawFramePinId; std::string motionVectorPinId; - std::function makeframe; + std::function + makeframe; std::function makeFrameWithPinId; bool sendDecodedFrame = false; int threshold; cv::Mat bgrImg; + bool motionFound = false; }; class DetailFfmpeg : public MvExtractDetailAbs { public: - DetailFfmpeg(MotionVectorExtractorProps props, std::function _makeFrameWithPinId, std::function _makeframe) : MvExtractDetailAbs(props, _makeFrameWithPinId, _makeframe) {} - ~DetailFfmpeg() - { - avcodec_free_context(&decoderContext); - } - void getMotionVectors(frame_container& frames, frame_sp& outFrame, frame_sp& decodedFrame); + DetailFfmpeg( + MotionVectorExtractorProps props, + std::function _makeFrameWithPinId, + std::function + _makeframe) + : MvExtractDetailAbs(props, _makeFrameWithPinId, _makeframe) {} + ~DetailFfmpeg() { avcodec_free_context(&decoderContext); } + void getMotionVectors(frame_container& frames, frame_sp& outFrame, + frame_sp& decodedFrame); void initDecoder(); - int decodeAndGetMotionVectors(AVPacket* pkt, frame_container& frames, frame_sp& outFrame, frame_sp& decodedFrame); + int decodeAndGetMotionVectors(AVPacket* pkt, frame_container& frames, + frame_sp& outFrame, frame_sp& decodedFrame); + private: AVFrame* avFrame = NULL; AVCodecContext* decoderContext = NULL; @@ -63,89 +72,112 @@ class DetailFfmpeg : public MvExtractDetailAbs class DetailOpenH264 : public MvExtractDetailAbs { public: - DetailOpenH264(MotionVectorExtractorProps props, std::function _makeFrameWithPinId, std::function _makeframe) : MvExtractDetailAbs(props, _makeFrameWithPinId, _makeframe) {} + DetailOpenH264( + MotionVectorExtractorProps props, + std::function _makeFrameWithPinId, + std::function + _makeframe) + : MvExtractDetailAbs(props, _makeFrameWithPinId, _makeframe) {} ~DetailOpenH264() { if (pDecoder) { pDecoder->Uninitialize(); WelsDestroyDecoder(pDecoder); } } - void getMotionVectors(frame_container& frames, frame_sp& outFrame, frame_sp& decodedFrame); + void getMotionVectors(frame_container& frames, frame_sp& outFrame, + frame_sp& decodedFrame); void initDecoder(); + private: ISVCDecoder* pDecoder; SBufferInfo pDstInfo; SDecodingParam sDecParam; SParserBsInfo parseInfo; }; -void DetailFfmpeg::initDecoder() -{ +void DetailFfmpeg::initDecoder() { int ret; AVCodec* dec = NULL; AVDictionary* opts = NULL; dec = avcodec_find_decoder(AV_CODEC_ID_H264); decoderContext = avcodec_alloc_context3(dec); - if (!decoderContext) - { + if (!decoderContext) { throw AIPException(AIP_FATAL, "Failed to allocate codec"); } /* Init the decoder */ av_dict_set(&opts, "flags2", "+export_mvs", 0); ret = avcodec_open2(decoderContext, dec, &opts); av_dict_free(&opts); - if (ret < 0) - { + if (ret < 0) { throw AIPException(AIP_FATAL, "failed open decoder"); } } -void DetailFfmpeg::getMotionVectors(frame_container& frames, frame_sp& outFrame, frame_sp& decodedFrame) -{ + +bool discardNoisyFrames(int black_pixel_percent, cv::Mat image) { + cv::Mat gray; + cv::cvtColor(image, gray, cv::COLOR_BGR2GRAY); + + cv::Mat binary; + cv::threshold(gray, binary, 128, 255, cv::THRESH_BINARY); + int totalPixels = binary.total(); + int whitePixels = cv::countNonZero(binary); + int blackPixels = totalPixels - whitePixels; + + double blackPixelPercentage = (static_cast(blackPixels) / static_cast(totalPixels)) * 100.0; + if (blackPixelPercentage > black_pixel_percent || whitePixels == totalPixels) + return true; + else + return false; +} + +void DetailFfmpeg::getMotionVectors(frame_container& frames, frame_sp& outFrame, + frame_sp& decodedFrame) { int ret = 0; AVPacket* pkt = NULL; avFrame = av_frame_alloc(); - if (!avFrame) - { + if (!avFrame) { LOG_ERROR << "Could not allocate frame\n"; } pkt = av_packet_alloc(); - if (!pkt) - { + if (!pkt) { LOG_ERROR << "Could not allocate AVPacket\n"; } - ret = decodeAndGetMotionVectors(pkt, frames, outFrame, decodedFrame); - av_packet_free(&pkt); - av_frame_free(&avFrame); + bgrImg.data = static_cast(decodedFrame->data()); + if (!discardNoisyFrames(99, bgrImg)) { + ret = decodeAndGetMotionVectors(pkt, frames, outFrame, decodedFrame); + av_packet_free(&pkt); + av_frame_free(&avFrame); + } } -int DetailFfmpeg::decodeAndGetMotionVectors(AVPacket* pkt, frame_container& frames, frame_sp& outFrame, frame_sp& decodedFrame) -{ + +int DetailFfmpeg::decodeAndGetMotionVectors(AVPacket* pkt, + frame_container& frames, + frame_sp& outFrame, + frame_sp& decodedFrame) { auto inFrame = frames.begin()->second; pkt->data = (uint8_t*)inFrame->data(); pkt->size = (int)inFrame->size(); int ret = avcodec_send_packet(decoderContext, pkt); - if (ret < 0) - { + if (ret < 0) { LOG_ERROR << stderr << "Error while sending a packet to the decoder: %s\n"; return ret; } - while (ret >= 0) - { + while (ret >= 0) { ret = avcodec_receive_frame(decoderContext, avFrame); - if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) - { + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { outFrame = makeFrameWithPinId(0, motionVectorPinId); break; } - else if (ret < 0) - { - LOG_ERROR << stderr << "Error while receiving a frame from the decoder: %s\n"; + else if (ret < 0) { + LOG_ERROR << stderr + << "Error while receiving a frame from the decoder: %s\n"; return ret; } - if (sendDecodedFrame) - { - SwsContext* sws_context = sws_getContext( - decoderContext->width, decoderContext->height, decoderContext->pix_fmt, - decoderContext->width, decoderContext->height, AV_PIX_FMT_BGR24, - SWS_BICUBIC | SWS_FULL_CHR_H_INT, NULL, NULL, NULL); + if (sendDecodedFrame) { + SwsContext* sws_context = + sws_getContext(decoderContext->width, decoderContext->height, + decoderContext->pix_fmt, decoderContext->width, + decoderContext->height, AV_PIX_FMT_BGR24, + SWS_BICUBIC | SWS_FULL_CHR_H_INT, NULL, NULL, NULL); if (!sws_context) { // Handle error } @@ -154,25 +186,23 @@ int DetailFfmpeg::decodeAndGetMotionVectors(AVPacket* pkt, frame_container& fram dstStrides[0] = decoderContext->width * 3; // Assuming BGR format uint8_t* dstData[AV_NUM_DATA_POINTERS]; dstData[0] = static_cast(decodedFrame->data()); - sws_scale(sws_context, avFrame->data, avFrame->linesize, 0, decoderContext->height, dstData, dstStrides); + sws_scale(sws_context, avFrame->data, avFrame->linesize, 0, + decoderContext->height, dstData, dstStrides); frames.insert(make_pair(rawFramePinId, decodedFrame)); } - if (ret >= 0) - { + if (ret >= 0) { AVFrameSideData* sideData; sideData = av_frame_get_side_data(avFrame, AV_FRAME_DATA_MOTION_VECTORS); - if (sideData) - { + if (sideData) { std::vector lineOverlays; CompositeOverlay compositeOverlay; const AVMotionVector* mvs = (const AVMotionVector*)sideData->data; - for (int i = 0; i < sideData->size / sizeof(*mvs); i++) - { + for (int i = 0; i < sideData->size / sizeof(*mvs); i++) { const AVMotionVector* mv = &mvs[i]; - if (std::abs(mv->motion_x) > threshold || std::abs(mv->motion_y) > threshold) - { + if (std::abs(mv->motion_x) > threshold || + std::abs(mv->motion_y) > threshold) { LineOverlay lineOverlay; lineOverlay.x1 = mv->src_x; lineOverlay.y1 = mv->src_y; @@ -187,8 +217,7 @@ int DetailFfmpeg::decodeAndGetMotionVectors(AVPacket* pkt, frame_container& fram compositeOverlay.add(&lineOverlay); } - if (lineOverlays.size()) - { + if (lineOverlays.size()) { DrawingOverlay drawingOverlay; drawingOverlay.add(&compositeOverlay); auto serializedSize = drawingOverlay.mGetSerializeSize(); @@ -198,8 +227,7 @@ int DetailFfmpeg::decodeAndGetMotionVectors(AVPacket* pkt, frame_container& fram frames.insert(make_pair(motionVectorPinId, outFrame)); } } - else - { + else { outFrame = makeFrameWithPinId(0, motionVectorPinId); } av_packet_unref(pkt); @@ -210,8 +238,7 @@ int DetailFfmpeg::decodeAndGetMotionVectors(AVPacket* pkt, frame_container& fram return 0; } -void DetailOpenH264::initDecoder() -{ +void DetailOpenH264::initDecoder() { sDecParam = { 0 }; if (!sendDecodedFrame) { sDecParam.bParseOnly = true; @@ -230,8 +257,9 @@ void DetailOpenH264::initDecoder() } } -void DetailOpenH264::getMotionVectors(frame_container& frames, frame_sp& outFrame, frame_sp& decodedFrame) -{ +void DetailOpenH264::getMotionVectors(frame_container& frames, + frame_sp& outFrame, + frame_sp& decodedFrame) { uint8_t* pData[3] = { NULL }; pData[0] = NULL; pData[1] = NULL; @@ -245,28 +273,34 @@ void DetailOpenH264::getMotionVectors(frame_container& frames, frame_sp& outFram int16_t* mMotionVectorData = nullptr; memset(&pDstInfo, 0, sizeof(SBufferInfo)); outFrame = makeFrameWithPinId(mMotionVectorSize, motionVectorPinId); - mMotionVectorData = static_cast(outFrame->data()); + mMotionVectorData = + static_cast(malloc(mMotionVectorSize * sizeof(int16_t))); - if (sDecParam.bParseOnly) - { - pDecoder->ParseBitstreamGetMotionVectors(pSrc, iSrcLen, ppDst, &parseInfo, &pDstInfo, &mMotionVectorSize, &mMotionVectorData); + if (sDecParam.bParseOnly) { + pDecoder->ParseBitstreamGetMotionVectors(pSrc, iSrcLen, ppDst, &parseInfo, + &pDstInfo, &mMotionVectorSize, + &mMotionVectorData); } - else - { - pDecoder->DecodeFrameGetMotionVectorsNoDelay(pSrc, iSrcLen, ppDst, &pDstInfo, &mMotionVectorSize, &mMotionVectorData); + else { + pDecoder->DecodeFrameGetMotionVectorsNoDelay(pSrc, iSrcLen, ppDst, + &pDstInfo, &mMotionVectorSize, + &mMotionVectorData); } - - if (mMotionVectorSize != mWidth * mHeight * 8) - { + int avgX = 0; + int avgY = 0; + if (mMotionVectorSize != mWidth * mHeight * 8) { std::vector circleOverlays; CompositeOverlay compositeOverlay; - - for (int i = 0; i < mMotionVectorSize; i += 4) - { + int totalX = 0; + int totalY = 0; + avgX = 0; + avgY = 0; + for (int i = 0; i < mMotionVectorSize; i += 4) { auto motionX = mMotionVectorData[i]; auto motionY = mMotionVectorData[i + 1]; - if (abs(motionX) > threshold || abs(motionY) > threshold) - { + totalX = totalX + motionX; + totalY = totalY + motionY; + if (abs(motionX) > threshold || abs(motionY) > threshold) { CircleOverlay circleOverlay; circleOverlay.x1 = mMotionVectorData[i + 2]; circleOverlay.y1 = mMotionVectorData[i + 3]; @@ -275,13 +309,17 @@ void DetailOpenH264::getMotionVectors(frame_container& frames, frame_sp& outFram circleOverlays.push_back(circleOverlay); } } - + avgX = totalX / (mMotionVectorSize / 4); + avgY = totalY / (mMotionVectorSize / 4); for (auto& circleOverlay : circleOverlays) { compositeOverlay.add(&circleOverlay); } + if (mMotionVectorSize != mWidth * mHeight * 8 && sendOverlayFrame == true) + { + std::vector circleOverlays; + CompositeOverlay compositeOverlay; - if (circleOverlays.size()) - { + if (circleOverlays.size()) { DrawingOverlay drawingOverlay; drawingOverlay.add(&compositeOverlay); auto mvSize = drawingOverlay.mGetSerializeSize(); @@ -290,131 +328,162 @@ void DetailOpenH264::getMotionVectors(frame_container& frames, frame_sp& outFram frames.insert(make_pair(motionVectorPinId, outFrame)); } } - - if ((!sDecParam.bParseOnly) && (pDstInfo.pDst[0] != nullptr) && (mMotionVectorSize != mWidth * mHeight * 8)) - { + if ((!sDecParam.bParseOnly) && (pDstInfo.pDst[0] != nullptr) && + (mMotionVectorSize != mWidth * mHeight * 8) && + (abs(avgX) > threshold || abs(avgY) > threshold)) { + int rowIndex; + unsigned char* pPtr = NULL; decodedFrame = makeFrameWithPinId(mHeight * 3 * mWidth, rawFramePinId); - uint8_t* yuvImagePtr = (uint8_t*)malloc(mHeight * 1.5 * pDstInfo.UsrData.sSystemBuffer.iStride[0]); + uint8_t* yuvImagePtr = (uint8_t*)malloc(mHeight * 1.5 * mWidth); auto yuvStartPointer = yuvImagePtr; - unsigned char* pY = pDstInfo.pDst[0]; - memcpy(yuvImagePtr, pY, pDstInfo.UsrData.sSystemBuffer.iStride[0] * mHeight); - unsigned char* pU = pDstInfo.pDst[1]; - yuvImagePtr += pDstInfo.UsrData.sSystemBuffer.iStride[0] * mHeight; - memcpy(yuvImagePtr, pU, pDstInfo.UsrData.sSystemBuffer.iStride[1] * mHeight / 2); - unsigned char* pV = pDstInfo.pDst[2]; - yuvImagePtr += pDstInfo.UsrData.sSystemBuffer.iStride[1] * mHeight / 2; - memcpy(yuvImagePtr, pV, pDstInfo.UsrData.sSystemBuffer.iStride[1] * mHeight / 2); + pPtr = pData[0]; + for (rowIndex = 0; rowIndex < mHeight; rowIndex++) { + memcpy(yuvImagePtr, pPtr, mWidth); + pPtr += pDstInfo.UsrData.sSystemBuffer.iStride[0]; + yuvImagePtr += mWidth; + } + int halfHeight = mHeight / 2; + int halfWidth = mWidth / 2; + pPtr = pData[1]; + for (rowIndex = 0; rowIndex < halfHeight; rowIndex++) { + memcpy(yuvImagePtr, pPtr, halfWidth); + pPtr += pDstInfo.UsrData.sSystemBuffer.iStride[1]; + yuvImagePtr += halfWidth; + } + pPtr = pData[2]; + for (rowIndex = 0; rowIndex < halfHeight; rowIndex++) { + memcpy(yuvImagePtr, pPtr, halfWidth); + pPtr += pDstInfo.UsrData.sSystemBuffer.iStride[1]; + yuvImagePtr += halfWidth; + } - cv::Mat yuvImgCV = cv::Mat(mHeight + mHeight / 2, mWidth, CV_8UC1, yuvStartPointer, pDstInfo.UsrData.sSystemBuffer.iStride[0]); + cv::Mat yuvImgCV = cv::Mat(mHeight + mHeight / 2, mWidth, CV_8UC1, + yuvStartPointer, mWidth); bgrImg.data = static_cast(decodedFrame->data()); - - cv::cvtColor(yuvImgCV, bgrImg, cv::COLOR_YUV2BGR_I420); - frames.insert(make_pair(rawFramePinId, decodedFrame)); + if (!discardNoisyFrames(99, bgrImg)) { + cv::cvtColor(yuvImgCV, bgrImg, cv::COLOR_YUV2BGR_I420); + frames.insert(make_pair(rawFramePinId, decodedFrame)); + } + free(yuvStartPointer); } + free(mMotionVectorData); } -MotionVectorExtractor::MotionVectorExtractor(MotionVectorExtractorProps props) : Module(TRANSFORM, "MotionVectorExtractor", props) -{ - if (props.MVExtract == MotionVectorExtractorProps::MVExtractMethod::FFMPEG) - { - mDetail.reset(new DetailFfmpeg(props, [&](size_t size, string& pinId) -> frame_sp { return makeFrame(size, pinId); }, [&](frame_sp& frame, size_t& size, string& pinId) -> frame_sp { return makeFrame(frame, size, pinId); })); + +MotionVectorExtractor::MotionVectorExtractor(MotionVectorExtractorProps props) + : Module(TRANSFORM, "MotionVectorExtractor", props) { + if (props.MVExtract == MotionVectorExtractorProps::MVExtractMethod::FFMPEG) { + mDetail.reset(new DetailFfmpeg( + props, + [&](size_t size, string& pinId) -> frame_sp { + return makeFrame(size, pinId); + }, + [&](frame_sp& frame, size_t& size, string& pinId) -> frame_sp { + return makeFrame(frame, size, pinId); + })); } - else if (props.MVExtract == MotionVectorExtractorProps::MVExtractMethod::OPENH264) - { - mDetail.reset(new DetailOpenH264(props, [&](size_t size, string& pinId) -> frame_sp { return makeFrame(size, pinId); }, [&](frame_sp& frame, size_t& size, string& pinId) -> frame_sp { return makeFrame(frame, size, pinId); })); + else if (props.MVExtract == + MotionVectorExtractorProps::MVExtractMethod::OPENH264) { + mDetail.reset(new DetailOpenH264( + props, + [&](size_t size, string& pinId) -> frame_sp { + return makeFrame(size, pinId); + }, + [&](frame_sp& frame, size_t& size, string& pinId) -> frame_sp { + return makeFrame(frame, size, pinId); + })); } - auto motionVectorOutputMetadata = framemetadata_sp(new FrameMetadata(FrameMetadata::OVERLAY_INFO_IMAGE)); + auto motionVectorOutputMetadata = + framemetadata_sp(new FrameMetadata(FrameMetadata::OVERLAY_INFO_IMAGE)); rawOutputMetadata = framemetadata_sp(new RawImageMetadata()); mDetail->motionVectorPinId = addOutputPin(motionVectorOutputMetadata); mDetail->rawFramePinId = addOutputPin(rawOutputMetadata); } -bool MotionVectorExtractor::init() -{ + +bool MotionVectorExtractor::init() { mDetail->initDecoder(); return Module::init(); } -bool MotionVectorExtractor::term() -{ - return Module::term(); -} -bool MotionVectorExtractor::validateInputPins() -{ - if (getNumberOfInputPins() != 1) - { - LOG_ERROR << "<" << getId() << ">::validateInputPins size is expected to be 1. Actual<" << getNumberOfInputPins() << ">"; +bool MotionVectorExtractor::term() { return Module::term(); } +bool MotionVectorExtractor::validateInputPins() { + if (getNumberOfInputPins() != 1) { + LOG_ERROR << "<" << getId() + << ">::validateInputPins size is expected to be 1. Actual<" + << getNumberOfInputPins() << ">"; return false; } framemetadata_sp metadata = getFirstInputMetadata(); FrameMetadata::FrameType frameType = metadata->getFrameType(); - if (frameType != FrameMetadata::H264_DATA) - { - LOG_ERROR << "<" << getId() << ">::validateInputPins input frameType is expected to be H264_DATA. Actual<" << frameType << ">"; + if (frameType != FrameMetadata::H264_DATA) { + LOG_ERROR << "<" << getId() + << ">::validateInputPins input frameType is expected to be " + "H264_DATA. Actual<" + << frameType << ">"; return false; } return true; } -bool MotionVectorExtractor::validateOutputPins() -{ +bool MotionVectorExtractor::validateOutputPins() { auto size = getNumberOfOutputPins(); - if (getNumberOfOutputPins() > 2) - { - LOG_ERROR << "<" << getId() << ">::validateOutputPins size is expected to be 2. Actual<" << getNumberOfOutputPins() << ">"; + if (getNumberOfOutputPins() > 2) { + LOG_ERROR << "<" << getId() + << ">::validateOutputPins size is expected to be 2. Actual<" + << getNumberOfOutputPins() << ">"; return false; } - pair me; // map element + pair me; // map element auto framefactoryByPin = getOutputFrameFactory(); - BOOST_FOREACH(me, framefactoryByPin) - { - FrameMetadata::FrameType frameType = me.second->getFrameMetadata()->getFrameType(); - if (frameType != FrameMetadata::OVERLAY_INFO_IMAGE && frameType != FrameMetadata::RAW_IMAGE) - { - LOG_ERROR << "<" << getId() << ">::validateOutputPins input frameType is expected to be MOTION_VECTOR_DATA or RAW_IMAGE. Actual<" << frameType << ">"; + BOOST_FOREACH(me, framefactoryByPin) { + FrameMetadata::FrameType frameType = + me.second->getFrameMetadata()->getFrameType(); + if (frameType != FrameMetadata::OVERLAY_INFO_IMAGE && + frameType != FrameMetadata::RAW_IMAGE) { + LOG_ERROR << "<" << getId() + << ">::validateOutputPins input frameType is expected to be " + "MOTION_VECTOR_DATA or RAW_IMAGE. Actual<" + << frameType << ">"; return false; } } return true; } -bool MotionVectorExtractor::shouldTriggerSOS() -{ - return mShouldTriggerSOS; -} -bool MotionVectorExtractor::process(frame_container& frames) -{ +bool MotionVectorExtractor::shouldTriggerSOS() { return mShouldTriggerSOS; } +bool MotionVectorExtractor::process(frame_container& frames) { frame_sp motionVectorFrame; frame_sp decodedFrame; mDetail->getMotionVectors(frames, motionVectorFrame, decodedFrame); send(frames); return true; } -void MotionVectorExtractor::setMetadata(frame_sp frame) -{ +void MotionVectorExtractor::setMetadata(frame_sp frame) { auto metadata = frame->getMetadata(); - if (!metadata->isSet()) - { + if (!metadata->isSet()) { return; } sps_pps_properties p; - H264ParserUtils::parse_sps(((const char*)frame->data()) + 5, frame->size() > 5 ? frame->size() - 5 : frame->size(), &p); + H264ParserUtils::parse_sps( + ((const char*)frame->data()) + 5, + frame->size() > 5 ? frame->size() - 5 : frame->size(), &p); mDetail->mWidth = p.width; mDetail->mHeight = p.height; - RawImageMetadata outputMetadata(mDetail->mWidth, mDetail->mHeight, ImageMetadata::BGR, CV_8UC3, 0, CV_8U, FrameMetadata::HOST, true); - auto rawOutMetadata = FrameMetadataFactory::downcast(rawOutputMetadata); + RawImageMetadata outputMetadata(mDetail->mWidth, mDetail->mHeight, + ImageMetadata::BGR, CV_8UC3, 0, CV_8U, + FrameMetadata::HOST, true); + auto rawOutMetadata = + FrameMetadataFactory::downcast(rawOutputMetadata); rawOutMetadata->setData(outputMetadata); mDetail->bgrImg = Utils::getMatHeader(rawOutMetadata); } -bool MotionVectorExtractor::processSOS(frame_sp& frame) -{ +bool MotionVectorExtractor::processSOS(frame_sp& frame) { setMetadata(frame); mShouldTriggerSOS = false; return true; } -bool MotionVectorExtractor::handlePropsChange(frame_sp& frame) -{ +bool MotionVectorExtractor::handlePropsChange(frame_sp& frame) { MotionVectorExtractorProps props; auto ret = Module::handlePropsChange(frame, props); mDetail->setProps(props); return ret; } -void MotionVectorExtractor::setProps(MotionVectorExtractorProps& props) -{ +void MotionVectorExtractor::setProps(MotionVectorExtractorProps& props) { Module::addPropsToQueue(props); } \ No newline at end of file diff --git a/base/src/Mp4ReaderSource.cpp b/base/src/Mp4ReaderSource.cpp index f2fd08136..1cfe133d4 100644 --- a/base/src/Mp4ReaderSource.cpp +++ b/base/src/Mp4ReaderSource.cpp @@ -629,20 +629,20 @@ class Mp4ReaderDetailAbs { int seekedToFrame = -1; uint64_t skipMsecsInFile = 0; - - if (!mState.startTimeStampFromFile) + + if (!mState.resolvedStartingTS) { LOG_ERROR << "Start timestamp is not saved in the file. Can't support seeking with timestamps."; return false; } - if (skipTS < mState.startTimeStampFromFile) + if (skipTS < mState.resolvedStartingTS) { LOG_INFO << "seek time outside range. Seeking to start of video."; skipMsecsInFile = 0; } else { - skipMsecsInFile = skipTS - mState.startTimeStampFromFile; + skipMsecsInFile = skipTS - mState.resolvedStartingTS; } LOG_INFO << "Attempting seek <" << mState.mVideoPath << "> @skipMsecsInFile <" << skipMsecsInFile << ">"; diff --git a/base/src/OverlayModule.cpp b/base/src/OverlayModule.cpp index c0ea6a0fa..a76ba4776 100644 --- a/base/src/OverlayModule.cpp +++ b/base/src/OverlayModule.cpp @@ -32,7 +32,7 @@ bool OverlayModule::validateInputPins() BOOST_FOREACH(me, inputMetadataByPin) { FrameMetadata::FrameType frameType = me.second->getFrameType(); - if (frameType != FrameMetadata::RAW_IMAGE && frameType != FrameMetadata::OVERLAY_INFO_IMAGE) + if (frameType != FrameMetadata::RAW_IMAGE && frameType != FrameMetadata::OVERLAY_INFO_IMAGE && frameType != FrameMetadata::FACE_LANDMARKS_INFO && frameType != FrameMetadata::FACEDETECTS_INFO) { LOG_ERROR << "<" << getId() << ">::validateInputPins input frameType is expected to be RAW_IMAGE OR OVERLAY_INFO_IMAGE. Actual<" << frameType << ">"; return false; @@ -70,8 +70,13 @@ bool OverlayModule::process(frame_container& frames) if (frameType == FrameMetadata::OVERLAY_INFO_IMAGE) { drawOverlay.deserialize(frame); + } + else if (frameType == FrameMetadata::FACE_LANDMARKS_INFO) { + drawOverlay.deserialize(frame); + } + else if (frameType == FrameMetadata::FACEDETECTS_INFO) { + drawOverlay.deserialize(frame); } - else if (frameType == FrameMetadata::RAW_IMAGE) { drawOverlay.draw(frame); diff --git a/base/src/RTSPClientSrc.cpp b/base/src/RTSPClientSrc.cpp index c5fca7213..656d67a58 100644 --- a/base/src/RTSPClientSrc.cpp +++ b/base/src/RTSPClientSrc.cpp @@ -5,7 +5,6 @@ using namespace std; - #include #include @@ -38,24 +37,23 @@ FrameMetadata::FrameType getFrameType_fromFFMPEG(AVMediaType avMediaType, AVCode case AV_CODEC_ID_BMP: return FrameMetadata::BMP_IMAGE; default: - return FrameMetadata::GENERAL; + return FrameMetadata::GENERAL; } } - return FrameMetadata::GENERAL; //for everything else we may not have a match + return FrameMetadata::GENERAL; // for everything else we may not have a match } - class RTSPClientSrc::Detail { public: - Detail(RTSPClientSrc* m,std::string path, bool useTCP) :myModule(m), path(path), bConnected(false), bUseTCP(useTCP){} + Detail(RTSPClientSrc* m, std::string path, bool useTCP) : myModule(m), path(path), bConnected(false), bUseTCP(useTCP) {} ~Detail() { destroy(); } void destroy() { - if(nullptr!= pFormatCtx) + if (nullptr != pFormatCtx) avformat_close_input(&pFormatCtx); } - + bool connect() { avformat_network_init(); @@ -65,12 +63,12 @@ class RTSPClientSrc::Detail AVDictionary* avdic = NULL; - av_dict_set(&avdic, "rtsp_transport", (bUseTCP)?"tcp":"udp", 0); + av_dict_set(&avdic, "rtsp_transport", (bUseTCP) ? "tcp" : "udp", 0); av_dict_set(&avdic, "max_delay", "100", 0); if (avformat_open_input(&pFormatCtx, path.c_str(), NULL, &avdic) != 0) { - LOG_ERROR << "can't open the URL." << path <nb_streams; i++) { auto pCodecCtx = pFormatCtx->streams[i]->codec; - LOG_INFO << av_get_media_type_string (pCodecCtx->codec_type)<<" Codec: " << avcodec_get_name(pCodecCtx->codec_id); - auto fType=getFrameType_fromFFMPEG(pCodecCtx->codec_type,pCodecCtx->codec_id); - string outPin=myModule->getOutputPinIdByType(fType); + LOG_INFO << av_get_media_type_string(pCodecCtx->codec_type) << " Codec: " << avcodec_get_name(pCodecCtx->codec_id); + auto fType = getFrameType_fromFFMPEG(pCodecCtx->codec_type, pCodecCtx->codec_id); + string outPin = myModule->getOutputPinIdByType(fType); if (!outPin.empty()) { streamsMap[i] = outPin; if (pCodecCtx->codec_type == AVMEDIA_TYPE_VIDEO) { videoStream = i; - auto meta= FrameMetadataFactory::downcast(myModule->getOutputMetadata(outPin)); + auto meta = FrameMetadataFactory::downcast(myModule->getOutputMetadata(outPin)); H264Metadata tmp(pCodecCtx->width, pCodecCtx->height, pCodecCtx->gop_size, pCodecCtx->max_b_frames); meta->setData(tmp); } } } - //by this time all the output pins should be satisfied if not we have a problem + // by this time all the output pins should be satisfied if not we have a problem if (streamsMap.size() < myModule->getNumberOfOutputPins()) { LOG_ERROR << "some output pins can not be satsified" << std::endl; @@ -128,31 +126,26 @@ class RTSPClientSrc::Detail bool readBuffer() { - if(!initDone) - { - std::chrono::time_point t = std::chrono::system_clock::now(); - beginTs = std::chrono::duration_cast(t.time_since_epoch()); - initDone = true; - } frame_container outFrames; bool got_something = false; - while(!got_something) + while (!got_something) { if (av_read_frame(pFormatCtx, &packet) >= 0) { - if (videoStream >= 0) //source has video + if (videoStream >= 0) // source has video { - if (packet.stream_index == videoStream) //got video + if (packet.stream_index == videoStream) // got video { got_something = true; } } else { - got_something = true; //does not need video and got something + got_something = true; // does not need video and got something } auto it = streamsMap.find(packet.stream_index); - if (it != streamsMap.end()) { // so we have an interest in sending this + if (it != streamsMap.end()) + { // so we have an interest in sending this frame_sp frm; auto naluType = H264Utils::getNALUType((const char*)packet.data); if (naluType == H264Utils::H264_NAL_TYPE_SEI) @@ -176,92 +169,80 @@ class RTSPClientSrc::Detail } std::chrono::time_point t = std::chrono::system_clock::now(); - auto dur = std::chrono::duration_cast(t.time_since_epoch()); + auto dur = std::chrono::duration_cast(t.time_since_epoch()); frm->timestamp = dur.count(); if (!outFrames.insert(make_pair(it->second, frm)).second) { LOG_WARNING << "oops! there is already another packet for pin " << it->second; } - auto diff = dur - beginTs; - if(diff.count() > 1000) - { - currentCameraFps = frameCount; - frameCount = 0; - beginTs = dur; - } - frameCount++; } av_packet_unref(&packet); } } - - if(outFrames.size()>0) - myModule->send(outFrames); + if (outFrames.size() > 0) + myModule->send(outFrames); return true; } bool isConncected() const { return bConnected; } - int frameCount = 0; - int currentCameraFps = 0; + private: AVPacket packet; AVFormatContext* pFormatCtx = nullptr; std::string path; bool bConnected; - int videoStream=-1; + int videoStream = -1; bool bUseTCP; std::map streamsMap; RTSPClientSrc* myModule; - std::chrono::milliseconds beginTs; - bool initDone = false; }; RTSPClientSrc::RTSPClientSrc(RTSPClientSrcProps _props) : Module(SOURCE, "RTSPClientSrc", _props), mProps(_props) { - mDetail.reset(new Detail(this,mProps.rtspURL, mProps.useTCP)); + mDetail.reset(new Detail(this, mProps.rtspURL, mProps.useTCP)); } -RTSPClientSrc::~RTSPClientSrc() { +RTSPClientSrc::~RTSPClientSrc() +{ } -bool RTSPClientSrc::init() { +bool RTSPClientSrc::init() +{ if (mDetail->connect()) { return Module::init(); } return false; } -bool RTSPClientSrc::term() { +bool RTSPClientSrc::term() +{ mDetail.reset(); return true; } void RTSPClientSrc::setProps(RTSPClientSrcProps& props) { mProps = props; - //TBD need to also reset the whole connection + // TBD need to also reset the whole connection } -RTSPClientSrcProps RTSPClientSrc::getProps() { + +RTSPClientSrcProps RTSPClientSrc::getProps() +{ return mProps; } -bool RTSPClientSrc::produce() { +bool RTSPClientSrc::produce() +{ return mDetail->readBuffer(); } -bool RTSPClientSrc::validateOutputPins() { - //smallest check at least one output pin should be there +bool RTSPClientSrc::validateOutputPins() +{ + // smallest check at least one output pin should be there return this->getNumberOfOutputPins() > 0; } void RTSPClientSrc::notifyPlay(bool play) {} -bool RTSPClientSrc::handleCommand(Command::CommandType type, frame_sp& frame) -{ - if (type == Command::CommandType::Relay) - { - return Module::handleCommand(type, frame); - } - return true; +//bool RTSPClientSrc::handleCommand(Command::CommandType type, frame_sp& frame) { return true; } +bool RTSPClientSrc::handleCommand(Command::CommandType type, frame_sp &frame) { + if (type == Command::CommandType::Relay) { + return Module::handleCommand(type, frame); + } + return true; } - -int RTSPClientSrc::getCurrentFps() -{ - return mDetail->currentCameraFps; -} - bool RTSPClientSrc::handlePropsChange(frame_sp& frame) { return true; } diff --git a/base/test/facedetectorXform_tests.cpp b/base/test/facedetectorXform_tests.cpp index 3b3ae4528..cc04c2903 100644 --- a/base/test/facedetectorXform_tests.cpp +++ b/base/test/facedetectorXform_tests.cpp @@ -13,6 +13,8 @@ #include "ApraFaceInfo.h" #include "FaceDetectsInfo.h" #include "ImageDecoderCV.h" +#include "OverlayModule.h" +#include "ImageViewerModule.h" BOOST_AUTO_TEST_SUITE(facedetector_tests) #ifdef ARM64 @@ -59,6 +61,40 @@ BOOST_AUTO_TEST_CASE(basic) Test_Utils::saveOrCompare("./data/testOutput/facesResult.raw", const_cast(static_cast(frame.data)), frame.step[0] * frame.rows, 0); } +BOOST_AUTO_TEST_CASE(basic_faces) +{ + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/faces.jpg"))); + auto metadata = framemetadata_sp(new FrameMetadata(FrameMetadata::ENCODED_IMAGE)); + fileReader->addOutputPin(metadata); + + auto decoder = boost::shared_ptr(new ImageDecoderCV(ImageDecoderCVProps())); + auto metadata2 = framemetadata_sp(new RawImageMetadata()); + decoder->addOutputPin(metadata2); + fileReader->setNext(decoder); + + FaceDetectorXformProps faceDetectorProps; + auto faceDetector = boost::shared_ptr(new FaceDetectorXform(faceDetectorProps)); + decoder->setNext(faceDetector); + + auto overlay = boost::shared_ptr(new OverlayModule(OverlayModuleProps())); + faceDetector->setNext(overlay); + + auto sink = boost::shared_ptr(new ImageViewerModule(ImageViewerModuleProps("imageview"))); + overlay->setNext(sink); + + PipeLine p("test"); + p.appendModule(fileReader); + p.init(); + + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + + LOG_INFO << "profiling done - stopping the pipeline"; + p.stop(); + p.term(); + p.wait_for_all(); +} + BOOST_AUTO_TEST_CASE(serialization_test) { diff --git a/base/test/motionvector_extractor_and_overlay_tests.cpp b/base/test/motionvector_extractor_and_overlay_tests.cpp index e999c7987..f08cae573 100644 --- a/base/test/motionvector_extractor_and_overlay_tests.cpp +++ b/base/test/motionvector_extractor_and_overlay_tests.cpp @@ -148,9 +148,7 @@ void motionVectorExtractAndOverlaySetProps(MotionVectorExtractorProps::MVExtract p.term(); p.wait_for_all(); } - -void motionVectorExtractAndOverlay_Render(MotionVectorExtractorProps::MVExtractMethod MvExtract) -{ +void motionVectorExtractAndOverlay_Render(MotionVectorExtractorProps::MVExtractMethod MvExtract) { LoggerProps loggerProps; loggerProps.logLevel = boost::log::trivial::severity_level::info; Logger::setLogLevel(boost::log::trivial::severity_level::info); @@ -185,7 +183,7 @@ void motionVectorExtractAndOverlay_Render(MotionVectorExtractorProps::MVExtractM p.stop(); p.term(); p.wait_for_all(); -} + } void rtspCamMotionVectorExtractAndOverlay_Render(MotionVectorExtractorProps::MVExtractMethod MvExtract) { @@ -197,6 +195,7 @@ void rtspCamMotionVectorExtractAndOverlay_Render(MotionVectorExtractorProps::MVE rtsp_client_tests_data d; bool overlayFrames = false; + const std::string url = ""; std::string username = ""; std::string password = ""; @@ -261,11 +260,6 @@ BOOST_AUTO_TEST_CASE(extract_motion_vectors_and_overlay_openh264) motionVectorExtractAndOverlay(MotionVectorExtractorProps::OPENH264); } -BOOST_AUTO_TEST_CASE(extract_motion_vectors_and_overlay_render_openh264, *boost::unit_test::disabled()) -{ - motionVectorExtractAndOverlay_Render(MotionVectorExtractorProps::OPENH264); -} - BOOST_AUTO_TEST_CASE(rtspcam_extract_motion_vectors_and_overlay_render_openh264, *boost::unit_test::disabled()) { rtspCamMotionVectorExtractAndOverlay_Render(MotionVectorExtractorProps::OPENH264); diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt new file mode 100644 index 000000000..c2f9918a3 --- /dev/null +++ b/samples/CMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.22) +include_directories( +../base +${CMAKE_CURRENT_SOURCE_DIR} +) + +add_subdirectory(face_detection_cpu) +add_subdirectory(create_thumbnail_from_mp4_video) +add_subdirectory(play_mp4_from_beginning) +add_subdirectory(timelapse-sample) \ No newline at end of file diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 000000000..64a63f76c --- /dev/null +++ b/samples/README.md @@ -0,0 +1,42 @@ + + + +This Samples demonstrate the capability and usage of ApraPipes Modules into different applications. + +## Running the samples: +- The samples will be build along with the ApraPipes build. +- Each sample has their own executable. +- The executables can be found in the location: + - _build/samples/sample/Debug. Note that here sample in the path is the particular sample. +- To run the samples run this command: +``` +./filename.exe arg1 arg2 ... +``` +- Note that samples may have the arguments. The details are given in the below section. + +## Samples +### 1. Timelapse: +This samples demonstrates the usage of ApraPipes modules which can be used to generate a timelapse or summary of the video. +#### - Modules Used: +- Mp4Reader: To Read the input video. +- MotionVectorExtractor: To extract the frames from the input video for configured thershold. +- ColorChange: To convert frames given by MotionVectorExtractor from BGR to RGB and then RGB to YUV420PLANAR. +- H264Encoder +- Mp4Writer: To write the generated timelapse video to the output path. +- arguments: + - arg1: input video path + - arg2: output path +- The Unit test "timelapse_summary_test.cpp" is located inside the aprapipessampleut executable. Make sure to replace the "inputVideoPath" and "outputPath" with suitable paths. + +### 2. Relay sample: +This sample demonstrate the use of relay command in the ApraPipes which can be used to swith the stream from live to recorder and vice versa. +#### - Modules Used: +- RtspClientSrc: To read the live stream from RTSP camera. +- Mp4Reader: To Read recorded stream from the Mp4Video. +- H264Decoder: To decode the h264 frames from RTSP and MP4 sources. +- colorConversion: To transform YUV420 frame to RAW RGB format. +- ImageViewer: To render RGB frames on the screen. +- arguments: + - arg1: RTSP camera URL + - arg2: MP4 video file path +- The Unit test "relay_sample_test.cpp" is located inside the aprapipessampleut executable. Make sure to replace the `rtspUrl` and `mp4VideoPath` with suitable paths. diff --git a/samples/cMakeLists.txt b/samples/cMakeLists.txt new file mode 100644 index 000000000..c2f9918a3 --- /dev/null +++ b/samples/cMakeLists.txt @@ -0,0 +1,10 @@ +cmake_minimum_required(VERSION 3.22) +include_directories( +../base +${CMAKE_CURRENT_SOURCE_DIR} +) + +add_subdirectory(face_detection_cpu) +add_subdirectory(create_thumbnail_from_mp4_video) +add_subdirectory(play_mp4_from_beginning) +add_subdirectory(timelapse-sample) \ No newline at end of file diff --git a/samples/create_thumbnail_from_mp4_video/CMakeLists.txt b/samples/create_thumbnail_from_mp4_video/CMakeLists.txt new file mode 100644 index 000000000..f80673668 --- /dev/null +++ b/samples/create_thumbnail_from_mp4_video/CMakeLists.txt @@ -0,0 +1,34 @@ +cmake_minimum_required(VERSION 3.22) +set(TARGET generateThumbnailFromMp4Video) +SET(SAMPLE_FILES + GenerateThumbnailsPipeline.cpp + pipelineMain.cpp +) +SET(SAMPLE_FILES_H + GenerateThumbnailsPipeline.h +) +SET(SOURCE + ${SAMPLE_FILES} + ${SAMPLE_FILES_H} +) +add_executable(${TARGET} ${SOURCE}) +target_include_directories ( ${TARGET} PRIVATE + ${JETSON_MULTIMEDIA_LIB_INCLUDE} + ${OpenCV_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} + ${LIBMP4_INC_DIR} + ${NVCODEC_INCLUDE_DIR} +) +target_link_libraries( + ${TARGET} + aprapipes + ${JPEG_LIBRARIES} + ${LIBMP4_LIB} + ${OPENH264_LIB} + ${Boost_LIBRARIES} + ${FFMPEG_LIBRARIES} + ${OpenCV_LIBRARIES} + ${NVCUDAToolkit_LIBS} + ${NVCODEC_LIB} + ${NVJPEGLIB_L4T} + ) \ No newline at end of file diff --git a/samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.cpp b/samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.cpp new file mode 100644 index 000000000..16ffe45cb --- /dev/null +++ b/samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.cpp @@ -0,0 +1,89 @@ + +#include "GenerateThumbnailsPipeline.h" +#include "CudaMemCopy.h" +#include "ExternalSinkModule.h" +#include "FileReaderModule.h" +#include "FileWriterModule.h" +#include "FrameMetadata.h" +#include "H264Decoder.h" +#include "H264Metadata.h" +#include "JPEGEncoderNVJPEG.h" +#include "Logger.h" +#include "Mp4ReaderSource.h" +#include "Mp4VideoMetadata.h" +#include "ValveModule.h" +#include + +GenerateThumbnailsPipeline::GenerateThumbnailsPipeline() + : pipeLine("thumnailSamplePipeline") {} + +bool GenerateThumbnailsPipeline::setUpPipeLine( + const std::string &videoPath, const std::string &outFolderPath) { + // Implementation + + bool parseFS = false; + auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); + auto frameType = FrameMetadata::FrameType::H264_DATA; + auto mp4ReaderProps = + Mp4ReaderSourceProps(videoPath, parseFS, 0, true, false, false); + //initializing source Mp4 reader to read Mp4 video + mMp4Reader = + boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); + mMp4Reader->addOutPutPin(h264ImageMetadata); + + auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); + mMp4Reader->addOutPutPin(mp4Metadata); + + std::vector mImagePin; + mImagePin = mMp4Reader->getAllOutputPinsByType(frameType); + + //initializing H264 decoder to decode frame in H264 format + mDecoder = + boost::shared_ptr(new H264Decoder(H264DecoderProps())); + //Selecting an image pin of H264 data frame type is necessary because the decoder processes H264 frames for decoding. + mMp4Reader->setNext(mDecoder, mImagePin); + + //Initializing the valve to send only one frame. It is currently set to 0, meaning no frames are captured. + mValve = boost::shared_ptr(new ValveModule(ValveModuleProps(0))); + mDecoder->setNext(mValve); + + //initialize cuda memory + auto stream = cudastream_sp(new ApraCudaStream); + mCudaCopy = boost::shared_ptr( + new CudaMemCopy(CudaMemCopyProps(cudaMemcpyHostToDevice, stream))); + mValve->setNext(mCudaCopy); + + //initializing Jpeg encoder to encode the frame in jpeg format + mJpegEncoder = boost::shared_ptr( + new JPEGEncoderNVJPEG(JPEGEncoderNVJPEGProps(stream))); + mCudaCopy->setNext(mJpegEncoder); + + //initilizing file writer as sink to write frame at given path + mFileWriter = boost::shared_ptr( + new FileWriterModule(FileWriterModuleProps(outFolderPath))); + mJpegEncoder->setNext(mFileWriter); + + return true; +} + +bool GenerateThumbnailsPipeline::startPipeLine() { + pipeLine.appendModule(mMp4Reader); + if (!pipeLine.init()) { + throw AIPException( + AIP_FATAL, + "Engine Pipeline init failed. Check IPEngine Logs for more details."); + return false; + } + pipeLine.run_all_threaded(); + //allowing only one frame to get captured. + mValve->allowFrames(1); + + return true; +} + +bool GenerateThumbnailsPipeline::stopPipeLine() { + pipeLine.stop(); + pipeLine.term(); + pipeLine.wait_for_all(); + return true; +} diff --git a/samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.h b/samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.h new file mode 100644 index 000000000..b421c74f0 --- /dev/null +++ b/samples/create_thumbnail_from_mp4_video/GenerateThumbnailsPipeline.h @@ -0,0 +1,30 @@ +#include +#include +#include "Mp4ReaderSource.h" +#include "H264Decoder.h" +#include "ColorConversionXForm.h" +#include "CudaMemCopy.h" +#include "FileWriterModule.h" +#include "JPEGEncoderNVJPEG.h" + + + +class GenerateThumbnailsPipeline +{ +public: + GenerateThumbnailsPipeline(); + bool setUpPipeLine(const std::string &videoPath,const std::string &outFolderPath); + bool startPipeLine(); + bool stopPipeLine(); + +private: + PipeLine pipeLine; + boost::shared_ptr mValve; + boost::shared_ptr mMp4Reader; + boost::shared_ptr mDecoder; + boost::shared_ptr mColorchange; + boost::shared_ptr mCudaCopy; + boost::shared_ptr mJpegEncoder; + boost::shared_ptr mFileWriter; +}; + diff --git a/samples/create_thumbnail_from_mp4_video/pipelineMain.cpp b/samples/create_thumbnail_from_mp4_video/pipelineMain.cpp new file mode 100644 index 000000000..0b0d4f11b --- /dev/null +++ b/samples/create_thumbnail_from_mp4_video/pipelineMain.cpp @@ -0,0 +1,32 @@ +#include "GenerateThumbnailsPipeline.h" +#include +#include + +void main(int argc, char *argv[]) { + if (argc < 3) { + std::cerr << "Usage: " << argv[0] << " " + << std::endl; + } + + std::string videoPath = argv[argc - 2]; + std::string outFolderPath = argv[argc - 1]; + + GenerateThumbnailsPipeline thumbnailPipeline; + if (!thumbnailPipeline.setUpPipeLine(videoPath, outFolderPath)) { + std::cerr << "Failed to setup pipeline." << std::endl; + } + + if (!thumbnailPipeline.startPipeLine()) { + std::cerr << "Failed to start pipeline." << std::endl; + } + + // Wait for the pipeline to run for 10 seconds + boost::this_thread::sleep_for(boost::chrono::seconds(5)); + + // Stop the pipeline + if (!thumbnailPipeline.stopPipeLine()) { + std::cerr << "Failed to stop pipeline." << std::endl; + } else { + std::cerr << "Saved Generated Thumbnail in <" << outFolderPath << ">"; + } +} \ No newline at end of file diff --git a/samples/create_thumbnail_from_mp4_video/test_generate_thumbnail_from_mp4_video.cpp b/samples/create_thumbnail_from_mp4_video/test_generate_thumbnail_from_mp4_video.cpp new file mode 100644 index 000000000..a4232aa65 --- /dev/null +++ b/samples/create_thumbnail_from_mp4_video/test_generate_thumbnail_from_mp4_video.cpp @@ -0,0 +1,31 @@ +#include "GenerateThumbnailsPipeline.h" +#include "test_utils.h" +#include +#include + +BOOST_AUTO_TEST_SUITE(generateThumbnails) + +BOOST_AUTO_TEST_CASE(generateThumbnails_from_mp4) { + auto generateThumbnailPipeline = + boost::shared_ptr( + new GenerateThumbnailsPipeline()); + std::string videoPath = "../../data/Mp4_videos/h264_video_metadata/20230514/0011/1686723796848.mp4"; + std::string outFolderPath = "data/generated_thumbnail/thumbnail_????.jpg"; + BOOST_CHECK_NO_THROW(generateThumbnailPipeline->setUpPipeLine(videoPath, outFolderPath)); + BOOST_CHECK_NO_THROW(generateThumbnailPipeline->startPipeLine()); + + boost::this_thread::sleep_for(boost::chrono::seconds(5)); + + const uint8_t *pReadDataTest = nullptr; + unsigned int readDataSizeTest = 0U; + + BOOST_TEST(Test_Utils::readFile("data/test_thumbnail/sample_thumbnail.jpg", + pReadDataTest, readDataSizeTest)); + Test_Utils::saveOrCompare( + "data/generated_thumbnail/thumbnail_0000.jpg", pReadDataTest, + readDataSizeTest, 0); + + generateThumbnailPipeline->stopPipeLine(); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/samples/face_detection_cpu/CMakeLists.txt b/samples/face_detection_cpu/CMakeLists.txt new file mode 100644 index 000000000..97cdd62de --- /dev/null +++ b/samples/face_detection_cpu/CMakeLists.txt @@ -0,0 +1,28 @@ +cmake_minimum_required(VERSION 3.22) +set(TARGET face_detection_cpu) +SET(SAMPLE_FILES + face_detection_cpu.cpp + pipelineMain.cpp +) +SET(SAMPLE_FILES_H + face_detection_cpu.h +) +SET(SOURCE + ${SAMPLE_FILES} + ${SAMPLE_FILES_H} +) + +add_executable(${TARGET} ${SOURCE}) + +target_include_directories ( ${TARGET} PRIVATE + ${OpenCV_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} +) + +target_link_libraries( + ${TARGET} + aprapipes + ${JPEG_LIBRARIES} + ${Boost_LIBRARIES} + ${OpenCV_LIBRARIES} + ) \ No newline at end of file diff --git a/samples/face_detection_cpu/face_detection_cpu.cpp b/samples/face_detection_cpu/face_detection_cpu.cpp new file mode 100644 index 000000000..a24f5a2ba --- /dev/null +++ b/samples/face_detection_cpu/face_detection_cpu.cpp @@ -0,0 +1,44 @@ +#include "WebCamSource.h" +#include "ImageViewerModule.h" +#include "face_detection_cpu.h" +#include "OverlayModule.h" +#include "ImageDecoderCV.h" +#include "FaceDetectorXform.h" +#include "ColorConversionXForm.h" +#include + +FaceDetectionCPU::FaceDetectionCPU() : faceDetectionCPUSamplePipeline("faceDetectionCPUSamplePipeline") {} + +bool FaceDetectionCPU::setupPipeline(const int &cameraId, const double &scaleFactor, const double &threshold, const std::string &faceDetectionConfiguration, const std::string &faceDetectionWeight) { + WebCamSourceProps webCamSourceprops(cameraId); + mSource = boost::shared_ptr(new WebCamSource(webCamSourceprops)); + + FaceDetectorXformProps faceDetectorProps(scaleFactor, threshold, faceDetectionConfiguration, faceDetectionWeight); + mFaceDetector = boost::shared_ptr(new FaceDetectorXform(faceDetectorProps)); + mSource->setNext(mFaceDetector); + + mOverlay = boost::shared_ptr(new OverlayModule(OverlayModuleProps())); + mFaceDetector->setNext(mOverlay); + + mColorConversion = boost::shared_ptr(new ColorConversion(ColorConversionProps(ColorConversionProps::RGB_TO_BGR))); + mOverlay->setNext(mColorConversion); + + mImageViewerSink = boost::shared_ptr(new ImageViewerModule(ImageViewerModuleProps("imageview"))); + mColorConversion->setNext(mImageViewerSink); + + return true; +} + +bool FaceDetectionCPU::startPipeline() { + faceDetectionCPUSamplePipeline.appendModule(mSource); + faceDetectionCPUSamplePipeline.init(); + faceDetectionCPUSamplePipeline.run_all_threaded(); + return true; +} + +bool FaceDetectionCPU::stopPipeline() { + faceDetectionCPUSamplePipeline.stop(); + faceDetectionCPUSamplePipeline.term(); + faceDetectionCPUSamplePipeline.wait_for_all(); + return true; +} \ No newline at end of file diff --git a/samples/face_detection_cpu/face_detection_cpu.h b/samples/face_detection_cpu/face_detection_cpu.h new file mode 100644 index 000000000..3d6360962 --- /dev/null +++ b/samples/face_detection_cpu/face_detection_cpu.h @@ -0,0 +1,23 @@ +#include +#include "WebCamSource.h" +#include "ImageViewerModule.h" +#include "OverlayModule.h" +#include "FacialLandmarksCV.h" +#include "ImageDecoderCV.h" +#include "FaceDetectorXform.h" +#include "ColorConversionXForm.h" + +class FaceDetectionCPU { +public: + FaceDetectionCPU(); + bool setupPipeline(const int &cameraId, const double &scaleFactor, const double &threshold, const std::string &faceDetectionConfiguration, const std::string &faceDetectionWeight); + bool startPipeline(); + bool stopPipeline(); +private: + PipeLine faceDetectionCPUSamplePipeline; + boost::shared_ptr mSource; + boost::shared_ptr mFaceDetector; + boost::shared_ptr mOverlay; + boost::shared_ptr mColorConversion; + boost::shared_ptr mImageViewerSink; +}; \ No newline at end of file diff --git a/samples/face_detection_cpu/pipelineMain.cpp b/samples/face_detection_cpu/pipelineMain.cpp new file mode 100644 index 000000000..3f35b8687 --- /dev/null +++ b/samples/face_detection_cpu/pipelineMain.cpp @@ -0,0 +1,22 @@ +#include "face_detection_cpu.h" +#include +#include + +void main() { + FaceDetectionCPU faceDetectionCPUSamplePipeline; + if (!faceDetectionCPUSamplePipeline.setupPipeline(0, 1.0, 0.7, "../../data/assets/deploy.prototxt", "../../data/assets/res10_300x300_ssd_iter_140000_fp16.caffemodel")) { + std::cerr << "Failed to setup pipeline." << std::endl; + } + + if (!faceDetectionCPUSamplePipeline.startPipeline()) { + std::cerr << "Failed to start pipeline." << std::endl; + } + + // Wait for the pipeline to run for 10 seconds + boost::this_thread::sleep_for(boost::chrono::seconds(50)); + + // Stop the pipeline + if (!faceDetectionCPUSamplePipeline.stopPipeline()) { + std::cerr << "Failed to stop pipeline." << std::endl; + } +} \ No newline at end of file diff --git a/samples/face_detection_cpu/test_face_detection_cpu.cpp b/samples/face_detection_cpu/test_face_detection_cpu.cpp new file mode 100644 index 000000000..6440070d2 --- /dev/null +++ b/samples/face_detection_cpu/test_face_detection_cpu.cpp @@ -0,0 +1,18 @@ +#include + +#include "face_detection_cpu.h" + +BOOST_AUTO_TEST_SUITE(test_face_detection_cpu) + +BOOST_AUTO_TEST_CASE(face_detection_cpu) +{ + auto faceDetectionCPUPipeline = boost::shared_ptr(new FaceDetectionCPU()); + faceDetectionCPUPipeline->setupPipeline(0, 1.0, 0.7, "./data/assets/deploy.prototxt", "./data/assets/res10_300x300_ssd_iter_140000_fp16.caffemodel"); + faceDetectionCPUPipeline->startPipeline(); + + boost::this_thread::sleep_for(boost::chrono::seconds(25)); + + faceDetectionCPUPipeline->stopPipeline(); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/samples/play_mp4_from_beginning/CMakeLists.txt b/samples/play_mp4_from_beginning/CMakeLists.txt new file mode 100644 index 000000000..7d46995bc --- /dev/null +++ b/samples/play_mp4_from_beginning/CMakeLists.txt @@ -0,0 +1,33 @@ +cmake_minimum_required(VERSION 3.22) +set(TARGET play_mp4_from_beginning) +SET(SAMPLE_FILES + PlayMp4VideoFromBeginning.cpp + pipelineMain.cpp +) +SET(SAMPLE_FILES_H + PlayMp4VideoFromBeginning.h +) +SET(SOURCE + ${SAMPLE_FILES} + ${SAMPLE_FILES_H} +) +add_executable(${TARGET} ${SOURCE}) +target_include_directories ( ${TARGET} PRIVATE + ${OpenCV_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} + ${LIBMP4_INC_DIR} +) +target_link_libraries( + ${TARGET} + aprapipes + ${JPEG_LIBRARIES} + ${LIBMP4_LIB} + ${OPENH264_LIB} + ${Boost_LIBRARIES} + ${OpenCV_LIBRARIES} + ${NVCUDAToolkit_LIBS} + ${NVJPEGLIB_L4T} + ${NVCODEC_LIB} + ${FFMPEG_LIBRARIES} + ) + \ No newline at end of file diff --git a/samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.cpp b/samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.cpp new file mode 100644 index 000000000..a3d9a0892 --- /dev/null +++ b/samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.cpp @@ -0,0 +1,87 @@ + +#include "PlayMp4VideoFromBeginning.h" +#include "ColorConversionXForm.h" +#include "CudaMemCopy.h" +#include "ExternalSinkModule.h" +#include "FileWriterModule.h" +#include "Frame.h" +#include "FrameContainerQueue.h" +#include "FrameMetadata.h" +#include "H264Decoder.h" +#include "H264Metadata.h" +#include "ImageViewerModule.h" +#include "JPEGEncoderNVJPEG.h" +#include "Mp4ReaderSource.h" +#include "Mp4VideoMetadata.h" +#include + +PlayMp4VideoFromBeginning::PlayMp4VideoFromBeginning() + : pipeLine("PlayMp4videoFromBeginingSamplePipline") {} + +bool PlayMp4VideoFromBeginning::setUpPipeLine(const std::string &videoPath) { + // Implementation + bool parseFS = false; + auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); + auto frameType = FrameMetadata::FrameType::H264_DATA; + auto mp4ReaderProps = + Mp4ReaderSourceProps(videoPath, parseFS, 0, true, true, false); + mp4ReaderProps.fps = 24; + // initializing source Mp4 reader to read Mp4 video + mMp4Reader = + boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); + mMp4Reader->addOutPutPin(h264ImageMetadata); + + auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); + mMp4Reader->addOutPutPin(mp4Metadata); + + std::vector mImagePin; + mImagePin = mMp4Reader->getAllOutputPinsByType(frameType); + + // initializing H264 decoder to decode frame in H264 format + mDecoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); + + // Selecting an image pin of H264 data frame type is necessary because the + // decoder processes H264 frames for decoding. + mMp4Reader->setNext(mDecoder, mImagePin); + + //initializing conversion type module to convert YUV420 frame to RGB + auto conversionType = + ColorConversionProps::ConversionType::YUV420PLANAR_TO_RGB; + auto metadata = framemetadata_sp(new RawImagePlanarMetadata( + 1280, 720, ImageMetadata::ImageType::YUV420, size_t(0), CV_8U)); + mColorchange = boost::shared_ptr( + new ColorConversion(ColorConversionProps(conversionType))); + mDecoder->setNext(mColorchange); + + //initializing imageViewer module as sink to show video on screen + mImageViewerSink = boost::shared_ptr( + new ImageViewerModule(ImageViewerModuleProps("imageview"))); + mColorchange->setNext(mImageViewerSink); + + return true; +} + +bool PlayMp4VideoFromBeginning::startPipeLine() { + pipeLine.appendModule(mMp4Reader); + if (!pipeLine.init()) { + throw AIPException( + AIP_FATAL, + "Engine Pipeline init failed. Check IPEngine Logs for more details."); + return false; + } + pipeLine.run_all_threaded(); + return true; +} + +bool PlayMp4VideoFromBeginning::stopPipeLine() { + pipeLine.stop(); + pipeLine.term(); + pipeLine.wait_for_all(); + return true; +} + +bool PlayMp4VideoFromBeginning::flushQueuesAndSeek() { + pipeLine.flushAllQueues(); + mMp4Reader->randomSeek(1686723796848, false); + return true; +} diff --git a/samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.h b/samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.h new file mode 100644 index 000000000..d489e752c --- /dev/null +++ b/samples/play_mp4_from_beginning/PlayMp4VideoFromBeginning.h @@ -0,0 +1,24 @@ +#include "ColorConversionXForm.h" +#include "H264Decoder.h" +#include "ImageViewerModule.h" +#include "Mp4ReaderSource.h" + +#include + +class PlayMp4VideoFromBeginning { +public: + PlayMp4VideoFromBeginning(); + bool setUpPipeLine(const std::string &videoPath); + bool startPipeLine(); + bool stopPipeLine(); + bool flushQueuesAndSeek(); + + + boost::shared_ptr mMp4Reader; + boost::shared_ptr mImageViewerSink; + boost::shared_ptr mDecoder; + boost::shared_ptr mColorchange; + +private: + PipeLine pipeLine; +}; \ No newline at end of file diff --git a/samples/play_mp4_from_beginning/pipelineMain.cpp b/samples/play_mp4_from_beginning/pipelineMain.cpp new file mode 100644 index 000000000..3274cf83e --- /dev/null +++ b/samples/play_mp4_from_beginning/pipelineMain.cpp @@ -0,0 +1,30 @@ +#include "PlayMp4VideoFromBeginning.h" +#include +#include + +void main(int argc, char *argv[]) { + if (argc < 2) { + std::cerr << "Usage: " << argv[0] << " " + << std::endl; + } + std::string videoPath = argv[argc - 1]; + + PlayMp4VideoFromBeginning pipelineInstance; + + if (!pipelineInstance.setUpPipeLine(videoPath)) { + std::cerr << "Failed to setup pipeline." << std::endl; + } + if (!pipelineInstance.startPipeLine()) { + std::cerr << "Failed to start pipeline." << std::endl; + } + // Wait for the pipeline to run for 10 seconds + boost::this_thread::sleep_for(boost::chrono::seconds(3)); + if (!pipelineInstance.flushQueuesAndSeek()) { + std::cerr << "Failed to flush Queues." << std::endl; + } + boost::this_thread::sleep_for(boost::chrono::seconds(5)); + // Stop the pipeline + if (!pipelineInstance.stopPipeLine()) { + std::cerr << "Failed to stop pipeline." << std::endl; + } +} \ No newline at end of file diff --git a/samples/play_mp4_from_beginning/test_play_mp4_video_from_beginning.cpp b/samples/play_mp4_from_beginning/test_play_mp4_video_from_beginning.cpp new file mode 100644 index 000000000..e3e170a23 --- /dev/null +++ b/samples/play_mp4_from_beginning/test_play_mp4_video_from_beginning.cpp @@ -0,0 +1,87 @@ +#include "Frame.h" +#include "FrameContainerQueue.h" +#include "PlayMp4VideoFromBeginning.h" +#include "test_utils.h" +#include + +BOOST_AUTO_TEST_SUITE(start_video_from_beginning) +class SinkModuleProps : public ModuleProps { +public: + SinkModuleProps() : ModuleProps(){}; +}; + +class SinkModule : public Module { +public: + SinkModule(SinkModuleProps props) : Module(SINK, "sinkModule", props){}; + boost::shared_ptr getQue() { return Module::getQue(); } + frame_container pop() { return Module::pop(); } + +protected: + bool process() { return false; } + bool validateOutputPins() { return true; } + bool validateInputPins() { return true; } +}; + +BOOST_AUTO_TEST_CASE(play_mp4_from_beginning_flush_queue_test) { + auto playMp4VideoFromBeginning = boost::shared_ptr( + new PlayMp4VideoFromBeginning()); + std::string videoPath = "C:/APRA/fork/ApraPipes/data/Mp4_videos/" + "h264_video_metadata/20230514/0011/1686723796848.mp4"; + playMp4VideoFromBeginning->setUpPipeLine(videoPath); + + auto sink = boost::shared_ptr(new SinkModule(SinkModuleProps())); + playMp4VideoFromBeginning->mColorchange->setNext(sink); + BOOST_CHECK(playMp4VideoFromBeginning->mMp4Reader->init()); + BOOST_CHECK(playMp4VideoFromBeginning->mDecoder->init()); + BOOST_CHECK(playMp4VideoFromBeginning->mColorchange->init()); + BOOST_CHECK(sink->init()); + + for (int i = 0; i <= 20; i++) { + playMp4VideoFromBeginning->mMp4Reader->step(); + playMp4VideoFromBeginning->mDecoder->step(); + } + playMp4VideoFromBeginning->mColorchange->step(); + + auto frames = sink->pop(); + auto frame = frames.size(); + auto sinkQue = sink->getQue(); + BOOST_CHECK_EQUAL(frames.size(), 1); + sinkQue->flush(); + BOOST_CHECK_EQUAL(sinkQue->size(), 0); + frame_sp outputFrame = frames.cbegin()->second; + Test_Utils::saveOrCompare( + "./data/mp4Reader_saveOrCompare/h264/testplay.raw", + const_cast(static_cast(outputFrame->data())), + outputFrame->size(), 0); +} + +BOOST_AUTO_TEST_CASE(play_mp4_from_beginning_seek_test) { + auto playMp4VideoFromBeginning = boost::shared_ptr( + new PlayMp4VideoFromBeginning()); + std::string videoPath = "C:/APRA/fork/ApraPipes/data/Mp4_videos/" + "h264_video_metadata/20230514/0011/1686723796848.mp4"; + playMp4VideoFromBeginning->setUpPipeLine(videoPath); + auto sink = boost::shared_ptr(new SinkModule(SinkModuleProps())); + playMp4VideoFromBeginning->mMp4Reader->setNext(sink); + + BOOST_CHECK(playMp4VideoFromBeginning->mMp4Reader->init()); + BOOST_CHECK(playMp4VideoFromBeginning->mDecoder->init()); + BOOST_CHECK(playMp4VideoFromBeginning->mColorchange->init()); + BOOST_CHECK(sink->init()); + + playMp4VideoFromBeginning->mMp4Reader->step(); + + auto frames = sink->pop(); + auto imgFrame = frames.begin()->second; + + uint64_t skipTS = 1686723796848; + playMp4VideoFromBeginning->mMp4Reader->randomSeek(skipTS, false); + playMp4VideoFromBeginning->mMp4Reader->step(); + frames = sink->pop(); + imgFrame = frames.begin()->second; + BOOST_TEST(imgFrame->timestamp == 1686723796848); + LOG_INFO << "Found next available frame " << imgFrame->timestamp - skipTS + << " msecs later from skipTS"; +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/samples/relay-sample/PipelineMain.cpp b/samples/relay-sample/PipelineMain.cpp new file mode 100644 index 000000000..0cf06427b --- /dev/null +++ b/samples/relay-sample/PipelineMain.cpp @@ -0,0 +1,50 @@ +#include "relay_sample.h" +#include +#include +#include + +int main(int argc, char *argv[]) { + + if (argc < 3) { + std::cerr << "Usage: " << argv[0] << " " << std::endl; + return 1; + } + + std::string rtspUrl = argv[argc - 2]; + std::string mp4VideoPath = argv[argc - 1]; + + LoggerProps loggerProps; + loggerProps.logLevel = boost::log::trivial::severity_level::info; + Logger::setLogLevel(boost::log::trivial::severity_level::info); + Logger::initLogger(loggerProps); + + RelayPipeline pipelineInstance; + + if (!pipelineInstance.setupPipeline(rtspUrl, mp4VideoPath)) { + std::cerr << "Failed to setup pipeline." << std::endl; + return 1; + } + + if (!pipelineInstance.startPipeline()) { + std::cerr << "Failed to start pipeline." << std::endl; + return 1; + } + + while (true) { + int k = getchar(); + if (k == 114) { + pipelineInstance.addRelayToRtsp(false); + pipelineInstance.addRelayToMp4(true); + } + if (k == 108) { + pipelineInstance.addRelayToMp4(false); + pipelineInstance.addRelayToRtsp(true); + } + if (k == 115) { + pipelineInstance.stopPipeline(); + break; + } + } + + return 0; +} \ No newline at end of file diff --git a/samples/relay-sample/cMakeLists.txt b/samples/relay-sample/cMakeLists.txt new file mode 100644 index 000000000..3443a9a98 --- /dev/null +++ b/samples/relay-sample/cMakeLists.txt @@ -0,0 +1,42 @@ +cmake_minimum_required(VERSION 3.22) +set(TARGET relaysample) +SET(CORE_FILES + relay_sample.cpp + PipelineMain.cpp +) +SET(CORE_FILES_H + relay_sample.h +) +SET(SOURCE + ${CORE_FILES} + ${CORE_FILES_H} +) + +add_executable(${TARGET} ${SOURCE}) + +target_include_directories (${TARGET} PRIVATE + ${JETSON_MULTIMEDIA_LIB_INCLUDE} + ${FFMPEG_INCLUDE_DIRS} + ${OpenCV_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} + ${LIBMP4_INC_DIR} + ${BARESIP_INC_DIR} + ${LIBRE_INC_DIR} + ${NVCODEC_INCLUDE_DIR} +) + +target_link_libraries( + ${TARGET} + aprapipes + ${JPEG_LIBRARIES} + ${LIBMP4_LIB} + ${OPENH264_LIB} + ${Boost_LIBRARIES} + ${FFMPEG_LIBRARIES} + ${OpenCV_LIBRARIES} + ${JETSON_LIBS} + ${NVCUDAToolkit_LIBS} + ${NVCODEC_LIB} + ${NVJPEGLIB_L4T} + ${CURSES_LIBRARIES} +) \ No newline at end of file diff --git a/samples/relay-sample/relay_sample.cpp b/samples/relay-sample/relay_sample.cpp new file mode 100644 index 000000000..f656dbbb8 --- /dev/null +++ b/samples/relay-sample/relay_sample.cpp @@ -0,0 +1,120 @@ +#include "relay_sample.h" +#include "ExternalSinkModule.h" +#include "FrameMetadata.h" +#include "H264Metadata.h" +#include "ImageViewerModule.h" +#include "KeyboardListener.h" +#include "Logger.h" +#include "Mp4VideoMetadata.h" +#include "PipeLine.h" +#include "RTSPClientSrc.h" +#include "stdafx.h" +#include +#include +#include +#include + +RelayPipeline::RelayPipeline() : pipeline("RelaySample") {} + +void RelayPipeline::addRelayToRtsp(bool open) { + rtspSource->relay(h264Decoder, open); +} + +void RelayPipeline::addRelayToMp4(bool open) { + mp4ReaderSource->relay(h264Decoder, open); +} + +//bool RelayPipeline::testPipeline() { +// +// auto sink = boost::shared_ptr(new ExternalSinkModule()); +// colorConversion->setNext(sink); +// +// BOOST_TEST(rtspSource->init()); +// BOOST_TEST(mp4ReaderSource->init()); +// BOOST_TEST(h264Decoder->init()); +// BOOST_TEST(colorConversion->init()); +// BOOST_TEST(sink->init()); +// +// for (int i = 0; i <= 10; i++) { +// mp4ReaderSource->step(); +// h264Decoder->step(); +// } +// colorConversion->step(); +// +// auto frames = sink->pop(); +// frame_sp outputFrame = frames.cbegin()->second; +// Test_Utils::saveOrCompare( +// "../.././data/frame_from_mp4.raw", +// const_cast(static_cast(outputFrame->data())), +// outputFrame->size(), 0); +// +// for (int i = 0; i <= 10; i++) { +// rtspSource->step(); +// h264Decoder->step(); +// } +// colorConversion->step(); +// +// frames = sink->pop(); +// outputFrame = frames.cbegin()->second; +// Test_Utils::saveOrCompare( +// "../.././data/frame_from_rtsp.raw", +// const_cast(static_cast(outputFrame->data())), +// outputFrame->size(), 0); +// +// return true; +//} + +bool RelayPipeline::setupPipeline(const std::string &rtspUrl, const std::string &mp4VideoPath) { + // RTSP + auto url = std::string(rtspUrl); + rtspSource = boost::shared_ptr( + new RTSPClientSrc(RTSPClientSrcProps(url, "", ""))); + auto rtspMetaData = framemetadata_sp(new H264Metadata(1280, 720)); + rtspSource->addOutputPin(rtspMetaData); + + // MP4 + bool parseFS = false; + auto h264ImageMetadata = framemetadata_sp(new H264Metadata(1280, 720)); + auto frameType = FrameMetadata::FrameType::H264_DATA; + auto mp4ReaderProps = + Mp4ReaderSourceProps(mp4VideoPath, parseFS, 0, true, true, false); + mp4ReaderProps.fps = 9; + mp4ReaderSource = + boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); + mp4ReaderSource->addOutPutPin(h264ImageMetadata); + auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); + mp4ReaderSource->addOutPutPin(mp4Metadata); + std::vector mImagePin; + mImagePin = mp4ReaderSource->getAllOutputPinsByType(frameType); + + h264Decoder = + boost::shared_ptr(new H264Decoder(H264DecoderProps())); + rtspSource->setNext(h264Decoder); + mp4ReaderSource->setNext(h264Decoder, mImagePin); + + colorConversion = boost::shared_ptr(new ColorConversion( + ColorConversionProps(ColorConversionProps::YUV420PLANAR_TO_RGB))); + h264Decoder->setNext(colorConversion); + + imageViewer = boost::shared_ptr( + new ImageViewerModule(ImageViewerModuleProps("Relay Sample"))); + colorConversion->setNext(imageViewer); + return true; +} + +bool RelayPipeline::startPipeline() { + pipeline.appendModule(rtspSource); + pipeline.appendModule(mp4ReaderSource); + pipeline.init(); + pipeline.run_all_threaded(); + addRelayToMp4(false); + return true; +} + +bool RelayPipeline::stopPipeline() { + pipeline.stop(); + pipeline.term(); + pipeline.wait_for_all(); + return true; +} + diff --git a/samples/relay-sample/relay_sample.h b/samples/relay-sample/relay_sample.h new file mode 100644 index 000000000..d12fd41f5 --- /dev/null +++ b/samples/relay-sample/relay_sample.h @@ -0,0 +1,31 @@ +#include "ImageViewerModule.h" +#include "KeyboardListener.h" +#include "Mp4VideoMetadata.h" +#include +#include + +#include "ImageViewerModule.h" +#include "RTSPClientSrc.h" +#include +#include +#include + +class RelayPipeline { +public: + RelayPipeline(); + + boost::shared_ptr rtspSource; + boost::shared_ptr mp4ReaderSource; + boost::shared_ptr h264Decoder; + boost::shared_ptr colorConversion; + boost::shared_ptr imageViewer; + + bool setupPipeline(const std::string &rtspUrl, const std::string &mp4VideoPath); + bool startPipeline(); + bool stopPipeline(); + void addRelayToRtsp(bool open); + void addRelayToMp4(bool open); + +private: + PipeLine pipeline; +}; \ No newline at end of file diff --git a/samples/relay-sample/relay_sample_test.cpp b/samples/relay-sample/relay_sample_test.cpp new file mode 100644 index 000000000..699e59557 --- /dev/null +++ b/samples/relay-sample/relay_sample_test.cpp @@ -0,0 +1,52 @@ +#include "relay_sample.h" +#include "test_utils.h" +#include +#include +#include "ExternalSinkModule.h" + +BOOST_AUTO_TEST_SUITE(relay_sample) + +BOOST_AUTO_TEST_CASE(relaySample) { + RelayPipeline relayPipeline; + std::string rtspUrl = "rtsp://root:m4m1g0@10.102.10.75/axis-media/media.amp?resolution=1280x720"; + std::string mp4VideoPath = "data/1714992199120.mp4"; + relayPipeline.setupPipeline(rtspUrl, mp4VideoPath); + + + auto sink = boost::shared_ptr(new ExternalSinkModule()); + relayPipeline.colorConversion->setNext(sink); + + BOOST_TEST(relayPipeline.rtspSource->init()); + BOOST_TEST(relayPipeline.mp4ReaderSource->init()); + BOOST_TEST(relayPipeline.h264Decoder->init()); + BOOST_TEST(relayPipeline.colorConversion->init()); + BOOST_TEST(sink->init()); + + for (int i = 0; i <= 10; i++) { + relayPipeline.mp4ReaderSource->step(); + relayPipeline.h264Decoder->step(); + } + relayPipeline.colorConversion->step(); + + auto frames = sink->pop(); + frame_sp outputFrame = frames.cbegin()->second; + Test_Utils::saveOrCompare( + "../.././data/frame_from_mp4.raw", + const_cast(static_cast(outputFrame->data())), + outputFrame->size(), 0); + + for (int i = 0; i <= 10; i++) { + relayPipeline.rtspSource->step(); + relayPipeline.h264Decoder->step(); + } + relayPipeline.colorConversion->step(); + + frames = sink->pop(); + outputFrame = frames.cbegin()->second; + Test_Utils::saveOrCompare( + "../.././data/frame_from_rtsp.raw", + const_cast(static_cast(outputFrame->data())), + outputFrame->size(), 0); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/samples/timelapse-sample/CMakeLists.txt b/samples/timelapse-sample/CMakeLists.txt new file mode 100644 index 000000000..9dd3893a0 --- /dev/null +++ b/samples/timelapse-sample/CMakeLists.txt @@ -0,0 +1,40 @@ +set(TARGET generateTimeLapse) +SET(SAMPLE_FILES + timelapse_summary.cpp + PipelineMain.cpp +) +SET(SAMPLE_FILES_H +timelapse_summary.h +) +SET(SOURCE + ${SAMPLE_FILES} + ${SAMPLE_FILES_H} +) +add_executable(${TARGET} ${SOURCE}) +target_include_directories ( ${TARGET} PRIVATE +${JETSON_MULTIMEDIA_LIB_INCLUDE} + ${OpenCV_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} + ${LIBMP4_INC_DIR} +) +target_include_directories (${TARGET} PUBLIC + ${JETSON_MULTIMEDIA_LIB_INCLUDE} + ${FFMPEG_INCLUDE_DIRS} + ${OpenCV_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} + ${LIBMP4_INC_DIR} + ${BARESIP_INC_DIR} + ${LIBRE_INC_DIR} + ${NVCODEC_INCLUDE_DIR} +) +target_link_libraries( + ${TARGET} + aprapipes + ${LIBMP4_LIB} + ${OPENH264_LIB} + ${Boost_LIBRARIES} + ${FFMPEG_LIBRARIES} + ${OpenCV_LIBRARIES} + ${NVCUDAToolkit_LIBS} + ${NVCODEC_LIB} + ) diff --git a/samples/timelapse-sample/PipelineMain.cpp b/samples/timelapse-sample/PipelineMain.cpp new file mode 100644 index 000000000..42c0d54f9 --- /dev/null +++ b/samples/timelapse-sample/PipelineMain.cpp @@ -0,0 +1,31 @@ +#include "timelapse_summary.h" +#include +#include + +void main(int argc, char *argv[]) { + if (argc < 3) { + std::cerr << "Usage: " << argv[0] << " " + << std::endl; + } + + std::string videoPath = argv[argc - 2]; + std::string outFolderPath = argv[argc - 1]; + + TimelapsePipeline timelapsePipeline; + if (!timelapsePipeline.setupPipeline(videoPath, outFolderPath)) { + std::cerr << "Failed to setup pipeline." << std::endl; + } + + if (!timelapsePipeline.startPipeline()) { + std::cerr << "Failed to start pipeline." << std::endl; + } + + // Wait for the pipeline to run for 10 seconds + boost::this_thread::sleep_for(boost::chrono::minutes(2)); + + // Stop the pipeline + if (!timelapsePipeline.stopPipeline()) { + std::cerr << "Failed to stop pipeline." << std::endl; + } + std::cerr << "Saved Generated TimeLapse in" << outFolderPath << std::endl; +} \ No newline at end of file diff --git a/samples/timelapse-sample/timelapse_summary.cpp b/samples/timelapse-sample/timelapse_summary.cpp new file mode 100644 index 000000000..2bc18052a --- /dev/null +++ b/samples/timelapse-sample/timelapse_summary.cpp @@ -0,0 +1,100 @@ +#include "timelapse_summary.h" +#include "ColorConversionXForm.h" +#include "CudaMemCopy.h" +#include "CudaStreamSynchronize.h" +#include "ExternalSinkModule.h" +#include "FileWriterModule.h" +#include "H264EncoderNVCodec.h" +#include "H264Metadata.h" +#include "ImageViewerModule.h" +#include "MotionVectorExtractor.h" +#include "Mp4ReaderSource.h" +#include "Mp4VideoMetadata.h" +#include "Mp4WriterSink.h" +#include "OverlayModule.h" +#include "PipeLine.h" +#include + +TimelapsePipeline::TimelapsePipeline() + : timelapseSamplePipeline("test"), mCudaStream_(new ApraCudaStream()), + mCuContext(new ApraCUcontext()), + mH264ImageMetadata(new H264Metadata(0, 0)) {} + +bool TimelapsePipeline::setupPipeline(const std::string &videoPath, + const std::string &outFolderPath) { + uint32_t gopLength = 25; + uint32_t bitRateKbps = 1000; + uint32_t frameRate = 30; + H264EncoderNVCodecProps::H264CodecProfile profile = + H264EncoderNVCodecProps::MAIN; + bool enableBFrames = false; + bool sendDecodedFrames = true; + + auto mp4ReaderProps = + Mp4ReaderSourceProps(videoPath, false, 0, true, false, false); + mp4ReaderProps.parseFS = true; + mp4ReaderProps.readLoop = false; + // mp4Reader module is being used here to read the .mp4 videos + mMp4Reader = + boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); + auto motionExtractorProps = MotionVectorExtractorProps( + MotionVectorExtractorProps::MVExtractMethod::OPENH264, sendDecodedFrames, + 2); + // motionVectorExtractor module is being used to get the frames for the + // defined thershold + mMotionExtractor = boost::shared_ptr( + new MotionVectorExtractor(motionExtractorProps)); + // convert frames from BGR to RGB + mColorchange1 = boost::shared_ptr(new ColorConversion( + ColorConversionProps(ColorConversionProps::BGR_TO_RGB))); + // convert frames from RGB to YUV420PLANAR + // the two step color change is done because H264Encoder takes YUV data + mColorchange2 = boost::shared_ptr(new ColorConversion( + ColorConversionProps(ColorConversionProps::RGB_TO_YUV420PLANAR))); + mSync = boost::shared_ptr( + new CudaStreamSynchronize(CudaStreamSynchronizeProps(mCudaStream_))); + mEncoder = boost::shared_ptr(new H264EncoderNVCodec( + H264EncoderNVCodecProps(bitRateKbps, mCuContext, gopLength, frameRate, + profile, enableBFrames))); + // write the output video + auto mp4WriterSinkProps = + Mp4WriterSinkProps(UINT32_MAX, 10, 24, outFolderPath, true); + mp4WriterSinkProps.recordedTSBasedDTS = false; + mMp4WriterSink = + boost::shared_ptr(new Mp4WriterSink(mp4WriterSinkProps)); + + mMp4Reader->addOutPutPin(mH264ImageMetadata); + auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); + mMp4Reader->addOutPutPin(mp4Metadata); + std::vector mImagePin = + mMp4Reader->getAllOutputPinsByType(FrameMetadata::H264_DATA); + std::vector mDecodedPin = + mMotionExtractor->getAllOutputPinsByType(FrameMetadata::RAW_IMAGE); + mMp4Reader->setNext(mMotionExtractor, mImagePin); + mMotionExtractor->setNext(mColorchange1, mDecodedPin); + mColorchange1->setNext(mColorchange2); + mCopy = boost::shared_ptr( + new CudaMemCopy(CudaMemCopyProps(cudaMemcpyHostToDevice, mCudaStream_))); + mColorchange2->setNext(mCopy); + mCopy->setNext(mSync); + mSync->setNext(mEncoder); + mEncoder->setNext(mMp4WriterSink); + + timelapseSamplePipeline.appendModule(mMp4Reader); + if (timelapseSamplePipeline.init()) { + return true; + } + return false; +} + +bool TimelapsePipeline::startPipeline() { + timelapseSamplePipeline.run_all_threaded(); + return true; +} + +bool TimelapsePipeline::stopPipeline() { + timelapseSamplePipeline.stop(); + timelapseSamplePipeline.term(); + timelapseSamplePipeline.wait_for_all(); + return true; +} diff --git a/samples/timelapse-sample/timelapse_summary.h b/samples/timelapse-sample/timelapse_summary.h new file mode 100644 index 000000000..0102add50 --- /dev/null +++ b/samples/timelapse-sample/timelapse_summary.h @@ -0,0 +1,24 @@ + +#include + +class TimelapsePipeline { +public: + TimelapsePipeline(); + bool setupPipeline(); + bool startPipeline(); + bool stopPipeline(); + +private: + PipeLine timelapseSamplePipeline; + cudastream_sp mCudaStream_; + apracucontext_sp mCuContext; + framemetadata_sp mH264ImageMetadata; + boost::shared_ptr mMp4Reader; + boost::shared_ptr mMotionExtractor; + boost::shared_ptr mColorchange1; + boost::shared_ptr mColorchange2; + boost::shared_ptr mSync; + boost::shared_ptr mEncoder; + boost::shared_ptr mMp4WriterSink; + boost::shared_ptr mCopy; +}; \ No newline at end of file diff --git a/samples/timelapse-sample/timelapse_summary_test.cpp b/samples/timelapse-sample/timelapse_summary_test.cpp new file mode 100644 index 000000000..e7bbce6b8 --- /dev/null +++ b/samples/timelapse-sample/timelapse_summary_test.cpp @@ -0,0 +1,22 @@ +#include +#include +#include "test_utils.h" +#include "timelapse_summary.h" + +BOOST_AUTO_TEST_SUITE(generate_timelapse) + +BOOST_AUTO_TEST_CASE(generateTimeLapse) +{ + auto timelapsePipeline = boost::shared_ptr(new TimelapsePipeline()); + + std::string inputVideoPath = "./data/timelapse_sample_videos"; + std::string outputPath = "./data/timelapse_videos"; + + BOOST_ASSERT(timelapsePipeline->setupPipeline(inputVideoPath,outputPath)); + BOOST_ASSERT(timelapsePipeline->startPipeline()); + boost::this_thread::sleep(boost::chrono::seconds(10)); + + BOOST_ASSERT(timelapsePipeline->stopPipeline()); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file