diff --git a/base/CMakeLists.txt b/base/CMakeLists.txt index 473385aa8..6fde5c2d3 100755 --- a/base/CMakeLists.txt +++ b/base/CMakeLists.txt @@ -99,7 +99,7 @@ find_package(OpenSSL REQUIRED) IF(ENABLE_CUDA) if((NOT DEFINED CMAKE_CUDA_ARCHITECTURES) OR (CMAKE_CUDA_ARCHITECTURES STREQUAL "")) - set(CMAKE_CUDA_ARCHITECTURES 52 60 70 75) + set(CMAKE_CUDA_ARCHITECTURES 52 60 70 75 80 86 87) endif() message ("CUDA ARCHS: ${CMAKE_CUDA_ARCHITECTURES}") @@ -119,7 +119,9 @@ IF(ENABLE_CUDA) find_library(GLESv2LIB GLESv2 REQUIRED) find_library(X11LIB X11 REQUIRED) find_library(NVBUFUTILSLIB nvbuf_utils REQUIRED) - find_library(EGLSTREAM_CAMCONSUMER_LIB nveglstream_camconsumer REQUIRED) + find_library(NVBUFUTILSLIB nvbufsurface REQUIRED) + find_library(NVBUFSURFTRANSFORMLIB nvbufsurftransform REQUIRED) + # find_library(EGLSTREAM_CAMCONSUMER_LIB nveglstream_camconsumer REQUIRED) find_library(NVARGUS_SOCKETCLINET_LIB nvargus_socketclient REQUIRED) find_package(Curses REQUIRED) @@ -128,10 +130,9 @@ IF(ENABLE_CUDA) libcuda.so.1.1 ${V4L2LIB} ${NVBUFUTILSLIB} - ${EGLLIB} + ${NVBUFSURFTRANSFORMLIB} ${GLESv2LIB} ${X11LIB} - ${EGLSTREAM_CAMCONSUMER_LIB} ${NVARGUS_SOCKETCLINET_LIB} ) include_directories(AFTER SYSTEM /usr/local/cuda/include) @@ -305,41 +306,36 @@ ENDIF(ENABLE_LINUX) SET(IP_FILES src/ApraLines.cpp - src/CalcHistogramCV.cpp - src/HistogramOverlay.cpp - src/ImageDecoderCV.cpp - src/ImageViewerModule.cpp - src/BMPConverter.cpp - src/ImageResizeCV.cpp - src/ImageEncoderCV.cpp - src/RotateCV.cpp - src/BrightnessContrastControlXform.cpp - src/VirtualPTZ.cpp - src/WebCamSource.cpp - src/FaceDetectorXform.cpp - src/TextOverlayXForm.cpp - src/ValveModule.cpp + # src/CalcHistogramCV.cpp + # src/HistogramOverlay.cpp + # src/ImageDecoderCV.cpp + # src/ImageViewerModule.cpp + # src/BMPConverter.cpp + # src/ImageResizeCV.cpp + # src/FacialLandmarksCV.cpp + # src/ImageEncoderCV.cpp + # src/RotateCV.cpp + # src/AffineTransform.cpp + # src/BrightnessContrastControlXform.cpp + # src/VirtualPTZ.cpp + # src/WebCamSource.cpp + # src/FaceDetectorXform.cpp + # src/TextOverlayXForm.cpp + # src/ValveModule.cpp src/ColorConversionXForm.cpp src/AbsColorConversionFactory.cpp src/ColorConversionStrategy.h src/AbsColorConversionFactory.h src/ArchiveSpaceManager.cpp - src/AbsControlModule.cpp - src/EndocamControlModule.cpp + # src/Overlay.cpp + # src/OverlayFactory.h + # src/OverlayFactory.cpp + # src/TestSignalGeneratorSrc.cpp + # src/AudioToTextXForm.cpp + # src/AbsControlModule.cpp + # src/ThumbnailListGenerator.cpp ) - - -IF(ENABLE_GST) - IF(NOT ENABLE_WINDOWS) - list(APPEND IP_FILES - src/GstWebRTCSink.cpp - ) - ENDIF(NOT ENABLE_WINDOWS) - list(APPEND IP_FILES - src/GstOnvifRtspSink.cpp - ) -ENDIF(ENABLE_GST) - + SET(IP_FILES_H include/HistogramOverlay.h include/CalcHistogramCV.h @@ -419,26 +415,24 @@ SET(CUDA_IP_FILES IF(ENABLE_ARM64) SET(CUDA_IP_FILES ${CUDA_IP_FILES} - src/JPEGDecoderL4TM.cpp - src/JPEGDecoderL4TMHelper.cpp + # src/JPEGDecoderL4TM.cpp + # src/JPEGDecoderL4TMHelper.cpp src/JPEGEncoderL4TM.cpp src/JPEGEncoderL4TMHelper.cpp src/AV4L2Buffer.cpp src/AV4L2ElementPlane.cpp - src/H264EncoderV4L2Helper.cpp - src/V4L2CUYUV420Converter.cpp - src/H264EncoderV4L2.cpp + # src/H264EncoderV4L2Helper.cpp + # src/V4L2CUYUV420Converter.cpp + # src/H264EncoderV4L2.cpp src/DMAFDWrapper.cpp - src/NvArgusCameraHelper.cpp - src/NvArgusCamera.cpp - src/NvV4L2Camera.cpp - src/NvV4L2CameraHelper.cpp - src/EglRenderer.cpp - src/EglRendererReview.cpp - src/ApraEglRenderer.cpp - src/NvEglRenderer.cpp + # src/NvArgusCameraHelper.cpp + # src/NvArgusCamera.cpp + # src/NvV4L2Camera.cpp + # src/NvV4L2CameraHelper.cpp + # src/EglRenderer.cpp + # src/NvEglRenderer.cpp src/DMAUtils.cpp - src/NvTransform.cpp + # src/NvTransform.cpp src/ApraEGLDisplay.cpp src/DMAFDToHostCopy.cpp src/Matrix.cpp @@ -613,10 +607,11 @@ ENDIF(ENABLE_ARM64) IF (ENABLE_CUDA) SET(CUDA_UT_FILES - test/cudamemcopy_tests.cpp - test/resizenppi_tests.cpp - test/rotatenppi_tests.cpp - test/masknppi_tests.cpp + # test/cudamemcopy_tests.cpp + # test/resizenppi_tests.cpp + # test/rotatenppi_tests.cpp + # test/ccnppi_tests.cpp + # test/memtypeconversion_tests.cpp ) IF(NOT ENABLE_ARM64) # following tests need CUDA but can not run on ARM ? @@ -637,16 +632,16 @@ ENDIF(ENABLE_CUDA) SET(UT_FILES test/utmain.cpp - # test/unit_tests.cpp - # test/cv_memory_leaks_tests.cpp - # test/module_tests.cpp + test/unit_tests.cpp + test/cv_memory_leaks_tests.cpp + test/module_tests.cpp # test/calchistogramcv_tests.cpp # test/filenamestrategy_tests.cpp test/test_utils.cpp test/test_utils.h - # test/filewritermodule_tests.cpp + test/filewritermodule_tests.cpp # test/logger_tests.cpp - # test/logger_stress_tests.cpp #todo this test needs to be improved and added +# test/logger_stress_tests.cpp #todo this test needs to be improved and added # test/quepushstrategy_tests.cpp # test/framesmuxer_tests.cpp # test/filereadermodule_tests.cpp @@ -656,10 +651,13 @@ SET(UT_FILES # test/bmpconverter_tests.cpp # test/rtsppusher_tests.cpp # test/findexstrategy_tests.cpp - test/jpegdecodercv_tests.cpp + # test/jpegdecodercv_tests.cpp # test/Imageresizecv_tests.cpp - test/ImageEncodeCV_tests.cpp + # test/faciallandmarkscv_tests.cpp + # test/imageviewermodule_tests.cpp + # test/ImageEncodeCV_tests.cpp # test/rotatecv_tests.cpp + # test/affinetransform_tests.cpp # test/brightness_contrast_tests.cpp # test/virtualptz_tests.cpp # test/webcam_source_tests.cpp @@ -668,25 +666,34 @@ SET(UT_FILES # test/pullstratergy_tests.cpp # test/QRReader_tests.cpp # test/textoverlayxform_tests.cpp - test/mp4writersink_tests.cpp + # test/mp4writersink_tests.cpp # test/pipeline_tests.cpp # test/multiple_pipeline_tests.cpp #todo this test needs to be improved and added # test/valveModule_tests.cpp # test/color_conversion_tests.cpp # test/archivespacemanager_tests.cpp # test/multimediaqueuexform_tests.cpp - test/mp4readersource_tests.cpp - test/fileRecovery_tests.cpp - # test/rtsp_client_tests.cpp + # test/mp4readersource_tests.cpp # test/rtsp_client_tests.cpp + # test/motionvector_extractor_and_overlay_tests.cpp + # test/mp4_reverse_play_tests.cpp + # test/ordered_cache_of_files_tests.cpp + # test/mp4_seek_tests.cpp + # test/mp4_simul_read_write_tests.cpp + # test/mp4_getlivevideots_tests.cpp + # test/mp4_dts_strategy_tests.cpp + # test/overlaymodule_tests.cpp + # test/testSignalGeneratorSrc_tests.cpp + # test/audioToTextXform_tests.cpp ${ARM64_UT_FILES} # ${CUDA_UT_FILES} ) IF(ENABLE_LINUX) list(APPEND UT_FILES - test/virtualcamerasink_tests.cpp - test/QRReader_tests.cpp + # test/gtkglrenderer_tests.cpp + # test/virtualcamerasink_tests.cpp + # test/QRReader_tests.cpp ) ENDIF(ENABLE_LINUX) diff --git a/base/include/ApraNvEglRenderer.h b/base/include/ApraNvEglRenderer.h index 1e39adf2c..a43f74e0b 100644 --- a/base/include/ApraNvEglRenderer.h +++ b/base/include/ApraNvEglRenderer.h @@ -39,6 +39,11 @@ #include #include #include +#include +#include FT_FREETYPE_H +#include +#include +#include #include @@ -95,9 +100,29 @@ class NvEglRenderer */ static NvEglRenderer *createEglRenderer(const char *name, uint32_t width, uint32_t height, uint32_t x_offset, - uint32_t y_offset); + uint32_t y_offset, + const char* ttfFilePath = NULL, + const char* message = NULL, + float scale = 0.0f, + float r = 0.0f, + float g = 0.0f, + float b = 0.0f,float fontsize = 0.0f,int textPosX = 0, int textPosY = 0, + std::string imagePath = "",int imagePosX = 0,int imagePosY = 0,uint32_t imageWidth = 0,uint32_t imageHeight = 0,float opacity = 1,bool mask = false, + float imageOpacity = 1.0f,float textOpacity = 1.0f); ~NvEglRenderer(); + std::string ttfFilePath; + std::string message; + float scale; + float r, g, b; + float fontSize; + int textPosX, textPosY; + float opacity; + std::string imagePath; + int imagePosX, imagePosY; + uint32_t imageWidth, imageHeight; + float imageOpacity,textOpacity; + bool mask; /** * Renders a buffer. * @@ -112,6 +137,41 @@ class NvEglRenderer */ int render(int fd); + // Set dmabuf import parameters for EGL_EXT_image_dma_buf_import + void setImportParams(int pitchBytes, int fourcc, int offsetBytes = 0, int width = 0, int height = 0) + { + render_pitch = pitchBytes; + render_fourcc = fourcc; + render_offset = offsetBytes; + render_width = width; + render_height = height; + render_num_planes = 1; + } + + // Set multi-plane import parameters (e.g., NV12 - 2 planes, YUV420 - 3 planes) + void setImportParamsPlanar(int fourcc, + int width, + int height, + int pitchPlane0, + int offsetPlane0, + int pitchPlane1, + int offsetPlane1, + int pitchPlane2 = 0, + int offsetPlane2 = 0, + int numPlanes = 2) + { + render_fourcc = fourcc; + render_width = width; + render_height = height; + render_pitch = pitchPlane0; + render_offset = offsetPlane0; + render_pitch1 = pitchPlane1; + render_offset1 = offsetPlane1; + render_pitch2 = pitchPlane2; + render_offset2 = offsetPlane2; + render_num_planes = numPlanes; + } + /** * Sets the rendering rate in frames per second (fps). * @@ -148,8 +208,19 @@ class NvEglRenderer * @param[in] y Vertical offset, in pixels. * @return 0 for success, -1 otherwise. */ - int setOverlayText(char *str, uint32_t x, uint32_t y); + static PFNGLGENVERTEXARRAYSOESPROC glGenVertexArraysOES; + static PFNGLBINDVERTEXARRAYOESPROC glBindVertexArrayOES; + static PFNGLDELETEVERTEXARRAYSOESPROC glDeleteVertexArraysOES; + int setOverlayText(char *str, uint32_t x, uint32_t y); + void setWindowOpacity(float opacity); + void RenderText(std::string text, float x, float y, float scale, float r, float g, float b); + GLuint initTextShader(); + int initFontAtlas(const char* fontPath, int fontSize); + GLuint loadImageTexture(const char* imagePath); + GLuint initImageShader(); + void RenderImage(GLuint texture, float x, float y, float width, float height); + private: Display * x_display; /**< Connection to the X server created using XOpenDisplay(). */ @@ -166,7 +237,9 @@ class NvEglRenderer GC gc; /**< Graphic Context */ XFontStruct *fontinfo; /**< Brush's font info */ char overlay_str[512]; /**< Overlay's text */ - + GLuint gl_program = 0; // OpenGL shader program handle + GLint alpha_location = -1; // Location of alpha uniform in shader + GLuint cached_image_texture = 0; // Cached image texture loaded once during initialization /** * Creates a GL texture used for rendering. * @@ -213,7 +286,15 @@ class NvEglRenderer * Constructor called by the wrapper createEglRenderer. */ NvEglRenderer(const char *name, uint32_t width, uint32_t height, - uint32_t x_offset, uint32_t y_offset); + uint32_t x_offset, uint32_t y_offset, + const char* ttfFilePath = NULL, + const char* message = NULL, + float scale = 0.0f, + float r = 0.0f, + float g = 0.0f, + float b = 0.0f,float fontsize = 0.0f,int textPosX = 0, int textPosY = 0, + std::string imagePath = "",int imagePosX = 0,int imagePosY = 0,uint32_t imageWidth = 0,uint32_t imageHeight = 0,float opacity = 1,bool mask = false, + float imageOpacity = 1.0f,float textOpacity = 1.0f); /** * Gets the pointers to the required EGL methods. */ @@ -232,6 +313,19 @@ class NvEglRenderer */ int renderInternal(); + /** + * Helper functions for rendering + */ + EGLImageKHR createEglImageFromDmaBuf(); + int renderVideoFrame(EGLImageKHR hEglImage); + void renderOverlays(); + void saveGLState(GLint& prevProgram, GLint& prevVAO, GLint& prevTexExternal, + GLint& prevTex2D, GLint& prevArrayBuffer, GLint& prevActiveTexUnit, + GLboolean& wasBlendEnabled); + void restoreGLState(GLint prevProgram, GLint prevVAO, GLint prevTexExternal, + GLint prevTex2D, GLint prevArrayBuffer, GLint prevActiveTexUnit, + GLboolean wasBlendEnabled); + /** * These EGL function pointers are required by the renderer. */ @@ -242,6 +336,18 @@ class NvEglRenderer static PFNEGLCLIENTWAITSYNCKHRPROC eglClientWaitSyncKHR; static PFNEGLGETSYNCATTRIBKHRPROC eglGetSyncAttribKHR; static PFNGLEGLIMAGETARGETTEXTURE2DOESPROC glEGLImageTargetTexture2DOES; + + // dma-buf import params + int render_pitch = 0; + int render_offset = 0; + int render_fourcc = 0; + int render_width = 0; + int render_height = 0; + int render_num_planes = 1; + int render_pitch1 = 0; + int render_offset1 = 0; + int render_pitch2 = 0; + int render_offset2 = 0; }; /** @} */ #endif \ No newline at end of file diff --git a/base/include/DMAAllocator.h b/base/include/DMAAllocator.h index e8753c4a0..e4167ebe9 100755 --- a/base/include/DMAAllocator.h +++ b/base/include/DMAAllocator.h @@ -1,97 +1,93 @@ #pragma once #include "Allocators.h" #include "DMAFDWrapper.h" -#include "nvbuf_utils.h" +#include "nvbufsurface.h" #include "ImageMetadata.h" #include "RawImageMetadata.h" #include "RawImagePlanarMetadata.h" #include "FrameMetadataFactory.h" -#include "ApraEGLDisplay.h" #include "Logger.h" #include - +using namespace std; class DMAAllocator : public HostAllocator { private: std::vector mDMAFDWrapperArr; int mFreeDMACount; - NvBufferColorFormat mColorFormat; + NvBufSurfaceColorFormat mColorFormat; EGLDisplay mEglDisplay; int mHeight; int mWidth; int mCount; + // Store metadata to allow lazy initialization when metadata is populated later + framemetadata_sp mMetadata; - static NvBufferColorFormat getColorFormat(ImageMetadata::ImageType imageType) + static NvBufSurfaceColorFormat getColorFormat(ImageMetadata::ImageType imageType) { - NvBufferColorFormat colorFormat; + NvBufSurfaceColorFormat colorFormat; switch (imageType) { - case ImageMetadata::UYVY: - colorFormat = NvBufferColorFormat_UYVY; - break; - case ImageMetadata::YUYV: - colorFormat = NvBufferColorFormat_YUYV; - break; - case ImageMetadata::RGBA: - colorFormat = NvBufferColorFormat_ABGR32; - break; - case ImageMetadata::BGRA: - colorFormat = NvBufferColorFormat_ARGB32; - break; - case ImageMetadata::YUV420: - colorFormat = NvBufferColorFormat_YUV420; - break; - case ImageMetadata::NV12: - colorFormat = NvBufferColorFormat_NV12; - break; - case ImageMetadata::YUV444: - colorFormat = NvBufferColorFormat_YUV444; - break; - default: - throw AIPException(AIP_FATAL, "Expected Actual<" + std::to_string(imageType) + ">"); + case ImageMetadata::UYVY: + colorFormat = NVBUF_COLOR_FORMAT_UYVY; + break; + case ImageMetadata::YUYV: + colorFormat = NVBUF_COLOR_FORMAT_YUYV; + break; + case ImageMetadata::RGBA: + colorFormat = NVBUF_COLOR_FORMAT_RGBA; + break; + case ImageMetadata::BGRA: + colorFormat = NVBUF_COLOR_FORMAT_BGRA; + break; + case ImageMetadata::YUV420: + colorFormat = NVBUF_COLOR_FORMAT_YUV420; + break; + case ImageMetadata::NV12: + colorFormat = NVBUF_COLOR_FORMAT_NV12; + break; + default: + throw AIPException(AIP_FATAL, "Expected Actual<" + std::to_string(imageType) + ">"); } - return colorFormat; } public: DMAAllocator(framemetadata_sp framemetadata) : mFreeDMACount(0), mCount(0) { - if (!framemetadata->isSet()) - { - return; - } - + // Always keep a handle to metadata for lazy initialization later + mMetadata = framemetadata; mEglDisplay = ApraEGLDisplay::getEGLDisplay(); - - auto imageType = ImageMetadata::RGBA; - - auto frameType = framemetadata->getFrameType(); - switch (frameType) - { - case FrameMetadata::FrameType::RAW_IMAGE: + mWidth = 0; + mHeight = 0; + // If metadata is already set, initialize now; otherwise we'll initialize lazily in allocateChunks + if (framemetadata->isSet()) { - auto inputRawMetadata = FrameMetadataFactory::downcast(framemetadata); - mWidth = inputRawMetadata->getWidth(); - mHeight = inputRawMetadata->getHeight(); - imageType = inputRawMetadata->getImageType(); - } - break; - case FrameMetadata::FrameType::RAW_IMAGE_PLANAR: - { - auto inputRawMetadata = FrameMetadataFactory::downcast(framemetadata); - mWidth = inputRawMetadata->getWidth(0); - mHeight = inputRawMetadata->getHeight(0); - imageType = inputRawMetadata->getImageType(); - } - break; - default: - throw AIPException(AIP_FATAL, "Expected Raw Image or RAW_IMAGE_PLANAR. Actual<" + std::to_string(frameType) + ">"); - break; + auto imageType = ImageMetadata::RGBA; + auto frameType = framemetadata->getFrameType(); + switch (frameType) + { + case FrameMetadata::FrameType::RAW_IMAGE: + { + auto inputRawMetadata = FrameMetadataFactory::downcast(framemetadata); + mWidth = inputRawMetadata->getWidth(); + mHeight = inputRawMetadata->getHeight(); + imageType = inputRawMetadata->getImageType(); + } + break; + case FrameMetadata::FrameType::RAW_IMAGE_PLANAR: + { + auto inputRawMetadata = FrameMetadataFactory::downcast(framemetadata); + mWidth = inputRawMetadata->getWidth(0); + mHeight = inputRawMetadata->getHeight(0); + imageType = inputRawMetadata->getImageType(); + } + break; + default: + throw AIPException(AIP_FATAL, "Expected Raw Image or RAW_IMAGE_PLANAR. Actual<" + std::to_string(frameType) + ">"); + } + mColorFormat = getColorFormat(imageType); } - - mColorFormat = getColorFormat(imageType); - }; + } ~DMAAllocator() { @@ -101,68 +97,86 @@ class DMAAllocator : public HostAllocator } } - static void setMetadata(framemetadata_sp &metadata, int width, int height, ImageMetadata::ImageType imageType) + static void setMetadata(framemetadata_sp &metadata, int width, int height, ImageMetadata::ImageType imageType, size_t pitchValues[4] = nullptr, size_t offsetValues[4] = nullptr) { auto eglDisplay = ApraEGLDisplay::getEGLDisplay(); auto colorFormat = getColorFormat(imageType); - auto dmaFDWrapper = DMAFDWrapper::create(0, width, height, colorFormat, NvBufferLayout_Pitch, eglDisplay); + auto dmaFDWrapper = DMAFDWrapper::create(0, width, height, colorFormat, NVBUF_LAYOUT_PITCH, eglDisplay); if (!dmaFDWrapper) { LOG_INFO << "Failed to allocate dmaFDWrapper"; throw AIPException(AIP_FATAL, "Memory Allocation Failed."); } - NvBufferParams fdParams; - if (NvBufferGetParams(dmaFDWrapper->getFd(), &fdParams)) + auto surf = dmaFDWrapper->getNvBufSurface(); + if (!surf) { - throw AIPException(AIP_FATAL, "NvBufferGetParams Failed."); - } - - LOG_DEBUG << "PixelFormat<" << fdParams.pixel_format << "> Planes<" << fdParams.num_planes << "> NvBufferSize<" << fdParams.nv_buffer_size << "> MemSize<" << fdParams.memsize << ">"; - for (auto i = 0; i < fdParams.num_planes; i++) - { - LOG_DEBUG << "Width<" << fdParams.width[i] << "> Height<" << fdParams.height[i] << "> Pitch<" << fdParams.pitch[i] << "> Offset<" << fdParams.offset[i] << "> PSize<" << fdParams.psize[i] << "> Layout<" << fdParams.layout[i] << ">"; + throw AIPException(AIP_FATAL, "NvBufSurface is null."); } + auto &fdParams = surf->surfaceList[0]; + LOG_DEBUG << "PixelFormat<" << fdParams.colorFormat << "> Layout<" << fdParams.layout << ">"; + LOG_DEBUG << "Width<" << fdParams.width << "> Height<" << fdParams.height << "> Pitch<" << fdParams.planeParams.pitch[0] << "> Offset<" << fdParams.planeParams.offset[0] << "> PSize<" << fdParams.planeParams.psize[0] << ">"; + LOG_DEBUG <<"pitch 1="<(metadata); + RawImageMetadata rawMetadata( + width, height, imageType, type, + fdParams.planeParams.pitch[0], CV_8U, FrameMetadata::MemType::DMABUF, false); + inputRawMetadata->setData(rawMetadata); + if (pitchValues != nullptr) + { + pitchValues[0] = fdParams.planeParams.pitch[0]; + } } - auto inputRawMetadata = FrameMetadataFactory::downcast(metadata); - RawImageMetadata rawMetadata(width, height, imageType, type, fdParams.pitch[0], CV_8U, FrameMetadata::MemType::DMABUF, false); - inputRawMetadata->setData(rawMetadata); - } - break; - case FrameMetadata::FrameType::RAW_IMAGE_PLANAR: // need to check for yuv444 - { - auto inputRawMetadata = FrameMetadataFactory::downcast(metadata); - size_t step[4] = {0, 0, 0, 0}; - for (auto i = 0; i < fdParams.num_planes; i++) + break; + case FrameMetadata::FrameType::RAW_IMAGE_PLANAR: // need to check for yuv444 { - step[i] = fdParams.pitch[i]; + auto inputRawMetadata = FrameMetadataFactory::downcast(metadata); + size_t step[4] = {0, 0, 0, 0}; + step[0] = fdParams.planeParams.pitch[0]; + // Fill pitches for all planes when available + step[1] = fdParams.planeParams.pitch[1]; + step[2] = fdParams.planeParams.pitch[2]; + if (pitchValues != nullptr) + { + pitchValues[0] = fdParams.planeParams.pitch[0]; + pitchValues[1] = fdParams.planeParams.pitch[1]; + pitchValues[2] = fdParams.planeParams.pitch[2]; + } + + if (offsetValues != nullptr) + { + offsetValues[0] = fdParams.planeParams.offset[0]; + offsetValues[1] = fdParams.planeParams.offset[1]; + offsetValues[2] = fdParams.planeParams.offset[2]; + } + + RawImagePlanarMetadata rawMetadata(width, height, imageType, step, CV_8U, FrameMetadata::MemType::DMABUF); + inputRawMetadata->setData(rawMetadata); } - RawImagePlanarMetadata rawMetadata(width, height, imageType, step, CV_8U, FrameMetadata::MemType::DMABUF); - inputRawMetadata->setData(rawMetadata); - } - break; - default: - throw AIPException(AIP_FATAL, "Expected Raw Image or RAW_IMAGE_PLANAR. Actual<" + std::to_string(frameType) + ">"); + break; + default: + throw AIPException(AIP_FATAL, "Expected Raw Image or RAW_IMAGE_PLANAR. Actual<" + std::to_string(frameType) + ">"); break; } @@ -171,9 +185,37 @@ class DMAAllocator : public HostAllocator void *allocateChunks(size_t n) { + // Lazy initialize using metadata if dimensions were not known at construction time + if ((mWidth == 0 || mHeight == 0) && mMetadata && mMetadata->isSet()) + { + auto imageType = ImageMetadata::RGBA; + auto frameType = mMetadata->getFrameType(); + switch (frameType) + { + case FrameMetadata::FrameType::RAW_IMAGE: + { + auto inputRawMetadata = FrameMetadataFactory::downcast(mMetadata); + mWidth = inputRawMetadata->getWidth(); + mHeight = inputRawMetadata->getHeight(); + imageType = inputRawMetadata->getImageType(); + } + break; + case FrameMetadata::FrameType::RAW_IMAGE_PLANAR: + { + auto inputRawMetadata = FrameMetadataFactory::downcast(mMetadata); + mWidth = inputRawMetadata->getWidth(0); + mHeight = inputRawMetadata->getHeight(0); + imageType = inputRawMetadata->getImageType(); + } + break; + default: + throw AIPException(AIP_FATAL, "Expected Raw Image or RAW_IMAGE_PLANAR. Actual<" + std::to_string(frameType) + ">"); + } + mColorFormat = getColorFormat(imageType); + } if (mFreeDMACount == 0) { - auto dmaFDWrapper = DMAFDWrapper::create(mCount++, mWidth, mHeight, mColorFormat, NvBufferLayout_Pitch, mEglDisplay); + auto dmaFDWrapper = DMAFDWrapper::create(mCount++, mWidth, mHeight, mColorFormat, NVBUF_LAYOUT_PITCH, mEglDisplay); if (!dmaFDWrapper) { LOG_INFO << "Failed to allocate dmaFDWrapper"; diff --git a/base/include/DMAFDWrapper.h b/base/include/DMAFDWrapper.h index 6c29fb419..61821c097 100644 --- a/base/include/DMAFDWrapper.h +++ b/base/include/DMAFDWrapper.h @@ -1,21 +1,24 @@ #pragma once #include "ApraEGLDisplay.h" // this is added to address the following issue: https://github.com/opencv/opencv/issues/7113 -#include "nvbuf_utils.h" +#include "nvbufsurface.h" #include "EGL/egl.h" #include "cudaEGL.h" +struct NvBufSurface; + class DMAFDWrapper { public: /* Always use this static method to create DMAFDWrapper */ static DMAFDWrapper *create(int index, int width, int height, - NvBufferColorFormat colorFormat, - NvBufferLayout layout, EGLDisplay eglDisplay); + NvBufSurfaceColorFormat colorFormat, + NvBufSurfaceLayout layout, EGLDisplay eglDisplay); virtual ~DMAFDWrapper(); /* Return DMA buffer handle */ int getFd() const { return m_fd; } + NvBufSurface* getNvBufSurface() const { return m_surf; } EGLImageKHR getEGLImage() const { return eglImage; } EGLDisplay getEGLDisplay() const { return eglDisplay; } void* getHostPtr(); @@ -38,6 +41,7 @@ class DMAFDWrapper private: int m_fd; + NvBufSurface* m_surf; EGLImageKHR eglImage; CUgraphicsResource pResource; CUeglFrame eglFrame; @@ -51,4 +55,4 @@ class DMAFDWrapper const int index; const void* clientData; -}; \ No newline at end of file +}; diff --git a/base/include/DMAUtils.h b/base/include/DMAUtils.h index 1f834f1f7..67a48b794 100644 --- a/base/include/DMAUtils.h +++ b/base/include/DMAUtils.h @@ -1,14 +1,14 @@ #pragma once -#include "nvbuf_utils.h" + #include "EGL/egl.h" #include "cudaEGL.h" - +#include class Frame; class DMAUtils { public: static uint8_t* getCudaPtrForFD(int fd, EGLImageKHR eglImage, CUgraphicsResource *pResource, CUeglFrame eglFrame, EGLDisplay eglDisplay); - static uint8_t* getCudaPtr(EGLImageKHR eglImage, CUgraphicsResource *pResource, CUeglFrame eglFrame, EGLDisplay eglDisplay); - static void freeCudaPtr(EGLImageKHR eglImage, CUgraphicsResource *pResource, EGLDisplay eglDisplay); + static uint8_t* getCudaPtr(EGLImageKHR eglImage, CUgraphicsResource *pResource, CUeglFrame *pEglFrame); + static void freeCudaPtr(EGLImageKHR eglImage, CUgraphicsResource *pResource, NvBufSurface *surf, EGLDisplay eglDisplay); }; \ No newline at end of file diff --git a/base/include/EglRenderer.h b/base/include/EglRenderer.h index c3cd1ea74..547521083 100644 --- a/base/include/EglRenderer.h +++ b/base/include/EglRenderer.h @@ -6,27 +6,103 @@ using CallbackFunction = std::function; class EglRendererProps : public ModuleProps { public: - EglRendererProps(uint32_t _x_offset,uint32_t _y_offset, uint32_t _width, uint32_t _height) : ModuleProps() - { - x_offset = _x_offset; - y_offset = _y_offset; - height = _height; - width = _width; - } - EglRendererProps(uint32_t _x_offset,uint32_t _y_offset) : ModuleProps() - { - x_offset = _x_offset; - y_offset = _y_offset; - height = 0; - width = 0; - } - uint32_t x_offset; - uint32_t y_offset; - uint32_t height; - uint32_t width; - // One more bool value which will be alwaysOnTop + struct TextInfo { + std::string fontPath = ""; //Path for TTF font file + std::string message = ""; + float scale = 0.0f; + float fontSize = 0.0f; + std::vector color = {0.0f, 0.0f, 0.0f}; // RGB + std::pair position = {0, 0}; + float opacity = 1.0f; + }; + + struct ImageInfo { + std::string path = ""; + std::pair position = {0, 0}; + std::pair size = {0, 0}; + float opacity = 1.0f; // width, height + }; + EglRendererProps() : ModuleProps() {}; + // All settings enabled + EglRendererProps(uint32_t _x_offset , uint32_t _y_offset , + uint32_t _width , uint32_t _height , + const TextInfo& _text , + const ImageInfo& _image , + float _opacity,bool _mask) + : ModuleProps(), + x_offset(_x_offset), y_offset(_y_offset), + width(_width), height(_height), + text(_text), image(_image), + opacity(_opacity), mask(_mask) + {} + // --- Geometry (x, y) --- + EglRendererProps(uint32_t _x_offset, uint32_t _y_offset) + : ModuleProps(), x_offset(_x_offset), y_offset(_y_offset) + {} + + // --- Geometry (x, y, width, height) --- + EglRendererProps(uint32_t _x_offset, uint32_t _y_offset, + uint32_t _width, uint32_t _height) + : ModuleProps(), x_offset(_x_offset), y_offset(_y_offset), + width(_width), height(_height) + {} + + // --- Geometry + Text --- + EglRendererProps(uint32_t _x_offset, uint32_t _y_offset, + uint32_t _width, uint32_t _height, + const TextInfo& _text) + : ModuleProps(), x_offset(_x_offset), y_offset(_y_offset), + width(_width), height(_height), text(_text) + {} + + // --- Geometry + Image --- + EglRendererProps(uint32_t _x_offset, uint32_t _y_offset, + uint32_t _width, uint32_t _height, + const ImageInfo& _image) + : ModuleProps(), x_offset(_x_offset), y_offset(_y_offset), + width(_width), height(_height), image(_image) + {} + + // --- Geometry + Text + Image --- + EglRendererProps(uint32_t _x_offset, uint32_t _y_offset, + uint32_t _width, uint32_t _height, + const TextInfo& _text, + const ImageInfo& _image) + : ModuleProps(), x_offset(_x_offset), y_offset(_y_offset), + width(_width), height(_height), + text(_text), image(_image) + {} + + // --- Geometry + Opacity + Mask --- + EglRendererProps(uint32_t _x_offset, uint32_t _y_offset, + uint32_t _width, uint32_t _height, + float _opacity, bool _mask) + : ModuleProps(), x_offset(_x_offset), y_offset(_y_offset), + width(_width), height(_height), + opacity(_opacity), mask(_mask) + {} + + + + + ~EglRendererProps() = default; + + // --- Geometry --- + uint32_t x_offset = 0; + uint32_t y_offset = 0; + uint32_t width = 0; + uint32_t height = 0; + + // --- Rendering data --- + TextInfo text; + ImageInfo image; + + // --- Display --- + float opacity = 1.0f; + bool mask = false; }; + class EglRenderer : public Module { public: diff --git a/base/include/JPEGEncoderL4TMHelper.h b/base/include/JPEGEncoderL4TMHelper.h index 2119d71f5..5feff099e 100755 --- a/base/include/JPEGEncoderL4TMHelper.h +++ b/base/include/JPEGEncoderL4TMHelper.h @@ -2,8 +2,8 @@ #define __NV_JPEG_ENCODER_H__ #include -#include -#include "libjpeg-8b/jpeglib.h" +#include +#include class JPEGEncoderL4TMHelper { diff --git a/base/include/NvTransform.h b/base/include/NvTransform.h index f4dc111c4..5ccea63a4 100644 --- a/base/include/NvTransform.h +++ b/base/include/NvTransform.h @@ -5,22 +5,86 @@ class NvTransformProps : public ModuleProps { public: - NvTransformProps(ImageMetadata::ImageType _imageType) : top(0) , left(0) , width(0) , height(0) - { - imageType = _imageType; - } - NvTransformProps(ImageMetadata::ImageType _imageType, int _width, int _height) : top(0) , left(0) , width(_width) , height(_height) - { - imageType = _imageType; - } - NvTransformProps(ImageMetadata::ImageType _imageType, int _width, int _height, int _top , int _left) : top(_top) , left(_left) , width(_width) , height(_height) - { - imageType = _imageType; - } - ImageMetadata::ImageType imageType; - int top,left,width,height; + enum class NvRotation + { + None_ = 0, + Rotate90 = 90, + Rotate180 = 180, + Rotate270 = 270 + }; + + enum class NvFlip + { + None_ = 0, + FlipX = 1, + FlipY = 2 + }; + + // Default crop constructor + NvTransformProps(ImageMetadata::ImageType _imageType) + : top(0), left(0), width(0), height(0), + rotation(NvRotation::None_), flip(NvFlip::None_) + { + imageType = _imageType; + } + + // Crop with width and height + NvTransformProps(ImageMetadata::ImageType _imageType, int _width, int _height) + : top(0), left(0), width(_width), height(_height), + rotation(NvRotation::None_), flip(NvFlip::None_) + { + imageType = _imageType; + } + + // Crop with width, height, top, left + NvTransformProps(ImageMetadata::ImageType _imageType, int _width, int _height, int _top, int _left) + : top(_top), left(_left), width(_width), height(_height), + rotation(NvRotation::None_), flip(NvFlip::None_) + { + imageType = _imageType; + } + + // Rotation constructor + NvTransformProps(ImageMetadata::ImageType _imageType, NvRotation _rotation) + : top(0), left(0), width(0), height(0), + rotation(_rotation),flip(NvFlip::None_) + { + imageType = _imageType; + } + + // Flip constructor + NvTransformProps(ImageMetadata::ImageType _imageType, NvFlip _flip) + : top(0), left(0), width(0), height(0), + rotation(NvRotation::None_), flip(_flip) + { + imageType = _imageType; + } + + //crop with rotation + NvTransformProps(ImageMetadata::ImageType _imageType, int _width, int _height, int _top, int _left, NvRotation _rotation) + : top(_top), left(_left), width(_width), height(_height), + rotation(_rotation), flip(NvFlip::None_) + { + imageType = _imageType; + } + + //crop with flip + NvTransformProps(ImageMetadata::ImageType _imageType, int _width, int _height, int _top, int _left, NvFlip _flip) + : top(_top), left(_left), width(_width), height(_height), + rotation(NvRotation::None_), flip(_flip) + { + imageType = _imageType; + } + + + + ImageMetadata::ImageType imageType; + int top, left, width, height; + NvRotation rotation; + NvFlip flip; }; + class NvTransform : public Module { diff --git a/base/src/AV4L2ElementPlane.cpp b/base/src/AV4L2ElementPlane.cpp index cb33dc859..5b4b043c2 100644 --- a/base/src/AV4L2ElementPlane.cpp +++ b/base/src/AV4L2ElementPlane.cpp @@ -33,7 +33,16 @@ void AV4L2ElementPlane::setPlaneFormat(uint32_t width, uint32_t height) mFormat.fmt.pix_mp.width = width; mFormat.fmt.pix_mp.height = height; mFormat.fmt.pix_mp.num_planes = mNumPlanes; - mFormat.fmt.pix_mp.plane_fmt[0].sizeimage = 2 * 1024 * 1024; // this line is not required for yuv420 - test this + // For OUTPUT YUV420M, explicitly set per-plane stride and size to match converters + if (mType == V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE && mPixelFormat == V4L2_PIX_FMT_YUV420M) + { + mFormat.fmt.pix_mp.plane_fmt[0].bytesperline = width; + mFormat.fmt.pix_mp.plane_fmt[1].bytesperline = width / 2; + mFormat.fmt.pix_mp.plane_fmt[2].bytesperline = width / 2; + mFormat.fmt.pix_mp.plane_fmt[0].sizeimage = width * height; + mFormat.fmt.pix_mp.plane_fmt[1].sizeimage = (width * height) / 4; + mFormat.fmt.pix_mp.plane_fmt[2].sizeimage = (width * height) / 4; + } auto ret = v4l2_ioctl(mFD, VIDIOC_S_FMT, &mFormat); if (ret) diff --git a/base/src/DMAFDToHostCopy.cpp b/base/src/DMAFDToHostCopy.cpp index afa687ab9..5ae34b361 100644 --- a/base/src/DMAFDToHostCopy.cpp +++ b/base/src/DMAFDToHostCopy.cpp @@ -103,6 +103,12 @@ bool DMAFDToHostCopy::process(frame_container &frames) auto dstPtr = static_cast(outFrame->data()); for (auto i = 0; i < mDetail->mNumPlanes; i++) { + // Validate plane data pointer before calling mCopyToData + if (mDetail->mImagePlanes[i]->data == nullptr) { + LOG_ERROR << "Plane " << i << " data pointer is null in DMAFDToHostCopy! Cannot copy data."; + return false; + } + mDetail->mImagePlanes[i]->mCopyToData(mDetail->mImagePlanes[i].get(), dstPtr); dstPtr += mDetail->mImagePlanes[i]->imageSize; } diff --git a/base/src/DMAFDWrapper.cpp b/base/src/DMAFDWrapper.cpp index 339b9b57d..1a7f3c4c9 100644 --- a/base/src/DMAFDWrapper.cpp +++ b/base/src/DMAFDWrapper.cpp @@ -1,14 +1,15 @@ #include "DMAFDWrapper.h" #include "DMAUtils.h" -#include "nvbuf_utils.h" +#include "nvbufsurface.h" #include "Logger.h" #include "AIPExceptions.h" +#include #include "cuda_runtime.h" DMAFDWrapper *DMAFDWrapper::create(int index, int width, int height, - NvBufferColorFormat colorFormat, - NvBufferLayout layout, EGLDisplay eglDisplay) + NvBufSurfaceColorFormat colorFormat, + NvBufSurfaceLayout layout, EGLDisplay eglDisplay) { DMAFDWrapper *buffer = new DMAFDWrapper(index, eglDisplay); if (!buffer) @@ -16,90 +17,96 @@ DMAFDWrapper *DMAFDWrapper::create(int index, int width, int height, return nullptr; } - NvBufferCreateParams inputParams = {0}; + NvBufSurfaceAllocateParams inputParams = {0}; - inputParams.width = width; - inputParams.height = height; - inputParams.layout = layout; - inputParams.colorFormat = colorFormat; - inputParams.payloadType = NvBufferPayload_SurfArray; - inputParams.nvbuf_tag = NvBufferTag_CAMERA; + inputParams.params.width = width; + inputParams.params.height = height; + inputParams.params.layout = layout; + inputParams.params.colorFormat = colorFormat; + inputParams.params.memType = NVBUF_MEM_SURFACE_ARRAY; + inputParams.memtag = NvBufSurfaceTag_CAMERA; - if (NvBufferCreateEx(&buffer->m_fd, &inputParams)) + if (NvBufSurfaceAllocate(&buffer->m_surf, 1, &inputParams)) { - LOG_ERROR << "Failed NvBufferCreateEx"; + LOG_ERROR << "Failed NvBufSurfaceAllocate"; delete buffer; return nullptr; } + buffer->m_surf->numFilled = 1; + buffer->m_fd = buffer->m_surf->surfaceList[0].bufferDesc; + // Use NvBufferMemMapEx - auto res = NvBufferMemMap(buffer->m_fd, 0, NvBufferMem_Read, &(buffer->hostPtr)); + auto res = NvBufSurfaceMap(buffer->m_surf, 0, 0, NVBUF_MAP_READ_WRITE); if (res) { - LOG_ERROR << "NvBufferMemMap Error<>" << res; + LOG_ERROR << "NvBufSurfaceMap Error<>" << res; delete buffer; return nullptr; } - if (colorFormat == NvBufferColorFormat_NV12 || - colorFormat == NvBufferColorFormat_YUV420 || - colorFormat == NvBufferColorFormat_YUV444) + // JP5: Set hostPtr to mapped address for plane 0 + buffer->hostPtr = buffer->m_surf->surfaceList[0].mappedAddr.addr[0]; + + if (colorFormat == NVBUF_COLOR_FORMAT_NV12 || + colorFormat == NVBUF_COLOR_FORMAT_YUV420) { - res = NvBufferMemMap(buffer->m_fd, 1, NvBufferMem_Read, &(buffer->hostPtrU)); + res = NvBufSurfaceMap(buffer->m_surf, 0, 1, NVBUF_MAP_READ_WRITE); if (res) { - LOG_ERROR << "NvBufferMemMap Error<>" << res; + LOG_ERROR << "NvBufSurfaceMap Error<>" << res; delete buffer; return nullptr; } + + // JP5: Set hostPtrU to mapped address for plane 1 + buffer->hostPtrU = buffer->m_surf->surfaceList[0].mappedAddr.addr[1]; } - if (colorFormat == NvBufferColorFormat_YUV420 || colorFormat == NvBufferColorFormat_YUV444) + if (colorFormat == NVBUF_COLOR_FORMAT_YUV420) { - res = NvBufferMemMap(buffer->m_fd, 2, NvBufferMem_Read, &(buffer->hostPtrV)); + res = NvBufSurfaceMap(buffer->m_surf, 0, 2, NVBUF_MAP_READ_WRITE); if (res) { - LOG_ERROR << "NvBufferMemMap Error<>" << res; + LOG_ERROR << "NvBufSurfaceMap Error<>" << res; delete buffer; return nullptr; } + + // JP5: Set hostPtrV to mapped address for plane 2 + buffer->hostPtrV = buffer->m_surf->surfaceList[0].mappedAddr.addr[2]; } - // if(colorFormat == NvBufferColorFormat_YUV444 ) - // { - // res = NvBufferMemMap(buffer->m_fd, 3, NvBufferMem_Read, &(buffer->hostPtrV)); - // if (res) - // { - // LOG_ERROR << "NvBufferMemMap Error<>" << res; - // delete buffer; - // return nullptr; - // } - // } - - if (colorFormat != NvBufferColorFormat_UYVY) + // Map NvBufSurface to EGLImage for JP6.2 CUDA interop + NvBufSurface *surf = buffer->m_surf; + if (NvBufSurfaceMapEglImage(surf, 0) != 0) { - buffer->eglImage = NvEGLImageFromFd(eglDisplay, buffer->m_fd); - if (buffer->eglImage == EGL_NO_IMAGE_KHR) - { - LOG_ERROR << "Failed to create eglImage"; - EGLint error = eglGetError(); - std::stringstream errorMsg; - errorMsg << "Failed to create eglImage. EGL Error: " << std::hex << error; - LOG_ERROR << errorMsg.str(); - LOG_ERROR << "Buffer details - fd: " << buffer->m_fd; - delete buffer; - return nullptr; - } - - cudaFree(0); - buffer->cudaPtr = DMAUtils::getCudaPtr(buffer->eglImage, &buffer->pResource, buffer->eglFrame, eglDisplay); + LOG_ERROR << "NvBufSurfaceMapEglImage failed"; + delete buffer; + return nullptr; } - + buffer->eglImage = surf->surfaceList[0].mappedAddr.eglImage; + LOG_INFO << "Mapped EGL image from NvBufSurface. FD: " << buffer->m_fd + << " EGLImage: " << buffer->eglImage; + + cudaFree(0); + buffer->cudaPtr = DMAUtils::getCudaPtr(buffer->eglImage, &buffer->pResource, &buffer->eglFrame); + + if (buffer->cudaPtr == nullptr) + { + LOG_ERROR << "Failed to get CUDA pointer from EGL image"; + delete buffer; + return nullptr; + } + + LOG_INFO << "Successfully created CUDA pointer: " << (void*)buffer->cudaPtr; + return buffer; } DMAFDWrapper::DMAFDWrapper(int _index, EGLDisplay _eglDisplay) : eglImage(EGL_NO_IMAGE_KHR), m_fd(-1), + m_surf(nullptr), index(_index), eglDisplay(_eglDisplay), hostPtr(nullptr), @@ -113,50 +120,53 @@ DMAFDWrapper::~DMAFDWrapper() { if (eglImage != EGL_NO_IMAGE_KHR) { - cudaFree(0); - DMAUtils::freeCudaPtr(eglImage, &pResource, eglDisplay); + if (m_surf) + { + auto res_unmap_egl = NvBufSurfaceUnMapEglImage(m_surf, 0); + if (res_unmap_egl) + { + LOG_ERROR << "NvBufSurfaceUnMapEglImage Error: " << res_unmap_egl; + } + } } if (hostPtr) { - auto res = NvBufferMemUnMap(m_fd, 0, &hostPtr); + auto res = NvBufSurfaceUnMap(getNvBufSurface(), 0, 0); if (res) { - LOG_ERROR << "NvBufferMemUnMap Error<>" << res; + LOG_ERROR << "NvBufSurfaceUnMap Error<>" << res; } } if (hostPtrU) { - auto res = NvBufferMemUnMap(m_fd, 1, &hostPtrU); + auto res = NvBufSurfaceUnMap(getNvBufSurface(), 0, 1); if (res) { - LOG_ERROR << "NvBufferMemUnMap Error<>" << res; + LOG_ERROR << "NvBufSurfaceUnMap Error<>" << res; } } if (hostPtrV) { - auto res = NvBufferMemUnMap(m_fd, 2, &hostPtrV); + auto res = NvBufSurfaceUnMap(getNvBufSurface(), 0, 2); if (res) { - LOG_ERROR << "NvBufferMemUnMap Error<>" << res; + LOG_ERROR << "NvBufSurfaceUnMap Error<>" << res; } } - if (m_fd >= 0) + if (m_surf) { - NvBufferDestroy(m_fd); + NvBufSurfaceDestroy(m_surf); + m_surf = nullptr; m_fd = -1; } } void *DMAFDWrapper::getHostPtr() { - if (NvBufferMemSyncForCpu(m_fd, 0, &hostPtr)) - { - throw AIPException(AIP_FATAL, "NvBufferMemSyncForCpu FAILED."); - } return hostPtr; } @@ -168,20 +178,12 @@ void *DMAFDWrapper::getHostPtrY() void *DMAFDWrapper::getHostPtrU() { - if (NvBufferMemSyncForCpu(m_fd, 1, &hostPtrU)) - { - throw AIPException(AIP_FATAL, "NvBufferMemSyncForCpu FAILED."); - } return hostPtrU; } void *DMAFDWrapper::getHostPtrV() { - if (NvBufferMemSyncForCpu(m_fd, 2, &hostPtrV)) - { - throw AIPException(AIP_FATAL, "NvBufferMemSyncForCpu FAILED."); - } return hostPtrV; } diff --git a/base/src/DMAUtils.cpp b/base/src/DMAUtils.cpp index b740d700c..5903ed71f 100644 --- a/base/src/DMAUtils.cpp +++ b/base/src/DMAUtils.cpp @@ -1,60 +1,92 @@ #include "DMAUtils.h" #include "Logger.h" +#include +#include +#include -uint8_t* DMAUtils::getCudaPtrForFD(int fd, EGLImageKHR eglImage, CUgraphicsResource *pResource, CUeglFrame eglFrame, EGLDisplay eglDisplay){ - eglImage = NvEGLImageFromFd(eglDisplay, fd); - if (eglImage == NULL) +// Forward declaration of NvBufSurface to avoid header dependencies if possible +//struct _NvBufSurface; +//typedef struct _NvBufSurface NvBufSurface; + +// This function is the equivalent of the old getCudaPtr. +// It directly uses the EGLImageKHR, which is created externally now. + +uint8_t* DMAUtils::getCudaPtr(EGLImageKHR eglImage, CUgraphicsResource *pResource, CUeglFrame *pEglFrame) +{ + // Ensure CUDA driver is initialized and a context is current + cuInit(0); + CUcontext current = nullptr; + cuCtxGetCurrent(¤t); + if (current == nullptr) { - LOG_ERROR << "DID not find eglImage for File Descriptor"; - return nullptr; + CUdevice dev = 0; + if (cuDeviceGet(&dev, 0) != CUDA_SUCCESS) + { + LOG_ERROR << "cuDeviceGet failed"; + return NULL; + } + CUcontext created = nullptr; + if (cuCtxCreate(&created, 0, dev) != CUDA_SUCCESS) + { + LOG_ERROR << "cuCtxCreate failed"; + return NULL; + } } - return getCudaPtr(eglImage, pResource, eglFrame, eglDisplay); -} -uint8_t* DMAUtils::getCudaPtr(EGLImageKHR eglImage, CUgraphicsResource *pResource, CUeglFrame eglFrame, EGLDisplay eglDisplay) -{ + // Register the EGL image as a CUDA graphics resource auto status = cuGraphicsEGLRegisterImage(pResource, eglImage, CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE); if (status != CUDA_SUCCESS) { LOG_ERROR << "cuGraphicsEGLRegisterImage failed: " << status << " cuda process stop"; return NULL; } - - status = cuGraphicsResourceGetMappedEglFrame(&eglFrame, *pResource, 0, 0); + // Get the mapped CUeglFrame from the resource + status = cuGraphicsResourceGetMappedEglFrame(pEglFrame, *pResource, 0, 0); if (status != CUDA_SUCCESS) { - LOG_ERROR << "cuGraphicsSubResourceGetMappedArray failed status<" << status << ">"; + LOG_ERROR << "cuGraphicsResourceGetMappedEglFrame failed status<" << status << ">"; return NULL; } - // stread sync + status = cuCtxSynchronize(); if (status != CUDA_SUCCESS) { LOG_ERROR << "cuCtxSynchronize failed status<" << status << ">"; return NULL; } - - return static_cast(eglFrame.frame.pPitch[0]); + + uint8_t* cudaPtr = static_cast(pEglFrame->frame.pPitch[0]); + + return cudaPtr; } -void DMAUtils::freeCudaPtr(EGLImageKHR eglImage, CUgraphicsResource *pResource, EGLDisplay eglDisplay) +// Updated freeCudaPtr function to handle cleanup for JetPack 6 +void DMAUtils::freeCudaPtr(EGLImageKHR eglImage, CUgraphicsResource *pResource, NvBufSurface *surf, EGLDisplay eglDisplay) { auto status = cuCtxSynchronize(); if (status != CUDA_SUCCESS) { LOG_ERROR << "cuCtxSynchronize failed after cc status<" << status << ">"; - return; + // Continue cleanup even if sync fails } status = cuGraphicsUnregisterResource(*pResource); if (status != CUDA_SUCCESS) { - LOG_ERROR << "cuGraphicsEGLUnRegisterResource failed: " << status; - return; + LOG_ERROR << "cuGraphicsUnregisterResource failed: " << status; + // Continue cleanup even if unregister fails + } + + // Unmap the EGLImage + auto res_unmap_egl = NvBufSurfaceUnMapEglImage(surf, 0); + if (res_unmap_egl) + { + LOG_ERROR << "NvBufSurfaceUnMapEglImage Error: " << res_unmap_egl; } - auto res = NvDestroyEGLImage(eglDisplay, eglImage); - if (res) + // Destroy the NvBufSurface + auto res_destroy = NvBufSurfaceDestroy(surf); + if (res_destroy) { - LOG_ERROR << "NvDestroyEGLImage Error<>" << res; + LOG_ERROR << "NvBufSurfaceDestroy Error: " << res_destroy; } } \ No newline at end of file diff --git a/base/src/EglRenderer.cpp b/base/src/EglRenderer.cpp index cbfe92975..6e0cb5b8d 100644 --- a/base/src/EglRenderer.cpp +++ b/base/src/EglRenderer.cpp @@ -2,15 +2,27 @@ #include "EglRenderer.h" #include "ApraNvEglRenderer.h" #include "DMAFDWrapper.h" +#include +#include +#include +#include +#include +#include +#include class EglRenderer::Detail { - public: - Detail(uint32_t _x_offset, uint32_t _y_offset, uint32_t _width, uint32_t _height) : x_offset(_x_offset), y_offset(_y_offset), width(_width), height(_height) - { - m_isEglWindowCreated = false; - } + + Detail(uint32_t _x_offset , uint32_t _y_offset , + uint32_t _width , uint32_t _height , + const EglRendererProps::TextInfo& _text, + const EglRendererProps::ImageInfo& _image, + float _opacity ,bool _mask) + : x_offset(_x_offset), y_offset(_y_offset), + width(_width), height(_height), + text(_text), image(_image), opacity(_opacity),mask(_mask) + {} ~Detail() { @@ -26,25 +38,39 @@ class EglRenderer::Detail LOG_DEBUG << "WILL INITIALIZE NEW WINDOW"; uint32_t displayHeight, displayWidth; NvEglRenderer::getDisplayResolution(displayWidth, displayHeight); - if (height != 0 && width != 0) - { - // x_offset += (displayWidth-width)/2; - // y_offset += (displayHeight-height)/2; - LOG_DEBUG << "X_OFFSET" << x_offset << "y_offset" << y_offset; - renderer = NvEglRenderer::createEglRenderer(__TIMESTAMP__, width, height, x_offset, y_offset); - } - else + + uint32_t renderW = (width == 0) ? _width : width; + uint32_t renderH = (height == 0) ? _height : height; + + if (width == 0 || height == 0) { - x_offset += (displayWidth - _width) / 2; - y_offset += (displayHeight - _height) / 2; - LOG_DEBUG << "X_OFFSET" << x_offset << "y_offset" << y_offset; - renderer = NvEglRenderer::createEglRenderer(__TIMESTAMP__, _width, _height, x_offset, y_offset); + x_offset += (displayWidth - renderW) / 2; + y_offset += (displayHeight - renderH) / 2; } + + LOG_DEBUG << "X_OFFSET " << x_offset << " y_offset " << y_offset; + + renderer = NvEglRenderer::createEglRenderer( + __TIMESTAMP__, + renderW, renderH, x_offset, y_offset, + text.fontPath.c_str(), text.message.c_str(), + text.scale, + text.color[0], text.color[1], text.color[2], + text.fontSize, + text.position.first, text.position.second, + image.path, + image.position.first, image.position.second, + image.size.first, image.size.second, + opacity,mask, + image.opacity,text.opacity + ); + if (!renderer) { LOG_INFO << "Failed to create EGL renderer"; return false; } + m_isEglWindowCreated = true; return true; } @@ -57,23 +83,42 @@ class EglRenderer::Detail LOG_DEBUG << "Window Exist"; m_isEglWindowCreated = false; delete renderer; + renderer = nullptr; } + return true; } - bool shouldTriggerSOS() + bool shouldTriggerSOS() const { - return !renderer; + return (!m_isEglWindowCreated) || (!renderer); } + // --- Renderer and Window Info --- NvEglRenderer *renderer = nullptr; - uint32_t x_offset, y_offset, width, height; + bool m_isEglWindowCreated = false; + + // --- Geometry --- + uint32_t x_offset = 0; + uint32_t y_offset = 0; + uint32_t width = 0; + uint32_t height = 0; + + // --- Rendering Data --- + EglRendererProps::TextInfo text; + EglRendererProps::ImageInfo image; + + //Display + float opacity = 1.0f; + bool mask = false; + + std::chrono::milliseconds m_frameDelay{27}; - bool m_isEglWindowCreated; }; + EglRenderer::EglRenderer(EglRendererProps props) : Module(SINK, "EglRenderer", props) { - mDetail.reset(new Detail(props.x_offset, props.y_offset, props.width, props.height)); + mDetail.reset(new Detail(props.x_offset, props.y_offset, props.width, props.height,props.text,props.image,props.opacity,props.mask)); } EglRenderer::~EglRenderer() {} @@ -138,6 +183,10 @@ bool EglRenderer::processSOS(frame_sp &frame) auto frameType = inputMetadata->getFrameType(); int width = 0; int height = 0; + int pitch = 0; + int fourcc = 0; + int offset0 = 0, offset1 = 0, offset2 = 0; + int pitch1_dmabuf = 0, pitch2_dmabuf = 0; switch (frameType) { @@ -146,6 +195,25 @@ bool EglRenderer::processSOS(frame_sp &frame) auto metadata = FrameMetadataFactory::downcast(inputMetadata); width = metadata->getWidth(); height = metadata->getHeight(); + pitch = static_cast(metadata->getStep()); + switch (metadata->getImageType()) + { + case ImageMetadata::RGBA: + fourcc = DRM_FORMAT_ABGR8888; // Tegra commonly expects ABGR for RGBA memory + break; + case ImageMetadata::BGRA: + fourcc = DRM_FORMAT_BGRA8888; // Correct mapping for BGRA layouts + break; + case ImageMetadata::UYVY: + fourcc = DRM_FORMAT_UYVY; + break; + case ImageMetadata::YUYV: + fourcc = DRM_FORMAT_YUYV; + break; + default: + fourcc = DRM_FORMAT_RGBA8888; + break; + } } break; case FrameMetadata::FrameType::RAW_IMAGE_PLANAR: @@ -153,6 +221,28 @@ bool EglRenderer::processSOS(frame_sp &frame) auto metadata = FrameMetadataFactory::downcast(inputMetadata); width = metadata->getWidth(0); height = metadata->getHeight(0); + + // Use actual dmabuf plane pitches/offsets from producer + auto dma = static_cast(frame->data()); + auto surf = dma->getNvBufSurface(); + auto &fdParams = surf->surfaceList[0]; + + pitch = static_cast(fdParams.planeParams.pitch[0]); + pitch1_dmabuf = static_cast(fdParams.planeParams.pitch[1]); + pitch2_dmabuf = static_cast(fdParams.planeParams.pitch[2]); + + offset0 = static_cast(fdParams.planeParams.offset[0]); + offset1 = static_cast(fdParams.planeParams.offset[1]); + offset2 = static_cast(fdParams.planeParams.offset[2]); + + if (metadata->getImageType() == ImageMetadata::NV12) + { + fourcc = DRM_FORMAT_NV12; + } + else if (metadata->getImageType() == ImageMetadata::YUV420) + { + fourcc = DRM_FORMAT_YUV420; + } } break; default: @@ -160,6 +250,31 @@ bool EglRenderer::processSOS(frame_sp &frame) } mDetail->init(height, width); + if (mDetail->renderer && fourcc != 0 && pitch != 0) + { + if (frameType == FrameMetadata::FrameType::RAW_IMAGE_PLANAR && + (fourcc == DRM_FORMAT_NV12 || fourcc == DRM_FORMAT_YUV420)) + { + int pitch1 = pitch1_dmabuf; + int pitch2 = (fourcc == DRM_FORMAT_YUV420) ? pitch2_dmabuf : 0; + int numPlanes = (fourcc == DRM_FORMAT_NV12) ? 2 : 3; + mDetail->renderer->setImportParamsPlanar( + fourcc, + width, + height, + pitch, + offset0, + pitch1, + offset1, + pitch2, + offset2, + numPlanes); + } + else + { + mDetail->renderer->setImportParams(pitch, fourcc, 0, width, height); + } + } return true; } @@ -188,7 +303,7 @@ bool EglRenderer::handleCommand(Command::CommandType type, frame_sp &frame) getCommand(cmd, frame); if(!mDetail->m_isEglWindowCreated) { - mDetail->init(cmd.width, cmd.height); + mDetail->init(cmd.height, cmd.width); } return true; } diff --git a/base/src/H264DecoderV4L2Helper.cpp b/base/src/H264DecoderV4L2Helper.cpp index ee30aec7f..94ee82d66 100644 --- a/base/src/H264DecoderV4L2Helper.cpp +++ b/base/src/H264DecoderV4L2Helper.cpp @@ -31,7 +31,8 @@ * ./decode_sample elementary_h264file.264 output_raw_file.yuv **/ #include "DMAFDWrapper.h" -#include "nvbuf_utils.h" +#include "nvbufsurface.h" +#include "nvbufsurftransform.h" #include #include #include @@ -129,157 +130,154 @@ * */ -Buffer::Buffer(enum v4l2_buf_type buf_type, enum v4l2_memory memory_type, - uint32_t index) - :buf_type(buf_type), - memory_type(memory_type), - index(index) + Buffer::Buffer(enum v4l2_buf_type buf_type, enum v4l2_memory memory_type, + uint32_t index) + :buf_type(buf_type), + memory_type(memory_type), + index(index) { - uint32_t i; - - memset(planes, 0, sizeof(planes)); - - mapped = false; - n_planes = 1; - for (i = 0; i < n_planes; i++) - { - this->planes[i].fd = -1; - this->planes[i].data = NULL; - this->planes[i].bytesused = 0; - this->planes[i].mem_offset = 0; - this->planes[i].length = 0; - this->planes[i].fmt.sizeimage = 0; - } +uint32_t i; + +memset(planes, 0, sizeof(planes)); + +mapped = false; +n_planes = 1; +for (i = 0; i < n_planes; i++) +{ + this->planes[i].fd = -1; + this->planes[i].data = NULL; + this->planes[i].bytesused = 0; + this->planes[i].mem_offset = 0; + this->planes[i].length = 0; + this->planes[i].fmt.sizeimage = 0; } - +} + Buffer::Buffer(enum v4l2_buf_type buf_type, enum v4l2_memory memory_type, - uint32_t n_planes, BufferPlaneFormat * fmt, uint32_t index) - :buf_type(buf_type), - memory_type(memory_type), - index(index), - n_planes(n_planes) + uint32_t n_planes, BufferPlaneFormat * fmt, uint32_t index) + :buf_type(buf_type), + memory_type(memory_type), + index(index), + n_planes(n_planes) { - uint32_t i; - - mapped = false; - - memset(planes, 0, sizeof(planes)); - for (i = 0; i < n_planes; i++) - { - this->planes[i].fd = -1; - this->planes[i].fmt = fmt[i]; - } +uint32_t i; + +mapped = false; + +memset(planes, 0, sizeof(planes)); +for (i = 0; i < n_planes; i++) +{ + this->planes[i].fd = -1; + this->planes[i].fmt = fmt[i]; } - +} + Buffer::~Buffer() { - if (mapped) - { - unmap(); - } +if (mapped) +{ + unmap(); } - +} + int Buffer::map() { - uint32_t j; - - if (memory_type != V4L2_MEMORY_MMAP) +uint32_t j; + +if (memory_type != V4L2_MEMORY_MMAP) +{ + cout << "Buffer " << index << "already mapped" << endl; + return -1; +} + +if (mapped) +{ + cout << "Buffer " << index << "already mapped" << endl; + return 0; +} + +for (j = 0; j < n_planes; j++) +{ + if (planes[j].fd == -1) { - LOG_INFO << "Buffer " << index << "already mapped" << endl; return -1; } - - if (mapped) - { - LOG_INFO << "Buffer " << index << "already mapped" << endl; - return 0; - } - - for (j = 0; j < n_planes; j++) + + planes[j].data = (unsigned char *) mmap(NULL, + planes[j].length, + PROT_READ | PROT_WRITE, + MAP_SHARED, + planes[j].fd, + planes[j].mem_offset); + if (planes[j].data == MAP_FAILED) { - if (planes[j].fd == -1) - { - return -1; - } - - planes[j].data = (unsigned char *) mmap(NULL, - planes[j].length, - PROT_READ | PROT_WRITE, - MAP_SHARED, - planes[j].fd, - planes[j].mem_offset); - if (planes[j].data == MAP_FAILED) - { - LOG_INFO << "Could not map buffer " << index << ", plane " << j << endl; - return -1; - } - + cout << "Could not map buffer " << index << ", plane " << j << endl; + return -1; } - mapped = true; - return 0; + } - +mapped = true; +return 0; +} + void Buffer::unmap() { - if (memory_type != V4L2_MEMORY_MMAP || !mapped) - { - LOG_INFO << "Cannot Unmap Buffer " << index << - ". Only mapped MMAP buffer can be unmapped" << endl; - return; - } - - for (uint32_t j = 0; j < n_planes; j++) - { - if (planes[j].data) - { - munmap(planes[j].data, planes[j].length); - } - planes[j].data = NULL; - } - mapped = false; +if (memory_type != V4L2_MEMORY_MMAP || !mapped) +{ + cout << "Cannot Unmap Buffer " << index << + ". Only mapped MMAP buffer can be unmapped" << endl; + return; } - -int -Buffer::fill_buffer_plane_format(uint32_t *num_planes, - Buffer::BufferPlaneFormat *planefmts, - uint32_t width, uint32_t height, uint32_t raw_pixfmt) + +for (uint32_t j = 0; j < n_planes; j++) { - switch (raw_pixfmt) + if (planes[j].data) { - case V4L2_PIX_FMT_YUV420M: - *num_planes = 3; - - planefmts[0].width = width; - planefmts[1].width = width / 2; - planefmts[2].width = width / 2; - - planefmts[0].height = height; - planefmts[1].height = height / 2; - planefmts[2].height = height / 2; - - planefmts[0].bytesperpixel = 1; - planefmts[1].bytesperpixel = 1; - planefmts[2].bytesperpixel = 1; - break; - case V4L2_PIX_FMT_NV12M: - *num_planes = 2; - - planefmts[0].width = width; - planefmts[1].width = width / 2; - - planefmts[0].height = height; - planefmts[1].height = height / 2; - - planefmts[0].bytesperpixel = 1; - planefmts[1].bytesperpixel = 2; - break; - default: - LOG_INFO << "Unsupported pixel format " << raw_pixfmt << endl; - return -1; + munmap(planes[j].data, planes[j].length); } - return 0; + planes[j].data = NULL; +} +mapped = false; +} + +int Buffer::fill_buffer_plane_format(uint32_t *num_planes, Buffer::BufferPlaneFormat *planefmts, uint32_t width, uint32_t height, uint32_t raw_pixfmt) +{ +switch (raw_pixfmt) +{ + case V4L2_PIX_FMT_YUV420M: + *num_planes = 3; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + planefmts[2].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + planefmts[2].height = height / 2; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 1; + planefmts[2].bytesperpixel = 1; + break; + case V4L2_PIX_FMT_NV12M: + *num_planes = 2; + + planefmts[0].width = width; + planefmts[1].width = width / 2; + + planefmts[0].height = height; + planefmts[1].height = height / 2; + + planefmts[0].bytesperpixel = 1; + planefmts[1].bytesperpixel = 2; + break; + default: + cout << "Unsupported pixel format " << raw_pixfmt << endl; + return -1; +} +return 0; } void h264DecoderV4L2Helper::read_input_chunk_frame_sp(frame_sp inpFrame, Buffer * buffer) @@ -436,12 +434,16 @@ void h264DecoderV4L2Helper::intitliazeSpeed(int _playbackFps, float _playbackSpe struct v4l2_crop crop; int ret_val; int32_t min_cap_buffers; - NvBufferCreateParams input_params = {0}; - NvBufferCreateParams cap_params = {0}; + NvBufSurfaceAllocateParams dstParams = {{0}}; + NvBufSurface *dst_nvbuf_surf = NULL; + NvBufSurfaceAllocateParams capParams = {{0}}; + NvBufSurface *cap_nvbuf_surf = NULL; + /* Get format on capture plane set by device. ** This may change after an resolution change event. */ + format.type = ctx->cp_buf_type; ret_val = v4l2_ioctl(ctx->fd, VIDIOC_G_FMT, &format); if (ret_val) @@ -468,7 +470,19 @@ void h264DecoderV4L2Helper::intitliazeSpeed(int _playbackFps, float _playbackSpe if (ctx->dst_dma_fd != -1) { - NvBufferDestroy(ctx->dst_dma_fd); + ret_val = NvBufSurfaceFromFd((int)ctx->dst_dma_fd, + (void**)(&dst_nvbuf_surf)); + if (ret_val) { + cerr << "Failed to Get NvBufSurface from FD" << endl; + ctx->in_error = 1; + } + + ret_val = NvBufSurfaceDestroy(dst_nvbuf_surf); + if (ret_val) { + cerr << "Failed to destroy NvBufSurface" << endl; + ctx->in_error = 1; + } + ctx->dst_dma_fd = -1; } @@ -519,12 +533,20 @@ void h264DecoderV4L2Helper::intitliazeSpeed(int _playbackFps, float _playbackSpe { if (ctx->dmabuff_fd[index] != 0) { - ret_val = NvBufferDestroy(ctx->dmabuff_fd[index]); - if (ret_val) - { - LOG_INFO << "Failed to Destroy NvBuffer" << endl; + ret_val = NvBufSurfaceFromFd((int)ctx->dmabuff_fd[index], + (void**)(&cap_nvbuf_surf)); + if (ret_val) { + cerr << "Failed to Get NvBufSurface from FD" << endl; + ctx->in_error = 1; + } + + ret_val = NvBufSurfaceDestroy(cap_nvbuf_surf); + if (ret_val) { + cerr << "Failed to destroy NvBufSurface" << endl; ctx->in_error = 1; } + + ctx->dmabuff_fd[index] = 0; } } } @@ -564,12 +586,12 @@ void h264DecoderV4L2Helper::intitliazeSpeed(int _playbackFps, float _playbackSpe if (format.fmt.pix_mp.quantization == V4L2_QUANTIZATION_DEFAULT) { LOG_INFO << "Decoder colorspace ITU-R BT.601 with standard range luma (16-235)" << endl; - cap_params.colorFormat = NvBufferColorFormat_NV12; + capParams.params.colorFormat = NVBUF_COLOR_FORMAT_NV12; } else { LOG_INFO << "Decoder colorspace ITU-R BT.601 with extended range luma (0-255)" << endl; - cap_params.colorFormat = NvBufferColorFormat_NV12_ER; + capParams.params.colorFormat = NVBUF_COLOR_FORMAT_NV12_ER; } // Request number of buffers more than minimum returned by ctrl. @@ -586,18 +608,21 @@ void h264DecoderV4L2Helper::intitliazeSpeed(int _playbackFps, float _playbackSpe for (uint32_t index = 0; index < ctx->cp_num_buffers; index++) { - cap_params.width = crop.c.width; - cap_params.height = crop.c.height; - cap_params.layout = NvBufferLayout_BlockLinear; - cap_params.payloadType = NvBufferPayload_SurfArray; - cap_params.nvbuf_tag = NvBufferTag_VIDEO_DEC; - ret_val = NvBufferCreateEx(&ctx->dmabuff_fd[index], &cap_params); + capParams.params.width = crop.c.width; + capParams.params.height = crop.c.height; + capParams.params.layout = NVBUF_LAYOUT_BLOCK_LINEAR; + capParams.params.memType = NVBUF_MEM_SURFACE_ARRAY; + capParams.memtag = NvBufSurfaceTag_VIDEO_DEC; + + ret_val = NvBufSurfaceAllocate(&cap_nvbuf_surf, 1, &capParams); if (ret_val) { - LOG_INFO << "Failed to create buffers" << endl; + cerr << "Creation of dmabuf failed" << endl; ctx->in_error = 1; break; } + cap_nvbuf_surf->numFilled = 1; + ctx->dmabuff_fd[index] = cap_nvbuf_surf->surfaceList[0].bufferDesc; } // Request buffers on capture plane. @@ -749,13 +774,14 @@ void * h264DecoderV4L2Helper::capture_thread(void *arg) break; } + LOG_INFO << "Transform check: display_width=" << ctx->display_width; if (ctx->display_width != 0) { /* Transformation parameters are defined ** which are passed to the NvBufferTransform ** for required conversion. */ - NvBufferRect src_rect, dest_rect; + NvBufSurfTransformRect src_rect, dest_rect; src_rect.top = 0; src_rect.left = 0; src_rect.width = ctx->display_width; @@ -765,18 +791,18 @@ void * h264DecoderV4L2Helper::capture_thread(void *arg) dest_rect.width = ctx->display_width; dest_rect.height = ctx->display_height; - NvBufferTransformParams transform_params; + NvBufSurfTransformParams transform_params; memset(&transform_params,0,sizeof (transform_params)); /* @transform_flag defines the flags for enabling the ** valid transforms. All the valid parameters are ** present in the nvbuf_utils header. */ - transform_params.transform_flag = NVBUFFER_TRANSFORM_FILTER; - transform_params.transform_flip = NvBufferTransform_None; - transform_params.transform_filter = NvBufferTransform_Filter_Smart; - transform_params.src_rect = src_rect; - transform_params.dst_rect = dest_rect; + transform_params.transform_flag = NVBUFSURF_TRANSFORM_FILTER; + transform_params.transform_flip = NvBufSurfTransform_None; + transform_params.transform_filter = NvBufSurfTransformInter_Nearest; + transform_params.src_rect = &src_rect; + transform_params.dst_rect = &dest_rect; // Written for NV12. if (ctx->cp_mem_type == V4L2_MEMORY_DMABUF) @@ -792,8 +818,39 @@ void * h264DecoderV4L2Helper::capture_thread(void *arg) auto dmaOutFrame = static_cast(outputFrame->data()); int f_d = dmaOutFrame->getFd(); +// JP5: Proper transform implementation following official sample + NvBufSurface* src_nvbuf_surf = nullptr; + NvBufSurface* dst_nvbuf_surf = nullptr; + + // Get source buffer from decoder + ret_val = NvBufSurfaceFromFd(ctx->dmabuff_fd[v4l2_buf.index], (void**)(&src_nvbuf_surf)); + if (ret_val != 0) { + LOG_ERROR << "Failed to get source NvBufSurface from FD " << ctx->dmabuff_fd[v4l2_buf.index] << ": " << ret_val; + ctx->in_error = 1; + break; + } + + // Get destination buffer from output frame + dst_nvbuf_surf = dmaOutFrame->getNvBufSurface(); + if (!dst_nvbuf_surf) { + LOG_ERROR << "Failed to get destination NvBufSurface"; + ctx->in_error = 1; + break; + } + + // Ensure destination is properly configured for pitch layout + dst_nvbuf_surf->surfaceList[0].layout = NVBUF_LAYOUT_PITCH; + + // Perform the transform + ret_val = NvBufSurfTransform(src_nvbuf_surf, dst_nvbuf_surf, &transform_params); + LOG_INFO << "NvBufSurfTransform result: " << ret_val << " (0=success)"; - ret_val = NvBufferTransform(decoded_buffer->planes[0].fd,f_d, &transform_params); + if (ret_val != 0) { + LOG_ERROR << "NvBufSurfTransform failed with error: " << ret_val; + // Try to get more details about the error + LOG_ERROR << "Source format: " << src_nvbuf_surf->surfaceList[0].colorFormat << " layout: " << src_nvbuf_surf->surfaceList[0].layout; + LOG_ERROR << "Dest format: " << dst_nvbuf_surf->surfaceList[0].colorFormat << " layout: " << dst_nvbuf_surf->surfaceList[0].layout; + } if (ret_val == -1) { ctx->in_error = 1; @@ -1465,6 +1522,27 @@ h264DecoderV4L2Helper::~h264DecoderV4L2Helper() { } +bool h264DecoderV4L2Helper::flush_frames() +{ + // Reset any local scheduling state + framesToSkip = 0; + iFramesToSkip = 0; + + // Clear any queued timestamps awaiting delivery + while (!framesTimestampEntry.empty()) + { + framesTimestampEntry.pop(); + } + + // Flush downstream pipeline queue if provided + if (flushPipelineQueue) + { + flushPipelineQueue(); + } + + return true; +} + bool h264DecoderV4L2Helper::term_helper() { LOG_DEBUG << "Terminating Helper WITH FD " << ctx.fd; @@ -1483,7 +1561,7 @@ bool h264DecoderV4L2Helper::term_helper() { for (uint32_t idx = 0; idx < ctx.cp_num_buffers; ++idx) { - ret = NvBufferDestroy(ctx.dmabuff_fd[idx]); + // TODO: Fix cleanup - ret = NvBufSurfaceDestroy(ctx.dmabuff_fd[...]); if (ret) { LOG_DEBUG << "Failed to Destroy Buffers" << endl; @@ -1518,7 +1596,7 @@ bool h264DecoderV4L2Helper::term_helper() { if (ctx.dmabuff_fd[i] != 0) { - ret = NvBufferDestroy(ctx.dmabuff_fd[i]); + // TODO: Fix cleanup - ret = NvBufSurfaceDestroy(ctx.dmabuff_fd[...]); ctx.dmabuff_fd[i] = 0; if (ret < 0) { @@ -1529,7 +1607,7 @@ bool h264DecoderV4L2Helper::term_helper() } if (ctx.dst_dma_fd != -1) { - NvBufferDestroy(ctx.dst_dma_fd); + // TODO: Fix cleanup - NvBufSurfaceDestroy(ctx.dst_dma_fd); ctx.dst_dma_fd = -1; } @@ -1557,55 +1635,5 @@ bool h264DecoderV4L2Helper::term_helper() LOG_DEBUG << "Decoder Run is successful" << endl; } - return true; - -} - -bool h264DecoderV4L2Helper::flush_frames() -{ - LOG_DEBUG << "Flushing all frames from decoder"; - - // Signal end of stream - struct v4l2_buffer v4l2_buf; - struct v4l2_plane planes[MAX_PLANES]; - Buffer *buffer; - - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, sizeof(planes)); - v4l2_buf.m.planes = planes; - - // Queue empty buffer to signal EOS - buffer = ctx.op_buffers[0]; - buffer->planes[0].bytesused = 0; - - int ret = q_buffer(&ctx, v4l2_buf, buffer, - ctx.op_buf_type, ctx.op_mem_type, ctx.op_num_planes); - if (ret) { - LOG_DEBUG << "Error queueing EOS buffer"; - ctx.in_error = 1; - return false; - } - - // Wait for all queued buffers to be processed - while (ctx.num_queued_op_buffers > 0 && !ctx.in_error) { - memset(&v4l2_buf, 0, sizeof(v4l2_buf)); - memset(planes, 0, sizeof(planes)); - v4l2_buf.m.planes = planes; - - ret = dq_buffer(&ctx, v4l2_buf, NULL, ctx.op_buf_type, ctx.op_mem_type, -1); - if (ret) { - LOG_DEBUG << "Error dequeuing buffer during flush"; - ctx.in_error = 1; - return false; - } - } - - // // Signal capture thread to stop - // ctx.eos = true; - // ctx.got_eos = 1; - - // Let capture thread process remaining frames - usleep(1000); - return true; } \ No newline at end of file diff --git a/base/src/H264DecoderV4L2Helper.h b/base/src/H264DecoderV4L2Helper.h index 3df2600cb..3c93747ed 100644 --- a/base/src/H264DecoderV4L2Helper.h +++ b/base/src/H264DecoderV4L2Helper.h @@ -29,7 +29,7 @@ /** * Specifies the decoder device node. */ -#define DECODER_DEV "/dev/nvhost-nvdec" +#define DECODER_DEV "/dev/v4l2-nvdec" #define MAX_BUFFERS 32 #define CHUNK_SIZE 4000000 /** diff --git a/base/src/H264EncoderV4L2.cpp b/base/src/H264EncoderV4L2.cpp index 8cb3f14f6..64950970f 100644 --- a/base/src/H264EncoderV4L2.cpp +++ b/base/src/H264EncoderV4L2.cpp @@ -91,7 +91,11 @@ bool H264EncoderV4L2::term() bool H264EncoderV4L2::process(frame_container &frames) { auto frame = frames.cbegin()->second; - // LOG_DEBUG << "Got Frames of Size " << frames.size() << " in H264EncoderV4L2"; + if (!frame->data() || frame->size() == 0) + { + LOG_ERROR << "H264EncoderV4L2 received empty/null input frame"; + return true; + } mHelper->process(frame); return true; diff --git a/base/src/H264EncoderV4L2Helper.cpp b/base/src/H264EncoderV4L2Helper.cpp index 0bb543b5f..b01f422f1 100644 --- a/base/src/H264EncoderV4L2Helper.cpp +++ b/base/src/H264EncoderV4L2Helper.cpp @@ -111,10 +111,10 @@ void H264EncoderV4L2Helper::termV4L2() void H264EncoderV4L2Helper::initV4L2() { - mFD = v4l2_open("/dev/nvhost-msenc", O_RDWR); + mFD = v4l2_open("/dev/v4l2-nvenc", O_RDWR); if (mFD == -1) { - throw AIPException(AIP_FATAL, "Could not open device nvhost-msenc"); + throw AIPException(AIP_FATAL, "Could not open device v4l2-nvenc"); } struct v4l2_capability caps; @@ -299,6 +299,7 @@ bool H264EncoderV4L2Helper::process(frame_sp& frame) LOG_DEBUG << "Got Free Buffer in process"; mConverter->process(frame, buffer); mOutputPlane->qBuffer(buffer->getIndex()); + return true; } bool H264EncoderV4L2Helper::processEOS() @@ -312,5 +313,6 @@ bool H264EncoderV4L2Helper::processEOS() mOutputPlane->setEOSFlag(buffer); mOutputPlane->qBuffer(buffer->getIndex()); - mCapturePlane->waitForDQThread(2000); // blocking call - waits for 2 secs for thread to exit + mCapturePlane->waitForDQThread(2000); + return true; } \ No newline at end of file diff --git a/base/src/JPEGEncoderL4TMHelper.cpp b/base/src/JPEGEncoderL4TMHelper.cpp index 3de560be7..65703d8cd 100755 --- a/base/src/JPEGEncoderL4TMHelper.cpp +++ b/base/src/JPEGEncoderL4TMHelper.cpp @@ -80,12 +80,7 @@ bool JPEGEncoderL4TMHelper::init(uint32_t width, uint32_t height, uint32_t _stri cinfo.input_components = 3; // YUV RGB cinfo.in_color_space = color_space; - if (scale != 1) - { - cinfo.image_scale = TRUE; - cinfo.scaled_image_width = width * scale; - cinfo.scaled_image_height = height * scale; - } +// Scaling functionality removed - not available in standard libjpeg // if (scale != 1) { ... } jpeg_set_defaults(&cinfo); jpeg_set_quality(&cinfo, quality, TRUE); @@ -177,7 +172,6 @@ int JPEGEncoderL4TMHelper::encode(const unsigned char *in_buf, unsigned char **o jpeg_write_scanlines(&cinfo, &row, 1); } } - jpeg_finish_compress(&cinfo); diff --git a/base/src/NvEglRenderer.cpp b/base/src/NvEglRenderer.cpp index 1c28ee222..0ac0de989 100644 --- a/base/src/NvEglRenderer.cpp +++ b/base/src/NvEglRenderer.cpp @@ -28,11 +28,43 @@ #include "ApraNvEglRenderer.h" #include "NvLogging.h" -#include "nvbuf_utils.h" +//#include "nvbuf_utils.h" #include "NvElement.h" #include #include #include "Logger.h" +#include +#include +#include FT_FREETYPE_H +#include +#include +#include +#define STB_IMAGE_IMPLEMENTATION +#include "stb_image.h" +#include +#include +#include + + +struct Character { + GLuint TextureID; // Glyph texture + int SizeX; // Width + int SizeY; // Height + int BearingX; // Offset from baseline to left/top + int BearingY; + GLuint Advance; // Offset to advance to next glyph +}; + +static std::map Characters; +static GLuint textVAO = 0, textVBO = 0; +static GLuint textShader = 0; +GLuint imageShader; +static int g_fontAscent = 0; +static int g_fontDescent = 0; +static int g_fontLineAdvance = 0; +static bool g_fontInitialized = false; +static std::string g_loadedFontPath; +static int g_loadedFontSize = 0; #define CAT_NAME "EglRenderer" @@ -51,9 +83,15 @@ PFNEGLCLIENTWAITSYNCKHRPROC NvEglRenderer::eglClientWaitSyncKHR; PFNEGLGETSYNCATTRIBKHRPROC NvEglRenderer::eglGetSyncAttribKHR; PFNGLEGLIMAGETARGETTEXTURE2DOESPROC NvEglRenderer::glEGLImageTargetTexture2DOES; +PFNGLGENVERTEXARRAYSOESPROC NvEglRenderer::glGenVertexArraysOES = nullptr; +PFNGLBINDVERTEXARRAYOESPROC NvEglRenderer::glBindVertexArrayOES = nullptr; +PFNGLDELETEVERTEXARRAYSOESPROC NvEglRenderer::glDeleteVertexArraysOES = nullptr; + using namespace std; -NvEglRenderer::NvEglRenderer(const char *name, uint32_t width, uint32_t height, uint32_t x_offset, uint32_t y_offset) // alwaysOnTOp +NvEglRenderer::NvEglRenderer(const char *name, uint32_t width, uint32_t height, uint32_t x_offset, uint32_t y_offset, const char *ttfFilePath,const char *message,float scale,float r,float g,float b,float fontsize,int textPosX, int textPosY, + string imagePath,int imagePosX,int imagePosY,uint32_t imageWidth,uint32_t imageHeight,float opacity,bool mask + ,float imageOpacity,float textOpacity) // alwaysOnTOp { int depth; int screen_num; @@ -62,6 +100,7 @@ NvEglRenderer::NvEglRenderer(const char *name, uint32_t width, uint32_t height, x_display = NULL; texture_id = 0; + cached_image_texture = 0; gc = NULL; fontinfo = NULL; @@ -156,12 +195,38 @@ NvEglRenderer::NvEglRenderer(const char *name, uint32_t width, uint32_t height, WhitePixel(x_display, DefaultScreen(x_display)) ); fontinfo = XLoadQueryFont(x_display, "9x15bold"); + this->ttfFilePath = ttfFilePath ? std::string(ttfFilePath) : std::string(); + this->message = message ? std::string(message) : std::string(); + this->scale = scale; + this->r = r; + this->g = g; + this->b = b; + this->fontSize = fontsize; + this->textPosX = textPosX; + this->textPosY = textPosY; + this->opacity = opacity; + this->imagePath = imagePath; + this->imagePosX = imagePosX; + this->imagePosY = imagePosY; + this->imageWidth = imageWidth; + this->imageHeight = imageHeight; + this->mask = mask; + this->imageOpacity = imageOpacity; + this->textOpacity = textOpacity; + + if(opacity < 1.0f) + { + setWindowOpacity(opacity); + } + else + { + setWindowOpacity(0.99f); + } pthread_mutex_lock(&render_lock); pthread_create(&render_thread, NULL, renderThread, this); pthread_setname_np(render_thread, "EglRenderer"); pthread_cond_wait(&render_cond, &render_lock); pthread_mutex_unlock(&render_lock); - return; } @@ -196,6 +261,7 @@ NvEglRenderer::renderThread(void *arg) EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_ALPHA_SIZE, 8, + EGL_STENCIL_SIZE, 8, EGL_SURFACE_TYPE, EGL_WINDOW_BIT, EGL_NONE, }; @@ -238,6 +304,10 @@ NvEglRenderer::renderThread(void *arg) eglMakeCurrent(renderer->egl_display, renderer->egl_surface, renderer->egl_surface, renderer->egl_context); + + glEnable(GL_BLEND); + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + if (eglGetError() != EGL_SUCCESS) { goto error; @@ -247,9 +317,66 @@ NvEglRenderer::renderThread(void *arg) { goto error; } + // Load OES VAO function pointers after making context current + NvEglRenderer::glGenVertexArraysOES = (PFNGLGENVERTEXARRAYSOESPROC)eglGetProcAddress("glGenVertexArraysOES"); + NvEglRenderer::glBindVertexArrayOES = (PFNGLBINDVERTEXARRAYOESPROC)eglGetProcAddress("glBindVertexArrayOES"); + NvEglRenderer::glDeleteVertexArraysOES = (PFNGLDELETEVERTEXARRAYSOESPROC)eglGetProcAddress("glDeleteVertexArraysOES"); + + textShader = renderer->initTextShader(); + // Initialize image shader used for 2D image rendering + imageShader = renderer->initImageShader(); + // setup VAO/VBO for text quads + if (glGenVertexArraysOES) + { + glGenVertexArraysOES(1, &textVAO); + } + glGenBuffers(1, &textVBO); + if (glBindVertexArrayOES) + { + glBindVertexArrayOES(textVAO); + glBindBuffer(GL_ARRAY_BUFFER, textVBO); + glBufferData(GL_ARRAY_BUFFER, sizeof(float) * 6 * 4, NULL, GL_DYNAMIC_DRAW); + glEnableVertexAttribArray(1); + glVertexAttribPointer(1, 4, GL_FLOAT, GL_FALSE, 4 * sizeof(float), 0); + glBindBuffer(GL_ARRAY_BUFFER, 0); + glBindVertexArrayOES(0); + } + else + { + // No VAO support: just allocate buffer now; we'll set attrib pointer per draw + glBindBuffer(GL_ARRAY_BUFFER, textVBO); + glBufferData(GL_ARRAY_BUFFER, sizeof(float) * 6 * 4, NULL, GL_DYNAMIC_DRAW); + glBindBuffer(GL_ARRAY_BUFFER, 0); + } renderer->create_texture(); + // Load image texture once during initialization + renderer->cached_image_texture = renderer->loadImageTexture(renderer->imagePath.c_str()); + if (renderer->cached_image_texture == 0) + { + fprintf(stderr, "Warning: Failed to load cached image texture\n"); + } + + // Initialize font atlas once if text is requested + if (!renderer->message.empty() && !renderer->ttfFilePath.empty()) + { + if (!g_fontInitialized || g_loadedFontPath != renderer->ttfFilePath || g_loadedFontSize != renderer->fontSize) + { + if (renderer->initFontAtlas(renderer->ttfFilePath.c_str(), renderer->fontSize) == 0) + { + g_fontInitialized = true; + g_loadedFontPath = renderer->ttfFilePath; + g_loadedFontSize = renderer->fontSize; + } + else + { + g_fontInitialized = false; + fprintf(stderr, "ERROR: Font atlas initialization failed for '%s'\n", renderer->ttfFilePath.c_str()); + } + } + } + pthread_mutex_lock(&renderer->render_lock); pthread_cond_broadcast(&renderer->render_cond); @@ -276,6 +403,10 @@ NvEglRenderer::renderThread(void *arg) { glDeleteTextures(1, &renderer->texture_id); } + if (renderer->cached_image_texture) + { + glDeleteTextures(1, &renderer->cached_image_texture); + } if (renderer->egl_display != EGL_NO_DISPLAY) { @@ -364,35 +495,361 @@ NvEglRenderer::render(int fd) return 0; } -int -NvEglRenderer::renderInternal() +EGLImageKHR +NvEglRenderer::createEglImageFromDmaBuf() { - EGLImageKHR hEglImage; - bool frame_is_late = false; - - EGLSyncKHR egl_sync; - int iErr; - hEglImage = NvEGLImageFromFd(egl_display, render_fd); - if (!hEglImage) + EGLAttrib attrs[32]; + int ai = 0; + attrs[ai++] = EGL_WIDTH; attrs[ai++] = (EGLint)render_width; + attrs[ai++] = EGL_HEIGHT; attrs[ai++] = (EGLint)render_height; + attrs[ai++] = EGL_LINUX_DRM_FOURCC_EXT; attrs[ai++] = (EGLint)render_fourcc; + // Plane 0 + attrs[ai++] = EGL_DMA_BUF_PLANE0_FD_EXT; attrs[ai++] = (EGLint)render_fd; + attrs[ai++] = EGL_DMA_BUF_PLANE0_MODIFIER_LO_EXT; attrs[ai++] = 0; + attrs[ai++] = EGL_DMA_BUF_PLANE0_MODIFIER_HI_EXT; attrs[ai++] = 0; + attrs[ai++] = EGL_DMA_BUF_PLANE0_OFFSET_EXT; attrs[ai++] = (EGLint)render_offset; + attrs[ai++] = EGL_DMA_BUF_PLANE0_PITCH_EXT; attrs[ai++] = (EGLint)render_pitch; + if (render_num_planes >= 2) { - return -1; + attrs[ai++] = EGL_DMA_BUF_PLANE1_FD_EXT; attrs[ai++] = (EGLint)render_fd; + attrs[ai++] = EGL_DMA_BUF_PLANE1_OFFSET_EXT; attrs[ai++] = (EGLint)render_offset1; + attrs[ai++] = EGL_DMA_BUF_PLANE1_PITCH_EXT; attrs[ai++] = (EGLint)render_pitch1; } - + if (render_num_planes >= 3) + { + attrs[ai++] = EGL_DMA_BUF_PLANE2_FD_EXT; attrs[ai++] = (EGLint)render_fd; + attrs[ai++] = EGL_DMA_BUF_PLANE2_OFFSET_EXT; attrs[ai++] = (EGLint)render_offset2; + attrs[ai++] = EGL_DMA_BUF_PLANE2_PITCH_EXT; attrs[ai++] = (EGLint)render_pitch2; + } + attrs[ai++] = EGL_NONE; + + return eglCreateImage( + egl_display, + EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + (EGLClientBuffer)NULL, + attrs + ); +} +int NvEglRenderer::renderVideoFrame(EGLImageKHR hEglImage) +{ glActiveTexture(GL_TEXTURE0); glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture_id); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_EXTERNAL_OES, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); glEGLImageTargetTexture2DOES(GL_TEXTURE_EXTERNAL_OES, hEglImage); + + glUseProgram(gl_program); + + // --- Circle Mask Control --- + GLint enableLoc = glGetUniformLocation(gl_program, "u_enableMask"); + GLint centerLoc = glGetUniformLocation(gl_program, "u_center"); + GLint radiusLoc = glGetUniformLocation(gl_program, "u_radius"); + + if (enableLoc >= 0) glUniform1i(enableLoc, mask ? 1 : 0); // enableMask is a bool member variable + if (centerLoc >= 0) glUniform2f(centerLoc, 0.5f, 0.5f); // center at middle of frame + if (radiusLoc >= 0) glUniform1f(radiusLoc, 0.45f); // circle radius (slightly less than half) + glDrawArrays(GL_TRIANGLES, 0, 6); - iErr = glGetError(); + EGLBoolean egl_status = eglGetError(); + if (egl_status != EGL_SUCCESS) + return -1; + + int iErr = glGetError(); if (iErr != GL_NO_ERROR) + return -1; + + return 0; +} + + +void +NvEglRenderer::saveGLState(GLint& prevProgram, GLint& prevVAO, GLint& prevTexExternal, + GLint& prevTex2D, GLint& prevArrayBuffer, GLint& prevActiveTexUnit, + GLboolean& wasBlendEnabled) +{ + wasBlendEnabled = glIsEnabled(GL_BLEND); + glGetIntegerv(GL_CURRENT_PROGRAM, &prevProgram); + if (glGetError() == GL_NO_ERROR && glBindVertexArrayOES) + { + glGetIntegerv(GL_VERTEX_ARRAY_BINDING_OES, &prevVAO); + } + glGetIntegerv(GL_ACTIVE_TEXTURE, &prevActiveTexUnit); + glGetIntegerv(GL_TEXTURE_BINDING_EXTERNAL_OES, &prevTexExternal); + glGetIntegerv(GL_TEXTURE_BINDING_2D, &prevTex2D); + glGetIntegerv(GL_ARRAY_BUFFER_BINDING, &prevArrayBuffer); +} + +void +NvEglRenderer::restoreGLState(GLint prevProgram, GLint prevVAO, GLint prevTexExternal, + GLint prevTex2D, GLint prevArrayBuffer, GLint prevActiveTexUnit, + GLboolean wasBlendEnabled) +{ + glUseProgram(prevProgram); + if (glBindVertexArrayOES && prevVAO) + { + glBindVertexArrayOES(prevVAO); + } + glActiveTexture(prevActiveTexUnit); + glBindTexture(GL_TEXTURE_2D, prevTex2D); + glBindTexture(GL_TEXTURE_EXTERNAL_OES, prevTexExternal); + glBindBuffer(GL_ARRAY_BUFFER, prevArrayBuffer); + + if (!wasBlendEnabled) + { + glDisable(GL_BLEND); + } + + // If VAO is unavailable, restore video attribute pointer for 'in_pos' + if (!glBindVertexArrayOES) + { + GLint inPosLoc = glGetAttribLocation(prevProgram, "in_pos"); + if (inPosLoc >= 0) + { + glVertexAttribPointer(inPosLoc, 4, GL_FLOAT, GL_FALSE, 0, (void*)0); + glEnableVertexAttribArray(inPosLoc); + } + } +} +void +NvEglRenderer::renderOverlays() +{ + // Ensure depth/stencil won't block overlays and enable blending + glDisable(GL_DEPTH_TEST); + glDisable(GL_STENCIL_TEST); + glEnable(GL_BLEND); + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + + // Save GL state before rendering overlays + GLint prevProgram; + GLint prevVAO = 0; + GLint prevTexExternal; + GLint prevTex2D; + GLint prevArrayBuffer; + GLint prevActiveTexUnit; + GLboolean wasBlendEnabled; + saveGLState(prevProgram, prevVAO, prevTexExternal, prevTex2D, prevArrayBuffer, + prevActiveTexUnit, wasBlendEnabled); + + GLint viewport[4] = {0, 0, 0, 0}; + glGetIntegerv(GL_VIEWPORT, viewport); + int viewportWidth = viewport[2]; + int viewportHeight = viewport[3]; + if (viewportWidth <= 0 && render_width > 0) + { + viewportWidth = static_cast(render_width); + } + if (viewportHeight <= 0 && render_height > 0) + { + viewportHeight = static_cast(render_height); + } + float orthoWidth = viewportWidth > 0 ? static_cast(viewportWidth) + : (render_width > 0 ? render_width : 1.0f); + float orthoHeight = viewportHeight > 0 ? static_cast(viewportHeight) + : (render_height > 0 ? render_height : 1.0f); + + // Render image overlay if available + if (cached_image_texture != 0) + { + glUseProgram(imageShader); + GLint projLoc = glGetUniformLocation(imageShader, "projection"); + if (projLoc != -1) + { + glm::mat4 projection = glm::ortho(0.0f, orthoWidth, orthoHeight, 0.0f); + glUniformMatrix4fv(projLoc, 1, GL_FALSE, glm::value_ptr(projection)); + } + RenderImage(cached_image_texture, imagePosX, imagePosY, imageWidth, imageHeight); + } + + // Render text overlay if message is not empty + if (!message.empty() && !ttfFilePath.empty()) + { + if (!g_fontInitialized) + { + // Attempt lazy init if not already initialized + if (initFontAtlas(ttfFilePath.c_str(), fontSize) == 0) + { + g_fontInitialized = true; + g_loadedFontPath = ttfFilePath; + g_loadedFontSize = fontSize; + } + else + { + fprintf(stderr, "ERROR: Skipping text render because font atlas is not initialized for '%s'\n", ttfFilePath.c_str()); + // Skip text rendering if font couldn't be initialized + goto restore_state; + } + } + // Text projection matrix (normalized device coordinates) + GLfloat textProjection[16] = { + 2.0f / orthoWidth, 0.0f, 0.0f, 0.0f, + 0.0f, -2.0f / orthoHeight, 0.0f, 0.0f, + 0.0f, 0.0f, -1.0f, 0.0f, + -1.0f, 1.0f, 0.0f, 1.0f + }; + + glUseProgram(textShader); + GLint projLoc = glGetUniformLocation(textShader, "projection"); + if (projLoc != -1) + glUniformMatrix4fv(projLoc, 1, GL_FALSE, textProjection); + + // Compute multiline-aware text dimensions + int maxLineW = 0; + int currentLineW = 0; + int numLines = 1; + for (auto it = message.begin(); it != message.end(); ++it) + { + if (*it == '\n') + { + if (currentLineW > maxLineW) maxLineW = currentLineW; + currentLineW = 0; + numLines++; + continue; + } + auto cit = Characters.find(*it); + if (cit == Characters.end()) + { + // Approximate advance for missing glyphs (e.g., space) + continue; + } + const Character& ch = cit->second; + currentLineW += static_cast(((int)ch.Advance >> 6) * scale); + } + if (currentLineW > maxLineW) maxLineW = currentLineW; + int textW = maxLineW > 0 ? maxLineW : 1; + int textH = (g_fontLineAdvance > 0 ? g_fontLineAdvance : 16) * numLines * scale; + if (textH < 1) textH = 1; + + // Save previous scissor state + GLboolean prevScissorEnabled = glIsEnabled(GL_SCISSOR_TEST); + GLint prevScissorBox[4]; + glGetIntegerv(GL_SCISSOR_BOX, prevScissorBox); + + // Determine if stencil buffer is available before using it for clipping + GLint stencilBits = 0; + glGetIntegerv(GL_STENCIL_BITS, &stencilBits); + const bool useStencil = stencilBits > 0; + + if (useStencil) + { + // Prepare stencil: 0 everywhere, 1 inside text rectangle + glEnable(GL_STENCIL_TEST); + glStencilMask(0xFF); + glDisable(GL_SCISSOR_TEST); + glClearStencil(0); + glClear(GL_STENCIL_BUFFER_BIT); + } + + // Compute scissor box for top-left coordinates to GL bottom-left origin + int ascentPixels = g_fontAscent > 0 ? static_cast(g_fontAscent * scale) : static_cast(16 * scale); + int adjustedtextPosY = static_cast(textPosY) - ascentPixels; + int adjustedTextH = textH + ascentPixels; + if (adjustedtextPosY < 0) + { + adjustedTextH += adjustedtextPosY; + adjustedtextPosY = 0; + } + + GLint scX = static_cast(textPosX); + GLint scY = static_cast(viewportHeight - adjustedtextPosY - adjustedTextH); + if (scY < 0) scY = 0; + if (scX < 0) scX = 0; + GLsizei scW = static_cast(textW); + GLsizei scH = static_cast(adjustedTextH); + if (viewportWidth > 0) + { + if (scX + scW > viewportWidth) + { + int excess = (scX + scW) - viewportWidth; + scW = static_cast(std::max(1, static_cast(scW) - excess)); + } + } + if (viewportHeight > 0) + { + if (scY + scH > viewportHeight) + { + int excess = (scY + scH) - viewportHeight; + scH = static_cast(std::max(1, static_cast(scH) - excess)); + } + } + + // Use scissor only to clip text rendering (no stencil) + glEnable(GL_SCISSOR_TEST); + glScissor(scX, scY, scW, scH); + if (useStencil) + { + glClearStencil(1); + glClear(GL_STENCIL_BUFFER_BIT); + + // Only pass where stencil == 1 + glStencilFunc(GL_EQUAL, 1, 0xFF); + glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP); + } + else + { + glDisable(GL_STENCIL_TEST); + } + + // For non-VAO path, set attrib pointer for text + if (!glBindVertexArrayOES) + { + glBindBuffer(GL_ARRAY_BUFFER, textVBO); + glEnableVertexAttribArray(1); + glVertexAttribPointer(1, 4, GL_FLOAT, GL_FALSE, 4 * sizeof(float), 0); + } + + RenderText(this->message, this->textPosX, this->textPosY, this->scale, this->r, this->g, this->b); + // Disable stencil and restore scissor state + if (useStencil) + { + glDisable(GL_STENCIL_TEST); + } + if (prevScissorEnabled) + { + glEnable(GL_SCISSOR_TEST); + glScissor(prevScissorBox[0], prevScissorBox[1], prevScissorBox[2], prevScissorBox[3]); + } + else + { + glDisable(GL_SCISSOR_TEST); + } + } + +restore_state: + // Restore GL state + restoreGLState(prevProgram, prevVAO, prevTexExternal, prevTex2D, prevArrayBuffer, + prevActiveTexUnit, wasBlendEnabled); +} + +int +NvEglRenderer::renderInternal() +{ + // Create EGL image from dmabuf + EGLImageKHR hEglImage = createEglImageFromDmaBuf(); + if (!hEglImage) { return -1; } - egl_sync = eglCreateSyncKHR(egl_display, EGL_SYNC_FENCE_KHR, NULL); + + // Render video frame + if (renderVideoFrame(hEglImage) < 0) + { + eglDestroyImageKHR(egl_display, hEglImage); + return -1; + } + + // Create sync object for frame timing + EGLSyncKHR egl_sync = eglCreateSyncKHR(egl_display, EGL_SYNC_FENCE_KHR, NULL); if (egl_sync == EGL_NO_SYNC_KHR) { + eglDestroyImageKHR(egl_display, hEglImage); return -1; } + + // Frame timing control if (last_render_time.tv_sec != 0) { pthread_mutex_lock(&render_lock); @@ -400,39 +857,38 @@ NvEglRenderer::renderInternal() last_render_time.tv_nsec += render_time_nsec; last_render_time.tv_sec += last_render_time.tv_nsec / 1000000000UL; last_render_time.tv_nsec %= 1000000000UL; - - pthread_cond_timedwait(&render_cond, &render_lock, - &last_render_time); - + pthread_cond_timedwait(&render_cond, &render_lock, &last_render_time); pthread_mutex_unlock(&render_lock); } else { struct timeval now; - gettimeofday(&now, NULL); last_render_time.tv_sec = now.tv_sec; last_render_time.tv_nsec = now.tv_usec * 1000L; } + + // Render overlays (image and text) + renderOverlays(); + + // Swap buffers eglSwapBuffers(egl_display, egl_surface); if (eglGetError() != EGL_SUCCESS) { + eglDestroySyncKHR(egl_display, egl_sync); + eglDestroyImageKHR(egl_display, hEglImage); return -1; } - if (eglClientWaitSyncKHR (egl_display, egl_sync, - EGL_SYNC_FLUSH_COMMANDS_BIT_KHR, EGL_FOREVER_KHR) == EGL_FALSE) - { - } - if (eglDestroySyncKHR(egl_display, egl_sync) != EGL_TRUE) - { - } - NvDestroyEGLImage(egl_display, hEglImage); + // Wait for sync and cleanup + eglClientWaitSyncKHR(egl_display, egl_sync, EGL_SYNC_FLUSH_COMMANDS_BIT_KHR, EGL_FOREVER_KHR); + eglDestroySyncKHR(egl_display, egl_sync); + eglDestroyImageKHR(egl_display, hEglImage); + // Render X11 overlay text if needed if (strlen(overlay_str) != 0) { - XSetForeground(x_display, gc, - BlackPixel(x_display, DefaultScreen(x_display))); + XSetForeground(x_display, gc, BlackPixel(x_display, DefaultScreen(x_display))); XSetFont(x_display, gc, fontinfo->fid); XDrawString(x_display, x_window, gc, overlay_str_x_offset, overlay_str_y_offset, overlay_str, strlen(overlay_str)); @@ -476,10 +932,13 @@ NvEglRenderer::setFPS(float fps) NvEglRenderer * NvEglRenderer::createEglRenderer(const char *name, uint32_t width, uint32_t height, uint32_t x_offset, - uint32_t y_offset) + uint32_t y_offset, const char *ttfFilePath, + const char *message, float scale, float r, float g, float b, float fontsize,int textPosX, int textPosY, + string imagePath,int imagePosX,int imagePosY,uint32_t imageWidth,uint32_t imageHeight,float opacity,bool mask + ,float imageOpacity,float textOpacity) { NvEglRenderer* renderer = new NvEglRenderer(name, width, height, - x_offset, y_offset); + x_offset, y_offset, ttfFilePath, message, scale, r, g, b, fontsize, textPosX, textPosY, imagePath, imagePosX, imagePosY, imageWidth, imageHeight,opacity,mask,imageOpacity,textOpacity); return renderer; } @@ -577,9 +1036,22 @@ NvEglRenderer::InitializeShaders(void) static const char kFragmentShader[] = "#extension GL_OES_EGL_image_external : require\n" - "precision mediump float;\n" "varying vec2 interp_tc; \n" - "uniform samplerExternalOES tex; \n" "void main() {\n" - "gl_FragColor = texture2D(tex, interp_tc);\n" "}\n"; + "precision mediump float;\n" + "varying vec2 interp_tc;\n" + "uniform samplerExternalOES tex;\n" + "uniform int u_enableMask;\n" // 0 = off, 1 = on + "uniform vec2 u_center;\n" // normalized [0,1] coordinates + "uniform float u_radius;\n" // normalized radius (0–1)\n" + "void main() {\n" + " vec4 color = texture2D(tex, interp_tc);\n" + " if (u_enableMask == 1) {\n" + " float dist = distance(interp_tc, u_center);\n" + " if (dist > u_radius)\n" + " discard;\n" + " }\n" + " gl_FragColor = color;\n" + "}\n"; + glEnable(GL_SCISSOR_TEST); program = glCreateProgram(); @@ -608,7 +1080,7 @@ NvEglRenderer::InitializeShaders(void) return -1; } - GLuint vbo; // Store vetex and tex coords + GLuint vbo; // Store vertex and tex coords glGenBuffers(1, &vbo); glBindBuffer(GL_ARRAY_BUFFER, vbo); glBufferData(GL_ARRAY_BUFFER, sizeof(vertexTexBuf), vertexTexBuf, GL_STATIC_DRAW); @@ -620,8 +1092,18 @@ NvEglRenderer::InitializeShaders(void) glVertexAttribPointer(pos_location, 4, GL_FLOAT, GL_FALSE, 0, (void*)0); glEnableVertexAttribArray(pos_location); + glActiveTexture(GL_TEXTURE0); - glUniform1i(glGetUniformLocation(program, "texSampler"), 0); + // Bind the sampler used by the fragment shader to unit 0 + GLint texLoc = glGetUniformLocation(program, "tex"); + if (texLoc != -1) + { + glUniform1i(texLoc, 0); + } + + this->alpha_location = 1; + this->gl_program = program; + if (glGetError() != GL_NO_ERROR) { return -1; @@ -642,4 +1124,308 @@ NvEglRenderer::create_texture() glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture_id); return 0; -} \ No newline at end of file +} +void NvEglRenderer::setWindowOpacity(float opacity) +{ + if (opacity < 0.0f) opacity = 0.0f; + if (opacity > 1.0f) opacity = 1.0f; + + unsigned long opacityValue = (unsigned long)(0xFFFFFFFFul * opacity); + Atom opacityAtom = XInternAtom(x_display, "_NET_WM_WINDOW_OPACITY", False); + XChangeProperty(x_display, x_window, opacityAtom, XA_CARDINAL, 32, + PropModeReplace, (unsigned char *)&opacityValue, 1); + XFlush(x_display); +} + +int NvEglRenderer::initFontAtlas(const char* fontPath, int fontSize) +{ + FT_Library ft; + if (FT_Init_FreeType(&ft)) { + fprintf(stderr, "ERROR: Could not init FreeType Library\n"); + return -1; + } + + FT_Face face; + if (FT_New_Face(ft, fontPath, 0, &face)) { + fprintf(stderr, "ERROR: Failed to load font: %s\n", fontPath); + return -1; + } + + FT_Set_Pixel_Sizes(face, 0, fontSize); + // Cache font metrics for baseline and line advance + g_fontAscent = static_cast(face->size->metrics.ascender >> 6); + g_fontDescent = static_cast(-(face->size->metrics.descender >> 6)); + g_fontLineAdvance = static_cast(face->size->metrics.height >> 6); + if (g_fontLineAdvance == 0) + { + g_fontLineAdvance = g_fontAscent + g_fontDescent; + } + glPixelStorei(GL_UNPACK_ALIGNMENT, 1); // disable byte-alignment restriction + + for (unsigned char c = 0; c < 128; c++) { + if (FT_Load_Char(face, c, FT_LOAD_RENDER)) { + fprintf(stderr, "ERROR: Failed to load Glyph %c\n", c); + continue; + } + + GLuint tex; + glGenTextures(1, &tex); + glBindTexture(GL_TEXTURE_2D, tex); + glTexImage2D( + GL_TEXTURE_2D, + 0, + GL_ALPHA, + face->glyph->bitmap.width, + face->glyph->bitmap.rows, + 0, + GL_ALPHA, + GL_UNSIGNED_BYTE, + face->glyph->bitmap.buffer + ); + + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + + Character character = { + tex, + (int)face->glyph->bitmap.width, + (int)face->glyph->bitmap.rows, + (int)face->glyph->bitmap_left, + (int)face->glyph->bitmap_top, + (GLuint)face->glyph->advance.x + }; + Characters.insert(std::pair(c, character)); + } + + FT_Done_Face(face); + FT_Done_FreeType(ft); + return 0; +} +GLuint NvEglRenderer::initTextShader() +{ + static const char* textVertexShaderSrc = R"( + attribute vec4 vertex; + varying vec2 TexCoords; + uniform mat4 projection; + void main() { + gl_Position = projection * vec4(vertex.xy, 0.0, 1.0); + TexCoords = vertex.zw; + } + )"; + + static const char* textFragmentShaderSrc = R"( + precision mediump float; + varying vec2 TexCoords; + uniform sampler2D text; + uniform vec3 textColor; + uniform float opacity; + void main() { + float alpha = texture2D(text, TexCoords).a; + vec4 color = vec4(textColor, alpha); + color.a *= opacity; + gl_FragColor = color; + } + )"; + + GLuint vs = glCreateShader(GL_VERTEX_SHADER); + glShaderSource(vs, 1, &textVertexShaderSrc, NULL); + glCompileShader(vs); + + GLuint fs = glCreateShader(GL_FRAGMENT_SHADER); + glShaderSource(fs, 1, &textFragmentShaderSrc, NULL); + glCompileShader(fs); + + GLuint shader = glCreateProgram(); + glAttachShader(shader, vs); + glAttachShader(shader, fs); + glBindAttribLocation(shader, 1, "vertex"); + glLinkProgram(shader); + + // Bind uniforms + glUseProgram(shader); + GLint textSamplerLoc = glGetUniformLocation(shader, "text"); + GLint opacityLoc = glGetUniformLocation(shader, "opacity"); + if (textSamplerLoc != -1) + glUniform1i(textSamplerLoc, 0); + if (opacityLoc != -1) + glUniform1f(opacityLoc, textOpacity); + glUseProgram(0); + + glDeleteShader(vs); + glDeleteShader(fs); + return shader; +} + +void NvEglRenderer::RenderText(std::string text, float x, float y, float scale, float r, float g, float b) +{ + // Assumes textShader is already bound and projection uniform set by caller + GLint colorLoc = glGetUniformLocation(textShader, "textColor"); + glUniform3f(colorLoc, r, g, b); + glActiveTexture(GL_TEXTURE0); + if (glBindVertexArrayOES) + { + glBindVertexArrayOES(textVAO); + } + else + { + // No VAO: ensure attribute pointer is set to textVBO at attrib 1 + glBindBuffer(GL_ARRAY_BUFFER, textVBO); + glEnableVertexAttribArray(1); + glVertexAttribPointer(1, 4, GL_FLOAT, GL_FALSE, 4 * sizeof(float), 0); + } + + const float originX = x; + // In screen coordinates: y=0 is top, y increases downward + // Baseline is at y position (y is top of text area) + float baselineY = y; + + for (auto c = text.begin(); c != text.end(); c++) { + if (*c == '\n') + { + // Move to next line (downward in screen space) + x = originX; + baselineY += g_fontLineAdvance * scale; + continue; + } + auto it = Characters.find(*c); + if (it == Characters.end()) + { + // Skip unsupported glyphs + continue; + } + const Character& ch = it->second; + + // Compute glyph position + float xpos = x + ch.BearingX * scale; + // ypos is the top of the glyph rectangle + // BearingY is offset from baseline to top, so top = baseline - BearingY + float ypos = baselineY - (ch.BearingY * scale); + + float w = ch.SizeX * scale; + float h = ch.SizeY * scale; + + // Build vertex array (ypos = top, ypos + h = bottom) + float vertices[6][4] = { + {xpos, ypos + h, 0.0f, 1.0f}, // flip V + {xpos, ypos, 0.0f, 0.0f}, + {xpos + w, ypos, 1.0f, 0.0f}, + {xpos, ypos + h, 0.0f, 1.0f}, + {xpos + w, ypos, 1.0f, 0.0f}, + {xpos + w, ypos + h, 1.0f, 1.0f} + }; + + + glBindTexture(GL_TEXTURE_2D, ch.TextureID); + glBindBuffer(GL_ARRAY_BUFFER, textVBO); + glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(vertices), vertices); + glDrawArrays(GL_TRIANGLES, 0, 6); + + // Advance cursor + x += (ch.Advance >> 6) * scale; + } + + if (glBindVertexArrayOES) + { + glBindVertexArrayOES(0); + } + glBindTexture(GL_TEXTURE_2D, 0); +} + +GLuint NvEglRenderer::loadImageTexture(const char* imagePath) +{ + int width, height, channels; + unsigned char* data = stbi_load(imagePath, &width, &height, &channels, STBI_rgb_alpha); + if (!data) { + fprintf(stderr, "Failed to load image: %s\n", imagePath); + return 0; + } + + GLuint tex; + glGenTextures(1, &tex); + glBindTexture(GL_TEXTURE_2D, tex); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, width, height, 0, + GL_RGBA, GL_UNSIGNED_BYTE, data); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glBindTexture(GL_TEXTURE_2D, 0); + + stbi_image_free(data); + return tex; +} + +GLuint NvEglRenderer::initImageShader() +{ + static const char* vsSrc = R"( + attribute vec4 vertex; + varying vec2 TexCoords; + uniform mat4 projection; + void main() { + gl_Position = projection * vec4(vertex.xy, 0.0, 1.0); + TexCoords = vertex.zw; + })"; + + static const char* fsSrc = R"( + precision mediump float; + varying vec2 TexCoords; + uniform sampler2D imageTex; + uniform float opacity; + void main() { + vec4 color = texture2D(imageTex, TexCoords); + color.a *= opacity; + gl_FragColor = color; + })"; + + GLuint vs = glCreateShader(GL_VERTEX_SHADER); + glShaderSource(vs, 1, &vsSrc, NULL); + glCompileShader(vs); + + GLuint fs = glCreateShader(GL_FRAGMENT_SHADER); + glShaderSource(fs, 1, &fsSrc, NULL); + glCompileShader(fs); + + GLuint shader = glCreateProgram(); + glAttachShader(shader, vs); + glAttachShader(shader, fs); + glBindAttribLocation(shader, 1, "vertex"); + glLinkProgram(shader); + + glUseProgram(shader); + GLint texLoc = glGetUniformLocation(shader, "imageTex"); + GLint opacityLoc = glGetUniformLocation(shader, "opacity"); + if (texLoc != -1) glUniform1i(texLoc, 0); + if (opacityLoc != -1) glUniform1f(opacityLoc, imageOpacity); + glUseProgram(0); + + glDeleteShader(vs); + glDeleteShader(fs); + return shader; +} + +void NvEglRenderer::RenderImage(GLuint texture, float x, float y, float width, float height) +{ + glUseProgram(imageShader); + glActiveTexture(GL_TEXTURE0); + glBindTexture(GL_TEXTURE_2D, texture); + + float vertices[6][4] = { + {x, y + height, 0.0f, 1.0f}, + {x, y, 0.0f, 0.0f}, + {x + width, y, 1.0f, 0.0f}, + {x, y + height, 0.0f, 1.0f}, + {x + width, y, 1.0f, 0.0f}, + {x + width, y + height, 1.0f, 1.0f} + }; + + glBindBuffer(GL_ARRAY_BUFFER, textVBO); + glBufferSubData(GL_ARRAY_BUFFER, 0, sizeof(vertices), vertices); + glEnableVertexAttribArray(1); + glVertexAttribPointer(1, 4, GL_FLOAT, GL_FALSE, 4 * sizeof(float), 0); + glDrawArrays(GL_TRIANGLES, 0, 6); + + glBindTexture(GL_TEXTURE_2D, 0); + glUseProgram(0); +} diff --git a/base/src/NvEglRenderer_bgd.cpp b/base/src/NvEglRenderer_bgd.cpp index a737f8c54..88a5d5d4f 100644 --- a/base/src/NvEglRenderer_bgd.cpp +++ b/base/src/NvEglRenderer_bgd.cpp @@ -33,6 +33,23 @@ #include #include #include "Logger.h" +#include +#include FT_FREETYPE_H +#include +#include + +struct Character { + GLuint TextureID; // Glyph texture + int SizeX; // Width + int SizeY; // Height + int BearingX; // Offset from baseline to left/top + int BearingY; + GLuint Advance; // Offset to advance to next glyph +}; + +static std::map Characters; +static GLuint textVAO = 0, textVBO = 0; +static GLuint textShader = 0; #define CAT_NAME "EglRenderer" @@ -237,6 +254,10 @@ NvEglRenderer::renderThread(void *arg) eglMakeCurrent(renderer->egl_display, renderer->egl_surface, renderer->egl_surface, renderer->egl_context); + + glEnable(GL_BLEND); + glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA); + if (eglGetError() != EGL_SUCCESS) { goto error; @@ -575,7 +596,8 @@ NvEglRenderer::InitializeShaders(void) static const char kFragmentShader[] = "#extension GL_OES_EGL_image_external : require\n" "precision mediump float;\n" "varying vec2 interp_tc; \n" - "uniform samplerExternalOES tex; \n" "void main() {\n" + "uniform samplerExternalOES tex; \n" "uniform float uAlpha;\n" + "void main() {\n" "gl_FragColor = texture2D(tex, interp_tc);\n" "}\n"; glEnable(GL_SCISSOR_TEST); @@ -623,6 +645,19 @@ NvEglRenderer::InitializeShaders(void) { return -1; } + GLint alphaLoc = glGetUniformLocation(program, "uAlpha"); + if (alphaLoc == -1) + { + fprintf(stderr, "Warning: uniform 'uAlpha' not found or optimized out.\n"); + } + else + { + glUniform1f(alphaLoc, 1.0f); // default fully opaque + } + + this->alpha_location = alphaLoc; + this->gl_program = program; + return 0; } @@ -639,4 +674,16 @@ NvEglRenderer::create_texture() glBindTexture(GL_TEXTURE_EXTERNAL_OES, texture_id); return 0; -} \ No newline at end of file +} +void NvEglRenderer::setWindowOpacity(float opacity) +{ + // Clamp 0.0–1.0 + if (opacity < 0.0f) opacity = 0.0f; + if (opacity > 1.0f) opacity = 1.0f; + + unsigned long opacityValue = (unsigned long)(0xFFFFFFFFul * opacity); + Atom opacityAtom = XInternAtom(x_display, "_NET_WM_WINDOW_OPACITY", False); + XChangeProperty(x_display, x_window, opacityAtom, XA_CARDINAL, 32, + PropModeReplace, (unsigned char *)&opacityValue, 1); + XFlush(x_display); +} diff --git a/base/src/NvTransform.cpp b/base/src/NvTransform.cpp index f48008b8c..3b459c71a 100644 --- a/base/src/NvTransform.cpp +++ b/base/src/NvTransform.cpp @@ -1,5 +1,5 @@ #include "NvTransform.h" -#include "nvbuf_utils.h" +//#include "nvbuf_utils.h" #include "FrameMetadata.h" #include "Frame.h" #include "Logger.h" @@ -7,237 +7,301 @@ #include "AIPExceptions.h" #include "DMAFDWrapper.h" #include "DMAAllocator.h" - #include "npp.h" +#include +#include class NvTransform::Detail { public: - Detail(NvTransformProps &_props) : props(_props) - { - src_rect.top = _props.top; - src_rect.left = _props.left; - src_rect.width = _props.width; - src_rect.height = _props.height; - - memset(&transParams, 0, sizeof(transParams)); - transParams.transform_filter = NvBufferTransform_Filter_Smart; - if (src_rect.width != 0) - { - transParams.src_rect = src_rect; - transParams.transform_flag = NVBUFFER_TRANSFORM_FILTER | NVBUFFER_TRANSFORM_CROP_SRC; - } - else - { - transParams.transform_flag = NVBUFFER_TRANSFORM_FILTER; - } - } - - ~Detail() - { - } - - bool compute(frame_sp &frame, int outFD) - { - auto dmaFDWrapper = static_cast(frame->data()); - auto ret_val = NvBufferTransform(dmaFDWrapper->getFd(), outFD, &transParams); - if (ret_val == -1) - { - LOG_INFO << "Transform failed============================================>>>>>>>>>>>>>>>>>>>>>>>>>>>" << endl; - } - return true; - } + Detail(NvTransformProps &_props) : props(_props) +{ + src_rect.top = _props.top; + src_rect.left = _props.left; + src_rect.width = _props.width; + src_rect.height = _props.height; + + + memset(&transParams, 0, sizeof(transParams)); + transParams.transform_filter = NvBufSurfTransformInter_Default; + + transParams.transform_flag = NVBUFSURF_TRANSFORM_FILTER; + + if (src_rect.width != 0 && src_rect.height != 0) + { + transParams.src_rect = &src_rect; + transParams.transform_flag |= NVBUFSURF_TRANSFORM_CROP_SRC | NVBUFSURF_TRANSFORM_ALLOW_ODD_CROP; + } + + transParams.transform_flip = NvBufSurfTransform_None; + if (_props.rotation != (NvTransformProps::NvRotation)0) + { + transParams.transform_flag |= NVBUFSURF_TRANSFORM_FLIP; + + switch (_props.rotation) + { + case NvTransformProps::NvRotation::Rotate90: + transParams.transform_flip = NvBufSurfTransform_Rotate90; + break; + case NvTransformProps::NvRotation::Rotate180: + transParams.transform_flip = NvBufSurfTransform_Rotate180; + break; + case NvTransformProps::NvRotation::Rotate270: + transParams.transform_flip = NvBufSurfTransform_Rotate270; + break; + default: + LOG_ERROR << "Invalid rotation angle. Supported: 0, 90, 180, 270."; + transParams.transform_flip = NvBufSurfTransform_None; + break; + } + } + if (_props.flip != (NvTransformProps::NvFlip)0) + { + transParams.transform_flag |= NVBUFSURF_TRANSFORM_FLIP; + + switch (_props.flip) + { + case NvTransformProps::NvFlip::FlipX: + transParams.transform_flip = NvBufSurfTransform_FlipX; + break; + case NvTransformProps::NvFlip::FlipY: + transParams.transform_flip = NvBufSurfTransform_FlipY; + break; + default: + LOG_ERROR << "Invalid flip value. Supported: None, FlipX, FlipY."; + transParams.transform_flip = NvBufSurfTransform_None; + break; + } + } +} + + + bool compute(frame_sp &frame, int outFD) + { + auto dmaFDWrapper = static_cast(frame->data()); + NvBufSurface *in_surf = nullptr; + NvBufSurface *out_surf = nullptr; + + if (NvBufSurfaceFromFd(dmaFDWrapper->getFd(), (void**)&in_surf) != 0) { + LOG_INFO << "Failed to create input surface"; + return false; + } + + if (NvBufSurfaceFromFd(outFD, (void**)&out_surf) != 0) { + LOG_INFO << "Failed to create output surface"; + return false; + } + + // Synchronize input to device before transform + for (int p = 0; p < 3; ++p) + { + NvBufSurfaceSyncForDevice(in_surf, 0, p); + } + + NvBufSurfTransform_Error err = NvBufSurfTransform(in_surf, out_surf, &transParams); + + if (err != NvBufSurfTransformError_Success) { + LOG_INFO << "Transform failed============================================>>>>>>>>>>>>>>>>>>>>>>>>>>>" << endl;; + } + + for (int p = 0; p < 3; ++p) + { + NvBufSurfaceSyncForCpu(out_surf, 0, p); + } + + return true; + } public: - NvBufferRect src_rect; - framemetadata_sp outputMetadata; - std::string outputPinId; - NvTransformProps props; + NvBufSurfTransformRect src_rect; + framemetadata_sp outputMetadata; + std::string outputPinId; + NvTransformProps props; private: - NvBufferTransformParams transParams; + NvBufSurfTransformParams transParams; }; NvTransform::NvTransform(NvTransformProps props) : Module(TRANSFORM, "NvTransform", props) { - mDetail.reset(new Detail(props)); + mDetail.reset(new Detail(props)); } NvTransform::~NvTransform() {} bool NvTransform::validateInputPins() { - if (getNumberOfInputPins() != 1) - { - LOG_INFO << "<" << getId() << ">::validateInputPins size is expected to be 1. Actual<" << getNumberOfInputPins() << ">"; - return false; - } - - framemetadata_sp metadata = getFirstInputMetadata(); - FrameMetadata::FrameType frameType = metadata->getFrameType(); - if (frameType != FrameMetadata::RAW_IMAGE && frameType != FrameMetadata::RAW_IMAGE_PLANAR) - { - LOG_INFO << "<" << getId() << ">::validateInputPins input frameType is expected to be RAW_IMAGE or RAW_IMAGE_PLANAR. Actual<" << frameType << ">"; - return false; - } - - FrameMetadata::MemType memType = metadata->getMemType(); - if (memType != FrameMetadata::MemType::DMABUF) - { - LOG_INFO << "<" << getId() << ">::validateInputPins input memType is expected to be DMABUF. Actual<" << memType << ">"; - return false; - } - - return true; + if (getNumberOfInputPins() != 1) + { + LOG_INFO << "<" << getId() << ">::validateInputPins size is expected to be 1. Actual<" << getNumberOfInputPins() << ">"; + return false; + } + + framemetadata_sp metadata = getFirstInputMetadata(); + FrameMetadata::FrameType frameType = metadata->getFrameType(); + if (frameType != FrameMetadata::RAW_IMAGE && frameType != FrameMetadata::RAW_IMAGE_PLANAR) + { + LOG_INFO << "<" << getId() << ">::validateInputPins input frameType is expected to be RAW_IMAGE or RAW_IMAGE_PLANAR. Actual<" << frameType << ">"; + return false; + } + + FrameMetadata::MemType memType = metadata->getMemType(); + if (memType != FrameMetadata::MemType::DMABUF) + { + LOG_INFO << "<" << getId() << ">::validateInputPins input memType is expected to be DMABUF. Actual<" << memType << ">"; + return false; + } + + return true; } bool NvTransform::validateOutputPins() { - if (getNumberOfOutputPins() != 1) - { - LOG_INFO << "<" << getId() << ">::validateOutputPins size is expected to be 1. Actual<" << getNumberOfOutputPins() << ">"; - return false; - } - - framemetadata_sp metadata = getFirstOutputMetadata(); - auto frameType = metadata->getFrameType(); - if (frameType != FrameMetadata::RAW_IMAGE && frameType != FrameMetadata::RAW_IMAGE_PLANAR) - { - LOG_INFO << "<" << getId() << ">::validateOutputPins input frameType is expected to be RAW_IMAGE or RAW_IMAGE_PLANAR. Actual<" << frameType << ">"; - return false; - } - - FrameMetadata::MemType memType = metadata->getMemType(); - if (memType != FrameMetadata::MemType::DMABUF) - { - LOG_INFO << "<" << getId() << ">::validateOutputPins input memType is expected to be DMABUF. Actual<" << memType << ">"; - return false; - } - - return true; + if (getNumberOfOutputPins() != 1) + { + LOG_INFO << "<" << getId() << ">::validateOutputPins size is expected to be 1. Actual<" << getNumberOfOutputPins() << ">"; + return false; + } + + framemetadata_sp metadata = getFirstOutputMetadata(); + auto frameType = metadata->getFrameType(); + if (frameType != FrameMetadata::RAW_IMAGE && frameType != FrameMetadata::RAW_IMAGE_PLANAR) + { + LOG_INFO << "<" << getId() << ">::validateOutputPins input frameType is expected to be RAW_IMAGE or RAW_IMAGE_PLANAR. Actual<" << frameType << ">"; + return false; + } + + FrameMetadata::MemType memType = metadata->getMemType(); + if (memType != FrameMetadata::MemType::DMABUF) + { + LOG_INFO << "<" << getId() << ">::validateOutputPins input memType is expected to be DMABUF. Actual<" << memType << ">"; + return false; + } + + return true; } void NvTransform::addInputPin(framemetadata_sp &metadata, string &pinId) { - Module::addInputPin(metadata, pinId); - switch (mDetail->props.imageType) - { - case ImageMetadata::BGRA: - case ImageMetadata::RGBA: - mDetail->outputMetadata = framemetadata_sp(new RawImageMetadata(FrameMetadata::MemType::DMABUF)); - break; - case ImageMetadata::NV12: - case ImageMetadata::YUV420: - case ImageMetadata::YUV444: - mDetail->outputMetadata = framemetadata_sp(new RawImagePlanarMetadata(FrameMetadata::MemType::DMABUF)); - break; - default: - throw AIPException(AIP_FATAL, "Unsupported Image Type<" + std::to_string(mDetail->props.imageType) + ">"); - } - - mDetail->outputMetadata->copyHint(*metadata.get()); - mDetail->outputPinId = addOutputPin(mDetail->outputMetadata); + Module::addInputPin(metadata, pinId); + switch (mDetail->props.imageType) + { + case ImageMetadata::BGRA: + case ImageMetadata::RGBA: + mDetail->outputMetadata = framemetadata_sp(new RawImageMetadata(FrameMetadata::MemType::DMABUF)); + break; + case ImageMetadata::NV12: + case ImageMetadata::YUV420: + case ImageMetadata::YUV444: + mDetail->outputMetadata = framemetadata_sp(new RawImagePlanarMetadata(FrameMetadata::MemType::DMABUF)); + break; + default: + throw AIPException(AIP_FATAL, "Unsupported Image Type<" + std::to_string(mDetail->props.imageType) + ">"); + } + + mDetail->outputMetadata->copyHint(*metadata.get()); + mDetail->outputPinId = addOutputPin(mDetail->outputMetadata); } bool NvTransform::init() { - if (!Module::init()) - { - return false; - } + if (!Module::init()) + { + return false; + } - return true; + return true; } bool NvTransform::term() { - return Module::term(); + return Module::term(); } bool NvTransform::process(frame_container &frames) { - auto frame = frames.cbegin()->second; - if(isFrameEmpty(frame)) - { - LOG_INFO << "Found Empty Frame "; - return true; - } - if(!mDetail->outputMetadata->getDataSize()) - { - return true; - } - auto outFrame = makeFrame(mDetail->outputMetadata->getDataSize(), mDetail->outputPinId); - if (!outFrame.get()) - { - LOG_INFO << "FAILED TO GET BUFFER"; - return false; - } - - auto dmaFdWrapper = static_cast(outFrame->data()); - dmaFdWrapper->tempFD = dmaFdWrapper->getFd(); - - mDetail->compute(frame, dmaFdWrapper->tempFD); - - frames.insert(make_pair(mDetail->outputPinId, outFrame)); - send(frames); - - // LOG_DEBUG << "Processed Frame " << frame->fIndex << " to " << mDetail->outputPinId; - - return true; + auto frame = frames.cbegin()->second; + if(isFrameEmpty(frame)) + { + LOG_INFO << "Found Empty Frame "; + return true; + } + if(!mDetail->outputMetadata->getDataSize()) + { + return true; + } + auto outFrame = makeFrame(mDetail->outputMetadata->getDataSize(), mDetail->outputPinId); + if (!outFrame.get()) + { + LOG_INFO << "FAILED TO GET BUFFER"; + return false; + } + + auto dmaFdWrapper = static_cast(outFrame->data()); + dmaFdWrapper->tempFD = dmaFdWrapper->getFd(); + + mDetail->compute(frame, dmaFdWrapper->tempFD); + + frames.insert(make_pair(mDetail->outputPinId, outFrame)); + send(frames); + + // LOG_DEBUG << "Processed Frame " << frame->fIndex << " to " << mDetail->outputPinId; + + return true; } bool NvTransform::processSOS(frame_sp &frame) { - auto metadata = frame->getMetadata(); - setMetadata(metadata); + auto metadata = frame->getMetadata(); + setMetadata(metadata); - return true; + return true; } void NvTransform::setMetadata(framemetadata_sp &metadata) { - auto frameType = metadata->getFrameType(); - int width = 0; - int height = 0; - int depth = CV_8U; - ImageMetadata::ImageType inputImageType = ImageMetadata::ImageType::MONO; - - switch (frameType) - { - case FrameMetadata::FrameType::RAW_IMAGE: - { - auto rawMetadata = FrameMetadataFactory::downcast(metadata); - width = rawMetadata->getWidth(); - height = rawMetadata->getHeight(); - depth = rawMetadata->getDepth(); - inputImageType = rawMetadata->getImageType(); - } - break; - case FrameMetadata::FrameType::RAW_IMAGE_PLANAR: - { - auto rawMetadata = FrameMetadataFactory::downcast(metadata); - width = rawMetadata->getWidth(0); - height = rawMetadata->getHeight(0); - depth = rawMetadata->getDepth(); - inputImageType = rawMetadata->getImageType(); - } - break; - default: - throw AIPException(AIP_NOTIMPLEMENTED, "Unsupported FrameType<" + std::to_string(frameType) + ">"); - } - - if (mDetail->props.width != 0) - { - width = mDetail->props.width; - height = mDetail->props.height; - } - - DMAAllocator::setMetadata(mDetail->outputMetadata, width, height, mDetail->props.imageType); + auto frameType = metadata->getFrameType(); + int width = 0; + int height = 0; + int depth = CV_8U; + ImageMetadata::ImageType inputImageType = ImageMetadata::ImageType::MONO; + + switch (frameType) + { + case FrameMetadata::FrameType::RAW_IMAGE: + { + auto rawMetadata = FrameMetadataFactory::downcast(metadata); + width = rawMetadata->getWidth(); + height = rawMetadata->getHeight(); + depth = rawMetadata->getDepth(); + inputImageType = rawMetadata->getImageType(); + } + break; + case FrameMetadata::FrameType::RAW_IMAGE_PLANAR: + { + auto rawMetadata = FrameMetadataFactory::downcast(metadata); + width = rawMetadata->getWidth(0); + height = rawMetadata->getHeight(0); + depth = rawMetadata->getDepth(); + inputImageType = rawMetadata->getImageType(); + } + break; + default: + throw AIPException(AIP_NOTIMPLEMENTED, "Unsupported FrameType<" + std::to_string(frameType) + ">"); + } + + if (mDetail->props.width != 0) + { + width = mDetail->props.width; + height = mDetail->props.height; + } + + DMAAllocator::setMetadata(mDetail->outputMetadata, width, height, mDetail->props.imageType); } bool NvTransform::processEOS(string &pinId) { - LOG_DEBUG<< "Resetting Output Metadata"; - // mDetail->outputMetadata.reset(); - return true; + LOG_DEBUG<< "Resetting Output Metadata"; + // mDetail->outputMetadata.reset(); + return true; } \ No newline at end of file diff --git a/base/src/V4L2CUYUV420Converter.cpp b/base/src/V4L2CUYUV420Converter.cpp index f6abb05e5..c3ae98bc1 100755 --- a/base/src/V4L2CUYUV420Converter.cpp +++ b/base/src/V4L2CUYUV420Converter.cpp @@ -3,8 +3,11 @@ #include "ApraEGLDisplay.h" #include "Frame.h" #include "AIPExceptions.h" - -#include "nvbuf_utils.h" +//#include "nvbuf_utils.h" +#include +//#include "nvbuf_utils.h" +#include +#include #include V4L2CUYUV420Converter::V4L2CUYUV420Converter(uint32_t srcWidth, uint32_t srcHeight, struct v4l2_format &format) : mFormat(format) @@ -26,13 +29,22 @@ V4L2CUYUV420Converter::~V4L2CUYUV420Converter() void V4L2CUYUV420Converter::process(frame_sp& frame, AV4L2Buffer *buffer) { auto data = static_cast(frame->data()); - + if (!data) + { + LOG_FATAL << "Input frame data is null in V4L2CUYUV420Converter::process"; + return; + } uint32_t i; auto numPlanes = buffer->getNumPlanes(); for (i = 0; i < numPlanes; i++) { buffer->v4l2_buf.m.planes[i].bytesused = mBytesUsedY; auto v4l2Data = buffer->planesInfo[i].data; + if (!v4l2Data) + { + LOG_FATAL << "Destination plane data is null for plane " << i; + return; + } auto height = mHeightY; auto width = mWidthY; auto bytesperline = mFormat.fmt.pix_mp.plane_fmt[i].bytesperline; @@ -53,9 +65,15 @@ void V4L2CUYUV420Converter::process(frame_sp& frame, AV4L2Buffer *buffer) for (i = 0; i < numPlanes; i++) { - if (NvBufferMemSyncForDevice(buffer->planesInfo[i].fd, i, (void **)(&buffer->planesInfo[i].data)) < 0) + NvBufSurface *surf = 0; + if (NvBufSurfaceFromFd(buffer->planesInfo[i].fd, reinterpret_cast(&surf)) != 0) + { + LOG_FATAL << "Failed to map DMABUF to NvBufSurface"; + return; + } + if (NvBufSurfaceSyncForDevice(surf, -1, i) != 0) { - LOG_FATAL << "NvBufferMemSyncForDevice failed<>" << i; + LOG_FATAL << "NvBufSurfaceSyncForDevice failed for plane " << i; } } } @@ -72,10 +90,31 @@ V4L2CUDMABufYUV420Converter::~V4L2CUDMABufYUV420Converter() void V4L2CUDMABufYUV420Converter::process(frame_sp& frame, AV4L2Buffer *buffer) { - auto ptr = static_cast(frame->data()); + auto ptr = static_cast(frame->data()); + if (!ptr) + { + LOG_FATAL << "DMAFDWrapper is null"; + return; + } + int fd = ptr->getFd(); + if (fd < 0) + { + LOG_FATAL << "Invalid DMABUF fd"; + return; + } buffer->v4l2_buf.m.planes[0].m.fd = ptr->getFd(); buffer->v4l2_buf.m.planes[0].bytesused = 1; - + NvBufSurface *surf = ptr->getNvBufSurface(); + if (!surf) + { + LOG_FATAL << "Failed to get NvBufSurface from DMAFDWrapper"; + return; + } + if (NvBufSurfaceSyncForDevice(surf, -1, 0) != 0) + { + LOG_FATAL << "NvBufSurfaceSyncForDevice failed"; + return; + } std::lock_guard lock(mCacheMutex); mCache.push_back(frame); } @@ -106,60 +145,73 @@ V4L2CURGBToYUV420Converter::~V4L2CURGBToYUV420Converter() void V4L2CURGBToYUV420Converter::process(frame_sp& frame, AV4L2Buffer *buffer) { - eglImage = NvEGLImageFromFd(eglDisplay, buffer->planesInfo[0].fd); + const int width = static_cast(mFormat.fmt.pix_mp.width); + const int height = static_cast(mFormat.fmt.pix_mp.height); + const int pitch0 = static_cast(mFormat.fmt.pix_mp.plane_fmt[0].bytesperline); + const int pitch1 = static_cast(mFormat.fmt.pix_mp.plane_fmt[1].bytesperline); + const int pitch2 = static_cast(mFormat.fmt.pix_mp.plane_fmt[2].bytesperline); + EGLAttrib plane0_offset = 0; + EGLAttrib plane1_offset = pitch0 * height; + EGLAttrib plane2_offset = plane1_offset + pitch1 * (height / 2); + EGLAttrib attribs[] = { + EGL_WIDTH, width, + EGL_HEIGHT, height, + EGL_LINUX_DRM_FOURCC_EXT, DRM_FORMAT_YUV420, + EGL_DMA_BUF_PLANE0_FD_EXT, buffer->planesInfo[0].fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, plane0_offset, + EGL_DMA_BUF_PLANE0_PITCH_EXT, pitch0, + EGL_DMA_BUF_PLANE1_FD_EXT, buffer->planesInfo[1].fd, + EGL_DMA_BUF_PLANE1_OFFSET_EXT, plane1_offset, + EGL_DMA_BUF_PLANE1_PITCH_EXT, pitch1, + EGL_DMA_BUF_PLANE2_FD_EXT, buffer->planesInfo[2].fd, + EGL_DMA_BUF_PLANE2_OFFSET_EXT, plane2_offset, + EGL_DMA_BUF_PLANE2_PITCH_EXT, pitch2, + EGL_NONE + }; + eglImage = eglCreateImage(eglDisplay, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, + (EGLClientBuffer)NULL, attribs); + if (eglImage == EGL_NO_IMAGE_KHR) { + LOG_ERROR << "eglCreateImage failed for YUV420 EGL image"; + return; + } status = cuGraphicsEGLRegisterImage(&pResource, eglImage, CU_GRAPHICS_MAP_RESOURCE_FLAGS_NONE); - if (status != CUDA_SUCCESS) - { - LOG_ERROR << "cuGraphicsEGLRegisterImage failed: " << status << " cuda process stop"; + if (status != CUDA_SUCCESS) { + LOG_ERROR << "cuGraphicsEGLRegisterImage failed: " << status; + eglDestroyImage(eglDisplay, eglImage); return; } - status = cuGraphicsResourceGetMappedEglFrame(&eglFrame, pResource, 0, 0); - if (status != CUDA_SUCCESS) - { - LOG_ERROR << "cuGraphicsSubResourceGetMappedArray failed status<" << status << ">"; + if (status != CUDA_SUCCESS) { + LOG_ERROR << "cuGraphicsResourceGetMappedEglFrame failed: " << status; return; } - - for (auto i = 0; i < 3; i++) - { + Npp8u* dst[3]; + for (int i = 0; i < 3; ++i) { dst[i] = static_cast(eglFrame.frame.pPitch[i]); } status = cuCtxSynchronize(); - if (status != CUDA_SUCCESS) - { - LOG_ERROR << "cuCtxSynchronize failed status<" << status << ">"; + if (status != CUDA_SUCCESS) { + LOG_ERROR << "cuCtxSynchronize failed: " << status; return; } auto data = static_cast(frame->data()); auto res = nppiRGBToYUV420_8u_C3P3R(static_cast(data), nsrcStep, dst, dstPitch, oSizeROI); - if (res != NPP_SUCCESS) - { + if (res != NPP_SUCCESS) { LOG_ERROR << "nppiRGBToYUV420_8u_C3P3R failed"; } status = cuCtxSynchronize(); - if (status != CUDA_SUCCESS) - { - LOG_ERROR << "cuCtxSynchronize failed after cc status<" << status << ">"; + if (status != CUDA_SUCCESS) { + LOG_ERROR << "cuCtxSynchronize failed after NPP: " << status; } - status = cuGraphicsUnregisterResource(pResource); - if (status != CUDA_SUCCESS) - { + if (status != CUDA_SUCCESS) { LOG_ERROR << "cuGraphicsEGLUnRegisterResource failed: " << status; } - - NvDestroyEGLImage(eglDisplay, eglImage); - - for (auto i = 0; i < 3; i++) - { - buffer->v4l2_buf.m.planes[i].bytesused = mBytesUsedY; - if (i != 0) - { - buffer->v4l2_buf.m.planes[i].bytesused = mBytesUsedUV; - } - } + eglDestroyImage(eglDisplay, eglImage); + buffer->v4l2_buf.m.planes[0].bytesused = mBytesUsedY; + buffer->v4l2_buf.m.planes[1].bytesused = mBytesUsedUV; + buffer->v4l2_buf.m.planes[2].bytesused = mBytesUsedUV; } \ No newline at end of file diff --git a/base/test/eglrenderer_test.cpp b/base/test/eglrenderer_test.cpp index f6ac0b2bc..e33193072 100644 --- a/base/test/eglrenderer_test.cpp +++ b/base/test/eglrenderer_test.cpp @@ -3,15 +3,19 @@ #include "FileReaderModule.h" #include "EglRenderer.h" #include "PipeLine.h" +#include "NvV4L2Camera.h" +#include "NvTransform.h" +#include "NvEglRenderer.h" +#include "ApraNvEglRenderer.h" BOOST_AUTO_TEST_SUITE(eglrenderer_tests) BOOST_AUTO_TEST_CASE(basic, *boost::unit_test::disabled()) { int width = 640; - int height = 480; + int height = 360; - FileReaderModuleProps fileReaderProps("./data/ArgusCamera"); + FileReaderModuleProps fileReaderProps("/home/developer/ApraPipes/data/Raw_YUV420_640x360/Image001_YUV420.raw"); fileReaderProps.fps = 30; auto fileReader = boost::shared_ptr(new FileReaderModule(fileReaderProps)); auto metadata = framemetadata_sp(new RawImageMetadata(width, height, ImageMetadata::ImageType::UYVY, CV_8UC1, 0, CV_8U, FrameMetadata::MemType::DMABUF, true)); @@ -38,4 +42,299 @@ BOOST_AUTO_TEST_CASE(basic, *boost::unit_test::disabled()) p.wait_for_all(); } +BOOST_AUTO_TEST_CASE(displayOnTop, *boost::unit_test::disabled()) +{ + LoggerProps logProps; + logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::severity_level::trace); + NvV4L2CameraProps nvCamProps(640,360, 10,false); + auto source = boost::shared_ptr(new NvV4L2Camera(nvCamProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0,0))); + //,"/home/developer/ApraPipes/data/Debrosee-ALPnL.ttf","HelloWorld",1.0f,1.0f,1.0f,1.0f,24,200,200,0.99))); + transform->setNext(sink); + + PipeLine p("test"); + p.appendModule(source); + BOOST_TEST(p.init()); + + Logger::setLogLevel(boost::log::trivial::severity_level::info); + + p.run_all_threaded(); + + boost::this_thread::sleep_for(boost::chrono::seconds(120)); + Logger::setLogLevel(boost::log::trivial::severity_level::error); + + p.stop(); + p.term(); + + p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(switch_display, *boost::unit_test::disabled()) +{ + NvV4L2CameraProps nvCamProps(640,360, 10,false); + auto source = boost::shared_ptr(new NvV4L2Camera(nvCamProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0,0))); + //,"/home/developer/ApraPipes/data/Debrosee-ALPnL.ttf","HelloWorld",1.0f,1.0f,1.0f,1.0f,24,10,50))); + transform->setNext(sink); + + PipeLine p("test"); + p.appendModule(source); + BOOST_TEST(p.init()); + + Logger::setLogLevel(boost::log::trivial::severity_level::info); + + p.run_all_threaded(); + + boost::this_thread::sleep_for(boost::chrono::seconds(120)); + Logger::setLogLevel(boost::log::trivial::severity_level::error); + + p.stop(); + p.term(); + + p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(open_close_window, *boost::unit_test::disabled()) +{ + NvV4L2CameraProps nvCamProps(640,360, 10,false); + auto source = boost::shared_ptr(new NvV4L2Camera(nvCamProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0,0))); + transform->setNext(sink); + + PipeLine p("test"); + p.appendModule(source); + BOOST_TEST(p.init()); + + Logger::setLogLevel(boost::log::trivial::severity_level::info); + + p.run_all_threaded(); + + boost::this_thread::sleep_for(boost::chrono::seconds(5)); + sink->closeWindow(); + + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + sink->createWindow(200,200); + + boost::this_thread::sleep_for(boost::chrono::seconds(120)); + Logger::setLogLevel(boost::log::trivial::severity_level::error); + + p.stop(); + p.term(); + + p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(ctor_default, *boost::unit_test::disabled()) +{ + LoggerProps logProps; + logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::trace); + + NvV4L2CameraProps camProps(640, 360, 10, false); + auto source = boost::shared_ptr(new NvV4L2Camera(camProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps())); + transform->setNext(sink); + + PipeLine p("default_ctor_test"); + p.appendModule(source); + BOOST_TEST(p.init()); + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + p.stop(); p.term(); p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(ctor_geometry_xy, *boost::unit_test::disabled()) +{ + LoggerProps logProps; logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::trace); + + NvV4L2CameraProps camProps(640, 360, 10, false); + auto source = boost::shared_ptr(new NvV4L2Camera(camProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(50, 50))); + transform->setNext(sink); + + PipeLine p("geometry_xy_test"); + p.appendModule(source); + BOOST_TEST(p.init()); + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + p.stop(); p.term(); p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(ctor_geometry_xywh, *boost::unit_test::disabled()) +{ + LoggerProps logProps; logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::trace); + + NvV4L2CameraProps camProps(640, 360, 10, false); + auto source = boost::shared_ptr(new NvV4L2Camera(camProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0, 0, 320, 240))); + transform->setNext(sink); + + PipeLine p("geometry_xywh_test"); + p.appendModule(source); + BOOST_TEST(p.init()); + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + p.stop(); p.term(); p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(ctor_geometry_text, *boost::unit_test::disabled()) +{ + LoggerProps logProps; logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::trace); + + NvV4L2CameraProps camProps(640, 360, 10, false); + auto source = boost::shared_ptr(new NvV4L2Camera(camProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + EglRendererProps::TextInfo text; + text.fontPath = "/home/developer/ApraPipes/data/Debrosee-ALPnL.ttf"; + text.message = "HelloText"; + text.color = {1.0f, 0.0f, 1.0f}; + text.fontSize = 24; + text.position = {50,50}; + text.scale = 1; + text.opacity = 1.0f; + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(100, 100, 320, 240, text))); + transform->setNext(sink); + + PipeLine p("geometry_text_test"); + p.appendModule(source); + BOOST_TEST(p.init()); + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + p.stop(); p.term(); p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(ctor_geometry_image, *boost::unit_test::disabled()) +{ + LoggerProps logProps; logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::trace); + + NvV4L2CameraProps camProps(640, 360, 10, false); + auto source = boost::shared_ptr(new NvV4L2Camera(camProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + EglRendererProps::ImageInfo img; + img.path = "/home/developer/ApraPipes/data/apra.jpeg"; + img.position = {0, 0}; + img.size = {128, 128}; + img.opacity = 0.75f; + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(200, 200, 320, 240, img))); + transform->setNext(sink); + + PipeLine p("geometry_image_test"); + p.appendModule(source); + BOOST_TEST(p.init()); + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + p.stop(); p.term(); p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(ctor_geometry_text_image, *boost::unit_test::disabled()) +{ + LoggerProps logProps; logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::trace); + + NvV4L2CameraProps camProps(640, 360, 10, false); + auto source = boost::shared_ptr(new NvV4L2Camera(camProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + EglRendererProps::TextInfo text; + text.fontPath = "/home/developer/ApraPipes/data/Debrosee-ALPnL.ttf"; + text.message = "Overlay Text"; + text.color = {0.0f, 1.0f, 0.0f}; + text.fontSize = 20; + text.position = {100, 50}; + + EglRendererProps::ImageInfo img; + img.path = "/home/developer/ApraPipes/data/apra.jpeg"; + img.position = {200, 150}; + img.size = {64, 64}; + + auto sink = boost::shared_ptr( + new EglRenderer(EglRendererProps(0, 0, 320, 240, text, img)) + ); + transform->setNext(sink); + + PipeLine p("geometry_text_image_test"); + p.appendModule(source); + BOOST_TEST(p.init()); + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + p.stop(); p.term(); p.wait_for_all(); +} + +BOOST_AUTO_TEST_CASE(ctor_geometry_opacity_mask, *boost::unit_test::disabled()) +{ + LoggerProps logProps; logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::trace); + + NvV4L2CameraProps camProps(640, 360, 10, false); + auto source = boost::shared_ptr(new NvV4L2Camera(camProps)); + + NvTransformProps nvprops(ImageMetadata::RGBA); + auto transform = boost::shared_ptr(new NvTransform(nvprops)); + source->setNext(transform); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0, 0, 320, 240, 0.5f, true))); + transform->setNext(sink); + + PipeLine p("geometry_opacity_mask_test"); + p.appendModule(source); + BOOST_TEST(p.init()); + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(10)); + p.stop(); p.term(); p.wait_for_all(); +} + BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/base/test/h264decoder_tests.cpp b/base/test/h264decoder_tests.cpp index 078565be6..8b715995b 100644 --- a/base/test/h264decoder_tests.cpp +++ b/base/test/h264decoder_tests.cpp @@ -5,402 +5,68 @@ #include "Logger.h" #include "H264Decoder.h" #include "test_utils.h" -#include "DMAFDToHostCopy.h" #include "PipeLine.h" #include "ExternalSinkModule.h" #include "H264Metadata.h" #include "Mp4ReaderSource.h" #include "Mp4VideoMetadata.h" #include "StatSink.h" -#include "CudaStreamSynchronize.h" -#include "RTSPClientSrc.h" -#ifdef ARM64 -#include "EglRenderer.h" -#include "AffineTransform.h" -#include "NvTransform.h" - -#else +#include "MemTypeConversion.h" + #include "CudaMemCopy.h" #include "nv_test_utils.h" -#endif +#include "ResizeNPPI.h" +#include "JPEGEncoderL4TM.h" BOOST_AUTO_TEST_SUITE(h264decoder_tests) -#ifdef ARM64 - -// struct rtsp_client_tests_data { -// rtsp_client_tests_data() -// { -// outFile = string("./data/testOutput/bunny.h264"); -// Test_Utils::FileCleaner fc; -// fc.pathsOfFiles.push_back(outFile); //clear any occurance before starting the tests -// } -// string outFile; -// string empty; -// }; -// BOOST_AUTO_TEST_CASE(rtsp_case3) -// { -// rtsp_client_tests_data d; -// auto url=string("rtsp://evo-dev-apra.blub0xSecurity.com:5544/76174d56-fdc0-4a2e-aad6-981f4f7f71ea"); -// auto rtspProps = RTSPClientSrcProps(url, d.empty, d.empty); -// rtspProps.fps = 18; -// auto mLive = boost::shared_ptr(new RTSPClientSrc(rtspProps)); -// auto meta = framemetadata_sp(new H264Metadata()); -// mLive->addOutputPin(meta); -// auto Decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); -// std::vector mImagePin; -// mImagePin = mLive->getAllOutputPinsByType(FrameMetadata::FrameType::H264_DATA); -// mLive->setNext(Decoder, mImagePin); -// auto copySource = boost::shared_ptr(new DMAFDToHostCopy); -// Decoder->setNext(copySource); -// auto writer1 = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/1/webrtcFrame_????.raw"))); -// copySource->setNext(writer1); -// boost::shared_ptr p; -// p = boost::shared_ptr(new PipeLine("test")); -// p->appendModule(mLive); -// if (!p->init()) -// { -// throw AIPException(AIP_FATAL, "Engine Pipeline init failed. Check IPEngine Logs for more details."); -// } -// p->run_all_threaded(); -// Test_Utils::sleep_for_seconds(5); -// p->stop(); -// p->term(); -// p->wait_for_all(); -// p.reset(); -// } -BOOST_AUTO_TEST_CASE(atl_test_pipeline,* boost::unit_test::disabled()) +BOOST_AUTO_TEST_CASE(mp4reader_decoder_eglrenderer_2) { - Logger::setLogLevel(boost::log::trivial::severity_level::debug); - std::string videoPath = "/home/developer/workspace/ApraPipes/1684307720.mp4"; - auto stream = cudastream_sp(new ApraCudaStream); - - auto mp4ReaderProps = Mp4ReaderSourceProps(videoPath, false); - auto m_reviewSource = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); - auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); - m_reviewSource->addOutPutPin(h264ImageMetadata); + Logger::setLogLevel("debug"); - auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); - m_reviewSource->addOutPutPin(mp4Metadata); + auto fileReader = + boost::shared_ptr(new FileReaderModule( + FileReaderModuleProps("./data/8bit_frame_1280x720_rgba.raw"))); + auto metadata = framemetadata_sp( + new RawImageMetadata(1280, 720, ImageMetadata::ImageType::RGBA, + CV_8UC4, 0, CV_8U, FrameMetadata::HOST, true)); + fileReader->addOutputPin(metadata); - StatSinkProps sinkProps; - sinkProps.logHealth = true; - sinkProps.logHealthFrequency = 100; - auto sink = boost::shared_ptr(new StatSink(sinkProps)); - m_reviewSource->setNext(sink); + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversionDMA = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::DMABUF, stream))); + fileReader->setNext(memconversionDMA); - auto m_h264Decode = boost::shared_ptr(new H264Decoder(H264DecoderProps())); - std::vector mImagePin; - mImagePin = m_reviewSource->getAllOutputPinsByType(FrameMetadata::FrameType::H264_DATA); - // m_reviewSource->setNext(m_h264Decode, mImagePin); + auto memconversionHost = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + memconversionDMA->setNext(memconversionHost); - auto m_nv12_to_yuv444Transform = boost::shared_ptr(new NvTransform(NvTransformProps(ImageMetadata::RGBA))); - m_h264Decode->setNext(m_nv12_to_yuv444Transform); - - AffineTransformProps affineProps(AffineTransformProps::CUBIC, stream, 0,4096, 0, 0, 1); - affineProps.qlen = 1; - affineProps.quePushStrategyType = QuePushStrategy::NON_BLOCKING_ANY; - auto m_reviewAffineTransform = boost::shared_ptr(new AffineTransform(affineProps)); - m_nv12_to_yuv444Transform->setNext(m_reviewAffineTransform); - - auto sync = boost::shared_ptr(new CudaStreamSynchronize(CudaStreamSynchronizeProps(stream))); - m_reviewAffineTransform->setNext(sync); - - EglRendererProps eglProps(455, 38, 1000, 1000); - eglProps.qlen = 2; - eglProps.fps = 20; - eglProps.quePushStrategyType = QuePushStrategy::NON_BLOCKING_ANY; - auto m_review_renderer = boost::shared_ptr(new EglRenderer(eglProps)); - sync->setNext(m_review_renderer); - - // m_playbackPipeline.appendModule(m_reviewSource); - // m_playbackPipeline.init(); - Logger::setLogLevel(boost::log::trivial::severity_level::debug); + auto fileWriter = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/MEMCPY_TEST/frame.raw",true))); + memconversionHost->setNext(fileWriter); boost::shared_ptr p; p = boost::shared_ptr(new PipeLine("test")); - p->appendModule(m_reviewSource); + p->appendModule(fileReader); - if (!p->init()) + if (!p->init()) { throw AIPException(AIP_FATAL, "Engine Pipeline init failed. Check IPEngine Logs for more details."); } p->run_all_threaded(); + Test_Utils::sleep_for_seconds(1); + p->stop(); + p->term(); + p->wait_for_all(); + p.reset(); } -BOOST_AUTO_TEST_CASE(memory_leak_check,* boost::unit_test::disabled()) -{ - - boost::shared_ptr m_playbackPipeline; - m_playbackPipeline = boost::shared_ptr(new PipeLine("test")); - - std::vector mediaList = { - "/media/developer/1250328450326F1B/Videos_to_test/2023-10-31/D1/P1/2023-10-31_12-42-39-913.mp4", - "/media/developer/1250328450326F1B/Videos_to_test/2023-10-31/D1/P1/2023-10-31_12-42-55-845.mp4", - "/media/developer/1250328450326F1B/Videos_to_test/2023-10-31/D1/P1/2023-10-31_12-50-53-781.mp4", - "/media/developer/1250328450326F1B/Videos_to_test/2023-10-31/D1/P1/2023-10-31_12-51-12-751.mp4", - "/media/developer/1250328450326F1B/Videos_to_test/2023-10-31/D1/P1/2023-10-31_12-51-34-695.mp4" - }; - - auto stream = cudastream_sp(new ApraCudaStream); - auto mp4ReaderProps = Mp4ReaderSourceProps(mediaList[0], false); - boost::shared_ptr m_reviewSource = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); - auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); - m_reviewSource->addOutPutPin(h264ImageMetadata); - - auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); - m_reviewSource->addOutPutPin(mp4Metadata); - - - StatSinkProps sinkProps; - sinkProps.logHealth = true; - sinkProps.logHealthFrequency = 100; - auto m3 = boost::shared_ptr(new StatSink(sinkProps)); - m_reviewSource->setNext(m3); - - auto decoderProps = H264DecoderProps(); - boost::shared_ptr m_h264Decode = boost::shared_ptr(new H264Decoder(decoderProps)); - std::vector mImagePin; - mImagePin = m_reviewSource->getAllOutputPinsByType(FrameMetadata::FrameType::H264_DATA); - m_reviewSource->setNext(m_h264Decode, mImagePin); - - EglRendererProps eglProps(455, 800, 400, 400); - boost::shared_ptr m_review_renderer = boost::shared_ptr(new EglRenderer(eglProps)); - m_h264Decode->setNext(m_review_renderer); - - m_playbackPipeline->appendModule(m_reviewSource); - m_playbackPipeline->init(); - m_playbackPipeline->run_all_threaded(); - - while (true) - { - - LOG_ERROR << "TOTAL NUMBER OF FILE TO PLAY" << mediaList.size(); - for (int i = 0; i < mediaList.size(); i++) - { - LOG_ERROR << "<========================== PLAYING =================================>>>>>>>>>>>>>>>>>>>>" << mediaList[i]; - auto currMediaProps = m_reviewSource->getProps(); - currMediaProps.videoPath = mediaList[i]; //"/home/developer/workspace/ApraPipes/1684824632.mp4"; - m_reviewSource->setProps(currMediaProps); - // m_review_renderer->createWindow(1000, 1000); - boost::this_thread::sleep_for(boost::chrono::seconds(15)); - // m_review_renderer->closeWindow(); - // boost::this_thread::sleep_for(boost::chrono::seconds(1)); - } - } - - Test_Utils::sleep_for_seconds(15000); -} - -BOOST_AUTO_TEST_CASE(fastPlayback,* boost::unit_test::disabled()) -{ - boost::shared_ptr m_playbackPipeline; - m_playbackPipeline = boost::shared_ptr(new PipeLine("test")); - - std::vector mediaList = { - "/home/vivek/2023-12-12_16-34-08-799.mp4"}; - - auto stream = cudastream_sp(new ApraCudaStream); - auto mp4ReaderProps = Mp4ReaderSourceProps(mediaList[0], false); - // mp4ReaderProps.fps = 30; - mp4ReaderProps.logHealth = true; - mp4ReaderProps.logHealthFrequency = 100; - mp4ReaderProps.quePushStrategyType = QuePushStrategy::BLOCKING; - boost::shared_ptr m_reviewSource = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); - auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); - m_reviewSource->addOutPutPin(h264ImageMetadata); - - auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); - m_reviewSource->addOutPutPin(mp4Metadata); - - auto decoderProps = H264DecoderProps(); - decoderProps.quePushStrategyType = QuePushStrategy::BLOCKING; - boost::shared_ptr m_h264Decode = boost::shared_ptr(new H264Decoder(decoderProps)); - std::vector mImagePin; - mImagePin = m_reviewSource->getAllOutputPinsByType(FrameMetadata::FrameType::H264_DATA); - m_reviewSource->setNext(m_h264Decode, mImagePin); - - EglRendererProps eglProps(0, 0, 1000, 1000); - eglProps.fps = 30; - eglProps.quePushStrategyType = QuePushStrategy::BLOCKING; - boost::shared_ptr m_review_renderer = boost::shared_ptr(new EglRenderer(eglProps)); - m_h264Decode->setNext(m_review_renderer); - Logger::setLogLevel(boost::log::trivial::severity_level::debug); - m_playbackPipeline->appendModule(m_reviewSource); - m_playbackPipeline->init(); - m_playbackPipeline->run_all_threaded(); - Logger::setLogLevel(boost::log::trivial::severity_level::debug); - - boost::this_thread::sleep_for(boost::chrono::seconds(10)); - m_review_renderer->play(false, true); - m_reviewSource->play(false, true); - - boost::this_thread::sleep_for(boost::chrono::seconds(10)); - // auto rederProps = m_reviewSource->getProps(); - // m_reviewSource->setProps(rederProps); - m_review_renderer->play(true, true); - m_reviewSource->play(true, true); - boost::this_thread::sleep_for(boost::chrono::seconds(10)); - - m_review_renderer->play(false, true); - m_reviewSource->play(false, true); - - boost::this_thread::sleep_for(boost::chrono::seconds(10)); - m_review_renderer->play(true, true); - m_reviewSource->play(true, true); - - Test_Utils::sleep_for_seconds(15000); -} - -void myCallbackFunction() -{ - // Your callback logic here - LOG_ERROR << "Callback function triggered!"; -} - -BOOST_AUTO_TEST_CASE(rotateRecordedClip,* boost::unit_test::disabled()) -{ - - boost::shared_ptr m_playbackPipeline; - m_playbackPipeline = boost::shared_ptr(new PipeLine("test")); - - std::vector mediaList = { - "/home/vivek/apra_test/Videos_to_test/2023-10-31/D1/P1/2023-10-31_12-42-39-913.mp4" - }; - - auto stream = cudastream_sp(new ApraCudaStream); - auto mp4ReaderProps = Mp4ReaderSourceProps(mediaList[0], false); - boost::shared_ptr m_reviewSource = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); - m_reviewSource->registerCallback(myCallbackFunction); - auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); - m_reviewSource->addOutPutPin(h264ImageMetadata); - - auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); - m_reviewSource->addOutPutPin(mp4Metadata); - - auto decoderProps = H264DecoderProps(); - boost::shared_ptr m_h264Decode = boost::shared_ptr(new H264Decoder(decoderProps)); - std::vector mImagePin; - mImagePin = m_reviewSource->getAllOutputPinsByType(FrameMetadata::FrameType::H264_DATA); - m_reviewSource->setNext(m_h264Decode, mImagePin); - - auto nv_transform = boost::shared_ptr(new NvTransform(NvTransformProps(ImageMetadata::RGBA))); // DMA - m_h264Decode->setNext(nv_transform); - - AffineTransformProps affineProps(AffineTransformProps::CUBIC, stream, 0, 4096, 0, 0, 1.0f); - affineProps.qlen = 1; - affineProps.quePushStrategyType = QuePushStrategy::NON_BLOCKING_ANY; - auto affine = boost::shared_ptr(new AffineTransform(affineProps)); - nv_transform->setNext(affine); - - EglRendererProps eglProps(0, 0, 1000, 1000); - boost::shared_ptr m_review_renderer = boost::shared_ptr(new EglRenderer(eglProps)); - affine->setNext(m_review_renderer); - - m_playbackPipeline->appendModule(m_reviewSource); - m_playbackPipeline->init(); - m_playbackPipeline->run_all_threaded(); - - // while (true) - // { - - // LOG_ERROR << "TOTAL NUMBER OF FILE TO PLAY" << mediaList.size(); - // // for (int i = 0; i < mediaList.size(); i++) - // // { - // // LOG_ERROR << "<========================== PLAYING =================================>>>>>>>>>>>>>>>>>>>>" << mediaList[i]; - // // auto currMediaProps = m_reviewSource->getProps(); - // // currMediaProps.videoPath = mediaList[i]; //"/home/developer/workspace/ApraPipes/1684824632.mp4"; - // // m_reviewSource->setProps(currMediaProps); - // // // m_review_renderer->createWindow(1000, 1000); - // // boost::this_thread::sleep_for(boost::chrono::seconds(1)); - // // auto affineProps = affine->getProps(); - // // affineProps.angle =90; - // // affine->setProps(affineProps); - // // boost::this_thread::sleep_for(boost::chrono::seconds(1)); - // // affineProps = affine->getProps(); - // // affineProps.angle =180; - // // affine->setProps(affineProps); - // // boost::this_thread::sleep_for(boost::chrono::seconds(1)); - // // affineProps = affine->getProps(); - // // affineProps.angle =270; - // // affine->setProps(affineProps); - // // boost::this_thread::sleep_for(boost::chrono::seconds(1)); - // // affineProps = affine->getProps(); - // // affineProps.angle =0; - // // affine->setProps(affineProps); - // // boost::this_thread::sleep_for(boost::chrono::seconds(1)); - // // // m_review_renderer->closeWindow(); - // // // boost::this_thread::sleep_for(boost::chrono::seconds(1)); - // // } - // } - - Test_Utils::sleep_for_seconds(15000); -} - -// BOOST_AUTO_TEST_CASE(mp4reader_decoder_eglrenderer,* boost::unit_test::disabled()) -// { -// Logger::setLogLevel("info"); - -// // metadata is known -// std::string videoPath = "/home/developer/mp4_data/newatl.mp4"; -// auto mp4ReaderProps = Mp4ReaderSourceProps(videoPath, false); -// auto mp4Reader = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); -// auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); -// mp4Reader->addOutPutPin(h264ImageMetadata); - -// auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); -// mp4Reader->addOutPutPin(mp4Metadata); - -// auto Decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); -// std::vector mImagePin; -// mImagePin = mp4Reader->getAllOutputPinsByType(FrameMetadata::FrameType::H264_DATA); -// mp4Reader->setNext(Decoder, mImagePin); - -// auto nv_transform = boost::shared_ptr(new NvTransform(NvTransformProps(ImageMetadata::RGBA))); // DMA -// Decoder->setNext(nv_transform); - -// auto stream = cudastream_sp(new ApraCudaStream); -// AffineTransformProps affineProps(AffineTransformProps::CUBIC, stream, 10, 45, 0, 1.0f); -// affineProps.qlen = 1; -// affineProps.quePushStrategyType = QuePushStrategy::NON_BLOCKING_ANY; -// auto affine = boost::shared_ptr(new AffineTransform(affineProps)); -// nv_transform->setNext(affine); - -// auto sync = boost::shared_ptr(new CudaStreamSynchronize(CudaStreamSynchronizeProps(stream))); -// affine->setNext(sync); - -// auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0, 0))); -// sync->setNext(sink); - -// boost::shared_ptr p; -// p = boost::shared_ptr(new PipeLine("test")); -// p->appendModule(mp4Reader); - -// if (!p->init()) -// { -// throw AIPException(AIP_FATAL, "Engine Pipeline init failed. Check IPEngine Logs for more details."); -// } - -// p->run_all_threaded(); - -// Test_Utils::sleep_for_seconds(15000); - -// // p->stop(); -// // p->term(); -// // p->wait_for_all(); -// // p.reset(); -// } - -BOOST_AUTO_TEST_CASE(mp4reader_decoder_eglrenderer,* boost::unit_test::disabled()) +BOOST_AUTO_TEST_CASE(mp4reader_decoder_eglrenderer) { Logger::setLogLevel("info"); - auto stream = cudastream_sp(new ApraCudaStream); + // metadata is known - std::string videoPath = "/home/developer/workspace/ApraPipes/1684824632.mp4"; - // std::string videoPath = "/media/developer/7C3B-7A0B/2023-07-11/DOCTOR/PATIENT/2023-07-11_16-14-50-880.mp4"; - auto mp4ReaderProps = Mp4ReaderSourceProps(videoPath, false); + std::string videoPath = "/home/developer/ApraPipes/data/Mp4_videos/h264_video/20221010/0012/1668063524439.mp4"; + auto mp4ReaderProps = Mp4ReaderSourceProps(videoPath, false, 0, true, false, false); auto mp4Reader = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); mp4Reader->addOutPutPin(h264ImageMetadata); @@ -408,72 +74,47 @@ BOOST_AUTO_TEST_CASE(mp4reader_decoder_eglrenderer,* boost::unit_test::disabled( auto mp4Metadata = framemetadata_sp(new Mp4VideoMetadata("v_1")); mp4Reader->addOutPutPin(mp4Metadata); - // StatSinkProps sinkProps; - // sinkProps.logHealth = true; - // sinkProps.logHealthFrequency = 100; - // auto sink2 = boost::shared_ptr(new StatSink(sinkProps)); - // mp4Reader->setNext(sink2); - - auto Decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); + auto Decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); std::vector mImagePin; mImagePin = mp4Reader->getAllOutputPinsByType(FrameMetadata::FrameType::H264_DATA); mp4Reader->setNext(Decoder, mImagePin); - //Adding transform - - auto nv_transform = boost::shared_ptr(new NvTransform(NvTransformProps(ImageMetadata::RGBA))); // DMA - Decoder->setNext(nv_transform); - - // AffineTransformProps affineProps(AffineTransformProps::CUBIC, stream, 15 ,4096, 0, 0, 1); - // affineProps.qlen = 1; - // affineProps.quePushStrategyType = QuePushStrategy::NON_BLOCKING_ANY; - // auto m_reviewAffineTransform = boost::shared_ptr(new AffineTransform(affineProps)); - // nv_transform->setNext(m_reviewAffineTransform); + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + Decoder->setNext(memconversion2); + + auto fileWriter = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/ggg/yash_frame.raw",true))); + memconversion2->setNext(fileWriter); - auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0,0))); - nv_transform->setNext(sink); boost::shared_ptr p; p = boost::shared_ptr(new PipeLine("test")); p->appendModule(mp4Reader); - if (!p->init()) { throw AIPException(AIP_FATAL, "Engine Pipeline init failed. Check IPEngine Logs for more details."); } p->run_all_threaded(); - boost::this_thread::sleep_for(boost::chrono::seconds(2)); - LOG_ERROR << "Play @nd Video"; - - - // auto currMediaProps = mp4Reader->getProps(); - // currMediaProps.videoPath = "/home/developer/workspace/ApraPipes/1684824653.mp4"; - // mp4Reader->setProps(currMediaProps); - // boost::this_thread::sleep_for(boost::chrono::seconds(20)); + Test_Utils::sleep_for_seconds(15); - auto currMediaProps = mp4Reader->getProps(); - currMediaProps.videoPath = "/home/developer/workspace/ApraPipes/1684824632.mp4";//"/home/developer/workspace/ApraPipes/1684824632.mp4"; - mp4Reader->setProps(currMediaProps); - boost::this_thread::sleep_for(boost::chrono::seconds(2)); - // Decoder->decoderEos(); - // mp4Reader->closeOpenFile(); p->stop(); p->term(); p->wait_for_all(); p.reset(); } -BOOST_AUTO_TEST_CASE(mp4reader_decoder_extsink) +/* +BOOST_AUTO_TEST_CASE(sample_mp4_file_decoder_cuda_device_to_host) { Logger::setLogLevel("info"); // metadata is known - std::string videoPath = "/home/developer/mp4_data/newatl.mp4"; - auto mp4ReaderProps = Mp4ReaderSourceProps(videoPath, false); + std::string videoPath = "./data/Mp4_videos/h264_video/20221010/0012/1668064027062.mp4"; + auto mp4ReaderProps = Mp4ReaderSourceProps(videoPath, false, 0, true, false, false); auto mp4Reader = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps)); auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); mp4Reader->addOutPutPin(h264ImageMetadata); @@ -486,52 +127,33 @@ BOOST_AUTO_TEST_CASE(mp4reader_decoder_extsink) mImagePin = mp4Reader->getAllOutputPinsByType(FrameMetadata::FrameType::H264_DATA); mp4Reader->setNext(Decoder, mImagePin); - auto m3 = boost::shared_ptr(new ExternalSinkModule()); - Decoder->setNext(m3); - - boost::shared_ptr p; - p = boost::shared_ptr(new PipeLine("test")); - p->appendModule(mp4Reader); - - if (!p->init()) - { - throw AIPException(AIP_FATAL, "Engine Pipeline init failed. Check IPEngine Logs for more details."); - } - - p->run_all_threaded(); - - Test_Utils::sleep_for_seconds(15); + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + Decoder->setNext(memconversion2); - p->stop(); - p->term(); - p->wait_for_all(); - p.reset(); -} + auto hostToDevice = boost::shared_ptr(new CudaMemCopy(CudaMemCopyProps(cudaMemcpyKind::cudaMemcpyHostToDevice, stream))); + memconversion2->setNext(hostToDevice); -#else -BOOST_AUTO_TEST_CASE(h264_to_yuv420) -{ - Logger::setLogLevel("info"); - // metadata is known - auto props = FileReaderModuleProps("./data/h264_data/FVDO_Freeway_4cif_???.H264", 0, -1); - props.readLoop = false; - auto fileReader = boost::shared_ptr(new FileReaderModule(props)); + auto resizeNPPI = boost::shared_ptr(new ResizeNPPI(ResizeNPPIProps(320, 180, stream))); + hostToDevice->setNext(resizeNPPI); - auto h264ImageMetadata = framemetadata_sp(new H264Metadata(0, 0)); - auto rawImagePin = fileReader->addOutputPin(h264ImageMetadata); - - auto Decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); - fileReader->setNext(Decoder); + auto deviceToHost = boost::shared_ptr(new CudaMemCopy(CudaMemCopyProps(cudaMemcpyKind::cudaMemcpyDeviceToHost, stream))); + resizeNPPI->setNext(deviceToHost); + + auto jpegEncoder = boost::shared_ptr(new JPEGEncoderL4TM()); + auto encodedImageMetadata = framemetadata_sp(new FrameMetadata(FrameMetadata::ENCODED_IMAGE)); + auto encodedImagePin = jpegEncoder->addOutputPin(encodedImageMetadata); + deviceToHost->setNext(jpegEncoder); + + auto fileWriter = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/ULT_TEST/frame_????.jpg"))); + jpegEncoder->setNext(fileWriter); - auto fileWriter = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/yuv420Frames/Yuv420_704x576????.raw"))); - Decoder->setNext(fileWriter); - fileReader->play(true); boost::shared_ptr p; p = boost::shared_ptr(new PipeLine("test")); - p->appendModule(fileReader); + p->appendModule(mp4Reader); if (!p->init()) { @@ -540,132 +162,13 @@ BOOST_AUTO_TEST_CASE(h264_to_yuv420) p->run_all_threaded(); - Test_Utils::sleep_for_seconds(6); - - p->stop(); - p->term(); - p->wait_for_all(); - p.reset(); - -} - -BOOST_AUTO_TEST_CASE(encoder_to_decoder) -{ - Logger::setLogLevel("info"); - auto cuContext = apracucontext_sp(new ApraCUcontext()); - - auto width = 640; - auto height = 360; - uint32_t gopLength = 25; - uint32_t bitRateKbps = 1000; - uint32_t frameRate = 30; - H264EncoderNVCodecProps::H264CodecProfile profile = H264EncoderNVCodecProps::BASELINE; - bool enableBFrames = true; - - auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/Raw_YUV420_640x360/Image???_YUV420.raw", 0, -1))); - auto metadata = framemetadata_sp(new RawImagePlanarMetadata(width, height, ImageMetadata::ImageType::YUV420, size_t(0), CV_8U)); - - fileReader->addOutputPin(metadata); - - auto cudaStream_ = boost::shared_ptr(new ApraCudaStream()); - auto copyProps = CudaMemCopyProps(cudaMemcpyKind::cudaMemcpyHostToDevice, cudaStream_); - copyProps.sync = true; - auto copy = boost::shared_ptr(new CudaMemCopy(copyProps)); - fileReader->setNext(copy); - - auto encoder = boost::shared_ptr(new H264EncoderNVCodec(H264EncoderNVCodecProps(bitRateKbps, cuContext, gopLength, frameRate, profile, enableBFrames))); - copy->setNext(encoder); - - auto Decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); - encoder->setNext(Decoder); - - auto m2 = boost::shared_ptr(new ExternalSinkModule()); - Decoder->setNext(m2); - - fileReader->play(true); - - BOOST_TEST(fileReader->init()); - BOOST_TEST(copy->init()); - BOOST_TEST(encoder->init()); - BOOST_TEST(Decoder->init()); - BOOST_TEST(m2->init()); - - int index = 0; - for (auto i = 0; i <= 43; i++) - { - - fileReader->step(); - copy->step(); - encoder->step(); - Decoder->step(); - - if (i >= 3) - { - auto frames = m2->pop(); - BOOST_TEST(frames.size() == 1); - auto outputFrame = frames.cbegin()->second; - BOOST_TEST(outputFrame->getMetadata()->getFrameType() == FrameMetadata::RAW_IMAGE_PLANAR); - - std::string fileName; - - if (index <= 9) - { - fileName = "/data/Raw_YUV420_640x360/Image00" + std::to_string(index) + "_YUV420.raw"; - } - else - { - fileName = "/data/Raw_YUV420_640x360/Image0" + std::to_string(index) + "_YUV420.raw"; - } - - Test_Utils::saveOrCompare(fileName.c_str(), const_cast(static_cast(outputFrame->data())), outputFrame->size(), 0); - index++; - } - } -} - -BOOST_AUTO_TEST_CASE(mp4reader_to_decoder_extSink) -{ - Logger::setLogLevel("info"); - - std::string startingVideoPath_2 = "./data/Mp4_videos/h264_video/20221010/0012/1668064027062.mp4"; - auto mp4ReaderProps_2 = Mp4ReaderSourceProps(startingVideoPath_2, false); - mp4ReaderProps_2.logHealth = true; - mp4ReaderProps_2.logHealthFrequency = 100; - mp4ReaderProps_2.fps = 30; - auto mp4Reader_2 = boost::shared_ptr(new Mp4ReaderSource(mp4ReaderProps_2)); - auto h264ImageMetadata_2 = framemetadata_sp(new H264Metadata(0, 0)); - mp4Reader_2->addOutPutPin(h264ImageMetadata_2); - auto mp4Metadata_2 = framemetadata_sp(new Mp4VideoMetadata("v_1")); - mp4Reader_2->addOutPutPin(mp4Metadata_2); - // metadata is known - - auto Decoder = boost::shared_ptr(new H264Decoder(H264DecoderProps())); - mp4Reader_2->setNext(Decoder); - - StatSinkProps sinkProps; - sinkProps.logHealth = true; - sinkProps.logHealthFrequency = 100; - auto sink = boost::shared_ptr(new StatSink(sinkProps)); - Decoder->setNext(sink); - - boost::shared_ptr p; - p = boost::shared_ptr(new PipeLine("test")); - - p->appendModule(mp4Reader_2); - - if (!p->init()) - { - throw AIPException(AIP_FATAL, "Engine Pipeline init failed. Check IPEngine Logs for more details."); - } + Test_Utils::sleep_for_seconds(15); - p->run_all_threaded(); - Test_Utils::sleep_for_seconds(10); p->stop(); p->term(); p->wait_for_all(); p.reset(); } - -#endif +*/ BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/base/test/h264encoderv4l2_tests.cpp b/base/test/h264encoderv4l2_tests.cpp index dceec25f7..51fd1bafa 100755 --- a/base/test/h264encoderv4l2_tests.cpp +++ b/base/test/h264encoderv4l2_tests.cpp @@ -15,47 +15,48 @@ BOOST_AUTO_TEST_SUITE(h264encoderv4l2_tests) -BOOST_AUTO_TEST_CASE(yuv420_640x360) +BOOST_AUTO_TEST_CASE(yuv420_640x360, *boost::unit_test::disabled()) { - // metadata is known - auto width = 640; - auto height = 360; + // metadata is known + auto width = 640; + auto height = 360; - auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/Raw_YUV420_640x360/Image???_YUV420.raw"))); - auto metadata = framemetadata_sp(new RawImagePlanarMetadata(width, height, ImageMetadata::ImageType::YUV420, size_t(0), CV_8U)); - auto rawImagePin = fileReader->addOutputPin(metadata); + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("../data/Raw_YUV420_640x360/Image020_YUV420.raw"))); + auto metadata = framemetadata_sp(new RawImagePlanarMetadata(width, height, ImageMetadata::ImageType::YUV420, size_t(0), CV_8U)); + auto rawImagePin = fileReader->addOutputPin(metadata); - H264EncoderV4L2Props encoderProps; - encoderProps.targetKbps = 1024; - auto encoder = boost::shared_ptr(new H264EncoderV4L2(encoderProps)); - fileReader->setNext(encoder); + H264EncoderV4L2Props encoderProps; + encoderProps.targetKbps = 1024; + auto encoder = boost::shared_ptr(new H264EncoderV4L2(encoderProps)); + fileReader->setNext(encoder); - auto fileWriter = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/Raw_YUV420_640x360.h264", true))); - encoder->setNext(fileWriter); + auto fileWriter = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/Raw_YUV420_640x360.h264", true))); + encoder->setNext(fileWriter); - BOOST_TEST(fileReader->init()); - BOOST_TEST(encoder->init()); - BOOST_TEST(fileWriter->init()); + BOOST_TEST(fileReader->init()); + BOOST_TEST(encoder->init()); + BOOST_TEST(fileWriter->init()); - PipeLine p("test"); - p.appendModule(fileReader); - BOOST_TEST(p.init()); - p.run_all_threaded(); - boost::this_thread::sleep_for(boost::chrono::seconds(UINT32_MAX)); - p.stop(); - p.term(); - p.wait_for_all(); + fileReader->play(true); - // Test_Utils::saveOrCompare("./data/testOutput/Raw_YUV420_640x360.h264", 0); + for (auto i = 0; i < 42; i++) + { + fileReader->step(); + encoder->step(); + fileWriter->step(); + } + + Test_Utils::saveOrCompare("./data/testOutput/Raw_YUV420_640x360.h264", 0); } + BOOST_AUTO_TEST_CASE(rgb24_1280x720, *boost::unit_test::disabled()) { // metadata is known auto width = 1280; auto height = 720; - auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/Raw_RGB24_1280x720"))); + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("../data/frame_1280x720_rgb.raw"))); auto metadata = framemetadata_sp(new RawImageMetadata(width, height, ImageMetadata::ImageType::RGB, CV_8UC3, size_t(0), CV_8U, FrameMetadata::HOST, true)); auto rawImagePin = fileReader->addOutputPin(metadata); diff --git a/base/test/h264encoderv4l2helper_tests.cpp b/base/test/h264encoderv4l2helper_tests.cpp index 8c9b248f9..d6b2698d3 100755 --- a/base/test/h264encoderv4l2helper_tests.cpp +++ b/base/test/h264encoderv4l2helper_tests.cpp @@ -8,7 +8,7 @@ #include "test_utils.h" -#include "nvbuf_utils.h" +//#include "nvbuf_utils.h" BOOST_AUTO_TEST_SUITE(h264encoderv4l2helper_tests) diff --git a/base/test/memtypeconversion_tests.cpp b/base/test/memtypeconversion_tests.cpp index c7450c16b..a29aec048 100644 --- a/base/test/memtypeconversion_tests.cpp +++ b/base/test/memtypeconversion_tests.cpp @@ -8,38 +8,259 @@ #include "FileWriterModule.h" #include "test_utils.h" #include "ExternalSinkModule.h" -#include "ImageDecoderCV.h" -#include "FileWriterModule.h" -#include "JPEGDecoderL4TM.h" -#if defined(__arm__) || defined(__aarch64__) +#define ENABLE_ARM64 +#if 1 +#include "NvV4L2Camera.h" #include "NvTransform.h" #include "EglRenderer.h" #endif BOOST_AUTO_TEST_SUITE(memtypeconversion_tests) +BOOST_AUTO_TEST_CASE(Host_to_Dma_to_Device_to_Host_RGBA_1280x720) +{ +#if 1 + LoggerProps logProps; + logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::severity_level::trace); + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("/home/developer/ApraPipes/data/8bit_frame_1280x720_rgba.raw"))); + auto metadata = framemetadata_sp(new RawImageMetadata(1280, 720, ImageMetadata::ImageType::RGBA, CV_8UC4, 0, CV_8U, FrameMetadata::HOST, true)); + fileReader->addOutputPin(metadata); + + auto memconversion1 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::DMABUF))); + fileReader->setNext(memconversion1); + + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + memconversion1->setNext(memconversion2); + + auto memconversion3 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + memconversion2->setNext(memconversion3); + + auto sink = boost::shared_ptr(new ExternalSinkModule()); + memconversion3->setNext(sink); + + BOOST_TEST(fileReader->init()); + BOOST_TEST(memconversion1->init()); + BOOST_TEST(memconversion2->init()); + BOOST_TEST(memconversion3->init()); + BOOST_TEST(sink->init()); + + fileReader->step(); + memconversion1->step(); + memconversion2->step(); + memconversion3->step(); + + auto outputPinId = memconversion3->getAllOutputPinsByType(FrameMetadata::RAW_IMAGE)[0]; + auto frames = sink->pop(); + BOOST_TEST((frames.find(outputPinId) != frames.end())); + auto outFrame = frames[outputPinId]; + BOOST_TEST(outFrame->getMetadata()->getFrameType() == FrameMetadata::RAW_IMAGE); + Test_Utils::saveOrCompare("./data/MemConversion_outputs/Host_to_Dma_to_Device_to_Host_RGBA_1280x720.raw", (const uint8_t *)outFrame->data(), outFrame->size(), 0); +#endif +} + +BOOST_AUTO_TEST_CASE(Host_to_Device_to_Dma_to_Device_to_Host_YUV420_400x400) +{ +#if 1 + LoggerProps logProps; + logProps.enableConsoleLog = true; + Logger::initLogger(logProps); + Logger::setLogLevel(boost::log::trivial::severity_level::trace); + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/yuv420_400x400.raw"))); + auto metadata = framemetadata_sp(new RawImagePlanarMetadata(400, 400, ImageMetadata::ImageType::YUV420, size_t(0), CV_8U)); + fileReader->addOutputPin(metadata); + + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion1 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + fileReader->setNext(memconversion1); + + auto memconversion2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::DMABUF, stream))); + memconversion1->setNext(memconversion2); + + auto memconversion3 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + memconversion2->setNext(memconversion3); + + auto memconversion4 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + memconversion3->setNext(memconversion4); + + auto sink = boost::shared_ptr(new ExternalSinkModule()); + memconversion4->setNext(sink); + + BOOST_TEST(fileReader->init()); + BOOST_TEST(memconversion1->init()); + BOOST_TEST(memconversion2->init()); + BOOST_TEST(memconversion3->init()); + BOOST_TEST(memconversion4->init()); + BOOST_TEST(sink->init()); + + fileReader->step(); + memconversion1->step(); + memconversion2->step(); + memconversion3->step(); + memconversion4->step(); + + auto outputPinId = memconversion4->getAllOutputPinsByType(FrameMetadata::RAW_IMAGE_PLANAR)[0]; + auto frames = sink->pop(); + BOOST_TEST((frames.find(outputPinId) != frames.end())); + auto outFrame = frames[outputPinId]; + BOOST_TEST(outFrame->getMetadata()->getFrameType() == FrameMetadata::RAW_IMAGE_PLANAR); + Test_Utils::saveOrCompare("./data/MemConversion_outputs/Host_to_Device_to_Dma_to_Device_to_Host_YUV420_400x400.raw", (const uint8_t *)outFrame->data(), outFrame->size(), 0); +#endif +} + +BOOST_AUTO_TEST_CASE(Host_to_Device_to_Dma_to_Host_BGRA_400x400) +{ +#if 1 + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/overlay_400x400_BGRA.raw"))); + auto metadata = framemetadata_sp(new RawImageMetadata(400, 400, ImageMetadata::ImageType::BGRA, CV_8UC4, 0, CV_8U, FrameMetadata::HOST, true)); + fileReader->addOutputPin(metadata); + + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion1 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + fileReader->setNext(memconversion1); + + auto memconversion2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::DMABUF, stream))); + memconversion1->setNext(memconversion2); + + auto memconversion3 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST))); + memconversion2->setNext(memconversion3); + + auto sink = boost::shared_ptr(new ExternalSinkModule()); + memconversion3->setNext(sink); + + BOOST_TEST(fileReader->init()); + BOOST_TEST(memconversion1->init()); + BOOST_TEST(memconversion2->init()); + BOOST_TEST(memconversion3->init()); + BOOST_TEST(sink->init()); + + fileReader->step(); + memconversion1->step(); + memconversion2->step(); + memconversion3->step(); + + auto outputPinId = memconversion3->getAllOutputPinsByType(FrameMetadata::RAW_IMAGE)[0]; + auto frames = sink->pop(); + BOOST_TEST((frames.find(outputPinId) != frames.end())); + auto outFrame = frames[outputPinId]; + BOOST_TEST(outFrame->getMetadata()->getFrameType() == FrameMetadata::RAW_IMAGE); + Test_Utils::saveOrCompare("./data/MemConversion_outputs/Host_to_Device_to_Dma_to_Host_BGRA_400x400.raw", (const uint8_t *)outFrame->data(), outFrame->size(), 0); +#endif +} + +BOOST_AUTO_TEST_CASE(Dma_to_Host, *boost::unit_test::disabled()) +{ +#if 1 + NvV4L2CameraProps nvCamProps(640, 360, 10,false); + auto source = boost::shared_ptr(new NvV4L2Camera(nvCamProps)); + + auto transform = boost::shared_ptr(new NvTransform(ImageMetadata::RGBA)); + source->setNext(transform); + + auto memconversion = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST))); + transform->setNext(memconversion); + + auto sink = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/nvv4l2/frame_????.raw"))); + memconversion->setNext(sink); + + PipeLine p("test"); + p.appendModule(source); + BOOST_TEST(p.init()); + + Logger::setLogLevel(boost::log::trivial::severity_level::info); + + p.run_all_threaded(); + + boost::this_thread::sleep_for(boost::chrono::seconds(120)); + Logger::setLogLevel(boost::log::trivial::severity_level::error); + + p.stop(); + p.term(); + p.wait_for_all(); +#endif +} + +BOOST_AUTO_TEST_CASE(Dma_to_Host_to_Dma, *boost::unit_test::disabled()) +{ +#if 1 + NvV4L2CameraProps nvCamProps(640, 360, 10, false); + auto source = boost::shared_ptr(new NvV4L2Camera(nvCamProps)); + + auto transform = boost::shared_ptr(new NvTransform(ImageMetadata::RGBA)); + source->setNext(transform); + + auto memconversion = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST))); + transform->setNext(memconversion); + + auto memconversion2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::DMABUF))); + memconversion->setNext(memconversion2); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0, 0))); + memconversion2->setNext(sink); + + PipeLine p("test"); + p.appendModule(source); + BOOST_TEST(p.init()); + + Logger::setLogLevel(boost::log::trivial::severity_level::info); + + p.run_all_threaded(); + + boost::this_thread::sleep_for(boost::chrono::seconds(120)); + Logger::setLogLevel(boost::log::trivial::severity_level::error); + + p.stop(); + p.term(); + p.wait_for_all(); +#endif +} + BOOST_AUTO_TEST_CASE(Device_to_Dma_RGBA, *boost::unit_test::disabled()) { -#if defined(__arm__) || defined(__aarch64__) - FileReaderModuleProps fileReaderProps("/media/developer/7979-7B01/2024-01-24/D1/P1/2024-01-24_16-37-49-137.jpeg"); - fileReaderProps.readLoop = true; - auto fileReader = boost::shared_ptr(new FileReaderModule(fileReaderProps)); - auto metadata = framemetadata_sp(new FrameMetadata(FrameMetadata::ENCODED_IMAGE)); +#if 1 + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/8bit_frame_1280x720_rgba.raw"))); + auto metadata = framemetadata_sp(new RawImageMetadata(1280, 720, ImageMetadata::ImageType::RGBA, CV_8UC4, 0, CV_8U, FrameMetadata::HOST, true)); fileReader->addOutputPin(metadata); - auto decoder = boost::shared_ptr(new JPEGDecoderL4TM()); - fileReader->setNext(decoder); - auto rawImageMetadata = framemetadata_sp(new RawImageMetadata()); - auto rawImagePin = decoder->addOutputPin(rawImageMetadata); + auto stream = cudastream_sp(new ApraCudaStream); + auto copy1 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + fileReader->setNext(copy1); + + auto memconversion = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::DMABUF, stream))); + copy1->setNext(memconversion); + + auto sink = boost::shared_ptr(new EglRenderer(EglRendererProps(0, 0))); + memconversion->setNext(sink); + + PipeLine p("test"); + p.appendModule(fileReader); + BOOST_TEST(p.init()); + + Logger::setLogLevel(boost::log::trivial::severity_level::info); + + p.run_all_threaded(); + boost::this_thread::sleep_for(boost::chrono::seconds(20)); + Logger::setLogLevel(boost::log::trivial::severity_level::error); - // auto fileWriter = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/reader/frame_????.raw"))); - // decoder->setNext(fileWriter); + p.stop(); + p.term(); + p.wait_for_all(); +#endif +} +BOOST_AUTO_TEST_CASE(Device_to_Dma_Planar, *boost::unit_test::disabled()) +{ +#if 1 + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/yuv420_400x400.raw"))); + auto metadata = framemetadata_sp(new RawImagePlanarMetadata(400, 400, ImageMetadata::ImageType::YUV420, size_t(0), CV_8U)); + fileReader->addOutputPin(metadata); auto stream = cudastream_sp(new ApraCudaStream); auto copy1 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); - decoder->setNext(copy1); + fileReader->setNext(copy1); auto memconversion = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::DMABUF, stream))); copy1->setNext(memconversion); @@ -64,4 +285,139 @@ BOOST_AUTO_TEST_CASE(Device_to_Dma_RGBA, *boost::unit_test::disabled()) #endif } -BOOST_AUTO_TEST_SUITE_END() +BOOST_AUTO_TEST_CASE(Dma_to_Device_Planar, *boost::unit_test::disabled()) +{ +#if 1 + NvV4L2CameraProps nvCamProps(640, 360, 10,false); + auto source = boost::shared_ptr(new NvV4L2Camera(nvCamProps)); + + auto transform = boost::shared_ptr(new NvTransform(ImageMetadata::NV12)); + source->setNext(transform); + + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + transform->setNext(memconversion); + + auto copy2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + memconversion->setNext(copy2); + + auto sink = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/nvv4l2/frame_????.raw"))); + copy2->setNext(sink); + + PipeLine p("test"); + p.appendModule(source); + BOOST_TEST(p.init()); + + Logger::setLogLevel(boost::log::trivial::severity_level::info); + + p.run_all_threaded(); + + boost::this_thread::sleep_for(boost::chrono::seconds(20)); + Logger::setLogLevel(boost::log::trivial::severity_level::error); + + p.stop(); + p.term(); + p.wait_for_all(); +#endif +} + +BOOST_AUTO_TEST_CASE(Dma_to_Device, *boost::unit_test::disabled()) +{ +#if 1 + NvV4L2CameraProps nvCamProps(640, 360, 10,false); + auto source = boost::shared_ptr(new NvV4L2Camera(nvCamProps)); + + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + source->setNext(memconversion); + + auto copy2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + memconversion->setNext(copy2); + + auto sink = boost::shared_ptr(new FileWriterModule(FileWriterModuleProps("./data/testOutput/nvv4l2/frame_????.raw"))); + copy2->setNext(sink); + + PipeLine p("test"); + p.appendModule(source); + BOOST_TEST(p.init()); + + Logger::setLogLevel(boost::log::trivial::severity_level::info); + + p.run_all_threaded(); + + boost::this_thread::sleep_for(boost::chrono::seconds(20)); + Logger::setLogLevel(boost::log::trivial::severity_level::error); + + p.stop(); + p.term(); + p.wait_for_all(); +#endif +} + +BOOST_AUTO_TEST_CASE(Host_to_Device_to_Host) +{ + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/RGB_320x180.raw"))); + auto metadata = framemetadata_sp(new RawImageMetadata(320, 180, ImageMetadata::ImageType::RGB, CV_8UC3, 0, CV_8U, FrameMetadata::HOST, true)); + fileReader->addOutputPin(metadata); + + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion1 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + fileReader->setNext(memconversion1); + + auto memconversion2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + memconversion1->setNext(memconversion2); + + auto sink = boost::shared_ptr(new ExternalSinkModule()); + memconversion2->setNext(sink); + + BOOST_TEST(fileReader->init()); + BOOST_TEST(memconversion1->init()); + BOOST_TEST(memconversion2->init()); + BOOST_TEST(sink->init()); + + fileReader->step(); + memconversion1->step(); + memconversion2->step(); + + auto outputPinId = memconversion2->getAllOutputPinsByType(FrameMetadata::RAW_IMAGE)[0]; + auto frames = sink->pop(); + BOOST_TEST((frames.find(outputPinId) != frames.end())); + auto outFrame = frames[outputPinId]; + BOOST_TEST(outFrame->getMetadata()->getFrameType() == FrameMetadata::RAW_IMAGE); + Test_Utils::saveOrCompare("./data/MemConversion_outputs/Host_to_Device_to_Host_RGB_320x180.raw", (const uint8_t *)outFrame->data(), outFrame->size(), 0); +} + +BOOST_AUTO_TEST_CASE(Host_to_Device_to_Host_PlanarImage) +{ + auto fileReader = boost::shared_ptr(new FileReaderModule(FileReaderModuleProps("./data/nv12-704x576.raw"))); + auto metadata = framemetadata_sp(new RawImagePlanarMetadata(704, 576, ImageMetadata::ImageType::NV12, size_t(0), CV_8U)); + fileReader->addOutputPin(metadata); + + auto stream = cudastream_sp(new ApraCudaStream); + auto memconversion1 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::CUDA_DEVICE, stream))); + fileReader->setNext(memconversion1); + + auto memconversion2 = boost::shared_ptr(new MemTypeConversion(MemTypeConversionProps(FrameMetadata::HOST, stream))); + memconversion1->setNext(memconversion2); + + auto sink = boost::shared_ptr(new ExternalSinkModule()); + memconversion2->setNext(sink); + + BOOST_TEST(fileReader->init()); + BOOST_TEST(memconversion1->init()); + BOOST_TEST(memconversion2->init()); + BOOST_TEST(sink->init()); + + fileReader->step(); + memconversion1->step(); + memconversion2->step(); + + auto outputPinId = memconversion2->getAllOutputPinsByType(FrameMetadata::RAW_IMAGE_PLANAR)[0]; + auto frames = sink->pop(); + BOOST_TEST((frames.find(outputPinId) != frames.end())); + auto outFrame = frames[outputPinId]; + BOOST_TEST(outFrame->getMetadata()->getFrameType() == FrameMetadata::RAW_IMAGE_PLANAR); + Test_Utils::saveOrCompare("./data/MemConversion_outputs/Host_to_Device_to_Host_NV12.raw", (const uint8_t *)outFrame->data(), outFrame->size(), 0); +} + +BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file diff --git a/base/test/nvtransform_tests.cpp b/base/test/nvtransform_tests.cpp index bc301dddc..b5e708208 100644 --- a/base/test/nvtransform_tests.cpp +++ b/base/test/nvtransform_tests.cpp @@ -1,13 +1,12 @@ #include - #include "PipeLine.h" #include "NvV4L2Camera.h" #include "EglRenderer.h" -#include "nvbuf_utils.h" #include "EGL/egl.h" #include "cudaEGL.h" #include "NvTransform.h" - +#include "FileWriterModule.h" +#include "FileReaderModule.h" #include #include #include @@ -18,112 +17,272 @@ #include "RawImageMetadata.h" #include "DMAFDWrapper.h" #include "DMAUtils.h" - +#include #include +#include "nvbufsurftransform.h" +#include "ExternalSourceModule.h" +#include "ExternalSinkModule.h" +#include "ApraData.h" +#include "FrameFactory.h" using sys_clock = std::chrono::system_clock; -BOOST_AUTO_TEST_SUITE(nv_transform_tests, *boost::unit_test::disabled()) +class NvTransformTest : public NvTransform { +public: + using NvTransform::NvTransform; + using NvTransform::addInputPin; + using NvTransform::processSOS; + using NvTransform::processEOS; + + bool processFrame(frame_container &frames) { + return NvTransform::process(frames); + } +}; -BOOST_AUTO_TEST_CASE(basic, *boost::unit_test::disabled()) +frame_sp makeYUV420Frame(const std::string& path, uint32_t width, uint32_t height) { - Logger::setLogLevel(boost::log::trivial::severity_level::info); + size_t sizeY = width * height; + size_t sizeUV = sizeY >> 2; + size_t size = sizeY + 2 * sizeUV; + size_t step[4] = { 640, 320, 320, 0 }; + + auto metadata = framemetadata_sp(new RawImagePlanarMetadata( + width, height, ImageMetadata::YUV420, step, CV_8U, FrameMetadata::MemType::DMABUF + )); - NvV4L2CameraProps sourceProps(1920, 1080, 10); - auto source = boost::shared_ptr(new NvV4L2Camera(sourceProps)); + auto frameFactory = framefactory_sp(new FrameFactory(metadata, 1)); + auto frame = frameFactory->create(size, frameFactory); + if (!frame || !frame->data()) + throw std::runtime_error("Failed to create frame or DMA buffer"); - auto nv_transform = boost::shared_ptr(new NvTransform(NvTransformProps(ImageMetadata::RGBA))); - source->setNext(nv_transform); + auto dma = static_cast(frame->data()); + std::ifstream file(path, std::ios::binary); + if (!file.is_open()) + throw std::runtime_error("Cannot open YUV file: " + path); - PipeLine p("test"); - p.appendModule(source); - BOOST_TEST(p.init()); + const int y_w = static_cast(width); + const int y_h = static_cast(height); + const int u_w = y_w >> 1; + const int u_h = y_h >> 1; + const int v_w = u_w; + const int v_h = u_h; - p.run_all_threaded(); - boost::this_thread::sleep_for(boost::chrono::seconds(10)); - p.stop(); - p.term(); - p.wait_for_all(); + NvBufSurface* surf = dma->getNvBufSurface(); + const size_t y_pitch = surf->surfaceList[0].planeParams.pitch[0]; + const size_t u_pitch = surf->surfaceList[0].planeParams.pitch[1]; + const size_t v_pitch = surf->surfaceList[0].planeParams.pitch[2]; + + std::vector rowY(y_w); + std::vector rowU(u_w); + std::vector rowV(v_w); + + // Y plane + uint8_t* dstY = static_cast(dma->getHostPtrY()); + for (int r = 0; r < y_h; ++r) { + file.read(reinterpret_cast(rowY.data()), y_w); + if (file.gcount() != y_w) throw std::runtime_error("Failed to read Y row"); + memcpy(dstY + r * y_pitch, rowY.data(), y_w); + } + + // U plane + uint8_t* dstU = static_cast(dma->getHostPtrU()); + for (int r = 0; r < u_h; ++r) { + file.read(reinterpret_cast(rowU.data()), u_w); + if (file.gcount() != u_w) throw std::runtime_error("Failed to read U row"); + memcpy(dstU + r * u_pitch, rowU.data(), u_w); + } + + // V plane + uint8_t* dstV = static_cast(dma->getHostPtrV()); + for (int r = 0; r < v_h; ++r) { + file.read(reinterpret_cast(rowV.data()), v_w); + if (file.gcount() != v_w) throw std::runtime_error("Failed to read V row"); + memcpy(dstV + r * v_pitch, rowV.data(), v_w); + } + + return frame; } +BOOST_AUTO_TEST_SUITE(nv_transform_tests, *boost::unit_test::disabled()) + BOOST_AUTO_TEST_CASE(test) { - EGLDisplay eglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); - if(eglDisplay == EGL_NO_DISPLAY) - { - throw AIPException(AIP_FATAL, "eglGetDisplay failed"); - } - - if (!eglInitialize(eglDisplay, NULL, NULL)) - { - throw AIPException(AIP_FATAL, "eglInitialize failed"); - } - DMAFDWrapper* dmafdWrapper = DMAFDWrapper::create(0,1024,1024,NvBufferColorFormat_ABGR32,NvBufferLayout_Pitch,eglDisplay); - auto mapped = dmafdWrapper->getHostPtr(); - memset(mapped,255,1024*1024*4); - auto rgbSize = 10; - for(auto i = 0; i < rgbSize; i++) - { - cout << (int)*(static_cast(mapped) + i) << " "; - } - cout <getHostPtr(); + memset(mapped,255,1024*1024*4); + + auto rgbSize = 10; + for(auto i = 0; i < rgbSize; i++) + std::cout << (int)*(static_cast(mapped) + i) << " "; + std::cout << std::endl; +} + +BOOST_AUTO_TEST_CASE(yuv_dma_crop) +{ + constexpr int src_width = 3840; + constexpr int src_height = 2160; + constexpr int dst_width = 3840; + constexpr int dst_height = 2160; + + auto input_frame = makeYUV420Frame("/home/developer/ApraPipes/data/4k.yuv", src_width, src_height); + BOOST_REQUIRE(input_frame != nullptr); + + NvTransformProps props(ImageMetadata::YUV420,NvTransformProps::NvFlip::FlipY); + auto nv_transform = std::make_shared(props); + + std::string inputPinId = "input"; + framemetadata_sp metadata = input_frame->getMetadata(); + nv_transform->addInputPin(metadata, inputPinId); + + std::string outputPinId = "output"; + frame_container frames; + frames[inputPinId] = input_frame; + + BOOST_REQUIRE(nv_transform->init()); + nv_transform->processSOS(input_frame); + nv_transform->processFrame(frames); + nv_transform->processEOS(outputPinId); + nv_transform->term(); + + frame_sp out_frame; + for (const auto &kv : frames) { + if (kv.first != inputPinId) { + out_frame = kv.second; + break; + } + } + + BOOST_REQUIRE(out_frame != nullptr); + auto out_dma = static_cast(out_frame->data()); + auto out_md = FrameMetadataFactory::downcast(out_frame->getMetadata()); + + const int y_w = out_md->getWidth(0); + const int y_h = out_md->getHeight(0); + const size_t y_pitch = out_md->getStep(0); + const int u_w = out_md->getWidth(1); + const int u_h = out_md->getHeight(1); + const size_t u_pitch = out_md->getStep(1); + const int v_w = out_md->getWidth(2); + const int v_h = out_md->getHeight(2); + const size_t v_pitch = out_md->getStep(2); + + std::ofstream f_out("/home/developer/ApraPipes/data/4k_cropped.yuv", std::ios::binary); + + // Write Y plane + const uint8_t* srcY = static_cast(out_dma->getHostPtrY()); + for (int r = 0; r < y_h; ++r) + f_out.write(reinterpret_cast(srcY + r * y_pitch), y_w); + + // Write U plane + const uint8_t* srcU = static_cast(out_dma->getHostPtrU()); + for (int r = 0; r < u_h; ++r) + f_out.write(reinterpret_cast(srcU + r * u_pitch), u_w); + + // Write V plane + const uint8_t* srcV = static_cast(out_dma->getHostPtrV()); + for (int r = 0; r < v_h; ++r) + f_out.write(reinterpret_cast(srcV + r * v_pitch), v_w); } -BOOST_AUTO_TEST_CASE(fdtest) +BOOST_AUTO_TEST_CASE(yuv_dma_transform_variants) { - size_t size = 1024*1024*4; - void *host = malloc(size); - void *host_ref = malloc(size); - - cudaFree(0); - - EGLImageKHR eglInImage; - CUgraphicsResource pInResource; - CUeglFrame eglInFrame; - EGLDisplay eglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); - if(eglDisplay == EGL_NO_DISPLAY) - { - throw AIPException(AIP_FATAL, "eglGetDisplay failed"); - } - - if (!eglInitialize(eglDisplay, NULL, NULL)) - { - throw AIPException(AIP_FATAL, "eglInitialize failed"); - } - DMAFDWrapper* dmafdWrapper = DMAFDWrapper::create(0,1024,1024,NvBufferColorFormat_ABGR32,NvBufferLayout_Pitch,eglDisplay); - auto mapped = dmafdWrapper->getHostPtr(); - - - auto src = DMAUtils::getCudaPtrForFD(dmafdWrapper->getFd(), eglInImage,&pInResource,eglInFrame, eglDisplay); - - int value = 128; - for(int i = 0; i < 10; i++) - { - value += i; - memset(mapped, value, size); - NvBufferMemSyncForDevice(dmafdWrapper->getFd(), 0, &mapped); - cudaMemcpy(host, src, size, cudaMemcpyDeviceToHost); - cudaDeviceSynchronize(); - memset(host_ref, value, size); - if(memcmp(host, host_ref, size) != 0) - { - std::cout << "failed" << std::endl; - } - - value += 1; - cudaMemset(src, value, size); - cudaDeviceSynchronize(); - NvBufferMemSyncForCpu(dmafdWrapper->getFd(), 0, &mapped); - memset(host_ref, value, size); - if(memcmp(mapped, host_ref, size) != 0) - { - std::cout << "failed 2" << std::endl; - } - } - - - - DMAUtils::freeCudaPtr(eglInImage,&pInResource, eglDisplay); + constexpr int src_width = 3840; + constexpr int src_height = 2160; + + auto input_frame = makeYUV420Frame("/home/developer/ApraPipes/data/4k.yuv", src_width, src_height); + BOOST_REQUIRE(input_frame != nullptr); + + struct TestCase { + std::string name; + NvTransformProps props; + }; + + std::vector tests = { + { "default", NvTransformProps(ImageMetadata::YUV420) }, + { "crop_w_h", NvTransformProps(ImageMetadata::YUV420, 1920, 1080) }, + { "crop_full", NvTransformProps(ImageMetadata::YUV420, 1920, 1080, 100, 100) }, + { "rotate_90", NvTransformProps(ImageMetadata::YUV420, NvTransformProps::NvRotation::Rotate90) }, + { "rotate_180", NvTransformProps(ImageMetadata::YUV420, NvTransformProps::NvRotation::Rotate180) }, + { "flip_x", NvTransformProps(ImageMetadata::YUV420, NvTransformProps::NvFlip::FlipX) }, + { "flip_y", NvTransformProps(ImageMetadata::YUV420, NvTransformProps::NvFlip::FlipY) } + }; + + EGLDisplay eglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); + if(eglDisplay == EGL_NO_DISPLAY) throw AIPException(AIP_FATAL, "eglGetDisplay failed"); + if(!eglInitialize(eglDisplay, NULL, NULL)) throw AIPException(AIP_FATAL, "eglInitialize failed"); + + int index = 0; + for (auto &test : tests) + { + std::cout << "\nRunning NvTransform variant: " << test.name << std::endl; + + auto nv_transform = std::make_shared(test.props); + + std::string inputPinId = "input" + std::to_string(index); + framemetadata_sp metadata = input_frame->getMetadata(); + nv_transform->addInputPin(metadata, inputPinId); + + frame_container frames; + frames[inputPinId] = input_frame; + + BOOST_REQUIRE(nv_transform->init()); + nv_transform->processSOS(input_frame); + nv_transform->processFrame(frames); + nv_transform->processEOS(inputPinId); + nv_transform->term(); + + frame_sp out_frame; + for (const auto &kv : frames) + { + if (kv.first != inputPinId) + { + out_frame = kv.second; + break; + } + } + + BOOST_REQUIRE(out_frame != nullptr); + auto out_dma = static_cast(out_frame->data()); + auto out_md = FrameMetadataFactory::downcast(out_frame->getMetadata()); + + const int y_w = out_md->getWidth(0); + const int y_h = out_md->getHeight(0); + const size_t y_pitch = out_md->getStep(0); + const int u_w = out_md->getWidth(1); + const int u_h = out_md->getHeight(1); + const size_t u_pitch = out_md->getStep(1); + const int v_w = out_md->getWidth(2); + const int v_h = out_md->getHeight(2); + const size_t v_pitch = out_md->getStep(2); + + std::string outPath = "./data/Testoutput" + test.name + ".yuv"; + std::ofstream f_out(outPath, std::ios::binary); + BOOST_REQUIRE(f_out.is_open()); + + // Write Y plane + const uint8_t* srcY = static_cast(out_dma->getHostPtrY()); + for (int r = 0; r < y_h; ++r) + f_out.write(reinterpret_cast(srcY + r * y_pitch), y_w); + + // Write U plane + const uint8_t* srcU = static_cast(out_dma->getHostPtrU()); + for (int r = 0; r < u_h; ++r) + f_out.write(reinterpret_cast(srcU + r * u_pitch), u_w); + + // Write V plane + const uint8_t* srcV = static_cast(out_dma->getHostPtrV()); + for (int r = 0; r < v_h; ++r) + f_out.write(reinterpret_cast(srcV + r * v_pitch), v_w); + + f_out.close(); + std::cout << "Saved output: " << outPath << std::endl; + + index++; + } } -BOOST_AUTO_TEST_SUITE_END() \ No newline at end of file +BOOST_AUTO_TEST_SUITE_END() diff --git a/base/vcpkg-configuration.json b/base/vcpkg-configuration.json new file mode 100644 index 000000000..6a865236a --- /dev/null +++ b/base/vcpkg-configuration.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg-tool/main/docs/vcpkg-configuration.schema.json", + "overlay-ports": [ + "../thirdparty/custom-overlay" + ], + "default-registry": { + "kind": "git", + "repository": "https://github.com/Apra-Labs/vcpkg.git", + "baseline": "6ba64191f73b82b71ffa5263b26189f7b2b82f92" + }, + "registries": [ + { + "kind": "git", + "repository": "https://github.com/Apra-Labs/vcpkg.git", + "baseline": "29a017687d56121cb9d200a7dc519c0de2c78a4a", + "packages": [ "boost*", "boost-*"] + } + ] +} \ No newline at end of file diff --git a/base/vcpkg.json b/base/vcpkg.json index 0e014b44f..f09fa4944 100644 --- a/base/vcpkg.json +++ b/base/vcpkg.json @@ -2,15 +2,46 @@ "$schema": "https://raw.githubusercontent.com/microsoft/vcpkg/master/scripts/vcpkg.schema.json", "name": "apra-pipes-cuda", "version": "0.0.1", - "builtin-baseline": "b7b806799d987389fa0900f8db5edba3f573d0ad", + "builtin-baseline": "6ba64191f73b82b71ffa5263b26189f7b2b82f92", + "overrides": [ + { + "name": "ffmpeg", + "version": "4.4.3" + }, + { + "name": "libarchive", + "version": "3.5.2" + } + ], "dependencies": [ + { + "name": "whisper", + "default-features": false, + "features": [ + "cuda" + ] + }, { "name": "opencv4", "default-features": false, - "features": ["contrib", "cuda", "cudnn", "dnn", "jpeg", "nonfree", "png", "tiff", "webp" ] + "features": [ + "contrib", + "cuda", + "cudnn", + "dnn", + "jpeg", + "nonfree", + "png", + "tiff", + "webp" + ] }, - "libjpeg-turbo", + "freeglut", "ffmpeg", + "openh264-apra", + "glfw3", + "glew", + "libjpeg-turbo", "bigint", "boost-math", "boost-system", @@ -27,12 +58,21 @@ "bzip2", "zlib", "sfml", + "brotli", + "hiredis", + "redis-plus-plus", + { + "name": "gtk3", + "platform": "!windows" + }, { "name": "glib", "default-features": false, - "features": ["libmount"], + "features": [ + "libmount" + ], "platform": "(linux & x64)", - "$reason" : "skip linux:arm64 and windows" + "$reason": "skip linux:arm64 and windows" }, { "name": "glib", @@ -40,23 +80,7 @@ "platform": "windows" }, { - "name": "hiredis", - "platform": "!arm64" - }, - { - "name": "redis-plus-plus", - "platform": "!arm64" - }, - { - "name": "gstreamer", - "features": ["plugins-good","plugins-bad","plugins-ugly" ], - "platform": "!linux", - "$reason": "current vcpkg system does not build gstreamer for linux, we build our own" - }, - { - "name":"gst-rtsp-server", - "platform": "!linux", - "$reason": "current vcpkg system does not build gstreamer for linux, we build our own" + "name": "libmp4" } ] } \ No newline at end of file diff --git a/data/CustomFont111.ttf b/data/CustomFont111.ttf new file mode 100644 index 000000000..8bb9f8750 Binary files /dev/null and b/data/CustomFont111.ttf differ diff --git a/data/Debrosee-ALPnL.ttf b/data/Debrosee-ALPnL.ttf new file mode 100644 index 000000000..311dd6114 Binary files /dev/null and b/data/Debrosee-ALPnL.ttf differ diff --git a/data/apra.jpeg b/data/apra.jpeg new file mode 100644 index 000000000..65e6f6ea6 Binary files /dev/null and b/data/apra.jpeg differ diff --git a/thirdparty/custom-overlay/baresip/portfile.cmake b/thirdparty/custom-overlay/baresip/portfile.cmake new file mode 100644 index 000000000..fbdd50c14 --- /dev/null +++ b/thirdparty/custom-overlay/baresip/portfile.cmake @@ -0,0 +1,28 @@ +# portfile.cmake for Baresip + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Apra-Labs/baresip + REF ea7840ff25a610e2968fc253aed1d774b7073cf9 + SHA512 12ddd8e44757233a10dca0307d04fd2c6436ba749c2573e11a7257440c2cbec5fb828ea4274f543b36691f5c2f7d9783df53efae0df3635c0208fac64ea4e934 + HEAD_REF forApraPipes +) + +vcpkg_configure_cmake( + SOURCE_PATH "${SOURCE_PATH}" + PREFER_NINJA +) + +vcpkg_build_cmake() + +vcpkg_install_cmake() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") + +file( + INSTALL "${SOURCE_PATH}/LICENSE" + DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" + RENAME license +) + diff --git a/thirdparty/custom-overlay/baresip/vcpkg.json b/thirdparty/custom-overlay/baresip/vcpkg.json new file mode 100644 index 000000000..23136e9d2 --- /dev/null +++ b/thirdparty/custom-overlay/baresip/vcpkg.json @@ -0,0 +1,12 @@ +{ + "name": "baresip", + "version": "3.2.0", + "description": "Baresip is a portable and modular SIP User-Agent with audio and video support.", + "homepage": "https://github.com/baresip/baresip", + "dependencies": [ + { + "name": "re", + "platform" : "!windows" + } + ] +} diff --git a/thirdparty/custom-overlay/cudnn/FindCUDNN.cmake b/thirdparty/custom-overlay/cudnn/FindCUDNN.cmake new file mode 100644 index 000000000..292efaebc --- /dev/null +++ b/thirdparty/custom-overlay/cudnn/FindCUDNN.cmake @@ -0,0 +1,104 @@ +# Distributed under the OSI-approved BSD 3-Clause License. + +#.rst: +# FindCUDNN +# -------- +# +# Result Variables +# ^^^^^^^^^^^^^^^^ +# +# This module will set the following variables in your project:: +# +# ``CUDNN_FOUND`` +# True if CUDNN found on the local system +# +# ``CUDNN_INCLUDE_DIRS`` +# Location of CUDNN header files. +# +# ``CUDNN_LIBRARIES`` +# The CUDNN libraries. +# +# ``CuDNN::CuDNN`` +# The CUDNN target +# + +include(FindPackageHandleStandardArgs) + +find_path(CUDNN_INCLUDE_DIR NAMES cudnn.h cudnn_v8.h cudnn_v7.h + HINTS ${CUDA_TOOLKIT_ROOT} $ENV{CUDA_PATH} $ENV{CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR} $ENV{CUDA_PATH}/../../../NVIDIA/CUDNN/v9.0 /usr/include /usr/include/x86_64-linux-gnu/ /usr/include/aarch64-linux-gnu/ + PATH_SUFFIXES cuda/include include include/12.3) +find_library(CUDNN_LIBRARY NAMES cudnn cudnn8 cudnn7 + HINTS ${CUDA_TOOLKIT_ROOT} $ENV{CUDA_PATH} $ENV{CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR} $ENV{CUDA_PATH}/../../../NVIDIA/CUDNN/v9.0 /usr/lib/x86_64-linux-gnu/ /usr/include/aarch64-linux-gnu/ /usr/ + PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64 cuda/lib/x64 lib/12.3/x64) + +if(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn.h") + file(READ ${CUDNN_INCLUDE_DIR}/cudnn.h CUDNN_HEADER_CONTENTS) +elseif(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_v8.h") + file(READ ${CUDNN_INCLUDE_DIR}/cudnn_v8.h CUDNN_HEADER_CONTENTS) +elseif(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_v7.h") + file(READ ${CUDNN_INCLUDE_DIR}/cudnn_v7.h CUDNN_HEADER_CONTENTS) +endif() +if(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_version.h") + file(READ "${CUDNN_INCLUDE_DIR}/cudnn_version.h" CUDNN_VERSION_H_CONTENTS) + string(APPEND CUDNN_HEADER_CONTENTS "${CUDNN_VERSION_H_CONTENTS}") + unset(CUDNN_VERSION_H_CONTENTS) +elseif(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_version_v8.h") + file(READ "${CUDNN_INCLUDE_DIR}/cudnn_version_v8.h" CUDNN_VERSION_H_CONTENTS) + string(APPEND CUDNN_HEADER_CONTENTS "${CUDNN_VERSION_H_CONTENTS}") + unset(CUDNN_VERSION_H_CONTENTS) +elseif(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_version_v7.h") + file(READ "${CUDNN_INCLUDE_DIR}/cudnn_version_v7.h" CUDNN_VERSION_H_CONTENTS) + string(APPEND CUDNN_HEADER_CONTENTS "${CUDNN_VERSION_H_CONTENTS}") + unset(CUDNN_VERSION_H_CONTENTS) +endif() +if(CUDNN_HEADER_CONTENTS) + string(REGEX MATCH "define CUDNN_MAJOR * +([0-9]+)" + _CUDNN_VERSION_MAJOR "${CUDNN_HEADER_CONTENTS}") + string(REGEX REPLACE "define CUDNN_MAJOR * +([0-9]+)" "\\1" + _CUDNN_VERSION_MAJOR "${_CUDNN_VERSION_MAJOR}") + string(REGEX MATCH "define CUDNN_MINOR * +([0-9]+)" + _CUDNN_VERSION_MINOR "${CUDNN_HEADER_CONTENTS}") + string(REGEX REPLACE "define CUDNN_MINOR * +([0-9]+)" "\\1" + _CUDNN_VERSION_MINOR "${_CUDNN_VERSION_MINOR}") + string(REGEX MATCH "define CUDNN_PATCHLEVEL * +([0-9]+)" + _CUDNN_VERSION_PATCH "${CUDNN_HEADER_CONTENTS}") + string(REGEX REPLACE "define CUDNN_PATCHLEVEL * +([0-9]+)" "\\1" + _CUDNN_VERSION_PATCH "${_CUDNN_VERSION_PATCH}") + if(NOT _CUDNN_VERSION_MAJOR) + set(_CUDNN_VERSION "?") + else() + set(_CUDNN_VERSION "${_CUDNN_VERSION_MAJOR}.${_CUDNN_VERSION_MINOR}.${_CUDNN_VERSION_PATCH}") + endif() +endif() + +set(CUDNN_INCLUDE_DIRS ${CUDNN_INCLUDE_DIR}) +set(CUDNN_LIBRARIES ${CUDNN_LIBRARY}) +mark_as_advanced(CUDNN_LIBRARY CUDNN_INCLUDE_DIR) + +find_package_handle_standard_args(CUDNN + REQUIRED_VARS CUDNN_INCLUDE_DIR CUDNN_LIBRARY + VERSION_VAR CUDNN_VERSION +) + +if(WIN32) + set(CUDNN_DLL_DIR ${CUDNN_INCLUDE_DIR}) + list(TRANSFORM CUDNN_DLL_DIR APPEND "/../bin") + find_file(CUDNN_LIBRARY_DLL NAMES cudnn64_${CUDNN_VERSION_MAJOR}.dll PATHS ${CUDNN_DLL_DIR}) +endif() + +if( CUDNN_FOUND AND NOT TARGET CuDNN::CuDNN ) + if( EXISTS "${CUDNN_LIBRARY_DLL}" ) + add_library( CuDNN::CuDNN SHARED IMPORTED ) + set_target_properties( CuDNN::CuDNN PROPERTIES + IMPORTED_LOCATION "${CUDNN_LIBRARY_DLL}" + IMPORTED_IMPLIB "${CUDNN_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${CUDNN_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" ) + else() + add_library( CuDNN::CuDNN UNKNOWN IMPORTED ) + set_target_properties( CuDNN::CuDNN PROPERTIES + IMPORTED_LOCATION "${CUDNN_LIBRARY}" + INTERFACE_INCLUDE_DIRECTORIES "${CUDNN_INCLUDE_DIR}" + IMPORTED_LINK_INTERFACE_LANGUAGES "C" ) + endif() +endif() diff --git a/thirdparty/custom-overlay/cudnn/portfile.cmake b/thirdparty/custom-overlay/cudnn/portfile.cmake new file mode 100644 index 000000000..f33292fa5 --- /dev/null +++ b/thirdparty/custom-overlay/cudnn/portfile.cmake @@ -0,0 +1,65 @@ +set(MINIMUM_CUDNN_VERSION "7.6.5") + +vcpkg_find_cuda(OUT_CUDA_TOOLKIT_ROOT CUDA_TOOLKIT_ROOT OUT_CUDA_VERSION CUDA_VERSION) + +# Try to find CUDNN if it exists; only download if it doesn't exist +find_path(CUDNN_INCLUDE_DIR NAMES cudnn.h cudnn_v8.h cudnn_v7.h + HINTS ${CUDA_TOOLKIT_ROOT} $ENV{CUDA_PATH} $ENV{CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR} $ENV{CUDA_PATH}/../../../NVIDIA/CUDNN/v9.0 /usr/include /usr/include/x86_64-linux-gnu/ /usr/include/aarch64-linux-gnu/ + PATH_SUFFIXES cuda/include include include/12.3) +message(STATUS "CUDNN_INCLUDE_DIR: ${CUDNN_INCLUDE_DIR}") +find_library(CUDNN_LIBRARY NAMES cudnn cudnn8 cudnn7 libcudnn libcudnn8 libcudnn7 + HINTS ${CUDA_TOOLKIT_ROOT} $ENV{CUDA_PATH} $ENV{CUDA_TOOLKIT_ROOT_DIR} $ENV{cudnn} $ENV{CUDNN} $ENV{CUDNN_ROOT_DIR} $ENV{CUDA_PATH}/../../../NVIDIA/CUDNN/v9.0 /usr/lib/aarch64-linux-gnu/ /usr/include/aarch64-linux-gnu/ /usr/ + PATH_SUFFIXES lib lib64 cuda/lib cuda/lib64 lib/x64 cuda/lib/x64 lib/12.3/x64) +message(STATUS "CUDNN_LIBRARY: ${CUDNN_LIBRARY}") +if(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn.h") + file(READ ${CUDNN_INCLUDE_DIR}/cudnn.h CUDNN_HEADER_CONTENTS) +elseif(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_v8.h") + file(READ ${CUDNN_INCLUDE_DIR}/cudnn_v8.h CUDNN_HEADER_CONTENTS) +elseif(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_v7.h") + file(READ ${CUDNN_INCLUDE_DIR}/cudnn_v7.h CUDNN_HEADER_CONTENTS) +endif() +if(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_version.h") + file(READ "${CUDNN_INCLUDE_DIR}/cudnn_version.h" CUDNN_VERSION_H_CONTENTS) + string(APPEND CUDNN_HEADER_CONTENTS "${CUDNN_VERSION_H_CONTENTS}") + unset(CUDNN_VERSION_H_CONTENTS) +elseif(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_version_v8.h") + file(READ "${CUDNN_INCLUDE_DIR}/cudnn_version_v8.h" CUDNN_VERSION_H_CONTENTS) + string(APPEND CUDNN_HEADER_CONTENTS "${CUDNN_VERSION_H_CONTENTS}") + unset(CUDNN_VERSION_H_CONTENTS) +elseif(EXISTS "${CUDNN_INCLUDE_DIR}/cudnn_version_v7.h") + file(READ "${CUDNN_INCLUDE_DIR}/cudnn_version_v7.h" CUDNN_VERSION_H_CONTENTS) + string(APPEND CUDNN_HEADER_CONTENTS "${CUDNN_VERSION_H_CONTENTS}") + unset(CUDNN_VERSION_H_CONTENTS) +endif() +if(CUDNN_HEADER_CONTENTS) + string(REGEX MATCH "define CUDNN_MAJOR * +([0-9]+)" + _CUDNN_VERSION_MAJOR "${CUDNN_HEADER_CONTENTS}") + string(REGEX REPLACE "define CUDNN_MAJOR * +([0-9]+)" "\\1" + _CUDNN_VERSION_MAJOR "${_CUDNN_VERSION_MAJOR}") + string(REGEX MATCH "define CUDNN_MINOR * +([0-9]+)" + _CUDNN_VERSION_MINOR "${CUDNN_HEADER_CONTENTS}") + string(REGEX REPLACE "define CUDNN_MINOR * +([0-9]+)" "\\1" + _CUDNN_VERSION_MINOR "${_CUDNN_VERSION_MINOR}") + string(REGEX MATCH "define CUDNN_PATCHLEVEL * +([0-9]+)" + _CUDNN_VERSION_PATCH "${CUDNN_HEADER_CONTENTS}") + string(REGEX REPLACE "define CUDNN_PATCHLEVEL * +([0-9]+)" "\\1" + _CUDNN_VERSION_PATCH "${_CUDNN_VERSION_PATCH}") + if(NOT _CUDNN_VERSION_MAJOR) + set(_CUDNN_VERSION "?") + else() + set(_CUDNN_VERSION "${_CUDNN_VERSION_MAJOR}.${_CUDNN_VERSION_MINOR}.${_CUDNN_VERSION_PATCH}") + endif() +endif() + +if (CUDNN_INCLUDE_DIR AND CUDNN_LIBRARY AND _CUDNN_VERSION VERSION_GREATER_EQUAL MINIMUM_CUDNN_VERSION) + message(STATUS "Found CUDNN ${_CUDNN_VERSION} located on system: (include ${CUDNN_INCLUDE_DIR} lib: ${CUDNN_LIBRARY})") + set(VCPKG_POLICY_EMPTY_PACKAGE enabled) +elseif(VCPKG_TARGET_IS_WINDOWS) + message(FATAL_ERROR "Please download CUDNN from official sources (such as https://developer.nvidia.com/rdp/cudnn-download ) and extract the zip into your CUDA_TOOLKIT_ROOT (${CUDA_TOOLKIT_ROOT}). (For example: tar.exe -xvf cudnn-11.2-windows-x64-v8.1.1.33.zip --strip 1 --directory \"${CUDA_TOOLKIT_ROOT}\"") +else() + message(FATAL_ERROR "Please install CUDNN using your system package manager (the same way you installed CUDA). For example: apt install libcudnn8-dev.") +endif() + +file(INSTALL "${CURRENT_PORT_DIR}/FindCUDNN.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +file(INSTALL "${CURRENT_PORT_DIR}/usage" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") +file(INSTALL "${CURRENT_PORT_DIR}/vcpkg-cmake-wrapper.cmake" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}") diff --git a/thirdparty/custom-overlay/cudnn/usage b/thirdparty/custom-overlay/cudnn/usage new file mode 100644 index 000000000..f528e0154 --- /dev/null +++ b/thirdparty/custom-overlay/cudnn/usage @@ -0,0 +1,10 @@ +The package cudnn provides CMake variables: + + find_package(CUDNN REQUIRED) + target_link_libraries(main PRIVATE ${CUDNN_LIBRARIES}) + target_include_directories(main PRIVATE ${CUDNN_INCLUDE_DIRS}) + +Or the following CMake target: + + find_package(CUDNN REQUIRED) + target_link_libraries(main PRIVATE CuDNN::CuDNN) diff --git a/thirdparty/custom-overlay/cudnn/vcpkg-cmake-wrapper.cmake b/thirdparty/custom-overlay/cudnn/vcpkg-cmake-wrapper.cmake new file mode 100644 index 000000000..5a69edec5 --- /dev/null +++ b/thirdparty/custom-overlay/cudnn/vcpkg-cmake-wrapper.cmake @@ -0,0 +1,6 @@ +set(CUDNN_PREV_MODULE_PATH ${CMAKE_MODULE_PATH}) +list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}) + +_find_package(${ARGS}) + +set(CMAKE_MODULE_PATH ${CUDNN_PREV_MODULE_PATH}) diff --git a/thirdparty/custom-overlay/cudnn/vcpkg.json b/thirdparty/custom-overlay/cudnn/vcpkg.json new file mode 100644 index 000000000..d5447da0b --- /dev/null +++ b/thirdparty/custom-overlay/cudnn/vcpkg.json @@ -0,0 +1,12 @@ +{ + "name": "cudnn", + "version": "7.6.5", + "port-version": 11, + "description": "NVIDIA's cuDNN deep neural network acceleration library.", + "homepage": "https://developer.nvidia.com/cudnn", + "license": null, + "supports": "(windows & x64 & !uwp) | (linux & x64) | (linux & arm64)", + "dependencies": [ + "cuda" + ] +} diff --git a/thirdparty/custom-overlay/libmp4/portfile.cmake b/thirdparty/custom-overlay/libmp4/portfile.cmake new file mode 100644 index 000000000..192d535b0 --- /dev/null +++ b/thirdparty/custom-overlay/libmp4/portfile.cmake @@ -0,0 +1,23 @@ +# portfile.cmake for libmp4 + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Apra-Labs/libmp4 + REF 98f8ae9637093c822f344ec95c8cffbb814dd336 + SHA512 34c8ced415b5b1e03c0b04148ca5647109a70226af1fdc3c0739c8d88e68294ebe187a59d44008d5bea3fbab7e09b19e311a03712710f62b53444a92e924db4c + HEAD_REF forApraPipes +) +vcpkg_configure_cmake( + SOURCE_PATH "${SOURCE_PATH}" + PREFER_NINJA +) + +vcpkg_build_cmake() + +vcpkg_install_cmake() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/share") + +file(INSTALL ${SOURCE_PATH}/COPYING DESTINATION ${CURRENT_PACKAGES_DIR}/share/${PORT} RENAME copyright) + diff --git a/thirdparty/custom-overlay/libmp4/vcpkg.json b/thirdparty/custom-overlay/libmp4/vcpkg.json new file mode 100644 index 000000000..a92d28155 --- /dev/null +++ b/thirdparty/custom-overlay/libmp4/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "libmp4", + "version": "1.0", + "description": "libmp4 is a C library to handle MP4 files (ISO base media file format", + "homepage": "https://github.com/Parrot-Developers/libmp4" + } + \ No newline at end of file diff --git a/thirdparty/custom-overlay/openh264-apra/0001-respect-default-library-option.patch b/thirdparty/custom-overlay/openh264-apra/0001-respect-default-library-option.patch new file mode 100644 index 000000000..15e3c7154 --- /dev/null +++ b/thirdparty/custom-overlay/openh264-apra/0001-respect-default-library-option.patch @@ -0,0 +1,57 @@ +From 328b15a962caa928373b55d85f9911f45442886e Mon Sep 17 00:00:00 2001 +From: Xavier Claessens +Date: Mon, 19 Oct 2020 17:03:25 -0400 +Subject: [PATCH] meson: Respect default_library option + +When using library() instead of shared_library() and static_library, +meson will build shared, static, or both depending on the +value of static_library option. + +As far as I know extract_all_objects() was uses as workaround for Meson +bugs fixed a while ago when using not installed static libraries. +--- + meson.build | 19 +++---------------- + 1 file changed, 3 insertions(+), 16 deletions(-) + +diff --git a/meson.build b/meson.build +index 283413375b..65641508de 100644 +--- a/meson.build ++++ b/meson.build +@@ -184,26 +184,13 @@ api_header_deps = [] + subdir ('codec') + subdir ('test') + +-all_objects = [ +- libcommon.extract_all_objects(), +- libprocessing.extract_all_objects(), +- libencoder.extract_all_objects(), +- libdecoder.extract_all_objects() +-] +- +-libopenh264_shared = shared_library('openh264', +- objects: all_objects, ++libopenh264 = library('openh264', ++ link_whole: [libcommon, libprocessing, libencoder, libdecoder], + install: true, + soversion: major_version, +- version: meson.project_version(), + vs_module_defs: 'openh264.def', + dependencies: deps) + +-libopenh264_static = static_library('openh264', +- objects: all_objects, +- install: true, +- dependencies: deps) +- + pkg_install_dir = '@0@/pkgconfig'.format(get_option('libdir')) + + foreach t : ['', '-static'] +@@ -235,7 +222,7 @@ foreach t : ['', '-static'] + endforeach + + openh264_dep = declare_dependency( +- link_with: libopenh264_shared, ++ link_with: libopenh264, + include_directories: include_directories('include'), + dependencies: deps + api_header_deps) + diff --git a/thirdparty/custom-overlay/openh264-apra/portfile.cmake b/thirdparty/custom-overlay/openh264-apra/portfile.cmake new file mode 100644 index 000000000..f7c94396c --- /dev/null +++ b/thirdparty/custom-overlay/openh264-apra/portfile.cmake @@ -0,0 +1,36 @@ +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Apra-Labs/openh264 + REF 4e3c4edd39c0192b98d10424fefe9c0b6bec1a2e + SHA512 f33c1e01f1d2ff04dcf16e563eacec0bf43235db8afc241ff63c99e5896a2c97efe47d5eeee225bcfc8c49a214d22c42aea640e4d1626b5cc116cc26d59a201d + HEAD_REF ForApraPipes + PATCHES + 0001-respect-default-library-option.patch # https://github.com/cisco/openh264/pull/3351 +) + +if((VCPKG_TARGET_ARCHITECTURE STREQUAL "x86" OR VCPKG_TARGET_ARCHITECTURE STREQUAL "x64")) + vcpkg_find_acquire_program(NASM) + get_filename_component(NASM_EXE_PATH ${NASM} DIRECTORY) + vcpkg_add_to_path(${NASM_EXE_PATH}) +elseif(VCPKG_TARGET_IS_WINDOWS) + vcpkg_find_acquire_program(GASPREPROCESSOR) + foreach(GAS_PATH ${GASPREPROCESSOR}) + get_filename_component(GAS_ITEM_PATH ${GAS_PATH} DIRECTORY) + vcpkg_add_to_path(${GAS_ITEM_PATH}) + endforeach(GAS_PATH) +endif() + +vcpkg_configure_meson( + SOURCE_PATH ${SOURCE_PATH} + OPTIONS -Dtests=disabled +) + +vcpkg_install_meson() +vcpkg_copy_pdbs() +vcpkg_fixup_pkgconfig() + +if(VCPKG_LIBRARY_LINKAGE STREQUAL "static") + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/bin" "${CURRENT_PACKAGES_DIR}/debug/bin") +endif() + +configure_file("${SOURCE_PATH}/LICENSE" "${CURRENT_PACKAGES_DIR}/share/${PORT}/copyright" COPYONLY) diff --git a/thirdparty/custom-overlay/openh264-apra/vcpkg.json b/thirdparty/custom-overlay/openh264-apra/vcpkg.json new file mode 100644 index 000000000..17c90a38e --- /dev/null +++ b/thirdparty/custom-overlay/openh264-apra/vcpkg.json @@ -0,0 +1,14 @@ +{ + "name": "openh264-apra", + "version-date": "2023-04-04", + "port-version": 1, + "description": "OpenH264 is a codec library which supports H.264 encoding and decoding. It is suitable for use in real time applications such as WebRTC.", + "homepage": "https://www.openh264.org/", + "supports": "!uwp", + "dependencies": [ + { + "name": "vcpkg-tool-meson", + "host": true + } + ] +} diff --git a/thirdparty/custom-overlay/re/portfile.cmake b/thirdparty/custom-overlay/re/portfile.cmake new file mode 100644 index 000000000..8b20fb58c --- /dev/null +++ b/thirdparty/custom-overlay/re/portfile.cmake @@ -0,0 +1,24 @@ +# portfile.cmake for lib_re + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Apra-Labs/re + REF 5e516154d4354df8a753849270d235f02e04ac5a + SHA512 b6875d8b98a06419619c7338ec53cc6c7078f24c3d5cacceac2ad43f201d8f302cdac14ce394c56e3ebf1b0b1692ea7feac4e58bb934e8923dead9608250e757 + HEAD_REF forApraPipes +) + +vcpkg_configure_cmake( + SOURCE_PATH "${SOURCE_PATH}" + PREFER_NINJA +) + +vcpkg_build_cmake() + +vcpkg_install_cmake() + +file( + INSTALL "${SOURCE_PATH}/LICENSE" + DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" + RENAME license +) diff --git a/thirdparty/custom-overlay/re/vcpkg.json b/thirdparty/custom-overlay/re/vcpkg.json new file mode 100644 index 000000000..34575f254 --- /dev/null +++ b/thirdparty/custom-overlay/re/vcpkg.json @@ -0,0 +1,7 @@ +{ + "name": "re", + "version": "3.2.0", + "description": "libre is a Generic library for real-time communications with async IO support.", + "homepage": "https://github.com/baresip/re" +} + \ No newline at end of file diff --git a/thirdparty/custom-overlay/whisper/fix-for-arm64.patch b/thirdparty/custom-overlay/whisper/fix-for-arm64.patch new file mode 100644 index 000000000..8c9c897f0 --- /dev/null +++ b/thirdparty/custom-overlay/whisper/fix-for-arm64.patch @@ -0,0 +1,127 @@ +diff --git a/ggml-cuda.cu b/ggml-cuda.cu +index 2db5043..c799e32 100644 +--- a/ggml-cuda.cu ++++ b/ggml-cuda.cu +@@ -12,9 +12,6 @@ + #include + #include + #include +-#include "ggml-cuda.h" +-#include "ggml.h" +-#include "ggml-backend-impl.h" + + #if defined(GGML_USE_HIPBLAS) + #include +@@ -108,6 +105,10 @@ + #include + #include + ++#include "ggml-cuda.h" ++#include "ggml.h" ++#include "ggml-backend-impl.h" ++ + #if CUDART_VERSION < 11020 + #define CU_DEVICE_ATTRIBUTE_VIRTUAL_MEMORY_MANAGEMENT_SUPPORTED CU_DEVICE_ATTRIBUTE_VIRTUAL_ADDRESS_MANAGEMENT_SUPPORTED + #define CUBLAS_TF32_TENSOR_OP_MATH CUBLAS_TENSOR_OP_MATH +diff --git a/ggml-quants.c b/ggml-quants.c +index 601d155..01921c6 100644 +--- a/ggml-quants.c ++++ b/ggml-quants.c +@@ -425,17 +425,86 @@ inline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) { + + #else + +-#define ggml_int16x8x2_t int16x8x2_t +-#define ggml_uint8x16x2_t uint8x16x2_t +-#define ggml_uint8x16x4_t uint8x16x4_t +-#define ggml_int8x16x2_t int8x16x2_t +-#define ggml_int8x16x4_t int8x16x4_t +- +-#define ggml_vld1q_s16_x2 vld1q_s16_x2 +-#define ggml_vld1q_u8_x2 vld1q_u8_x2 +-#define ggml_vld1q_u8_x4 vld1q_u8_x4 +-#define ggml_vld1q_s8_x2 vld1q_s8_x2 +-#define ggml_vld1q_s8_x4 vld1q_s8_x4 ++typedef struct ggml_int16x8x2_t { ++ int16x8_t val[2]; ++} ggml_int16x8x2_t; ++ ++inline static ggml_int16x8x2_t ggml_vld1q_s16_x2(const int16_t * ptr) { ++ ggml_int16x8x2_t res; ++ ++ res.val[0] = vld1q_s16(ptr + 0); ++ res.val[1] = vld1q_s16(ptr + 8); ++ ++ return res; ++} ++ ++typedef struct ggml_uint8x16x2_t { ++ uint8x16_t val[2]; ++} ggml_uint8x16x2_t; ++ ++inline static ggml_uint8x16x2_t ggml_vld1q_u8_x2(const uint8_t * ptr) { ++ ggml_uint8x16x2_t res; ++ ++ res.val[0] = vld1q_u8(ptr + 0); ++ res.val[1] = vld1q_u8(ptr + 16); ++ ++ return res; ++} ++ ++typedef struct ggml_uint8x16x4_t { ++ uint8x16_t val[4]; ++} ggml_uint8x16x4_t; ++ ++inline static ggml_uint8x16x4_t ggml_vld1q_u8_x4(const uint8_t * ptr) { ++ ggml_uint8x16x4_t res; ++ ++ res.val[0] = vld1q_u8(ptr + 0); ++ res.val[1] = vld1q_u8(ptr + 16); ++ res.val[2] = vld1q_u8(ptr + 32); ++ res.val[3] = vld1q_u8(ptr + 48); ++ ++ return res; ++} ++ ++typedef struct ggml_int8x16x2_t { ++ int8x16_t val[2]; ++} ggml_int8x16x2_t; ++ ++inline static ggml_int8x16x2_t ggml_vld1q_s8_x2(const int8_t * ptr) { ++ ggml_int8x16x2_t res; ++ ++ res.val[0] = vld1q_s8(ptr + 0); ++ res.val[1] = vld1q_s8(ptr + 16); ++ ++ return res; ++} ++ ++typedef struct ggml_int8x16x4_t { ++ int8x16_t val[4]; ++} ggml_int8x16x4_t; ++ ++inline static ggml_int8x16x4_t ggml_vld1q_s8_x4(const int8_t * ptr) { ++ ggml_int8x16x4_t res; ++ ++ res.val[0] = vld1q_s8(ptr + 0); ++ res.val[1] = vld1q_s8(ptr + 16); ++ res.val[2] = vld1q_s8(ptr + 32); ++ res.val[3] = vld1q_s8(ptr + 48); ++ ++ return res; ++} ++ ++// #define ggml_int16x8x2_t int16x8x2_t ++// #define ggml_uint8x16x2_t uint8x16x2_t ++// #define ggml_uint8x16x4_t uint8x16x4_t ++// #define ggml_int8x16x2_t int8x16x2_t ++// #define ggml_int8x16x4_t int8x16x4_t ++ ++// #define ggml_vld1q_s16_x2 vld1q_s16_x2 ++// #define ggml_vld1q_u8_x2 vld1q_u8_x2 ++// #define ggml_vld1q_u8_x4 vld1q_u8_x4 ++// #define ggml_vld1q_s8_x2 vld1q_s8_x2 ++// #define ggml_vld1q_s8_x4 vld1q_s8_x4 + + #endif + diff --git a/thirdparty/custom-overlay/whisper/portfile.cmake b/thirdparty/custom-overlay/whisper/portfile.cmake new file mode 100644 index 000000000..1ff52d319 --- /dev/null +++ b/thirdparty/custom-overlay/whisper/portfile.cmake @@ -0,0 +1,40 @@ +vcpkg_check_linkage(ONLY_STATIC_LIBRARY) + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO Apra-Labs/whisper.cpp + REF c3bff0d121e2af823344939643d64a27e4a76ea2 #v1.5.4 + SHA512 d51a32c91340d2b9f18bf5221e134e57a0259bc3a1c803ef427adc6e3de5f54c556232cd4ef070b9c07f93968efd942a61cfe311c2cbca013a928f0eb8055e6f # This is a temporary value. We will modify this value in the next section. + HEAD_REF kj/add-Config-for-vcpkg + PATCHES "fix-for-arm64.patch" +) + +vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS + FEATURES + "cuda" WHISPER_CUBLAS +) + +set(WHISPER_CUBLAS OFF) +if("cuda" IN_LIST FEATURES) + set(WHISPER_CUBLAS ON) +endif() + + +vcpkg_cmake_configure( + SOURCE_PATH "${SOURCE_PATH}" + DISABLE_PARALLEL_CONFIGURE +) + +vcpkg_cmake_install() +vcpkg_cmake_config_fixup( + CONFIG_PATH lib/cmake/whisper + PACKAGE_NAME whisper + ) +vcpkg_copy_pdbs() + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/include") + +file(INSTALL "${SOURCE_PATH}/LICENSE" DESTINATION "${CURRENT_PACKAGES_DIR}/share/${PORT}" RENAME copyright) +configure_file("${CMAKE_CURRENT_LIST_DIR}/usage" "${CURRENT_PACKAGES_DIR}/share/${PORT}/usage" COPYONLY) + +file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/share") \ No newline at end of file diff --git a/thirdparty/custom-overlay/whisper/usage b/thirdparty/custom-overlay/whisper/usage new file mode 100644 index 000000000..b997cd604 --- /dev/null +++ b/thirdparty/custom-overlay/whisper/usage @@ -0,0 +1,4 @@ +whisper provides CMake targets: + +find_package(whisper CONFIG REQUIRED) +target_link_libraries(main PRIVATE whisper::whisper) \ No newline at end of file diff --git a/thirdparty/custom-overlay/whisper/vcpkg.json b/thirdparty/custom-overlay/whisper/vcpkg.json new file mode 100644 index 000000000..0290a42fe --- /dev/null +++ b/thirdparty/custom-overlay/whisper/vcpkg.json @@ -0,0 +1,28 @@ +{ + "name": "whisper", + "version": "1.5.4", + "homepage": "https://github.com/Apra-Labs/whisper.cpp", + "description": "Fork of whisper.cpp a High-performance inference of OpenAI's Whisper automatic speech recognition (ASR) model in cpp.", + "license": "MIT", + "dependencies": [ + { + "name": "vcpkg-cmake", + "host": true + }, + { + "name": "vcpkg-cmake-config", + "host": true + } + ], + "default-features": [ + "default-features" + ], + "features": { + "cuda": { + "description": "Build Whisper with CUDA support", + "dependencies": [ + "cuda" + ] + } + } +} \ No newline at end of file diff --git a/thirdparty/gst-build b/thirdparty/gst-build deleted file mode 160000 index 7697a3f3c..000000000 --- a/thirdparty/gst-build +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7697a3f3cd51d70c73333f095311ef1471d0fc17 diff --git a/thirdparty/libmp4 b/thirdparty/libmp4 deleted file mode 160000 index a2c0f1009..000000000 --- a/thirdparty/libmp4 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a2c0f10090f7751014228153aa1945f2cd6e0508