diff --git a/gstreamer/gst-plugin-codec2/c2-engine/c2-engine.cc b/gstreamer/gst-plugin-codec2/c2-engine/c2-engine.cc index 80a8b99f..73431789 100644 --- a/gstreamer/gst-plugin-codec2/c2-engine/c2-engine.cc +++ b/gstreamer/gst-plugin-codec2/c2-engine/c2-engine.cc @@ -637,9 +637,15 @@ gst_c2_engine_queue (GstC2Engine * engine, GstC2QueueItem * item) try { qc2audio::QC2Status status = qc2audio::QC2_OK; std::shared_ptr outbuffer; +#if (CODEC2_CONFIG_VERSION_MAJOR == 2 && CODEC2_CONFIG_VERSION_MINOR >= 3) + std::shared_ptr c2pool = + c2module->GetLinearBufferPool(size); + status = c2pool->allocate(&outbuffer); +#else std::shared_ptr c2circlePool = - c2module->GetLinearCirclePool(size); + c2module->GetLinearBufferPool(size); status = c2circlePool->take(&outbuffer, nullptr); +#endif // (CODEC2_CONFIG_VERSION_MAJOR == 2 && CODEC2_CONFIG_VERSION_MINOR >= 3) c2buffer = GstC2Utils::CreateBuffer(buffer, outbuffer); } catch (std::exception& e) { GST_ERROR ("Failed to fetch memory block, error: '%s'!", e.what()); diff --git a/gstreamer/gst-plugin-examples/gst-camera-360-prebuffered-data-app/main.cc b/gstreamer/gst-plugin-examples/gst-camera-360-prebuffered-data-app/main.cc index 7b1fe49c..9a9e740a 100644 --- a/gstreamer/gst-plugin-examples/gst-camera-360-prebuffered-data-app/main.cc +++ b/gstreamer/gst-plugin-examples/gst-camera-360-prebuffered-data-app/main.cc @@ -103,9 +103,10 @@ namespace camera = qmmf; /* Default parameters */ -#define MAX_QUEUE_SIZE 300 -#define DELAY_TO_START_RECORDING 30 -#define RECORD_DURATION 30 +#define MAX_QUEUE_SIZE 300 +#define DELAY_TO_START_RECORDING 30 +#define RECORD_DURATION 30 +#define MP4MUX_DRAIN_MARGIN_SECONDS 60 /* Fixed stream resolutions per spec */ #define RDI_RING_BUFFER_WIDTH 4096 @@ -161,6 +162,7 @@ struct _GstStreamInf { GstElement *encoder; GstElement *filesink; GstElement *appsink; + gulong new_sample_handler_id; GstPad *qmmf_pad; GstCaps *qmmf_caps; gint width; @@ -217,6 +219,10 @@ struct _GstAppContext { gboolean switch_to_live_cam2; guint process_src_id_cam2; + GCond queued_buffers_drained_signal; + gboolean queued_buffers_drained_cam1; + gboolean queued_buffers_drained_cam2; + /* Camera ID (logical camera) */ guint camera_id; @@ -715,6 +721,38 @@ check_for_exit (GstAppContext *appctx) return e; } +static void +signal_queued_buffers_drained (GstAppContext *appctx, gint cam_id) +{ + if (!appctx) + return; + + g_mutex_lock (&appctx->lock); + if (cam_id == 0) + appctx->queued_buffers_drained_cam1 = TRUE; + else + appctx->queued_buffers_drained_cam2 = TRUE; + g_mutex_unlock (&appctx->lock); + g_cond_signal (&appctx->queued_buffers_drained_signal); +} + +static void +wait_for_queued_buffers_drained (GstAppContext *appctx, gint cam_id) +{ + if (!appctx) + return; + + g_mutex_lock (&appctx->lock); + while (!(cam_id == 0 ? appctx->queued_buffers_drained_cam1 + : appctx->queued_buffers_drained_cam2) && + !appctx->exit) { + gint64 wait_time = g_get_monotonic_time () + G_GINT64_CONSTANT (1000000); + g_cond_wait_until (&appctx->queued_buffers_drained_signal, &appctx->lock, + wait_time); + } + g_mutex_unlock (&appctx->lock); +} + static gboolean wait_for_eos (GstAppContext *appctx) { @@ -767,6 +805,7 @@ handle_interrupt_signal (gpointer userdata) clear_buffers_queue_cam2 (appctx); g_cond_signal (&appctx->eos_signal); + g_cond_signal (&appctx->queued_buffers_drained_signal); if (appctx->mloop && g_main_loop_is_running (appctx->mloop)) g_main_loop_quit (appctx->mloop); @@ -849,12 +888,17 @@ set_encoder_props (GstElement *encoder, const gchar *encoder_name) } static void -set_mp4mux_robust_props (GstElement *mp4mux) +set_mp4mux_robust_props (GstElement *mp4mux, GstAppContext *appctx) { + const guint64 reserved_duration = + ((guint64) appctx->delay_to_start_recording + + (guint64) appctx->record_duration + + MP4MUX_DRAIN_MARGIN_SECONDS) * GST_SECOND; + g_object_set (G_OBJECT (mp4mux), "reserved-moov-update-period", (guint64) 1000000, "reserved-bytes-per-sec", (guint) 10000, - "reserved-max-duration", (guint64) 8000000000ULL, + "reserved-max-duration", reserved_duration, NULL); } @@ -1016,7 +1060,7 @@ create_encoder_stream (GstAppContext *appctx, GstStreamInf *stream, g_object_set (G_OBJECT (stream->capsfilter), "caps", stream->qmmf_caps, NULL); set_encoder_props (stream->encoder, appctx->encoder_name); - set_mp4mux_robust_props (stream->mp4mux); + set_mp4mux_robust_props (stream->mp4mux, appctx); /* Output file path */ if (stream->output_path[0] != '\0') { @@ -1134,11 +1178,11 @@ create_appsink_stream (GstAppContext *appctx, GstStreamInf *stream, /* Connect the correct callback based on which camera this stream belongs to */ if (stream->logical_stream_type == CAM1_LOGICAL_STREAM_TYPE) - g_signal_connect (stream->appsink, "new-sample", - G_CALLBACK (on_new_sample_cam1), appctx); + stream->new_sample_handler_id = g_signal_connect (stream->appsink, + "new-sample", G_CALLBACK (on_new_sample_cam1), appctx); else - g_signal_connect (stream->appsink, "new-sample", - G_CALLBACK (on_new_sample_cam2), appctx); + stream->new_sample_handler_id = g_signal_connect (stream->appsink, + "new-sample", G_CALLBACK (on_new_sample_cam2), appctx); gst_bin_add_many (GST_BIN (appctx->main_pipeline), stream->capsfilter, stream->appsink, NULL); @@ -1313,6 +1357,15 @@ link_stream (GstAppContext *appctx, GstStreamInf *stream) static void unlink_stream (GstAppContext *appctx, GstStreamInf *stream) { + if (stream->appsink) { + gst_app_sink_set_emit_signals (GST_APP_SINK (stream->appsink), FALSE); + if (stream->new_sample_handler_id != 0) { + g_signal_handler_disconnect (stream->appsink, + stream->new_sample_handler_id); + stream->new_sample_handler_id = 0; + } + } + if (stream->qmmf_pad) gst_pad_set_active (stream->qmmf_pad, FALSE); @@ -1645,6 +1698,7 @@ process_queued_buffers (gpointer user_data) gst_app_src_end_of_stream (src); g_print ("[INFO] CAM%d buffer queue empty, sending EOS\n", cam_id + 1); gst_object_unref (appsrc_elem); + signal_queued_buffers_drained (appctx, cam_id); g_free (ctx); return FALSE; } @@ -1678,6 +1732,13 @@ start_pushing_buffers (GstAppContext *appctx, gint cam_id) g_print ("[INFO] Starting to push CAM%d queued buffers to appsrc pipeline\n", cam_id + 1); + g_mutex_lock (&appctx->lock); + if (cam_id == 0) + appctx->queued_buffers_drained_cam1 = FALSE; + else + appctx->queued_buffers_drained_cam2 = FALSE; + g_mutex_unlock (&appctx->lock); + guint src_id = g_timeout_add (10, process_queued_buffers, ctx); if (cam_id == 0) @@ -2353,10 +2414,12 @@ prebuffering_usecase_360 (GstAppContext *appctx) if (!is_rdi) start_pushing_buffers (appctx, 1); /* CAM2 (IPE only) */ - /* Release ring buffer appsink streams */ + wait_for_queued_buffers_drained (appctx, 0); release_stream (appctx, str0_cam1); - if (!is_rdi) + if (!is_rdi) { + wait_for_queued_buffers_drained (appctx, 1); release_stream (appctx, str6_cam2); + } g_print ("[INFO] Live recording started for %u seconds\n", appctx->record_duration); @@ -2493,6 +2556,7 @@ main (gint argc, gchar *argv[]) g_mutex_init (&appctx->lock); g_cond_init (&appctx->eos_signal); g_cond_init (&appctx->live_pts_signal); + g_cond_init (&appctx->queued_buffers_drained_signal); /* Defaults */ appctx->stream_cnt = 0; @@ -2504,6 +2568,8 @@ main (gint argc, gchar *argv[]) appctx->first_live_pts = GST_CLOCK_TIME_NONE; appctx->switch_to_live = FALSE; appctx->switch_to_live_cam2 = FALSE; + appctx->queued_buffers_drained_cam1 = FALSE; + appctx->queued_buffers_drained_cam2 = FALSE; appctx->record_duration = RECORD_DURATION; appctx->recording_start_pts = GST_CLOCK_TIME_NONE; appctx->recording_end_pts = GST_CLOCK_TIME_NONE; @@ -3000,6 +3066,7 @@ main (gint argc, gchar *argv[]) g_mutex_clear (&appctx->lock); g_cond_clear (&appctx->eos_signal); g_cond_clear (&appctx->live_pts_signal); + g_cond_clear (&appctx->queued_buffers_drained_signal); if (appctx->appsrc_pipeline_cam2) gst_object_unref (appctx->appsrc_pipeline_cam2); diff --git a/gstreamer/gst-plugin-mlqnn/ml-qnn-engine.cc b/gstreamer/gst-plugin-mlqnn/ml-qnn-engine.cc index fc58f6e2..662eb68c 100644 --- a/gstreamer/gst-plugin-mlqnn/ml-qnn-engine.cc +++ b/gstreamer/gst-plugin-mlqnn/ml-qnn-engine.cc @@ -1252,6 +1252,25 @@ gst_ml_qnn_engine_free (GstMLQnnEngine * engine) } if (engine->graph_infos) { + const GraphInfo_t *graph_info = engine->graph_infos[0]; + Qnn_Tensor_t *tensor; + + // Clear the client buffer pointers set during execute(), some of the + // graph info below may alias memory owned by the QNN system context or + // model library, which must not be walked with stale buffer pointers + // when it gets freed. + for (guint idx = 0; idx < graph_info->numInputTensors; idx++) { + tensor = &(graph_info->inputTensors[idx]); + QNN_TENSOR_CLIENTBUF (tensor).data = NULL; + QNN_TENSOR_CLIENTBUF (tensor).dataSize = 0; + } + + for (guint idx = 0; idx < graph_info->numOutputTensors; ++idx) { + tensor = &(graph_info->outputTensors[idx]); + QNN_TENSOR_CLIENTBUF (tensor).data = NULL; + QNN_TENSOR_CLIENTBUF (tensor).dataSize = 0; + } + // A model library owns its graph info and frees it through its own API. if (engine->FreeGraph != NULL) { engine->FreeGraph (&(engine->graph_infos), engine->n_graphs); diff --git a/gstreamer/gst-python-examples/gst-gui-launcher-app.py b/gstreamer/gst-python-examples/gst-gui-launcher-app.py index eea8d289..84a223ee 100644 --- a/gstreamer/gst-python-examples/gst-gui-launcher-app.py +++ b/gstreamer/gst-python-examples/gst-gui-launcher-app.py @@ -628,20 +628,20 @@ def create_pipeline(self, application, src): sink_7::position='<960, 540>' sink_7::dimensions='<960, 540>' sink_7::alpha=0.5 \ mixer. ! queue ! fpsdisplaysink signal-fps-measurements=true text-overlay=true video-sink='gtksink' " + in_src + f"! queue ! tee name=split \ split. ! queue ! mixer. \ - split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model={models_dir}/yolox_quantized.tflite ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=BGRA,width=640,height=360 ! queue ! mixer. \ + split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model={models_dir}/yolox_quantized.tflite ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=RGBA,width=640,height=360 ! queue ! mixer. \ split. ! queue ! mixer. \ - split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model={models_dir}/inception_v3_quantized.tflite ! queue ! qtimlpostprocess settings='{{\"confidence\": 40.0}}' results=2 module=mobilenet-softmax labels={labels_dir}/classification.json ! video/x-raw,format=BGRA,width=640,height=360 ! queue ! mixer. \ + split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model={models_dir}/inception_v3_quantized.tflite ! queue ! qtimlpostprocess settings='{{\"confidence\": 40.0}}' results=2 module=mobilenet-softmax labels={labels_dir}/classification.json ! video/x-raw,format=RGBA,width=640,height=360 ! queue ! mixer. \ split. ! queue ! mixer. \ - split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model={models_dir}/hrnet_pose_quantized.tflite ! queue ! qtimlpostprocess settings={labels_dir}/hrnet_pose_settings.json results=2 module=hrnet labels={labels_dir}/hrnet_pose.json ! video/x-raw,format=BGRA,width=640,height=360 ! queue ! mixer. \ + split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model={models_dir}/hrnet_pose_quantized.tflite ! queue ! qtimlpostprocess settings={labels_dir}/hrnet_pose_settings.json results=2 module=hrnet labels={labels_dir}/hrnet_pose.json ! video/x-raw,format=RGBA,width=640,height=360 ! queue ! mixer. \ split. ! queue ! mixer. \ - split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model={models_dir}/deeplabv3_plus_mobilenet_quantized.tflite ! queue ! qtimlpostprocess module=deeplab-argmax labels={labels_dir}/deeplabv3_resnet50.json ! video/x-raw,format=BGRA,width=256,height=144 ! queue ! mixer. \ + split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model={models_dir}/deeplabv3_plus_mobilenet_quantized.tflite ! queue ! qtimlpostprocess module=deeplab-argmax labels={labels_dir}/deeplabv3_resnet50.json ! video/x-raw,format=RGBA ! queue ! mixer. \ " elif application == "ObjectDetection": pipeline = "gst-launch-1.0 " + in_src + f"! queue ! tee name=split \ split. ! queue ! qtivcomposer name=mixer ! queue ! gtksink \ split. ! queue ! qtimlvconverter ! queue ! qtimltflite {TFLITE_DELEGATE} model='{models_dir}/yolox_quantized.tflite' ! queue ! \ qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels='{labels_dir}/yolox.json' ! \ - video/x-raw,format=BGRA,width=640,height=360 ! queue ! mixer." + video/x-raw,format=RGBA,width=640,height=360 ! queue ! mixer." elif application == "Face Detection": if src == "On-Device-Camera": in_src = "qtiqmmfsrc name=camsrc ! video/x-raw,format=NV12,width=640,height=480,framerate=30/1" @@ -696,10 +696,10 @@ def create_pipeline(self, application, src): filesrc location={video_in} ! qtdemux ! h264parse ! v4l2h264dec capture-io-mode=4 output-io-mode=4 ! video/x-raw,format=NV12 ! tee name=split_2 ! queue ! mixer. \ filesrc location={video_in} ! qtdemux ! h264parse ! v4l2h264dec capture-io-mode=4 output-io-mode=4 ! video/x-raw,format=NV12 ! tee name=split_3 ! queue ! mixer. \ filesrc location={video_in} ! qtdemux ! h264parse ! v4l2h264dec capture-io-mode=4 output-io-mode=4 ! video/x-raw,format=NV12 ! tee name=split_4 ! queue ! mixer. \ - stage_01_inference. ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=BGRA,width=640,height=360 ! queue ! mixer. \ - stage_02_inference. ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=BGRA,width=640,height=360 ! queue ! mixer. \ - stage_03_inference. ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=BGRA,width=640,height=360 ! queue ! mixer. \ - stage_04_inference. ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=BGRA,width=640,height=360 ! queue ! mixer. " + stage_01_inference. ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=RGBA,width=640,height=360 ! queue ! mixer. \ + stage_02_inference. ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=RGBA,width=640,height=360 ! queue ! mixer. \ + stage_03_inference. ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=RGBA,width=640,height=360 ! queue ! mixer. \ + stage_04_inference. ! queue ! qtimlpostprocess settings='{{\"confidence\": 51.0}}' results=10 module=yolov8 labels={labels_dir}/yolox.json ! video/x-raw,format=RGBA,width=640,height=360 ! queue ! mixer. " if pipeline is not None: pipeline_thread = threading.Thread(target=self.execute, args=(pipeline,)) pipeline_thread.start() diff --git a/gstreamer/gst-sample-apps/CMakeLists.txt b/gstreamer/gst-sample-apps/CMakeLists.txt index f061e2ae..6ef282fd 100644 --- a/gstreamer/gst-sample-apps/CMakeLists.txt +++ b/gstreamer/gst-sample-apps/CMakeLists.txt @@ -3,6 +3,7 @@ set(GST_CAMERA_SAMPLE_APPS gst-camera-per-port-example gst-camera-burst-capture-example gst-camera-metadata-example + gst-fastcv-padding-strip-example ) if (ENABLE_GST_SAMPLE_APPS_CAMERA) diff --git a/gstreamer/gst-sample-apps/gst-fastcv-padding-strip-example/CMakeLists.txt b/gstreamer/gst-sample-apps/gst-fastcv-padding-strip-example/CMakeLists.txt index 7b25056c..d70ea25d 100644 --- a/gstreamer/gst-sample-apps/gst-fastcv-padding-strip-example/CMakeLists.txt +++ b/gstreamer/gst-sample-apps/gst-fastcv-padding-strip-example/CMakeLists.txt @@ -47,7 +47,6 @@ target_compile_definitions(${GST_EXAMPLE_BIN} PRIVATE target_link_libraries(${GST_EXAMPLE_BIN} PRIVATE ${GST_LIBRARIES} - gstappsutils ${EXTRA_LIBS} )