From b201c7d61bd92c450571c7c048499f1d5390739f Mon Sep 17 00:00:00 2001 From: yuanjinghua Date: Tue, 29 Aug 2023 01:04:01 +0800 Subject: [PATCH 1/7] Solve recording blur problem when use software encoder for some resolutions called by memory alignment Signed-off-by: yuanjinghua Change-Id: I657b1d1607b9ca0a3fc744231579200eef861547 --- .../memsource/gst_consumer_surface_pool.cpp | 4 + .../memsource/gst_consumer_surface_pool.h | 2 + .../include/gst_video_capture_pool.h | 3 + .../src/gst_video_capture_pool.cpp | 76 +++++++++++++++++++ 4 files changed, 85 insertions(+) diff --git a/services/engine/gstreamer/plugins/source/memsource/gst_consumer_surface_pool.cpp b/services/engine/gstreamer/plugins/source/memsource/gst_consumer_surface_pool.cpp index 53613dbc2..d6628e3f7 100644 --- a/services/engine/gstreamer/plugins/source/memsource/gst_consumer_surface_pool.cpp +++ b/services/engine/gstreamer/plugins/source/memsource/gst_consumer_surface_pool.cpp @@ -384,6 +384,9 @@ static GstFlowReturn gst_consumer_surface_pool_alloc_buffer(GstBufferPool *pool, if (gst_is_consumer_surface_memory(mem)) { *surfacemem = reinterpret_cast(mem); add_buffer_info(surfacepool, *surfacemem, *buffer); + if (surfacepool->update_video_meta != nullptr) { + surfacepool->update_video_meta(surfacepool, *surfacemem, *buffer); + } } gst_consumer_surface_pool_dump_gstbuffer(surfacepool, *buffer); return GST_FLOW_OK; @@ -460,6 +463,7 @@ static void gst_consumer_surface_pool_init(GstConsumerSurfacePool *pool) pool->priv = priv; pool->buffer_available = nullptr; pool->find_buffer = nullptr; + pool->update_video_meta = nullptr; pool->get_surface_buffer = gst_consumer_surface_pool_get_surface_buffer; pool->release_surface_buffer = gst_consumer_surface_pool_release_surface_buffer; priv->available_buf_count = 0; diff --git a/services/engine/gstreamer/plugins/source/memsource/gst_consumer_surface_pool.h b/services/engine/gstreamer/plugins/source/memsource/gst_consumer_surface_pool.h index c2865be2f..0cb9b423f 100644 --- a/services/engine/gstreamer/plugins/source/memsource/gst_consumer_surface_pool.h +++ b/services/engine/gstreamer/plugins/source/memsource/gst_consumer_surface_pool.h @@ -18,6 +18,7 @@ #include #include +#include "gst_consumer_surface_memory.h" #include "iconsumer_surface.h" #include "surface.h" @@ -55,6 +56,7 @@ struct _GstConsumerSurfacePool { OHOS::sptr &surface_buffer, gint32 &fencefd); void (*release_surface_buffer) (GstConsumerSurfacePool *pool, OHOS::sptr &surface_buffer, gint32 &fencefd); + void (*update_video_meta) (GstConsumerSurfacePool *pool, GstConsumerSurfaceMemory *mem, GstBuffer *buffer); }; struct _GstConsumerSurfacePoolClass { diff --git a/services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h b/services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h index a405789e0..9ad1177a5 100644 --- a/services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h +++ b/services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h @@ -39,6 +39,9 @@ struct _GstVideoCapturePool { GMutex pool_lock; gboolean cached_data; std::shared_ptr poolMgr; + guint plans = 0; + gint stride[GST_VIDEO_MAX_PLANES] = {0, 0, 0, 0}; + gsize offset[GST_VIDEO_MAX_PLANES] = {0, 0, 0, 0}; }; struct _GstVideoCapturePoolClass { diff --git a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp index 42a90068f..9ba206b59 100644 --- a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp +++ b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp @@ -16,6 +16,8 @@ #include "gst_video_capture_pool.h" #include #include "buffer_type_meta.h" +#include "diaplay_type.h" +#include "gst/video/gstvideometa.h" #include "surface.h" #include "scope_guard.h" #include "securec.h" @@ -29,6 +31,24 @@ using namespace OHOS; GST_DEBUG_CATEGORY_STATIC(gst_video_capture_pool_debug_category); #define GST_CAT_DEFAULT gst_video_capture_pool_debug_category +namespace { + const std::unordered_map FORMAT_PLANE_MAP = { + { PIXEL_FMT_RGBA_8888, 1 }, + { PIXEL_FMT_YCRCB_420_SP, 2}, + { PIXEL_FMT_YCBCR_420_SP, 2 }, + { PIXEL_FMT_YCBCR_420_P, 3 }, + }; + const std::unordered_map FORMAT_PLANE_MAP = { + { PIXEL_FMT_RGBA_8888, GST_VIDEO_FORMAT_RGBA }, + { PIXEL_FMT_YCRCB_420_SP, GST_VIDEO_FORMAT_NV21}, + { PIXEL_FMT_YCBCR_420_SP, GST_VIDEO_FORMAT_NV12 }, + { PIXEL_FMT_YCBCR_420_P, GST_VIDEO_FORMAT_I420 }, + }; + constexpr int32_t VIDEO_PLANE_0 = 0; + constexpr int32_t VIDEO_PLANE_1 = 1; + constexpr int32_t VIDEO_PLANE_2 = 2; +} + enum { PROP_0, PROP_CACHED_DATA, @@ -42,6 +62,8 @@ static GstFlowReturn gst_video_capture_pool_find_buffer(GstBufferPool *gstpool, static GstFlowReturn gst_video_capture_pool_get_buffer(GstConsumerSurfacePool *surfacepool, GstBuffer **buffer, bool *releasebuffer); static GstFlowReturn gst_video_capture_pool_release_buffer(GstConsumerSurfacePool *surfacepool, bool *releasebuffer); +static void gst_video_capture_pool_update_video_meta(GstConsumerSurfacePool *surfacepool, GstConsumerSurfaceMemory *mem, + GstBuffer *buffer); static void gst_video_capture_pool_class_init(GstVideoCapturePoolClass *klass) { @@ -63,6 +85,7 @@ static void gst_video_capture_pool_init(GstVideoCapturePool *pool) GstConsumerSurfacePool *surfacepool = GST_CONSUMER_SURFACE_POOL(pool); surfacepool->buffer_available = gst_video_capture_pool_buffer_available; surfacepool->find_buffer = gst_video_capture_pool_find_buffer; + surfacepool->update_video_meta = gst_video_capture_pool_update_video_meta; pool->cached_data = false; pool->poolMgr = nullptr; @@ -242,4 +265,57 @@ static GstFlowReturn gst_video_capture_pool_release_buffer(GstConsumerSurfacePoo surfacepool->release_surface_buffer(surfacepool, surfacebuffer, fencefd); return GST_FLOW_OK; +} + +static void gst_video_capture_pool_update_video_meta(GstConsumerSurfacePool *surfacepool, GstConsumerSurfaceMemory *mem, + GstBuffer *buffer); +{ + g_return_if_fail(surfacepool != nullptr && buffer != nullptr && mem != nullptr && mem->is_eos_frame == FALSE); + g_return_if_fail(mem->buffer_handle != nullptr && FORMAT_PLANE_MAP.count((PixelFormat)mem->pixel_format) != 0); + g_return_if_fail(mem->buffer_handle->stride > 0 && mem->buffer_handle->stride != (int32_t)mem->width); + + GstVideoBufferPool *pool = GST_VIDEO_CAPTURE_POOL_CAST(surfacepool); + g_return_if_fail(pool != nullptr); + + int32_t stride = mem->buffer_handle->stride; + if (stride != pool->stride[VIDEO_PLANE_0]) { // this judge optimize stride/offset info calculation. + switch (mem->pixel_format) { + case PIXEL_FMT_YCRCB_420_SP: // fall-through + case PIXEL_FMT_YCBCR_420_SP: + pool->stride[VIDEO_PLANE_0] = stride; + pool->stride[VIDEO_PLANE_1] = stride; + pool->offset[VIDEO_PLANE_0] = 0; + pool->offset[VIDEO_PLANE_1] = stride * mem->height; // plane 0 size is (stride * mem->height) + break; + case PIXEL_FMT_YCBCR_420_P: + pool->stride[VIDEO_PLANE_0] = stride; + pool->stride[VIDEO_PLANE_1] = stride >> 1; + pool->stride[VIDEO_PLANE_2] = stride >> 1; + pool->offset[VIDEO_PLANE_0] = 0; + pool->offset[VIDEO_PLANE_1] = stride * mem->height; // plane 0 size is (stride * mem->height) + pool->offset[VIDEO_PLANE_2] = pool->offset[VIDEO_PLANE_1] + stride * ((mem->height + 1) >> 1); + break; + case PIXEL_FMT_RGBA_8888: + pool->stride[VIDEO_PLANE_0] = stride; + pool->offset[VIDEO_PLANE_0] = 0; + break; + default: + GST_DEBUG_OBJECT(pool, "update buffer video meta ignore pixelformat %d", mem->pixel_format); + return; + } + pool->planes = FORMAT_PLANE_MAP.at((PixelFormat)mem->pixel_format); + } + GstVideoMeta *video_meta = gst_buffer_get_video_meta(buffer); + if (video_meta != nullptr) { + video_meta->n_planes = pool->planes; + video_meta->stride[VIDEO_PLANE_0] = pool->stride[VIDEO_PLANE_0]; + video_meta->stride[VIDEO_PLANE_1] = pool->stride[VIDEO_PLANE_1]; + video_meta->stride[VIDEO_PLANE_2] = pool->stride[VIDEO_PLANE_2]; + video_meta->offset[VIDEO_PLANE_0] = pool->offset[VIDEO_PLANE_0]; + video_meta->offset[VIDEO_PLANE_1] = pool->offset[VIDEO_PLANE_1]; + video_meta->offset[VIDEO_PLANE_2] = pool->offset[VIDEO_PLANE_2]; + } else { + gst_buffer_add_video_meta_full(buffer, GST_VIDEO_FRAME_FLAG_NONE, TO_GST_MAP.at((PixelFormat)mem->pixel_format), + mem->width, mem->height, pool->planes, pool->offset, pool->stride); + } } \ No newline at end of file -- Gitee From 77795fba325acf261fc46074059f054e702b32c4 Mon Sep 17 00:00:00 2001 From: acefighteryuan Date: Mon, 28 Aug 2023 17:29:34 +0000 Subject: [PATCH 2/7] update services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp. Signed-off-by: acefighteryuan --- .../plugins/source/videocapture/src/gst_video_capture_pool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp index 9ba206b59..c5f605ec3 100644 --- a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp +++ b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp @@ -16,7 +16,7 @@ #include "gst_video_capture_pool.h" #include #include "buffer_type_meta.h" -#include "diaplay_type.h" +#include "display_type.h" #include "gst/video/gstvideometa.h" #include "surface.h" #include "scope_guard.h" -- Gitee From 71a7ef933ca37a7c7d1f7e35bd88ce07abfc8dca Mon Sep 17 00:00:00 2001 From: acefighteryuan Date: Tue, 29 Aug 2023 01:09:49 +0000 Subject: [PATCH 3/7] update services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp. Signed-off-by: acefighteryuan --- .../plugins/source/videocapture/src/gst_video_capture_pool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp index c5f605ec3..26e86c040 100644 --- a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp +++ b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp @@ -38,7 +38,7 @@ namespace { { PIXEL_FMT_YCBCR_420_SP, 2 }, { PIXEL_FMT_YCBCR_420_P, 3 }, }; - const std::unordered_map FORMAT_PLANE_MAP = { + const std::unordered_map TO_GST_MAP = { { PIXEL_FMT_RGBA_8888, GST_VIDEO_FORMAT_RGBA }, { PIXEL_FMT_YCRCB_420_SP, GST_VIDEO_FORMAT_NV21}, { PIXEL_FMT_YCBCR_420_SP, GST_VIDEO_FORMAT_NV12 }, -- Gitee From 5160b5cae53e474200c734fd0b33da22d13105a3 Mon Sep 17 00:00:00 2001 From: acefighteryuan Date: Tue, 29 Aug 2023 01:18:50 +0000 Subject: [PATCH 4/7] update services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp. Signed-off-by: acefighteryuan --- .../plugins/source/videocapture/src/gst_video_capture_pool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp index 26e86c040..179d8bb6f 100644 --- a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp +++ b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp @@ -268,7 +268,7 @@ static GstFlowReturn gst_video_capture_pool_release_buffer(GstConsumerSurfacePoo } static void gst_video_capture_pool_update_video_meta(GstConsumerSurfacePool *surfacepool, GstConsumerSurfaceMemory *mem, - GstBuffer *buffer); + GstBuffer *buffer) { g_return_if_fail(surfacepool != nullptr && buffer != nullptr && mem != nullptr && mem->is_eos_frame == FALSE); g_return_if_fail(mem->buffer_handle != nullptr && FORMAT_PLANE_MAP.count((PixelFormat)mem->pixel_format) != 0); -- Gitee From 5ea56a85371c40759698ab322e5ee7631ef67fd2 Mon Sep 17 00:00:00 2001 From: acefighteryuan Date: Tue, 29 Aug 2023 01:35:30 +0000 Subject: [PATCH 5/7] update services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp. Signed-off-by: acefighteryuan --- .../plugins/source/videocapture/src/gst_video_capture_pool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp index 179d8bb6f..cd225e451 100644 --- a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp +++ b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp @@ -274,7 +274,7 @@ static void gst_video_capture_pool_update_video_meta(GstConsumerSurfacePool *sur g_return_if_fail(mem->buffer_handle != nullptr && FORMAT_PLANE_MAP.count((PixelFormat)mem->pixel_format) != 0); g_return_if_fail(mem->buffer_handle->stride > 0 && mem->buffer_handle->stride != (int32_t)mem->width); - GstVideoBufferPool *pool = GST_VIDEO_CAPTURE_POOL_CAST(surfacepool); + GstVideoCapturePool *pool = GST_VIDEO_CAPTURE_POOL_CAST(surfacepool); g_return_if_fail(pool != nullptr); int32_t stride = mem->buffer_handle->stride; -- Gitee From 796a228f66ce60475e67fec294eb211191bf9c8b Mon Sep 17 00:00:00 2001 From: acefighteryuan Date: Tue, 29 Aug 2023 01:43:10 +0000 Subject: [PATCH 6/7] update services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h. Signed-off-by: acefighteryuan --- .../source/videocapture/include/gst_video_capture_pool.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h b/services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h index 9ad1177a5..37fb1fdc5 100644 --- a/services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h +++ b/services/engine/gstreamer/plugins/source/videocapture/include/gst_video_capture_pool.h @@ -39,7 +39,7 @@ struct _GstVideoCapturePool { GMutex pool_lock; gboolean cached_data; std::shared_ptr poolMgr; - guint plans = 0; + guint planes = 0; gint stride[GST_VIDEO_MAX_PLANES] = {0, 0, 0, 0}; gsize offset[GST_VIDEO_MAX_PLANES] = {0, 0, 0, 0}; }; -- Gitee From 531a9c753dc8962e44b2209931ada187f032f978 Mon Sep 17 00:00:00 2001 From: acefighteryuan Date: Tue, 29 Aug 2023 01:54:12 +0000 Subject: [PATCH 7/7] update services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp. Signed-off-by: acefighteryuan --- .../plugins/source/videocapture/src/gst_video_capture_pool.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp index cd225e451..7d754a8e1 100644 --- a/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp +++ b/services/engine/gstreamer/plugins/source/videocapture/src/gst_video_capture_pool.cpp @@ -38,7 +38,7 @@ namespace { { PIXEL_FMT_YCBCR_420_SP, 2 }, { PIXEL_FMT_YCBCR_420_P, 3 }, }; - const std::unordered_map TO_GST_MAP = { + const std::unordered_map TO_GST_MAP = { { PIXEL_FMT_RGBA_8888, GST_VIDEO_FORMAT_RGBA }, { PIXEL_FMT_YCRCB_420_SP, GST_VIDEO_FORMAT_NV21}, { PIXEL_FMT_YCBCR_420_SP, GST_VIDEO_FORMAT_NV12 }, -- Gitee