From 4de1c90b1971260c30eb59910788857ec3ac8f33 Mon Sep 17 00:00:00 2001 From: qano Date: Mon, 13 May 2024 19:45:56 +0800 Subject: [PATCH] livephot develop Signed-off-by: qano --- bundle.json | 8 +- .../src/output/deferred_photo_proxy_napi.cpp | 11 +- .../src/output/photo_output_napi.cpp | 207 ++++++- frameworks/native/camera/BUILD.gn | 5 + .../camera/src/output/camera_photo_proxy.cpp | 101 ++++ .../src/output/deferred_photo_proxy.cpp | 60 ++- .../native/camera/src/output/photo_output.cpp | 27 +- .../camera/src/session/capture_session.cpp | 105 +++- .../native/camera/test/moduletest/BUILD.gn | 13 + .../src/camera_framework_moduletest.cpp | 2 +- .../native/camera/test/unittest/BUILD.gn | 22 + frameworks/native/ndk/BUILD.gn | 4 + .../camera/include/camera_photo_proxy.h | 52 ++ .../camera/include/output/capture_output.h | 2 +- .../include/output/deferred_photo_proxy.h | 32 +- .../camera/include/output/photo_output.h | 15 +- .../camera/include/session/capture_session.h | 29 +- interfaces/kits/js/camera_napi/BUILD.gn | 5 + .../camera_napi/include/camera_napi_utils.h | 6 + .../output/deferred_photo_proxy_napi.h | 6 +- .../include/output/photo_output_napi.h | 21 +- multimedia_camera_framework.gni | 14 - services/camera_service/BUILD.gn | 33 +- .../camera_service_ipc_interface_code.h | 5 +- .../binder/base/include/icapture_session.h | 8 + .../client/include/hcapture_session_proxy.h | 5 + .../client/src/hcapture_session_proxy.cpp | 57 ++ .../server/include/hcapture_session_stub.h | 4 + .../server/src/hcapture_session_stub.cpp | 37 ++ .../include/avcodec/audio_capturer_session.h | 53 ++ .../include/avcodec/audio_encoder.h | 55 ++ .../include/avcodec/audio_video_muxer.h | 61 +++ .../include/avcodec/avcodec_task_manager.h | 76 +++ .../avcodec/camera_server_photo_proxy.h | 70 +++ .../include/avcodec/common/audio_record.h | 157 ++++++ .../include/avcodec/common/frame_record.h | 209 ++++++++ .../include/avcodec/common/sample_callback.h | 33 ++ .../include/avcodec/common/sample_info.h | 135 +++++ .../include/avcodec/drain_manager.h | 61 +++ .../avcodec/moving_photo_video_cache.h | 80 +++ .../include/avcodec/video_encoder.h | 61 +++ services/camera_service/include/camera_log.h | 16 + .../camera_service/include/hcamera_device.h | 2 + .../camera_service/include/hcapture_session.h | 83 ++- .../camera_service/include/hstream_repeat.h | 9 +- .../src/avcodec/audio_capturer_session.cpp | 156 ++++++ .../src/avcodec/audio_encoder.cpp | 255 +++++++++ .../src/avcodec/audio_video_muxer.cpp | 136 +++++ .../src/avcodec/avcodec_task_manager.cpp | 247 +++++++++ .../src/avcodec/camera_server_photo_proxy.cpp | 168 ++++++ .../src/avcodec/moving_photo_video_cache.cpp | 186 +++++++ .../src/avcodec/sample_callback.cpp | 97 ++++ .../src/avcodec/video_encoder.cpp | 329 ++++++++++++ .../camera_service/src/hcamera_device.cpp | 24 + .../camera_service/src/hcapture_session.cpp | 507 +++++++++++++++++- .../camera_service/src/hstream_repeat.cpp | 47 +- .../include/base/blocking_queue.h | 13 +- .../include/base/task_manager/task_manager.h | 3 +- .../src/base/task_manager/task_manager.cpp | 5 +- .../src/base/task_manager/thread_pool.cpp | 2 + .../src/deferred_processing_service.cpp | 3 +- services/etc/camera_service.cfg | 5 +- 62 files changed, 4116 insertions(+), 124 deletions(-) create mode 100644 frameworks/native/camera/src/output/camera_photo_proxy.cpp create mode 100644 interfaces/inner_api/native/camera/include/camera_photo_proxy.h create mode 100644 services/camera_service/include/avcodec/audio_capturer_session.h create mode 100644 services/camera_service/include/avcodec/audio_encoder.h create mode 100644 services/camera_service/include/avcodec/audio_video_muxer.h create mode 100644 services/camera_service/include/avcodec/avcodec_task_manager.h create mode 100644 services/camera_service/include/avcodec/camera_server_photo_proxy.h create mode 100644 services/camera_service/include/avcodec/common/audio_record.h create mode 100644 services/camera_service/include/avcodec/common/frame_record.h create mode 100644 services/camera_service/include/avcodec/common/sample_callback.h create mode 100644 services/camera_service/include/avcodec/common/sample_info.h create mode 100644 services/camera_service/include/avcodec/drain_manager.h create mode 100644 services/camera_service/include/avcodec/moving_photo_video_cache.h create mode 100644 services/camera_service/include/avcodec/video_encoder.h create mode 100644 services/camera_service/src/avcodec/audio_capturer_session.cpp create mode 100644 services/camera_service/src/avcodec/audio_encoder.cpp create mode 100644 services/camera_service/src/avcodec/audio_video_muxer.cpp create mode 100644 services/camera_service/src/avcodec/avcodec_task_manager.cpp create mode 100644 services/camera_service/src/avcodec/camera_server_photo_proxy.cpp create mode 100644 services/camera_service/src/avcodec/moving_photo_video_cache.cpp create mode 100644 services/camera_service/src/avcodec/sample_callback.cpp create mode 100644 services/camera_service/src/avcodec/video_encoder.cpp diff --git a/bundle.json b/bundle.json index 2bc25d7da..9c908eb6a 100644 --- a/bundle.json +++ b/bundle.json @@ -24,9 +24,12 @@ "ability_runtime", "access_token", "ace_engine", + "audio_framework", + "av_codec", "bundle_framework", "c_utils", "common_event_service", + "data_share", "device_manager", "drivers_interface_camera", "drivers_interface_display", @@ -39,6 +42,8 @@ "hitrace", "ipc", "image_framework", + "media_foundation", + "media_library", "napi", "os_account", "player_framework", @@ -47,8 +52,7 @@ "sensor", "thermal_manager", "window_manager", - "memmgr", - "memmgr_override" + "memmgr" ], "third_party": [ ] diff --git a/frameworks/js/camera_napi/src/output/deferred_photo_proxy_napi.cpp b/frameworks/js/camera_napi/src/output/deferred_photo_proxy_napi.cpp index 8394b64e1..82d8d6de1 100644 --- a/frameworks/js/camera_napi/src/output/deferred_photo_proxy_napi.cpp +++ b/frameworks/js/camera_napi/src/output/deferred_photo_proxy_napi.cpp @@ -13,7 +13,9 @@ * limitations under the License. */ +#include "deferred_photo_proxy.h" #include "image_napi.h" +#include "photo_proxy.h" #include "pixel_map_napi.h" #include "hilog/log.h" #include "camera_log.h" @@ -23,7 +25,6 @@ namespace OHOS { namespace CameraStandard { thread_local napi_ref DeferredPhotoProxyNapi::sConstructor_ = nullptr; thread_local napi_value DeferredPhotoProxyNapi::sThumbnailPixelMap_ = nullptr; -thread_local sptr DeferredPhotoProxyNapi::sDeferredPhotoProxy_ = nullptr; thread_local uint32_t DeferredPhotoProxyNapi::deferredPhotoProxyTaskId = DEFERRED_PHOTO_PROXY_TASKID; DeferredPhotoProxyNapi::DeferredPhotoProxyNapi() : env_(nullptr), wrapper_(nullptr), thumbnailPixelMap_(nullptr) { @@ -53,7 +54,9 @@ napi_value DeferredPhotoProxyNapi::DeferredPhotoProxyNapiConstructor(napi_env en if (status == napi_ok && thisVar != nullptr) { std::unique_ptr obj = std::make_unique(); obj->env_ = env; - obj->deferredPhotoProxy_ = sDeferredPhotoProxy_; + + obj->deferredPhotoProxy_ = static_cast(sPhotoProxy_.GetRefPtr()); + obj->photoProxy_ = obj->deferredPhotoProxy_; status = napi_wrap(env, thisVar, reinterpret_cast(obj.get()), DeferredPhotoProxyNapi::DeferredPhotoProxyNapiDestructor, nullptr, nullptr); if (status == napi_ok) { @@ -115,9 +118,9 @@ napi_value DeferredPhotoProxyNapi::CreateDeferredPhotoProxy(napi_env env, sptr #include +#include #include #include +#include "output/deferred_photo_proxy_napi.h" #include "camera_buffer_handle_utils.h" #include "camera_error_code.h" +#include "camera_log.h" +#include "camera_napi_const.h" #include "camera_napi_security_utils.h" #include "camera_napi_template_utils.h" #include "camera_napi_utils.h" @@ -25,12 +30,17 @@ #include "camera_output_capability.h" #include "image_napi.h" #include "image_receiver.h" +#include "js_native_api.h" #include "pixel_map_napi.h" #include "image_packer.h" +#include "refbase.h" #include "video_key_info.h" #include "output/photo_output_napi.h" #include "output/photo_napi.h" -#include "output/deferred_photo_proxy_napi.h" +#include "camera_photo_proxy.h" +#include "ipc_skeleton.h" +#include "media_library_manager.h" +#include "media_library_comm_napi.h" namespace OHOS { namespace CameraStandard { @@ -42,7 +52,9 @@ thread_local uint32_t PhotoOutputNapi::photoOutputTaskId = CAMERA_PHOTO_OUTPUT_T static uv_sem_t g_captureStartSem; static bool g_isSemInited; static std::mutex g_photoImageMutex; -PhotoListener::PhotoListener(napi_env env, const sptr photoSurface) : env_(env), photoSurface_(photoSurface) + +PhotoListener::PhotoListener(napi_env env, const sptr photoSurface, wptr photoOutput) + : env_(env), photoSurface_(photoSurface), photoOutput_(photoOutput) { if (bufferProcessor_ == nullptr && photoSurface != nullptr) { bufferProcessor_ = std::make_shared (photoSurface); @@ -175,8 +187,67 @@ void PhotoListener::DeepCopyBuffer(sptr newSurfaceBuffer, sptr surfaceBuffer, bool isHighQuality) const +{ + MEDIA_INFO_LOG("ExecutePhotoAsset"); + napi_value result[ARGS_TWO] = {nullptr, nullptr}; + napi_value callback = nullptr; + napi_value retVal; + napi_get_undefined(env_, &result[PARAM0]); + napi_get_undefined(env_, &result[PARAM1]); + // deep copy buffer + sptr newSurfaceBuffer = SurfaceBuffer::Create(); + DeepCopyBuffer(newSurfaceBuffer, surfaceBuffer); + BufferHandle *bufferHandle = newSurfaceBuffer->GetBufferHandle(); + if (bufferHandle == nullptr) { + napi_value errorCode; + napi_create_int32(env_, CameraErrorCode::INVALID_ARGUMENT, &errorCode); + result[PARAM0] = errorCode; + MEDIA_ERR_LOG("invalid bufferHandle"); + } + newSurfaceBuffer->Map(); + auto photoOutput = photoOutput_.promote(); + string uri = ""; + int32_t cameraShotType = 0; + CreateMediaLibrary(surfaceBuffer, bufferHandle, isHighQuality, uri, cameraShotType); + MEDIA_INFO_LOG("CreateMediaLibrary result %{public}s, type %{public}d", uri.c_str(), cameraShotType); + result[PARAM1] = Media::MediaLibraryCommNapi::CreatePhotoAssetNapi(env_, uri, cameraShotType); + napi_get_reference_value(env_, capturePhotoAssetCb_, &callback); + napi_call_function(env_, nullptr, callback, ARGS_TWO, result, &retVal); + // return buffer to buffer queue + photoSurface_->ReleaseBuffer(surfaceBuffer, -1); + newSurfaceBuffer->Unmap(); +} + +void PhotoListener::CreateMediaLibrary(sptr surfaceBuffer, BufferHandle *bufferHandle, + bool isHighQuality, std::string &uri, int32_t &cameraShotType) const +{ + int64_t imageId = 0; + int32_t deferredProcessingType; + surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::imageId, imageId); + surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::deferredProcessingType, deferredProcessingType); + MEDIA_ERR_LOG("PhotoListener ExecutePhotoAsset imageId:%{public}" PRId64 ", deferredProcessingType:%{public}d", + imageId, deferredProcessingType); + // get buffer handle and photo info + int32_t photoWidth; + int32_t photoHeight; + surfaceBuffer->GetExtraData()->ExtraGet(OHOS::CameraStandard::dataWidth, photoWidth); + surfaceBuffer->GetExtraData()->ExtraGet(OHOS::CameraStandard::dataHeight, photoHeight); + MEDIA_INFO_LOG("photoWidth:%{public}d, photoHeight: %{public}d", photoWidth, photoHeight); + int32_t format = bufferHandle->format; + sptr photoProxy; + std::string imageIdStr = std::to_string(imageId); + photoProxy = new(std::nothrow) CameraPhotoProxy(bufferHandle, format, photoWidth, photoHeight, isHighQuality); + photoProxy->SetDeferredAttrs(imageIdStr, deferredProcessingType); + auto photoOutput = photoOutput_.promote(); + if (photoOutput && photoOutput->GetSession()) { + photoOutput->GetSession()->CreateMediaLibrary(photoProxy, uri, cameraShotType); + } +} + void PhotoListener::UpdateJSCallback(sptr photoSurface) const { + MEDIA_DEBUG_LOG("PhotoListener UpdateJSCallback enter"); sptr surfaceBuffer = nullptr; int32_t fence = -1; int64_t timestamp; @@ -190,18 +261,23 @@ void PhotoListener::UpdateJSCallback(sptr photoSurface) const int32_t isDegradedImage; surfaceBuffer->GetExtraData()->ExtraGet(OHOS::Camera::isDegradedImage, isDegradedImage); MEDIA_INFO_LOG("PhotoListener UpdateJSCallback isDegradedImage:%{public}d", isDegradedImage); - - if (isDegradedImage == 0) { + if ((callbackFlag & CAPTURE_PHOTO_ASSET) != 0) { + ExecutePhotoAsset(surfaceBuffer, isDegradedImage == 0); + } else if (isDegradedImage == 0 && (callbackFlag & CAPTURE_PHOTO) != 0) { ExecutePhoto(surfaceBuffer); - } else { + } else if (isDegradedImage != 0 && (callbackFlag & CAPTURE_DEFERRED_PHOTO) != 0) { ExecuteDeferredPhoto(surfaceBuffer); + } else { + MEDIA_INFO_LOG("PhotoListener on error callback"); } } void PhotoListener::UpdateJSCallbackAsync(sptr photoSurface) const { + MEDIA_DEBUG_LOG("PhotoListener UpdateJSCallbackAsync enter"); uv_loop_s* loop = nullptr; napi_get_uv_event_loop(env_, &loop); + MEDIA_INFO_LOG("PhotoListener UpdateJSCallbackAsync get loop"); if (!loop) { MEDIA_ERR_LOG("PhotoListener:UpdateJSCallbackAsync() failed to get event loop"); return; @@ -213,6 +289,7 @@ void PhotoListener::UpdateJSCallbackAsync(sptr photoSurface) const } std::unique_ptr callbackInfo = std::make_unique(photoSurface, this); work->data = callbackInfo.get(); + MEDIA_DEBUG_LOG("PhotoListener UpdateJSCallbackAsync uv_queue_work_with_qos start"); int ret = uv_queue_work_with_qos( loop, work, [](uv_work_t* work) {}, [](uv_work_t* work, int status) { @@ -244,15 +321,28 @@ void PhotoListener::SaveCallbackReference(const std::string &eventType, napi_val switch (eventTypeEnum) { case PhotoOutputEventType::CAPTURE_PHOTO_AVAILABLE: curCallbackRef = &capturePhotoCb_; + callbackFlag |= CAPTURE_PHOTO; break; case PhotoOutputEventType::CAPTURE_DEFERRED_PHOTO_AVAILABLE: curCallbackRef = &captureDeferredPhotoCb_; + callbackFlag |= CAPTURE_DEFERRED_PHOTO; + break; + case PhotoOutputEventType::CAPTURE_PHOTO_ASSET_AVAILABLE: + curCallbackRef = &capturePhotoAssetCb_; + callbackFlag |= CAPTURE_PHOTO_ASSET; break; default: MEDIA_ERR_LOG("Incorrect photo callback event type received from JS"); return; } - + if (eventTypeEnum == PhotoOutputEventType::CAPTURE_PHOTO_ASSET_AVAILABLE) { + auto photoOutput = photoOutput_.promote(); + if (photoOutput) { + photoOutput->AddDeferType(DeferredDeliveryImageType::DELIVERY_PHOTO); + } else { + MEDIA_ERR_LOG("cannot get photoOutput"); + } + } napi_ref callbackRef = nullptr; const int32_t refCount = 1; napi_status status = napi_create_reference(env_, callback, refCount, &callbackRef); @@ -268,9 +358,16 @@ void PhotoListener::RemoveCallbackRef(napi_env env, napi_value callback, const s if (eventType == CONST_CAPTURE_PHOTO_AVAILABLE) { napi_delete_reference(env_, capturePhotoCb_); capturePhotoCb_ = nullptr; + callbackFlag &= ~CAPTURE_PHOTO; } else if (eventType == CONST_CAPTURE_DEFERRED_PHOTO_AVAILABLE) { napi_delete_reference(env_, captureDeferredPhotoCb_); captureDeferredPhotoCb_ = nullptr; + callbackFlag &= ~CAPTURE_DEFERRED_PHOTO; + } else if (eventType == CONST_CAPTURE_PHOTO_ASSET_AVAILABLE) { + napi_delete_reference(env_, capturePhotoAssetCb_); + photoOutput_->DeferImageDeliveryFor(DeferredDeliveryImageType::DELIVERY_NONE); + callbackFlag &= ~CAPTURE_PHOTO_ASSET; + capturePhotoAssetCb_ = nullptr; } MEDIA_INFO_LOG("RemoveCallbackReference: js callback no find"); @@ -1048,13 +1145,17 @@ napi_value PhotoOutputNapi::Init(napi_env env, napi_value exports) int32_t refCount = 1; napi_property_descriptor photo_output_props[] = { + DECLARE_NAPI_FUNCTION("isMovingPhotoSupported", IsMovingPhotoSupported), + DECLARE_NAPI_FUNCTION("enableMovingPhoto", EnableMovingPhoto), DECLARE_NAPI_FUNCTION("getDefaultCaptureSetting", GetDefaultCaptureSetting), DECLARE_NAPI_FUNCTION("capture", Capture), DECLARE_NAPI_FUNCTION("confirmCapture", ConfirmCapture), DECLARE_NAPI_FUNCTION("release", Release), DECLARE_NAPI_FUNCTION("isMirrorSupported", IsMirrorSupported), DECLARE_NAPI_FUNCTION("setMirror", SetMirror), DECLARE_NAPI_FUNCTION("enableQuickThumbnail", EnableQuickThumbnail), - DECLARE_NAPI_FUNCTION("isQuickThumbnailSupported", IsQuickThumbnailSupported), DECLARE_NAPI_FUNCTION("on", On), - DECLARE_NAPI_FUNCTION("once", Once), DECLARE_NAPI_FUNCTION("off", Off), + DECLARE_NAPI_FUNCTION("isQuickThumbnailSupported", IsQuickThumbnailSupported), + DECLARE_NAPI_FUNCTION("on", On), + DECLARE_NAPI_FUNCTION("once", Once), + DECLARE_NAPI_FUNCTION("off", Off), DECLARE_NAPI_FUNCTION("deferImageDelivery", DeferImageDeliveryFor), DECLARE_NAPI_FUNCTION("deferImageDeliveryFor", DeferImageDeliveryFor), DECLARE_NAPI_FUNCTION("isDeferredImageDeliverySupported", IsDeferredImageDeliverySupported), @@ -1806,7 +1907,63 @@ napi_value PhotoOutputNapi::SetMirror(napi_env env, napi_callback_info info) } else { MEDIA_ERR_LOG("SetMirror call Failed!"); } + return result; +} + +napi_value PhotoOutputNapi::IsMovingPhotoSupported(napi_env env, napi_callback_info info) +{ + MEDIA_DEBUG_LOG("IsMotionPhotoSupported is called"); + napi_status status; + napi_value result = nullptr; + size_t argc = ARGS_ZERO; + napi_value argv[ARGS_ZERO]; + napi_value thisVar = nullptr; + + CAMERA_NAPI_GET_JS_ARGS(env, info, argc, argv, thisVar); + + napi_get_undefined(env, &result); + PhotoOutputNapi* photoOutputNapi = nullptr; + status = napi_unwrap(env, thisVar, reinterpret_cast(&photoOutputNapi)); + auto session = photoOutputNapi->GetPhotoOutput()->GetSession(); + if (status == napi_ok && photoOutputNapi != nullptr && session != nullptr) { + bool isSupported = session->IsMovingPhotoSupported(); + napi_get_boolean(env, isSupported, &result); + } else { + napi_get_boolean(env, false, &result); + MEDIA_ERR_LOG("IsMotionPhotoSupported call Failed!"); + } + return result; +} +napi_value PhotoOutputNapi::EnableMovingPhoto(napi_env env, napi_callback_info info) +{ + MEDIA_DEBUG_LOG("enableMovingPhoto is called"); + napi_status status; + napi_value result = nullptr; + size_t argc = ARGS_ONE; + napi_value argv[ARGS_ONE] = { 0 }; + napi_value thisVar = nullptr; + CAMERA_NAPI_GET_JS_ARGS(env, info, argc, argv, thisVar); + NAPI_ASSERT(env, argc == ARGS_ONE, "requires one parameter"); + napi_valuetype valueType = napi_undefined; + napi_typeof(env, argv[0], &valueType); + if (valueType != napi_boolean && !CameraNapiUtils::CheckError(env, INVALID_ARGUMENT)) { + return result; + } + napi_get_undefined(env, &result); + PhotoOutputNapi* photoOutputNapi = nullptr; + status = napi_unwrap(env, thisVar, reinterpret_cast(&photoOutputNapi)); + auto session = photoOutputNapi->GetPhotoOutput()->GetSession(); + if (status == napi_ok && photoOutputNapi != nullptr && session != nullptr) { + bool isEnableMovingPhoto; + napi_get_value_bool(env, argv[PARAM0], &isEnableMovingPhoto); + session->LockForControl(); + int32_t retCode = session->EnableMovingPhoto(isEnableMovingPhoto); + session->UnlockForControl(); + if (retCode != 0 && !CameraNapiUtils::CheckError(env, retCode)) { + return result; + } + } return result; } @@ -1890,7 +2047,7 @@ void PhotoOutputNapi::RegisterPhotoAvailableCallbackListener( } if (photoListener_ == nullptr) { MEDIA_INFO_LOG("new photoListener and register surface consumer listener"); - sptr photoListener = new (std::nothrow) PhotoListener(env, sPhotoSurface_); + sptr photoListener = new (std::nothrow) PhotoListener(env, sPhotoSurface_, photoOutput_); SurfaceError ret = sPhotoSurface_->RegisterConsumerListener((sptr&)photoListener); if (ret != SURFACE_ERROR_OK) { MEDIA_ERR_LOG("register surface consumer listener failed!"); @@ -1936,7 +2093,7 @@ void PhotoOutputNapi::RegisterDeferredPhotoProxyAvailableCallbackListener( } if (photoListener_ == nullptr) { MEDIA_INFO_LOG("new deferred photoListener and register surface consumer listener"); - sptr photoListener = new (std::nothrow) PhotoListener(env, sPhotoSurface_); + sptr photoListener = new (std::nothrow) PhotoListener(env, sPhotoSurface_, photoOutput_); SurfaceError ret = sPhotoSurface_->RegisterConsumerListener((sptr&)photoListener); if (ret != SURFACE_ERROR_OK) { MEDIA_ERR_LOG("register surface consumer listener failed!"); @@ -1954,6 +2111,33 @@ void PhotoOutputNapi::UnregisterDeferredPhotoProxyAvailableCallbackListener( } } +void PhotoOutputNapi::RegisterPhotoAssetAvailableCallbackListener( + napi_env env, napi_value callback, const std::vector& args, bool isOnce) +{ + if (sPhotoSurface_ == nullptr) { + MEDIA_ERR_LOG("sPhotoSurface_ is null!"); + return; + } + if (photoListener_ == nullptr) { + MEDIA_INFO_LOG("new photoListener and register surface consumer listener"); + sptr photoListener = new (std::nothrow) PhotoListener(env, sPhotoSurface_, photoOutput_); + SurfaceError ret = sPhotoSurface_->RegisterConsumerListener((sptr&)photoListener); + if (ret != SURFACE_ERROR_OK) { + MEDIA_ERR_LOG("register surface consumer listener failed!"); + } + photoListener_ = photoListener; + } + photoListener_->SaveCallbackReference(CONST_CAPTURE_PHOTO_ASSET_AVAILABLE, callback); +} + +void PhotoOutputNapi::UnregisterPhotoAssetAvailableCallbackListener( + napi_env env, napi_value callback, const std::vector& args) +{ + if (photoListener_ != nullptr) { + photoListener_->RemoveCallbackRef(env, callback, CONST_CAPTURE_PHOTO_ASSET_AVAILABLE); + } +} + void PhotoOutputNapi::RegisterCaptureStartCallbackListener( napi_env env, napi_value callback, const std::vector& args, bool isOnce) { @@ -2126,6 +2310,9 @@ const PhotoOutputNapi::EmitterFunctions& PhotoOutputNapi::GetEmitterFunctions() { CONST_CAPTURE_DEFERRED_PHOTO_AVAILABLE, { &PhotoOutputNapi::RegisterDeferredPhotoProxyAvailableCallbackListener, &PhotoOutputNapi::UnregisterDeferredPhotoProxyAvailableCallbackListener } }, + { CONST_CAPTURE_PHOTO_ASSET_AVAILABLE, { + &PhotoOutputNapi::RegisterPhotoAssetAvailableCallbackListener, + &PhotoOutputNapi::UnregisterPhotoAssetAvailableCallbackListener } }, { CONST_CAPTURE_START, { &PhotoOutputNapi::RegisterCaptureStartCallbackListener, &PhotoOutputNapi::UnregisterCaptureStartCallbackListener } }, diff --git a/frameworks/native/camera/BUILD.gn b/frameworks/native/camera/BUILD.gn index df70e21c6..296f83ff8 100644 --- a/frameworks/native/camera/BUILD.gn +++ b/frameworks/native/camera/BUILD.gn @@ -80,6 +80,7 @@ ohos_shared_library("camera_framework") { "src/input/camera_manager.cpp", "src/input/prelaunch_config.cpp", "src/output/camera_output_capability.cpp", + "src/output/camera_photo_proxy.cpp", "src/output/capture_output.cpp", "src/output/deferred_photo_proxy.cpp", "src/output/metadata_output.cpp", @@ -136,6 +137,8 @@ ohos_shared_library("camera_framework") { external_deps = [ "access_token:libtokenid_sdk", "c_utils:utils", + "data_share:datashare_common", + "data_share:datashare_consumer", "device_manager:devicemanagersdk", "drivers_interface_camera:libbuffer_handle_sequenceable_1.0", "drivers_interface_camera:libcamera_proxy_1.0", @@ -153,6 +156,8 @@ ohos_shared_library("camera_framework") { "image_framework:image", "image_framework:image_native", "ipc:ipc_core", + "media_library:media_library", + "media_library:media_library_manager", "os_account:libaccountkits", "os_account:os_account_innerkits", "safwk:system_ability_fwk", diff --git a/frameworks/native/camera/src/output/camera_photo_proxy.cpp b/frameworks/native/camera/src/output/camera_photo_proxy.cpp new file mode 100644 index 000000000..4ebf85373 --- /dev/null +++ b/frameworks/native/camera/src/output/camera_photo_proxy.cpp @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include "camera_log.h" +#include "camera_photo_proxy.h" + +namespace OHOS { +namespace CameraStandard { + +CameraPhotoProxy::CameraPhotoProxy() +{ + photoId_ = ""; + deferredProcType_ = 0; + photoWidth_ = 0; + photoHeight_ = 0; + bufferHandle_ = nullptr; + fileSize_ = 0; +} + +CameraPhotoProxy::CameraPhotoProxy(const BufferHandle* bufferHandle, int32_t format, + int32_t photoWidth, int32_t photoHeight, bool isHighQuality) +{ + MEDIA_INFO_LOG("CameraPhotoProxy"); + bufferHandle_ = bufferHandle; + format_ = format; + photoWidth_ = photoWidth; + photoHeight_ = photoHeight; + fileSize_ = 0; + isHighQuality_ = isHighQuality; + MEDIA_INFO_LOG("format = %{public}d, width = %{public}d, height = %{public}d", + format_, photoWidth, photoHeight); +} + +CameraPhotoProxy::~CameraPhotoProxy() +{ + std::lock_guard lock(mutex_); + MEDIA_INFO_LOG("~CameraPhotoProxy"); + fileSize_ = 0; +} + +void CameraPhotoProxy::ReadFromParcel(MessageParcel &parcel) +{ + std::lock_guard lock(mutex_); + photoId_ = parcel.ReadString(); + deferredProcType_ = parcel.ReadInt32(); + isDeferredPhoto_ = parcel.ReadInt32(); + format_ = parcel.ReadInt32(); + photoWidth_ = parcel.ReadInt32(); + photoHeight_ = parcel.ReadInt32(); + isHighQuality_ = parcel.ReadBool(); + bufferHandle_ = ReadBufferHandle(parcel); + MEDIA_INFO_LOG("PhotoProxy::ReadFromParcel"); +} + +void CameraPhotoProxy::WriteToParcel(MessageParcel &parcel) +{ + std::lock_guard lock(mutex_); + parcel.WriteString(photoId_); + parcel.WriteInt32(deferredProcType_); + parcel.WriteInt32(isDeferredPhoto_); + parcel.WriteInt32(format_); + parcel.WriteInt32(photoWidth_); + parcel.WriteInt32(photoHeight_); + parcel.WriteBool(isHighQuality_); + if (bufferHandle_) { + MEDIA_DEBUG_LOG("PhotoProxy::WriteToParcel %{public}d", bufferHandle_->fd); + bool ret = WriteBufferHandle(parcel, *bufferHandle_); + if (ret == false) { + MEDIA_ERR_LOG("Failure, Reason: WriteBufferHandle return false"); + } + } else { + MEDIA_ERR_LOG("PhotoProxy::WriteToParcel without bufferHandle_"); + } + MEDIA_INFO_LOG("PhotoProxy::WriteToParcel"); +} + +void CameraPhotoProxy::SetDeferredAttrs(std::string photoId, int32_t deferredProcType) +{ + isDeferredPhoto_ = 1; + photoId_ = photoId; + deferredProcType_ = deferredProcType; +} +} // namespace CameraStandard +} // namespace OHOS diff --git a/frameworks/native/camera/src/output/deferred_photo_proxy.cpp b/frameworks/native/camera/src/output/deferred_photo_proxy.cpp index 333bf6ba8..e2adbcba8 100644 --- a/frameworks/native/camera/src/output/deferred_photo_proxy.cpp +++ b/frameworks/native/camera/src/output/deferred_photo_proxy.cpp @@ -21,6 +21,7 @@ #include "camera_buffer_handle_utils.h" #include "camera_log.h" #include "output/deferred_photo_proxy.h" +#include "photo_proxy.h" namespace OHOS { namespace CameraStandard { @@ -29,8 +30,8 @@ DeferredPhotoProxy::DeferredPhotoProxy() { photoId_ = ""; deferredProcType_ = 0; - thumbnailWidth_ = 0; - thumbnailHeight_ = 0; + photoWidth_ = 0; + photoHeight_ = 0; bufferHandle_ = nullptr; fileDataAddr_ = nullptr; fileSize_ = 0; @@ -44,8 +45,6 @@ DeferredPhotoProxy::DeferredPhotoProxy(const BufferHandle* bufferHandle, MEDIA_INFO_LOG("DeferredPhotoProxy"); photoId_ = imageId; deferredProcType_ = deferredProcType; - thumbnailWidth_ = 0; - thumbnailHeight_ = 0; bufferHandle_ = bufferHandle; fileDataAddr_ = nullptr; fileSize_ = 0; @@ -56,20 +55,20 @@ DeferredPhotoProxy::DeferredPhotoProxy(const BufferHandle* bufferHandle, } DeferredPhotoProxy::DeferredPhotoProxy(const BufferHandle* bufferHandle, - std::string imageId, int32_t deferredProcType, int32_t thumbnailWidth, int32_t thumbnailHeight) + std::string imageId, int32_t deferredProcType, int32_t photoWidth, int32_t photoHeight) { MEDIA_INFO_LOG("DeferredPhotoProxy"); photoId_ = imageId; deferredProcType_ = deferredProcType; - thumbnailWidth_ = thumbnailWidth; - thumbnailHeight_ = thumbnailHeight; + photoWidth_ = photoWidth; + photoHeight_ = photoHeight; bufferHandle_ = bufferHandle; fileDataAddr_ = nullptr; fileSize_ = 0; isMmaped_ = false; buffer_ = nullptr; MEDIA_INFO_LOG("imageId: = %{public}s, deferredProcType = %{public}d, width = %{public}d, height = %{public}d", - imageId.c_str(), deferredProcType, thumbnailWidth, thumbnailHeight); + imageId.c_str(), deferredProcType, photoWidth, photoHeight); } DeferredPhotoProxy::DeferredPhotoProxy(const BufferHandle* bufferHandle, @@ -78,8 +77,6 @@ DeferredPhotoProxy::DeferredPhotoProxy(const BufferHandle* bufferHandle, MEDIA_INFO_LOG("DeferredPhotoProxy"); photoId_ = imageId; deferredProcType_ = deferredProcType; - thumbnailWidth_ = 0; - thumbnailHeight_ = 0; bufferHandle_ = bufferHandle; fileDataAddr_ = nullptr; fileSize_ = fileSize; @@ -108,8 +105,8 @@ void DeferredPhotoProxy::ReadFromParcel(MessageParcel &parcel) std::lock_guard lock(mutex_); photoId_ = parcel.ReadString(); deferredProcType_ = parcel.ReadInt32(); - thumbnailWidth_ = parcel.ReadInt32(); - thumbnailHeight_ = parcel.ReadInt32(); + photoWidth_ = parcel.ReadInt32(); + photoHeight_ = parcel.ReadInt32(); bufferHandle_ = ReadBufferHandle(parcel); MEDIA_INFO_LOG("DeferredPhotoProxy::ReadFromParcel"); } @@ -119,8 +116,8 @@ void DeferredPhotoProxy::WriteToParcel(MessageParcel &parcel) std::lock_guard lock(mutex_); parcel.WriteString(photoId_); parcel.WriteInt32(deferredProcType_); - parcel.WriteInt32(thumbnailWidth_); - parcel.WriteInt32(thumbnailHeight_); + parcel.WriteInt32(photoWidth_); + parcel.WriteInt32(photoHeight_); WriteBufferHandle(parcel, *bufferHandle_); MEDIA_INFO_LOG("DeferredPhotoProxy::WriteToParcel"); } @@ -132,14 +129,14 @@ std::string DeferredPhotoProxy::GetPhotoId() return photoId_; } -DeferredProcType DeferredPhotoProxy::GetDeferredProcType() +Media::DeferredProcType DeferredPhotoProxy::GetDeferredProcType() { MEDIA_INFO_LOG("DeferredPhotoProxy::GetDeferredProcType"); std::lock_guard lock(mutex_); if (deferredProcType_ == 0) { - return BACKGROUND; + return Media::DeferredProcType::BACKGROUND; } else { - return OFFLINE; + return Media::DeferredProcType::OFFLINE; } } @@ -159,6 +156,16 @@ void* DeferredPhotoProxy::GetFileDataAddr() return fileDataAddr_; } +Media::PhotoFormat DeferredPhotoProxy::GetFormat() +{ + return Media::PhotoFormat::RGBA; +} + +Media::PhotoQuality DeferredPhotoProxy::GetPhotoQuality() +{ + return Media::PhotoQuality::LOW; +} + size_t DeferredPhotoProxy::GetFileSize() { MEDIA_INFO_LOG("DeferredPhotoProxy::GetFileSize"); @@ -174,12 +181,27 @@ size_t DeferredPhotoProxy::GetFileSize() int32_t DeferredPhotoProxy::GetWidth() { - return thumbnailWidth_; + return photoWidth_; } int32_t DeferredPhotoProxy::GetHeight() { - return thumbnailHeight_; + return photoHeight_; +} + +void DeferredPhotoProxy::Release() +{ + MEDIA_INFO_LOG("CameraPhotoProxy release start"); +} + +std::string DeferredPhotoProxy::GetDisplayName() +{ + return ""; +} + +std::string DeferredPhotoProxy::GetExtension() +{ + return ""; } } // namespace CameraStandard } // namespace OHOS diff --git a/frameworks/native/camera/src/output/photo_output.cpp b/frameworks/native/camera/src/output/photo_output.cpp index 072c44bcf..f45accda4 100644 --- a/frameworks/native/camera/src/output/photo_output.cpp +++ b/frameworks/native/camera/src/output/photo_output.cpp @@ -383,7 +383,10 @@ int32_t PhotoOutput::Capture(std::shared_ptr photoCaptureSe auto itemStream = static_cast(GetStream().GetRefPtr()); int32_t errCode = CAMERA_UNKNOWN_ERROR; if (itemStream) { + MEDIA_DEBUG_LOG("Capture start"); + captureSession->StartMovingPhotoCapture(); errCode = itemStream->Capture(photoCaptureSettings->GetCaptureMetadataSetting()); + MEDIA_DEBUG_LOG("Capture End"); } else { MEDIA_ERR_LOG("PhotoOutput::Capture() itemStream is nullptr"); } @@ -412,7 +415,10 @@ int32_t PhotoOutput::Capture() auto itemStream = static_cast(GetStream().GetRefPtr()); int32_t errCode = CAMERA_UNKNOWN_ERROR; if (itemStream) { + MEDIA_DEBUG_LOG("Capture start"); + captureSession->StartMovingPhotoCapture(); errCode = itemStream->Capture(captureMetadataSetting); + MEDIA_DEBUG_LOG("Capture end"); } else { MEDIA_ERR_LOG("PhotoOutput::Capture() itemStream is nullptr"); } @@ -576,6 +582,23 @@ int32_t PhotoOutput::DeferImageDeliveryFor(DeferredDeliveryImageType type) return 0; } +int32_t PhotoOutput::AddDeferType(DeferredDeliveryImageType type) +{ + MEDIA_INFO_LOG("PhotoOutput AddDeferType type:%{public}d!", type); + deferredType_ = type; + return 0; +} + +void PhotoOutput::SetSession(wptr captureSession) +{ + MEDIA_INFO_LOG("PhotoOutput SetSession"); + CaptureOutput::SetSession(captureSession); + if (deferredType_ == DeferredDeliveryImageType::DELIVERY_NONE) { + return; + } + DeferImageDeliveryFor(deferredType_); +} + int32_t PhotoOutput::IsDeferredImageDeliverySupported(DeferredDeliveryImageType type) { MEDIA_INFO_LOG("IsDeferredImageDeliverySupported type:%{public}d!", type); @@ -675,7 +698,7 @@ int32_t PhotoOutput::EnableAutoHighQualityPhoto(bool enabled) MEDIA_ERR_LOG("PhotoOutput EnableAutoHighQualityPhoto error"); return OPERATION_NOT_ALLOWED; } - + if (isAutoHighQualityPhotoSupported == -1) { MEDIA_ERR_LOG("PhotoOutput EnableAutoHighQualityPhoto not supported"); return INVALID_ARGUMENT; @@ -685,7 +708,7 @@ int32_t PhotoOutput::EnableAutoHighQualityPhoto(bool enabled) return res; } -void PhotoOutput::ProcessSnapshotDurationUpdates(int32_t snapshotDuration) +void PhotoOutput::ProcessSnapshotDurationUpdates(int32_t snapshotDuration) __attribute__((no_sanitize("cfi"))) { std::lock_guard lock(outputCallbackMutex_); if (appCallback_ != nullptr) { diff --git a/frameworks/native/camera/src/session/capture_session.cpp b/frameworks/native/camera/src/session/capture_session.cpp index c9b2842a1..66f7c01d3 100644 --- a/frameworks/native/camera/src/session/capture_session.cpp +++ b/frameworks/native/camera/src/session/capture_session.cpp @@ -37,6 +37,7 @@ #include "output/preview_output.h" #include "output/video_output.h" #include "camera_error_code.h" +#include "media_photo_asset_proxy.h" namespace OHOS { namespace CameraStandard { @@ -869,6 +870,18 @@ void CaptureSession::SetCallback(std::shared_ptr callback) return; } +void CaptureSession::CreateMediaLibrary(sptr photoProxy, std::string &uri, int32_t &cameraShotType) +{ + int32_t errorCode = CAMERA_OK; + std::lock_guard lock(sessionCallbackMutex_); + if (captureSession_) { + errorCode = captureSession_->CreateMediaLibrary(photoProxy, uri, cameraShotType); + if (errorCode != CAMERA_OK) { + MEDIA_ERR_LOG("Failed to create media library, errorCode: %{public}d", errorCode); + } + } +} + std::shared_ptr CaptureSession::GetApplicationCallback() { std::lock_guard lock(sessionCallbackMutex_); @@ -2037,7 +2050,7 @@ void CaptureSession::ProcessSnapshotDurationUpdates(const uint64_t timestamp, if (ret != CAM_META_SUCCESS || metadataItem.count <= 0) { return; } - int32_t duration = static_cast(metadataItem.data.ui32[0]); + const int32_t duration = static_cast(metadataItem.data.ui32[0]); if (duration != prevDuration_.load()) { ((sptr &)photoOutput_)->ProcessSnapshotDurationUpdates(duration); } @@ -3873,6 +3886,96 @@ int32_t CaptureSession::EnableFeature(SceneFeature feature, bool isEnable) return retCode; } +bool CaptureSession::IsMovingPhotoSupported() +{ + CAMERA_SYNC_TRACE; + MEDIA_DEBUG_LOG("Enter IsMovingPhotoSupported"); + if (inputDevice_ == nullptr) { + MEDIA_ERR_LOG("CaptureSession::IsMovingPhotoSupported camera device is null"); + return false; + } + auto deviceInfo = inputDevice_->GetCameraDeviceInfo(); + if (deviceInfo == nullptr) { + MEDIA_ERR_LOG("CaptureSession::IsMovingPhotoSupported camera deviceInfo is null"); + return false; + } + std::shared_ptr metadata = deviceInfo->GetMetadata(); + camera_metadata_item_t metadataItem; + vector modes = {}; + int ret = Camera::FindCameraMetadataItem(metadata->get(), OHOS_ABILITY_MOVING_PHOTO, &metadataItem); + if (ret == CAM_META_SUCCESS) { + uint32_t step = 3; + for (uint32_t index = 0; index < metadataItem.count - 1;) { + if (metadataItem.data.i32[index + 1] == 1) { + modes.push_back(metadataItem.data.i32[index]); + } + MEDIA_DEBUG_LOG("IsMovingPhotoSupported mode:%{public}d", metadataItem.data.i32[index]); + index += step; + } + } + return std::find(modes.begin(), modes.end(), GetMode()) != modes.end(); +} + +int32_t CaptureSession::EnableMovingPhoto(bool isEnable) +{ + CAMERA_SYNC_TRACE; + MEDIA_INFO_LOG("Enter EnableMovingPhoto, isEnable:%{public}d", isEnable); + if (!IsMovingPhotoSupported()) { + MEDIA_ERR_LOG("IsMovingPhotoSupported is false"); + return CameraErrorCode::SERVICE_FATL_ERROR; + } + if (!(IsSessionConfiged() || IsSessionCommited())) { + MEDIA_ERR_LOG("CaptureSession Failed EnableMovingPhoto!, session not configed"); + return CameraErrorCode::SERVICE_FATL_ERROR; + } + bool status = false; + int32_t ret; + camera_metadata_item_t item; + ret = Camera::FindCameraMetadataItem(changedMetadata_->get(), OHOS_CONTROL_MOVING_PHOTO, &item); + uint8_t enableValue = static_cast(isEnable ? OHOS_CAMERA_MOVING_PHOTO_ON : OHOS_CAMERA_MOVING_PHOTO_OFF); + if (ret == CAM_META_ITEM_NOT_FOUND) { + status = changedMetadata_->addEntry(OHOS_CONTROL_MOVING_PHOTO, &enableValue, 1); + } else if (ret == CAM_META_SUCCESS) { + status = changedMetadata_->updateEntry(OHOS_CONTROL_MOVING_PHOTO, &enableValue, 1); + } + if (!status) { + MEDIA_ERR_LOG("CaptureSession::EnableMovingPhoto Failed to enable"); + } + if (captureSession_) { + int32_t errCode = captureSession_->EnableMovingPhoto(isEnable); + if (errCode != CAMERA_OK) { + MEDIA_ERR_LOG("Failed to EnableMovingPhoto!, %{public}d", errCode); + return errCode; + } + } else { + MEDIA_ERR_LOG("CaptureSession::EnableMovingPhoto() captureSession_ is nullptr"); + } + isMovingPhotoEnabled_ = isEnable; + return CameraErrorCode::SUCCESS; +} + +bool CaptureSession::IsMovingPhotoEnabled() +{ + return isMovingPhotoEnabled_; +} + +int32_t CaptureSession::StartMovingPhotoCapture() +{ + CAMERA_SYNC_TRACE; + MEDIA_INFO_LOG("StartMovingPhotoCapture"); + if (!IsMovingPhotoSupported()) { + MEDIA_ERR_LOG("IsMovingPhotoSupported is false"); + return CameraErrorCode::SERVICE_FATL_ERROR; + } + if (captureSession_) { + int32_t errCode = captureSession_->StartMovingPhotoCapture(); + if (errCode != CAMERA_OK) { + MEDIA_ERR_LOG("Failed to StartMovingPhotoCapture!, %{public}d", errCode); + } + } + return CameraErrorCode::SUCCESS; +} + void CaptureSession::SetMacroStatusCallback(std::shared_ptr callback) { std::lock_guard lock(sessionCallbackMutex_); diff --git a/frameworks/native/camera/test/moduletest/BUILD.gn b/frameworks/native/camera/test/moduletest/BUILD.gn index 1afa610b5..96d3d0114 100644 --- a/frameworks/native/camera/test/moduletest/BUILD.gn +++ b/frameworks/native/camera/test/moduletest/BUILD.gn @@ -23,11 +23,15 @@ ohos_moduletest("camera_framework_moduletest") { "${graphic_surface_path}/surface/include", "${graphic_surface_path}/surface/include", "${multimedia_camera_framework_path}/services/camera_service/include", + "${multimedia_camera_framework_path}/services/camera_service/include/avcodec", + "${multimedia_camera_framework_path}/services/camera_service/include/avcodec/common", "${multimedia_camera_framework_path}/services/camera_service/binder/base/include", "${multimedia_camera_framework_path}/services/camera_service/binder/client/include", "${multimedia_camera_framework_path}/services/camera_service/binder/server/include", "${multimedia_camera_framework_path}/interfaces/inner_api/native/camera/include", "${multimedia_camera_framework_path}/interfaces/inner_api/native/test", + "${multimedia_camera_framework_path}/services/deferred_processing_service/include/base/task_manager", + "${multimedia_camera_framework_path}/services/deferred_processing_service/include/base/task_manager/task_group", "${base_security_path}/interfaces/innerkits/accesstoken/include", "${base_security_path}/interfaces/innerkits/token_setproc/include", ] @@ -48,7 +52,12 @@ ohos_moduletest("camera_framework_moduletest") { "access_token:libnativetoken", "access_token:libprivacy_sdk", "access_token:libtoken_setproc", + "audio_framework:audio_capturer", + "audio_framework:audio_client", + "av_codec:av_codec_client", "c_utils:utils", + "data_share:datashare_common", + "data_share:datashare_consumer", "drivers_interface_camera:libcamera_proxy_1.0", "drivers_interface_camera:libcamera_proxy_1.1", "drivers_interface_camera:metadata", @@ -60,6 +69,10 @@ ohos_moduletest("camera_framework_moduletest") { "hitrace:hitrace_meter", "init:libbegetutil", "ipc:ipc_core", + "media_foundation:media_foundation", + "media_foundation:native_media_core", + "media_library:media_library", + "media_library:media_library_manager", "safwk:system_ability_fwk", "samgr:samgr_proxy", ] diff --git a/frameworks/native/camera/test/moduletest/src/camera_framework_moduletest.cpp b/frameworks/native/camera/test/moduletest/src/camera_framework_moduletest.cpp index 9e0f6bf56..4b701d575 100644 --- a/frameworks/native/camera/test/moduletest/src/camera_framework_moduletest.cpp +++ b/frameworks/native/camera/test/moduletest/src/camera_framework_moduletest.cpp @@ -4253,7 +4253,7 @@ HWTEST_F(CameraFrameworkModuleTest, camera_fwcoverage_moduletest_007, TestSize.L bool isSupported = true; int32_t flashModeSupported = camSession->IsFlashModeSupported(flashMode, isSupported); - EXPECT_EQ(setFlashMode, 7400103); + EXPECT_EQ(flashModeSupported, 7400103); std::vector getSupportedFlashModes = camSession->GetSupportedFlashModes(); EXPECT_EQ(getSupportedFlashModes.empty(), true); diff --git a/frameworks/native/camera/test/unittest/BUILD.gn b/frameworks/native/camera/test/unittest/BUILD.gn index fd2c495f6..b95ee80cb 100644 --- a/frameworks/native/camera/test/unittest/BUILD.gn +++ b/frameworks/native/camera/test/unittest/BUILD.gn @@ -22,11 +22,15 @@ ohos_unittest("camera_framework_unittest_v1_1") { "./include", "${graphic_surface_path}/surface/include", "${multimedia_camera_framework_path}/services/camera_service/include", + "${multimedia_camera_framework_path}/services/camera_service/include/avcodec", + "${multimedia_camera_framework_path}/services/camera_service/include/avcodec/common", "${multimedia_camera_framework_path}/services/camera_service/binder/base/include", "${multimedia_camera_framework_path}/services/camera_service/binder/client/include", "${multimedia_camera_framework_path}/services/camera_service/binder/server/include", "${multimedia_camera_framework_path}/interfaces/inner_api/native/camera/include", "${multimedia_camera_framework_path}/interfaces/inner_api/native/test", + "${multimedia_camera_framework_path}/services/deferred_processing_service/include/base/task_manager", + "${multimedia_camera_framework_path}/services/deferred_processing_service/include/base/task_manager/task_group", "${base_security_path}/interfaces/innerkits/accesstoken/include", "${base_security_path}/interfaces/innerkits/token_setproc/include", ] @@ -47,7 +51,12 @@ ohos_unittest("camera_framework_unittest_v1_1") { "access_token:libnativetoken", "access_token:libprivacy_sdk", "access_token:libtoken_setproc", + "audio_framework:audio_capturer", + "audio_framework:audio_client", + "av_codec:av_codec_client", "c_utils:utils", + "data_share:datashare_common", + "data_share:datashare_consumer", "drivers_interface_camera:libcamera_proxy_1.0", "drivers_interface_camera:libcamera_proxy_1.1", "drivers_interface_camera:metadata", @@ -59,6 +68,10 @@ ohos_unittest("camera_framework_unittest_v1_1") { "hitrace:hitrace_meter", "image_framework:image_native", "ipc:ipc_core", + "media_foundation:media_foundation", + "media_foundation:native_media_core", + "media_library:media_library", + "media_library:media_library_manager", "safwk:system_ability_fwk", ] @@ -97,6 +110,8 @@ ohos_unittest("camera_ndk_unittest_v1_1") { "./include", "${multimedia_camera_framework_path}/interfaces/kits/native/include", "${multimedia_camera_framework_path}/interfaces/inner_api/native/test", + "${multimedia_camera_framework_path}/services/deferred_processing_service/include/base/task_manager", + "${multimedia_camera_framework_path}/services/deferred_processing_service/include/base/task_manager/task_group", "${base_security_path}/interfaces/innerkits/accesstoken/include", "${base_security_path}/interfaces/innerkits/token_setproc/include", ] @@ -118,7 +133,10 @@ ohos_unittest("camera_ndk_unittest_v1_1") { "access_token:libnativetoken", "access_token:libprivacy_sdk", "access_token:libtoken_setproc", + "av_codec:av_codec_client", "c_utils:utils", + "data_share:datashare_common", + "data_share:datashare_consumer", "drivers_interface_camera:libcamera_proxy_1.0", "drivers_interface_camera:libcamera_proxy_1.1", "drivers_interface_camera:metadata", @@ -130,6 +148,10 @@ ohos_unittest("camera_ndk_unittest_v1_1") { "hitrace:hitrace_meter", "image_framework:image_native", "ipc:ipc_core", + "media_foundation:media_foundation", + "media_foundation:native_media_core", + "media_library:media_library", + "media_library:media_library_manager", "safwk:system_ability_fwk", "sensor:sensor_interface_native", ] diff --git a/frameworks/native/ndk/BUILD.gn b/frameworks/native/ndk/BUILD.gn index 701a447d5..8220d3a90 100644 --- a/frameworks/native/ndk/BUILD.gn +++ b/frameworks/native/ndk/BUILD.gn @@ -68,6 +68,8 @@ ohos_shared_library("ohcamera") { external_deps = [ "c_utils:utils", + "data_share:datashare_common", + "data_share:datashare_consumer", "graphic_surface:surface", "hilog:libhilog", "hisysevent:libhisysevent", @@ -75,6 +77,8 @@ ohos_shared_library("ohcamera") { "image_framework:image", "image_framework:image_native", "ipc:ipc_core", + "media_library:media_library", + "media_library:media_library_manager", ] innerapi_tags = [ "ndk" ] diff --git a/interfaces/inner_api/native/camera/include/camera_photo_proxy.h b/interfaces/inner_api/native/camera/include/camera_photo_proxy.h new file mode 100644 index 000000000..9326e3791 --- /dev/null +++ b/interfaces/inner_api/native/camera/include/camera_photo_proxy.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_CAMERA_PHOTO_PROXY_H +#define OHOS_CAMERA_PHOTO_PROXY_H + +#include "message_parcel.h" +#include "surface.h" + +namespace OHOS { +namespace CameraStandard { + +class CameraPhotoProxy : public RefBase { +public: + CameraPhotoProxy(); + CameraPhotoProxy(const BufferHandle* bufferHandle, int32_t format, int32_t photoWidth, + int32_t photoHeight, bool isHighQuality); + virtual ~CameraPhotoProxy(); + void ReadFromParcel(MessageParcel &parcel); + void WriteToParcel(MessageParcel &parcel); + void SetDeferredAttrs(std::string photoId, int32_t deferredProcType); + +private: + int32_t CameraFreeBufferHandle(BufferHandle *handle); + const BufferHandle* bufferHandle_; + int32_t format_; + int32_t photoWidth_; + int32_t photoHeight_; + void* fileDataAddr_; + size_t fileSize_; + bool isHighQuality_; + std::mutex mutex_; + std::string photoId_; + int32_t deferredProcType_; + int32_t isDeferredPhoto_; + sptr photoSurface_; +}; +} // namespace CameraStandard +} // namespace OHOS +#endif // OHOS_CAMERA_DEFERRED_PHOTO_PROXY_H \ No newline at end of file diff --git a/interfaces/inner_api/native/camera/include/output/capture_output.h b/interfaces/inner_api/native/camera/include/output/capture_output.h index 32cb46e26..fa018ae2b 100644 --- a/interfaces/inner_api/native/camera/include/output/capture_output.h +++ b/interfaces/inner_api/native/camera/include/output/capture_output.h @@ -111,7 +111,7 @@ public: StreamType GetStreamType(); sptr GetStream(); sptr GetSession(); - void SetSession(wptr captureSession); + virtual void SetSession(wptr captureSession); std::mutex asyncOpMutex_; std::mutex outputCallbackMutex_; int32_t SetPhotoProfile(Profile& profile); diff --git a/interfaces/inner_api/native/camera/include/output/deferred_photo_proxy.h b/interfaces/inner_api/native/camera/include/output/deferred_photo_proxy.h index ad0c85a8f..e8af4992e 100644 --- a/interfaces/inner_api/native/camera/include/output/deferred_photo_proxy.h +++ b/interfaces/inner_api/native/camera/include/output/deferred_photo_proxy.h @@ -18,39 +18,43 @@ #include #include "message_parcel.h" +#include "buffer_handle.h" +#include "photo_proxy.h" +#include namespace OHOS { namespace CameraStandard { -enum DeferredProcType { - BACKGROUND = 0, - OFFLINE, -}; -class DeferredPhotoProxy : public RefBase { +class DeferredPhotoProxy : public Media::PhotoProxy { public: DeferredPhotoProxy(); DeferredPhotoProxy(const BufferHandle* bufferHandle, std::string imageId, int32_t deferredProcType); DeferredPhotoProxy(const BufferHandle* bufferHandle, std::string imageId, int32_t deferredProcType, - int32_t thumbnailWidth, int32_t thumbnailHeight); + int32_t photoWidth, int32_t photoHeight); DeferredPhotoProxy(const BufferHandle* bufferHandle, std::string imageId, int32_t deferredProcType, uint8_t* buffer, size_t fileSize); virtual ~DeferredPhotoProxy(); void ReadFromParcel(MessageParcel &parcel); void WriteToParcel(MessageParcel &parcel); - std::string GetPhotoId(); - DeferredProcType GetDeferredProcType(); - void* GetFileDataAddr(); - size_t GetFileSize(); - int32_t GetWidth(); - int32_t GetHeight(); + std::string GetDisplayName() override; + std::string GetExtension() override; + std::string GetPhotoId() override; + Media::DeferredProcType GetDeferredProcType() override; + void* GetFileDataAddr() override; + size_t GetFileSize() override; + int32_t GetWidth() override; + int32_t GetHeight() override; + Media::PhotoFormat GetFormat() override; + Media::PhotoQuality GetPhotoQuality() override; + void Release() override; private: uint8_t* buffer_; const BufferHandle* bufferHandle_; std::string photoId_; int32_t deferredProcType_; - int32_t thumbnailWidth_; - int32_t thumbnailHeight_; + int32_t photoWidth_; + int32_t photoHeight_; void* fileDataAddr_; size_t fileSize_; bool isMmaped_; diff --git a/interfaces/inner_api/native/camera/include/output/photo_output.h b/interfaces/inner_api/native/camera/include/output/photo_output.h index 23c480d1e..4588cb57a 100644 --- a/interfaces/inner_api/native/camera/include/output/photo_output.h +++ b/interfaces/inner_api/native/camera/include/output/photo_output.h @@ -322,6 +322,18 @@ public: */ int32_t IsDeferredImageDeliverySupported(DeferredDeliveryImageType type); + /** + * @brief Add the deferredImageDelivery type when on photoAssetAvailable. + * + */ + int32_t AddDeferType(DeferredDeliveryImageType type); + + /** + * @brief Set the captureSession. + * + */ + void SetSession(wptr captureSession) override; + /** * @brief To check the deferredImageDelivery capability is enable or not. * @@ -330,7 +342,7 @@ public: int32_t IsDeferredImageDeliveryEnabled(DeferredDeliveryImageType type); void ProcessSnapshotDurationUpdates(int32_t snapshotDuration); - + /** * @brief To check the auto high quality photo is supported or not. * @@ -359,6 +371,7 @@ public: sptr deferredSurface_; private: + DeferredDeliveryImageType deferredType_ = DeferredDeliveryImageType::DELIVERY_NONE; std::shared_ptr appCallback_; sptr cameraSvcCallback_; std::shared_ptr defaultCaptureSetting_; diff --git a/interfaces/inner_api/native/camera/include/session/capture_session.h b/interfaces/inner_api/native/camera/include/session/capture_session.h index 1681f0d43..4ff2ab84f 100644 --- a/interfaces/inner_api/native/camera/include/session/capture_session.h +++ b/interfaces/inner_api/native/camera/include/session/capture_session.h @@ -39,6 +39,7 @@ #include "refbase.h" #include "color_space_info_parse.h" #include "capture_scene_const.h" +#include "camera_photo_proxy.h" namespace OHOS { namespace CameraStandard { @@ -351,6 +352,15 @@ public: */ void SetCallback(std::shared_ptr callback); + /** + * @brief Set the moving photo callback. + * + * @param photoProxy Requested for the pointer where moving photo callback is present. + * @param uri get uri for medialibary. + * @param cameraShotType get cameraShotType for medialibary. + */ + void CreateMediaLibrary(sptr photoProxy, std::string &uri, int32_t &cameraShotType); + /** * @brief Get the application callback information. * @@ -967,6 +977,21 @@ public: */ int32_t EnableMacro(bool isEnable); + /** + * @brief Check current status is support motion photo. + */ + bool IsMovingPhotoSupported(); + + /** + * @brief Enable motion photo. + */ + int32_t EnableMovingPhoto(bool isEnable); + + /** + * @brief startMotionPhotoCapture. + */ + int32_t StartMovingPhotoCapture(); + /** * @brief Check current status is support moon capture boost or not. */ @@ -1054,7 +1079,7 @@ public: * @return errCode. */ int32_t SetSensorSensitivity(uint32_t sensitivity); - + /** * @brief Get camera sensor sensitivity. * @param sensitivity current sensitivity value. @@ -1137,6 +1162,7 @@ public: void EnableDeferredType(DeferredDeliveryImageType deferredType); void SetUserId(); + bool IsMovingPhotoEnabled(); bool IsImageDeferred(); int32_t EnableAutoHighQualityPhoto(bool enabled); @@ -1152,6 +1178,7 @@ protected: std::map beautyTypeAndLevels_; std::shared_ptr metadataResultProcessor_ = nullptr; bool isImageDeferred_; + std::atomic isMovingPhotoEnabled_ { false }; static const std::unordered_map metaExposureModeMap_; static const std::unordered_map fwkExposureModeMap_; diff --git a/interfaces/kits/js/camera_napi/BUILD.gn b/interfaces/kits/js/camera_napi/BUILD.gn index cf02b6fdd..c4704f7ab 100644 --- a/interfaces/kits/js/camera_napi/BUILD.gn +++ b/interfaces/kits/js/camera_napi/BUILD.gn @@ -86,6 +86,8 @@ ohos_shared_library("camera_napi") { deps = [ "${multimedia_camera_framework_path}/frameworks/native/camera:camera_framework" ] external_deps = [ "c_utils:utils", + "data_share:datashare_common", + "data_share:datashare_consumer", "drivers_interface_camera:libcamera_proxy_1.0", "drivers_interface_camera:libcamera_proxy_1.1", "graphic_surface:surface", @@ -95,6 +97,9 @@ ohos_shared_library("camera_napi") { "image_framework:image", "image_framework:image_native", "ipc:ipc_core", + "media_library:media_library", + "media_library:media_library_manager", + "media_library:medialibrary_nutils", "napi:ace_napi", ] sanitize = { diff --git a/interfaces/kits/js/camera_napi/include/camera_napi_utils.h b/interfaces/kits/js/camera_napi/include/camera_napi_utils.h index bb4bdc814..4be5af2a5 100644 --- a/interfaces/kits/js/camera_napi/include/camera_napi_utils.h +++ b/interfaces/kits/js/camera_napi/include/camera_napi_utils.h @@ -23,6 +23,12 @@ #include "js_native_api_types.h" #include "napi/native_node_api.h" +#ifdef NAPI_ASSERT +#undef NAPI_ASSERT +#endif + +#define NAPI_ASSERT(env, assertion, message) NAPI_ASSERT_BASE(env, assertion, message, nullptr) + #define CAMERA_NAPI_GET_JS_ARGS(env, info, argc, argv, thisVar) \ do { \ void* data; \ diff --git a/interfaces/kits/js/camera_napi/include/output/deferred_photo_proxy_napi.h b/interfaces/kits/js/camera_napi/include/output/deferred_photo_proxy_napi.h index 89bdffbe7..34a82bb35 100644 --- a/interfaces/kits/js/camera_napi/include/output/deferred_photo_proxy_napi.h +++ b/interfaces/kits/js/camera_napi/include/output/deferred_photo_proxy_napi.h @@ -16,14 +16,15 @@ #ifndef DEFERRED_PHOTO_PROXY_NAPI_H_ #define DEFERRED_PHOTO_PROXY_NAPI_H_ -#include "camera_napi_utils.h" #include "deferred_photo_proxy.h" +#include "photo_proxy_napi.h" +#include "camera_napi_utils.h" namespace OHOS { namespace CameraStandard { static const char DEFERRED_PHOTO_NAPI_CLASS_NAME[] = "DeferredPhotoProxy"; -class DeferredPhotoProxyNapi { +class DeferredPhotoProxyNapi : public Media::PhotoProxyNapi { public: static napi_value Init(napi_env env, napi_value exports); static napi_value CreateDeferredPhotoProxy(napi_env env, sptr deferredPhotoProxy); @@ -42,7 +43,6 @@ private: static thread_local napi_ref sConstructor_; static thread_local napi_value sThumbnailPixelMap_; - static thread_local sptr sDeferredPhotoProxy_; static thread_local uint32_t deferredPhotoProxyTaskId; napi_env env_; diff --git a/interfaces/kits/js/camera_napi/include/output/photo_output_napi.h b/interfaces/kits/js/camera_napi/include/output/photo_output_napi.h index 337305b54..7badccd9d 100644 --- a/interfaces/kits/js/camera_napi/include/output/photo_output_napi.h +++ b/interfaces/kits/js/camera_napi/include/output/photo_output_napi.h @@ -35,6 +35,7 @@ static const std::string CONST_CAPTURE_FRAME_SHUTTER = "frameShutter"; static const std::string CONST_CAPTURE_ERROR = "error"; static const std::string CONST_CAPTURE_PHOTO_AVAILABLE = "photoAvailable"; static const std::string CONST_CAPTURE_DEFERRED_PHOTO_AVAILABLE = "deferredPhotoProxyAvailable"; +static const std::string CONST_CAPTURE_PHOTO_ASSET_AVAILABLE = "photoAssetAvailable"; static const std::string CONST_CAPTURE_FRAME_SHUTTER_END = "frameShutterEnd"; static const std::string CONST_CAPTURE_READY = "captureReady"; static const std::string CONST_CAPTURE_ESTIMATED_CAPTURE_DURATION = "estimatedCaptureDuration"; @@ -60,6 +61,7 @@ enum PhotoOutputEventType { CAPTURE_INVALID_TYPE, CAPTURE_PHOTO_AVAILABLE, CAPTURE_DEFERRED_PHOTO_AVAILABLE, + CAPTURE_PHOTO_ASSET_AVAILABLE, CAPTURE_ESTIMATED_CAPTURE_DURATION, CAPTURE_START_WITH_INFO }; @@ -71,6 +73,7 @@ static EnumHelper PhotoOutputEventTypeHelper({ {CAPTURE_ERROR, CONST_CAPTURE_ERROR}, {CAPTURE_PHOTO_AVAILABLE, CONST_CAPTURE_PHOTO_AVAILABLE}, {CAPTURE_DEFERRED_PHOTO_AVAILABLE, CONST_CAPTURE_DEFERRED_PHOTO_AVAILABLE}, + {CAPTURE_PHOTO_ASSET_AVAILABLE, CONST_CAPTURE_PHOTO_ASSET_AVAILABLE}, {CAPTURE_FRAME_SHUTTER_END, CONST_CAPTURE_FRAME_SHUTTER_END}, {CAPTURE_READY, CONST_CAPTURE_READY}, {CAPTURE_ESTIMATED_CAPTURE_DURATION, CONST_CAPTURE_ESTIMATED_CAPTURE_DURATION}, @@ -97,9 +100,13 @@ private: sptr photoSurface_ = nullptr; }; +constexpr int32_t CAPTURE_PHOTO = 1 << 0; +constexpr int32_t CAPTURE_DEFERRED_PHOTO = 1 << 1; +constexpr int32_t CAPTURE_PHOTO_ASSET = 1 << 2; + class PhotoListener : public IBufferConsumerListener { public: - explicit PhotoListener(napi_env env, const sptr photoSurface); + explicit PhotoListener(napi_env env, const sptr photoSurface, wptr photoOutput); ~PhotoListener() = default; void OnBufferAvailable() override; void SaveCallbackReference(const std::string &eventType, napi_value callback); @@ -110,14 +117,20 @@ private: std::mutex mutex_; napi_env env_; sptr photoSurface_; + wptr photoOutput_; shared_ptr bufferProcessor_; void UpdateJSCallback(sptr photoSurface) const; void UpdateJSCallbackAsync(sptr photoSurface) const; void ExecutePhoto(sptr surfaceBfuffer) const; void ExecuteDeferredPhoto(sptr surfaceBuffer) const; void DeepCopyBuffer(sptr newSurfaceBuffer, sptr surfaceBuffer) const; + void ExecutePhotoAsset(sptr surfaceBuffer, bool isHighQuality) const; + void CreateMediaLibrary(sptr surfaceBuffer, BufferHandle *bufferHandle, + bool isHighQuality, std::string &uri, int32_t &cameraShotType) const; napi_ref capturePhotoCb_; napi_ref captureDeferredPhotoCb_; + napi_ref capturePhotoAssetCb_; + int32_t callbackFlag = 0; }; class RawPhotoListener : public IBufferConsumerListener { @@ -254,6 +267,8 @@ public: static napi_value EnableAutoHighQualityPhoto(napi_env env, napi_callback_info info); static int32_t MapQualityLevelFromJs(int32_t jsQuality, PhotoCaptureSetting::QualityLevel& nativeQuality); static int32_t MapImageRotationFromJs(int32_t jsRotation, PhotoCaptureSetting::RotationConfig& nativeRotation); + static napi_value IsMovingPhotoSupported(napi_env env, napi_callback_info info); + static napi_value EnableMovingPhoto(napi_env env, napi_callback_info info); PhotoOutputNapi(); ~PhotoOutputNapi() override; @@ -282,6 +297,10 @@ private: napi_env env, napi_value callback, const std::vector& args, bool isOnce); void UnregisterDeferredPhotoProxyAvailableCallbackListener( napi_env env, napi_value callback, const std::vector& args); + void RegisterPhotoAssetAvailableCallbackListener( + napi_env env, napi_value callback, const std::vector& args, bool isOnce); + void UnregisterPhotoAssetAvailableCallbackListener( + napi_env env, napi_value callback, const std::vector& args); void RegisterCaptureStartCallbackListener( napi_env env, napi_value callback, const std::vector& args, bool isOnce); void UnregisterCaptureStartCallbackListener(napi_env env, napi_value callback, const std::vector& args); diff --git a/multimedia_camera_framework.gni b/multimedia_camera_framework.gni index adb07bce5..59556fea1 100644 --- a/multimedia_camera_framework.gni +++ b/multimedia_camera_framework.gni @@ -21,20 +21,6 @@ base_security_path = "//base/security/access_token" driver_disply_path = "//drivers/peripheral/display" window_manager_path = "//foundation/window/window_manager" system_safwk_path = "//utils/system/safwk" -memmgr_root = "//foundation/resourceschedule/memmgr" -memmgr_plugin_root = "//foundation/resourceschedule/memmgr_override" - -use_memmgr_plugin = false -if (defined(global_parts_info) && - defined(global_parts_info.resourceschedule_memmgr_override)) { - use_memmgr_plugin = true -} - -use_memmgr = false -if (defined(global_parts_info) && - defined(global_parts_info.resourceschedule_memmgr)) { - use_memmgr = true -} use_sensor = false if (defined(global_parts_info) && defined(global_parts_info.sensors_sensor)) { diff --git a/services/camera_service/BUILD.gn b/services/camera_service/BUILD.gn index d1a887e49..983f16ebb 100644 --- a/services/camera_service/BUILD.gn +++ b/services/camera_service/BUILD.gn @@ -18,6 +18,7 @@ ohos_shared_library("camera_service") { branch_protector_ret = "pac_ret" install_enable = true sources = [ + "${multimedia_camera_framework_path}/frameworks/native/camera/src/output/camera_photo_proxy.cpp", "binder/client/src/hcamera_device_callback_proxy.cpp", "binder/client/src/hcamera_listener_proxy.cpp", "binder/client/src/hcamera_service_callback_proxy.cpp", @@ -31,6 +32,14 @@ ohos_shared_library("camera_service") { "binder/server/src/hstream_capture_stub.cpp", "binder/server/src/hstream_metadata_stub.cpp", "binder/server/src/hstream_repeat_stub.cpp", + "src/avcodec/audio_capturer_session.cpp", + "src/avcodec/audio_encoder.cpp", + "src/avcodec/audio_video_muxer.cpp", + "src/avcodec/avcodec_task_manager.cpp", + "src/avcodec/camera_server_photo_proxy.cpp", + "src/avcodec/moving_photo_video_cache.cpp", + "src/avcodec/sample_callback.cpp", + "src/avcodec/video_encoder.cpp", "src/camera_fwk_metadata_utils.cpp", "src/camera_timer.cpp", "src/camera_util.cpp", @@ -64,6 +73,8 @@ ohos_shared_library("camera_service") { "//foundation/multimedia/camera_framework/services/camera_service/include", "${multimedia_camera_framework_path}/services/camera_service/include/dfx", "${multimedia_camera_framework_path}/services/camera_service/include/smooth_zoom", + "${multimedia_camera_framework_path}/services/camera_service/include/avcodec", + "${multimedia_camera_framework_path}/services/camera_service/include/avcodec/common", "//foundation/multimedia/camera_framework/services/camera_service/binder/base/include", "//foundation/multimedia/camera_framework/services/camera_service/binder/client/include", "//foundation/multimedia/camera_framework/services/camera_service/binder/server/include", @@ -103,9 +114,20 @@ ohos_shared_library("camera_service") { "access_token:libaccesstoken_sdk", "access_token:libprivacy_sdk", "access_token:libtokenid_sdk", + "audio_framework:audio_capturer", + "audio_framework:audio_client", + "av_codec:av_codec_client", + "av_codec:native_media_acodec", + "av_codec:native_media_aenc", + "av_codec:native_media_avcencinfo", + "av_codec:native_media_avmuxer", + "av_codec:native_media_codecbase", + "av_codec:native_media_venc", "bundle_framework:appexecfwk_base", "bundle_framework:appexecfwk_core", "c_utils:utils", + "data_share:datashare_common", + "data_share:datashare_consumer", "drivers_interface_camera:libcamera_proxy_1.0", "drivers_interface_camera:libcamera_proxy_1.1", "drivers_interface_camera:libcamera_proxy_1.2", @@ -123,6 +145,11 @@ ohos_shared_library("camera_service") { "hitrace:hitrace_meter", "ipc:ipc_core", "ipc:ipc_single", + "media_foundation:media_foundation", + "media_foundation:native_media_core", + "media_library:media_library", + "media_library:media_library_manager", + "memmgr:memmgrclient", "os_account:libaccountkits", "os_account:os_account_innerkits", "safwk:system_ability_fwk", @@ -136,12 +163,6 @@ ohos_shared_library("camera_service") { defines += [ "CAMERA_USE_SENSOR" ] } - if (use_memmgr_plugin) { - external_deps += [ "memmgr_override:memmgrclient" ] - } else if (use_memmgr) { - external_deps += [ "memmgr:memmgrclient" ] - } - include_dirs += [ "//drivers/peripheral/camera/interfaces/include", "//drivers/peripheral/camera/interfaces/hdi_ipc", diff --git a/services/camera_service/binder/base/include/camera_service_ipc_interface_code.h b/services/camera_service/binder/base/include/camera_service_ipc_interface_code.h index 756d77b63..8f120d0bf 100644 --- a/services/camera_service/binder/base/include/camera_service_ipc_interface_code.h +++ b/services/camera_service/binder/base/include/camera_service_ipc_interface_code.h @@ -133,7 +133,10 @@ enum CaptureSessionInterfaceCode { CAMERA_CAPTURE_GET_ACTIVE_COLOR_SPACE, CAMERA_CAPTURE_SET_COLOR_SPACE, CAMERA_CAPTURE_SESSION_SET_SMOOTH_ZOOM, - CAMERA_CAPTURE_SESSION_SET_FEATURE_MODE + CAMERA_CAPTURE_SESSION_SET_FEATURE_MODE, + CAMERA_CAPTURE_SESSION_ENABLE_MOTION_PHOTO, + CAMERA_CAPTURE_SESSION_START_MOVING_PHOTO_CAPTURE, + CAMERA_CAPTURE_SESSION_CREATE_MEDIA_LIBRARY_MANAGER, }; /** diff --git a/services/camera_service/binder/base/include/icapture_session.h b/services/camera_service/binder/base/include/icapture_session.h index 7e60ce91c..9de0a0943 100644 --- a/services/camera_service/binder/base/include/icapture_session.h +++ b/services/camera_service/binder/base/include/icapture_session.h @@ -21,6 +21,7 @@ #include "icapture_session_callback.h" #include "iremote_broker.h" #include "istream_common.h" +#include "camera_photo_proxy.h" namespace OHOS { namespace CameraStandard { @@ -85,6 +86,13 @@ public: virtual int32_t SetFeatureMode(int32_t featureMode) = 0; + virtual int32_t EnableMovingPhoto(bool isEnable) = 0; + + virtual int32_t StartMovingPhotoCapture() = 0; + + virtual int32_t CreateMediaLibrary(sptr &photoProxy, + std::string &uri, int32_t &cameraShotType) = 0; + DECLARE_INTERFACE_DESCRIPTOR(u"ICaptureSession"); }; } // namespace CameraStandard diff --git a/services/camera_service/binder/client/include/hcapture_session_proxy.h b/services/camera_service/binder/client/include/hcapture_session_proxy.h index c24856321..643145404 100644 --- a/services/camera_service/binder/client/include/hcapture_session_proxy.h +++ b/services/camera_service/binder/client/include/hcapture_session_proxy.h @@ -60,6 +60,11 @@ public: int32_t SetFeatureMode(int32_t featureMode) override; + int32_t EnableMovingPhoto(bool isEnable) override; + + int32_t StartMovingPhotoCapture() override; + + int32_t CreateMediaLibrary(sptr &photoProxy, std::string &uri, int32_t &cameraShotType) override; private: static inline BrokerDelegator delegator_; }; diff --git a/services/camera_service/binder/client/src/hcapture_session_proxy.cpp b/services/camera_service/binder/client/src/hcapture_session_proxy.cpp index a6a728a08..ae0912540 100644 --- a/services/camera_service/binder/client/src/hcapture_session_proxy.cpp +++ b/services/camera_service/binder/client/src/hcapture_session_proxy.cpp @@ -16,6 +16,7 @@ #include "hcapture_session_proxy.h" #include "camera_log.h" #include "camera_service_ipc_interface_code.h" +#include namespace OHOS { namespace CameraStandard { @@ -333,5 +334,61 @@ int32_t HCaptureSessionProxy::SetFeatureMode(int32_t featureMode) } return error; } + +int32_t HCaptureSessionProxy::EnableMovingPhoto(bool isEnable) +{ + MessageParcel data; + MessageParcel reply; + MessageOption option; + + data.WriteInterfaceToken(GetDescriptor()); + data.WriteBool(isEnable); + int error = Remote()->SendRequest( + static_cast(CaptureSessionInterfaceCode::CAMERA_CAPTURE_SESSION_ENABLE_MOTION_PHOTO), + data, reply, option); + if (error != ERR_NONE) { + MEDIA_ERR_LOG("HCaptureSessionProxy enable moving photo failed, error: %{public}d", error); + } + return error; +} + +int32_t HCaptureSessionProxy::StartMovingPhotoCapture() +{ + MessageParcel data; + MessageParcel reply; + MessageOption option; + + data.WriteInterfaceToken(GetDescriptor()); + int error = Remote()->SendRequest( + static_cast(CaptureSessionInterfaceCode::CAMERA_CAPTURE_SESSION_START_MOVING_PHOTO_CAPTURE), + data, reply, option); + if (error != ERR_NONE) { + MEDIA_ERR_LOG("HCaptureSessionProxy start moving capture, error: %{public}d", error); + } + return error; +} + +int32_t HCaptureSessionProxy::CreateMediaLibrary(sptr &photoProxy, + std::string &uri, int32_t &cameraShotType) +{ + MessageParcel data; + MessageParcel reply; + MessageOption option; + if (photoProxy == nullptr) { + MEDIA_ERR_LOG("HCaptureSessionProxy CreateMediaLibrary photoProxy is null"); + return IPC_PROXY_ERR; + } + data.WriteInterfaceToken(GetDescriptor()); + photoProxy->WriteToParcel(data); + int error = Remote()->SendRequest( + static_cast(CaptureSessionInterfaceCode::CAMERA_CAPTURE_SESSION_CREATE_MEDIA_LIBRARY_MANAGER), + data, reply, option); + if (error != ERR_NONE) { + MEDIA_ERR_LOG("HCaptureSessionProxy CreateMediaLibrary failed, error: %{public}d", error); + } + uri = reply.ReadString(); + cameraShotType = reply.ReadInt32(); + return error; +} } // namespace CameraStandard } // namespace OHOS diff --git a/services/camera_service/binder/server/include/hcapture_session_stub.h b/services/camera_service/binder/server/include/hcapture_session_stub.h index d8b190a08..48db83f59 100644 --- a/services/camera_service/binder/server/include/hcapture_session_stub.h +++ b/services/camera_service/binder/server/include/hcapture_session_stub.h @@ -16,6 +16,7 @@ #ifndef OHOS_CAMERA_HCAPTURE_SESSION_STUB_H #define OHOS_CAMERA_HCAPTURE_SESSION_STUB_H +#include "camera_photo_proxy.h" #include "icamera_ipc_checker.h" #include "icapture_session.h" #include "iremote_stub.h" @@ -41,6 +42,9 @@ private: int32_t HandleSetColorSpace(MessageParcel& data); int32_t HandleSetSmoothZoom(MessageParcel& data, MessageParcel& reply); int32_t HandleSetFeatureMode(MessageParcel& data); + int32_t HandleEnableMovingPhoto(MessageParcel& data); + int32_t HandleStartMovingPhotoCapture(); + int32_t HandleCreateMediaLibrary(MessageParcel& data, MessageParcel &reply); }; } // namespace CameraStandard } // namespace OHOS diff --git a/services/camera_service/binder/server/src/hcapture_session_stub.cpp b/services/camera_service/binder/server/src/hcapture_session_stub.cpp index f9b04901c..3a906cb3a 100644 --- a/services/camera_service/binder/server/src/hcapture_session_stub.cpp +++ b/services/camera_service/binder/server/src/hcapture_session_stub.cpp @@ -15,9 +15,11 @@ #include "hcapture_session_stub.h" #include "camera_log.h" +#include "camera_server_photo_proxy.h" #include "camera_util.h" #include "ipc_skeleton.h" #include "camera_service_ipc_interface_code.h" +#include "camera_photo_proxy.h" namespace OHOS { namespace CameraStandard { @@ -79,6 +81,15 @@ int HCaptureSessionStub::OnRemoteRequest( case static_cast(CaptureSessionInterfaceCode::CAMERA_CAPTURE_SESSION_SET_FEATURE_MODE): errCode = HandleSetFeatureMode(data); break; + case static_cast(CaptureSessionInterfaceCode::CAMERA_CAPTURE_SESSION_ENABLE_MOTION_PHOTO): + errCode = HandleEnableMovingPhoto(data); + break; + case static_cast(CaptureSessionInterfaceCode::CAMERA_CAPTURE_SESSION_START_MOVING_PHOTO_CAPTURE): + errCode = HandleStartMovingPhotoCapture(); + break; + case static_cast(CaptureSessionInterfaceCode::CAMERA_CAPTURE_SESSION_CREATE_MEDIA_LIBRARY_MANAGER): + errCode = HandleCreateMediaLibrary(data, reply); + break; default: MEDIA_ERR_LOG("HCaptureSessionStub request code %{public}u not handled", code); errCode = IPCObjectStub::OnRemoteRequest(code, data, reply, option); @@ -218,5 +229,31 @@ int HCaptureSessionStub::HandleSetFeatureMode(MessageParcel &data) int featureMode = static_cast(data.ReadUint32()); return SetFeatureMode(featureMode); } + +int32_t HCaptureSessionStub::HandleEnableMovingPhoto(MessageParcel &data) +{ + bool isEnabled = data.ReadBool(); + return EnableMovingPhoto(isEnabled); +} + +int32_t HCaptureSessionStub::HandleStartMovingPhotoCapture() +{ + return StartMovingPhotoCapture(); +} + +int32_t HCaptureSessionStub::HandleCreateMediaLibrary(MessageParcel& data, MessageParcel &reply) +{ + sptr photoProxy = new CameraPhotoProxy(); + photoProxy->ReadFromParcel(data); + CHECK_AND_RETURN_RET_LOG(photoProxy != nullptr, IPC_STUB_INVALID_DATA_ERR, + "HCaptureSessionStub HandleCreateMediaLibrary photoProxy is null"); + std::string uri; + int32_t cameraShotType = 0; + int32_t ret = CreateMediaLibrary(photoProxy, uri, cameraShotType); + CHECK_AND_RETURN_RET_LOG(reply.WriteString(uri) && reply.WriteInt32(cameraShotType), IPC_STUB_WRITE_PARCEL_ERR, + "HCaptureSessionStub HandleCreateMediaLibrary Write uri and cameraShotType failed"); + return ret; +} + } // namespace CameraStandard } // namespace OHOS diff --git a/services/camera_service/include/avcodec/audio_capturer_session.h b/services/camera_service/include/avcodec/audio_capturer_session.h new file mode 100644 index 000000000..bc80f2e0c --- /dev/null +++ b/services/camera_service/include/avcodec/audio_capturer_session.h @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AUDIO_CAPTURER_SESSION_H +#define AUDIO_CAPTURER_SESSION_H + +#include "audio_capturer.h" +#include "blocking_queue.h" +#include "refbase.h" +#include "audio_record.h" +#include +#include +#include "task_manager.h" + +namespace OHOS { +namespace CameraStandard { +using namespace AudioStandard; +using namespace std::chrono; +using namespace DeferredProcessing; +constexpr uint32_t DEFAULT_AUDIO_THREAD_NUMBER = 1; +constexpr uint32_t DEFAULT_AUDIO_CACHE_NUMBER = 200; +class AudioCapturerSession : public RefBase { +public: + explicit AudioCapturerSession(); + ~AudioCapturerSession(); + bool StartAudioCapture(); + void ProcessAudioBuffer(); + void Stop(); + void Release(); + void GetAudioRecords(int64_t startTime, int64_t endTime, vector> &audioRecords); + +private: + // Already guard by hcapture_session + std::unique_ptr audioCapturer_ = nullptr; + BlockingQueue> audioBufferQueue_; + std::atomic startAudioCapture_ { false }; + unique_ptr audioTaskManager_ = nullptr; +}; +} // CameraStandard +} // OHOS +#endif // AUDIO_CAPTURER_SESSION_H \ No newline at end of file diff --git a/services/camera_service/include/avcodec/audio_encoder.h b/services/camera_service/include/avcodec/audio_encoder.h new file mode 100644 index 000000000..574807968 --- /dev/null +++ b/services/camera_service/include/avcodec/audio_encoder.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AVCODEC_SAMPLE_AUDIO_ENCODER_H +#define AVCODEC_SAMPLE_AUDIO_ENCODER_H + +#include "native_avcodec_audiocodec.h" +#include "sample_info.h" +#include "audio_record.h" +#include + +namespace OHOS { +namespace CameraStandard { +class AudioEncoder { +public: + AudioEncoder() = default; + ~AudioEncoder(); + + int32_t Create(const std::string &codecMime); + int32_t Config(); + int32_t Start(); + bool EncodeAudioBuffer(vector> audioRecords); + int32_t PushInputData(sptr info); + int32_t FreeOutputData(uint32_t bufferIndex); + int32_t Stop(); + int32_t Release(); + +private: + int32_t SetCallback(CodecUserData *codecAudioData); + void RestartAudioCodec(); + int32_t Configure(); + bool EnqueueBuffer(sptr audioRecord); + bool EncodeAudioBuffer(sptr audioRecord); + std::atomic isStarted_ { false }; + std::mutex encoderMutex_; + OH_AVCodec *encoder_ = nullptr; + std::mutex contextMutex_; + CodecUserData *context_ = nullptr; + std::mutex serialMutex_; +}; +} // CameraStandard +} // OHOS +#endif // AVCODEC_SAMPLE_AUDIO_ENCODER_H \ No newline at end of file diff --git a/services/camera_service/include/avcodec/audio_video_muxer.h b/services/camera_service/include/avcodec/audio_video_muxer.h new file mode 100644 index 000000000..6282fe4b2 --- /dev/null +++ b/services/camera_service/include/avcodec/audio_video_muxer.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AVCODEC_AUDIO_VIDEO_MUXER_H +#define AVCODEC_AUDIO_VIDEO_MUXER_H + +#include "native_avmuxer.h" +#include "native_avcodec_base.h" +#include "native_avformat.h" +#include "native_avbuffer.h" +#include +#include +#include +#include "media_photo_asset_proxy.h" + +namespace OHOS { +namespace CameraStandard { +enum TrackType { + AUDIO_TRACK = 0, + VIDEO_TRACK, + META_TRACK +}; +class AudioVideoMuxer : public RefBase { +public: + explicit AudioVideoMuxer(); + ~AudioVideoMuxer(); + + int32_t Create(int32_t fd, OH_AVOutputFormat format, std::shared_ptr photoAssetProxy); + int32_t AddTrack(int &trackId, OH_AVFormat *format, TrackType type); + int32_t Start(); + int32_t WriteSampleBuffer(OH_AVBuffer *sample, TrackType type); + int32_t Stop(); + int32_t Release(); + int32_t SetRotation(int32_t rotation); + int32_t GetVideoFd(); + std::shared_ptr GetPhotoAssetProxy(); + std::atomic releaseSignal_ = 2; + +private: + OH_AVMuxer *muxer_ = nullptr; + int32_t fd_ = -1; + std::shared_ptr photoAssetProxy_; + int audioTrackId_ = -1; + int videoTrackId_ = -1; + int metaTrackId_ = -1; +}; +} // CameraStandard +} // OHOS +#endif // AVCODEC_SAMPLE_VIDEO_ENCODER_H \ No newline at end of file diff --git a/services/camera_service/include/avcodec/avcodec_task_manager.h b/services/camera_service/include/avcodec/avcodec_task_manager.h new file mode 100644 index 000000000..127957c87 --- /dev/null +++ b/services/camera_service/include/avcodec/avcodec_task_manager.h @@ -0,0 +1,76 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CAMERA_FRAMEWORK_LIVEPHOTO_GENERATOR_H +#define CAMERA_FRAMEWORK_LIVEPHOTO_GENERATOR_H + +#include +#include +#include +#include +#include +#include +#include "frame_record.h" +#include "audio_record.h" +#include "audio_capturer_session.h" +#include "native_avmuxer.h" +#include "refbase.h" +#include "surface_buffer.h" +#include "video_encoder.h" +#include "audio_encoder.h" +#include "audio_video_muxer.h" +#include "iconsumer_surface.h" +#include "blocking_queue.h" +#include "task_manager.h" + +#include "media_photo_asset_proxy.h" + +namespace OHOS { +namespace CameraStandard { +using namespace std; +using namespace DeferredProcessing; +using namespace Media; +using CacheCbFunc = function, bool)>; +constexpr uint32_t DEFAULT_THREAD_NUMBER = 6; +constexpr uint32_t DEFAULT_ENCODER_THREAD_NUMBER = 1; +constexpr uint32_t GET_FD_EXPIREATION_TIME = 500; +class AvcodecTaskManager : public RefBase, public std::enable_shared_from_this { +public: + explicit AvcodecTaskManager(sptr audioCapturerSession); + ~AvcodecTaskManager(); + void EncodeVideoBuffer(sptr frameRecord, CacheCbFunc cacheCallback); + void CollectAudioBuffer(vector> audioRecordVec, sptr muxer); + void DoMuxerVideo(vector> frameRecords, string taskName); + sptr CreateAVMuxer(sptr frameRecord); + void SubmitTask(function task); + void SetVideoFd(int32_t videoFd, shared_ptr photoAssetProxy); + void Stop(); + +private: + void FinishMuxer(sptr muxer); + void Release(); + unique_ptr videoEncoder_ = nullptr; + unique_ptr audioEncoder_ = nullptr; + unique_ptr taskManager_ = nullptr; + unique_ptr audioEncoderManager_ = nullptr; + unique_ptr videoEncoderManager_ = nullptr; + sptr audioCapturerSession_ = nullptr; + condition_variable cvEmpty_; + mutex videoFdMutex_; + queue>> videoFdQueue_; +}; +} // CameraStandard +} // OHOS +#endif // CAMERA_FRAMEWORK_LIVEPHOTO_GENERATOR_H \ No newline at end of file diff --git a/services/camera_service/include/avcodec/camera_server_photo_proxy.h b/services/camera_service/include/avcodec/camera_server_photo_proxy.h new file mode 100644 index 000000000..5782e104e --- /dev/null +++ b/services/camera_service/include/avcodec/camera_server_photo_proxy.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2023-2023 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef OHOS_CAMERA_SERVER_PHOTO_PROXY_H +#define OHOS_CAMERA_SERVER_PHOTO_PROXY_H + +#include "message_parcel.h" +#include "surface.h" +#include "photo_proxy.h" + +namespace OHOS { +namespace CameraStandard { +using namespace Media; +static const std::string prefix = "IMG_"; +static const std::string suffix = ".jpg"; +static const std::string connector = "_"; +static const char placeholder = '0'; +static const int yearWidth = 4; +static const int otherWidth = 2; +static const int startYear = 1900; +class CameraServerPhotoProxy : public PhotoProxy { +public: + CameraServerPhotoProxy(); + virtual ~CameraServerPhotoProxy(); + void ReadFromParcel(MessageParcel &parcel); + std::string GetDisplayName() override; + std::string GetExtension() override; + std::string GetPhotoId() override; + Media::DeferredProcType GetDeferredProcType() override; + void* GetFileDataAddr() override; + size_t GetFileSize() override; + int32_t GetWidth() override; + int32_t GetHeight() override; + PhotoFormat GetFormat() override; + PhotoQuality GetPhotoQuality() override; + void SetDisplayName(std::string displayName); + void Release() override; + +private: + const BufferHandle* bufferHandle_; + int32_t format_; + int32_t photoWidth_; + int32_t photoHeight_; + void* fileDataAddr_; + size_t fileSize_; + bool isMmaped_; + std::mutex mutex_; + std::string photoId_; + int32_t deferredProcType_; + int32_t isDeferredPhoto_; + sptr photoSurface_; + std::string displayName_; + bool isHighQuality_; + int32_t CameraFreeBufferHandle(BufferHandle *handle); +}; +} // namespace CameraStandard +} // namespace OHOS +#endif // OHOS_CAMERA_SERVER_PHOTO_PROXY_H \ No newline at end of file diff --git a/services/camera_service/include/avcodec/common/audio_record.h b/services/camera_service/include/avcodec/common/audio_record.h new file mode 100644 index 000000000..081dc29bd --- /dev/null +++ b/services/camera_service/include/avcodec/common/audio_record.h @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CAMERA_FRAMEWORK_AUDIO_RECORD_H +#define CAMERA_FRAMEWORK_AUDIO_RECORD_H +#include +#include +#include +#include "native_avcodec_base.h" +#include "native_avbuffer.h" +#include +#include "camera_log.h" +#include "sample_info.h" + + +namespace OHOS { +namespace CameraStandard { +using namespace std; +using ByteArrayPtr = std::unique_ptr; + +class AudioRecord : public RefBase { +public: + explicit AudioRecord(int64_t timestamp) : timestamp_(timestamp) + { + frameId_ = std::to_string(timestamp); + } + ~AudioRecord() = default; + OH_AVBuffer *encodedBuffer = nullptr; + + std::string frameId_; + + void SetStatusReadyConvertStatus() + { + status = STATUS_READY_CONVERT; + } + + void SetStatusFinishEncodeStatus() + { + status = STATUS_FINISH_ENCODE; + canReleased_.notify_one(); + } + + bool IsIdle() + { + return status == STATUS_NONE; + } + + bool IsReadyConvert() + { + return status == STATUS_READY_CONVERT; + } + + bool IsFinishCache() + { + return status == STATUS_FINISH_ENCODE; + } + + void CacheEncodedBuffer(OH_AVBuffer *buffer) + { + MEDIA_DEBUG_LOG("cacheEncodedBuffer start"); + encodedBuffer = buffer; + SetStatusFinishEncodeStatus(); + } + + void SetEncodedResult(bool encodedResult) + { + isEncoded_ = encodedResult; + } + + bool IsEncoded() + { + return isEncoded_; + } + + const std::string& GetFrameId() const + { + return frameId_; + } + + uint32_t GetBufferSize() + { + return bufferSize; + } + + int64_t GetTimeStamp() + { + return timestamp_; + } + + struct HashFunction { + std::size_t operator()(const sptr& obj) const + { + return std::hash()(obj->GetFrameId()); + } + }; + + struct EqualFunction { + bool operator()(const sptr& obj1, const sptr& obj2) const + { + return obj1->GetFrameId() == obj2->GetFrameId(); + } + }; + + void ReleaseAudioBuffer() + { + std::unique_lock lock(mutex_); + if (IsReadyConvert()) { + MEDIA_DEBUG_LOG("ReleaseAudioBuffer when isReadyConvert"); + canReleased_.wait_for(lock, std::chrono::milliseconds(BUFFER_RELEASE_EXPIREATION_TIME), + [this] { return IsFinishCache(); }); + MEDIA_DEBUG_LOG("releaseSurfaceBuffer go %{public}s", frameId_.c_str()); + } + if (audioBuffer_ != nullptr) { + delete audioBuffer_; + audioBuffer_ = nullptr; + } + } + + uint8_t* GetAudioBuffer() + { + std::unique_lock lock(mutex_); + return audioBuffer_; + } + + void SetAudioBuffer(uint8_t* audioBuffer) + { + std::unique_lock lock(mutex_); + audioBuffer_ = audioBuffer; + } + +private: + static const int32_t STATUS_NONE = 0; + static const int32_t STATUS_READY_CONVERT = 1; + static const int32_t STATUS_FINISH_ENCODE = 2; + int status = STATUS_NONE; + std::atomic isEncoded_ { false }; + uint32_t bufferSize; + int64_t timestamp_; + std::mutex mutex_; + std::condition_variable canReleased_; + uint8_t* audioBuffer_ = nullptr; +}; +} // CameraStandard +} // OHOS +#endif //CAMERA_FRAMEWORK_AUDIO_RECORD_H diff --git a/services/camera_service/include/avcodec/common/frame_record.h b/services/camera_service/include/avcodec/common/frame_record.h new file mode 100644 index 000000000..3d461213a --- /dev/null +++ b/services/camera_service/include/avcodec/common/frame_record.h @@ -0,0 +1,209 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CAMERA_FRAMEWORK_FRAME_RECORD_H +#define CAMERA_FRAMEWORK_FRAME_RECORD_H +#include "iconsumer_surface.h" +#include +#include +#include +#include +#include "native_avcodec_base.h" +#include "native_avbuffer.h" +#include +#include "camera_log.h" +#include "output/camera_output_capability.h" +#include "surface_type.h" +#include "sample_info.h" + +namespace OHOS { +namespace CameraStandard { +using namespace std; + +class FrameRecord : public RefBase { +public: + explicit FrameRecord(sptr videoBuffer, int64_t timestamp, GraphicTransformType type) + : videoBuffer_(videoBuffer), timestamp_(timestamp), transformType_(type) + { + frameId_ = std::to_string(timestamp); + size = make_shared(); + size->width = videoBuffer->GetSurfaceBufferWidth(); + size->height = videoBuffer->GetSurfaceBufferHeight(); + bufferSize = videoBuffer->GetSize(); + } + + ~FrameRecord() + { + MEDIA_DEBUG_LOG("FrameRecord release start %{public}p", OH_AVBuffer_GetAddr(encodedBuffer)); + if (encodedBuffer) { + OH_AVBuffer_Destroy(encodedBuffer); + } + encodedBuffer = nullptr; + } + + OH_AVBuffer *encodedBuffer = nullptr; + std::string frameId_; + + void SetStatusReadyConvertStatus() + { + status = STATUS_READY_CONVERT; + } + + bool IsIdle() + { + return status == STATUS_NONE; + } + + bool IsReadyConvert() + { + return status == STATUS_READY_CONVERT; + } + + bool IsFinishCache() + { + return status == STATUS_FINISH_ENCODE; + } + + sptr GetSurfaceBuffer() + { + std::unique_lock lock(mutex_); + return videoBuffer_; + } + + void SetSurfaceBuffer(sptr buffer) + { + std::unique_lock lock(mutex_); + videoBuffer_ = buffer; + } + + void ReleaseSurfaceBuffer(sptr surface) + { + std::unique_lock lock(mutex_); + if (IsReadyConvert()) { + MEDIA_DEBUG_LOG("releaseSurfaceBuffer when isReadyConvert %{public}p", this); + canReleased_.wait_for(lock, std::chrono::milliseconds(BUFFER_RELEASE_EXPIREATION_TIME), + [this] { return IsFinishCache(); }); + MEDIA_DEBUG_LOG("releaseSurfaceBuffer go %{public}p", this); + } + if (videoBuffer_) { + SurfaceError surfaceRet = surface->AttachBufferToQueue(videoBuffer_); + if (surfaceRet != SURFACE_ERROR_OK) { + MEDIA_ERR_LOG("Failed to attach buffer %{public}d", surfaceRet); + return; + } + surfaceRet = surface->ReleaseBuffer(videoBuffer_, -1); + if (surfaceRet != SURFACE_ERROR_OK) { + MEDIA_ERR_LOG("Failed to Release Buffer %{public}d", surfaceRet); + return; + } + videoBuffer_ = nullptr; + MEDIA_INFO_LOG("release buffer end %{public}s", frameId_.c_str()); + } + } + + void NotifyBufferRelease() + { + MEDIA_DEBUG_LOG("notifyBufferRelease"); + status = STATUS_FINISH_ENCODE; + canReleased_.notify_one(); + } + + OH_AVBuffer *GetEncodeBuffer() + { + return encodedBuffer; + } + + void CacheIDRBuffer(OH_AVBuffer *buffer) + { + MEDIA_DEBUG_LOG("cacheIDRBuffer start"); + encodedBuffer = buffer; + } + + void SetEncodedResult(bool encodedResult) + { + isEncoded_ = encodedResult; + } + + bool IsEncoded() + { + return isEncoded_; + } + + const std::string& GetFrameId() const + { + return frameId_; + } + + uint32_t GetBufferSize() + { + return bufferSize; + } + + int64_t GetTimeStamp() + { + return timestamp_; + } + + shared_ptr GetFrameSize() + { + return size; + } + + int32_t GetRotation() + { + auto it = transformTypeToValue.find(transformType_); + return it == transformTypeToValue.end() ? 0 : it->second; + } + + struct HashFunction { + std::size_t operator()(const sptr& obj) const + { + return std::hash()(obj->GetFrameId()); + } + }; + + struct EqualFunction { + bool operator()(const sptr& obj1, const sptr& obj2) const + { + return obj1->GetFrameId() == obj2->GetFrameId(); + } + }; + + const unordered_map transformTypeToValue = { + {GRAPHIC_FLIP_H_ROT90, 90}, + {GRAPHIC_FLIP_H_ROT180, 180}, + {GRAPHIC_FLIP_H_ROT270, 270}, + {GRAPHIC_ROTATE_90, 270}, + {GRAPHIC_ROTATE_180, 180}, + {GRAPHIC_ROTATE_270, 90} + }; + +private: + static const int32_t STATUS_NONE = 0; + static const int32_t STATUS_READY_CONVERT = 1; + static const int32_t STATUS_FINISH_ENCODE = 2; + std::atomic status = STATUS_NONE; + std::atomic isEncoded_ { false }; + shared_ptr size; + uint32_t bufferSize; + sptr videoBuffer_; + int64_t timestamp_; + GraphicTransformType transformType_; + std::mutex mutex_; + std::condition_variable canReleased_; +}; +} // CameraStandard +} // OHOS +#endif //CAMERA_FRAMEWORK_CODEC_BUFFER_INFO_H diff --git a/services/camera_service/include/avcodec/common/sample_callback.h b/services/camera_service/include/avcodec/common/sample_callback.h new file mode 100644 index 000000000..e72386761 --- /dev/null +++ b/services/camera_service/include/avcodec/common/sample_callback.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef AVCODEC_SAMPLE_SAMPLE_CALLBACK_H +#define AVCODEC_SAMPLE_SAMPLE_CALLBACK_H +#include "native_avcodec_base.h" +#include "native_avbuffer.h" +namespace OHOS { +namespace CameraStandard { +class SampleCallback { +public: + static void OnCodecError(OH_AVCodec *codec, int32_t errorCode, void *userData); + static void OnCodecFormatChange(OH_AVCodec *codec, OH_AVFormat *format, void *userData); + static void OnInputBufferAvailable(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData); + static void OnOutputBufferAvailable(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData); + static void OnOutputFormatChanged(OH_AVCodec *codec, OH_AVFormat *format, void *userData); + static void OnNeedInputBuffer(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData); + static void OnNewOutputBuffer(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData); +}; +} // CameraStandard +} // OHOS +#endif // AVCODEC_SAMPLE_SAMPLE_CALLBACK_H \ No newline at end of file diff --git a/services/camera_service/include/avcodec/common/sample_info.h b/services/camera_service/include/avcodec/common/sample_info.h new file mode 100644 index 000000000..c9118e8b0 --- /dev/null +++ b/services/camera_service/include/avcodec/common/sample_info.h @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AVCODEC_SAMPLE_SAMPLE_INFO_H +#define AVCODEC_SAMPLE_SAMPLE_INFO_H +#include +#include +#include +#include +#include +#include "camera_log.h" +#include "native_avcodec_base.h" +#include "native_avbuffer.h" +#include "native_audio_channel_layout.h" +#include + +namespace OHOS { +namespace CameraStandard { +constexpr std::string_view MIME_VIDEO_AVC = "video/avc"; +constexpr std::string_view MIME_VIDEO_HEVC = "video/hevc"; + +constexpr int32_t BITRATE_10M = 10 * 1024 * 1024; // 10Mbps +constexpr int32_t BITRATE_20M = 20 * 1024 * 1024; // 20Mbps +constexpr int32_t BITRATE_30M = 30 * 1024 * 1024; // 30Mbps +constexpr uint32_t DEFAULT_SAMPLERATE = 48000; +constexpr uint64_t DEFAULT_BITRATE = 48000; +constexpr uint32_t DEFAULT_CHANNEL_COUNT = 2; +constexpr OH_AudioChannelLayout CHANNEL_LAYOUT = OH_AudioChannelLayout::CH_LAYOUT_STEREO ; +constexpr OH_BitsPerSample SAMPLE_FORMAT = OH_BitsPerSample::SAMPLE_S16LE; +constexpr int32_t COMPLIANCE_LEVEL = 0; +constexpr OH_BitsPerSample BITS_PER_CODED_SAMPLE = OH_BitsPerSample::SAMPLE_S16LE; +constexpr uint32_t DEFAULT_MAX_INPUT_SIZE = 1024 * DEFAULT_CHANNEL_COUNT * sizeof(short); +constexpr int32_t VIDEO_FRAME_INTERVAL = 33333; +constexpr int32_t AUDIO_FRAME_INTERVAL = 21333; +constexpr double VIDOE_FRAME_RATE = 30.0; +constexpr int32_t CACHE_FRAME_COUNT = 45; +constexpr int32_t BUFFER_RELEASE_EXPIREATION_TIME = 150; +constexpr int32_t BUFFER_ENCODE_EXPIREATION_TIME = 10; +constexpr OH_AVPixelFormat VIDOE_PIXEL_FORMAT = AV_PIXEL_FORMAT_NV21; + +class CodecAVBufferInfo : public RefBase { +public: + explicit CodecAVBufferInfo(uint32_t argBufferIndex, OH_AVBuffer *argBuffer) + : bufferIndex(argBufferIndex), buffer(argBuffer) + { + // get output buffer attr + OH_AVBuffer_GetBufferAttr(argBuffer, &attr); + }; + ~CodecAVBufferInfo() = default; + uint32_t bufferIndex = 0; + OH_AVBuffer *buffer = nullptr; + OH_AVCodecBufferAttr attr = {0, 0, 0, AVCODEC_BUFFER_FLAGS_NONE}; + + OH_AVBuffer *GetCopyAVBuffer() + { + int32_t capacity = OH_AVBuffer_GetCapacity(buffer); + MEDIA_INFO_LOG("CodecBufferInfo deep copy enter %{public}d", capacity); + OH_AVBuffer *destBuffer = OH_AVBuffer_Create(capacity); + auto sourceAddr = OH_AVBuffer_GetAddr(buffer); + auto destAddr = OH_AVBuffer_GetAddr(destBuffer); + errno_t cpyRet = memcpy_s(reinterpret_cast(destAddr), capacity, + reinterpret_cast(sourceAddr), attr.size); + if (cpyRet != 0) { + MEDIA_ERR_LOG("CodecBufferInfo memcpy_s failed. %{public}d", cpyRet); + } + OH_AVErrCode errorCode = OH_AVBuffer_SetBufferAttr(destBuffer, &attr); + if (errorCode != 0) { + MEDIA_ERR_LOG("CodecBufferInfo OH_AVBuffer_SetBufferAttr failed. %{public}d", errorCode); + } + return destBuffer; + } + + void AddCopyAVBuffer(OH_AVBuffer *IDRBuffer) + { + if (IDRBuffer == nullptr) { + return; + } + int32_t capacity = OH_AVBuffer_GetCapacity(buffer); + MEDIA_INFO_LOG("CodecBufferInfo deep copy enter %{public}d", capacity); + OH_AVCodecBufferAttr destAttr = {0, 0, 0, AVCODEC_BUFFER_FLAGS_NONE}; + OH_AVBuffer_GetBufferAttr(IDRBuffer, &destAttr); + auto sourceAddr = OH_AVBuffer_GetAddr(buffer); + auto destAddr = OH_AVBuffer_GetAddr(IDRBuffer) + destAttr.size; + errno_t cpyRet = memcpy_s(reinterpret_cast(destAddr), capacity, + reinterpret_cast(sourceAddr), attr.size); + if (cpyRet != 0) { + MEDIA_ERR_LOG("CodecBufferInfo memcpy_s failed. %{public}d", cpyRet); + } + destAttr.size = destAttr.size + attr.size; + destAttr.flags &= attr.flags; + OH_AVBuffer_SetBufferAttr(IDRBuffer, &destAttr); + } +}; + +class CodecUserData : public RefBase { +public: + CodecUserData() = default; + ~CodecUserData() + { + inputMutex_.lock(); + while (!inputBufferInfoQueue_.empty()) { + inputBufferInfoQueue_.pop(); + } + inputMutex_.unlock(); + outputMutex_.lock(); + while (!outputBufferInfoQueue_.empty()) { + outputBufferInfoQueue_.pop(); + } + outputMutex_.unlock(); + }; + uint32_t inputFrameCount_ = 0; + std::mutex inputMutex_; + std::condition_variable inputCond_; + std::queue> inputBufferInfoQueue_; + + uint32_t outputFrameCount_ = 0; + std::mutex outputMutex_; + std::condition_variable outputCond_; + std::queue> outputBufferInfoQueue_; +}; +} // CameraStandard +} // OHOS +#endif // AVCODEC_SAMPLE_SAMPLE_INFO_H \ No newline at end of file diff --git a/services/camera_service/include/avcodec/drain_manager.h b/services/camera_service/include/avcodec/drain_manager.h new file mode 100644 index 000000000..cd5aee359 --- /dev/null +++ b/services/camera_service/include/avcodec/drain_manager.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CAMERA_FRAMEWORK_DRAIN_MANAGER_H +#define CAMERA_FRAMEWORK_DRAIN_MANAGER_H + +#include +#include +#include +#include "frame_record.h" +#include + +namespace OHOS { +namespace CameraStandard { +class DrainImageCallback : public RefBase { +public: + ~DrainImageCallback() = default; + virtual void OnDrainImage(sptr frame) = 0; + virtual void OnDrainImageFinish(bool isFinished) = 0; +}; + +class DrainImageManager : public RefBase { +private: + sptr drainImageCallback; + std::function drainFinishCallback; + +public: + int drainCount; + DrainImageManager(sptr callback, int count) : drainImageCallback(callback), drainCount(count) {} + void DrainImage(sptr frame) + { + if (drainCount <= 0) { + return; + } + drainImageCallback->OnDrainImage(frame); + drainCount--; + if (drainCount == 0) { + DrainFinish(true); + } + } + + void DrainFinish(bool isFinished) + { + drainImageCallback->OnDrainImageFinish(isFinished); + } +}; +} // CameraStandard +} // OHOS +#endif //CAMERA_FRAMEWORK_DRAIN_MANAGER_H diff --git a/services/camera_service/include/avcodec/moving_photo_video_cache.h b/services/camera_service/include/avcodec/moving_photo_video_cache.h new file mode 100644 index 000000000..544726475 --- /dev/null +++ b/services/camera_service/include/avcodec/moving_photo_video_cache.h @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CAMERA_FRAMEWORK_MOVING_PHOTO_VIDEO_CACHE_H +#define CAMERA_FRAMEWORK_MOVING_PHOTO_VIDEO_CACHE_H + +#include +#include +#include +#include +#include +#include +#include "native_avmuxer.h" +#include "refbase.h" +#include "video_encoder.h" +#include "audio_encoder.h" +#include "audio_video_muxer.h" +#include "iconsumer_surface.h" +#include +#include +#include "frame_record.h" +#include +#include "avcodec_task_manager.h" + +namespace OHOS { +namespace CameraStandard { +using namespace std; +using CachedFrameSet = unordered_set, FrameRecord::HashFunction, FrameRecord::EqualFunction>; +using EncodedEndCbFunc = function>, string)>; +class CachedFrameCallbackHandle; +class MovingPhotoVideoCache : public RefBase { +public: + explicit MovingPhotoVideoCache(sptr taskManager); + ~MovingPhotoVideoCache(); + void CacheFrame(sptr frameRecord); + void OnImageEncoded(sptr frameRecord, bool encodeResult); + void GetFrameCachedResult(vector> frameRecords, + EncodedEndCbFunc encodedEndCbFunc, string taskName); + void DoMuxerVideo(vector> frameRecords, string taskName); + void ClearCallbackHandler(); + void ClearCache(); +private: + mutex callbackVecLock_; // Guard cachedFrameCallbackHandles + vector> cachedFrameCallbackHandles_; + mutex taskManagerLock_; // Guard cachedFrameCallbackHandles + sptr taskManager_; +}; + +class CachedFrameCallbackHandle : public RefBase { +public: + CachedFrameCallbackHandle(vector> frameRecords, + EncodedEndCbFunc encodedEndCbFunc, string taskName); + ~CachedFrameCallbackHandle(); + void OnCacheFrameFinish(sptr frameRecord, bool cachedSuccess); + void AbortCapture(); + CachedFrameSet GetCacheRecord(); +private: + CachedFrameSet cacheRecords_; //set + vector> errorCacheRecords_; + vector> successCacheRecords_; + EncodedEndCbFunc encodedEndCbFunc_; + atomic isAbort_ { false }; + mutex cacheFrameMutex_; + string taskName_; +}; +} // CameraStandard +} // OHOS +#endif // CAMERA_FRAMEWORK_MOVING_PHOTO_VIDEO_CACHE_H \ No newline at end of file diff --git a/services/camera_service/include/avcodec/video_encoder.h b/services/camera_service/include/avcodec/video_encoder.h new file mode 100644 index 000000000..d4c75f612 --- /dev/null +++ b/services/camera_service/include/avcodec/video_encoder.h @@ -0,0 +1,61 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef AVCODEC_SAMPLE_VIDEO_ENCODER_H +#define AVCODEC_SAMPLE_VIDEO_ENCODER_H + +#include "frame_record.h" +#include "native_avcodec_videoencoder.h" +#include "output/camera_output_capability.h" +#include "sample_info.h" + +namespace OHOS { +namespace CameraStandard { +using namespace std; +class VideoEncoder { +public: + VideoEncoder() = default; + ~VideoEncoder(); + + int32_t Create(const std::string &codecMime); + int32_t Config(); + int32_t Start(); + int32_t PushInputData(sptr info); + int32_t NotifyEndOfStream(); + int32_t FreeOutputData(uint32_t bufferIndex); + bool EncodeSurfaceBuffer(sptr frameRecord); + int32_t Stop(); + int32_t Release(); + int32_t GetSurface(); + int32_t ReleaseSurfaceBuffer(sptr frameRecord); + +private: + int32_t SetCallback(CodecUserData *codecUserData); + int32_t Configure(); + void RestartVideoCodec(shared_ptr size, int32_t rotation); + bool EnqueueBuffer(sptr frameRecord, int32_t keyFrameInterval); + std::atomic isStarted_ { false }; + std::mutex encoderMutex_; + OH_AVCodec *encoder_ = nullptr; + std::mutex contextMutex_; + CodecUserData *context_ = nullptr; + shared_ptr size_; + int32_t rotation_; + std::mutex surfaceMutex_; // guard codecSurface_ + sptr codecSurface_; +}; +} // CameraStandard +} // OHOS +#endif // AVCODEC_SAMPLE_VIDEO_ENCODER_H \ No newline at end of file diff --git a/services/camera_service/include/camera_log.h b/services/camera_service/include/camera_log.h index 69826007a..bcfd78d2f 100644 --- a/services/camera_service/include/camera_log.h +++ b/services/camera_service/include/camera_log.h @@ -77,6 +77,22 @@ } \ } while (0) +#define CHECK_AND_BREAK_LOG(cond, fmt, ...) \ + if (1) { \ + if (!(cond)) { \ + MEDIA_WARNING_LOG(fmt, ##__VA_ARGS__); \ + break; \ + } \ + } else void (0) + +#define CHECK_AND_CONTINUE_LOG(cond, fmt, ...) \ + if (1) { \ + if (!(cond)) { \ + MEDIA_WARNING_LOG(fmt, ##__VA_ARGS__); \ + continue; \ + } \ + } else void (0) + #define POINTER_MASK 0x00FFFFFF #define CAMERA_SYNC_TRACE HITRACE_METER_NAME(HITRACE_TAG_ZCAMERA, __PRETTY_FUNCTION__) diff --git a/services/camera_service/include/hcamera_device.h b/services/camera_service/include/hcamera_device.h index bd743d3af..8dcbf7a1e 100644 --- a/services/camera_service/include/hcamera_device.h +++ b/services/camera_service/include/hcamera_device.h @@ -114,6 +114,8 @@ public: int64_t GetSecureCameraSeq(uint64_t* secureSeqId); + bool CheckMovingPhotoSupported(int32_t mode); + private: class FoldScreenListener; std::mutex opMutex_; // Lock the operations updateSettings_, streamOperator_, and hdiCameraDevice_. diff --git a/services/camera_service/include/hcapture_session.h b/services/camera_service/include/hcapture_session.h index ffe8c9fd9..e02f3684a 100644 --- a/services/camera_service/include/hcapture_session.h +++ b/services/camera_service/include/hcapture_session.h @@ -16,6 +16,7 @@ #ifndef OHOS_CAMERA_H_CAPTURE_SESSION_H #define OHOS_CAMERA_H_CAPTURE_SESSION_H +#include #include #include #include @@ -23,7 +24,7 @@ #include #include #include - +#include #include "accesstoken_kit.h" #include "camera_util.h" #include "hcamera_device.h" @@ -33,20 +34,35 @@ #include "hstream_repeat.h" #include "icapture_session.h" #include "istream_common.h" +#include "camera_photo_proxy.h" #include "perm_state_change_callback_customize.h" #include "privacy_kit.h" #include "state_customized_cbk.h" +#include "surface.h" #include "v1_0/istream_operator.h" #include "v1_1/istream_operator.h" #include "v1_2/istream_operator.h" #include "v1_3/istream_operator_callback.h" #include "hcamera_restore_param.h" +#include "iconsumer_surface.h" +#include "blocking_queue.h" +#include "audio_capturer.h" +#include "audio_info.h" +#include "avcodec_task_manager.h" +#include "moving_photo_video_cache.h" +#include "drain_manager.h" +#include "audio_capturer_session.h" +#include "safe_map.h" namespace OHOS { namespace CameraStandard { using OHOS::HDI::Camera::V1_0::CaptureEndedInfo; using OHOS::HDI::Camera::V1_0::CaptureErrorInfo; using namespace OHOS::HDI::Display::Graphic::Common::V1_0; +using namespace AudioStandard; +using namespace std::chrono; +using namespace DeferredProcessing; +using namespace Media; class PermissionStatusChangeCb; class CameraUseStateChangeCb; @@ -119,6 +135,7 @@ public: int32_t OnCaptureReady(int32_t captureId, const std::vector& streamIds, uint64_t timestamp) override; virtual const sptr GetStreamByStreamID(int32_t streamId) = 0; + virtual void StartRecord(const std::string taskName) = 0; virtual const sptr GetHdiStreamByStreamID(int32_t streamId) = 0; @@ -126,6 +143,39 @@ private: std::mutex cbMutex_; }; +class SessionDrainImageCallback; +class MovingPhotoListener : public IBufferConsumerListener { +public: + explicit MovingPhotoListener(sptr surface); + ~MovingPhotoListener(); + void OnBufferAvailable() override; + void DrainOutImage(sptr drainImageCallback); + void RemoveDrainImageManager(sptr drainImageCallback); + void StopDrainOut(); +private: + sptr surface_; + BlockingQueue> recorderBufferQueue_; + SafeMap, sptr> callbackMap_; +}; + +class SessionDrainImageCallback : public DrainImageCallback { +public: + explicit SessionDrainImageCallback(std::vector>& frameCacheList, + wptr listener, + wptr cache, + string taskName); + ~SessionDrainImageCallback(); + void OnDrainImage(sptr frame) override; + void OnDrainImageFinish(bool isFinished) override; + +private: + std::vector> frameCacheList_; + wptr listener_; + wptr videoCache_; + string taskName_; +}; + + class HCaptureSession : public HCaptureSessionStub, public StreamOperatorCallback { public: explicit HCaptureSession(const uint32_t callingTokenId, int32_t opMode); @@ -156,7 +206,7 @@ public: bool QueryZoomPerformance(std::vector& crossZoomAndTime, int32_t operationMode); int32_t SetSmoothZoom(int32_t smoothZoomType, int32_t operationMode, float targetZoomRatio, float &duration) override; - + int32_t EnableMovingPhoto(bool isEnable) override; static void dumpSessions(std::string& dumpString); void dumpSessionInfo(std::string& dumpString); static void CameraSessionSummary(std::string& dumpString); @@ -165,11 +215,19 @@ public: int32_t GetopMode(); int32_t OperatePermissionCheck(uint32_t interfaceCode) override; + int32_t StartMovingPhotoCapture() override; + int32_t CreateMediaLibrary(sptr &photoProxy, std::string &uri, int32_t &cameraShotType) override; const sptr GetStreamByStreamID(int32_t streamId) override; const sptr GetHdiStreamByStreamID(int32_t streamId) override; int32_t SetFeatureMode(int32_t featureMode) override; + void StartRecord(const std::string taskName) override; private: + string lastDisplayName_ = ""; + int32_t saveIndex = 0; + volatile bool isSetMotionPhoto_ = false; + std::mutex livePhotoStreamLock_; // Guard livePhotoStreamRepeat_ + sptr livePhotoStreamRepeat_; inline void SetCameraDevice(sptr device) { std::lock_guard lock(cameraDeviceLock_); @@ -181,7 +239,7 @@ private: std::lock_guard lock(cameraDeviceLock_); return cameraDevice_; } - + string CreateDisplayName(); int32_t ValidateSessionInputs(); int32_t ValidateSessionOutputs(); int32_t AddOutputStream(sptr stream); @@ -197,6 +255,11 @@ private: void ClearSketchRepeatStream(); void ExpandSketchRepeatStream(); + void ExpandMovingPhotoRepeatStream(); + void ClearMovingPhotoRepeatStream(); + void StopMovingPhoto(); + int32_t CreateMovingPhotoStreamRepeat(int32_t format, int32_t width, int32_t height, + sptr producer); int32_t CheckIfColorSpaceMatchesFormat(ColorSpace colorSpace); void CancelStreamsAndGetStreamInfos(std::vector& streamInfos); void RestartStreams(); @@ -204,6 +267,12 @@ private: void SetColorSpaceForStreams(); void ProcessMetaZoomArray(std::vector& zoomAndTimeArray, sptr& cameraDevice); + void StartMovingPhotoStream(); + bool InitAudioCapture(); + bool StartAudioCapture(); + void ProcessAudioBuffer(); + void StartOnceRecord(std::string taskName); + int32_t StartPreviewStream(const std::shared_ptr& settings); std::string GetSessionState(); @@ -225,6 +294,14 @@ private: ColorSpace currColorSpace_ = ColorSpace::COLOR_SPACE_UNKNOWN; ColorSpace currCaptureColorSpace_ = ColorSpace::COLOR_SPACE_UNKNOWN; bool isSessionStarted_ = false; + + std::mutex movingPhotoStatusLock_; // Guard movingPhotoStatus + sptr surface_; + sptr livephotoListener_; + sptr audioCapturerSession_; + sptr metaSurface_; + sptr videoCache_; + sptr taskManager_; }; class PermissionStatusChangeCb : public Security::AccessToken::PermStateChangeCallbackCustomize { diff --git a/services/camera_service/include/hstream_repeat.h b/services/camera_service/include/hstream_repeat.h index 5c263cebc..4a72c8423 100644 --- a/services/camera_service/include/hstream_repeat.h +++ b/services/camera_service/include/hstream_repeat.h @@ -17,6 +17,7 @@ #define OHOS_CAMERA_H_STREAM_REPEAT_H #include +#include #include #include @@ -31,7 +32,8 @@ namespace CameraStandard { enum class RepeatStreamType { PREVIEW, VIDEO, - SKETCH + SKETCH, + LIVEPHOTO }; enum class RepeatStreamStatus { @@ -66,6 +68,8 @@ public: int32_t UpdateSketchRatio(float sketchRatio) override; sptr GetSketchStream(); RepeatStreamType GetRepeatStreamType(); + void SetMetaProducer(sptr metaProducer); + void SetMovingPhotoStartCallback(std::function callback); void DumpStreamInfo(std::string& dumpString) override; int32_t OperatePermissionCheck(uint32_t interfaceCode) override; int32_t SetFrameRate(int32_t minFrameRate, int32_t maxFrameRate) override; @@ -88,6 +92,9 @@ private: SketchStatus sketchStatus_ = SketchStatus::STOPED; RepeatStreamStatus repeatStreamStatus_ = RepeatStreamStatus::STOPED; bool mEnableSecure = false; + sptr metaProducer_; + std::mutex movingPhotoCallbackLock_; + std::function startMovingPhotoCallback_; }; } // namespace CameraStandard } // namespace OHOS diff --git a/services/camera_service/src/avcodec/audio_capturer_session.cpp b/services/camera_service/src/avcodec/audio_capturer_session.cpp new file mode 100644 index 000000000..84c8a4d97 --- /dev/null +++ b/services/camera_service/src/avcodec/audio_capturer_session.cpp @@ -0,0 +1,156 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "audio_capturer_session.h" + +#include +#include +#include +#include "audio_record.h" +#include "camera_log.h" +#include "sample_info.h" +#include "datetime_ex.h" + +namespace OHOS { +namespace CameraStandard { + +AudioCapturerSession::AudioCapturerSession() + : audioBufferQueue_("audioBuffer", DEFAULT_AUDIO_CACHE_NUMBER) +{ + audioTaskManager_ = make_unique("audioCaptureTaskManager", DEFAULT_AUDIO_THREAD_NUMBER, true); + AudioCapturerOptions capturerOptions; + capturerOptions.streamInfo.samplingRate = static_cast(AudioSamplingRate::SAMPLE_RATE_48000); + capturerOptions.streamInfo.encoding = AudioEncodingType::ENCODING_PCM; + capturerOptions.streamInfo.format = AudioSampleFormat::SAMPLE_S16LE; + capturerOptions.streamInfo.channels = AudioChannel::STEREO; + capturerOptions.capturerInfo.sourceType = SourceType::SOURCE_TYPE_MIC; + capturerOptions.capturerInfo.capturerFlags = 0; + audioCapturer_ = AudioCapturer::Create(capturerOptions); + if (audioCapturer_ == nullptr) { + MEDIA_ERR_LOG("AudioCapturerSession::Create AudioCapturer failed"); + } +} + +AudioCapturerSession::~AudioCapturerSession() +{ + MEDIA_INFO_LOG("~AudioCapturerSession enter"); + audioBufferQueue_.SetActive(false); + audioBufferQueue_.Clear(); + Release(); +} + +bool AudioCapturerSession::StartAudioCapture() +{ + MEDIA_INFO_LOG("Starting moving photo audio stream"); + if (audioCapturer_ == nullptr) { + MEDIA_ERR_LOG("audioCapturer is not create"); + return false; + } + if (!audioCapturer_->Start()) { + MEDIA_ERR_LOG("Start stream failed"); + audioCapturer_->Release(); + return false; + } + startAudioCapture_ = true; + MEDIA_INFO_LOG("Capturing started"); + AudioCapturerParams getCapturerParams; + if (audioCapturer_->GetParams(getCapturerParams) == 0) { + MEDIA_INFO_LOG("Get Audio format: %{public}d", getCapturerParams.audioSampleFormat); + MEDIA_INFO_LOG("Get Audio sampling rate: %{public}d", getCapturerParams.samplingRate); + MEDIA_INFO_LOG("Get Audio channels: %{public}d", getCapturerParams.audioChannel); + } + audioTaskManager_->SubmitTask([this]() { + this->ProcessAudioBuffer(); + }); + return true; +} + +void AudioCapturerSession::GetAudioRecords(int64_t startTime, int64_t endTime, vector> &audioRecords) +{ + vector> allRecords = audioBufferQueue_.GetAllElements(); + for (const auto& record : allRecords) { + if (record->GetTimeStamp() >= startTime && record->GetTimeStamp() < endTime) { + audioRecords.push_back(record); + } + } +} + +void AudioCapturerSession::ProcessAudioBuffer() +{ + if (audioCapturer_ == nullptr) { + MEDIA_ERR_LOG("AudioCapturer_ is not init"); + return; + } + size_t bufferLen = DEFAULT_MAX_INPUT_SIZE; + while (true) { + CHECK_AND_BREAK_LOG(startAudioCapture_, "Audio capture work done, thread out"); + auto buffer = std::make_unique(bufferLen); + if (buffer == nullptr) { + MEDIA_ERR_LOG("Failed to allocate buffer"); + return; + } + size_t bytesRead = 0; + while (bytesRead < bufferLen) { + int32_t len = audioCapturer_->Read(*(buffer.get() + bytesRead), bufferLen - bytesRead, true); + if (len >= 0) { + bytesRead += len; + } else { + bytesRead = len; + break; + } + } + if (audioBufferQueue_.Full()) { + sptr audioRecord = audioBufferQueue_.Pop(); + audioRecord->ReleaseAudioBuffer(); + MEDIA_DEBUG_LOG("audio release popBuffer"); + } + int64_t timeOffset = 20; + sptr audioRecord = new AudioRecord(GetTickCount() - timeOffset); + audioRecord->SetAudioBuffer(buffer.get()); + MEDIA_DEBUG_LOG("audio push buffer frameId: %{public}s", audioRecord->GetFrameId().c_str()); + buffer.release(); + audioBufferQueue_.Push(audioRecord); + if (bytesRead < 0) { + MEDIA_ERR_LOG("Bytes read failed. error code %{public}zu", bytesRead); + break; + } else if (bytesRead == 0) { + continue; + } + } +} + +void AudioCapturerSession::Stop() +{ + MEDIA_INFO_LOG("Audio capture stop enter"); + if (startAudioCapture_) { + startAudioCapture_ = false; + MEDIA_INFO_LOG("Audio capture stop out"); + if (audioCapturer_ != nullptr && audioCapturer_->Stop()) { + MEDIA_INFO_LOG("Audio capturer stop success"); + } + } +} + +void AudioCapturerSession::Release() +{ + Stop(); + if (audioCapturer_ != nullptr) { + MEDIA_INFO_LOG("Audio capture Release enter"); + audioCapturer_->Release(); + } + MEDIA_INFO_LOG("Audio capture released"); +} +} // namespace CameraStandard +} // namespace OHOS diff --git a/services/camera_service/src/avcodec/audio_encoder.cpp b/services/camera_service/src/avcodec/audio_encoder.cpp new file mode 100644 index 000000000..7f296bff6 --- /dev/null +++ b/services/camera_service/src/avcodec/audio_encoder.cpp @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "audio_encoder.h" +#include +#include +#include "audio_record.h" +#include "sample_info.h" +#include "surface_type.h" +#include "external_window.h" +#include "sample_callback.h" +#include "camera_log.h" +#include + +namespace OHOS { +namespace CameraStandard { + +AudioEncoder::~AudioEncoder() +{ + MEDIA_INFO_LOG("~AudioEncoder enter"); + Release(); +} + +int32_t AudioEncoder::Create(const std::string &codecMime) +{ + std::lock_guard lock(encoderMutex_); + encoder_ = OH_AudioCodec_CreateByMime(codecMime.data(), true); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Create failed"); + return 0; +} + +int32_t AudioEncoder::Config() +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + std::unique_lock contextLock(contextMutex_); + context_ = new CodecUserData; + // Configure audio encoder + int32_t ret = Configure(); + CHECK_AND_RETURN_RET_LOG(ret == 0, 1, "Configure failed"); + // SetCallback for audio encoder + ret = SetCallback(context_); + CHECK_AND_RETURN_RET_LOG(ret == 0, 1, "Set callback failed"); + contextLock.unlock(); + // Prepare audio encoder + ret = OH_AudioCodec_Prepare(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Prepare failed, ret: %{public}d", ret); + + return 0; +} + +int32_t AudioEncoder::Start() +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + int ret = OH_AudioCodec_Start(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Start failed, ret: %{public}d", ret); + isStarted_ = true; + return 0; +} + +int32_t AudioEncoder::PushInputData(sptr info) +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + int32_t ret = AV_ERR_OK; + ret = OH_AVBuffer_SetBufferAttr(info->buffer, &info->attr); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Set avbuffer attr failed, ret: %{public}d", ret); + ret = OH_AudioCodec_PushInputBuffer(encoder_, info->bufferIndex); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Push input data failed, ret: %{public}d", ret); + return 0; +} + +int32_t AudioEncoder::FreeOutputData(uint32_t bufferIndex) +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + MEDIA_INFO_LOG("FreeOutputData bufferIndex: %{public}u", bufferIndex); + int32_t ret = OH_AudioCodec_FreeOutputBuffer(encoder_, bufferIndex); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, + "Free output data failed, ret: %{public}d", ret); + return 0; +} + +int32_t AudioEncoder::Stop() +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + int ret = OH_AudioCodec_Flush(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Flush failed, ret: %{public}d", ret); + ret = OH_AudioCodec_Stop(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Stop failed, ret: %{public}d", ret); + isStarted_ = false; + return 0; +} + +int32_t AudioEncoder::Release() +{ + std::unique_lock contextLock(contextMutex_); + if (context_ != nullptr) { + delete context_; + context_ = nullptr; + } + contextLock.unlock(); + std::lock_guard lock(encoderMutex_); + if (encoder_ != nullptr) { + OH_AudioCodec_Destroy(encoder_); + encoder_ = nullptr; + } + isStarted_ = false; + return 0; +} + +void AudioEncoder::RestartAudioCodec() +{ + Release(); + Create(OH_AVCODEC_MIMETYPE_AUDIO_AAC); + Config(); + Start(); +} + +bool AudioEncoder::EnqueueBuffer(sptr audioRecord) +{ + int enququeRetryCount = 10; + uint8_t* buffer = audioRecord->GetAudioBuffer(); + if (buffer == nullptr) { + MEDIA_ERR_LOG("Enqueue audio buffer is empty"); + return false; + } + while (enququeRetryCount > 0) { + enququeRetryCount--; + std::unique_lock contextLock(contextMutex_); + CHECK_AND_RETURN_RET_LOG(context_ != nullptr, false, "AudioEncoder has been released"); + std::unique_lock lock(context_->inputMutex_); + bool condRet = context_->inputCond_.wait_for(lock, std::chrono::milliseconds(BUFFER_ENCODE_EXPIREATION_TIME), + [this]() { return !isStarted_ || !context_->inputBufferInfoQueue_.empty(); }); + CHECK_AND_CONTINUE_LOG(!context_->inputBufferInfoQueue_.empty(), + "Buffer queue is empty, continue, cond ret: %{public}d", condRet); + sptr bufferInfo = context_->inputBufferInfoQueue_.front(); + context_->inputBufferInfoQueue_.pop(); + context_->inputFrameCount_++; + lock.unlock(); + contextLock.unlock(); + bufferInfo->attr.pts = audioRecord->GetTimeStamp(); + bufferInfo->attr.size = DEFAULT_MAX_INPUT_SIZE; + bufferInfo->attr.flags = AVCODEC_BUFFER_FLAGS_NONE; + auto bufferAddr = OH_AVBuffer_GetAddr(bufferInfo->buffer); + int32_t bufferCap = OH_AVBuffer_GetCapacity(bufferInfo->buffer); + errno_t cpyRet = memcpy_s(bufferAddr, bufferCap, buffer, DEFAULT_MAX_INPUT_SIZE); + CHECK_AND_RETURN_RET_LOG(cpyRet == 0, false, "encoder memcpy_s failed. %{public}d", cpyRet); + int32_t ret = PushInputData(bufferInfo); + CHECK_AND_RETURN_RET_LOG(ret == 0, false, "Push data failed"); + MEDIA_DEBUG_LOG("Success frame id is : %{public}s", audioRecord->GetFrameId().c_str()); + return true; + } + MEDIA_ERR_LOG("Failed frame id is : %{public}s", audioRecord->GetFrameId().c_str()); + return false; +} + +bool AudioEncoder::EncodeAudioBuffer(sptr audioRecord) +{ + if (encoder_ == nullptr || !isStarted_) { + RestartAudioCodec(); + } + if (encoder_ == nullptr) { + return false; + } + audioRecord->SetStatusReadyConvertStatus(); + if (!EnqueueBuffer(audioRecord)) { + return false; + } + int retryCount = 10; + while (retryCount > 0) { + retryCount--; + std::unique_lock contextLock(contextMutex_); + CHECK_AND_RETURN_RET_LOG(context_ != nullptr, false, "AudioEncoder has been released"); + std::unique_lock lock(context_->outputMutex_); + bool condRet = context_->outputCond_.wait_for(lock, std::chrono::milliseconds(BUFFER_ENCODE_EXPIREATION_TIME), + [this]() { return !isStarted_ || !context_->outputBufferInfoQueue_.empty(); }); + CHECK_AND_CONTINUE_LOG(!context_->outputBufferInfoQueue_.empty(), + "Buffer queue is empty, continue, cond ret: %{public}d", condRet); + sptr bufferInfo = context_->outputBufferInfoQueue_.front(); + context_->outputBufferInfoQueue_.pop(); + context_->outputFrameCount_++; + lock.unlock(); + contextLock.unlock(); + MEDIA_DEBUG_LOG("Out buffer count: %{public}u, size: %{public}d, flag: %{public}u, pts:%{public}" PRId64, + context_->outputFrameCount_, bufferInfo->attr.size, bufferInfo->attr.flags, bufferInfo->attr.pts); + OH_AVBuffer *audioBuffer = bufferInfo->GetCopyAVBuffer(); + audioRecord->CacheEncodedBuffer(audioBuffer); + int32_t ret = FreeOutputData(bufferInfo->bufferIndex); + CHECK_AND_BREAK_LOG(ret == 0, "FreeOutputData failed"); + MEDIA_DEBUG_LOG("Success frame id is : %{public}s", audioRecord->GetFrameId().c_str()); + audioRecord->SetEncodedResult(true); + return true; + } + MEDIA_ERR_LOG("Failed frame id is : %{public}s", audioRecord->GetFrameId().c_str()); + return false; +} + +bool AudioEncoder::EncodeAudioBuffer(vector> audioRecords) +{ + std::lock_guard lock(serialMutex_); + MEDIA_INFO_LOG("EncodeAudioBuffer enter"); + bool isSuccess = true; + for (sptr audioRecord : audioRecords) { + if (!audioRecord->IsEncoded() && !EncodeAudioBuffer(audioRecord)) { + isSuccess = false; + MEDIA_ERR_LOG("Failed frame id is : %{public}s", audioRecord->GetFrameId().c_str()); + } + } + return isSuccess; +} + + +int32_t AudioEncoder::SetCallback(CodecUserData *codecUserData) +{ + int32_t ret = OH_AudioCodec_RegisterCallback(encoder_, + {SampleCallback::OnCodecError, SampleCallback::OnOutputFormatChanged, + SampleCallback::OnInputBufferAvailable, SampleCallback::OnOutputBufferAvailable}, codecUserData); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Set callback failed, ret: %{public}d", ret); + return 0; +} + +int32_t AudioEncoder::Configure() +{ + OH_AVFormat *format = OH_AVFormat_Create(); + CHECK_AND_RETURN_RET_LOG(format != nullptr, 1, "AVFormat create failed"); + + OH_AVFormat_SetIntValue(format, OH_MD_KEY_AUD_CHANNEL_COUNT, DEFAULT_CHANNEL_COUNT); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_AUD_SAMPLE_RATE, DEFAULT_SAMPLERATE); + OH_AVFormat_SetLongValue(format, OH_MD_KEY_BITRATE, DEFAULT_BITRATE); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_AUDIO_SAMPLE_FORMAT, SAMPLE_FORMAT); + OH_AVFormat_SetLongValue(format, OH_MD_KEY_CHANNEL_LAYOUT, CHANNEL_LAYOUT); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_MAX_INPUT_SIZE, DEFAULT_MAX_INPUT_SIZE); + int ret = OH_AudioCodec_Configure(encoder_, format); + OH_AVFormat_Destroy(format); + format = nullptr; + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Config failed, ret: %{public}d", ret); + return 0; +} +} // CameraStandard +} // OHOS \ No newline at end of file diff --git a/services/camera_service/src/avcodec/audio_video_muxer.cpp b/services/camera_service/src/avcodec/audio_video_muxer.cpp new file mode 100644 index 000000000..69ccf0f57 --- /dev/null +++ b/services/camera_service/src/avcodec/audio_video_muxer.cpp @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "audio_video_muxer.h" +#include "camera_log.h" + +namespace OHOS { +namespace CameraStandard { + +AudioVideoMuxer::AudioVideoMuxer() +{ +} + +AudioVideoMuxer::~AudioVideoMuxer() +{ + MEDIA_INFO_LOG("~AudioVideoMuxer enter"); +} + +int32_t AudioVideoMuxer::Create(int32_t fd, OH_AVOutputFormat format, + std::shared_ptr photoAssetProxy) +{ + muxer_ = OH_AVMuxer_Create(fd, format); + fd_ = fd; + photoAssetProxy_ = photoAssetProxy; + CHECK_AND_RETURN_RET_LOG(muxer_ != nullptr, 1, "Create failed"); + return 0; +} + +int32_t AudioVideoMuxer::Start() +{ + CHECK_AND_RETURN_RET_LOG(muxer_ != nullptr, 1, "muxer_ is null"); + int ret = OH_AVMuxer_Start(muxer_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Start failed, ret: %{public}d", ret); + return 0; +} + +int32_t AudioVideoMuxer::SetRotation(int32_t rotation) +{ + MEDIA_ERR_LOG("SetRotation rotation : %{public}d", rotation); + CHECK_AND_RETURN_RET_LOG(muxer_ != nullptr, 1, "muxer_ is null"); + int ret = OH_AVMuxer_SetRotation(muxer_, rotation); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "SetRotation failed, ret: %{public}d", ret); + return 0; +} + +int32_t AudioVideoMuxer::WriteSampleBuffer(OH_AVBuffer *sample, TrackType type) +{ + CHECK_AND_RETURN_RET_LOG(muxer_ != nullptr, 1, "muxer_ is null"); + int32_t ret = AV_ERR_OK; + int trackId = -1; + switch (type) { + case TrackType::AUDIO_TRACK: + trackId = audioTrackId_; + break; + case TrackType::VIDEO_TRACK: + trackId = videoTrackId_; + break; + case TrackType::META_TRACK: + trackId = metaTrackId_; + break; + default: + MEDIA_ERR_LOG("TrackType type = %{public}d not supported", type); + } + ret = OH_AVMuxer_WriteSampleBuffer(muxer_, trackId, sample); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "WriteSampleBuffer failed, ret: %{public}d", ret); + return 0; +} + +int32_t AudioVideoMuxer::GetVideoFd() +{ + return fd_; +} + +std::shared_ptr AudioVideoMuxer::GetPhotoAssetProxy() +{ + return photoAssetProxy_; +} + + +int32_t AudioVideoMuxer::AddTrack(int &trackId, OH_AVFormat *format, TrackType type) +{ + CHECK_AND_RETURN_RET_LOG(muxer_ != nullptr, 1, "muxer_ is null"); + int ret = OH_AVMuxer_AddTrack(muxer_, &trackId, format); + switch (type) { + case TrackType::AUDIO_TRACK: + audioTrackId_ = trackId; + break; + case TrackType::VIDEO_TRACK: + videoTrackId_ = trackId; + break; + case TrackType::META_TRACK: + metaTrackId_ = trackId; + break; + default: + MEDIA_ERR_LOG("TrackType type = %{public}d not supported", type); + } + OH_AVFormat_Destroy(format); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK || trackId < 0, 1, "AddTrack failed, ret: %{public}d", ret); + return 0; +} + +int32_t AudioVideoMuxer::Stop() +{ + CHECK_AND_RETURN_RET_LOG(muxer_ != nullptr, 1, "muxer_ is null"); + int ret = OH_AVMuxer_Stop(muxer_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Stop failed, ret: %{public}d", ret); + return 0; +} + +int32_t AudioVideoMuxer::Release() +{ + MEDIA_INFO_LOG("AudioVideoMuxer::Release enter"); + if (muxer_ != nullptr) { + OH_AVMuxer_Destroy(muxer_); + muxer_ = nullptr; + close(fd_); + } + return 0; +} + +} // CameraStandard +} // OHOS \ No newline at end of file diff --git a/services/camera_service/src/avcodec/avcodec_task_manager.cpp b/services/camera_service/src/avcodec/avcodec_task_manager.cpp new file mode 100644 index 000000000..b5294c5c2 --- /dev/null +++ b/services/camera_service/src/avcodec/avcodec_task_manager.cpp @@ -0,0 +1,247 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "avcodec_task_manager.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include "audio_capturer_session.h" +#include "audio_record.h" +#include "audio_video_muxer.h" +#include "external_window.h" +#include "frame_record.h" +#include "native_avbuffer.h" +#include "native_avbuffer_info.h" +#include "native_buffer_inner.h" +#include "camera_log.h" +#include "sample_info.h" + +namespace { + using namespace std::string_literals; + using namespace std::chrono_literals; +} +namespace OHOS { +namespace CameraStandard { + +AvcodecTaskManager::~AvcodecTaskManager() +{ + Release(); +} + +AvcodecTaskManager::AvcodecTaskManager(sptr audioCaptureSession) +{ + audioCapturerSession_ = audioCaptureSession; + // Create Task Manager + taskManager_ = make_unique("AvcodecTaskManager", DEFAULT_THREAD_NUMBER, false); + audioEncoderManager_ = make_unique("AudioTaskManager", DEFAULT_ENCODER_THREAD_NUMBER, true); + videoEncoderManager_ = make_unique("VideoTaskManager", DEFAULT_ENCODER_THREAD_NUMBER, true); + videoEncoder_ = make_unique(); + audioEncoder_ = make_unique(); +} + +void AvcodecTaskManager::EncodeVideoBuffer(sptr frameRecord, CacheCbFunc cacheCallback) +{ + videoEncoderManager_->SubmitTask([this, frameRecord, cacheCallback]() { + bool isEncodeSuccess = false; + if (!videoEncoder_ && !frameRecord) { + return; + } + isEncodeSuccess = videoEncoder_->EncodeSurfaceBuffer(frameRecord); + if (isEncodeSuccess) { + videoEncoder_->ReleaseSurfaceBuffer(frameRecord); + } + frameRecord->SetEncodedResult(isEncodeSuccess); + if (isEncodeSuccess) { + MEDIA_INFO_LOG("encode image success %{public}s, refCount: %{public}d", + frameRecord->GetFrameId().c_str(), frameRecord->GetSptrRefCount()); + } else { + MEDIA_ERR_LOG("encode image fail %{public}s", frameRecord->GetFrameId().c_str()); + } + if (cacheCallback) { + cacheCallback(frameRecord, isEncodeSuccess); + } + }); +} + +void AvcodecTaskManager::SubmitTask(function task) +{ + taskManager_->SubmitTask(task); +} + +void AvcodecTaskManager::SetVideoFd(int32_t videoFd, shared_ptr photoAssetProxy) +{ + lock_guard lock(videoFdMutex_); + MEDIA_INFO_LOG("Set videoFd: %{public}d", videoFd); + videoFdQueue_.push(std::make_pair(videoFd, photoAssetProxy)); + cvEmpty_.notify_all(); +} + +sptr AvcodecTaskManager::CreateAVMuxer(sptr frameRecord) +{ + unique_lock lock(videoFdMutex_); + if (videoFdQueue_.empty()) { + bool waitResult = false; + waitResult = cvEmpty_.wait_for(lock, std::chrono::milliseconds(GET_FD_EXPIREATION_TIME), + [this] { return !videoFdQueue_.empty(); }); + if (!waitResult || videoFdQueue_.empty()) { + return nullptr; + } + } + sptr muxer = new AudioVideoMuxer(); + OH_AVOutputFormat format = AV_OUTPUT_FORMAT_MPEG_4; + int32_t fd = videoFdQueue_.front().first; + shared_ptr photoAssetProxy = videoFdQueue_.front().second; + MEDIA_INFO_LOG("CreateAVMuxer with videoFd: %{public}d", fd); + videoFdQueue_.pop(); + muxer->Create(fd, format, photoAssetProxy); + OH_AVFormat *formatVideo = OH_AVFormat_Create(); + OH_AVFormat_SetStringValue(formatVideo, OH_MD_KEY_CODEC_MIME, OH_AVCODEC_MIMETYPE_VIDEO_AVC); + muxer->SetRotation(frameRecord->GetRotation()); + MEDIA_INFO_LOG("CreateAVMuxer with first frame %{public}s", frameRecord->GetFrameId().c_str()); + OH_AVFormat_SetIntValue(formatVideo, OH_MD_KEY_WIDTH, frameRecord->GetFrameSize()->width); + OH_AVFormat_SetIntValue(formatVideo, OH_MD_KEY_HEIGHT, frameRecord->GetFrameSize()->height); + int videoTrackId = -1; + muxer->AddTrack(videoTrackId, formatVideo, VIDEO_TRACK); + MEDIA_INFO_LOG("Succeed create videoTrackId %{public}d", videoTrackId); + OH_AVFormat *formatAudio = OH_AVFormat_Create(); + OH_AVFormat_SetStringValue(formatAudio, OH_MD_KEY_CODEC_MIME, OH_AVCODEC_MIMETYPE_AUDIO_AAC); + OH_AVFormat_SetIntValue(formatAudio, OH_MD_KEY_AUD_SAMPLE_RATE, DEFAULT_SAMPLERATE); + OH_AVFormat_SetIntValue(formatAudio, OH_MD_KEY_AUD_CHANNEL_COUNT, DEFAULT_CHANNEL_COUNT); + int audioTrackId = -1; + muxer->AddTrack(audioTrackId, formatAudio, AUDIO_TRACK); + MEDIA_INFO_LOG("Succeed create audioTrackId %{public}d", audioTrackId); + muxer->Start(); + return muxer; +} + +void AvcodecTaskManager::FinishMuxer(sptr muxer) +{ + MEDIA_INFO_LOG("doMxuer video is finished"); + if (muxer) { + shared_ptr proxy = muxer->GetPhotoAssetProxy(); + MEDIA_INFO_LOG("PhotoAssetProxy notify enter"); + if (proxy) { + proxy->NotifyVideoSaveFinished(); + } + muxer->Stop(); + muxer->Release(); + } +} + + +void AvcodecTaskManager::DoMuxerVideo(vector> frameRecords, string taskName) +{ + if (frameRecords.empty()) { + MEDIA_ERR_LOG("DoMuxerVideo error of empty encoded frame"); + return; + } + taskManager_->SubmitTask([this, frameRecords, taskName]() { + MEDIA_INFO_LOG("CreateAVMuxer with %{public}s %{public}s", + frameRecords.front()->GetFrameId().c_str(), taskName.c_str()); + sptr muxer = this->CreateAVMuxer(frameRecords[0]); + if (muxer == nullptr) { + MEDIA_ERR_LOG("CreateAVMuxer failed"); + return; + } + // CollectAudioBuffer + vector> audioRecords; + if (audioCapturerSession_) { + audioCapturerSession_->GetAudioRecords(frameRecords.front()->GetTimeStamp(), + frameRecords.back()->GetTimeStamp(), audioRecords); + } + for (size_t index = 0; index < frameRecords.size(); index++) { + OH_AVCodecBufferAttr attr = {0, 0, 0, AVCODEC_BUFFER_FLAGS_NONE}; + OH_AVBuffer *buffer = frameRecords[index]->encodedBuffer; + OH_AVBuffer_GetBufferAttr(buffer, &attr); + MEDIA_DEBUG_LOG("DoMuxerVideo frameRecord addr %{public}s videoBuffer addr %{public}p size: %{public}d", + frameRecords[index]->GetFrameId().c_str(), OH_AVBuffer_GetAddr(buffer), attr.size); + attr.pts = index * VIDEO_FRAME_INTERVAL; + OH_AVBuffer_SetBufferAttr(buffer, &attr); + muxer->WriteSampleBuffer(buffer, VIDEO_TRACK); + } + CollectAudioBuffer(audioRecords, muxer); + FinishMuxer(muxer); + }); +} + +void AvcodecTaskManager::CollectAudioBuffer(vector> audioRecordVec, sptr muxer) +{ + MEDIA_INFO_LOG("CollectAudioBuffer start with size %{public}zu", audioRecordVec.size()); + bool isEncodeSuccess = false; + if (!audioEncoder_ || audioRecordVec.empty() || !muxer) { + MEDIA_ERR_LOG("CollectAudioBuffer cannot find useful data"); + return; + } + isEncodeSuccess = audioEncoder_->EncodeAudioBuffer(audioRecordVec); + MEDIA_DEBUG_LOG("encode audio buffer result %{public}d", isEncodeSuccess); + for (size_t index = 0; index < audioRecordVec.size(); index++) { + OH_AVCodecBufferAttr attr = {0, 0, 0, AVCODEC_BUFFER_FLAGS_NONE}; + OH_AVBuffer *buffer = audioRecordVec[index]->encodedBuffer; + OH_AVBuffer_GetBufferAttr(buffer, &attr); + attr.pts = index * AUDIO_FRAME_INTERVAL; + if (index == audioRecordVec.size() - 1) { + attr.flags = AVCODEC_BUFFER_FLAGS_EOS; + } + OH_AVBuffer_SetBufferAttr(buffer, &attr); + muxer->WriteSampleBuffer(buffer, AUDIO_TRACK); + } + MEDIA_INFO_LOG("CollectAudioBuffer finished"); +} + +void AvcodecTaskManager::Release() +{ + MEDIA_INFO_LOG("AvcodecTaskManager release start"); + videoEncoderManager_->SubmitTask([this]() { + if (videoEncoder_ != nullptr) { + videoEncoder_->Release(); + } + }); + audioEncoderManager_->SubmitTask([this]() { + if (audioEncoder_ != nullptr) { + audioEncoder_->Release(); + } + unique_lock lock(videoFdMutex_); + while (!videoFdQueue_.empty()) { + int32_t fd = videoFdQueue_.front().first; + MEDIA_INFO_LOG("close with videoFd: %{public}d", fd); + close(fd); + videoFdQueue_.pop(); + } + MEDIA_INFO_LOG("AvcodecTaskManager release end"); + }); +} + +void AvcodecTaskManager::Stop() +{ + MEDIA_INFO_LOG("AvcodecTaskManager Stop start"); + videoEncoderManager_->SubmitTask([this]() { + if (videoEncoder_ != nullptr) { + videoEncoder_->Stop(); + } + }); + audioEncoderManager_->SubmitTask([this]() { + if (audioEncoder_ != nullptr) { + audioEncoder_->Stop(); + } + MEDIA_INFO_LOG("AvcodecTaskManager Stop end"); + }); +} +} // CameraStandard +} // OHOS \ No newline at end of file diff --git a/services/camera_service/src/avcodec/camera_server_photo_proxy.cpp b/services/camera_service/src/avcodec/camera_server_photo_proxy.cpp new file mode 100644 index 000000000..57fc439b0 --- /dev/null +++ b/services/camera_service/src/avcodec/camera_server_photo_proxy.cpp @@ -0,0 +1,168 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#include +#include "camera_log.h" +#include "datetime_ex.h" +#include "camera_server_photo_proxy.h" +#include "format.h" +#include "photo_proxy.h" + +namespace OHOS { +namespace CameraStandard { + +CameraServerPhotoProxy::CameraServerPhotoProxy() +{ + photoId_ = ""; + deferredProcType_ = 0; + photoWidth_ = 0; + photoHeight_ = 0; + bufferHandle_ = nullptr; + fileDataAddr_ = nullptr; + fileSize_ = 0; + isMmaped_ = false; +} + +CameraServerPhotoProxy::~CameraServerPhotoProxy() +{ + std::lock_guard lock(mutex_); + MEDIA_INFO_LOG("~CameraServerPhotoProxy"); + if (isMmaped_) { + munmap(fileDataAddr_, fileSize_); + } + CameraFreeBufferHandle(const_cast(bufferHandle_)); + fileDataAddr_ = nullptr; + fileSize_ = 0; +} + +int32_t CameraServerPhotoProxy::CameraFreeBufferHandle(BufferHandle *handle) +{ + if (handle == nullptr) { + MEDIA_ERR_LOG("CameraFreeBufferHandle with nullptr handle"); + return 0; + } + if (handle->fd >= 0) { + close(handle->fd); + handle->fd = -1; + } + const uint32_t reserveFds = handle->reserveFds; + for (uint32_t i = 0; i < reserveFds; i++) { + if (handle->reserve[i] >= 0) { + close(handle->reserve[i]); + handle->reserve[i] = -1; + } + } + free(handle); + return 0; +} + +void CameraServerPhotoProxy::SetDisplayName(std::string displayName) +{ + displayName_ = displayName; +} + +void CameraServerPhotoProxy::ReadFromParcel(MessageParcel &parcel) +{ + std::lock_guard lock(mutex_); + photoId_ = parcel.ReadString(); + deferredProcType_ = parcel.ReadInt32(); + isDeferredPhoto_ = parcel.ReadInt32(); + format_ = parcel.ReadInt32(); + photoWidth_ = parcel.ReadInt32(); + photoHeight_ = parcel.ReadInt32(); + isHighQuality_ = parcel.ReadBool(); + bufferHandle_ = ReadBufferHandle(parcel); + MEDIA_INFO_LOG("PhotoProxy::ReadFromParcel"); +} + +std::string CameraServerPhotoProxy::GetPhotoId() +{ + MEDIA_INFO_LOG("PhotoProxy::GetPhotoId photoId: = %{public}s", photoId_.c_str()); + std::lock_guard lock(mutex_); + return photoId_; +} + +Media::DeferredProcType CameraServerPhotoProxy::GetDeferredProcType() +{ + MEDIA_INFO_LOG("PhotoProxy::GetDeferredProcType"); + std::lock_guard lock(mutex_); + if (deferredProcType_ == 0) { + return Media::DeferredProcType::BACKGROUND; + } else { + return Media::DeferredProcType::OFFLINE; + } +} + +void* CameraServerPhotoProxy::GetFileDataAddr() +{ + MEDIA_INFO_LOG("PhotoProxy::GetFileDataAddr"); + std::lock_guard lock(mutex_); + + if (!isMmaped_) { + MEDIA_INFO_LOG("PhotoProxy::GetFileDataAddr mmap"); + fileDataAddr_ = mmap(nullptr, bufferHandle_->size, PROT_READ | PROT_WRITE, MAP_SHARED, bufferHandle_->fd, 0); + } + return fileDataAddr_; +} + +size_t CameraServerPhotoProxy::GetFileSize() +{ + MEDIA_INFO_LOG("PhotoProxy::GetFileSize"); + std::lock_guard lock(mutex_); + + fileSize_ = bufferHandle_->size; + return fileSize_; +} + +int32_t CameraServerPhotoProxy::GetWidth() +{ + return photoWidth_; +} + +int32_t CameraServerPhotoProxy::GetHeight() +{ + return photoHeight_; +} + +PhotoFormat CameraServerPhotoProxy::GetFormat() +{ + return isHighQuality_ ? Media::PhotoFormat::JPG : Media::PhotoFormat::RGBA; +} + +PhotoQuality CameraServerPhotoProxy::GetPhotoQuality() +{ + return isHighQuality_ ? Media::PhotoQuality::HIGH : Media::PhotoQuality::LOW; +} + +void CameraServerPhotoProxy::Release() +{ + MEDIA_INFO_LOG("CameraPhotoProxy release start"); +} + +std::string CameraServerPhotoProxy::GetDisplayName() +{ + return displayName_; +} + +std::string CameraServerPhotoProxy::GetExtension() +{ + return suffix; +} +} // namespace CameraStandard +} // namespace OHOS diff --git a/services/camera_service/src/avcodec/moving_photo_video_cache.cpp b/services/camera_service/src/avcodec/moving_photo_video_cache.cpp new file mode 100644 index 000000000..7b7bd023c --- /dev/null +++ b/services/camera_service/src/avcodec/moving_photo_video_cache.cpp @@ -0,0 +1,186 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "moving_photo_video_cache.h" +#include +#include +#include +#include +#include +#include +#include "external_window.h" +#include "native_buffer_inner.h" +#include "camera_log.h" +#include "refbase.h" +#include "surface_buffer.h" + +namespace { + using namespace std::string_literals; + using namespace std::chrono_literals; +} +namespace OHOS { +namespace CameraStandard { + +MovingPhotoVideoCache::~MovingPhotoVideoCache() +{ + MEDIA_DEBUG_LOG("~MovingPhotoVideoCache enter"); + taskManagerLock_.lock(); + taskManager_ = nullptr; + taskManagerLock_.unlock(); + std::lock_guard lock(callbackVecLock_); + cachedFrameCallbackHandles_.clear(); +} + +MovingPhotoVideoCache::MovingPhotoVideoCache(sptr taskManager) : taskManager_(taskManager) +{ +} + +void MovingPhotoVideoCache::CacheFrame(sptr frameRecord) +{ + MEDIA_DEBUG_LOG("CacheFrame enter"); + const std::string imgId = frameRecord->GetFrameId(); + frameRecord->SetStatusReadyConvertStatus(); + std::lock_guard lock(taskManagerLock_); + if (taskManager_) { + taskManager_->EncodeVideoBuffer(frameRecord, std::bind(&MovingPhotoVideoCache::OnImageEncoded, + this, std::placeholders::_1, std::placeholders::_2)); + } +} + +void MovingPhotoVideoCache::DoMuxerVideo(std::vector> frameRecords, string taskName) +{ + MEDIA_INFO_LOG("DoMuxerVideo enter"); + std::sort(frameRecords.begin(), frameRecords.end(), + [](const sptr& a, const sptr& b) { + return a->GetTimeStamp() < b->GetTimeStamp(); + }); + std::lock_guard lock(taskManagerLock_); + if (taskManager_) { + taskManager_->DoMuxerVideo(frameRecords, taskName); + taskManager_->SubmitTask([this]() { + this->ClearCallbackHandler(); + }); + } +} + +// Call this function after buffer has been encoded +void MovingPhotoVideoCache::OnImageEncoded(sptr frameRecord, bool encodeResult) +{ + std::lock_guard lock(callbackVecLock_); + for (auto cachedFrameCallbackHandle : cachedFrameCallbackHandles_) { + cachedFrameCallbackHandle->OnCacheFrameFinish(frameRecord, encodeResult); + } +} + +void MovingPhotoVideoCache::GetFrameCachedResult(std::vector> frameRecords, + EncodedEndCbFunc encodedEndCbFunc, string taskName) +{ + callbackVecLock_.lock(); + MEDIA_INFO_LOG("GetFrameCachedResult enter frameRecords size: %{public}zu", frameRecords.size()); + sptr cacheFrameHandler = + new CachedFrameCallbackHandle(frameRecords, encodedEndCbFunc, taskName); + cachedFrameCallbackHandles_.push_back(cacheFrameHandler); + callbackVecLock_.unlock(); + for (auto frameRecord : frameRecords) { + if (frameRecord->IsEncoded()) { + cacheFrameHandler->OnCacheFrameFinish(frameRecord, frameRecord->IsEncoded()); + } + } +} + +void MovingPhotoVideoCache::ClearCallbackHandler() +{ + // OnImageEncoded has callbackVecLock_ + MEDIA_INFO_LOG("ClearCallbackHandler enter"); + std::lock_guard lock(callbackVecLock_); + MEDIA_DEBUG_LOG("ClearCallbackHandler get callbackVecLock_"); + cachedFrameCallbackHandles_.erase(std::remove_if(cachedFrameCallbackHandles_.begin(), + cachedFrameCallbackHandles_.end(), + [](const sptr& obj) {return obj->GetCacheRecord().empty();}), + cachedFrameCallbackHandles_.end()); +} + +void MovingPhotoVideoCache::ClearCache() +{ + MEDIA_INFO_LOG("ClearCache enter"); + // clear cache and muxer success buffer + std::lock_guard lock(callbackVecLock_); + for (auto cachedFrameCallbackHandle : cachedFrameCallbackHandles_) { + cachedFrameCallbackHandle->AbortCapture(); + } + cachedFrameCallbackHandles_.clear(); +} + +CachedFrameCallbackHandle::CachedFrameCallbackHandle(std::vector> frameRecords, + EncodedEndCbFunc encodedEndCbFunc, string taskName) + : encodedEndCbFunc_(encodedEndCbFunc), isAbort_(false), taskName_(taskName) +{ + std::lock_guard lock(cacheFrameMutex_); + cacheRecords_.insert(frameRecords.begin(), frameRecords.end()); +} + +CachedFrameCallbackHandle::~CachedFrameCallbackHandle() +{ + MEDIA_INFO_LOG("~CachedFrameCallbackHandle enter"); +} + +void CachedFrameCallbackHandle::OnCacheFrameFinish(sptr frameRecord, bool cachedSuccess) +{ + MEDIA_INFO_LOG("OnCacheFrameFinish enter cachedSuccess: %{public}d", cachedSuccess); + std::lock_guard lock(cacheFrameMutex_); + if (isAbort_) { + // Handle abort + MEDIA_INFO_LOG("OnCacheFrameFinish is abort"); + return; + } + auto it = cacheRecords_.find(frameRecord); + if (it != cacheRecords_.end()) { + cacheRecords_.erase(it); + if (cachedSuccess) { + successCacheRecords_.push_back(frameRecord); + } else { + errorCacheRecords_.push_back(frameRecord); + } + if (!cacheRecords_.empty()) { + // Still waiting for more cache encoded buffer + return; + } + MEDIA_INFO_LOG("encodedEndCbFunc_ is called success count: %{public}zu", successCacheRecords_.size()); + // All buffer have been encoded + if (encodedEndCbFunc_ != nullptr) { + encodedEndCbFunc_(successCacheRecords_, taskName_); + } + } +} + +// This function is called when prestop capture +void CachedFrameCallbackHandle::AbortCapture() +{ + std::lock_guard lock(cacheFrameMutex_); + isAbort_ = true; + cacheRecords_.clear(); + if (encodedEndCbFunc_ != nullptr) { + encodedEndCbFunc_(std::move(successCacheRecords_), taskName_); + } +} + +CachedFrameSet CachedFrameCallbackHandle::GetCacheRecord() +{ + std::lock_guard lock(cacheFrameMutex_); + return cacheRecords_; +} + +} // CameraStandard +} // OHOS \ No newline at end of file diff --git a/services/camera_service/src/avcodec/sample_callback.cpp b/services/camera_service/src/avcodec/sample_callback.cpp new file mode 100644 index 000000000..f5de96951 --- /dev/null +++ b/services/camera_service/src/avcodec/sample_callback.cpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "sample_callback.h" +#include "sample_info.h" +#include "camera_log.h" + +namespace OHOS { +namespace CameraStandard { +void SampleCallback::OnCodecError(OH_AVCodec *codec, int32_t errorCode, void *userData) +{ + (void)codec; + (void)errorCode; + (void)userData; + MEDIA_ERR_LOG("On decoder error, error code: %{public}d", errorCode); +} + +void SampleCallback::OnCodecFormatChange(OH_AVCodec *codec, OH_AVFormat *format, void *userData) +{ + if (userData == nullptr) { + return; + } + (void)codec; + MEDIA_ERR_LOG("OnCodecFormatChange"); +} + + +void SampleCallback::OnNeedInputBuffer(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData) +{ + MEDIA_WARNING_LOG("OnNeedInputBuffer"); + if (userData == nullptr) { + return; + } + (void)codec; + CodecUserData *codecUserData = static_cast(userData); + std::unique_lock lock(codecUserData->inputMutex_); + codecUserData->inputBufferInfoQueue_.emplace(new CodecAVBufferInfo(index, buffer)); + codecUserData->inputCond_.notify_all(); +} + +void SampleCallback::OnNewOutputBuffer(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData) +{ + MEDIA_WARNING_LOG("OnNewOutputBuffer"); + if (userData == nullptr) { + return; + } + (void)codec; + CodecUserData *codecUserData = static_cast(userData); + std::unique_lock lock(codecUserData->outputMutex_); + codecUserData->outputBufferInfoQueue_.emplace(new CodecAVBufferInfo(index, buffer)); + codecUserData->outputCond_.notify_all(); +} + +void SampleCallback::OnOutputFormatChanged(OH_AVCodec *codec, OH_AVFormat *format, void *userData) +{ + (void)codec; + (void)format; + (void)userData; + MEDIA_WARNING_LOG("OnOutputFormatChanged received"); +} + +void SampleCallback::OnInputBufferAvailable(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData) +{ + MEDIA_WARNING_LOG("OnInputBufferAvailable %{public}d", index); + if (userData == nullptr) { + return; + } + (void)codec; + CodecUserData *codecAudioData = static_cast(userData); + std::unique_lock lock(codecAudioData->inputMutex_); + codecAudioData->inputBufferInfoQueue_.emplace(new CodecAVBufferInfo(index, buffer)); + codecAudioData->inputCond_.notify_all(); +} + +void SampleCallback::OnOutputBufferAvailable(OH_AVCodec *codec, uint32_t index, OH_AVBuffer *buffer, void *userData) +{ + MEDIA_WARNING_LOG("OnOutputBufferAvailable"); + (void)codec; + CodecUserData *codecAudioData = static_cast(userData); + std::unique_lock lock(codecAudioData->outputMutex_); + codecAudioData->outputBufferInfoQueue_.emplace(new CodecAVBufferInfo(index, buffer)); + codecAudioData->outputCond_.notify_all(); +} +} // CameraStandard +} // OHOS \ No newline at end of file diff --git a/services/camera_service/src/avcodec/video_encoder.cpp b/services/camera_service/src/avcodec/video_encoder.cpp new file mode 100644 index 000000000..14d755357 --- /dev/null +++ b/services/camera_service/src/avcodec/video_encoder.cpp @@ -0,0 +1,329 @@ +/* + * Copyright (c) 2024-2024 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "video_encoder.h" +#include +#include +#include +#include "frame_record.h" +#include "surface_type.h" +#include "external_window.h" +#include "sample_callback.h" +#include "camera_log.h" +#include +#include +#include +#include +#include +#include +#include "surface_utils.h" +#include + +namespace OHOS { +namespace CameraStandard { + +VideoEncoder::~VideoEncoder() +{ + MEDIA_INFO_LOG("~VideoEncoder enter"); + Release(); +} + +int32_t VideoEncoder::Create(const std::string &codecMime) +{ + std::lock_guard lock(encoderMutex_); + encoder_ = OH_VideoEncoder_CreateByMime(codecMime.data()); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Create failed"); + return 0; +} + +int32_t VideoEncoder::Config() +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + std::unique_lock contextLock(contextMutex_); + context_ = new CodecUserData; + // Configure video encoder + int32_t ret = Configure(); + CHECK_AND_RETURN_RET_LOG(ret == 0, 1, "Configure failed"); + // SetCallback for video encoder + ret = SetCallback(context_); + CHECK_AND_RETURN_RET_LOG(ret == 0, 1, "Set callback failed"); + contextLock.unlock(); + return 0; +} + +int32_t VideoEncoder::Start() +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + // Prepare video encoder + int ret = OH_VideoEncoder_Prepare(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Prepare failed, ret: %{public}d", ret); + // Start video encoder + ret = OH_VideoEncoder_Start(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Start failed, ret: %{public}d", ret); + isStarted_ = true; + return 0; +} + +int32_t VideoEncoder::GetSurface() +{ + std::lock_guard lock(encoderMutex_); + OHNativeWindow *nativeWindow; + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + int ret = OH_VideoEncoder_GetSurface(encoder_, &nativeWindow); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Get surface failed, ret: %{public}d", ret); + uint64_t surfaceId; + ret = OH_NativeWindow_GetSurfaceId(nativeWindow, &surfaceId); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Get surfaceId failed, ret: %{public}d", ret); + sptr surface = SurfaceUtils::GetInstance()->GetSurface(surfaceId); + CHECK_AND_RETURN_RET_LOG(surface != nullptr, 1, "Surface is null"); + surfaceMutex_.lock(); + codecSurface_ = surface; + surfaceMutex_.unlock(); + return 0; +} + +int32_t VideoEncoder::ReleaseSurfaceBuffer(sptr frameRecord) +{ + CHECK_AND_RETURN_RET_LOG(frameRecord->GetSurfaceBuffer() != nullptr, 1, + "SurfaceBuffer is released %{public}s", frameRecord->GetFrameId().c_str()); + sptr syncFence = SyncFence::INVALID_FENCE; + BufferRequestConfig requestConfig = { + .width = frameRecord->GetSurfaceBuffer()->GetWidth(), + .height = frameRecord->GetSurfaceBuffer()->GetHeight(), + .strideAlignment = 0x8, // default stride is 8 Bytes. + .format = frameRecord->GetSurfaceBuffer()->GetFormat(), + .usage = frameRecord->GetSurfaceBuffer()->GetUsage(), + .timeout = 0, + }; + sptr releaseBuffer; + { + std::lock_guard lock(surfaceMutex_); + CHECK_AND_RETURN_RET_LOG(codecSurface_ != nullptr, 1, "codecSurface_ is null"); + SurfaceError ret = codecSurface_->RequestBuffer(releaseBuffer, syncFence, requestConfig); + if (ret != SURFACE_ERROR_OK) { + MEDIA_ERR_LOG("RequestBuffer failed. %{public}d", ret); + return ret; + } + constexpr uint32_t waitForEver = -1; + (void)syncFence->Wait(waitForEver); + + if (!releaseBuffer) { + MEDIA_ERR_LOG("Failed to requestBuffer, %{public}s", frameRecord->GetFrameId().c_str()); + return ret; + } + ret = codecSurface_->DetachBufferFromQueue(releaseBuffer); + if (ret != SURFACE_ERROR_OK) { + MEDIA_ERR_LOG("Failed to detach buffer"); + return ret; + } + } + frameRecord->SetSurfaceBuffer(releaseBuffer); + // after request surfaceBuffer + frameRecord->NotifyBufferRelease(); + MEDIA_INFO_LOG("release codec surface buffer end"); + return 0; +} + +int32_t VideoEncoder::PushInputData(sptr info) +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Decoder is null"); + int32_t ret = AV_ERR_OK; + ret = OH_AVBuffer_SetBufferAttr(info->buffer, &info->attr); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Set avbuffer attr failed, ret: %{public}d", ret); + ret = OH_VideoEncoder_PushInputBuffer(encoder_, info->bufferIndex); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Push input data failed, ret: %{public}d", ret); + return 0; +} + +int32_t VideoEncoder::NotifyEndOfStream() +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + int32_t ret = OH_VideoEncoder_NotifyEndOfStream(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, + "Notify end of stream failed, ret: %{public}d", ret); + return 0; +} + +int32_t VideoEncoder::FreeOutputData(uint32_t bufferIndex) +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + int32_t ret = OH_VideoEncoder_FreeOutputBuffer(encoder_, bufferIndex); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, + "Free output data failed, ret: %{public}d", ret); + return 0; +} + +int32_t VideoEncoder::Stop() +{ + std::lock_guard lock(encoderMutex_); + CHECK_AND_RETURN_RET_LOG(encoder_ != nullptr, 1, "Encoder is null"); + int ret = OH_VideoEncoder_Flush(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Flush failed, ret: %{public}d", ret); + ret = OH_VideoEncoder_Stop(encoder_); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Stop failed, ret: %{public}d", ret); + isStarted_ = false; + return 0; +} + +void VideoEncoder::RestartVideoCodec(shared_ptr size, int32_t rotation) +{ + Release(); + size_ = size; + rotation_ = rotation; + Create(MIME_VIDEO_AVC.data()); + Config(); + GetSurface(); + Start(); +} + +bool VideoEncoder::EnqueueBuffer(sptr frameRecord, int32_t keyFrameInterval) +{ + if (!isStarted_ || encoder_ == nullptr || size_ == nullptr) { + RestartVideoCodec(frameRecord->GetFrameSize(), frameRecord->GetRotation()); + } + if (keyFrameInterval == 0) { + std::lock_guard lock(encoderMutex_); + OH_AVFormat *format = OH_AVFormat_Create(); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_REQUEST_I_FRAME, true); + OH_VideoEncoder_SetParameter(encoder_, format); + } + sptr buffer = frameRecord->GetSurfaceBuffer(); + if (buffer == nullptr) { + MEDIA_ERR_LOG("Enqueue video buffer is empty"); + return false; + } + std::lock_guard lock(surfaceMutex_); + CHECK_AND_RETURN_RET_LOG(codecSurface_ != nullptr, false, "codecSurface_ is null"); + SurfaceError surfaceRet = codecSurface_->AttachBufferToQueue(buffer); + if (surfaceRet != SURFACE_ERROR_OK) { + MEDIA_ERR_LOG("Failed to attach buffer, surfaceRet: %{public}d", surfaceRet); + // notify release buffer when attach failed + frameRecord->NotifyBufferRelease(); + return false; + } + constexpr int32_t invalidFence = -1; + BufferFlushConfig flushConfig = { + .damage = { + .w = buffer->GetWidth(), + .h = buffer->GetHeight(), + }, + .timestamp = frameRecord->GetTimeStamp(), + }; + surfaceRet = codecSurface_->FlushBuffer(buffer, invalidFence, flushConfig); + CHECK_AND_RETURN_RET_LOG(surfaceRet == 0, false, "FlushBuffer failed"); + MEDIA_DEBUG_LOG("Success frame id is : %{public}s", frameRecord->GetFrameId().c_str()); + return true; +} + +bool VideoEncoder::EncodeSurfaceBuffer(sptr frameRecord) +{ + int32_t keyFrameInterval = 0; + if (!EnqueueBuffer(frameRecord, keyFrameInterval)) { + return false; + } + // IDR frame is need if keyFrameInterval is 0 + int32_t needRestoreNumber = 2; + int32_t retryCount = 10; + while (retryCount > 0) { + retryCount--; + std::unique_lock contextLock(contextMutex_); + CHECK_AND_RETURN_RET_LOG(context_ != nullptr, false, "VideoEncoder has been released"); + std::unique_lock lock(context_->outputMutex_); + bool condRet = context_->outputCond_.wait_for(lock, std::chrono::milliseconds(BUFFER_ENCODE_EXPIREATION_TIME), + [this]() { return !isStarted_ || !context_->outputBufferInfoQueue_.empty(); }); + CHECK_AND_CONTINUE_LOG(!context_->outputBufferInfoQueue_.empty(), + "Buffer queue is empty, continue, cond ret: %{public}d", condRet); + sptr bufferInfo = context_->outputBufferInfoQueue_.front(); + context_->outputBufferInfoQueue_.pop(); + context_->outputFrameCount_++; + MEDIA_INFO_LOG("Out buffer count: %{public}u, size: %{public}d, flag: %{public}u, pts:%{public}" PRId64, + context_->outputFrameCount_, bufferInfo->attr.size, bufferInfo->attr.flags, bufferInfo->attr.pts); + lock.unlock(); + contextLock.unlock(); + if (bufferInfo->attr.flags == AVCODEC_BUFFER_FLAGS_CODEC_DATA) { + // first return IDR frame + OH_AVBuffer *IDRBuffer = bufferInfo->GetCopyAVBuffer(); + frameRecord->CacheIDRBuffer(IDRBuffer); + } else { + // then return I frame + bufferInfo->AddCopyAVBuffer(frameRecord->encodedBuffer); + } + int32_t ret = FreeOutputData(bufferInfo->bufferIndex); + CHECK_AND_BREAK_LOG(ret == 0, "FreeOutputData failed"); + if (--needRestoreNumber == 0) { + MEDIA_DEBUG_LOG("Success frame id is : %{public}s, refCount: %{public}d", + frameRecord->GetFrameId().c_str(), frameRecord->GetSptrRefCount()); + return true; + } + } + MEDIA_ERR_LOG("Failed frame id is : %{public}s", frameRecord->GetFrameId().c_str()); + return false; +} + +int32_t VideoEncoder::Release() +{ + std::unique_lock contextLock(contextMutex_); + if (context_ != nullptr) { + delete context_; + context_ = nullptr; + } + contextLock.unlock(); + std::lock_guard lock(encoderMutex_); + if (encoder_ != nullptr) { + OH_VideoEncoder_Destroy(encoder_); + encoder_ = nullptr; + } + isStarted_ = false; + return 0; +} + +int32_t VideoEncoder::SetCallback(CodecUserData *codecUserData) +{ + int32_t ret = AV_ERR_OK; + ret = OH_VideoEncoder_RegisterCallback(encoder_, + {SampleCallback::OnCodecError, SampleCallback::OnCodecFormatChange, + SampleCallback::OnNeedInputBuffer, SampleCallback::OnNewOutputBuffer}, codecUserData); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Set callback failed, ret: %{public}d", ret); + return 0; +} + +int32_t VideoEncoder::Configure() +{ + OH_AVFormat *format = OH_AVFormat_Create(); + CHECK_AND_RETURN_RET_LOG(format != nullptr, 1, "AVFormat create failed"); + + OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, size_->width); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, size_->height); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_ROTATION, rotation_); + OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, VIDOE_FRAME_RATE); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_VIDEO_ENCODE_BITRATE_MODE, CBR); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_BITRATE, BITRATE_30M); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, VIDOE_PIXEL_FORMAT); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, 0); + + int ret = OH_VideoEncoder_Configure(encoder_, format); + CHECK_AND_RETURN_RET_LOG(ret == AV_ERR_OK, 1, "Config failed, ret: %{public}d", ret); + + OH_AVFormat_Destroy(format); + format = nullptr; + return 0; +} +} // CameraStandard +} // OHOS \ No newline at end of file diff --git a/services/camera_service/src/hcamera_device.cpp b/services/camera_service/src/hcamera_device.cpp index 97eadb1c9..507b0201e 100644 --- a/services/camera_service/src/hcamera_device.cpp +++ b/services/camera_service/src/hcamera_device.cpp @@ -471,6 +471,30 @@ void HCameraDevice::CheckZoomChange(const std::shared_ptr cameraAbility; + int32_t ret = cameraHostManager_->GetCameraAbility(cameraID_, cameraAbility); + if (cameraAbility == nullptr) { + return false; + } + camera_metadata_item_t metadataItem; + std::vector modes = {}; + ret = OHOS::Camera::FindCameraMetadataItem(cameraAbility->get(), OHOS_ABILITY_MOVING_PHOTO, + &metadataItem); + if (ret == CAM_META_SUCCESS) { + uint32_t step = 3; + for (uint32_t index = 0; index < metadataItem.count - 1;) { + if (metadataItem.data.i32[index + 1] == 1) { + modes.push_back(metadataItem.data.i32[index]); + } + MEDIA_DEBUG_LOG("IsMovingPhotoSupported mode:%{public}d", metadataItem.data.i32[index]); + index += step; + } + } + return std::find(modes.begin(), modes.end(), mode) != modes.end(); +} + void HCameraDevice::ResetZoomTimer() { CameraTimer::GetInstance()->Unregister(zoomTimerId_); diff --git a/services/camera_service/src/hcapture_session.cpp b/services/camera_service/src/hcapture_session.cpp index 91a33c862..436f20af4 100644 --- a/services/camera_service/src/hcapture_session.cpp +++ b/services/camera_service/src/hcapture_session.cpp @@ -19,15 +19,19 @@ #include #include #include +#include +#include #include #include #include +#include #include #include -#include - +#include +#include "avcodec_task_manager.h" #include "bundle_mgr_interface.h" #include "camera_log.h" +#include "camera_server_photo_proxy.h" #include "camera_service_ipc_interface_code.h" #include "camera_util.h" #include "errors.h" @@ -43,13 +47,17 @@ #include "iservice_registry.h" #include "istream_common.h" #include "metadata_utils.h" +#include "moving_photo_video_cache.h" +#include "datetime_ex.h" #include "refbase.h" +#include "surface.h" #include "system_ability_definition.h" #include "v1_0/types.h" #include "display/composer/v1_1/display_composer_type.h" #include "smooth_zoom.h" #include "hcamera_restore_param.h" #include "camera_report_uitls.h" +#include "media_library_manager.h" using namespace OHOS::AAFwk; namespace OHOS { @@ -177,6 +185,7 @@ int32_t HCaptureSession::BeginConfig() } UnlinkInputAndOutputs(); ClearSketchRepeatStream(); + ClearMovingPhotoRepeatStream(); }); if (errCode == CAMERA_OK) { MEDIA_INFO_LOG("HCaptureSession::BeginConfig execute success"); @@ -277,6 +286,58 @@ int32_t HCaptureSession::AddOutputStream(sptr stream) return CAMERA_OK; } +void HCaptureSession::StartMovingPhotoStream() +{ + int32_t errorCode = 0; + stateMachine_.StateGuard([&errorCode, this](CaptureSessionState currentState) { + if (currentState != CaptureSessionState::SESSION_CONFIG_COMMITTED) { + MEDIA_ERR_LOG("EnableMovingPhoto, invalid session state: %{public}d, start after preview", currentState); + errorCode = CAMERA_INVALID_STATE; + return; + } + auto repeatStreams = streamContainer_.GetStreams(StreamType::REPEAT); + bool isPreviewStarted = false; + for (auto& item : repeatStreams) { + auto curStreamRepeat = CastStream(item); + auto repeatType = curStreamRepeat->GetRepeatStreamType(); + if (repeatType != RepeatStreamType::PREVIEW) { + continue; + } + if (curStreamRepeat->GetPreparedCaptureId() != CAPTURE_ID_UNSET && curStreamRepeat->producer_ != nullptr) { + isPreviewStarted = true; + break; + } + } + if (!isPreviewStarted) { + MEDIA_ERR_LOG("EnableMovingPhoto, preview is not streaming"); + return; + } + std::shared_ptr settings = nullptr; + auto cameraDevice = GetCameraDevice(); + if (cameraDevice != nullptr) { + settings = cameraDevice->CloneCachedSettings(); + DumpMetadata(settings); + } + for (auto& item : repeatStreams) { + auto curStreamRepeat = CastStream(item); + auto repeatType = curStreamRepeat->GetRepeatStreamType(); + if (repeatType != RepeatStreamType::LIVEPHOTO) { + continue; + } + if (isSetMotionPhoto_) { + errorCode = curStreamRepeat->Start(settings); + std::lock_guard lock(movingPhotoStatusLock_); + audioCapturerSession_ != nullptr && audioCapturerSession_->StartAudioCapture(); + } else { + errorCode = curStreamRepeat->Stop(); + StopMovingPhoto(); + } + break; + } + }); + MEDIA_INFO_LOG("HCaptureSession::StartMovingPhotoStream result:%{public}d", errorCode); +} + int32_t HCaptureSession::AddOutput(StreamType streamType, sptr stream) { int32_t errorCode = CAMERA_INVALID_ARG; @@ -518,6 +579,69 @@ void HCaptureSession::ExpandSketchRepeatStream() MEDIA_DEBUG_LOG("Exit HCaptureSession::ExpandSketchRepeatStream()"); } +void HCaptureSession::ExpandMovingPhotoRepeatStream() +{ + if (!GetCameraDevice()->CheckMovingPhotoSupported(GetopMode())) { + MEDIA_DEBUG_LOG("movingPhoto is not supported"); + return; + } + MEDIA_DEBUG_LOG("Enter HCaptureSession::ExpandMovingPhotoRepeatStream()"); + auto repeatStreams = streamContainer_.GetStreams(StreamType::REPEAT); + for (auto& stream : repeatStreams) { + if (stream == nullptr) { + continue; + } + auto streamRepeat = CastStream(stream); + if (streamRepeat->GetRepeatStreamType() == RepeatStreamType::PREVIEW) { + std::lock_guard lock(movingPhotoStatusLock_); + if (surface_) { + continue; + } + surface_ = Surface::CreateSurfaceAsConsumer("movingPhoto"); + surface_->SetDefaultUsage(BUFFER_USAGE_VIDEO_ENCODER); + livephotoListener_ = new(std::nothrow) MovingPhotoListener(surface_); + metaSurface_ = Surface::CreateSurfaceAsConsumer("movingPhotoMeta"); + surface_->RegisterConsumerListener((sptr &)livephotoListener_); + surface_->SetDefaultWidthAndHeight(streamRepeat->width_, streamRepeat->height_); + CreateMovingPhotoStreamRepeat(streamRepeat->format_, streamRepeat->width_, + streamRepeat->height_, surface_->GetProducer()); + std::lock_guard streamLock(livePhotoStreamLock_); + livePhotoStreamRepeat_->SetMetaProducer(metaSurface_->GetProducer()); + AddOutputStream(livePhotoStreamRepeat_); + if (!audioCapturerSession_) { + audioCapturerSession_ = new AudioCapturerSession(); + } + if (!taskManager_ && audioCapturerSession_) { + taskManager_ = new AvcodecTaskManager(audioCapturerSession_); + } + if (!videoCache_ && taskManager_) { + videoCache_ = new MovingPhotoVideoCache(taskManager_); + } + } + } + MEDIA_DEBUG_LOG("Exit HCaptureSession::ExpandMovingPhotoRepeatStream()"); +} + +int32_t HCaptureSession::CreateMovingPhotoStreamRepeat( + int32_t format, int32_t width, int32_t height, sptr producer) +{ + CAMERA_SYNC_TRACE; + std::lock_guard lock(livePhotoStreamLock_); + if (width <= 0 || height <= 0) { + MEDIA_ERR_LOG("HCameraService::CreateLivePhotoStreamRepeat args is illegal"); + return CAMERA_INVALID_ARG; + } + if (livePhotoStreamRepeat_ != nullptr) { + livePhotoStreamRepeat_->Release(); + } + auto streamRepeat = new (std::nothrow) HStreamRepeat(producer, format, width, height, RepeatStreamType::LIVEPHOTO); + CHECK_AND_RETURN_RET_LOG(streamRepeat != nullptr, CAMERA_ALLOC_ERROR, "HStreamRepeat allocation failed"); + MEDIA_DEBUG_LOG("para is:%{public}dx%{public}d,%{public}d", width, height, format); + livePhotoStreamRepeat_ = streamRepeat; + MEDIA_INFO_LOG("HCameraService::CreateLivePhotoStreamRepeat end"); + return CAMERA_OK; +} + const sptr HCaptureSession::GetStreamByStreamID(int32_t streamId) { auto stream = streamContainer_.GetStream(streamId); @@ -557,6 +681,55 @@ void HCaptureSession::ClearSketchRepeatStream() MEDIA_DEBUG_LOG("Exit HCaptureSession::ClearSketchRepeatStream()"); } +void HCaptureSession::ClearMovingPhotoRepeatStream() +{ + MEDIA_DEBUG_LOG("Enter HCaptureSession::ClearMovingPhotoRepeatStream()"); + // Already added session lock in BeginConfig() + auto repeatStreams = streamContainer_.GetStreams(StreamType::REPEAT); + for (auto& repeatStream : repeatStreams) { + if (repeatStream == nullptr) { + continue; + } + auto movingPhotoStream = CastStream(repeatStream); + if (movingPhotoStream->GetRepeatStreamType() != RepeatStreamType::LIVEPHOTO) { + continue; + } + StopMovingPhoto(); + std::lock_guard lock(movingPhotoStatusLock_); + if (surface_) { + surface_->UnregisterConsumerListener(); + surface_ = nullptr; + } + metaSurface_ = nullptr; + livephotoListener_ = nullptr; + videoCache_ = nullptr; + taskManager_ = nullptr; + audioCapturerSession_ = nullptr; + MEDIA_DEBUG_LOG("HCaptureSession::ClearLivePhotoRepeatStream() stream id is:%{public}d", + movingPhotoStream->GetFwkStreamId()); + RemoveOutputStream(repeatStream); + } + MEDIA_DEBUG_LOG("Exit HCaptureSession::ClearLivePhotoRepeatStream()"); +} + +void HCaptureSession::StopMovingPhoto() __attribute__((no_sanitize("cfi"))) +{ + MEDIA_DEBUG_LOG("Enter HCaptureSession::StopMovingPhoto"); + std::lock_guard lock(movingPhotoStatusLock_); + if (livephotoListener_) { + livephotoListener_->StopDrainOut(); + } + if (videoCache_) { + videoCache_->ClearCache(); + } + if (audioCapturerSession_) { + audioCapturerSession_->Stop(); + } + if (taskManager_) { + taskManager_->Stop(); + } +} + int32_t HCaptureSession::CommitConfig() { CAMERA_SYNC_TRACE; @@ -577,6 +750,8 @@ int32_t HCaptureSession::CommitConfig() if (errorCode != CAMERA_OK) { return; } + // expand moving photo always + ExpandMovingPhotoRepeatStream(); ExpandSketchRepeatStream(); auto device = GetCameraDevice(); if (device == nullptr) { @@ -588,8 +763,7 @@ int32_t HCaptureSession::CommitConfig() uint64_t secureSeqId = 0L; device ->GetSecureCameraSeq(&secureSeqId); if (((GetopMode() == secureMode) ^ (secureSeqId != 0))) { - MEDIA_ERR_LOG("CaptureSession::CommitConfig is not allowed commit mode = %{public}d." - "secureCamera is should be consistent with secureMode", GetopMode()); + MEDIA_ERR_LOG("secureCamera is not allowed commit mode = %{public}d.", GetopMode()); errorCode = CAMERA_INVALID_ARG; return; } @@ -912,6 +1086,13 @@ void HCaptureSession::ProcessMetaZoomArray( cameraDevice->UpdateSettingOnce(metaZoomArray); } +int32_t HCaptureSession::EnableMovingPhoto(bool isEnable) +{ + isSetMotionPhoto_ = isEnable; + StartMovingPhotoStream(); + return CAMERA_OK; +} + int32_t HCaptureSession::Start() { CAMERA_SYNC_TRACE; @@ -939,20 +1120,7 @@ int32_t HCaptureSession::Start() settings = cameraDevice->CloneCachedSettings(); DumpMetadata(settings); } - - auto repeatStreams = streamContainer_.GetStreams(StreamType::REPEAT); - for (auto& item : repeatStreams) { - auto curStreamRepeat = CastStream(item); - auto repeatType = curStreamRepeat->GetRepeatStreamType(); - if (repeatType != RepeatStreamType::PREVIEW) { - continue; - } - errorCode = curStreamRepeat->Start(settings); - if (errorCode != CAMERA_OK) { - MEDIA_ERR_LOG("HCaptureSession::Start(), Failed to start preview, rc: %{public}d", errorCode); - break; - } - } + errorCode = StartPreviewStream(settings); if (errorCode == CAMERA_OK) { isSessionStarted_ = true; } @@ -961,6 +1129,52 @@ int32_t HCaptureSession::Start() return errorCode; } +int32_t HCaptureSession::StartPreviewStream(const std::shared_ptr& settings) +{ + int32_t errorCode = CAMERA_OK; + auto repeatStreams = streamContainer_.GetStreams(StreamType::REPEAT); + bool hasDerferedPreview = false; + // start preview + for (auto& item : repeatStreams) { + auto curStreamRepeat = CastStream(item); + auto repeatType = curStreamRepeat->GetRepeatStreamType(); + if (repeatType != RepeatStreamType::PREVIEW) { + continue; + } + errorCode = curStreamRepeat->Start(settings); + hasDerferedPreview = curStreamRepeat->producer_ == nullptr; + if (isSetMotionPhoto_ && hasDerferedPreview) { + curStreamRepeat->SetMovingPhotoStartCallback([this]() { + MEDIA_INFO_LOG("StartMovingPhotoStream when addDeferedSurface"); + this->StartMovingPhotoStream(); + }); + } + if (errorCode != CAMERA_OK) { + MEDIA_ERR_LOG("HCaptureSession::Start(), Failed to start preview, rc: %{public}d", errorCode); + break; + } + } + // start movingPhoto + for (auto& item : repeatStreams) { + auto curStreamRepeat = CastStream(item); + auto repeatType = curStreamRepeat->GetRepeatStreamType(); + if (repeatType != RepeatStreamType::LIVEPHOTO) { + continue; + } + int32_t movingPhotoErrorCode = CAMERA_OK; + if (isSetMotionPhoto_ && !hasDerferedPreview) { + movingPhotoErrorCode = curStreamRepeat->Start(settings); + std::lock_guard lock(movingPhotoStatusLock_); + audioCapturerSession_ != nullptr && audioCapturerSession_->StartAudioCapture(); + } + if (movingPhotoErrorCode != CAMERA_OK) { + MEDIA_ERR_LOG("Failed to start movingPhoto, rc: %{public}d", movingPhotoErrorCode); + break; + } + } + return errorCode; +} + int32_t HCaptureSession::Stop() { CAMERA_SYNC_TRACE; @@ -977,6 +1191,9 @@ int32_t HCaptureSession::Stop() auto repeatStream = CastStream(item); if (repeatStream->GetRepeatStreamType() == RepeatStreamType::PREVIEW) { errorCode = repeatStream->Stop(); + } else if (repeatStream->GetRepeatStreamType() == RepeatStreamType::LIVEPHOTO) { + repeatStream->Stop(); + StopMovingPhoto(); } else { repeatStream->Stop(); } @@ -1042,6 +1259,8 @@ int32_t HCaptureSession::Release(CaptureSessionReleaseType type) errorCode = CAMERA_INVALID_STATE; return; } + // stop movingPhoto + StopMovingPhoto(); // Clear outputs ReleaseStreams(); @@ -1065,6 +1284,16 @@ int32_t HCaptureSession::Release(CaptureSessionReleaseType type) SetCallback(emptyCallback); stateMachine_.Transfer(CaptureSessionState::SESSION_RELEASED); isSessionStarted_ = false; + std::lock_guard lock(movingPhotoStatusLock_); + if (surface_) { + surface_->UnregisterConsumerListener(); + } + surface_ = nullptr; + metaSurface_ = nullptr; + livephotoListener_ = nullptr; + videoCache_ = nullptr; + taskManager_ = nullptr; + audioCapturerSession_ = nullptr; }); MEDIA_INFO_LOG("HCaptureSession::Release execute success"); return errorCode; @@ -1212,6 +1441,91 @@ void HCaptureSession::StopUsingPermissionCallback(const uint32_t callingTokenId, } } +int32_t HCaptureSession::StartMovingPhotoCapture() +{ + auto timestamp = high_resolution_clock::now().time_since_epoch(); + StartRecord(std::to_string(timestamp.count())); + return CAMERA_OK; +} + +std::string HCaptureSession::CreateDisplayName() +{ + struct tm currentTime; + std::string formattedTime = ""; + if (GetSystemCurrentTime(¤tTime)) { + std::stringstream ss; + ss << prefix << std::setw(yearWidth) << std::setfill(placeholder) << currentTime.tm_year + startYear + << std::setw(otherWidth) << std::setfill(placeholder) << (currentTime.tm_mon + 1) + << std::setw(otherWidth) << std::setfill(placeholder) << currentTime.tm_mday + << connector << std::setw(otherWidth) << std::setfill(placeholder) << currentTime.tm_hour + << std::setw(otherWidth) << std::setfill(placeholder) << currentTime.tm_min + << std::setw(otherWidth) << std::setfill(placeholder) << currentTime.tm_sec; + formattedTime = ss.str(); + } else { + MEDIA_ERR_LOG("Failed to get current time."); + } + if (lastDisplayName_ == formattedTime) { + saveIndex++; + formattedTime = formattedTime + connector + std::to_string(saveIndex) + suffix; + MEDIA_INFO_LOG("GetDisplayName is %{private}s", formattedTime.c_str()); + return formattedTime; + } + lastDisplayName_ = formattedTime; + formattedTime = formattedTime + suffix; + saveIndex = 0; + MEDIA_INFO_LOG("GetDisplayName is %{private}s", formattedTime.c_str()); + return formattedTime; +} + +int32_t HCaptureSession::CreateMediaLibrary(sptr &photoProxy, + std::string &uri, int32_t &cameraShotType) +{ + sptr object = nullptr; + auto samgr = SystemAbilityManagerClient::GetInstance().GetSystemAbilityManager(); + if (samgr == nullptr) { + MEDIA_ERR_LOG("Failed to get System ability manager"); + return CAMERA_UNKNOWN_ERROR; + } + object = samgr->GetSystemAbility(CAMERA_SERVICE_ID); + if (object == nullptr) { + MEDIA_ERR_LOG("object is null"); + return CAMERA_UNKNOWN_ERROR; + } + auto mediaLibraryManager = Media::MediaLibraryManager::GetMediaLibraryManager(); + if (mediaLibraryManager == nullptr) { + MEDIA_ERR_LOG("Error to init mediaLibraryManager"); + return CAMERA_UNKNOWN_ERROR; + } + mediaLibraryManager->InitMediaLibraryManager(object); + const static int32_t INVALID_UID = -1; + const static int32_t BASE_USER_RANGE = 200000; + int uid = IPCSkeleton::GetCallingUid(); + if (uid <= INVALID_UID) { + MEDIA_ERR_LOG("Get INVALID_UID UID %{public}d", uid); + } + int32_t userId = uid / BASE_USER_RANGE; + MEDIA_DEBUG_LOG("get uid:%{public}d, userId:%{public}d, tokenId:%{public}d", uid, userId, + IPCSkeleton::GetCallingTokenID()); + auto type = isSetMotionPhoto_ ? CameraShotType::MOVING_PHOTO : CameraShotType::IMAGE; + cameraShotType = static_cast(type); + auto photoAssetProxy = mediaLibraryManager->CreatePhotoAssetProxy(type, uid, userId); + MessageParcel data; + photoProxy->WriteToParcel(data); + sptr cameraServerPhotoProxy = new CameraServerPhotoProxy(); + cameraServerPhotoProxy->ReadFromParcel(data); + cameraServerPhotoProxy->SetDisplayName(CreateDisplayName()); + photoAssetProxy->AddPhotoProxy((sptr&)cameraServerPhotoProxy); + uri = photoAssetProxy->GetPhotoAssetUri(); + if (isSetMotionPhoto_) { + int32_t videoFd = photoAssetProxy->GetVideoFd(); + MEDIA_DEBUG_LOG("videFd:%{public}d", videoFd); + if (taskManager_) { + taskManager_->SetVideoFd(videoFd, photoAssetProxy); + } + } + return CAMERA_OK; +} + void PermissionStatusChangeCb::PermStateChangeCallback(Security::AccessToken::PermStateChangeInfo& result) { auto item = captureSession_.promote(); @@ -1258,6 +1572,66 @@ int32_t StreamOperatorCallback::OnCaptureStarted(int32_t captureId, const std::v return CAMERA_OK; } +void HCaptureSession::StartRecord(const std::string taskName) +{ + if (isSetMotionPhoto_) { + taskManager_->SubmitTask([this, taskName]() { + this->StartOnceRecord(taskName); + }); + } +} + +SessionDrainImageCallback::SessionDrainImageCallback(std::vector>& frameCacheList, + wptr listener, + wptr cache, + std::string taskName) + : frameCacheList_(frameCacheList), listener_(listener), videoCache_(cache), taskName_(taskName) +{ +} + +SessionDrainImageCallback::~SessionDrainImageCallback() +{ + MEDIA_INFO_LOG("~SessionDrainImageCallback enter"); +} + +void SessionDrainImageCallback::OnDrainImage(sptr frame) +{ + MEDIA_INFO_LOG("OnDrainImage enter"); + frameCacheList_.push_back(frame); + auto videoCache = videoCache_.promote(); + if (frame->IsIdle() && videoCache) { + videoCache_->CacheFrame(frame); + } else if (frame->IsFinishCache() && videoCache) { + videoCache_->OnImageEncoded(frame, true); + } else { + MEDIA_INFO_LOG("videoCache and frame is not useful"); + } +} + +void SessionDrainImageCallback::OnDrainImageFinish(bool isFinished) +{ + MEDIA_INFO_LOG("OnDrainImageFinish enter"); + auto videoCache = videoCache_.promote(); + videoCache_->GetFrameCachedResult(frameCacheList_, + std::bind(&MovingPhotoVideoCache::DoMuxerVideo, videoCache, + std::placeholders::_1, std::placeholders::_2), taskName_); + auto listener = listener_.promote(); + if (listener && isFinished) { + listener->RemoveDrainImageManager(this); + } +} + +void HCaptureSession::StartOnceRecord(string taskName) +{ + MEDIA_INFO_LOG("StartOnceRecord enter %{public}s", taskName.c_str()); + // frameCacheList only used by now thread + std::vector> frameCacheList; + sptr imageCallback = new SessionDrainImageCallback(frameCacheList, + livephotoListener_, videoCache_, taskName); + livephotoListener_->DrainOutImage(imageCallback); + MEDIA_INFO_LOG("StartOnceRecord end"); +} + int32_t StreamOperatorCallback::OnCaptureStarted_V1_2( int32_t captureId, const std::vector& infos) { @@ -1323,7 +1697,7 @@ int32_t StreamOperatorCallback::OnCaptureError(int32_t captureId, const std::vec int32_t StreamOperatorCallback::OnFrameShutter( int32_t captureId, const std::vector& streamIds, uint64_t timestamp) { - MEDIA_INFO_LOG("StreamOperatorCallback::OnFrameShutter"); + MEDIA_INFO_LOG("StreamOperatorCallback::OnFrameShutter ts is:%{public}" PRId64, timestamp); std::lock_guard lock(cbMutex_); for (auto& streamId : streamIds) { sptr curStream = GetHdiStreamByStreamID(streamId); @@ -1497,5 +1871,100 @@ std::list> StreamContainer::GetAllStreams() } return totalOrderedStreams; } +MovingPhotoListener::MovingPhotoListener(sptr surface) + : surface_(surface), recorderBufferQueue_("videoBuffer", CACHE_FRAME_COUNT) +{ +} + +MovingPhotoListener::~MovingPhotoListener() +{ + while (!recorderBufferQueue_.Empty()) { + MEDIA_ERR_LOG("surface_ release surface buffer"); + sptr popFrame= recorderBufferQueue_.Pop(); + popFrame->ReleaseSurfaceBuffer(surface_); + } + surface_ = nullptr; + recorderBufferQueue_.SetActive(false); + recorderBufferQueue_.Clear(); + MEDIA_ERR_LOG("HStreamRepeat::LivePhotoListener ~ end"); +} + +void MovingPhotoListener::RemoveDrainImageManager(sptr callback) +{ + callbackMap_.Erase(callback); + MEDIA_INFO_LOG("RemoveDrainImageManager drainImageManagerVec_ Start %d", callbackMap_.Size()); +} + +void MovingPhotoListener::StopDrainOut() +{ + MEDIA_INFO_LOG("StopDrainOut drainImageManagerVec_ Start %d", callbackMap_.Size()); + callbackMap_.Iterate([](const sptr callback, sptr manager) { + manager->DrainFinish(false); + }); + callbackMap_.Clear(); +} + +void MovingPhotoListener::OnBufferAvailable() +{ + MEDIA_DEBUG_LOG("surface_ OnBufferAvailable %{public}u, transform %{public}d", + surface_->GetQueueSize(), surface_->GetTransform()); + + if (!surface_) { + MEDIA_ERR_LOG("streamRepeat surface is null"); + return; + } + int64_t timestamp; + OHOS::Rect damage; + sptr buffer; + sptr syncFence = SyncFence::INVALID_FENCE; + SurfaceError surfaceRet = surface_->AcquireBuffer(buffer, syncFence, timestamp, damage); + if (surfaceRet != SURFACE_ERROR_OK) { + MEDIA_ERR_LOG("Failed to acquire surface buffer"); + return; + } + surfaceRet = surface_->DetachBufferFromQueue(buffer); + if (surfaceRet != SURFACE_ERROR_OK) { + MEDIA_ERR_LOG("Failed to detach buffer. %{public}d", surfaceRet); + return; + } + if (recorderBufferQueue_.Full()) { + MEDIA_DEBUG_LOG("surface_ release surface buffer"); + sptr popFrame= recorderBufferQueue_.Pop(); + popFrame->ReleaseSurfaceBuffer(surface_); + MEDIA_DEBUG_LOG("surface_ release surface buffer: %{public}s, refCount: %{public}d", + popFrame->GetFrameId().c_str(), popFrame->GetSptrRefCount()); + } + MEDIA_DEBUG_LOG("surface_ push buffer %{public}d x %{public}d, stride is %{public}d", + buffer->GetSurfaceBufferWidth(), buffer->GetSurfaceBufferHeight(), buffer->GetStride()); + sptr frameRecord = new FrameRecord(buffer, NanosecToMillisec(timestamp), surface_->GetTransform()); + recorderBufferQueue_.Push(frameRecord); + vector> callbacks; + callbackMap_.Iterate([frameRecord, &callbacks](const sptr callback, + sptr manager) { + callbacks.push_back(callback); + }); + for (sptr drainImageCallback : callbacks) { + sptr drainImageManager; + if (callbackMap_.Find(drainImageCallback, drainImageManager)) { + drainImageManager->DrainImage(frameRecord); + } + } +} + +void MovingPhotoListener::DrainOutImage(sptr drainImageCallback) +{ + sptr drainImageManager = + new DrainImageManager(drainImageCallback, recorderBufferQueue_.Size() + CACHE_FRAME_COUNT); + { + MEDIA_INFO_LOG("DrainOutImage enter %{public}zu", recorderBufferQueue_.Size()); + callbackMap_.Insert(drainImageCallback, drainImageManager); + } + // Convert recorderBufferQueue_ to a vector + std::vector> frameList = recorderBufferQueue_.GetAllElements(); + for (const auto& frame : frameList) { + MEDIA_ERR_LOG("DrainOutImage enter DrainImage"); + drainImageManager->DrainImage(frame); + } +} } // namespace CameraStandard } // namespace OHOS diff --git a/services/camera_service/src/hstream_repeat.cpp b/services/camera_service/src/hstream_repeat.cpp index f6daf0664..07983d1fa 100644 --- a/services/camera_service/src/hstream_repeat.cpp +++ b/services/camera_service/src/hstream_repeat.cpp @@ -73,7 +73,17 @@ int32_t HStreamRepeat::LinkInput(sptr void HStreamRepeat::SetStreamInfo(StreamInfo_V1_1& streamInfo) { HStreamCommon::SetStreamInfo(streamInfo); + sptr bufferProducerSequenceable = new BufferProducerSequenceable(metaProducer_); + HDI::Camera::V1_1::ExtendedStreamInfo metaExtendedStreamInfo { + .type = static_cast(4), .width = 0, .height = 0, .format = 0, + .dataspace = 0, .bufferQueue = bufferProducerSequenceable + }; switch (repeatStreamType_) { + case RepeatStreamType::LIVEPHOTO: + streamInfo.v1_0.intent_ = StreamIntent::VIDEO; + streamInfo.v1_0.encodeType_ = ENCODE_TYPE_H264; + streamInfo.extendedStreamInfos = { metaExtendedStreamInfo }; + break; case RepeatStreamType::VIDEO: streamInfo.v1_0.intent_ = StreamIntent::VIDEO; streamInfo.v1_0.encodeType_ = ENCODE_TYPE_H264; @@ -84,13 +94,9 @@ void HStreamRepeat::SetStreamInfo(StreamInfo_V1_1& streamInfo) if (mEnableSecure) { MEDIA_INFO_LOG("HStreamRepeat::SetStreamInfo Enter"); HDI::Camera::V1_1::ExtendedStreamInfo extendedStreamInfo { - .type = static_cast( - HDI::Camera::V1_3::ExtendedStreamInfoType::EXTENDED_STREAM_INFO_SECURE), - .width = 0, - .height = 0, - .format = 0, - .dataspace = 0, - .bufferQueue = nullptr + .type = static_cast( + HDI::Camera::V1_3::ExtendedStreamInfoType::EXTENDED_STREAM_INFO_SECURE), + .width = 0, .height = 0, .format = 0, .dataspace = 0, .bufferQueue = nullptr }; MEDIA_INFO_LOG("HStreamRepeat::SetStreamInfo end"); streamInfo.extendedStreamInfos = { extendedStreamInfo }; @@ -102,17 +108,25 @@ void HStreamRepeat::SetStreamInfo(StreamInfo_V1_1& streamInfo) HDI::Camera::V1_1::ExtendedStreamInfo extendedStreamInfo { .type = static_cast( HDI::Camera::V1_2::EXTENDED_STREAM_INFO_SKETCH), - .width = 0, - .height = 0, - .format = 0, - .dataspace = 0, - .bufferQueue = nullptr + .width = 0, .height = 0, .format = 0, .dataspace = 0, .bufferQueue = nullptr }; streamInfo.extendedStreamInfos = { extendedStreamInfo }; break; } } +void HStreamRepeat::SetMetaProducer(sptr metaProducer) +{ + std::lock_guard lock(producerLock_); + metaProducer_ = metaProducer; +} + +void HStreamRepeat::SetMovingPhotoStartCallback(std::function callback) +{ + std::lock_guard lock(movingPhotoCallbackLock_); + startMovingPhotoCallback_ = callback; +} + void HStreamRepeat::UpdateSketchStatus(SketchStatus status) { if (repeatStreamType_ != RepeatStreamType::SKETCH) { @@ -404,6 +418,11 @@ int32_t HStreamRepeat::AddDeferredSurface(const sptr& pro MEDIA_ERR_LOG("HStreamRepeat::AttachBufferQueue(), Failed to AttachBufferQueue %{public}d", rc); } MEDIA_INFO_LOG("HStreamRepeat::AddDeferredSurface end %{public}d", rc); + std::lock_guard lock(movingPhotoCallbackLock_); + if (startMovingPhotoCallback_) { + startMovingPhotoCallback_(); + startMovingPhotoCallback_ = nullptr; + } return CAMERA_OK; } @@ -472,9 +491,9 @@ int32_t HStreamRepeat::SetFrameRate(int32_t minFrameRate, int32_t maxFrameRate) } OHOS::Camera::MetadataUtils::ConvertMetadataToVec(dynamicSetting, repeatSettings); } - + auto streamOperator = GetStreamOperator(); - + CamRetCode rc = HDI::Camera::V1_0::NO_ERROR; if (streamOperator != nullptr) { std::lock_guard startStopLock(streamStartStopLock_); diff --git a/services/deferred_processing_service/include/base/blocking_queue.h b/services/deferred_processing_service/include/base/blocking_queue.h index 624ca511e..76da88711 100644 --- a/services/deferred_processing_service/include/base/blocking_queue.h +++ b/services/deferred_processing_service/include/base/blocking_queue.h @@ -16,12 +16,12 @@ #ifndef OHOS_DEFERRED_PROCESSING_SERVICE_BLOCKING_QUEUE_H #define OHOS_DEFERRED_PROCESSING_SERVICE_BLOCKING_QUEUE_H -#include #include #include #include #include #include +#include namespace OHOS { namespace CameraStandard { @@ -131,6 +131,17 @@ public: cvEmpty_.notify_one(); } } + std::vector GetAllElements() + { + std::unique_lock lock(mutex_); + std::vector elements; + std::queue tempQueue = que_; + while (!tempQueue.empty()) { + elements.push_back(tempQueue.front()); + tempQueue.pop(); + } + return std::move(elements); + } private: void ClearUnlocked() diff --git a/services/deferred_processing_service/include/base/task_manager/task_manager.h b/services/deferred_processing_service/include/base/task_manager/task_manager.h index 0e4eb2a4c..47acae39c 100644 --- a/services/deferred_processing_service/include/base/task_manager/task_manager.h +++ b/services/deferred_processing_service/include/base/task_manager/task_manager.h @@ -29,7 +29,7 @@ namespace CameraStandard { namespace DeferredProcessing { class TaskManager { public: - TaskManager(const std::string& name, uint32_t numThreads); + TaskManager(const std::string& name, uint32_t numThreads, bool serial); ~TaskManager(); void CreateDelayedTaskGroupIfNeed(); void BeginBackgroundTasks(); @@ -48,6 +48,7 @@ private: const std::string name_; const uint32_t numThreads_; + bool serial_ = true; std::unique_ptr pool_; std::unique_ptr taskRegistry_; TaskGroupHandle defaultTaskHandle_; diff --git a/services/deferred_processing_service/src/base/task_manager/task_manager.cpp b/services/deferred_processing_service/src/base/task_manager/task_manager.cpp index 7a31b4cc6..8f96052a2 100644 --- a/services/deferred_processing_service/src/base/task_manager/task_manager.cpp +++ b/services/deferred_processing_service/src/base/task_manager/task_manager.cpp @@ -21,9 +21,10 @@ namespace OHOS { namespace CameraStandard { namespace DeferredProcessing { -TaskManager::TaskManager(const std::string& name, uint32_t numThreads) +TaskManager::TaskManager(const std::string& name, uint32_t numThreads, bool serial) : name_(name), numThreads_(numThreads), + serial_(serial), pool_(nullptr), taskRegistry_(nullptr), defaultTaskHandle_(INVALID_TASK_GROUP_HANDLE), @@ -39,7 +40,7 @@ void TaskManager::Initialize() pool_ = ThreadPool::Create(name_, numThreads_); taskRegistry_ = std::make_unique(name_, pool_.get()); auto ret = RegisterTaskGroup("defaultTaskGroup", - [this](std::any param) { DoDefaultWorks(std::move(param)); }, true, false, defaultTaskHandle_); + [this](std::any param) { DoDefaultWorks(std::move(param)); }, serial_, false, defaultTaskHandle_); DP_INFO_LOG("register default task group: %d, handle: %d", ret, static_cast(defaultTaskHandle_)); (void)(ret); return; diff --git a/services/deferred_processing_service/src/base/task_manager/thread_pool.cpp b/services/deferred_processing_service/src/base/task_manager/thread_pool.cpp index c799eb5e7..9970f60fe 100644 --- a/services/deferred_processing_service/src/base/task_manager/thread_pool.cpp +++ b/services/deferred_processing_service/src/base/task_manager/thread_pool.cpp @@ -71,8 +71,10 @@ void ThreadPool::WorkerLoop(const std::string& threadName) { DP_DEBUG_LOG("(%s) entered.", threadName.c_str()); while (!isStopped_.load()) { + DP_DEBUG_LOG("(%s) task excute start entered.", threadName.c_str()); auto task = GetTask(); if (task) { + DP_DEBUG_LOG("(%s) task excuting entered.", threadName.c_str()); task(); } else { DP_DEBUG_LOG("empty task."); diff --git a/services/deferred_processing_service/src/deferred_processing_service.cpp b/services/deferred_processing_service/src/deferred_processing_service.cpp index 620c89791..2f056837b 100644 --- a/services/deferred_processing_service/src/deferred_processing_service.cpp +++ b/services/deferred_processing_service/src/deferred_processing_service.cpp @@ -91,7 +91,8 @@ TaskManager* DeferredProcessingService::GetPhotoTaskManager(int userId) if (photoTaskManagerMap_.count(userId) == 0) { constexpr uint32_t numThreads = 1; std::shared_ptr taskManager = - std::make_shared("PhotoProcTaskManager_userid_" + std::to_string(userId), numThreads); + std::make_shared("PhotoProcTaskManager_userid_" + std::to_string(userId), + numThreads, true); EventsMonitor::GetInstance().RegisterTaskManager(userId, taskManager.get()); photoTaskManagerMap_[userId] = taskManager; } diff --git a/services/etc/camera_service.cfg b/services/etc/camera_service.cfg index c05cb6172..94e9f49e5 100644 --- a/services/etc/camera_service.cfg +++ b/services/etc/camera_service.cfg @@ -9,7 +9,10 @@ "ohos.permission.GET_SENSITIVE_PERMISSIONS", "ohos.permission.PERMISSION_USED_STATS", "ohos.permission.ACCESS_SERVICE_DM", - "ohos.permission.GET_BUNDLE_INFO_PRIVILEGED" + "ohos.permission.GET_BUNDLE_INFO_PRIVILEGED", + "ohos.permission.READ_IMAGEVIDEO", + "ohos.permission.WRITE_IMAGEVIDEO", + "ohos.permission.MICROPHONE" ], "permission_acls" : ["ohos.permission.GET_SENSITIVE_PERMISSIONS"] } -- Gitee