From d583a3ae6acaba9816c3c61e3787280afe15d43d Mon Sep 17 00:00:00 2001 From: linyuanpeng Date: Thu, 23 Nov 2023 15:52:06 +0800 Subject: [PATCH] =?UTF-8?q?=E7=A9=BA=E9=97=B4=E9=9F=B3=E9=A2=91IMU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: linyuanpeng --- bundle.json | 3 +- frameworks/native/audioeffect/BUILD.gn | 1 + .../include/audio_effect_chain_manager.h | 30 +++- .../src/audio_effect_chain_manager.cpp | 153 +++++++++++++++++- .../native/audiocommon/include/audio_effect.h | 1 + services/audio_service/BUILD.gn | 1 + 6 files changed, 184 insertions(+), 5 deletions(-) diff --git a/bundle.json b/bundle.json index 583a43d2a7..0128195280 100644 --- a/bundle.json +++ b/bundle.json @@ -62,7 +62,8 @@ "power_manager", "resource_schedule_service", "safwk", - "samgr" + "samgr", + "sensor" ], "third_party": [ "bounds_checking_function", diff --git a/frameworks/native/audioeffect/BUILD.gn b/frameworks/native/audioeffect/BUILD.gn index 8a0e1806ca..7226c9c12a 100644 --- a/frameworks/native/audioeffect/BUILD.gn +++ b/frameworks/native/audioeffect/BUILD.gn @@ -56,6 +56,7 @@ ohos_shared_library("audio_effect") { "c_utils:utils", "hilog:libhilog", "ipc:ipc_single", + "sensor:sensor_interface_native", ] version_script = "../../../audio_framework.versionscript" diff --git a/frameworks/native/audioeffect/include/audio_effect_chain_manager.h b/frameworks/native/audioeffect/include/audio_effect_chain_manager.h index 869e89fcac..bcc7ce31b3 100644 --- a/frameworks/native/audioeffect/include/audio_effect_chain_manager.h +++ b/frameworks/native/audioeffect/include/audio_effect_chain_manager.h @@ -31,6 +31,7 @@ #include "audio_effect_chain_adapter.h" #include "audio_effect.h" +#include "sensor_agent.h" namespace OHOS { namespace AudioStandard { @@ -44,6 +45,9 @@ const uint32_t FACTOR_TWO = 2; const uint32_t BASE_TEN = 10; const std::string DEFAULT_DEVICE_SINK = "Speaker"; const uint32_t SIZE_OF_SPATIALIZATION_STATE = 2; +const uint32_t NONE_SPATIALIZER_ENGINE = 0; +const uint32_t ARM_SPATIALIZER_ENGINE = 1; +const uint32_t DSP_SPATIALIZER_ENGINE = 2; const std::vector HVS_SUPPORTED_CHANNELLAYOUTS { CH_LAYOUT_STEREO, @@ -56,9 +60,28 @@ const std::vector HVS_SUPPORTED_CHANNELLAYOUTS { CH_LAYOUT_9POINT1POINT4, CH_LAYOUT_9POINT1POINT6 }; + +class HeadTracker { +public: + HeadTracker(); + ~HeadTracker(); + int32_t SensorInit(); + int32_t SensorSetConfig(int32_t spatializerEngineState); + int32_t SensorActive(); + int32_t SensorDeactive(); + HeadPostureData GetHeadPostureData(); + void SetHeadPostureData(HeadPostureData headPostureData); +private: + static void HeadPostureDataProcCb(SensorEvent *event); + static HeadPostureData headPostureData_; + SensorUser sensorUser_; + int64_t sensorSamplingInterval_ = 30000000; // 30000000 ns = 30 ms + static std::mutex headTrackerMutex_; +}; + class AudioEffectChain { public: - AudioEffectChain(std::string scene); + AudioEffectChain(std::string scene, std::shared_ptr headTracker); ~AudioEffectChain(); std::string GetEffectMode(); void SetEffectMode(std::string mode); @@ -76,6 +99,7 @@ public: void StoreOldEffectChainInfo(std::string &sceneMode, AudioEffectConfig &ioBufferConfig); AudioEffectConfig GetIoBufferConfig(); void InitEffectChain(); + void SetHeadTrackingDisabled(); private: std::mutex reloadMutex; std::string sceneType; @@ -85,6 +109,7 @@ private: AudioEffectConfig ioBufferConfig; AudioBuffer audioBufIn; AudioBuffer audioBufOut; + std::shared_ptr headTracker_; }; class AudioEffectChainManager { @@ -112,6 +137,7 @@ public: int32_t InitAudioEffectChainDynamic(std::string sceneType); int32_t UpdateSpatializationState(std::vector spatializationState); private: + void UpdateSensorState(); std::map EffectToLibraryEntryMap_; std::map EffectToLibraryNameMap_; std::map> EffectChainToEffectsMap_; @@ -126,6 +152,8 @@ private: std::mutex dynamicMutex_; bool spatializatonEnabled_ = true; bool headTrackingEnabled_ = false; + std::shared_ptr headTracker_; + bool offloadEnabled_ = false; }; } // namespace AudioStandard } // namespace OHOS diff --git a/frameworks/native/audioeffect/src/audio_effect_chain_manager.cpp b/frameworks/native/audioeffect/src/audio_effect_chain_manager.cpp index 88ef66d27b..3d60b30a22 100644 --- a/frameworks/native/audioeffect/src/audio_effect_chain_manager.cpp +++ b/frameworks/native/audioeffect/src/audio_effect_chain_manager.cpp @@ -179,7 +179,7 @@ int32_t EffectChainManagerInitCb(const char *sceneType) namespace OHOS { namespace AudioStandard { -AudioEffectChain::AudioEffectChain(std::string scene) +AudioEffectChain::AudioEffectChain(std::string scene, std::shared_ptr headTracker) { sceneType = scene; effectMode = AUDIO_SUPPORTED_SCENE_MODES.find(EFFECT_DEFAULT)->second; @@ -193,6 +193,7 @@ AudioEffectChain::AudioEffectChain(std::string scene) ioBufferConfig.outputCfg.channels = DEFAULT_NUM_CHANNEL; ioBufferConfig.outputCfg.format = DATA_FORMAT_F32; ioBufferConfig.outputCfg.channelLayout = DEFAULT_NUM_CHANNELLAYOUT; + headTracker_ = headTracker; } AudioEffectChain::~AudioEffectChain() @@ -331,12 +332,17 @@ void AudioEffectChain::ApplyEffectChain(float *bufIn, float *bufOut, uint32_t fr return; } + int32_t replyData = 0; + auto imuData = headTracker_->GetHeadPostureData(); + AudioEffectTransInfo cmdInfo = {sizeof(HeadPostureData), &imuData}; + AudioEffectTransInfo replyInfo = {sizeof(int32_t), &replyData}; audioBufIn.frameLength = frameLen; audioBufOut.frameLength = frameLen; int32_t count = 0; { std::lock_guard lock(reloadMutex); for (AudioEffectHandle handle: standByEffectHandles) { + (*handle)->command(handle, EFFECT_CMD_SET_IMU, &cmdInfo, &replyInfo); if (count % FACTOR_TWO == 0) { audioBufIn.raw = bufIn; audioBufOut.raw = bufOut; @@ -415,6 +421,27 @@ void AudioEffectChain::StoreOldEffectChainInfo(std::string &sceneMode, AudioEffe return; } +void AudioEffectChain::SetHeadTrackingDisabled() +{ + if (IsEmptyEffectHandles()) { + return; + } + + { + std::lock_guard lock(reloadMutex); + for (AudioEffectHandle handle: standByEffectHandles) { + int32_t replyData = 0; + HeadPostureData imuDataDisabled = {1, 1.0, 0.0, 0.0, 0.0}; + AudioEffectTransInfo cmdInfo = {sizeof(HeadPostureData), &imuDataDisabled}; + AudioEffectTransInfo replyInfo = {sizeof(int32_t), &replyData}; + int32_t ret = (*handle)->command(handle, EFFECT_CMD_SET_IMU, &cmdInfo, &replyInfo); + if (ret != 0) { + AUDIO_WARNING_LOG("SetHeadTrackingDisabled failed"); + } + } + } +} + int32_t FindEffectLib(const std::string &effect, const std::vector> &effectLibraryList, AudioEffectLibEntry **libEntry, std::string &libName) @@ -485,6 +512,8 @@ AudioEffectChainManager::AudioEffectChainManager() deviceType_ = DEVICE_TYPE_SPEAKER; deviceSink_ = DEFAULT_DEVICE_SINK; isInitialized_ = false; + headTracker_ = std::make_shared(); + headTracker_->SensorInit(); } AudioEffectChainManager::~AudioEffectChainManager() @@ -541,7 +570,7 @@ int32_t AudioEffectChainManager::SetOutputDeviceSink(int32_t device, std::string sceneMode = AUDIO_SUPPORTED_SCENE_MODES.find(EFFECT_DEFAULT)->second; } SceneTypeToEffectChainMap_.erase(key); - audioEffectChain = new AudioEffectChain(sceneType); + audioEffectChain = new AudioEffectChain(sceneType, headTracker_); if (SceneTypeToEffectChainMap_.count(sceneTypeAndDeviceKey)) { SceneTypeToEffectChainMap_[sceneTypeAndDeviceKey] = audioEffectChain; } else { @@ -663,7 +692,7 @@ int32_t AudioEffectChainManager::CreateAudioEffectChainDynamic(std::string scene SceneTypeToEffectChainCountMap_[sceneTypeAndDeviceKey]++; return SUCCESS; } else { - audioEffectChain = new AudioEffectChain(sceneType); + audioEffectChain = new AudioEffectChain(sceneType, headTracker_); SceneTypeToEffectChainMap_.insert(std::make_pair(sceneTypeAndDeviceKey, audioEffectChain)); if (!SceneTypeToEffectChainCountMap_.count(sceneTypeAndDeviceKey)) { SceneTypeToEffectChainCountMap_.insert(std::make_pair(sceneTypeAndDeviceKey, 1)); @@ -875,9 +904,127 @@ int32_t AudioEffectChainManager::UpdateSpatializationState(std::vector spa } if (headTrackingEnabled_ != spatializationState[1]) { headTrackingEnabled_ = spatializationState[1]; + UpdateSensorState(); } return SUCCESS; } +void AudioEffectChainManager::UpdateSensorState() +{ + if (headTrackingEnabled_) { + if (offloadEnabled_) { + headTracker_->SensorSetConfig(DSP_SPATIALIZER_ENGINE); + } else { + headTracker_->SensorSetConfig(ARM_SPATIALIZER_ENGINE); + } + headTracker_->SensorActive(); + return; + } + + if (headTracker_->SensorDeactive() != 0) { + AUDIO_ERR_LOG("SensorDeactive failed"); + } + HeadPostureData headPostureData = {1, 1.0, 0.0, 0.0, 0.0}; + headTracker_->SetHeadPostureData(headPostureData); + for (auto it = SceneTypeToEffectChainMap_.begin(); it != SceneTypeToEffectChainMap_.end(); ++it) { + auto *audioEffectChain = it->second; + if (audioEffectChain == nullptr) { + continue; + } + audioEffectChain->SetHeadTrackingDisabled(); + } +} + +HeadPostureData HeadTracker::headPostureData_ = {1, 1.0, 0.0, 0.0, 0.0}; +std::mutex HeadTracker::headTrackerMutex_; + +void HeadTracker::HeadPostureDataProcCb(SensorEvent *event) +{ + std::lock_guard lock(headTrackerMutex_); + + if (event == nullptr) { + AUDIO_ERR_LOG("Audio HeadTracker Sensor event is nullptr!"); + return; + } + + if (event[0].data == nullptr) { + AUDIO_ERR_LOG("Audio HeadTracker Sensor event[0].data is nullptr!"); + return; + } + + if (event[0].dataLen < sizeof(HeadPostureData)) { + AUDIO_ERR_LOG("Event dataLen less than head posture data size, event.dataLen:%{public}u", event[0].dataLen); + return; + } + HeadPostureData *headPostureDataTmp = reinterpret_cast(event[0].data); + headPostureData_.order = headPostureDataTmp->order; + headPostureData_.w = headPostureDataTmp->w; + headPostureData_.x = headPostureDataTmp->x; + headPostureData_.y = headPostureDataTmp->y; + headPostureData_.z = headPostureDataTmp->z; +} + +HeadTracker::HeadTracker() +{ + AUDIO_INFO_LOG("HeadTracker ctor!"); +} + +HeadTracker::~HeadTracker() +{ + UnsubscribeSensor(SENSOR_TYPE_ID_HEADPOSTURE, &sensorUser_); +} + +int32_t HeadTracker::SensorInit() +{ + sensorUser_.callback = HeadPostureDataProcCb; + return SubscribeSensor(SENSOR_TYPE_ID_HEADPOSTURE, &sensorUser_); +} + +int32_t HeadTracker::SensorSetConfig(int32_t spatializerEngineState) +{ + int32_t ret; + switch (spatializerEngineState) { + case NONE_SPATIALIZER_ENGINE: + AUDIO_ERR_LOG("system has no spatializer engine!"); + ret = ERROR; + break; + case ARM_SPATIALIZER_ENGINE: + AUDIO_DEBUG_LOG("system uses arm spatializer engine!"); + ret = SetBatch(SENSOR_TYPE_ID_HEADPOSTURE, &sensorUser_, + sensorSamplingInterval_, sensorSamplingInterval_); + break; + case DSP_SPATIALIZER_ENGINE: + AUDIO_DEBUG_LOG("system uses dsp spatializer engine!"); + // 2 * sampling for DSP + ret = SetBatch(SENSOR_TYPE_ID_HEADPOSTURE, &sensorUser_, + sensorSamplingInterval_, sensorSamplingInterval_ * 2); + break; + default: + AUDIO_ERR_LOG("spatializerEngineState error!"); + ret = ERROR; + break; + } + return ret; +} + +int32_t HeadTracker::SensorActive() +{ + return ActivateSensor(SENSOR_TYPE_ID_HEADPOSTURE, &sensorUser_); +} + +int32_t HeadTracker::SensorDeactive() +{ + return DeactivateSensor(SENSOR_TYPE_ID_HEADPOSTURE, &sensorUser_); +} +HeadPostureData HeadTracker::GetHeadPostureData() +{ + std::lock_guard lock(headTrackerMutex_); + return headPostureData_; +} +void HeadTracker::SetHeadPostureData(HeadPostureData headPostureData) +{ + std::lock_guard lock(headTrackerMutex_); + headPostureData_ = headPostureData; } } +} \ No newline at end of file diff --git a/interfaces/inner_api/native/audiocommon/include/audio_effect.h b/interfaces/inner_api/native/audiocommon/include/audio_effect.h index ac75993a35..6ef519dea6 100644 --- a/interfaces/inner_api/native/audiocommon/include/audio_effect.h +++ b/interfaces/inner_api/native/audiocommon/include/audio_effect.h @@ -179,6 +179,7 @@ enum AudioEffectCommandCode { EFFECT_CMD_SET_PARAM = 4, EFFECT_CMD_GET_PARAM = 5, EFFECT_CMD_GET_CONFIG = 6, + EFFECT_CMD_SET_IMU = 7 }; enum AudioEffectParamSetCode { diff --git a/services/audio_service/BUILD.gn b/services/audio_service/BUILD.gn index 6b3113d0f3..5086d44c02 100644 --- a/services/audio_service/BUILD.gn +++ b/services/audio_service/BUILD.gn @@ -305,6 +305,7 @@ ohos_shared_library("audio_service") { "ipc:ipc_single", "safwk:system_ability_fwk", "samgr:samgr_proxy", + "sensor:sensor_interface_native", ] if (ressche_enable == true) { -- Gitee