diff --git a/static_core/irtoc/scripts/common.irt b/static_core/irtoc/scripts/common.irt index 3fc4376ef978302e5034efec78b1e4243f04f5a2..c661816a637ce0b7a713a23a026bb95b754eca96 100644 --- a/static_core/irtoc/scripts/common.irt +++ b/static_core/irtoc/scripts/common.irt @@ -127,6 +127,7 @@ module Constants THREAD_FRAME_OFFSET = "cross_values::GetManagedThreadFrameOffset(GetArch())" THREAD_EXCEPTION_OFFSET = "cross_values::GetManagedThreadExceptionOffset(GetArch())" THREAD_INTERPRETER_CACHE_OFFSET = "cross_values::GetManagedThreadInterpreterCacheOffset(GetArch())" + THREAD_FLATTENED_STRING_CACHE_OFFSET = "cross_values::GetManagedThreadFlattenedStringCacheOffset(GetArch())" THREAD_FLAG_OFFSET = "cross_values::GetManagedThreadFlagOffset(GetArch())" THREAD_VM_OFFSET = "cross_values::GetThreadVmOffset(GetArch())" MARK_WORD_OFFSET = "cross_values::GetObjectHeaderMarkWordOffset(GetArch())" diff --git a/static_core/plugins/ets/runtime/ets_coroutine.cpp b/static_core/plugins/ets/runtime/ets_coroutine.cpp index ad8b3bd08779dec8ab4805e3f5c6ebf78721815c..109f93ab0ef6c54e92254ea5e40bbb53aa65de88 100644 --- a/static_core/plugins/ets/runtime/ets_coroutine.cpp +++ b/static_core/plugins/ets/runtime/ets_coroutine.cpp @@ -259,6 +259,25 @@ void EtsCoroutine::OnHostWorkerChanged() auto *worker = GetWorker(); auto *ptr = worker->GetLocalStorage().Get(); GetLocalStorage().Set(ptr); + + if (GetType() == Coroutine::Type::MUTATOR) { + // update the string cache pointer + auto *curCoro = EtsCoroutine::GetCurrent(); + ASSERT(curCoro != nullptr); + auto setStringCachePtr = [this, worker]() { + auto *cache = worker->GetLocalStorage().Get(); + SetFlattenedStringCache(cache); + }; + // We need to put the current coro into the managed state to be GC-safe, because we manipulate a raw + // ObjectHeader* + if (ManagedThread::IsManagedScope()) { + setStringCachePtr(); + } else { + // maybe we will find a more performant solution in future... + ScopedManagedCodeThread s(curCoro); + setStringCachePtr(); + } + } } void EtsCoroutine::OnContextSwitchedTo() diff --git a/static_core/plugins/ets/runtime/ets_vm.cpp b/static_core/plugins/ets/runtime/ets_vm.cpp index ab1dff4cbff7ba3a0a8bea2be9f7f9cbd0de8a52..02d2b107c3168bd9eb7af3d6c51c4add28902142 100644 --- a/static_core/plugins/ets/runtime/ets_vm.cpp +++ b/static_core/plugins/ets/runtime/ets_vm.cpp @@ -691,6 +691,12 @@ void PandaEtsVM::VisitVmRoots(const GCRootVisitor &visitor) } return true; }); + // NOTE(konstanting): worth moving to gc_root.cpp with a separate root type. Requires the ManagedCpu introduction. + GetCoroutineManager()->EnumerateWorkers([visitor](CoroutineWorker *worker) { + // apply visitor to worker + worker->VisitGCRoots(visitor); + return true; + }); if (LIKELY(Runtime::GetOptions().IsUseStringCaches())) { visitor(mem::GCRoot(mem::RootType::ROOT_VM, doubleToStringCache_->GetCoreType())); visitor(mem::GCRoot(mem::RootType::ROOT_VM, floatToStringCache_->GetCoreType())); @@ -766,6 +772,12 @@ void PandaEtsVM::UpdateVmRefs(const GCRootUpdater &gcRootUpdater) UpdateManagedEntrypointArgRefs(coroutine, gcRootUpdater); return true; }); + // NOTE(konstanting): worth moving to gc_root.cpp with a separate root type. Requires the ManagedCpu introduction. + GetCoroutineManager()->EnumerateWorkers([gcRootUpdater](CoroutineWorker *worker) { + // apply updater to worker + worker->UpdateGCRoots(gcRootUpdater); + return true; + }); objStateTable_->EnumerateObjectStates([&gcRootUpdater](EtsObjectStateInfo *info) { auto *obj = info->GetEtsObject()->GetCoreType(); diff --git a/static_core/runtime/asm_defines/asm_defines.def b/static_core/runtime/asm_defines/asm_defines.def index 92e6a6f76311f13dc9958f1b18536310b4d6412e..0426b735029eebd085935638ab68851498f2cd3b 100644 --- a/static_core/runtime/asm_defines/asm_defines.def +++ b/static_core/runtime/asm_defines/asm_defines.def @@ -90,6 +90,7 @@ DEFINE_VALUE(MANAGED_THREAD_LANGUAGE_EXTENSION_DATA_OFFSET, ManagedThread::GetLa DEFINE_VALUE(MANAGED_THREAD_INTERNAL_ID_OFFSET, ManagedThread::GetInternalIdOffset()) DEFINE_VALUE(MANAGED_THREAD_RUNTIME_CALL_ENABLED_OFFSET, ManagedThread::GetRuntimeCallEnabledOffset()) DEFINE_VALUE(MANAGED_THREAD_INTERPRETER_CACHE_OFFSET, ManagedThread::GetInterpreterCacheOffset()) +DEFINE_VALUE(MANAGED_THREAD_FLATTENED_STRING_CACHE_OFFSET, ManagedThread::GetFlattenedStringCacheOffset()) DEFINE_VALUE(MT_MANAGED_THREAD_LOCKED_OBJECT_CAPACITY_OFFSET, MTManagedThread::GetLockedObjectCapacityOffset()) DEFINE_VALUE(MT_MANAGED_THREAD_LOCKED_OBJECT_SIZE_OFFSET, MTManagedThread::GetLockedObjectSizeOffset()) diff --git a/static_core/runtime/coroutines/coroutine_manager.h b/static_core/runtime/coroutines/coroutine_manager.h index fa83e207967ad4bb2101a0a5fa035b2fed48f40e..87d1d8e545a87830ef54189fc7cb2ce1ba7619fd 100644 --- a/static_core/runtime/coroutines/coroutine_manager.h +++ b/static_core/runtime/coroutines/coroutine_manager.h @@ -111,6 +111,7 @@ public: std::optional &&epInfo, Coroutine::Type type, CoroutinePriority priority); using NativeEntrypointFunc = Coroutine::NativeEntrypointInfo::NativeEntrypointFunc; + using EnumerateWorkerCallback = std::function; NO_COPY_SEMANTIC(CoroutineManager); NO_MOVE_SEMANTIC(CoroutineManager); @@ -248,6 +249,15 @@ public: { } + /** + * @brief enumerate workers and apply @param cb to them + * @return true if @param cb call was successful (returned true) for every worekr and false otherwise + */ + bool EnumerateWorkers(const EnumerateWorkerCallback &cb) const + { + return EnumerateWorkersImpl(cb); + } + virtual bool IsExclusiveWorkersLimitReached() const { return false; @@ -372,6 +382,9 @@ protected: /// Can be used in descendants to create custom coroutines manually CoroutineFactory GetCoroutineFactory(); + /// Worker enumerator, returns true iff cb call succeeds for every worker + virtual bool EnumerateWorkersImpl(const EnumerateWorkerCallback &cb) const = 0; + /// limit the number of IDs for performance reasons static constexpr size_t MAX_COROUTINE_ID = std::min(0xffffU, Coroutine::MAX_COROUTINE_ID); static constexpr size_t UNINITIALIZED_COROUTINE_ID = 0x0U; diff --git a/static_core/runtime/coroutines/coroutine_worker.cpp b/static_core/runtime/coroutines/coroutine_worker.cpp index 4c5eeafe13584421adb6c8b84672d481e34acc81..a126477afec903d484f97dee1857534a51133605 100644 --- a/static_core/runtime/coroutines/coroutine_worker.cpp +++ b/static_core/runtime/coroutines/coroutine_worker.cpp @@ -13,8 +13,9 @@ * limitations under the License. */ -#include "coroutines/coroutine_manager.h" -#include "coroutines/coroutine_worker.h" +#include "runtime/coroutines/coroutine_manager.h" +#include "runtime/coroutines/coroutine_worker.h" +#include "runtime/include/thread_scopes.h" namespace ark { @@ -33,4 +34,43 @@ void CoroutineWorker::OnCoroBecameActive(Coroutine *co) TriggerSchedulerExternally(co); } +void CoroutineWorker::OnBeforeScheduleLoopStart() +{ + CreateWorkerLocalObjects(); +} + +void CoroutineWorker::VisitGCRoots(const GCRootVisitor &visitor) +{ + auto *stringCache = GetLocalStorage().Get(); + // NOTE(konstanting): worker local objects deserve a special root type + visitor(mem::GCRoot(mem::RootType::ROOT_VM, stringCache)); +} + +void CoroutineWorker::UpdateGCRoots(const GCRootUpdater &gcRootUpdater) +{ + auto *stringCachePtr = GetLocalStorage().GetPtr(); + gcRootUpdater(stringCachePtr); +} + +void CoroutineWorker::CreateWorkerLocalObjects() +{ + // native part + // ... + + // managed part + auto *schLoopCoro = Coroutine::GetCurrent(); + ASSERT(schLoopCoro != nullptr); + ASSERT_NATIVE_CODE(); + ScopedManagedCodeThread s(schLoopCoro); + // create the string flattening cache + /* + * probably should be done like: + * auto* cache = StringFlatteningCache::Create(...); + */ + //Class *klass = nullptr; + //auto *cache = coretypes::Array::Create(klass, 100500); + //ASSERT(cache != nullptr); + //GetLocalStorage().Set(cache); +} + } // namespace ark diff --git a/static_core/runtime/coroutines/coroutine_worker.h b/static_core/runtime/coroutines/coroutine_worker.h index 9cfae6cf4443d5086017e13312b3afcc674d963a..0c98aca989bc1db7ec9fd163d2cf37af8df3f79d 100644 --- a/static_core/runtime/coroutines/coroutine_worker.h +++ b/static_core/runtime/coroutines/coroutine_worker.h @@ -39,7 +39,7 @@ enum class CoroutinePriority { /// Represents a coroutine worker, which can host multiple coroutines and schedule them. class CoroutineWorker { public: - enum class DataIdx { INTEROP_CTX_PTR, EXTERNAL_IFACES, LAST_ID }; + enum class DataIdx { INTEROP_CTX_PTR, EXTERNAL_IFACES, STRING_CACHE, LAST_ID }; using LocalStorage = StaticLocalStorage; using Id = int32_t; @@ -111,6 +111,17 @@ public: void TriggerSchedulerExternally(Coroutine *requester); + // GC stuff + virtual void VisitGCRoots(const GCRootVisitor &visitor); + virtual void UpdateGCRoots(const GCRootUpdater &gcRootUpdater); + +protected: + /// should be called right before the schedule loop is entered for the first time + void OnBeforeScheduleLoopStart(); + +private: + void CreateWorkerLocalObjects(); + private: Runtime *runtime_ = nullptr; PandaVM *vm_ = nullptr; diff --git a/static_core/runtime/coroutines/local_storage.h b/static_core/runtime/coroutines/local_storage.h index cddec614363fa54b056f30749ba56acec9b4c197..00e871f56442b06eca682048e89201ddfc148b9a 100644 --- a/static_core/runtime/coroutines/local_storage.h +++ b/static_core/runtime/coroutines/local_storage.h @@ -72,13 +72,30 @@ public: } template - T Get() + T Get() const { static_assert((ToIndex(IDX) < NUM_ENTRIES), "idx should be correct"); // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) return reinterpret_cast(entries_[ToIndex(IDX)].data.ptr); } + template + T *GetPtr() + { + static_assert((ToIndex(IDX) < NUM_ENTRIES), "idx should be correct"); + // NOLINTNEXTLINE(cppcoreguidelines-pro-type-union-access) + return reinterpret_cast(&(entries_[ToIndex(IDX)].data.ptr)); + } + + template + constexpr size_t GetOffset() const + { + static_assert((ToIndex(IDX) < NUM_ENTRIES), "idx should be correct"); + // NOTE(konstanting): TO BE IMPLEMENTED! + UNREACHABLE(); + return 0; + } + private: std::array entries_; }; diff --git a/static_core/runtime/coroutines/stackful_coroutine_manager.cpp b/static_core/runtime/coroutines/stackful_coroutine_manager.cpp index 78f5df0241ea05f6a44fb6e616d3b08e6cbdf74a..ed3f7777a583b806190e6deffb434b619ac83c91 100644 --- a/static_core/runtime/coroutines/stackful_coroutine_manager.cpp +++ b/static_core/runtime/coroutines/stackful_coroutine_manager.cpp @@ -539,6 +539,17 @@ bool StackfulCoroutineManager::EnumerateThreadsImpl(const ThreadManager::Callbac return true; } +bool StackfulCoroutineManager::EnumerateWorkersImpl(const EnumerateWorkerCallback &cb) const +{ + os::memory::LockHolder lock(workersLock_); + for (auto *w : workers_) { + if (!cb(w)) { + return false; + } + } + return true; +} + void StackfulCoroutineManager::SuspendAllThreads() { os::memory::LockHolder lock(coroListLock_); diff --git a/static_core/runtime/coroutines/stackful_coroutine_manager.h b/static_core/runtime/coroutines/stackful_coroutine_manager.h index 97370c47e279671e7571803a7ee161dceca4b770..db8af86b2dbe7b785e16a855c519a1706ec762d2 100644 --- a/static_core/runtime/coroutines/stackful_coroutine_manager.h +++ b/static_core/runtime/coroutines/stackful_coroutine_manager.h @@ -145,6 +145,8 @@ protected: bool EnumerateThreadsImpl(const ThreadManager::Callback &cb, unsigned int incMask, unsigned int xorMask) const override; + bool EnumerateWorkersImpl(const EnumerateWorkerCallback &cb) const override; + CoroutineContext *CreateCoroutineContext(bool coroHasEntrypoint) override; void DeleteCoroutineContext(CoroutineContext *ctx) override; diff --git a/static_core/runtime/coroutines/stackful_coroutine_worker.cpp b/static_core/runtime/coroutines/stackful_coroutine_worker.cpp index bcc7671d1f546162def5ab69471c59d9c1a2381e..db01960fe4d7fdb571da81b832616bfa4a594e38 100644 --- a/static_core/runtime/coroutines/stackful_coroutine_worker.cpp +++ b/static_core/runtime/coroutines/stackful_coroutine_worker.cpp @@ -64,17 +64,17 @@ void StackfulCoroutineWorker::AddCreatedCoroutineAndSwitchToIt(Coroutine *newCor { // precondition: called within the current worker, no cross-worker calls allowed ASSERT(GetCurrentContext()->GetWorker() == this); + RegisterIncomingActiveCoroutine(newCoro); + // suspend current coro... auto *coro = Coroutine::GetCurrent(); ScopedNativeCodeThread n(coro); coro->RequestSuspend(false); - - newCoro->LinkToExternalHolder(IsMainWorker() || InExclusiveMode()); + // ..and resume the new one auto *currentCtx = GetCurrentContext(); auto *nextCtx = newCoro->GetContext(); nextCtx->RequestResume(); Coroutine::SetCurrent(newCoro); - RegisterIncomingActiveCoroutine(newCoro); SwitchCoroutineContext(currentCtx, nextCtx); @@ -282,6 +282,8 @@ void StackfulCoroutineWorker::ScheduleLoop() void StackfulCoroutineWorker::ScheduleLoopBody() { + OnBeforeScheduleLoopStart(); + // run the loop while (IsActive()) { RequestScheduleImpl(); os::memory::LockHolder lkRunnables(runnablesLock_); diff --git a/static_core/runtime/coroutines/stackful_coroutine_worker.h b/static_core/runtime/coroutines/stackful_coroutine_worker.h index 4e66c4fa9720665e9e045f5fa5333a24a5b3cb96..24effa925d49dcfbb3b8e721281d4b0be794a52a 100644 --- a/static_core/runtime/coroutines/stackful_coroutine_worker.h +++ b/static_core/runtime/coroutines/stackful_coroutine_worker.h @@ -255,6 +255,7 @@ private: bool IsPotentiallyBlocked(); void MigrateCoroutinesImpl(StackfulCoroutineWorker *to, size_t migrateCount) REQUIRES(runnablesLock_); + /* events */ /// called right before the coroutineContext is switched void OnBeforeContextSwitch(StackfulCoroutineContext *from, StackfulCoroutineContext *to); /// called right after the coroutineContext is switched (in case if no migration happened) diff --git a/static_core/runtime/coroutines/threaded_coroutine_manager.cpp b/static_core/runtime/coroutines/threaded_coroutine_manager.cpp index c3da01a1186c43cfac3952ed78cadf3e83576a98..8c1eb7334f7ea709f40698c70a8624bf0a9b3e1e 100644 --- a/static_core/runtime/coroutines/threaded_coroutine_manager.cpp +++ b/static_core/runtime/coroutines/threaded_coroutine_manager.cpp @@ -314,6 +314,17 @@ bool ThreadedCoroutineManager::EnumerateThreadsImpl(const ThreadManager::Callbac return true; } +bool ThreadedCoroutineManager::EnumerateWorkersImpl(const EnumerateWorkerCallback &cb) const +{ + os::memory::LockHolder lock(workersLock_); + for (auto *w : workers_) { + if (!cb(w)) { + return false; + } + } + return true; +} + void ThreadedCoroutineManager::SuspendAllThreads() { os::memory::LockHolder lList(coroListLock_); diff --git a/static_core/runtime/coroutines/threaded_coroutine_manager.h b/static_core/runtime/coroutines/threaded_coroutine_manager.h index 7dd1f1830d44a8cddad6ea370c35095912e0cbc7..cbdc1de84acd03efb86e6f621f1cd53770d03cdf 100644 --- a/static_core/runtime/coroutines/threaded_coroutine_manager.h +++ b/static_core/runtime/coroutines/threaded_coroutine_manager.h @@ -81,6 +81,8 @@ public: protected: bool EnumerateThreadsImpl(const ThreadManager::Callback &cb, unsigned int incMask, unsigned int xorMask) const override; + bool EnumerateWorkersImpl(const EnumerateWorkerCallback &cb) const override; + CoroutineContext *CreateCoroutineContext(bool coroHasEntrypoint) override; void DeleteCoroutineContext(CoroutineContext *ctx) override; diff --git a/static_core/runtime/include/managed_thread.h b/static_core/runtime/include/managed_thread.h index 2278a385abd79c3b2c5b997e74be4db7dfeb65d0..ca855594ff1bbb58c926356adc3553a7d4a85ce6 100644 --- a/static_core/runtime/include/managed_thread.h +++ b/static_core/runtime/include/managed_thread.h @@ -376,6 +376,11 @@ public: return MEMBER_OFFSET(ManagedThread, interpreterCache_); } + static constexpr uint32_t GetFlattenedStringCacheOffset() + { + return MEMBER_OFFSET(ManagedThread, flattenedStringCache_); + } + void *GetLanguageExtensionsData() const { return languageExtensionData_; @@ -437,6 +442,9 @@ public: PANDA_PUBLIC_API void SetCustomTLSData(const char *key, CustomTLSData *data); PANDA_PUBLIC_API bool EraseCustomTLSData(const char *key); + void SetFlattenedStringCache(ObjectHeader *cacheInstance); + ObjectHeader *GetFlattenedStringCache() const; + #if EVENT_METHOD_ENTER_ENABLED || EVENT_METHOD_EXIT_ENABLED uint32_t RecordMethodEnter() { @@ -705,6 +713,9 @@ private: PandaMap> customTlsCache_ GUARDED_BY(Locks::customTlsLock_); + // NOTE(konstanting): this is to be moved once we decouple Thread from ManagedThread + ObjectHeader *flattenedStringCache_ {nullptr}; + mem::GCG1BarrierSet::G1PostBarrierRingBufferType *g1PostBarrierRingBuffer_ {nullptr}; // Keep these here to speed up interpreter mem::BarrierType preBarrierType_ {mem::BarrierType::PRE_WRB_NONE}; diff --git a/static_core/runtime/thread.cpp b/static_core/runtime/thread.cpp index d144747093f67024ac1d8cd3125bc0a43ef36c6c..32e7fb2150ad2390f9efd74404e229795db6ecc5 100644 --- a/static_core/runtime/thread.cpp +++ b/static_core/runtime/thread.cpp @@ -678,9 +678,13 @@ void MTManagedThread::ProcessCreatedThread() void ManagedThread::UpdateGCRoots(const GCRootUpdater &gcRootUpdater) { - if ((exception_ != nullptr)) { + if (exception_ != nullptr) { gcRootUpdater(&exception_); } + if (flattenedStringCache_ != nullptr) { + // This is the cached pointer. We need to update it; visiting it as root is nor required. + gcRootUpdater(&flattenedStringCache_); + } for (auto **localObject : localObjects_) { gcRootUpdater(localObject); } @@ -849,6 +853,16 @@ bool ManagedThread::EraseCustomTLSData(const char *key) return customTlsCache_.erase(key) != 0; } +void ManagedThread::SetFlattenedStringCache(ObjectHeader *cacheInstance) +{ + flattenedStringCache_ = cacheInstance; +} + +ObjectHeader *ManagedThread::GetFlattenedStringCache() const +{ + return flattenedStringCache_; +} + LanguageContext ManagedThread::GetLanguageContext() { return Runtime::GetCurrent()->GetLanguageContext(threadLang_);