diff --git a/BUILD.gn b/BUILD.gn index 0f8e7fe84d7bd8d9216411a4bd284f2846d65196..56f82aac3289dc581a32ed515661cfd7a68d939e 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -325,6 +325,7 @@ ecma_source = [ "ecmascript/mem/c_string.cpp", "ecmascript/mem/chunk.cpp", "ecmascript/mem/compress_collector.cpp", + "ecmascript/mem/concurrent_sweeper.cpp", "ecmascript/mem/ecma_heap_manager.cpp", "ecmascript/mem/free_object_kind.cpp", "ecmascript/mem/free_object_list.cpp", @@ -342,6 +343,9 @@ ecma_source = [ "ecmascript/napi/jsnapi.cpp", "ecmascript/object_factory.cpp", "ecmascript/object_operator.cpp", + "ecmascript/platform/platform.cpp", + "ecmascript/platform/runner.cpp", + "ecmascript/platform/task_queue.cpp", "ecmascript/layout_info.cpp", "ecmascript/regexp/dyn_chunk.cpp", "ecmascript/regexp/regexp_executor.cpp", diff --git a/ecmascript/ecma_handle_scope-inl.h b/ecmascript/ecma_handle_scope-inl.h index 53bbceb2cd958ece6e8cb20012929b4d0df4b0dc..981007e76fe08586526b264514903d564888bf4c 100644 --- a/ecmascript/ecma_handle_scope-inl.h +++ b/ecmascript/ecma_handle_scope-inl.h @@ -24,10 +24,12 @@ inline EcmaHandleScope::EcmaHandleScope(JSThread *thread) : thread_(thread), prevNext_(thread->handleScopeStorageNext_), prevEnd_(thread->handleScopeStorageEnd_), prevHandleStorageIndex_(thread->currentHandleStorageIndex_) { + thread->HandleScopeCountAdd(); } inline EcmaHandleScope::~EcmaHandleScope() { + thread_->HandleScopeCountDec(); thread_->handleScopeStorageNext_ = prevNext_; if (thread_->handleScopeStorageEnd_ != prevEnd_) { thread_->handleScopeStorageEnd_ = prevEnd_; @@ -37,6 +39,8 @@ inline EcmaHandleScope::~EcmaHandleScope() uintptr_t EcmaHandleScope::NewHandle(JSThread *thread, JSTaggedType value) { + // Each Handle must be managed by HandleScope, otherwise it may cause Handle leakage. + ASSERT(thread->HandleScopeCount_ > 0); auto result = thread->handleScopeStorageNext_; if (result == thread->handleScopeStorageEnd_) { result = reinterpret_cast(thread->ExpandHandleStorage()); diff --git a/ecmascript/ecma_vm.cpp b/ecmascript/ecma_vm.cpp index 31bb8274e624a0672003da22504623473a4e518e..678c44c54942c84e6820a0dab0863139c2a6b6ce 100644 --- a/ecmascript/ecma_vm.cpp +++ b/ecmascript/ecma_vm.cpp @@ -35,6 +35,7 @@ #include "ecmascript/mem/heap.h" #include "ecmascript/tagged_dictionary.h" #include "ecmascript/object_factory.h" +#include "ecmascript/platform/platform.h" #include "ecmascript/regexp/regexp_parser_cache.h" #include "ecmascript/runtime_call_id.h" #include "ecmascript/runtime_trampolines.h" @@ -116,6 +117,7 @@ EcmaVM::EcmaVM(RuntimeOptions options) bool EcmaVM::Initialize() { trace::ScopedTrace scoped_trace("EcmaVM::Initialize"); + Platform::GetCurrentPlatform()->Initialize(); RuntimeTrampolines::InitializeRuntimeTrampolines(thread_); @@ -132,9 +134,9 @@ bool EcmaVM::Initialize() UNREACHABLE(); } + [[maybe_unused]] EcmaHandleScope scope(thread_); if (!snapshotDeserializeEnable_ || !VerifyFilePath(fileName_)) { LOG_ECMA(DEBUG) << "EcmaVM::Initialize run builtins"; - [[maybe_unused]] EcmaHandleScope scope(thread_); JSHandle dynClassClassHandle = factory_->NewEcmaDynClass(nullptr, JSHClass::SIZE, JSType::HCLASS); JSHClass *dynclass = reinterpret_cast(dynClassClassHandle.GetTaggedValue().GetTaggedObject()); @@ -231,6 +233,7 @@ bool EcmaVM::InitializeFinish() } EcmaVM::~EcmaVM() { + Platform::GetCurrentPlatform()->Destory(); vmInitialized_ = false; ClearNativeMethodsData(); diff --git a/ecmascript/js_thread.cpp b/ecmascript/js_thread.cpp index f2352867b9ef8d99579a91b8946df96b78b13597..305c69df346039ebf2d00ad32c242fd11f87bc17 100644 --- a/ecmascript/js_thread.cpp +++ b/ecmascript/js_thread.cpp @@ -52,6 +52,7 @@ JSThread::~JSThread() } handleStorageNodes_.clear(); currentHandleStorageIndex_ = -1; + handleScopeCount_ = 0; handleScopeStorageNext_ = handleScopeStorageEnd_ = nullptr; EcmaVM::Cast(GetVM())->GetChunk()->Delete(globalStorage_); @@ -190,6 +191,14 @@ uintptr_t *JSThread::ExpandHandleStorage() void JSThread::ShrinkHandleStorage(int prevIndex) { currentHandleStorageIndex_ = prevIndex; + int32_t lastIndex = handleStorageNodes_.size() - 1; + if (lastIndex > MIN_HANDLE_STORAGE_SIZE && currentHandleStorageIndex_ < MIN_HANDLE_STORAGE_SIZE) { + for (int i = MIN_HANDLE_STORAGE_SIZE; i < lastIndex; i++) { + auto node = handleStorageNodes_.back(); + delete node; + handleStorageNodes_.pop_back(); + } + } } void JSThread::NotifyStableArrayElementsGuardians(JSHandle receiver) diff --git a/ecmascript/js_thread.h b/ecmascript/js_thread.h index eef175a525e535a575b3759f799927c91aea043a..c771970edf28a1e7f0b13ee06211cee3c718429e 100644 --- a/ecmascript/js_thread.h +++ b/ecmascript/js_thread.h @@ -122,6 +122,16 @@ public: return currentHandleStorageIndex_; } + void HandleScopeCountAdd() + { + handleScopeCount_++; + } + + void HandleScopeCountDec() + { + handleScopeCount_--; + } + void SetException(JSTaggedValue exception); JSTaggedValue GetException() const @@ -192,6 +202,7 @@ private: static const uint32_t NODE_BLOCK_SIZE = 1U << NODE_BLOCK_SIZE_LOG2; static constexpr uint32_t MAX_RUNTIME_FUNCTIONS = kungfu::EXTERN_RUNTIME_STUB_MAXCOUNT - kungfu::EXTERNAL_RUNTIME_STUB_BEGIN - 1; + static constexpr int32_t MIN_HANDLE_STORAGE_SIZE = 2; EcmaGlobalStorage *globalStorage_ {nullptr}; @@ -207,6 +218,7 @@ private: JSTaggedType *handleScopeStorageEnd_ {nullptr}; std::vector *> handleStorageNodes_ {}; int32_t currentHandleStorageIndex_ {-1}; + int32_t handleScopeCount_ {0}; JSTaggedValue exception_ {JSTaggedValue::Hole()}; bool stableArrayElementsGuardians_ {true}; GlobalEnvConstants globalConst_; // Place-Holder diff --git a/ecmascript/mem/allocator-inl.h b/ecmascript/mem/allocator-inl.h index bf0d150fcbc0a28e2fc5243701bd585e0dee33a7..0117a740a935a854378ee542fd82986b9ebef590 100644 --- a/ecmascript/mem/allocator-inl.h +++ b/ecmascript/mem/allocator-inl.h @@ -19,6 +19,7 @@ #include #include "ecmascript/mem/allocator.h" +#include "ecmascript/mem/concurrent_sweeper.h" #include "ecmascript/mem/heap-inl.h" #include "ecmascript/mem/space.h" #include "ecmascript/free_object.h" @@ -67,13 +68,13 @@ uintptr_t BumpPointerAllocator::Allocate(size_t size) return result; } -FreeListAllocator::FreeListAllocator(const Space *space) +FreeListAllocator::FreeListAllocator(const Space *space) : heap_(space->GetHeap()), type_(space->GetSpaceType()) { freeList_ = std::make_unique(); - heap_ = space->GetHeap(); uintptr_t begin = space->GetCurrentRegion()->GetBegin(); size_t size = space->GetCurrentRegion()->GetSize(); FreeObject::Cast(begin)->SetAvailable(size); + FreeObject::Cast(begin)->SetNext(nullptr); freeList_->Free(begin, size); } @@ -96,18 +97,39 @@ uintptr_t FreeListAllocator::Allocate(size_t size) } FreeObject *object = freeList_->Allocator(size); if (LIKELY(object != nullptr && !object->IsEmpty())) { - FreeBumpPoint(); - bpAllocator_.Reset(object->GetBegin(), object->GetEnd()); - ret = bpAllocator_.Allocate(size); - if (ret != 0 && bpAllocator_.Available() > 0) { - FreeObject::FillFreeObject(heap_->GetEcmaVM(), bpAllocator_.GetTop(), bpAllocator_.Available()); + return Allocate(object, size); + } + + if (sweeping_) { + // Concurrent sweep maybe sweep same region + heap_->GetSweeper()->FillSweptRegion(type_); + object = freeList_->Allocator(size); + if (LIKELY(object != nullptr && !object->IsEmpty())) { + return Allocate(object, size); + } + + // Parallel + heap_->GetSweeper()->ParallelSweepSpace(type_); + object = freeList_->Allocator(size); + if (LIKELY(object != nullptr && !object->IsEmpty())) { + return Allocate(object, size); } - return ret; } return 0; } +uintptr_t FreeListAllocator::Allocate(FreeObject *object, size_t size) +{ + FreeBumpPoint(); + bpAllocator_.Reset(object->GetBegin(), object->GetEnd()); + auto ret = bpAllocator_.Allocate(size); + if (ret != 0 && bpAllocator_.Available() > 0) { + FreeObject::FillFreeObject(heap_->GetEcmaVM(), bpAllocator_.GetTop(), bpAllocator_.Available()); + } + return ret; +} + void FreeListAllocator::FreeBumpPoint() { auto begin = bpAllocator_.GetTop(); @@ -116,7 +138,7 @@ void FreeListAllocator::FreeBumpPoint() bpAllocator_.Reset(); } -void FreeListAllocator::Free(uintptr_t begin, uintptr_t end) +void FreeListAllocator::Free(uintptr_t begin, uintptr_t end, bool isAdd) { ASSERT(heap_ != nullptr); size_t size = end - begin; @@ -125,7 +147,7 @@ void FreeListAllocator::Free(uintptr_t begin, uintptr_t end) } FreeObject::FillFreeObject(heap_->GetEcmaVM(), begin, size); - freeList_->Free(begin, static_cast(end - begin)); + freeList_->Free(begin, size, isAdd); } void FreeListAllocator::RebuildFreeList() @@ -133,5 +155,18 @@ void FreeListAllocator::RebuildFreeList() bpAllocator_.Reset(); freeList_->Rebuild(); } + +void FreeListAllocator::FillFreeList(FreeObjectKind *kind) +{ + freeList_->AddKind(kind); +} + +size_t FreeListAllocator::GetAvailableSize() const +{ + if (sweeping_) { + heap_->GetSweeper()->WaitingTaskFinish(type_); + } + return freeList_->GetFreeObjectSize(); +} } // namespace panda::ecmascript #endif // ECMASCRIPT_MEM_ALLOCATOR_INL_H diff --git a/ecmascript/mem/allocator.h b/ecmascript/mem/allocator.h index 25b3d3d4cc8841bdc9475f92e0af6125b9978c59..0ea0cf0f086e2a42e943e59a71aea7e8d5c45c78 100644 --- a/ecmascript/mem/allocator.h +++ b/ecmascript/mem/allocator.h @@ -91,28 +91,37 @@ public: inline void AddFree(Region *region); inline void RebuildFreeList(); + inline void FillFreeList(FreeObjectKind *kind); void Swap(FreeListAllocator &other) { heap_ = other.heap_; bpAllocator_.Swap(other.bpAllocator_); freeList_.swap(other.freeList_); + type_ = other.type_; + sweeping_ = other.sweeping_; } inline void FreeBumpPoint(); - inline void Free(uintptr_t begin, uintptr_t end); + inline void Free(uintptr_t begin, uintptr_t end, bool isAdd = true); inline void SplitFreeObject(FreeObject *current, size_t allocateSize); - size_t GetAvailableSize() const + inline size_t GetAvailableSize() const; + + void SetSweeping(bool sweeping) { - return freeList_->GetFreeObjectSize(); + sweeping_ = sweeping; } private: + inline uintptr_t Allocate(FreeObject *object, size_t size); + BumpPointerAllocator bpAllocator_; std::unique_ptr freeList_; Heap *heap_{nullptr}; + MemSpaceType type_ = OLD_SPACE; + bool sweeping_ = false; }; } // namespace panda::ecmascript diff --git a/ecmascript/mem/compress_collector.cpp b/ecmascript/mem/compress_collector.cpp index db3759594c3e193fd0be8a585f48de9cd9037310..7ba6c97c94fafc05995577188647e99c4d84abca 100644 --- a/ecmascript/mem/compress_collector.cpp +++ b/ecmascript/mem/compress_collector.cpp @@ -13,9 +13,10 @@ * limitations under the License. */ +#include "ecmascript/mem/compress_collector.h" + #include "ecmascript/ecma_vm.h" #include "ecmascript/mem/clock_scope.h" -#include "ecmascript/mem/compress_collector.h" #include "ecmascript/mem/compress_gc_marker-inl.h" #include "ecmascript/mem/ecma_heap_manager.h" #include "ecmascript/mem/heap-inl.h" @@ -58,6 +59,7 @@ void CompressCollector::RunPhases() void CompressCollector::InitializePhase() { heap_->GetThreadPool()->WaitTaskFinish(); + heap_->GetSweeper()->EnsureAllTaskFinish(); auto compressSpace = const_cast(heap_->GetCompressSpace()); if (compressSpace->GetCommittedSize() == 0) { compressSpace->Initialize(); @@ -70,8 +72,6 @@ void CompressCollector::InitializePhase() FreeListAllocator compressAllocator(compressSpace); oldSpaceAllocator_.Swap(compressAllocator); fromSpaceAllocator_.Reset(fromSpace); - auto heapManager = heap_->GetHeapManager(); - nonMovableAllocator_.Swap(heapManager->GetNonMovableSpaceAllocator()); auto callback = [](Region *current) { // ensure mark bitmap @@ -116,7 +116,6 @@ void CompressCollector::FinishPhase() workList_->Finish(youngAndOldAliveSize_); auto heapManager = heap_->GetHeapManager(); heapManager->GetOldSpaceAllocator().Swap(oldSpaceAllocator_); - heapManager->GetNonMovableSpaceAllocator().Swap(nonMovableAllocator_); heapManager->GetNewSpaceAllocator().Swap(fromSpaceAllocator_); } @@ -230,60 +229,8 @@ void CompressCollector::SweepPhases() heap_->GetEcmaVM()->GetJSThread()->IterateWeakEcmaGlobalStorage(gcUpdateWeak); heap_->GetEcmaVM()->ProcessReferences(gcUpdateWeak); - SweepSpace(const_cast(heap_->GetNonMovableSpace()), nonMovableAllocator_); - SweepSpace(const_cast(heap_->GetHugeObjectSpace())); -} - -void CompressCollector::SweepSpace(HugeObjectSpace *space) -{ - Region *currentRegion = space->GetRegionList().GetFirst(); - while (currentRegion != nullptr) { - Region *next = currentRegion->GetNext(); - auto markBitmap = currentRegion->GetMarkBitmap(); - bool isMarked = false; - markBitmap->IterateOverMarkedChunks([&isMarked]([[maybe_unused]] void *mem) { isMarked = true; }); - if (!isMarked) { - space->GetRegionList().RemoveNode(currentRegion); - space->ClearAndFreeRegion(currentRegion); - } - currentRegion = next; - } -} - -void CompressCollector::SweepSpace(Space *space, FreeListAllocator &allocator) -{ - allocator.RebuildFreeList(); - space->EnumerateRegions([this, &allocator](Region *current) { - auto markBitmap = current->GetMarkBitmap(); - ASSERT(markBitmap != nullptr); - uintptr_t freeStart = current->GetBegin(); - markBitmap->IterateOverMarkedChunks([this, ¤t, &freeStart, &allocator](void *mem) { - ASSERT(current->InRange(ToUintPtr(mem))); - auto header = reinterpret_cast(mem); - auto klass = header->GetClass(); - JSType jsType = klass->GetObjectType(); - auto size = klass->SizeFromJSHClass(jsType, header); - size = AlignUp(size, static_cast(MemAlignment::MEM_ALIGN_OBJECT)); - - uintptr_t freeEnd = ToUintPtr(mem); - if (freeStart != freeEnd) { - FreeLiveRange(allocator, current, freeStart, freeEnd); - } - freeStart = freeEnd + size; - }); - uintptr_t freeEnd = current->GetEnd(); - if (freeStart != freeEnd) { - FreeLiveRange(allocator, current, freeStart, freeEnd); - } - }); -} - -void CompressCollector::FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, - uintptr_t freeEnd) -{ - allocator.Free(freeStart, freeEnd); - nonMoveSpaceFreeSize_ += freeEnd - freeStart; - heap_->ClearSlotsRange(current, freeStart, freeEnd); + // SweepSpace(const_cast(heap_->GetHugeObjectSpace())); + heap_->GetSweeper()->SweepPhases(true); } uintptr_t CompressCollector::AllocateOld(size_t size) diff --git a/ecmascript/mem/compress_collector.h b/ecmascript/mem/compress_collector.h index 535a3f47a05f1a9942922a8ec2cc5bb6e114a874..8cfb405ed12ae38dd66b517404d5e390ae5bcdb3 100644 --- a/ecmascript/mem/compress_collector.h +++ b/ecmascript/mem/compress_collector.h @@ -61,7 +61,6 @@ private: os::memory::Mutex mtx_; BumpPointerAllocator fromSpaceAllocator_{}; FreeListAllocator oldSpaceAllocator_{}; - FreeListAllocator nonMovableAllocator_{}; size_t youngAndOldAliveSize_ = 0; size_t nonMoveSpaceFreeSize_ = 0; diff --git a/ecmascript/mem/concurrent_sweeper.cpp b/ecmascript/mem/concurrent_sweeper.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b92d572204078ae88fd702c3d90fd6103e93a8f2 --- /dev/null +++ b/ecmascript/mem/concurrent_sweeper.cpp @@ -0,0 +1,253 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/mem/concurrent_sweeper.h" + +#include "ecmascript/js_hclass-inl.h" +#include "ecmascript/mem/allocator-inl.h" +#include "ecmascript/mem/ecma_heap_manager.h" +#include "ecmascript/mem/free_object_list.h" +#include "ecmascript/mem/heap.h" +#include "ecmascript/platform/platform.h" + +namespace panda::ecmascript { +ConcurrentSweeper::ConcurrentSweeper(Heap *heap, bool concurrentSweep) + : heap_(heap), concurrentSweep_(concurrentSweep) +{ +} + +void ConcurrentSweeper::SweepPhases(bool compressGC) +{ + if (concurrentSweep_) { + // Add all region to region list. Ensure all task finish + trace::ScopedTrace scoped_trace("ConcurrentSweeper::SweepPhases"); + if (!compressGC) { + heap_->GetOldSpace()->EnumerateRegions([this](Region *current) { AddRegion(OLD_SPACE, current); }); + } + heap_->GetNonMovableSpace()->EnumerateRegions([this](Region *current) { AddRegion(NON_MOVABLE, current); }); + + // Prepare + isSweeping_ = true; + startSpaceType_ = compressGC ? NON_MOVABLE : OLD_SPACE; + for (int type = startSpaceType_; type < FREE_LIST_NUM; type++) { + auto spaceType = static_cast(type); + FreeListAllocator &allocator = heap_->GetHeapManager()->GetFreeListAllocator(spaceType); + remainderTaskNum_[type] = FREE_LIST_NUM - startSpaceType_; + allocator.SetSweeping(true); + allocator.RebuildFreeList(); + } + + if (!compressGC) { + Platform::GetCurrentPlatform()->PostTask(std::make_unique(this, OLD_SPACE)); + } + Platform::GetCurrentPlatform()->PostTask(std::make_unique(this, NON_MOVABLE)); + } else { + if (!compressGC) { + SweepSpace(const_cast(heap_->GetOldSpace()), heap_->GetHeapManager()->GetOldSpaceAllocator()); + } + SweepSpace(const_cast(heap_->GetNonMovableSpace()), + heap_->GetHeapManager()->GetNonMovableSpaceAllocator()); + } + SweepHugeSpace(); +} + +void ConcurrentSweeper::SweepSpace(MemSpaceType type, bool isMain) +{ + trace::ScopedTrace scoped_trace("Sweeper::SweepSpace"); + FreeListAllocator &allocator = heap_->GetHeapManager()->GetFreeListAllocator(type); + Region *current = GetRegionSafe(type); + while (current != nullptr) { + FreeRegion(current, allocator, isMain); + // Main thread sweeping region is added; + if (!isMain) { + AddSweptRegionSafe(type, current); + } + current = GetRegionSafe(type); + } + + if (!isMain) { + os::memory::LockHolder holder(mutexs_[type]); + remainderTaskNum_[type]--; + if (remainderTaskNum_[type] == 0) { + cvs_[type].SignalAll(); + } + } +} + +void ConcurrentSweeper::SweepSpace(Space *space, FreeListAllocator &allocator) +{ + allocator.RebuildFreeList(); + space->EnumerateRegions([this, &allocator](Region *current) { FreeRegion(current, allocator); }); +} + +void ConcurrentSweeper::SweepHugeSpace() +{ + trace::ScopedTrace scoped_trace("SweepSpace HugeObject"); + HugeObjectSpace *space = const_cast(heap_->GetHugeObjectSpace()); + Region *currentRegion = space->GetRegionList().GetFirst(); + + while (currentRegion != nullptr) { + Region *next = currentRegion->GetNext(); + auto markBitmap = currentRegion->GetMarkBitmap(); + bool isMarked = false; + markBitmap->IterateOverMarkedChunks([&isMarked]([[maybe_unused]] void *mem) { isMarked = true; }); + if (!isMarked) { + space->GetRegionList().RemoveNode(currentRegion); + space->ClearAndFreeRegion(currentRegion); + } + currentRegion = next; + } +} + +void ConcurrentSweeper::FreeRegion(Region *current, FreeListAllocator &allocator, bool isMain) +{ + auto markBitmap = current->GetMarkBitmap(); + ASSERT(markBitmap != nullptr); + uintptr_t freeStart = current->GetBegin(); + markBitmap->IterateOverMarkedChunks([this, ¤t, &freeStart, &allocator, isMain](void *mem) { + ASSERT(current->InRange(ToUintPtr(mem))); + auto header = reinterpret_cast(mem); + auto klass = header->GetClass(); + JSType jsType = klass->GetObjectType(); + auto size = klass->SizeFromJSHClass(jsType, header); + size = AlignUp(size, static_cast(MemAlignment::MEM_ALIGN_OBJECT)); + + uintptr_t freeEnd = ToUintPtr(mem); + if (freeStart != freeEnd) { + FreeLiveRange(allocator, current, freeStart, freeEnd, isMain); + } + freeStart = freeEnd + size; + }); + uintptr_t freeEnd = current->GetEnd(); + if (freeStart != freeEnd) { + FreeLiveRange(allocator, current, freeStart, freeEnd, isMain); + } +} + +void ConcurrentSweeper::FillSweptRegion(MemSpaceType type) +{ + trace::ScopedTrace scoped_trace("Sweeper::FillSweptRegion"); + if (sweptList_[type].empty()) { + return; + } + FreeListAllocator &allocator = heap_->GetHeapManager()->GetFreeListAllocator(type); + Region *region = nullptr; + while ((region = GetSweptRegionSafe(type)) != nullptr) { + region->EnumerateKinds([&allocator](FreeObjectKind *kind) { + if (kind == nullptr || kind->Empty()) { + return; + } + allocator.FillFreeList(kind); + }); + } +} + +void ConcurrentSweeper::FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, + uintptr_t freeEnd, bool isMain) +{ + allocator.Free(freeStart, freeEnd, isMain); + heap_->ClearSlotsRange(current, freeStart, freeEnd); +} + +void ConcurrentSweeper::AddRegion(MemSpaceType type, Region *region) +{ + sweepingList_[type].emplace_back(region); +} + +Region *ConcurrentSweeper::GetRegionSafe(MemSpaceType type) +{ + os::memory::LockHolder holder(mutexs_[type]); + Region *region = nullptr; + if (!sweepingList_[type].empty()) { + region = sweepingList_[type].back(); + sweepingList_[type].pop_back(); + } + return region; +} + +void ConcurrentSweeper::AddSweptRegionSafe(MemSpaceType type, Region *region) +{ + os::memory::LockHolder holder(mutexs_[type]); + sweptList_[type].emplace_back(region); +} + +Region *ConcurrentSweeper::GetSweptRegionSafe(MemSpaceType type) +{ + os::memory::LockHolder holder(mutexs_[type]); + Region *region = nullptr; + if (!sweptList_[type].empty()) { + region = sweptList_[type].back(); + sweptList_[type].pop_back(); + } + return region; +} + +void ConcurrentSweeper::EnsureAllTaskFinish() +{ + if (!isSweeping_) { + return; + } + for (int i = startSpaceType_; i < FREE_LIST_NUM; i++) { + WaitingTaskFinish(static_cast(i)); + } + isSweeping_ = false; +} + +void ConcurrentSweeper::WaitingTaskFinish(MemSpaceType type) +{ + if (remainderTaskNum_[type] > 0) { + SweepSpace(type); + { + os::memory::LockHolder holder(mutexs_[type]); + while (remainderTaskNum_[type] > 0) { + cvs_[type].Wait(&mutexs_[type]); + } + } + } + FinishSweeping(type); +} + +void ConcurrentSweeper::ParallelSweepSpace(MemSpaceType type) +{ + if (remainderTaskNum_[type] > 0) { + SweepSpace(type); + } + + if (remainderTaskNum_[type] <= 0) { + FinishSweeping(type); + } else { + FillSweptRegion(type); + } +} + +void ConcurrentSweeper::FinishSweeping(MemSpaceType type) +{ + FillSweptRegion(type); + FreeListAllocator &allocator = heap_->GetHeapManager()->GetFreeListAllocator(type); + allocator.SetSweeping(false); + if (type == OLD_SPACE) { + heap_->RecomputeLimits(); + } +} + +bool ConcurrentSweeper::SweeperTask::Run() +{ + for (size_t i = 0; i < FREE_LIST_NUM - sweeper_->startSpaceType_; i++) { + auto type = static_cast((i + type_) % FREE_LIST_NUM); + sweeper_->SweepSpace(type, false); + } + return true; +} +} // namespace panda::ecmascript diff --git a/ecmascript/mem/concurrent_sweeper.h b/ecmascript/mem/concurrent_sweeper.h new file mode 100644 index 0000000000000000000000000000000000000000..59cb42c75328a572f354377eccb3fc064af940d6 --- /dev/null +++ b/ecmascript/mem/concurrent_sweeper.h @@ -0,0 +1,95 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PANDA_ECMASCRIPT_MEM_CONCURRENT_SWEEPER_H +#define PANDA_ECMASCRIPT_MEM_CONCURRENT_SWEEPER_H + +#include +#include + +#include "ecmascript/mem/space.h" +#include "ecmascript/platform/task.h" +#include "os/mutex.h" + +namespace panda::ecmascript { +class FreeListAllocator; + +class ConcurrentSweeper { +public: + ConcurrentSweeper(Heap *heap, bool concurrentSweep); + ~ConcurrentSweeper() = default; + + NO_COPY_SEMANTIC(ConcurrentSweeper); + NO_MOVE_SEMANTIC(ConcurrentSweeper); + + void SweepPhases(bool compressGC = false); + + void EnsureAllTaskFinish(); + // Ensure task finish + void WaitingTaskFinish(MemSpaceType type); + // Parallel sweep space, does not guarantee the end of the task + void ParallelSweepSpace(MemSpaceType type); + + void FillSweptRegion(MemSpaceType type); + + bool IsConcurrentSweepEnabled() + { + return concurrentSweep_; + } + +private: + class SweeperTask : public Task { + public: + SweeperTask(ConcurrentSweeper *sweeper, MemSpaceType type) : sweeper_(sweeper), type_(type){}; + ~SweeperTask() override = default; + bool Run() override; + + NO_COPY_SEMANTIC(SweeperTask); + NO_MOVE_SEMANTIC(SweeperTask); + + private: + ConcurrentSweeper *sweeper_; + MemSpaceType type_; + }; + + void SweepSpace(MemSpaceType type, bool isMain = true); + void SweepSpace(Space *space, FreeListAllocator &allocator); + void SweepHugeSpace(); + void FinishSweeping(MemSpaceType type); + + void FreeRegion(Region *current, FreeListAllocator &allocator, bool isMain = true); + void FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, uintptr_t freeEnd, + bool isMain); + + void AddRegion(MemSpaceType type, Region *region); + Region *GetRegionSafe(MemSpaceType type); + + void AddSweptRegionSafe(MemSpaceType type, Region *region); + Region *GetSweptRegionSafe(MemSpaceType type); + + std::array mutexs_; + std::array cvs_; + std::array remainderTaskNum_ = {0, 0}; + + std::array, FREE_LIST_NUM> sweepingList_; + std::array, FREE_LIST_NUM> sweptList_; + + Heap *heap_; + bool concurrentSweep_ = false; + bool isSweeping_ = false; + MemSpaceType startSpaceType_ = MemSpaceType::OLD_SPACE; +}; +} // namespace panda::ecmascript +#endif // PANDA_ECMASCRIPT_MEM_CONCURRENT_SWEEPER_H diff --git a/ecmascript/mem/ecma_heap_manager-inl.h b/ecmascript/mem/ecma_heap_manager-inl.h index 4b65d6c33767a955b2ca0a60e62c019c37e0d961..a981c130cf957ca92baa6865d9bf871e3d64ffd9 100644 --- a/ecmascript/mem/ecma_heap_manager-inl.h +++ b/ecmascript/mem/ecma_heap_manager-inl.h @@ -83,18 +83,18 @@ TaggedObject *EcmaHeapManager::AllocateNonMovableOrHugeObject(JSHClass *hclass, if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) { return AllocateHugeObject(hclass, size); } - auto object = reinterpret_cast(nonMovableAllocator_.Allocate(size)); + auto object = reinterpret_cast(freeListAllocator_[NON_MOVABLE].Allocate(size)); if (UNLIKELY(object == nullptr)) { if (heap_->CheckAndTriggerNonMovableGC()) { - object = reinterpret_cast(nonMovableAllocator_.Allocate(size)); + object = reinterpret_cast(freeListAllocator_[NON_MOVABLE].Allocate(size)); } if (UNLIKELY(object == nullptr)) { // hclass must be nonmovable - if (!heap_->FillNonMovableSpaceAndTryGC(&nonMovableAllocator_)) { + if (!heap_->FillNonMovableSpaceAndTryGC(&freeListAllocator_[NON_MOVABLE])) { LOG_ECMA_MEM(FATAL) << "OOM : extend failed"; UNREACHABLE(); } - object = reinterpret_cast(nonMovableAllocator_.Allocate(size)); + object = reinterpret_cast(freeListAllocator_[NON_MOVABLE].Allocate(size)); if (UNLIKELY(object == nullptr)) { heap_->ThrowOutOfMemoryError(size); UNREACHABLE(); @@ -140,18 +140,18 @@ TaggedObject *EcmaHeapManager::AllocateOldGenerationOrHugeObject(JSHClass *hclas if (size > MAX_REGULAR_HEAP_OBJECT_SIZE) { return AllocateHugeObject(hclass, size); } - auto object = reinterpret_cast(oldSpaceAllocator_.Allocate(size)); + auto object = reinterpret_cast(freeListAllocator_[OLD_SPACE].Allocate(size)); if (UNLIKELY(object == nullptr)) { if (heap_->CheckAndTriggerOldGC()) { - object = reinterpret_cast(oldSpaceAllocator_.Allocate(size)); + object = reinterpret_cast(freeListAllocator_[OLD_SPACE].Allocate(size)); } if (UNLIKELY(object == nullptr)) { // hclass must nonmovable - if (!heap_->FillOldSpaceAndTryGC(&oldSpaceAllocator_)) { + if (!heap_->FillOldSpaceAndTryGC(&freeListAllocator_[OLD_SPACE])) { LOG_ECMA_MEM(FATAL) << "OOM : extend failed"; UNREACHABLE(); } - object = reinterpret_cast(oldSpaceAllocator_.Allocate(size)); + object = reinterpret_cast(freeListAllocator_[OLD_SPACE].Allocate(size)); if (UNLIKELY(object == nullptr)) { heap_->ThrowOutOfMemoryError(size); UNREACHABLE(); diff --git a/ecmascript/mem/ecma_heap_manager.cpp b/ecmascript/mem/ecma_heap_manager.cpp index ea3d0b982696759e6cf968acb841bb36268aec9f..f4ee34190bef73fb06dd34200037d899ac02f4a3 100644 --- a/ecmascript/mem/ecma_heap_manager.cpp +++ b/ecmascript/mem/ecma_heap_manager.cpp @@ -20,8 +20,7 @@ namespace panda::ecmascript { EcmaHeapManager::EcmaHeapManager(Heap *heap) : heap_(heap), newSpaceAllocator_(heap->GetNewSpace()), - nonMovableAllocator_(heap->GetNonMovableSpace()), - oldSpaceAllocator_(heap->GetOldSpace()), + freeListAllocator_ {FreeListAllocator(heap->GetOldSpace()), FreeListAllocator(heap_->GetNonMovableSpace())}, machineCodeSpaceAllocator_(heap->GetMachineCodeSpace()) { ASSERT(heap != nullptr); diff --git a/ecmascript/mem/ecma_heap_manager.h b/ecmascript/mem/ecma_heap_manager.h index 6bfd120149acc41d73eeeccef3f94f125b9fcfb2..be0877c798e805284fbab6e7849b89e48b6a8dc8 100644 --- a/ecmascript/mem/ecma_heap_manager.h +++ b/ecmascript/mem/ecma_heap_manager.h @@ -48,9 +48,14 @@ public: return heap_; } + FreeListAllocator &GetFreeListAllocator(MemSpaceType type) + { + return freeListAllocator_[type]; + } + FreeListAllocator &GetOldSpaceAllocator() { - return oldSpaceAllocator_; + return freeListAllocator_[OLD_SPACE]; } BumpPointerAllocator &GetNewSpaceAllocator() @@ -60,7 +65,7 @@ public: FreeListAllocator &GetNonMovableSpaceAllocator() { - return nonMovableAllocator_; + return freeListAllocator_[NON_MOVABLE]; } const BumpPointerAllocator &GetSnapShotSpaceAllocator() const @@ -76,8 +81,7 @@ public: private: Heap *heap_{nullptr}; BumpPointerAllocator newSpaceAllocator_; - FreeListAllocator nonMovableAllocator_; - FreeListAllocator oldSpaceAllocator_; + std::array freeListAllocator_; BumpPointerAllocator snapshotSpaceAllocator_; FreeListAllocator machineCodeSpaceAllocator_; }; diff --git a/ecmascript/mem/free_object_kind.cpp b/ecmascript/mem/free_object_kind.cpp index 951871f25276b489301f68b343c81a5cb7d650c0..94829d653a0a83d7dd4a93bf0dcad83f9ac7c791 100644 --- a/ecmascript/mem/free_object_kind.cpp +++ b/ecmascript/mem/free_object_kind.cpp @@ -31,6 +31,9 @@ void FreeObjectKind::Rebuild() { freeObject_ = nullptr; available_ = 0; + isAdded_ = false; + next_ = nullptr; + prev_ = nullptr; } FreeObject *FreeObjectKind::SearchSmallFreeObject(size_t size) diff --git a/ecmascript/mem/free_object_kind.h b/ecmascript/mem/free_object_kind.h index 869689c5fb87a2e109e96d34b1f1ce770dadb99d..057968a16d1ed4c98c9b5cb1941c33b0fb766859 100644 --- a/ecmascript/mem/free_object_kind.h +++ b/ecmascript/mem/free_object_kind.h @@ -27,9 +27,9 @@ class FreeObject; class FreeObjectKind { public: - FreeObjectKind(KindType type, uintptr_t begin, size_t size) : kindType_(type) + FreeObjectKind(KindType type) : kindType_(type) { - Free(begin, size); + Rebuild(); } ~FreeObjectKind() = default; @@ -60,6 +60,7 @@ private: FreeObjectKind *prev_ = nullptr; KindType kindType_ = INVALID_KIND_TYPE; size_t available_ = 0; + bool isAdded_ = false; FreeObject *freeObject_ = nullptr; friend class FreeObjectList; diff --git a/ecmascript/mem/free_object_list.cpp b/ecmascript/mem/free_object_list.cpp index 43e220f78e4fa0f7dc54117e286f06528e31be06..37b71c9fc9e508ca568ccc868b339ade2aed7f74 100644 --- a/ecmascript/mem/free_object_list.cpp +++ b/ecmascript/mem/free_object_list.cpp @@ -16,21 +16,21 @@ #include "ecmascript/mem/free_object_list.h" #include "ecmascript/free_object.h" +#include "ecmascript/mem/free_object_kind.h" #include "ecmascript/mem/free_object_list-inl.h" #include "ecmascript/mem/mem.h" namespace panda::ecmascript { -FreeObjectList::FreeObjectList() +FreeObjectList::FreeObjectList() : kinds_(new FreeObjectKind *[NUMBER_OF_KINDS](), NUMBER_OF_KINDS) { - kinds_ = Span(new FreeObjectKind *[NUMBER_OF_KINDS](), NUMBER_OF_KINDS); + for (int i = 0; i < NUMBER_OF_KINDS; i++) { + kinds_[i] = nullptr; + } noneEmptyKindBitMap_ = 0; } FreeObjectList::~FreeObjectList() { - for (auto it : kinds_) { - delete it; - } delete[] kinds_.data(); noneEmptyKindBitMap_ = 0; } @@ -49,33 +49,39 @@ FreeObject *FreeObjectList::Allocator(size_t size) KindType lastType = type - 1; for (type = CalcNextNoneEmptyIndex(type); type > lastType && type < NUMBER_OF_KINDS; - type = CalcNextNoneEmptyIndex(type + 1)) { + type = CalcNextNoneEmptyIndex(type + 1)) { lastType = type; - FreeObjectKind *top = kinds_[type]; - if (top == nullptr || top->Available() < size) { - continue; - } - FreeObject *current = nullptr; - if (type <= SMALL_KIND_MAX_INDEX) { - current = top->SearchSmallFreeObject(size); - } else { - current = top->SearchLargeFreeObject(size); - } - if (top->Empty()) { - RemoveKind(top); - } - if (current != nullptr) { - size_t currentSize = current->Available(); - available_ -= currentSize; - if (currentSize >= size) { - return current; + FreeObjectKind *current = kinds_[type]; + while (current != nullptr) { + if (current->Available() < size) { + current = current->next_; + continue; } + FreeObjectKind *next = nullptr; + FreeObject *object = nullptr; + if (type <= SMALL_KIND_MAX_INDEX) { + object = current->SearchSmallFreeObject(size); + } else { + next = current->next_; + object = current->SearchLargeFreeObject(size); + } + if (current->Empty()) { + RemoveKind(current); + } + if (object != nullptr) { + size_t objectSize = object->Available(); + available_ -= objectSize; + if (objectSize >= size) { + return object; + } + } + current = next; } } return nullptr; } -void FreeObjectList::Free(uintptr_t start, size_t size) +void FreeObjectList::Free(uintptr_t start, size_t size, bool isAdd) { if (start == 0 || size == 0) { return; @@ -86,26 +92,28 @@ void FreeObjectList::Free(uintptr_t start, size_t size) return; } - auto kind = kinds_[type]; + Region *region = Region::ObjectAddressToRange(reinterpret_cast(start)); + auto kind = region->GetFreeObjectKind(type); if (kind == nullptr) { - kind = new FreeObjectKind(type, start, size); - if (!AddKind(kind)) { - delete kind; - return; + LOG_ECMA(FATAL) << "The kind of region is nullptr"; + return; + } + kind->Free(start, size); + + if (isAdd) { + if (kind->isAdded_) { + available_ += size; + } else { + AddKind(kind); } - } else { - kind->Free(start, size); } - available_ += size; } void FreeObjectList::Rebuild() { - for (auto kind : kinds_) { - if (kind != nullptr) { - kind->Rebuild(); - RemoveKind(kind); - } + EnumerateKinds([](FreeObjectKind *kind) { kind->Rebuild(); }); + for (int i = 0; i < NUMBER_OF_KINDS; i++) { + kinds_[i] = nullptr; } available_ = 0; noneEmptyKindBitMap_ = 0; @@ -118,20 +126,22 @@ size_t FreeObjectList::GetFreeObjectSize() const bool FreeObjectList::AddKind(FreeObjectKind *kind) { - if (kind == nullptr || kind->Empty()) { + if (kind == nullptr || kind->Empty() || kind->isAdded_) { return false; } KindType type = kind->kindType_; FreeObjectKind *top = kinds_[type]; if (kind == top) { - return true; + return false; } if (top != nullptr) { top->prev_ = kind; } + kind->isAdded_ = true; kind->next_ = top; kinds_[type] = kind; SetNoneEmptyBit(type); + available_ += kind->Available(); return true; } @@ -151,11 +161,30 @@ void FreeObjectList::RemoveKind(FreeObjectKind *kind) if (kind->next_ != nullptr) { kind->next_->prev_ = kind->prev_; } - kind->prev_ = nullptr; - kind->next_ = nullptr; if (kinds_[type] == nullptr) { ClearNoneEmptyBit(type); } - delete kind; + available_ -= kind->Available(); + kind->Rebuild(); +} + +template +void FreeObjectList::EnumerateKinds(const Callback &cb) const +{ + for (KindType i = 0; i < NUMBER_OF_KINDS; i++) { + EnumerateKinds(i, cb); + } +} + +template +void FreeObjectList::EnumerateKinds(KindType type, const Callback &cb) const +{ + FreeObjectKind *current = kinds_[type]; + while (current != nullptr) { + // maybe reset + FreeObjectKind *next = current->next_; + cb(current); + current = next; + } } } // namespace panda::ecmascript diff --git a/ecmascript/mem/free_object_list.h b/ecmascript/mem/free_object_list.h index c7db11126e86734ad79e2ac5cedcfa0837b2f150..71c6f02941b79629d1a6512dfe2a46f56c8bf185 100644 --- a/ecmascript/mem/free_object_list.h +++ b/ecmascript/mem/free_object_list.h @@ -29,15 +29,30 @@ public: FreeObject *Allocator(size_t size); - void Free(uintptr_t start, size_t size); + void Free(uintptr_t start, size_t size, bool isAdd = true); void Rebuild(); + bool AddKind(FreeObjectKind *kind); + + void RemoveKind(FreeObjectKind *kind); + + template + void EnumerateKinds(const Callback &cb) const; + + template + void EnumerateKinds(KindType type, const Callback &cb) const; + NO_COPY_SEMANTIC(FreeObjectList); NO_MOVE_SEMANTIC(FreeObjectList); size_t GetFreeObjectSize() const; + static int numberOfKinds() + { + return NUMBER_OF_KINDS; + } + private: static constexpr int NUMBER_OF_KINDS = 39; static constexpr size_t MIN_SIZE = 16; @@ -58,9 +73,6 @@ private: inline void ClearNoneEmptyBit(KindType type); inline size_t CalcNextNoneEmptyIndex(KindType start); - bool AddKind(FreeObjectKind *kind); - void RemoveKind(FreeObjectKind *kind); - size_t available_ = 0; uint64_t noneEmptyKindBitMap_; Span kinds_ {}; diff --git a/ecmascript/mem/heap.cpp b/ecmascript/mem/heap.cpp index 9294b267354982e618573b87620122b2fe07d67d..e51c9837948d5d18ada9fedea533a75d1587e76a 100644 --- a/ecmascript/mem/heap.cpp +++ b/ecmascript/mem/heap.cpp @@ -20,12 +20,15 @@ #include "ecmascript/ecma_vm.h" #include "ecmascript/mem/assert_scope-inl.h" #include "ecmascript/mem/compress_collector.h" +#include "ecmascript/mem/concurrent_sweeper.h" #include "ecmascript/mem/ecma_heap_manager.h" #include "ecmascript/mem/mark_stack.h" +#include "ecmascript/mem/mem.h" #include "ecmascript/mem/mem_controller.h" #include "ecmascript/mem/old_space_collector.h" #include "ecmascript/mem/semi_space_collector.h" #include "ecmascript/mem/semi_space_worker.h" +#include "ecmascript/mem/space.h" #include "ecmascript/mem/verification.h" namespace panda::ecmascript { @@ -68,6 +71,8 @@ void Heap::Initialize() semiSpaceCollector_ = new SemiSpaceCollector(this, false); compressCollector_ = new CompressCollector(this, false); } + bool concurrentSweep = ecmaVm_->GetOptions().IsEnableParalledYoungGc(); + sweeper_ = std::make_shared(this, concurrentSweep); oldSpaceCollector_ = new OldSpaceCollector(this); } @@ -87,6 +92,7 @@ void Heap::FlipCompressSpace() void Heap::Destroy() { pool_->WaitTaskFinish(); + sweeper_->EnsureAllTaskFinish(); toSpace_->Destroy(); delete toSpace_; toSpace_ = nullptr; @@ -156,10 +162,9 @@ void Heap::CollectGarbage(TriggerGCType gcType) } break; case TriggerGCType::OLD_GC: - if (oldSpace_->GetHeapObjectSize() < OLD_SPACE_LIMIT_BEGIN) { - oldSpaceCollector_->RunPhases(); - } else { - compressCollector_->RunPhases(); + oldSpaceCollector_->RunPhases(); + if (!sweeper_->IsConcurrentSweepEnabled()) { + RecomputeLimits(); } RecomputeLimits(); break; @@ -167,7 +172,9 @@ void Heap::CollectGarbage(TriggerGCType gcType) case TriggerGCType::HUGE_GC: case TriggerGCType::MACHINE_CODE_GC: oldSpaceCollector_->RunPhases(); - RecomputeLimits(); + if (!sweeper_->IsConcurrentSweepEnabled()) { + RecomputeLimits(); + } break; case TriggerGCType::COMPRESS_FULL_GC: compressCollector_->RunPhases(); @@ -177,7 +184,6 @@ void Heap::CollectGarbage(TriggerGCType gcType) UNREACHABLE(); break; } - // post gc heap verify { if (ecmaVm_->GetOptions().IsPreGcHeapVerifyEnabled()) { @@ -228,10 +234,23 @@ void Heap::RecomputeLimits() bool Heap::CheckAndTriggerOldGC() { - if (oldSpace_->GetHeapObjectSize() <= oldSpaceAllocLimit_) { + size_t oldSpaceSize = oldSpace_->GetHeapObjectSize(); + if (oldSpaceSize <= oldSpaceAllocLimit_) { return false; } - CollectGarbage(TriggerGCType::OLD_GC); + if (oldSpaceSize + oldSpace_->GetCommittedSize() >= MAX_OLD_SPACE_SIZE) { + isNeedFullGC_ = true; + } + if (isNeedFullGC_) { + CollectGarbage(TriggerGCType::COMPRESS_FULL_GC); + isNeedFullGC_ = false; + } else { + CollectGarbage(TriggerGCType::OLD_GC); + } + // If old space object size + if (oldSpace_->GetHeapObjectSize() * 1.2 < oldSpace_->GetCommittedSize()) { + isNeedFullGC_ = true; + } return true; } diff --git a/ecmascript/mem/heap.h b/ecmascript/mem/heap.h index bc112f2f877b7714dab00ce70cf545c5d8276cb2..5c30d07fbfcdfe7896321b4e8fb7748e79ea0c81 100644 --- a/ecmascript/mem/heap.h +++ b/ecmascript/mem/heap.h @@ -16,6 +16,8 @@ #ifndef ECMASCRIPT_MEM_HEAP_H #define ECMASCRIPT_MEM_HEAP_H +#include + #include "ecmascript/thread/thread_pool.h" #include "ecmascript/mem/mark_stack.h" #include "ecmascript/mem/space.h" @@ -31,6 +33,7 @@ class FreeListAllocator; class RegionFactory; class HeapTracker; class MemController; +class ConcurrentSweeper; class Heap { public: @@ -118,6 +121,11 @@ public: return compressCollector_; } + std::shared_ptr GetSweeper() const + { + return sweeper_; + } + EcmaVM *GetEcmaVM() const { return ecmaVm_; @@ -276,12 +284,14 @@ private: SemiSpaceCollector *semiSpaceCollector_ {nullptr}; OldSpaceCollector *oldSpaceCollector_ {nullptr}; CompressCollector *compressCollector_ {nullptr}; + std::shared_ptr sweeper_; EcmaHeapManager *heapManager_ {nullptr}; RegionFactory *regionFactory_ {nullptr}; HeapTracker *tracker_ {nullptr}; MemController *memController_ {nullptr}; ThreadPool *pool_ {nullptr}; size_t oldSpaceAllocLimit_ {OLD_SPACE_LIMIT_BEGIN}; + bool isNeedFullGC_{false}; inline void SetMaximumCapacity(SemiSpace *space, size_t maximumCapacity); }; diff --git a/ecmascript/mem/old_space_collector-inl.h b/ecmascript/mem/old_space_collector-inl.h index a8f88de5598d382c9d9a2786cbff2bc571958915..f5e4a958de8af2dd66c8af41b0bd802db3dac5fa 100644 --- a/ecmascript/mem/old_space_collector-inl.h +++ b/ecmascript/mem/old_space_collector-inl.h @@ -24,14 +24,13 @@ #include "ecmascript/js_hclass-inl.h" namespace panda::ecmascript { -void OldSpaceCollector::MarkObject(TaggedObject *object) +void OldSpaceCollector::MarkObject(uint64_t threadId, TaggedObject *object) { Region *objectRegion = Region::ObjectAddressToRange(object); auto markBitmap = objectRegion->GetMarkBitmap(); - if (!markBitmap->Test(object)) { - markBitmap->Set(object); - markStack_.PushBack(object); + if (!markBitmap->AtomicTestAndSet(object)) { + workList_->Push(threadId, object); } } @@ -42,14 +41,6 @@ void OldSpaceCollector::RecordWeakReference(JSTaggedType *ref) weakProcessQueue_.PushBack(ref); } } - -void OldSpaceCollector::FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, - uintptr_t freeEnd) -{ - allocator.Free(freeStart, freeEnd); - freeSize_ += freeEnd - freeStart; - heap_->ClearSlotsRange(current, freeStart, freeEnd); -} } // namespace panda::ecmascript #endif // ECMASCRIPT_MEM_SEMI_SAPACE_COLLECTOR_INL_H diff --git a/ecmascript/mem/old_space_collector.cpp b/ecmascript/mem/old_space_collector.cpp index 3572760030bc5a72f1440173fb54d5204987a955..d7f72cccbc8d0de3fd11807457b0710772b8a439 100644 --- a/ecmascript/mem/old_space_collector.cpp +++ b/ecmascript/mem/old_space_collector.cpp @@ -15,6 +15,8 @@ #include "ecmascript/mem/old_space_collector-inl.h" +#include "sys/time.h" + #include "ecmascript/ecma_vm.h" #include "ecmascript/mem/clock_scope.h" #include "ecmascript/mem/ecma_heap_manager.h" @@ -27,7 +29,10 @@ #include "ecmascript/vmstat/runtime_stat.h" namespace panda::ecmascript { -OldSpaceCollector::OldSpaceCollector(Heap *heap) : heap_(heap), rootManager_(heap->GetEcmaVM()) {} +OldSpaceCollector::OldSpaceCollector(Heap *heap) : heap_(heap), rootManager_(heap->GetEcmaVM()) +{ + workList_ = new OldGCWorker(heap_, heap_->GetThreadPool()->GetThreadNum()); +} void OldSpaceCollector::RunPhases() { @@ -45,11 +50,10 @@ void OldSpaceCollector::RunPhases() void OldSpaceCollector::InitializePhase() { + heap_->GetThreadPool()->WaitTaskFinish(); markStack_.BeginMarking(heap_, heap_->GetMarkStack()); weakProcessQueue_.BeginMarking(heap_, heap_->GetProcessQueue()); auto heapManager = heap_->GetHeapManager(); - oldSpaceAllocator_.Swap(heapManager->GetOldSpaceAllocator()); - nonMovableAllocator_.Swap(heapManager->GetNonMovableSpaceAllocator()); machineCodeSpaceAllocator_.Swap(heapManager->GetMachineCodeSpaceAllocator()); heap_->EnumerateRegions([](Region *current) { // ensure mark bitmap @@ -60,6 +64,7 @@ void OldSpaceCollector::InitializePhase() bitmap->ClearAllBits(); } }); + workList_->Initialize(); freeSize_ = 0; hugeSpaceFreeSize_ = 0; oldSpaceCommitSize_ = heap_->GetOldSpace()->GetCommittedSize(); @@ -69,12 +74,11 @@ void OldSpaceCollector::InitializePhase() void OldSpaceCollector::FinishPhase() { // swap - markStack_.FinishMarking(heap_->GetMarkStack()); weakProcessQueue_.FinishMarking(heap_->GetProcessQueue()); auto heapManager = heap_->GetHeapManager(); - heapManager->GetOldSpaceAllocator().Swap(oldSpaceAllocator_); - heapManager->GetNonMovableSpaceAllocator().Swap(nonMovableAllocator_); heapManager->GetMachineCodeSpaceAllocator().Swap(machineCodeSpaceAllocator_); + size_t aliveSize = 0; + workList_->Finish(aliveSize); } void OldSpaceCollector::MarkingPhase() @@ -83,35 +87,36 @@ void OldSpaceCollector::MarkingPhase() RootVisitor gcMarkYoung = [this]([[maybe_unused]] Root type, ObjectSlot slot) { JSTaggedValue value(slot.GetTaggedType()); if (value.IsHeapObject()) { - MarkObject(value.GetTaggedObject()); + MarkObject(0, value.GetTaggedObject()); } }; RootRangeVisitor gcMarkRangeYoung = [this]([[maybe_unused]] Root type, ObjectSlot start, ObjectSlot end) { for (ObjectSlot slot = start; slot < end; slot++) { JSTaggedValue value(slot.GetTaggedType()); if (value.IsHeapObject()) { - MarkObject(value.GetTaggedObject()); + MarkObject(0, value.GetTaggedObject()); } } }; rootManager_.VisitVMRoots(gcMarkYoung, gcMarkRangeYoung); - ProcessMarkStack(); + ProcessMarkStack(0); + heap_->GetThreadPool()->WaitTaskFinish(); } -void OldSpaceCollector::ProcessMarkStack() +void OldSpaceCollector::ProcessMarkStack(uint64_t threadId) { while (true) { - auto obj = markStack_.PopBack(); - if (UNLIKELY(obj == nullptr)) { + TaggedObject *obj = nullptr; + if (!workList_->Pop(threadId, &obj)) { break; } auto jsHclass = obj->GetClass(); // mark dynClass - MarkObject(jsHclass); + MarkObject(threadId, jsHclass); rootManager_.MarkObjectBody( - obj, jsHclass, [this]([[maybe_unused]] TaggedObject *root, ObjectSlot start, ObjectSlot end) { + obj, jsHclass, [this, &threadId]([[maybe_unused]] TaggedObject *root, ObjectSlot start, ObjectSlot end) { for (ObjectSlot slot = start; slot < end; slot++) { JSTaggedValue value(slot.GetTaggedType()); if (value.IsWeak()) { @@ -119,59 +124,13 @@ void OldSpaceCollector::ProcessMarkStack() continue; } if (value.IsHeapObject()) { - MarkObject(value.GetTaggedObject()); + MarkObject(threadId, value.GetTaggedObject()); } } }); } } -void OldSpaceCollector::SweepSpace(Space *space, FreeListAllocator &allocator) -{ - allocator.RebuildFreeList(); - space->EnumerateRegions([this, &allocator](Region *current) { - auto markBitmap = current->GetMarkBitmap(); - ASSERT(markBitmap != nullptr); - uintptr_t freeStart = current->GetBegin(); - markBitmap->IterateOverMarkedChunks([this, ¤t, &freeStart, &allocator](void *mem) { - ASSERT(current->InRange(ToUintPtr(mem))); - auto header = reinterpret_cast(mem); - auto klass = header->GetClass(); - JSType jsType = klass->GetObjectType(); - auto size = klass->SizeFromJSHClass(jsType, header); - size = AlignUp(size, static_cast(MemAlignment::MEM_ALIGN_OBJECT)); - - uintptr_t freeEnd = ToUintPtr(mem); - if (freeStart != freeEnd) { - FreeLiveRange(allocator, current, freeStart, freeEnd); - } - freeStart = freeEnd + size; - }); - uintptr_t freeEnd = current->GetEnd(); - if (freeStart != freeEnd) { - FreeLiveRange(allocator, current, freeStart, freeEnd); - } - }); -} - -void OldSpaceCollector::SweepSpace(HugeObjectSpace *space) -{ - Region *currentRegion = space->GetRegionList().GetFirst(); - - while (currentRegion != nullptr) { - Region *next = currentRegion->GetNext(); - auto markBitmap = currentRegion->GetMarkBitmap(); - bool isMarked = false; - markBitmap->IterateOverMarkedChunks([&isMarked]([[maybe_unused]] void *mem) { isMarked = true; }); - if (!isMarked) { - space->GetRegionList().RemoveNode(currentRegion); - hugeSpaceFreeSize_ += currentRegion->GetCapacity(); - space->ClearAndFreeRegion(currentRegion); - } - currentRegion = next; - } -} - void OldSpaceCollector::SweepPhases() { trace::ScopedTrace scoped_trace("OldSpaceCollector::SweepPhases"); @@ -209,9 +168,7 @@ void OldSpaceCollector::SweepPhases() heap_->GetEcmaVM()->GetJSThread()->IterateWeakEcmaGlobalStorage(gcUpdateWeak); heap_->GetEcmaVM()->ProcessReferences(gcUpdateWeak); - SweepSpace(const_cast(heap_->GetOldSpace()), oldSpaceAllocator_); - SweepSpace(const_cast(heap_->GetNonMovableSpace()), nonMovableAllocator_); - SweepSpace(const_cast(heap_->GetHugeObjectSpace())); - SweepSpace(const_cast(heap_->GetMachineCodeSpace()), machineCodeSpaceAllocator_); + heap_->GetSweeper()->SweepPhases(); + // SweepSpace(const_cast(heap_->GetMachineCodeSpace()), machineCodeSpaceAllocator_); } } // namespace panda::ecmascript diff --git a/ecmascript/mem/old_space_collector.h b/ecmascript/mem/old_space_collector.h index 64940351b60214b02f4a9a2ac2d0653162e1fc07..423d5466255aeb49120cbfc3c3a8fd5dbab1a660 100644 --- a/ecmascript/mem/old_space_collector.h +++ b/ecmascript/mem/old_space_collector.h @@ -21,6 +21,7 @@ #include "ecmascript/mem/allocator.h" #include "ecmascript/mem/mark_stack-inl.h" #include "ecmascript/mem/mark_word.h" +#include "ecmascript/mem/semi_space_worker.h" #include "ecmascript/mem/slots.h" #include "ecmascript/mem/heap_roots.h" #include "ecmascript/mem/remembered_set.h" @@ -50,26 +51,24 @@ private: void SweepPhases(); void FinishPhase(); - void ProcessMarkStack(); + void ProcessMarkStack(uint64_t threadId); void MarkObjectBody(TaggedObject *object, JSHClass *klass, const EcmaObjectRangeVisitor &visitor); - inline void MarkObject(TaggedObject *object); - inline void FreeLiveRange(FreeListAllocator &allocator, Region *current, uintptr_t freeStart, uintptr_t freeEnd); + inline void MarkObject(uint64_t threadId, TaggedObject *object); inline void RecordWeakReference(JSTaggedType *ref); - void SweepSpace(Space *space, FreeListAllocator &allocator); - void SweepSpace(HugeObjectSpace *space); // Only sweep huge space. Heap *heap_; HeapRootManager rootManager_; MarkStack markStack_; + OldGCWorker *workList_{nullptr}; ProcessQueue weakProcessQueue_; - FreeListAllocator oldSpaceAllocator_ {}; - FreeListAllocator nonMovableAllocator_ {}; FreeListAllocator machineCodeSpaceAllocator_ {}; size_t freeSize_{0}; size_t hugeSpaceFreeSize_ = 0; size_t oldSpaceCommitSize_ = 0; size_t nonMoveSpaceCommitSize_ = 0; + + friend class OldGCWorker; }; } // namespace ecmascript } // namespace panda diff --git a/ecmascript/mem/region.h b/ecmascript/mem/region.h index c80d1a9c0159fba30ba3d60d4a041dfd0a7e2f80..064fa97b6d47cff719b69e70a87c267696e685db 100644 --- a/ecmascript/mem/region.h +++ b/ecmascript/mem/region.h @@ -16,6 +16,7 @@ #ifndef ECMASCRIPT_MEM_REGION_H #define ECMASCRIPT_MEM_REGION_H +#include "ecmascript/mem/free_object_list.h" #include "ecmascript/mem/mem.h" #include "mem/gc/bitmap.h" @@ -41,9 +42,11 @@ enum RegionFlags { // NOLINTNEXTLINE(hicpp-signed-bitwise) IS_IN_OLD_GENERATION = 1 << 6, // NOLINTNEXTLINE(hicpp-signed-bitwise) + IS_IN_NON_MOVABLE_GENERATION = 1 << 7, + // NOLINTNEXTLINE(hicpp-signed-bitwise) IS_IN_YOUNG_OR_OLD_GENERATION = IS_IN_YOUNG_GENERATION | IS_IN_OLD_GENERATION, // NOLINTNEXTLINE(hicpp-signed-bitwise) - IS_INVALID = 1 << 7, + IS_INVALID = 1 << 8, }; class Region { @@ -220,6 +223,36 @@ public: return res; } + void InitializeKind() + { + kinds_ = Span(new FreeObjectKind *[FreeObjectList::numberOfKinds()](), + FreeObjectList::numberOfKinds()); + for (int i = 0; i < FreeObjectList::numberOfKinds(); i++) { + kinds_[i] = new FreeObjectKind(i); + } + } + + void DestoryKind() + { + for (auto kind : kinds_) { + delete kind; + } + delete[] kinds_.data(); + } + + FreeObjectKind *GetFreeObjectKind(KindType type) + { + return kinds_[type]; + } + + template + void EnumerateKinds(Callback cb) + { + for (auto kind : kinds_) { + cb(kind); + } + } + private: Space *space_; uintptr_t flags_; // Memory alignment, only low 32bits are used now @@ -232,6 +265,7 @@ private: RangeBitmap *markBitmap_{nullptr}; RememberedSet *crossRegionSet_{nullptr}; RememberedSet *oldToNewSet_{nullptr}; + Span kinds_; friend class SnapShot; }; } // namespace ecmascript diff --git a/ecmascript/mem/semi_space_collector.cpp b/ecmascript/mem/semi_space_collector.cpp index ef6975066f4328a0f4a42b5b9701b7fd21bfccc2..4fed609c601fc9d559fb30c2ee0eaf9998af6010 100644 --- a/ecmascript/mem/semi_space_collector.cpp +++ b/ecmascript/mem/semi_space_collector.cpp @@ -59,8 +59,8 @@ void SemiSpaceCollector::RunPhases() void SemiSpaceCollector::InitializePhase() { heap_->GetThreadPool()->WaitTaskFinish(); - gcTime_++; auto fromSpace = heap_->GetFromSpace(); + heap_->GetSweeper()->EnsureAllTaskFinish(); if (fromSpace->GetCommittedSize() == 0) { heap_->InitializeFromSpace(); } @@ -126,6 +126,7 @@ bool SemiSpaceCollector::ParallelHandleOldToNew(uint32_t threadId, Region *regio }); } }; + heap_->EnumerateOldSpaceRegions(cb, region); ProcessMarkStack(threadId); return true; diff --git a/ecmascript/mem/semi_space_collector.h b/ecmascript/mem/semi_space_collector.h index 6e5a31f62468b10c25ffdcd70d77bb02382042f0..3a76149e5c181dabf648422ac28ecca3f1f6721f 100644 --- a/ecmascript/mem/semi_space_collector.h +++ b/ecmascript/mem/semi_space_collector.h @@ -91,7 +91,6 @@ private: size_t semiCopiedSize_{0}; size_t commitSize_ = 0; uintptr_t ageMark_{0}; - size_t gcTime_{0}; friend class TlabAllocator; friend class SemiSpaceWorker; friend class SemiSpaceMarker; diff --git a/ecmascript/mem/semi_space_worker.cpp b/ecmascript/mem/semi_space_worker.cpp index dbaa62b62335f572f5a05d69ff4b9923a8080017..0df831f442d5bac5585cb79ecb728c8a322e6c55 100644 --- a/ecmascript/mem/semi_space_worker.cpp +++ b/ecmascript/mem/semi_space_worker.cpp @@ -19,6 +19,7 @@ #include "ecmascript/mem/compress_collector.h" #include "ecmascript/mem/heap.h" #include "ecmascript/mem/mark_stack.h" +#include "ecmascript/mem/old_space_collector.h" #include "ecmascript/mem/region_factory.h" #include "ecmascript/mem/tlab_allocator-inl.h" @@ -197,4 +198,32 @@ void CompressGCWorker::Initialize() holder.aliveSize_ = 0; } } + +void OldGCWorker::Initialize() +{ + spaceTop_ = markSpace_; + markSpaceEnd_ = markSpace_ + SPACE_SIZE; + for (uint32_t i = 0; i < threadNum_; i++) { + WorkNodeHolder &holder = workList_[i]; + holder.pushNode_ = AllocalWorkNode(); + holder.popNode_ = AllocalWorkNode(); + holder.weakQueue_ = new ProcessQueue(); + holder.weakQueue_->BeginMarking(heap_, continuousQueue_[i]); + } +} + +void OldGCWorker::PushWorkNodeToGlobal(uint32_t threadId) +{ + WorkNode *&pushNode = workList_[threadId].pushNode_; + if (!pushNode->IsEmpty()) { + globalWork_.Push(pushNode); + pushNode = AllocalWorkNode(); + + auto pool = heap_->GetThreadPool(); + if (pool->GetTaskCount() < pool->GetThreadNum() - 1) { + pool->Submit(std::bind(&OldSpaceCollector::ProcessMarkStack, heap_->GetOldSpaceCollector(), + std::placeholders::_1)); + } + } +} } // namespace panda::ecmascript diff --git a/ecmascript/mem/semi_space_worker.h b/ecmascript/mem/semi_space_worker.h index 2572c475e1e2bb830f8aac2ea3e9c2fb668d3efa..0a88f31f0e07a0bfb273c4ed5da607778a92fa37 100644 --- a/ecmascript/mem/semi_space_worker.h +++ b/ecmascript/mem/semi_space_worker.h @@ -233,6 +233,19 @@ public: NO_COPY_SEMANTIC(CompressGCWorker); NO_MOVE_SEMANTIC(CompressGCWorker); }; -} // namespace panda::ecmascript +class OldGCWorker : public Worker { +public: + OldGCWorker() = delete; + OldGCWorker(Heap *heap, uint32_t threadNum) : Worker(heap, threadNum) {} + + ~OldGCWorker() override = default; + + void PushWorkNodeToGlobal(uint32_t threadId) override; + void Initialize() override; + + NO_COPY_SEMANTIC(OldGCWorker); + NO_MOVE_SEMANTIC(OldGCWorker); +}; +} // namespace panda::ecmascript #endif // ECMASCRIPT_MEM_SEMI_SPACE_WORKER_H diff --git a/ecmascript/mem/space.cpp b/ecmascript/mem/space.cpp index cb83b9416ccd5e0552e14a65bf76a0db9880fe05..4b821d8e487420e6174c0de8eb53523dbe9aa895 100644 --- a/ecmascript/mem/space.cpp +++ b/ecmascript/mem/space.cpp @@ -20,6 +20,7 @@ #include "ecmascript/mem/remembered_set.h" #include "ecmascript/mem/space-inl.h" #include "libpandabase/utils/logger.h" +#include "space.h" namespace panda::ecmascript { void Space::AddRegion(Region *region) @@ -37,11 +38,15 @@ void Space::Initialize() } else if (spaceType_ == MemSpaceType::SNAPSHOT_SPACE) { region->SetFlag(RegionFlags::IS_IN_SNAPSHOT_GENERATION); } else if (spaceType_ == MemSpaceType::OLD_SPACE) { + region->InitializeKind(); region->SetFlag(RegionFlags::IS_IN_OLD_GENERATION); } else if (spaceType_ == MemSpaceType::MACHINE_CODE_SPACE) { region->SetFlag(RegionFlags::IS_IN_OLD_GENERATION); int res = region->SetCodeExecutableAndReadable(); LOG_ECMA_MEM(DEBUG) << "Initialize SetCodeExecutableAndReadable" << res; + } else if (spaceType_ == MemSpaceType::NON_MOVABLE) { + region->InitializeKind(); + region->SetFlag(RegionFlags::IS_IN_NON_MOVABLE_GENERATION); } AddRegion(region); @@ -75,6 +80,9 @@ void Space::ClearAndFreeRegion(Region *region) delete rememberedSet; } DecrementCommitted(region->GetCapacity()); + if (spaceType_ == MemSpaceType::OLD_SPACE || spaceType_ == MemSpaceType::NON_MOVABLE) { + region->DestoryKind(); + } const_cast(heap_->GetRegionFactory())->FreeRegion(region); } @@ -177,6 +185,7 @@ bool OldSpace::Expand() Region *region = const_cast(GetHeap()->GetRegionFactory())->AllocateAlignedRegion(this, DEFAULT_REGION_SIZE); region->SetFlag(RegionFlags::IS_IN_OLD_GENERATION); + region->InitializeKind(); AddRegion(region); return true; } @@ -231,8 +240,7 @@ size_t OldSpace::GetHeapObjectSize() const { size_t result; size_t availableSize = GetHeap()->GetHeapManager()->GetOldSpaceAllocator().GetAvailableSize(); - size_t regionSize = GetRegionList().GetLength() * DEFAULT_REGION_SIZE; - result = regionSize - availableSize; + result = GetCommittedSize() - availableSize; result += GetHeap()->GetHugeObjectSpace()->GetHeapObjectSize(); return result; } @@ -250,6 +258,8 @@ bool NonMovableSpace::Expand() } Region *region = const_cast(GetHeap()->GetRegionFactory())->AllocateAlignedRegion(this, DEFAULT_REGION_SIZE); + region->SetFlag(IS_IN_NON_MOVABLE_GENERATION); + region->InitializeKind(); AddRegion(region); return true; } @@ -382,6 +392,12 @@ uintptr_t HugeObjectSpace::Allocate(size_t objectSize) return region->GetBegin(); } +void HugeObjectSpace::Free(Region *region) +{ + GetRegionList().RemoveNode(region); + ClearAndFreeRegion(region); +} + bool HugeObjectSpace::ContainObject(TaggedObject *object) const { auto region = GetRegionList().GetFirst(); @@ -401,12 +417,7 @@ bool HugeObjectSpace::IsLive(TaggedObject *object) const size_t HugeObjectSpace::GetHeapObjectSize() const { - size_t result = 0; - EnumerateRegions([&result](Region *current) { - auto obj = reinterpret_cast(current->GetBegin()); - result += obj->GetObjectSize(); - }); - return result; + return GetCommittedSize(); } void HugeObjectSpace::IterateOverObjects(const std::function &objectVisitor) const diff --git a/ecmascript/mem/space.h b/ecmascript/mem/space.h index 7e6012f538ac35246ea9bd61b406878eb6703e89..7e6be985645bcd57fdbfbb858ba3c10c7f88b04b 100644 --- a/ecmascript/mem/space.h +++ b/ecmascript/mem/space.h @@ -23,19 +23,23 @@ #include "ecmascript/mem/region.h" #include "libpandabase/utils/type_helpers.h" #include "libpandafile/file.h" +#include "utils/logger.h" namespace panda::ecmascript { class Heap; class Program; enum MemSpaceType { - SEMI_SPACE, - OLD_SPACE, + OLD_SPACE = 0, NON_MOVABLE, HUGE_OBJECT_SPACE, + SEMI_SPACE, SNAPSHOT_SPACE, + COMPRESS_SPACE, MACHINE_CODE_SPACE, - SPACE_TYPE_LAST // Count of different types + SPACE_TYPE_LAST, // Count of different types + + FREE_LIST_NUM = NON_MOVABLE - OLD_SPACE + 1, }; enum TriggerGCType { @@ -238,6 +242,7 @@ public: NO_COPY_SEMANTIC(HugeObjectSpace); NO_MOVE_SEMANTIC(HugeObjectSpace); uintptr_t Allocate(size_t objectSize); + void Free(Region *region); size_t GetHeapObjectSize() const; bool ContainObject(TaggedObject *object) const; bool IsLive(TaggedObject *object) const; diff --git a/ecmascript/mem/tlab_allocator-inl.h b/ecmascript/mem/tlab_allocator-inl.h index 3a36d26b60004c612f6447f9d85c872df0feccec..4b05298d9c2b3fee6b9983326fffd9f3cb49e10b 100644 --- a/ecmascript/mem/tlab_allocator-inl.h +++ b/ecmascript/mem/tlab_allocator-inl.h @@ -20,11 +20,13 @@ #include "ecmascript/free_object.h" #include "ecmascript/mem/compress_collector.h" +#include "ecmascript/mem/mem.h" #include "ecmascript/mem/semi_space_collector-inl.h" #include "ecmascript/mem/tlab_allocator.h" +#include "mem/mem.h" namespace panda::ecmascript { -static constexpr size_t SEMIGC_YOUNG_BUFFER_SIZE = 32 * 1024; +static constexpr size_t SEMIGC_YOUNG_BUFFER_SIZE = 31 * 1024; static constexpr size_t OLD_BUFFER_SIZE = 255 * 1024; TlabAllocator::TlabAllocator(Heap *heap, TriggerGCType gcType) @@ -136,6 +138,7 @@ bool TlabAllocator::ExpandOld() } else if (gcType_ == TriggerGCType::COMPRESS_FULL_GC) { buffer = heap_->GetCompressCollector()->AllocateOld(OLD_BUFFER_SIZE); } else { + LOG(ERROR, RUNTIME) << "ExpandOld"; UNREACHABLE(); } diff --git a/ecmascript/napi/include/jsnapi.h b/ecmascript/napi/include/jsnapi.h index 4bf334dd88b8f6f6a223b5ce6004fb519431c144..e5b48ae4333910c688b7f160f17d58143268fe69 100644 --- a/ecmascript/napi/include/jsnapi.h +++ b/ecmascript/napi/include/jsnapi.h @@ -73,6 +73,7 @@ public: } Local(const EcmaVM *vm, const Global ¤t); + Local(const EcmaVM *vm, const Local ¤t) : Local(current) {}; ~Local() = default; @@ -86,6 +87,11 @@ public: return GetAddress(); } + inline Local ToLocal(const EcmaVM *vm) const + { + return Local(*this); + } + inline bool IsEmpty() const { return GetAddress() == nullptr; @@ -106,6 +112,8 @@ public: return IsEmpty() || GetAddress()->IsHole(); } + void FreeGlobalHandleAddr() {} + private: explicit inline Local(uintptr_t addr) : address_(addr) {} inline T *GetAddress() const @@ -230,18 +238,6 @@ private: uintptr_t escapeHandle_ = 0U; }; -class PUBLIC_API JSExecutionScope { -public: - explicit JSExecutionScope(const EcmaVM *vm); - ~JSExecutionScope(); - DISALLOW_COPY(JSExecutionScope); - DISALLOW_MOVE(JSExecutionScope); - -private: - void *last_current_thread_ = nullptr; - bool is_revert_ = false; -}; - class PUBLIC_API JSValueRef { public: static Local Undefined(const EcmaVM *vm); diff --git a/ecmascript/napi/jsnapi.cpp b/ecmascript/napi/jsnapi.cpp index 4ab3e3ad293b811593eb9c4ee31c9cf95affc24a..eb3f4a1bc72c2386d57fe8cd7dabdb3e0988d1c0 100644 --- a/ecmascript/napi/jsnapi.cpp +++ b/ecmascript/napi/jsnapi.cpp @@ -377,20 +377,23 @@ LocalScope::LocalScope(const EcmaVM *vm) : thread_(vm->GetJSThread()) prevNext_ = thread->GetHandleScopeStorageNext(); prevEnd_ = thread->GetHandleScopeStorageEnd(); prevHandleStorageIndex_ = thread->GetCurrentHandleStorageIndex(); + thread->HandleScopeCountAdd(); } LocalScope::LocalScope(const EcmaVM *vm, JSTaggedType value) : thread_(vm->GetJSThread()) { auto thread = reinterpret_cast(thread_); + ecmascript::EcmaHandleScope::NewHandle(thread, value); prevNext_ = thread->GetHandleScopeStorageNext(); prevEnd_ = thread->GetHandleScopeStorageEnd(); prevHandleStorageIndex_ = thread->GetCurrentHandleStorageIndex(); - ecmascript::EcmaHandleScope::NewHandle(thread, value); + thread->HandleScopeCountAdd(); } LocalScope::~LocalScope() { auto thread = reinterpret_cast(thread_); + thread->HandleScopeCountDec(); thread->SetHandleScopeStorageNext(static_cast(prevNext_)); if (thread->GetHandleScopeStorageEnd() != prevEnd_) { thread->SetHandleScopeStorageEnd(static_cast(prevEnd_)); @@ -1340,18 +1343,6 @@ JSTaggedValue Callback::RegisterCallbackWithNewTarget(ecmascript::EcmaRuntimeCal return JSNApiHelper::ToJSHandle(result).GetTaggedValue(); } -// ------------------------------------- JSExecutionScope ------------------------------ -JSExecutionScope::JSExecutionScope(const EcmaVM *vm) -{ - (void)vm; -} - -JSExecutionScope::~JSExecutionScope() -{ - last_current_thread_ = nullptr; - is_revert_ = false; -} - // ----------------------------------- JSValueRef -------------------------------------- Local JSValueRef::Undefined(const EcmaVM *vm) { diff --git a/ecmascript/napi/test/jsi_test.cpp b/ecmascript/napi/test/jsi_test.cpp index 07a4784b0d290c54660e93a50d4438ea78fbcc37..f94d53c51bb70109f6bfbd0b8bbd9237e26bbbe0 100644 --- a/ecmascript/napi/test/jsi_test.cpp +++ b/ecmascript/napi/test/jsi_test.cpp @@ -113,13 +113,6 @@ int main() std::cout << "SetAccessorProperty result: " << getset << std::endl; global->SetProperty(*runtime, "GetSet", getSetTest); - std::vector> arguments; - arguments.emplace_back(runtime->NewString("Hello world")); - - consoleObj = global->GetProperty(*runtime, "console"); - logFunc = consoleObj->GetProperty(*runtime, "log"); - shared_ptr testObj = logFunc->Call(*runtime, runtime->NewUndefined(), arguments, 1); - shared_ptr testArrayValue = runtime->NewString("1"); shared_ptr testValue = runtime->NewArray(); testValue->SetProperty(*runtime, "0", testArrayValue); @@ -132,8 +125,7 @@ int main() std::cout << "GetProperty test: " << result->IsArray(*runtime) << ";" << result->GetArrayLength(*runtime) << std::endl; - std::vector argument; - runtime->ExecuteJsBin("native.aex"); + runtime->ExecuteJsBin("test.abc"); return 0; } diff --git a/ecmascript/object_factory.cpp b/ecmascript/object_factory.cpp index 7282588009227f3e99881e2fcce43a598c034e9d..0a2b76ff36f982b482efdffdefbf30efc41f88e0 100644 --- a/ecmascript/object_factory.cpp +++ b/ecmascript/object_factory.cpp @@ -793,10 +793,12 @@ FreeObject *ObjectFactory::FillFreeObject(uintptr_t address, size_t size, Remove if (size >= FreeObject::SIZE_OFFSET && size < FreeObject::SIZE) { object = reinterpret_cast(address); object->SetClass(freeObjectWithOneFieldClass_); + object->SetNext(nullptr); } else if (size >= FreeObject::SIZE) { object = reinterpret_cast(address); object->SetClass(freeObjectWithTwoFieldClass_); object->SetAvailable(size); + object->SetNext(nullptr); } else if (size == FreeObject::NEXT_OFFSET) { object = reinterpret_cast(address); object->SetClass(freeObjectWithNoneFieldClass_); diff --git a/ecmascript/platform/platform.cpp b/ecmascript/platform/platform.cpp new file mode 100644 index 0000000000000000000000000000000000000000..f589f3807e9a1f370137a6b60e6c52deba0057db --- /dev/null +++ b/ecmascript/platform/platform.cpp @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/platform/platform.h" + +#include "sys/sysinfo.h" + +namespace panda::ecmascript { +void Platform::Initialize(int threadNum) +{ + os::memory::LockHolder lock(mutex_); + if (isInitialized_++ <= 0) { + runner_ = std::make_unique(TheMostSuitableThreadNum(threadNum)); + } +} + +void Platform::Destory() +{ + os::memory::LockHolder lock(mutex_); + if (--isInitialized_ <= 0) { + runner_->Terminate(); + } +} + +int Platform::TheMostSuitableThreadNum(int threadNum) const +{ + if (threadNum > 0) { + return std::min(threadNum, MAX_PLATFORM_THREAD_NUM); + } + int numOfCpuCore = get_nprocs() - 1; + return std::min(numOfCpuCore, MAX_PLATFORM_THREAD_NUM); +} +} // namespace panda::ecmascript diff --git a/ecmascript/platform/platform.h b/ecmascript/platform/platform.h new file mode 100644 index 0000000000000000000000000000000000000000..46411111fafc515ef5d9806d67b7734a4e402865 --- /dev/null +++ b/ecmascript/platform/platform.h @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PANDA_ECMASCRIPT_PALTFORM_PLATFORM_H +#define PANDA_ECMASCRIPT_PALTFORM_PLATFORM_H + +#include + +#include "ecmascript/platform/runner.h" +#include "os/mutex.h" + +namespace panda::ecmascript { +class Platform { +public: + static Platform *GetCurrentPlatform() + { + static Platform platform; + return &platform; + } + + Platform() = default; + ~Platform() = default; + + NO_COPY_SEMANTIC(Platform); + NO_MOVE_SEMANTIC(Platform); + + void Initialize(int threadNum = DEFAULT_PLATFORM_THREAD_NUM); + void Destory(); + + void PostTask(std::unique_ptr task) const + { + ASSERT(isInitialized_ > 0); + runner_->PostTask(std::move(task)); + } + +private: + static constexpr uint32_t MAX_PLATFORM_THREAD_NUM = 7; + static constexpr uint32_t DEFAULT_PLATFORM_THREAD_NUM = 0; + + int TheMostSuitableThreadNum(int threadNum) const; + + std::unique_ptr runner_; + int isInitialized_ = 0; + os::memory::Mutex mutex_; +}; +} // namespace panda::ecmascript +#endif // PANDA_ECMASCRIPT_PALTFORM_PLATFORM_H diff --git a/ecmascript/platform/runner.cpp b/ecmascript/platform/runner.cpp new file mode 100644 index 0000000000000000000000000000000000000000..12d7dcfcdaf93a3dd112ef6ade04cd028cb9bbb0 --- /dev/null +++ b/ecmascript/platform/runner.cpp @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/platform/runner.h" + +#include +#include + +#include "os/thread.h" + +namespace panda::ecmascript { +Runner::Runner(int threadNum) +{ + for (int i = 0; i < threadNum; i++) { + std::unique_ptr thread = std::make_unique(&Runner::Run, this); + os::thread::SetThreadName(thread->native_handle(), "GC_WorkerThread"); + threadPool_.emplace_back(std::move(thread)); + } +} + +void Runner::Terminate() +{ + taskQueue_.Terminate(); + int threadNum = threadPool_.size(); + for (int i = 0; i < threadNum; i++) { + threadPool_.at(i)->join(); + } + threadPool_.clear(); +} + +void Runner::Run() +{ + while (std::unique_ptr task = taskQueue_.PopTask()) { + task->Run(); + } +} +} // namespace panda::ecmascript diff --git a/ecmascript/platform/runner.h b/ecmascript/platform/runner.h new file mode 100644 index 0000000000000000000000000000000000000000..eea822ecf025e12df9b23b61a041705c9626f530 --- /dev/null +++ b/ecmascript/platform/runner.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PANDA_ECMASCRIPT_PLATFORM_RUNNER_H +#define PANDA_ECMASCRIPT_PLATFORM_RUNNER_H + +#include +#include +#include + +#include "ecmascript/platform/task_queue.h" + +namespace panda::ecmascript { +class Runner { +public: + explicit Runner(int threadNum); + ~Runner() = default; + + NO_COPY_SEMANTIC(Runner); + NO_MOVE_SEMANTIC(Runner); + + void PostTask(std::unique_ptr task) + { + taskQueue_.PostTask(std::move(task)); + } + + void Terminate(); + +private: + void Run(); + + std::vector> threadPool_ {}; + TaskQueue taskQueue_ {}; +}; +} // namespace panda::ecmascript +#endif // PANDA_ECMASCRIPT_PLATFORM_RUNNER_H diff --git a/ecmascript/platform/task.h b/ecmascript/platform/task.h new file mode 100644 index 0000000000000000000000000000000000000000..e2bcc7b5f2d6596152ce652e1f794d79db6ca53e --- /dev/null +++ b/ecmascript/platform/task.h @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PANDA_ECMASCRIPT_PLATFORM_TASK_H +#define PANDA_ECMASCRIPT_PLATFORM_TASK_H + +#include "macros.h" + +namespace panda::ecmascript { +class Task { +public: + Task() = default; + virtual ~Task() = default; + virtual bool Run() = 0; + + NO_COPY_SEMANTIC(Task); + NO_MOVE_SEMANTIC(Task); +}; +} // namespace panda::ecmascript +#endif // PANDA_ECMASCRIPT_PLATFORM_TASK_H diff --git a/ecmascript/platform/task_queue.cpp b/ecmascript/platform/task_queue.cpp new file mode 100644 index 0000000000000000000000000000000000000000..b90c20556984aafedb5315534488305cb2fc6a5a --- /dev/null +++ b/ecmascript/platform/task_queue.cpp @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "ecmascript/platform/task_queue.h" + +namespace panda::ecmascript { +void TaskQueue::PostTask(std::unique_ptr task) +{ + os::memory::LockHolder holder(mtx_); + ASSERT(!terminate_); + tasks_.push(std::move(task)); + cv_.Signal(); +} + +std::unique_ptr TaskQueue::PopTask() +{ + os::memory::LockHolder holder(mtx_); + while (true) { + if (!tasks_.empty()) { + std::unique_ptr task = std::move(tasks_.front()); + tasks_.pop(); + return task; + } + if (terminate_) { + cv_.SignalAll(); + return nullptr; + } + cv_.Wait(&mtx_); + } +} + +void TaskQueue::Terminate() +{ + os::memory::LockHolder holder(mtx_); + terminate_ = true; + cv_.SignalAll(); +} +} // namespace panda::ecmascript diff --git a/ecmascript/platform/task_queue.h b/ecmascript/platform/task_queue.h new file mode 100644 index 0000000000000000000000000000000000000000..121eb7cc51ee34b2f81ffae408dbcf889b0d84f6 --- /dev/null +++ b/ecmascript/platform/task_queue.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PANDA_ECMASCRIPT_PLATFORM_TASK_QUEUE_H +#define PANDA_ECMASCRIPT_PLATFORM_TASK_QUEUE_H + +#include +#include +#include +#include + +#include "ecmascript/platform/task.h" +#include "os/mutex.h" + +namespace panda::ecmascript { +class TaskQueue { +public: + TaskQueue() = default; + ~TaskQueue() = default; + + NO_COPY_SEMANTIC(TaskQueue); + NO_MOVE_SEMANTIC(TaskQueue); + + void PostTask(std::unique_ptr task); + std::unique_ptr PopTask(); + + void Terminate(); + +private: + std::queue> tasks_; + + std::atomic_bool terminate_ = false; + os::memory::Mutex mtx_; + os::memory::ConditionVariable cv_; +}; +} // namespace panda::ecmascript +#endif // PANDA_ECMASCRIPT_PLATFORM_TASK_QUEUE_H diff --git a/ecmascript/tests/concurrent_sweep_test.cpp b/ecmascript/tests/concurrent_sweep_test.cpp new file mode 100644 index 0000000000000000000000000000000000000000..25baa08b63be3c94f2f89685d81452e949c95fb0 --- /dev/null +++ b/ecmascript/tests/concurrent_sweep_test.cpp @@ -0,0 +1,72 @@ +/* + * Copyright (c) 2021 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "gtest/gtest.h" + +#include "ecmascript/ecma_vm.h" +#include "ecmascript/global_env.h" +#include "ecmascript/js_handle.h" +#include "include/runtime_options.h" + +using namespace panda::ecmascript; + +namespace panda::test { +class ConcurrentSweepTest : public testing::Test { +public: + void SetUp() override + { + RuntimeOptions options; + options.SetShouldLoadBootPandaFiles(false); + options.SetShouldInitializeIntrinsics(false); + options.SetBootIntrinsicSpaces({"ecmascript"}); + options.SetBootClassSpaces({"ecmascript"}); + options.SetRuntimeType("ecmascript"); + options.SetRunGcInPlace(true); + [[maybe_unused]] bool success = Runtime::Create(options); + ASSERT_TRUE(success) << "Cannot create Runtime"; + instance = Runtime::GetCurrent()->GetPandaVM(); + thread = EcmaVM::Cast(instance)->GetAssociatedJSThread(); + scope = new EcmaHandleScope(thread); + thread->SetIsEcmaInterpreter(true); + thread->GetEcmaVM()->GetFactory()->SetTriggerGc(true); + } + + void TearDown() override + { + thread->ClearException(); + thread->GetEcmaVM()->GetFactory()->SetTriggerGc(false); + delete scope; + [[maybe_unused]] bool success = Runtime::Destroy(); + ASSERT_TRUE(success) << "Cannot destroy Runtime"; + } + + PandaVM *instance {nullptr}; + EcmaHandleScope *scope {nullptr}; + JSThread *thread; +}; + +TEST_F(ConcurrentSweepTest, ConcurrentSweep) +{ + auto vm = EcmaVM::Cast(instance); + const uint8_t *utf8 = reinterpret_cast("test"); + JSHandle test1(thread, EcmaString::CreateFromUtf8(utf8, 4, vm)); + if (vm->IsInitialized()) { + vm->CollectGarbage(ecmascript::MemSpaceType::OLD_SPACE); + } + JSHandle test2(thread, EcmaString::CreateFromUtf8(utf8, 4, vm)); + ASSERT_EQ(test1->GetLength(), 4); + ASSERT_NE(test1.GetTaggedValue().GetHeapObject(), test2.GetTaggedValue().GetHeapObject()); +} +} // namespace panda::test