diff --git a/common_components/heap/allocator/region_desc.h b/common_components/heap/allocator/region_desc.h index 062f4dd12825295ac238013038c03a126cff72e1..b65680cc5de0a3d9da59d74798178d641ff2550c 100755 --- a/common_components/heap/allocator/region_desc.h +++ b/common_components/heap/allocator/region_desc.h @@ -206,40 +206,6 @@ public: return nullptr; } - RegionBitmap* GetEnqueueBitmap() - { - RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.enqueueBitmap_, std::memory_order_acquire); - if (reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR) { - return nullptr; - } - return bitmap; - } - - RegionBitmap* GetOrAllocEnqueueBitmap() - { - do { - RegionBitmap *bitmap = __atomic_load_n(&metadata.liveInfo_.enqueueBitmap_, std::memory_order_acquire); - if (UNLIKELY_CC(reinterpret_cast(bitmap) == RegionLiveDesc::TEMPORARY_PTR)) { - continue; - } - if (LIKELY_CC(bitmap != nullptr)) { - return bitmap; - } - RegionBitmap* newValue = reinterpret_cast(RegionLiveDesc::TEMPORARY_PTR); - if (__atomic_compare_exchange_n(&metadata.liveInfo_.enqueueBitmap_, &bitmap, newValue, false, - std::memory_order_seq_cst, std::memory_order_relaxed)) { - RegionBitmap *allocated = - HeapBitmapManager::GetHeapBitmapManager().AllocateRegionBitmap(GetRegionBaseSize()); - __atomic_store_n(&metadata.liveInfo_.enqueueBitmap_, allocated, std::memory_order_release); - DLOG(REGION, "region %p(base=%#zx)@%#zx liveinfo %p alloc enqueuebitmap %p", - this, GetRegionBase(), GetRegionStart(), &metadata.liveInfo_, metadata.liveInfo_.enqueueBitmap_); - return allocated; - } - } while (true); - - return nullptr; - } - void ResetMarkBit() { SetMarkedRegionFlag(0); @@ -281,21 +247,6 @@ public: return marked; } - bool EnqueueObject(const BaseObject* obj) - { - if (IsLargeRegion()) { - if (metadata.regionBits.AtomicGetValue(RegionBitOffset::BIT_OFFSET_ENQUEUED_REGION, 1) != 1) { - SetEnqueuedRegionFlag(1); - return false; - } - return true; - } - size_t offset = GetAddressOffset(reinterpret_cast(obj)); - bool marked = GetOrAllocEnqueueBitmap()->MarkBits(offset); - CHECK_CC(IsEnqueuedObject(obj)); - return marked; - } - bool IsResurrectedObject(const BaseObject* obj) { if (IsLargeRegion()) { @@ -322,19 +273,6 @@ public: return markBitmap->IsMarked(offset); } - bool IsEnqueuedObject(const BaseObject* obj) - { - if (IsLargeRegion()) { - return (metadata.regionBits.AtomicGetValue(RegionBitOffset::BIT_OFFSET_ENQUEUED_REGION, 1) == 1); - } - RegionBitmap* enqueBitmap = GetEnqueueBitmap(); - if (enqueBitmap == nullptr) { - return false; - } - size_t offset = GetAddressOffset(reinterpret_cast(obj)); - return enqueBitmap->IsMarked(offset); - } - RegionRSet* GetRSet() { return metadata.regionRSet; @@ -1052,13 +990,11 @@ private: { markBitmap_ = nullptr; resurrectBitmap_ = nullptr; - enqueueBitmap_ = nullptr; } private: RegionDesc *relatedRegion_ {nullptr}; RegionBitmap *markBitmap_ {nullptr}; RegionBitmap *resurrectBitmap_ {nullptr}; - RegionBitmap *enqueueBitmap_ {nullptr}; friend class RegionDesc; }; diff --git a/common_components/heap/allocator/region_space.h b/common_components/heap/allocator/region_space.h index 18f143ae400ecd23c2ce255d0c8d24c3877dd78f..1b21323354de35ea7d60af9a0eb8978170b40487 100755 --- a/common_components/heap/allocator/region_space.h +++ b/common_components/heap/allocator/region_space.h @@ -309,12 +309,6 @@ public: return regionInfo->ResurrentObject(obj); } - static bool EnqueueObject(const BaseObject* obj) - { - RegionDesc* regionInfo = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); - return regionInfo->EnqueueObject(obj); - } - static bool IsMarkedObject(const BaseObject* obj) { RegionDesc* regionInfo = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); @@ -327,12 +321,6 @@ public: return regionInfo->IsResurrectedObject(obj); } - static bool IsEnqueuedObject(const BaseObject* obj) - { - RegionDesc* regionInfo = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); - return regionInfo->IsEnqueuedObject(obj); - } - static bool IsNewObjectSinceTrace(const BaseObject* object) { RegionDesc* region = RegionDesc::GetAliveRegionDescAt(reinterpret_cast(object)); diff --git a/common_components/heap/collector/region_bitmap.h b/common_components/heap/collector/region_bitmap.h index 0aeac4f6f4ed0a7e8a72ae17ac356b9ad4b2931b..6e20f45cf6d8321d9d1f37af3954afc058e7d81b 100755 --- a/common_components/heap/collector/region_bitmap.h +++ b/common_components/heap/collector/region_bitmap.h @@ -30,13 +30,7 @@ static constexpr size_t kMarkedBytesPerBit = 8; static constexpr size_t kBitsPerWord = sizeof(uint64_t) * kBitsPerByte; static constexpr size_t kBytesPerWord = sizeof(uint64_t) / sizeof(uint8_t); struct RegionBitmap { - static constexpr uint8_t factor = 16; - std::atomic partLiveBytes[factor]; - std::atomic liveBytes; - // 1 bit marks 8 bytes in region, 64 bits per word. - // word count = region size / (8 * 64) = region size / 512, should be dynamically decided at runtime. - std::atomic wordCnt; std::atomic markWords[0]; static size_t GetRegionBitmapSize(size_t regionSize) @@ -45,18 +39,17 @@ struct RegionBitmap { return sizeof(RegionBitmap) + ((regionSize / (kMarkedBytesPerBit * kBitsPerWord)) * sizeof(uint64_t)); } - explicit RegionBitmap(size_t regionSize) : liveBytes(0), wordCnt(regionSize / (kMarkedBytesPerBit * kBitsPerWord)) - {} + explicit RegionBitmap(size_t regionSize) {} bool MarkBits(size_t start) { size_t headWordIdx = (start / kMarkedBytesPerBit) / kBitsPerWord; size_t headMaskBitStart = (start / kMarkedBytesPerBit) % kBitsPerWord; uint64_t headMaskBits = static_cast(1) << headMaskBitStart; - uint64_t old = markWords[headWordIdx].load(); + uint64_t old = markWords[headWordIdx].load(std::memory_order_relaxed); bool isMarked = ((old & headMaskBits) != 0); if (!isMarked) { - old = markWords[headWordIdx].fetch_or(headMaskBits); + old = markWords[headWordIdx].fetch_or(headMaskBits, std::memory_order_relaxed); isMarked = ((old & headMaskBits) != 0); return isMarked; } @@ -68,7 +61,7 @@ struct RegionBitmap { size_t headWordIdx = (start / kMarkedBytesPerBit) / kBitsPerWord; size_t headMaskBitStart = (start / kMarkedBytesPerBit) % kBitsPerWord; uint64_t mask = static_cast(1) << headMaskBitStart; - bool ret = ((markWords[headWordIdx].load() & mask) != 0); + bool ret = ((markWords[headWordIdx].load(std::memory_order_relaxed) & mask) != 0); return ret; } }; diff --git a/common_components/heap/collector/trace_collector.cpp b/common_components/heap/collector/trace_collector.cpp index a2e7bd42b5cb8392b0b2e2704873f2d0f5b2d06e..b42ce75325c27a8fd4e51b293245f0cdec859e0f 100755 --- a/common_components/heap/collector/trace_collector.cpp +++ b/common_components/heap/collector/trace_collector.cpp @@ -187,14 +187,7 @@ void TraceCollector::ProcessMarkStack([[maybe_unused]] uint32_t threadIndex, Tas WorkStack remarkStack; auto fetchFromSatbBuffer = [this, &workStack, &remarkStack]() { SatbBuffer::Instance().TryFetchOneRetiredNode(remarkStack); - while (!remarkStack.empty()) { - BaseObject *obj = remarkStack.back(); - remarkStack.pop_back(); - if (Heap::IsHeapAddress(obj) && (!MarkObject(obj))) { - workStack.push_back(obj); - DLOG(TRACE, "tracing take from satb buffer: obj %p", obj); - } - } + workStack.insert(remarkStack); }; size_t iterationCnt = 0; constexpr size_t maxIterationLoopNum = 1000; @@ -208,16 +201,17 @@ void TraceCollector::ProcessMarkStack([[maybe_unused]] uint32_t threadIndex, Tas // get next object from work stack. BaseObject *obj = workStack.back(); workStack.pop_back(); + auto region = RegionDesc::GetAliveRegionDescAt(reinterpret_cast((void *)obj)); region->AddLiveByteCount(obj->GetSize()); [[maybe_unused]] auto beforeSize = workStack.count(); TraceObjectRefFields(obj, &visitor); DLOG(TRACE, "[tracing] visit finished, workstack size: before=%d, after=%d, newly added=%d", beforeSize, workStack.count(), workStack.count() - beforeSize); + // try to fork new task if needed. - if (threadPool != nullptr) { - TryForkTask(threadPool, workStack, globalQueue); - } + ASSERT_LOGF(threadPool != nullptr, "thread pool is null"); + TryForkTask(threadPool, workStack, globalQueue); } // Try some task from satb buffer, bound the loop to make sure it converges in time @@ -469,30 +463,29 @@ bool TraceCollector::MarkSatbBuffer(WorkStack& workStack) { OHOS_HITRACE(HITRACE_LEVEL_COMMERCIAL, "CMCGC::MarkSatbBuffer", ""); COMMON_PHASE_TIMER("MarkSatbBuffer"); - auto visitSatbObj = [this, &workStack]() { - WorkStack remarkStack; - auto func = [&remarkStack](Mutator& mutator) { - const SatbBuffer::TreapNode* node = mutator.GetSatbBufferNode(); - if (node != nullptr) { - const_cast(node)->GetObjects(remarkStack); - } - }; - MutatorManager::Instance().VisitAllMutators(func); - SatbBuffer::Instance().GetRetiredObjects(remarkStack); - - while (!remarkStack.empty()) { // LCOV_EXCL_BR_LINE - BaseObject* obj = remarkStack.back(); - remarkStack.pop_back(); - if (Heap::IsHeapAddress(obj)) { - if (!this->MarkObject(obj)) { - workStack.push_back(obj); - DLOG(TRACE, "satb buffer add obj %p", obj); - } - } + + WorkStack remarkStack; + auto func = [&remarkStack](Mutator& mutator) { + const SatbBuffer::TreapNode* node = mutator.GetSatbBufferNode(); + if (node != nullptr) { + const_cast(node)->GetObjects(remarkStack); } }; + MutatorManager::Instance().VisitAllMutators(func); + SatbBuffer::Instance().GetRetiredObjects(remarkStack); + + // root objects are pre-forwarded, use to-version of them + while (!remarkStack.empty()) { + BaseObject* obj = remarkStack.back(); + remarkStack.pop_back(); + + if (obj->IsForwarded()) { + workStack.push_back(obj->GetForwardingPointer()); + } else { + workStack.push_back(obj); + } + } - visitSatbObj(); return true; } diff --git a/common_components/heap/w_collector/w_collector.cpp b/common_components/heap/w_collector/w_collector.cpp index 345e1ebbae2a12ffbdf2e87f8625285f73ef2700..820a08b7997617dedf1968ca8c9dd16bfb589951 100755 --- a/common_components/heap/w_collector/w_collector.cpp +++ b/common_components/heap/w_collector/w_collector.cpp @@ -139,11 +139,9 @@ bool WCollector::TryUntagRefField(BaseObject* obj, RefField<>& field, BaseObject return false; } -static void TraceRefField(BaseObject *obj, BaseObject *targetObj, RefField<> &field, - WorkStack &workStack, RegionDesc *targetRegion); // note each ref-field will not be traced twice, so each old pointer the tracer meets must come from previous gc. -static void TraceRefField(BaseObject *obj, RefField<> &field, WorkStack &workStack, - WeakStack &weakStack, const GCReason gcReason) +template +static void TraceRefField(BaseObject* obj, RefField<>& field, WorkStack& workStack, WeakStack& weakStack) { RefField<> oldField(field); BaseObject* targetObj = oldField.GetTargetObject(); @@ -155,28 +153,24 @@ static void TraceRefField(BaseObject *obj, RefField<> &field, WorkStack &workSta DCHECK_CC(Heap::IsHeapAddress(targetObj)); auto targetRegion = RegionDesc::GetAliveRegionDescAt(reinterpret_cast((void*)targetObj)); - if (gcReason != GC_REASON_YOUNG && oldField.IsWeak()) { - DLOG(TRACE, "trace: skip weak obj when full gc, object: %p@%p, targetObj: %p", obj, &field, targetObj); - // weak ref is cleared after roots pre-forward, so there might be a to-version weak ref which also need to be - // cleared, offset recorded here will help us find it - weakStack.push_back(std::make_shared*, size_t>>( - &field, reinterpret_cast(&field) - reinterpret_cast(obj))); - return; - } - - // cannot skip objects in EXEMPTED_FROM_REGION, because its rset is incomplete - if (gcReason == GC_REASON_YOUNG && !targetRegion->IsInYoungSpace()) { - DLOG(TRACE, "trace: skip non-young object %p@%p, target object: %p<%p>(%zu)", - obj, &field, targetObj, targetObj->GetTypeInfo(), targetObj->GetSize()); - return; + if constexpr (youngGC) { + // cannot skip objects in EXEMPTED_FROM_REGION, because its rset is incomplete + if (!targetRegion->IsInYoungSpace()) { + DLOG(TRACE, "trace: skip non-young object %p@%p, target object: %p<%p>(%zu)", + obj, &field, targetObj, targetObj->GetTypeInfo(), targetObj->GetSize()); + return; + } + } else { + if (oldField.IsWeak()) { + DLOG(TRACE, "trace: skip weak obj when full gc, object: %p@%p, targetObj: %p", obj, &field, targetObj); + // weak ref is cleared after roots pre-forward, so there might be a to-version weak ref which also need to + // be cleared. Offset recorded here help us find it + weakStack.push_back(std::make_shared*, size_t>>( + &field, reinterpret_cast(&field) - reinterpret_cast(obj))); + return; + } } - common::TraceRefField(obj, targetObj, field, workStack, targetRegion); -} -// note each ref-field will not be traced twice, so each old pointer the tracer meets must come from previous gc. -static void TraceRefField(BaseObject *obj, BaseObject *targetObj, RefField<> &field, - WorkStack &workStack, RegionDesc *targetRegion) -{ if (targetRegion->IsNewObjectSinceTrace(targetObj)) { DLOG(TRACE, "trace: skip new obj %p<%p>(%zu)", targetObj, targetObj->GetTypeInfo(), targetObj->GetSize()); return; @@ -199,13 +193,11 @@ TraceCollector::TraceRefFieldVisitor WCollector::CreateTraceObjectRefFieldsVisit if (gcReason_ == GCReason::GC_REASON_YOUNG) { visitor.SetVisitor([obj = visitor.GetClosure(), workStack, weakStack](RefField<> &field) { - const GCReason gcReason = GCReason::GC_REASON_YOUNG; - TraceRefField(*obj, field, *workStack, *weakStack, gcReason); + TraceRefField(*obj, field, *workStack, *weakStack); }); } else { visitor.SetVisitor([obj = visitor.GetClosure(), workStack, weakStack](RefField<> &field) { - const GCReason gcReason = GCReason::GC_REASON_HEU; - TraceRefField(*obj, field, *workStack, *weakStack, gcReason); + TraceRefField(*obj, field, *workStack, *weakStack); }); } return visitor; diff --git a/common_components/mutator/mutator.h b/common_components/mutator/mutator.h index f5a4950dc0d69356c489d90145df3521d1b85b7a..d47456a43f8c448b61d241cc7b3746a5d87ec2bc 100755 --- a/common_components/mutator/mutator.h +++ b/common_components/mutator/mutator.h @@ -313,7 +313,7 @@ private: void RememberObjectImpl(const BaseObject* obj) { if (LIKELY_CC(Heap::IsHeapAddress(obj))) { - if (SatbBuffer::Instance().ShouldEnqueue(obj)) { + if (SatbBuffer::ShouldEnqueue(obj)) { SatbBuffer::Instance().EnsureGoodNode(satbNode_); satbNode_->Push(obj); } diff --git a/common_components/mutator/satb_buffer.cpp b/common_components/mutator/satb_buffer.cpp index cf3eddbc46e2528e00fb2d18580c9c3f18b9b7b5..39667c9850e505db4baee578ef81e52d52e59e5f 100755 --- a/common_components/mutator/satb_buffer.cpp +++ b/common_components/mutator/satb_buffer.cpp @@ -25,18 +25,24 @@ SatbBuffer& SatbBuffer::Instance() noexcept { return *g_instance; } bool SatbBuffer::ShouldEnqueue(const BaseObject* obj) { - if (UNLIKELY_CC(obj == nullptr)) { - return false; - } - if (Heap::GetHeap().GetGCReason() == GC_REASON_YOUNG && !RegionSpace::IsYoungSpaceObject(obj)) { + ASSERT_LOGF(obj != nullptr, "obj should not be null"); + + auto region = common::RegionDesc::GetAliveRegionDescAt(reinterpret_cast(obj)); + if (Heap::GetHeap().GetGCReason() == GC_REASON_YOUNG && !region->IsInYoungSpace()) { return false; } - if (RegionSpace::IsNewObjectSinceTrace(obj)) { + + if (region->IsNewObjectSinceTrace(obj)) { return false; } - if (RegionSpace::IsMarkedObject(obj)) { + + // get previous mark state and try mark + if (region->MarkObject(obj)) { + // already marked return false; } - return !RegionSpace::EnqueueObject(obj); + + // not marked before but now marked, put it in satb + return true; } } // namespace common diff --git a/common_components/mutator/satb_buffer.h b/common_components/mutator/satb_buffer.h index 10783b4be9b86f031b3471bb9366529edbd61e6f..a2b4119eff9daf549f02b0c206ae7b69721bb55c 100755 --- a/common_components/mutator/satb_buffer.h +++ b/common_components/mutator/satb_buffer.h @@ -208,7 +208,7 @@ public: LOGF_CHECK(node->IsEmpty()) << "get an unempty node from free nodes"; } } - bool ShouldEnqueue(const BaseObject* obj); + static bool ShouldEnqueue(const BaseObject* obj); // must not have thread racing void Init() diff --git a/common_components/mutator/tests/satb_buffer_test.cpp b/common_components/mutator/tests/satb_buffer_test.cpp index feec5b10277ecfe6fe68dde373f8a329cdd06025..25760586931660a8350c8b4596a79142f6e73a6d 100755 --- a/common_components/mutator/tests/satb_buffer_test.cpp +++ b/common_components/mutator/tests/satb_buffer_test.cpp @@ -44,11 +44,6 @@ class SatbBufferTest : public BaseTestWithScope { ThreadHolder::TryBindMutatorScope *scope_ {nullptr}; }; -HWTEST_F_L0(SatbBufferTest, NullptrReturnsFalse) -{ - EXPECT_FALSE(SatbBuffer::Instance().ShouldEnqueue(nullptr)); -} - HWTEST_F_L0(SatbBufferTest, IsYoungSpaceObject1) { auto* mutator = common::Mutator::GetMutator(); @@ -148,32 +143,4 @@ void ClearMarkBit(RegionBitmap* bitmap, size_t offset) } } while (!__atomic_compare_exchange_n(addr, &oldVal, newVal, false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)); } - -HWTEST_F_L0(SatbBufferTest, EnqueueObject) -{ - RegionSpace& theAllocator = reinterpret_cast(Heap::GetHeap().GetAllocator()); - uintptr_t addr = theAllocator.AllocOldRegion(); - ASSERT_NE(addr, 0U); - - uintptr_t objAddress = addr + 0x100; - BaseObject* obj = reinterpret_cast(objAddress); - Heap::GetHeap().SetGCReason(GC_REASON_HEU); - - RegionDesc* region = RegionDesc::GetRegionDescAt(reinterpret_cast(obj)); - region->SetRegionType(RegionDesc::RegionType::FROM_REGION); - region->SetTraceLine(); - region->SetMarkedRegionFlag(0); - RegionBitmap* markBitmap = region->GetMarkBitmap(); - size_t offset = region->GetAddressOffset(reinterpret_cast(obj)); - if (markBitmap != nullptr) { - ClearMarkBit(markBitmap, offset); - } - - region->SetEnqueuedRegionFlag(1); - RegionBitmap* enqueueBitmap = region->GetOrAllocEnqueueBitmap(); - enqueueBitmap->MarkBits(offset); - - bool result = SatbBuffer::Instance().ShouldEnqueue(obj); - EXPECT_FALSE(result); -} } // namespace common::test