From 78e20d49e206c980dae4b26ca820e8f50d725e9e Mon Sep 17 00:00:00 2001 From: shizhili Date: Fri, 8 Dec 2023 09:37:11 +0800 Subject: [PATCH 1/2] backport support kv storage for consumequeue --- ...-Support-KV-Storage-for-ConsumeQueue.patch | 29480 ++++++++++++++++ rocketmq.spec | 6 +- 2 files changed, 29485 insertions(+), 1 deletion(-) create mode 100644 patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch diff --git a/patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch b/patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch new file mode 100644 index 0000000..a4a09ae --- /dev/null +++ b/patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch @@ -0,0 +1,29480 @@ +From 0a046b85611bdc93c529650dc951f4cabbb61db9 Mon Sep 17 00:00:00 2001 +From: zhiliatom +Date: Fri, 8 Dec 2023 09:25:41 +0800 +Subject: [PATCH] [ISSUE #7064] [RIP-66-2] Support KV(RocksDB) Storage for + ConsumeQueue + +--- + .../rocketmq/broker/BrokerController.java | 74 +- + .../broker/controller/ReplicasManager.java | 4 +- + .../offset/RocksDBConsumerOffsetManager.java | 8 +- + .../broker/processor/AckMessageProcessor.java | 2 +- + .../processor/AdminBrokerProcessor.java | 28 +- + .../ChangeInvisibleTimeProcessor.java | 4 +- + .../processor/PopBufferMergeService.java | 4 +- + .../broker/processor/PopMessageProcessor.java | 2 +- + .../broker/processor/PopReviveService.java | 6 +- + .../RocksDBSubscriptionGroupManager.java | 10 +- + .../topic/RocksDBTopicConfigManager.java | 8 +- + .../processor/PopReviveServiceTest.java | 4 +- + common/pom.xml | 2 +- + .../org/apache/rocketmq/common/MixAll.java | 1 + + .../rocketmq/common/attribute/CQType.java | 3 +- + .../common/config/AbstractRocksDBStorage.java | 112 +- + .../common/config/ConfigRocksDBStorage.java | 2 +- + .../common/message/MessageExtBrokerInner.java | 13 + + .../rocketmq/common/topic/TopicValidator.java | 2 + + .../rocketmq/common/utils/DataConverter.java | 2 +- + .../rocketmq/example/quickstart/Producer.java | 2 +- + metrics | 241 ++ + patch009-modify-CRLF-to-LF.patch | 127 + + patch010-backport-add-some-fixes | 1286 ++++++ + patch011-backport-optimize-config | 1390 +++++++ + ...optimize-opentelemetry-metric-config.patch | 2081 ++++++++++ + ...-backport-enhance-rockdbconfigtojson.patch | 2920 +++++++++++++ + patch013-backport-enhance-admin-output.patch | 892 ++++ + ...ueue-Selection-Strategy-Optimization.patch | 2023 +++++++++ + patch015-backport-fix-some-bugs.patch | 1894 +++++++++ + ...rt-Optimize-fault-tolerant-mechanism.patch | 520 +++ + ...port-Convergent-thread-pool-creation.patch | 2243 ++++++++++ + ...ckport-enhancement-of-tiered-storage.patch | 601 +++ + patch019-backport-some-bugfix.patch | 1499 +++++++ + patch020-backport-add-goaway-mechanism.patch | 3696 +++++++++++++++++ + patch021-backport-some-enhancements.patch | 344 ++ + pom.xml | 4 +- + store/pom.xml | 4 + + .../org/apache/rocketmq/store/CommitLog.java | 94 +- + .../rocketmq/store/CommitLogDispatcher.java | 5 +- + .../apache/rocketmq/store/ConsumeQueue.java | 93 +- + .../rocketmq/store/DefaultMessageStore.java | 279 +- + .../apache/rocketmq/store/MessageStore.java | 54 +- + .../rocketmq/store/RocksDBMessageStore.java | 169 + + .../apache/rocketmq/store/RunningFlags.java | 22 +- + .../store/config/MessageStoreConfig.java | 33 +- + .../store/dledger/DLedgerCommitLog.java | 53 +- + .../apache/rocketmq/store/ha/HAService.java | 3 +- + .../ha/autoswitch/AutoSwitchHAClient.java | 2 +- + .../ha/autoswitch/AutoSwitchHAService.java | 7 +- + .../plugin/AbstractPluginMessageStore.java | 42 +- + .../queue/AbstractConsumeQueueStore.java | 105 + + .../store/queue/BatchConsumeQueue.java | 20 + + .../store/queue/ConsumeQueueInterface.java | 27 +- + .../store/queue/ConsumeQueueStore.java | 293 +- + .../queue/ConsumeQueueStoreInterface.java | 289 ++ + .../rocketmq/store/queue/MultiDispatch.java | 76 + + .../store/queue/QueueOffsetOperator.java | 8 + + .../store/queue/RocksDBConsumeQueue.java | 437 ++ + .../queue/RocksDBConsumeQueueOffsetTable.java | 641 +++ + .../store/queue/RocksDBConsumeQueueStore.java | 441 ++ + .../store/queue/RocksDBConsumeQueueTable.java | 312 ++ + .../ConsumeQueueCompactionFilterFactory.java | 47 + + .../rocksdb/ConsumeQueueRocksDBStorage.java | 133 + + .../store/rocksdb/RocksDBOptionsFactory.java | 161 + + .../store/timer/TimerMessageStore.java | 65 +- + .../store/DefaultMessageStoreTest.java | 5 +- + .../rocketmq/store/MultiDispatchTest.java | 8 +- + .../store/RocksDBMessageStoreTest.java | 1060 +++++ + .../apache/rocketmq/store/StoreTestUtil.java | 9 +- + .../rocketmq/store/ha/HAServerTest.java | 16 +- + .../store/ha/autoswitch/AutoSwitchHATest.java | 3 +- + .../tieredstore/TieredMessageStore.java | 15 +- + .../ExportMetadataInRocksDBCommand.java | 8 +- + .../metadata/RocksDBConfigToJsonCommand.java | 118 + + 75 files changed, 26543 insertions(+), 668 deletions(-) + create mode 100644 metrics + create mode 100644 patch009-modify-CRLF-to-LF.patch + create mode 100644 patch010-backport-add-some-fixes + create mode 100644 patch011-backport-optimize-config + create mode 100644 patch011-backport-optimize-opentelemetry-metric-config.patch + create mode 100644 patch012-backport-enhance-rockdbconfigtojson.patch + create mode 100644 patch013-backport-enhance-admin-output.patch + create mode 100644 patch014-backport-Queue-Selection-Strategy-Optimization.patch + create mode 100644 patch015-backport-fix-some-bugs.patch + create mode 100644 patch016-backport-Optimize-fault-tolerant-mechanism.patch + create mode 100644 patch017-backport-Convergent-thread-pool-creation.patch + create mode 100644 patch018-backport-enhancement-of-tiered-storage.patch + create mode 100644 patch019-backport-some-bugfix.patch + create mode 100644 patch020-backport-add-goaway-mechanism.patch + create mode 100644 patch021-backport-some-enhancements.patch + create mode 100644 store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java + create mode 100644 store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java + create mode 100644 store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java + create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +index d4bded600..9f1fd0ad0 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +@@ -16,7 +16,33 @@ + */ + package org.apache.rocketmq.broker; + ++import java.io.IOException; ++import java.net.InetSocketAddress; ++import java.util.AbstractMap; ++import java.util.ArrayList; ++import java.util.Arrays; ++import java.util.Collections; ++import java.util.HashMap; ++import java.util.List; ++import java.util.Map; ++import java.util.Objects; ++import java.util.Optional; ++import java.util.concurrent.BlockingQueue; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.CountDownLatch; ++import java.util.concurrent.ExecutorService; ++import java.util.concurrent.LinkedBlockingQueue; ++import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.ScheduledFuture; ++import java.util.concurrent.TimeUnit; ++import java.util.concurrent.locks.Lock; ++import java.util.concurrent.locks.ReentrantLock; ++import java.util.function.Function; ++import java.util.stream.Collectors; ++ + import com.google.common.collect.Lists; ++ + import org.apache.rocketmq.acl.AccessValidator; + import org.apache.rocketmq.acl.plain.PlainAccessValidator; + import org.apache.rocketmq.broker.client.ClientHousekeepingService; +@@ -126,7 +152,7 @@ import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.MessageArrivingListener; + import org.apache.rocketmq.store.MessageStore; + import org.apache.rocketmq.store.PutMessageResult; +-import org.apache.rocketmq.store.StoreType; ++import org.apache.rocketmq.store.RocksDBMessageStore; + import org.apache.rocketmq.store.config.BrokerRole; + import org.apache.rocketmq.store.config.MessageStoreConfig; + import org.apache.rocketmq.store.dledger.DLedgerCommitLog; +@@ -141,31 +167,6 @@ import org.apache.rocketmq.store.timer.TimerCheckpoint; + import org.apache.rocketmq.store.timer.TimerMessageStore; + import org.apache.rocketmq.store.timer.TimerMetrics; + +-import java.io.IOException; +-import java.net.InetSocketAddress; +-import java.util.AbstractMap; +-import java.util.ArrayList; +-import java.util.Arrays; +-import java.util.Collections; +-import java.util.HashMap; +-import java.util.List; +-import java.util.Map; +-import java.util.Objects; +-import java.util.Optional; +-import java.util.concurrent.BlockingQueue; +-import java.util.concurrent.ConcurrentHashMap; +-import java.util.concurrent.ConcurrentMap; +-import java.util.concurrent.CountDownLatch; +-import java.util.concurrent.ExecutorService; +-import java.util.concurrent.LinkedBlockingQueue; +-import java.util.concurrent.ScheduledExecutorService; +-import java.util.concurrent.ScheduledFuture; +-import java.util.concurrent.TimeUnit; +-import java.util.concurrent.locks.Lock; +-import java.util.concurrent.locks.ReentrantLock; +-import java.util.function.Function; +-import java.util.stream.Collectors; +- + public class BrokerController { + protected static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); + private static final Logger LOG_PROTECTION = LoggerFactory.getLogger(LoggerName.PROTECTION_LOGGER_NAME); +@@ -308,7 +309,7 @@ public class BrokerController { + this.setStoreHost(new InetSocketAddress(this.getBrokerConfig().getBrokerIP1(), getListenPort())); + this.brokerStatsManager = messageStoreConfig.isEnableLmq() ? new LmqBrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()) : new BrokerStatsManager(this.brokerConfig.getBrokerClusterName(), this.brokerConfig.isEnableDetailStat()); + this.broadcastOffsetManager = new BroadcastOffsetManager(this); +- if (isEnableRocksDBStore()) { ++ if (this.messageStoreConfig.isEnableRocksDBStore()) { + this.topicConfigManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqTopicConfigManager(this) : new RocksDBTopicConfigManager(this); + this.subscriptionGroupManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqSubscriptionGroupManager(this) : new RocksDBSubscriptionGroupManager(this); + this.consumerOffsetManager = messageStoreConfig.isEnableLmq() ? new RocksDBLmqConsumerOffsetManager(this) : new RocksDBConsumerOffsetManager(this); +@@ -747,7 +748,12 @@ public class BrokerController { + public boolean initializeMessageStore() { + boolean result = true; + try { +- DefaultMessageStore defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); ++ DefaultMessageStore defaultMessageStore; ++ if (this.messageStoreConfig.isEnableRocksDBStore()) { ++ defaultMessageStore = new RocksDBMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); ++ } else { ++ defaultMessageStore = new DefaultMessageStore(this.messageStoreConfig, this.brokerStatsManager, this.messageArrivingListener, this.brokerConfig, topicConfigManager.getTopicConfigTable()); ++ } + + if (messageStoreConfig.isEnableDLegerCommitLog()) { + DLedgerRoleChangeHandler roleChangeHandler = +@@ -944,16 +950,16 @@ public class BrokerController { + this.transactionalMessageService = ServiceProvider.loadClass(TransactionalMessageService.class); + if (null == this.transactionalMessageService) { + this.transactionalMessageService = new TransactionalMessageServiceImpl( +- new TransactionalMessageBridge(this, this.getMessageStore())); ++ new TransactionalMessageBridge(this, this.getMessageStore())); + LOG.warn("Load default transaction message hook service: {}", +- TransactionalMessageServiceImpl.class.getSimpleName()); ++ TransactionalMessageServiceImpl.class.getSimpleName()); + } + this.transactionalMessageCheckListener = ServiceProvider.loadClass( +- AbstractTransactionalMessageCheckListener.class); ++ AbstractTransactionalMessageCheckListener.class); + if (null == this.transactionalMessageCheckListener) { + this.transactionalMessageCheckListener = new DefaultTransactionalMessageCheckListener(); + LOG.warn("Load default discard message hook service: {}", +- DefaultTransactionalMessageCheckListener.class.getSimpleName()); ++ DefaultTransactionalMessageCheckListener.class.getSimpleName()); + } + this.transactionalMessageCheckListener.setBrokerController(this); + this.transactionalMessageCheckService = new TransactionalMessageCheckService(this); +@@ -2412,8 +2418,4 @@ public class BrokerController { + public void setColdDataCgCtrService(ColdDataCgCtrService coldDataCgCtrService) { + this.coldDataCgCtrService = coldDataCgCtrService; + } +- +- public boolean isEnableRocksDBStore() { +- return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.messageStoreConfig.getStoreType()); +- } + } +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +index a989e6e68..a1d711cb2 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +@@ -224,7 +224,7 @@ public class ReplicasManager { + + public synchronized void changeBrokerRole(final Long newMasterBrokerId, final String newMasterAddress, + final Integer newMasterEpoch, +- final Integer syncStateSetEpoch, final Set syncStateSet) { ++ final Integer syncStateSetEpoch, final Set syncStateSet) throws Exception { + if (newMasterBrokerId != null && newMasterEpoch > this.masterEpoch) { + if (newMasterBrokerId.equals(this.brokerControllerId)) { + changeToMaster(newMasterEpoch, syncStateSetEpoch, syncStateSet); +@@ -234,7 +234,7 @@ public class ReplicasManager { + } + } + +- public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set syncStateSet) { ++ public void changeToMaster(final int newMasterEpoch, final int syncStateSetEpoch, final Set syncStateSet) throws Exception { + synchronized (this) { + if (newMasterEpoch > this.masterEpoch) { + LOGGER.info("Begin to change to master, brokerName:{}, replicas:{}, new Epoch:{}", this.brokerConfig.getBrokerName(), this.brokerAddress, newMasterEpoch); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +index 5695a3356..05b53b0bc 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/offset/RocksDBConsumerOffsetManager.java +@@ -33,7 +33,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + + public RocksDBConsumerOffsetManager(BrokerController brokerController) { + super(brokerController); +- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); ++ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + } + + @Override +@@ -49,7 +49,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + @Override + protected void removeConsumerOffset(String topicAtGroup) { + try { +- byte[] keyBytes = topicAtGroup.getBytes(DataConverter.charset); ++ byte[] keyBytes = topicAtGroup.getBytes(DataConverter.CHARSET_UTF8); + this.rocksDBConfigManager.delete(keyBytes); + } catch (Exception e) { + LOG.error("kv remove consumerOffset Failed, {}", topicAtGroup); +@@ -58,7 +58,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + + @Override + protected void decode0(final byte[] key, final byte[] body) { +- String topicAtGroup = new String(key, DataConverter.charset); ++ String topicAtGroup = new String(key, DataConverter.CHARSET_UTF8); + RocksDBOffsetSerializeWrapper wrapper = JSON.parseObject(body, RocksDBOffsetSerializeWrapper.class); + + this.offsetTable.put(topicAtGroup, wrapper.getOffsetTable()); +@@ -93,7 +93,7 @@ public class RocksDBConsumerOffsetManager extends ConsumerOffsetManager { + } + + private void putWriteBatch(final WriteBatch writeBatch, final String topicGroupName, final ConcurrentMap offsetMap) throws Exception { +- byte[] keyBytes = topicGroupName.getBytes(DataConverter.charset); ++ byte[] keyBytes = topicGroupName.getBytes(DataConverter.CHARSET_UTF8); + RocksDBOffsetSerializeWrapper wrapper = new RocksDBOffsetSerializeWrapper(); + wrapper.setOffsetTable(offsetMap); + byte[] valueBytes = JSON.toJSONBytes(wrapper, SerializerFeature.BrowserCompatible); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +index 244b459d6..59a3e63b2 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +@@ -253,7 +253,7 @@ public class AckMessageProcessor implements NettyRequestProcessor { + + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(reviveTopic); +- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(rqId); + if (ackMsg instanceof BatchAckMsg) { + msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +index e77120e15..dd4ec960f 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +@@ -539,14 +539,18 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + + final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); + // delete pop retry topics first +- for (String group : groups) { +- final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); +- if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { +- deleteTopicInBroker(popRetryTopic); ++ try { ++ for (String group : groups) { ++ final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); ++ if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { ++ deleteTopicInBroker(popRetryTopic); ++ } + } ++ // delete topic ++ deleteTopicInBroker(topic); ++ } catch (Throwable t) { ++ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); + } +- // delete topic +- deleteTopicInBroker(topic); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); + return response; +@@ -2081,7 +2085,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + public RemotingCommand cleanExpiredConsumeQueue() { + LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: start."); + final RemotingCommand response = RemotingCommand.createResponseCommand(null); +- brokerController.getMessageStore().cleanExpiredConsumerQueue(); ++ try { ++ brokerController.getMessageStore().cleanExpiredConsumerQueue(); ++ } catch (Throwable t) { ++ return buildErrorResponse(ResponseCode.SYSTEM_ERROR, t.getMessage()); ++ } + LOGGER.info("AdminBrokerProcessor#cleanExpiredConsumeQueue: end."); + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); +@@ -2781,7 +2789,11 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { + + final ReplicasManager replicasManager = this.brokerController.getReplicasManager(); + if (replicasManager != null) { +- replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet()); ++ try { ++ replicasManager.changeBrokerRole(requestHeader.getMasterBrokerId(), requestHeader.getMasterAddress(), requestHeader.getMasterEpoch(), requestHeader.getSyncStateSetEpoch(), syncStateSetInfo.getSyncStateSet()); ++ } catch (Exception e) { ++ throw new RemotingCommandException(e.getMessage()); ++ } + } + response.setCode(ResponseCode.SUCCESS); + response.setRemark(null); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +index 2ccdf07f6..bdfffff09 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ChangeInvisibleTimeProcessor.java +@@ -180,7 +180,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { + } + + msgInner.setTopic(reviveTopic); +- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(rqId); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +@@ -216,7 +216,7 @@ public class ChangeInvisibleTimeProcessor implements NettyRequestProcessor { + ck.addDiff(0); + ck.setBrokerName(brokerName); + +- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(reviveQid); + msgInner.setTags(PopAckConstants.CK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java +index b7ba8ad4a..8a85dd8fe 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopBufferMergeService.java +@@ -633,7 +633,7 @@ public class PopBufferMergeService extends ServiceThread { + ackMsg.setQueueId(point.getQueueId()); + ackMsg.setPopTime(point.getPopTime()); + msgInner.setTopic(popMessageProcessor.reviveTopic); +- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(pointWrapper.getReviveQueueId()); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +@@ -673,7 +673,7 @@ public class PopBufferMergeService extends ServiceThread { + batchAckMsg.setQueueId(point.getQueueId()); + batchAckMsg.setPopTime(point.getPopTime()); + msgInner.setTopic(popMessageProcessor.reviveTopic); +- msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(batchAckMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(pointWrapper.getReviveQueueId()); + msgInner.setTags(PopAckConstants.BATCH_ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +index 0d9bdf143..f5d07c5aa 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +@@ -685,7 +685,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + + msgInner.setTopic(reviveTopic); +- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(reviveQid); + msgInner.setTags(PopAckConstants.CK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +index d5174d3d1..4f80752e1 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +@@ -356,7 +356,7 @@ public class PopReviveService extends ServiceThread { + } + for (MessageExt messageExt : messageExts) { + if (PopAckConstants.CK_TAG.equals(messageExt.getTags())) { +- String raw = new String(messageExt.getBody(), DataConverter.charset); ++ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={},find ck, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } +@@ -371,7 +371,7 @@ public class PopReviveService extends ServiceThread { + firstRt = point.getReviveTime(); + } + } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) { +- String raw = new String(messageExt.getBody(), DataConverter.charset); ++ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={},find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } +@@ -395,7 +395,7 @@ public class PopReviveService extends ServiceThread { + } + } + } else if (PopAckConstants.BATCH_ACK_TAG.equals(messageExt.getTags())) { +- String raw = new String(messageExt.getBody(), DataConverter.charset); ++ String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + if (brokerController.getBrokerConfig().isEnablePopLog()) { + POP_LOGGER.info("reviveQueueId={}, find batch ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +index 6503970af..e9a81a8d6 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/RocksDBSubscriptionGroupManager.java +@@ -30,7 +30,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + + public RocksDBSubscriptionGroupManager(BrokerController brokerController) { + super(brokerController, false); +- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); ++ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + } + + @Override +@@ -53,7 +53,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); + + try { +- byte[] keyBytes = groupName.getBytes(DataConverter.charset); ++ byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8); + byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible); + this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); + } catch (Exception e) { +@@ -68,7 +68,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + SubscriptionGroupConfig oldConfig = this.subscriptionGroupTable.putIfAbsent(groupName, subscriptionGroupConfig); + if (oldConfig == null) { + try { +- byte[] keyBytes = groupName.getBytes(DataConverter.charset); ++ byte[] keyBytes = groupName.getBytes(DataConverter.CHARSET_UTF8); + byte[] valueBytes = JSON.toJSONBytes(subscriptionGroupConfig, SerializerFeature.BrowserCompatible); + this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); + } catch (Exception e) { +@@ -82,7 +82,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + protected SubscriptionGroupConfig removeSubscriptionGroupConfig(String groupName) { + SubscriptionGroupConfig subscriptionGroupConfig = this.subscriptionGroupTable.remove(groupName); + try { +- this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.charset)); ++ this.rocksDBConfigManager.delete(groupName.getBytes(DataConverter.CHARSET_UTF8)); + } catch (Exception e) { + log.error("kv delete sub Failed, {}", subscriptionGroupConfig.toString()); + } +@@ -91,7 +91,7 @@ public class RocksDBSubscriptionGroupManager extends SubscriptionGroupManager { + + @Override + protected void decode0(byte[] key, byte[] body) { +- String groupName = new String(key, DataConverter.charset); ++ String groupName = new String(key, DataConverter.CHARSET_UTF8); + SubscriptionGroupConfig subscriptionGroupConfig = JSON.parseObject(body, SubscriptionGroupConfig.class); + + this.subscriptionGroupTable.put(groupName, subscriptionGroupConfig); +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +index 7da0d7c8a..fddecf2d9 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/RocksDBTopicConfigManager.java +@@ -30,7 +30,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { + + public RocksDBTopicConfigManager(BrokerController brokerController) { + super(brokerController, false); +- this.rocksDBConfigManager = new RocksDBConfigManager(this.brokerController.getMessageStoreConfig().getMemTableFlushInterval()); ++ this.rocksDBConfigManager = new RocksDBConfigManager(brokerController.getMessageStoreConfig().getMemTableFlushIntervalMs()); + } + + @Override +@@ -49,7 +49,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { + + @Override + protected void decode0(byte[] key, byte[] body) { +- String topicName = new String(key, DataConverter.charset); ++ String topicName = new String(key, DataConverter.CHARSET_UTF8); + TopicConfig topicConfig = JSON.parseObject(body, TopicConfig.class); + + this.topicConfigTable.put(topicName, topicConfig); +@@ -66,7 +66,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { + String topicName = topicConfig.getTopicName(); + TopicConfig oldTopicConfig = this.topicConfigTable.put(topicName, topicConfig); + try { +- byte[] keyBytes = topicName.getBytes(DataConverter.charset); ++ byte[] keyBytes = topicName.getBytes(DataConverter.CHARSET_UTF8); + byte[] valueBytes = JSON.toJSONBytes(topicConfig, SerializerFeature.BrowserCompatible); + this.rocksDBConfigManager.put(keyBytes, keyBytes.length, valueBytes); + } catch (Exception e) { +@@ -79,7 +79,7 @@ public class RocksDBTopicConfigManager extends TopicConfigManager { + protected TopicConfig removeTopicConfig(String topicName) { + TopicConfig topicConfig = this.topicConfigTable.remove(topicName); + try { +- this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.charset)); ++ this.rocksDBConfigManager.delete(topicName.getBytes(DataConverter.CHARSET_UTF8)); + } catch (Exception e) { + log.error("kv remove topic Failed, {}", topicConfig.toString()); + } +diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java +index 1c3a0cd45..78b76264f 100644 +--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java ++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/PopReviveServiceTest.java +@@ -234,7 +234,7 @@ public class PopReviveServiceTest { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + + msgInner.setTopic(REVIVE_TOPIC); +- msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ck).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(REVIVE_QUEUE_ID); + msgInner.setTags(PopAckConstants.CK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +@@ -269,7 +269,7 @@ public class PopReviveServiceTest { + SocketAddress host, long deliverMs, String ackUniqueId) { + MessageExtBrokerInner msgInner = new MessageExtBrokerInner(); + msgInner.setTopic(reviveTopic); +- msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.charset)); ++ msgInner.setBody(JSON.toJSONString(ackMsg).getBytes(DataConverter.CHARSET_UTF8)); + msgInner.setQueueId(reviveQid); + msgInner.setTags(PopAckConstants.ACK_TAG); + msgInner.setBornTimestamp(System.currentTimeMillis()); +diff --git a/common/pom.xml b/common/pom.xml +index 6104c3ac6..a28ed228f 100644 +--- a/common/pom.xml ++++ b/common/pom.xml +@@ -109,7 +109,7 @@ + rocketmq-logback-classic + + +- io.github.aliyunmq ++ org.apache.rocketmq + rocketmq-rocksdb + + +diff --git a/common/src/main/java/org/apache/rocketmq/common/MixAll.java b/common/src/main/java/org/apache/rocketmq/common/MixAll.java +index 1233a5422..407ef2842 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/MixAll.java ++++ b/common/src/main/java/org/apache/rocketmq/common/MixAll.java +@@ -492,6 +492,7 @@ public class MixAll { + public static int compareLong(long x, long y) { + return Long.compare(x, y); + } ++ + public static boolean isLmq(String lmqMetaData) { + return lmqMetaData != null && lmqMetaData.startsWith(LMQ_PREFIX); + } +diff --git a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java +index 73ef21880..9148d5a18 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java ++++ b/common/src/main/java/org/apache/rocketmq/common/attribute/CQType.java +@@ -19,5 +19,6 @@ package org.apache.rocketmq.common.attribute; + + public enum CQType { + SimpleCQ, +- BatchCQ ++ BatchCQ, ++ RocksDBCQ + } +diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +index 6f19a9815..20319abba 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +@@ -17,7 +17,6 @@ + package org.apache.rocketmq.common.config; + + import java.nio.ByteBuffer; +-import java.nio.charset.Charset; + import java.util.ArrayList; + import java.util.List; + import java.util.Map; +@@ -27,11 +26,11 @@ import java.util.concurrent.Semaphore; + import java.util.concurrent.ThreadPoolExecutor; + import java.util.concurrent.TimeUnit; + +-import com.google.common.collect.Lists; + import com.google.common.collect.Maps; + + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.utils.DataConverter; + import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +@@ -47,7 +46,6 @@ import org.rocksdb.Priority; + import org.rocksdb.ReadOptions; + import org.rocksdb.RocksDB; + import org.rocksdb.RocksDBException; +-import org.rocksdb.RocksIterator; + import org.rocksdb.Statistics; + import org.rocksdb.Status; + import org.rocksdb.WriteBatch; +@@ -58,7 +56,6 @@ import static org.rocksdb.RocksDB.NOT_FOUND; + public abstract class AbstractRocksDBStorage { + protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); + +- private static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); + private static final String SPACE = " | "; + + protected String dbPath; +@@ -223,10 +220,6 @@ public abstract class AbstractRocksDBStorage { + } + } + +- protected WrappedRocksIterator newIterator(ColumnFamilyHandle cfHandle, ReadOptions readOptions) { +- return new WrappedRocksIterator(this.db.newIterator(cfHandle, readOptions)); +- } +- + protected void rangeDelete(ColumnFamilyHandle cfHandle, WriteOptions writeOptions, + final byte[] startKey, final byte[] endKey) throws RocksDBException { + if (!hold()) { +@@ -243,46 +236,6 @@ public abstract class AbstractRocksDBStorage { + } + } + +- protected void manualCompactionDefaultCfMaxLevel(final CompactionOptions compactionOptions) throws Exception { +- final ColumnFamilyHandle defaultCFHandle = this.defaultCFHandle; +- final byte[] defaultCFName = defaultCFHandle.getName(); +- List fileMetaDataList = this.db.getLiveFilesMetaData(); +- if (fileMetaDataList == null || fileMetaDataList.isEmpty()) { +- return; +- } +- +- List defaultLiveFileDataList = Lists.newArrayList(); +- List inputFileNames = Lists.newArrayList(); +- int maxLevel = 0; +- for (LiveFileMetaData fileMetaData : fileMetaDataList) { +- if (compareTo(fileMetaData.columnFamilyName(), defaultCFName) != 0) { +- continue; +- } +- defaultLiveFileDataList.add(fileMetaData); +- if (fileMetaData.level() > maxLevel) { +- maxLevel = fileMetaData.level(); +- } +- } +- if (maxLevel == 0) { +- LOGGER.info("manualCompactionDefaultCfFiles skip level 0."); +- return; +- } +- +- for (LiveFileMetaData fileMetaData : defaultLiveFileDataList) { +- if (fileMetaData.level() != maxLevel || fileMetaData.beingCompacted()) { +- continue; +- } +- inputFileNames.add(fileMetaData.path() + fileMetaData.fileName()); +- } +- if (!inputFileNames.isEmpty()) { +- List outputLists = this.db.compactFiles(compactionOptions, defaultCFHandle, +- inputFileNames, maxLevel, -1, null); +- LOGGER.info("manualCompactionDefaultCfFiles OK. src: {}, dst: {}", inputFileNames, outputLists); +- } else { +- LOGGER.info("manualCompactionDefaultCfFiles Empty."); +- } +- } +- + protected void manualCompactionDefaultCfRange(CompactRangeOptions compactRangeOptions) { + if (!hold()) { + return; +@@ -494,50 +447,6 @@ public abstract class AbstractRocksDBStorage { + this.db.flushWal(true); + } + +- protected class WrappedRocksIterator { +- private final RocksIterator iterator; +- +- public WrappedRocksIterator(final RocksIterator iterator) { +- this.iterator = iterator; +- } +- +- public byte[] key() { +- return iterator.key(); +- } +- +- public byte[] value() { +- return iterator.value(); +- } +- +- public void next() { +- iterator.next(); +- } +- +- public void prev() { +- iterator.prev(); +- } +- +- public void seek(byte[] target) { +- iterator.seek(target); +- } +- +- public void seekForPrev(byte[] target) { +- iterator.seekForPrev(target); +- } +- +- public void seekToFirst() { +- iterator.seekToFirst(); +- } +- +- public boolean isValid() { +- return iterator.isValid(); +- } +- +- public void close() { +- iterator.close(); +- } +- } +- + private String getStatusError(RocksDBException e) { + if (e == null || e.getStatus() == null) { + return "null"; +@@ -574,7 +483,7 @@ public abstract class AbstractRocksDBStorage { + sb = new StringBuilder(256); + map.put(metaData.level(), sb); + } +- sb.append(new String(metaData.columnFamilyName(), CHARSET_UTF8)).append(SPACE). ++ sb.append(new String(metaData.columnFamilyName(), DataConverter.CHARSET_UTF8)).append(SPACE). + append(metaData.fileName()).append(SPACE). + append("s: ").append(metaData.size()).append(SPACE). + append("a: ").append(metaData.numEntries()).append(SPACE). +@@ -595,21 +504,4 @@ public abstract class AbstractRocksDBStorage { + } catch (Exception ignored) { + } + } +- +- public int compareTo(byte[] v1, byte[] v2) { +- int len1 = v1.length; +- int len2 = v2.length; +- int lim = Math.min(len1, len2); +- +- int k = 0; +- while (k < lim) { +- byte c1 = v1[k]; +- byte c2 = v2[k]; +- if (c1 != c2) { +- return c1 - c2; +- } +- k++; +- } +- return len1 - len2; +- } + } +\ No newline at end of file +diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +index 463bd8fed..b40f8046e 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java ++++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +@@ -203,7 +203,7 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage { + setUseDirectReads(true); + } + +- private static String getDBLogDir() { ++ public static String getDBLogDir() { + String rootPath = System.getProperty("user.home"); + if (StringUtils.isEmpty(rootPath)) { + return ""; +diff --git a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java +index 0c72ebb7b..91599653c 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java ++++ b/common/src/main/java/org/apache/rocketmq/common/message/MessageExtBrokerInner.java +@@ -70,4 +70,17 @@ public class MessageExtBrokerInner extends MessageExt { + public void setVersion(MessageVersion version) { + this.version = version; + } ++ ++ public void removeWaitStorePropertyString() { ++ if (this.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) { ++ // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message. ++ // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it. ++ String waitStoreMsgOKValue = this.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); ++ this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties())); ++ // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later ++ this.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue); ++ } else { ++ this.setPropertiesString(MessageDecoder.messageProperties2String(this.getProperties())); ++ } ++ } + } +diff --git a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java +index 61265c05d..c19592a44 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java ++++ b/common/src/main/java/org/apache/rocketmq/common/topic/TopicValidator.java +@@ -31,6 +31,7 @@ public class TopicValidator { + public static final String RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC = "TRANS_CHECK_MAX_TIME_TOPIC"; + public static final String RMQ_SYS_SELF_TEST_TOPIC = "SELF_TEST_TOPIC"; + public static final String RMQ_SYS_OFFSET_MOVED_EVENT = "OFFSET_MOVED_EVENT"; ++ public static final String RMQ_SYS_ROCKSDB_OFFSET_TOPIC = "CHECKPOINT_TOPIC"; + + public static final String SYSTEM_TOPIC_PREFIX = "rmq_sys_"; + public static final String SYNC_BROKER_MEMBER_GROUP_PREFIX = SYSTEM_TOPIC_PREFIX + "SYNC_BROKER_MEMBER_"; +@@ -55,6 +56,7 @@ public class TopicValidator { + SYSTEM_TOPIC_SET.add(RMQ_SYS_TRANS_CHECK_MAX_TIME_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_SELF_TEST_TOPIC); + SYSTEM_TOPIC_SET.add(RMQ_SYS_OFFSET_MOVED_EVENT); ++ SYSTEM_TOPIC_SET.add(RMQ_SYS_ROCKSDB_OFFSET_TOPIC); + + NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_SCHEDULE_TOPIC); + NOT_ALLOWED_SEND_TOPIC_SET.add(RMQ_SYS_TRANS_HALF_TOPIC); +diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java +index 8b50de12b..cc96770b2 100644 +--- a/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java ++++ b/common/src/main/java/org/apache/rocketmq/common/utils/DataConverter.java +@@ -20,7 +20,7 @@ import java.nio.ByteBuffer; + import java.nio.charset.Charset; + + public class DataConverter { +- public static Charset charset = Charset.forName("UTF-8"); ++ public static final Charset CHARSET_UTF8 = Charset.forName("UTF-8"); + + public static byte[] Long2Byte(Long v) { + ByteBuffer tmp = ByteBuffer.allocate(8); +diff --git a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java +index aac295030..8662328ea 100644 +--- a/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java ++++ b/example/src/main/java/org/apache/rocketmq/example/quickstart/Producer.java +@@ -71,7 +71,7 @@ public class Producer { + TAG /* Tag */, + ("Hello RocketMQ " + i).getBytes(RemotingHelper.DEFAULT_CHARSET) /* Message body */ + ); +- ++ msg.setDelayTimeLevel(1); + /* + * Call send message to deliver message to one of brokers. + */ +diff --git a/metrics b/metrics +new file mode 100644 +index 000000000..146e90fae +--- /dev/null ++++ b/metrics +@@ -0,0 +1,241 @@ ++# TYPE target info ++# HELP target Target metadata ++target_info{} 1 ++# TYPE otel_scope_info info ++# HELP otel_scope_info Scope metadata ++otel_scope_info{otel_scope_name="broker-meter"} 1 ++# TYPE rocketmq_storage_message_reserve_time gauge ++# HELP rocketmq_storage_message_reserve_time default view for gauge. ++rocketmq_storage_message_reserve_time_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1.9491132515E10 1696905287361 ++# TYPE rocketmq_timer_enqueue_latency gauge ++# HELP rocketmq_timer_enqueue_latency default view for gauge. ++rocketmq_timer_enqueue_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_timer_enqueue_lag gauge ++# HELP rocketmq_timer_enqueue_lag default view for gauge. ++rocketmq_timer_enqueue_lag{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_timing_messages gauge ++# HELP rocketmq_timing_messages default view for gauge. ++rocketmq_timing_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local",topic="BenchmarkTest"} 0.0 1696905287361 ++# TYPE rocketmq_storage_flush_behind_bytes gauge ++# HELP rocketmq_storage_flush_behind_bytes default view for gauge. ++rocketmq_storage_flush_behind_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1230.0 1696905287361 ++# TYPE rocketmq_processor_watermark gauge ++# HELP rocketmq_processor_watermark default view for gauge. ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="reply"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="transaction"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="send"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="async_put"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="lite_pull"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="pull"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="heartbeat"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="consumer_manager"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="admin"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="client_manager"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="query_message"} 0.0 1696905287361 ++rocketmq_processor_watermark{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",processor="ack"} 0.0 1696905287361 ++# TYPE rocketmq_pop_buffer_scan_time_consume histogram ++# HELP rocketmq_pop_buffer_scan_time_consume Time consuming of pop buffer scan ++rocketmq_pop_buffer_scan_time_consume_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 5267005.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 39651.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="1.0"} 5266935.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="10.0"} 5266998.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="100.0"} 5267001.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="1000.0"} 5267005.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="2000.0"} 5267005.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="3000.0"} 5267005.0 1696905287361 ++rocketmq_pop_buffer_scan_time_consume_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",le="+Inf"} 5267005.0 1696905287361 ++# TYPE rocketmq_pop_revive_lag gauge ++# HELP rocketmq_pop_revive_lag default view for gauge. ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="1"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="2"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="3"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="4"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="5"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="6"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="7"} 1.0 1696905287361 ++rocketmq_pop_revive_lag_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="0"} 1.0 1696905287361 ++# TYPE rocketmq_pop_revive_latency gauge ++# HELP rocketmq_pop_revive_latency default view for gauge. ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="1"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="2"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="3"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="4"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="5"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="6"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="7"} 0.0 1696905287361 ++rocketmq_pop_revive_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",queue_id="0"} 0.0 1696905287361 ++# TYPE rocketmq_storage_size gauge ++# HELP rocketmq_storage_size default view for gauge. ++rocketmq_storage_size_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 1.5905103872E10 1696905287361 ++# TYPE rocketmq_pop_offset_buffer_size gauge ++# HELP rocketmq_pop_offset_buffer_size default view for gauge. ++rocketmq_pop_offset_buffer_size{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 0.0 1696905287361 ++# TYPE rocketmq_timer_dequeue_latency gauge ++# HELP rocketmq_timer_dequeue_latency default view for gauge. ++rocketmq_timer_dequeue_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_broker_permission gauge ++# HELP rocketmq_broker_permission default view for gauge. ++rocketmq_broker_permission{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 6.0 1696905287361 ++# TYPE rocketmq_consumer_lag_messages gauge ++# HELP rocketmq_consumer_lag_messages default view for gauge. ++rocketmq_consumer_lag_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_throughput_out_total counter ++# HELP rocketmq_throughput_out_total default view for counter. ++rocketmq_throughput_out_total{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 ++# TYPE rocketmq_timer_dequeue_lag gauge ++# HELP rocketmq_timer_dequeue_lag default view for gauge. ++rocketmq_timer_dequeue_lag{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_messages_out_total counter ++# HELP rocketmq_messages_out_total default view for counter. ++rocketmq_messages_out_total{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 ++# TYPE rocketmq_messages_in_total counter ++# HELP rocketmq_messages_in_total default view for counter. ++rocketmq_messages_in_total{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 ++# TYPE rocketmq_consumer_queueing_latency gauge ++# HELP rocketmq_consumer_queueing_latency default view for gauge. ++rocketmq_consumer_queueing_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_consumer_connections gauge ++# HELP rocketmq_consumer_connections default view for gauge. ++rocketmq_consumer_connections{otel_scope_name="broker-meter",cluster="DefaultCluster",consume_mode="push",consumer_group="MQ_INST_test%GID_222",is_system="false",language="java",node_id="broker-a",node_type="broker",protocol_type="remoting",version="v4_8_0"} 1.0 1696905287361 ++# TYPE rocketmq_rpc_latency histogram ++# HELP rocketmq_rpc_latency Rpc latency ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success"} 128.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="1.0"} 17.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="3.0"} 18.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="5.0"} 19.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="7.0"} 19.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="10.0"} 20.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="100.0"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="1000.0"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="2000.0"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="3000.0"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="heart_beat",response_code="success",result="success",le="+Inf"} 24.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success"} 8.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="1.0"} 3.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="3.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="5.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="7.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="10.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="100.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="1000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="2000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="3000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="query_consumer_offset",response_code="success",result="success",le="+Inf"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="1.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="3.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="5.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="7.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="10.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="100.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="1000.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="2000.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="3000.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="subscription_not_latest",result="success",le="+Inf"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success"} 108.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="1.0"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="3.0"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="5.0"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="7.0"} 54.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="10.0"} 55.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="100.0"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="1000.0"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="2000.0"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="3000.0"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="get_consumer_list_by_group",response_code="success",result="success",le="+Inf"} 60.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success"} 210.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="1.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="3.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="5.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="7.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="10.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="100.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="1000.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="2000.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="3000.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="success",result="success",le="+Inf"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success"} 155.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success"} 2811756.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="1.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="3.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="5.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="7.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="10.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="100.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="1000.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="2000.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="3000.0"} 0.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="true",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="pull_message",response_code="pull_not_found",result="success",le="+Inf"} 155.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success"} 225.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="1.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="3.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="5.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="7.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="10.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="100.0"} 4.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="1000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="2000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="3000.0"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="send_message_v2",response_code="success",result="success",le="+Inf"} 5.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway"} 84.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="1.0"} 551.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="3.0"} 557.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="5.0"} 558.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="7.0"} 560.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="10.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="100.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="1000.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="2000.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="3000.0"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="update_consumer_offset",response_code="success",result="oneway",le="+Inf"} 565.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success"} 6.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="1.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="3.0"} 1.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="5.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="7.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="10.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="100.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="1000.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="2000.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="3000.0"} 2.0 1696905287361 ++rocketmq_rpc_latency_milliseconds_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_long_polling="false",node_id="broker-a",node_type="broker",protocol_type="remoting",request_code="unregister_client",response_code="success",result="success",le="+Inf"} 2.0 1696905287361 ++# TYPE rocketmq_storage_dispatch_behind_bytes gauge ++# HELP rocketmq_storage_dispatch_behind_bytes default view for gauge. ++rocketmq_storage_dispatch_behind_bytes{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker",storage_medium="disk",storage_type="local"} 0.0 1696905287361 ++# TYPE rocketmq_consumer_ready_messages gauge ++# HELP rocketmq_consumer_ready_messages default view for gauge. ++rocketmq_consumer_ready_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_consumer_lag_latency gauge ++# HELP rocketmq_consumer_lag_latency default view for gauge. ++rocketmq_consumer_lag_latency_milliseconds{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_throughput_in_total counter ++# HELP rocketmq_throughput_in_total default view for counter. ++rocketmq_throughput_in_total{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 ++# TYPE rocketmq_pop_checkpoint_buffer_size gauge ++# HELP rocketmq_pop_checkpoint_buffer_size default view for gauge. ++rocketmq_pop_checkpoint_buffer_size{otel_scope_name="broker-meter",cluster="DefaultCluster",node_id="broker-a",node_type="broker"} 0.0 1696905287361 ++# TYPE rocketmq_message_size histogram ++# HELP rocketmq_message_size Incoming messages size ++rocketmq_message_size_count{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 5.0 1696905287361 ++rocketmq_message_size_sum{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 1230.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="1024.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="4096.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="524288.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="1048576.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="2097152.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="4194304.0"} 5.0 1696905287361 ++rocketmq_message_size_bucket{otel_scope_name="broker-meter",cluster="DefaultCluster",is_system="false",message_type="normal",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl",le="+Inf"} 5.0 1696905287361 ++# TYPE rocketmq_consumer_inflight_messages gauge ++# HELP rocketmq_consumer_inflight_messages default view for gauge. ++rocketmq_consumer_inflight_messages{otel_scope_name="broker-meter",cluster="DefaultCluster",consumer_group="MQ_INST_test%GID_222",is_retry="false",is_system="false",node_id="broker-a",node_type="broker",topic="MQ_INST_test%zzl"} 0.0 1696905287361 ++# TYPE rocketmq_producer_connections gauge ++# HELP rocketmq_producer_connections default view for gauge. ++rocketmq_producer_connections{otel_scope_name="broker-meter",cluster="DefaultCluster",language="java",node_id="broker-a",node_type="broker",protocol_type="remoting",version="v4_8_0"} 1.0 1696905287361 +diff --git a/patch009-modify-CRLF-to-LF.patch b/patch009-modify-CRLF-to-LF.patch +new file mode 100644 +index 000000000..c9f1f1843 +--- /dev/null ++++ b/patch009-modify-CRLF-to-LF.patch +@@ -0,0 +1,127 @@ ++From 81cd734ee33586544694675bd1081f2c81024d22 Mon Sep 17 00:00:00 2001 ++From: zhiliatom ++Date: Thu, 28 Sep 2023 09:58:13 +0800 ++Subject: [PATCH] modify line separater from CRLF to LF ++ ++--- ++ .../rocketmq/common/constant/LoggerName.java | 110 +++++++++--------- ++ 1 file changed, 55 insertions(+), 55 deletions(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++index cb04b00b3..5c7afab9f 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +++++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++@@ -1,55 +1,55 @@ ++-/* ++- * Licensed to the Apache Software Foundation (ASF) under one or more ++- * contributor license agreements. See the NOTICE file distributed with ++- * this work for additional information regarding copyright ownership. ++- * The ASF licenses this file to You under the Apache License, Version 2.0 ++- * (the "License"); you may not use this file except in compliance with ++- * the License. You may obtain a copy of the License at ++- * ++- * http://www.apache.org/licenses/LICENSE-2.0 ++- * ++- * Unless required by applicable law or agreed to in writing, software ++- * distributed under the License is distributed on an "AS IS" BASIS, ++- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++- * See the License for the specific language governing permissions and ++- * limitations under the License. ++- */ ++-package org.apache.rocketmq.common.constant; ++- ++-public class LoggerName { ++- public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv"; ++- public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; ++- public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; ++- public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; ++- public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; ++- public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; ++- public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; ++- public static final String CLIENT_LOGGER_NAME = "RocketmqClient"; ++- public static final String ROCKETMQ_TRAFFIC_NAME = "RocketmqTraffic"; ++- public static final String ROCKETMQ_REMOTING_NAME = "RocketmqRemoting"; ++- public static final String TOOLS_LOGGER_NAME = "RocketmqTools"; ++- public static final String COMMON_LOGGER_NAME = "RocketmqCommon"; ++- public static final String STORE_LOGGER_NAME = "RocketmqStore"; ++- public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError"; ++- public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction"; ++- public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock"; ++- public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats"; ++- public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats"; ++- public static final String DLQ_LOGGER_NAME = "RocketmqDLQ"; ++- public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats"; ++- public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial"; ++- public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount"; ++- public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl"; ++- public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize"; ++- public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication"; ++- public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection"; ++- public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; ++- public static final String FILTER_LOGGER_NAME = "RocketmqFilter"; ++- public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; ++- public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover"; ++- public static final String STDOUT_LOGGER_NAME = "STDOUT"; ++- public static final String PROXY_LOGGER_NAME = "RocketmqProxy"; ++- public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark"; ++- public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr"; ++- public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB"; ++-} +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.common.constant; +++ +++public class LoggerName { +++ public static final String FILTERSRV_LOGGER_NAME = "RocketmqFiltersrv"; +++ public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; +++ public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; +++ public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; +++ public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; +++ public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; +++ public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; +++ public static final String CLIENT_LOGGER_NAME = "RocketmqClient"; +++ public static final String ROCKETMQ_TRAFFIC_NAME = "RocketmqTraffic"; +++ public static final String ROCKETMQ_REMOTING_NAME = "RocketmqRemoting"; +++ public static final String TOOLS_LOGGER_NAME = "RocketmqTools"; +++ public static final String COMMON_LOGGER_NAME = "RocketmqCommon"; +++ public static final String STORE_LOGGER_NAME = "RocketmqStore"; +++ public static final String STORE_ERROR_LOGGER_NAME = "RocketmqStoreError"; +++ public static final String TRANSACTION_LOGGER_NAME = "RocketmqTransaction"; +++ public static final String REBALANCE_LOCK_LOGGER_NAME = "RocketmqRebalanceLock"; +++ public static final String ROCKETMQ_STATS_LOGGER_NAME = "RocketmqStats"; +++ public static final String DLQ_STATS_LOGGER_NAME = "RocketmqDLQStats"; +++ public static final String DLQ_LOGGER_NAME = "RocketmqDLQ"; +++ public static final String CONSUMER_STATS_LOGGER_NAME = "RocketmqConsumerStats"; +++ public static final String COMMERCIAL_LOGGER_NAME = "RocketmqCommercial"; +++ public static final String ACCOUNT_LOGGER_NAME = "RocketmqAccount"; +++ public static final String FLOW_CONTROL_LOGGER_NAME = "RocketmqFlowControl"; +++ public static final String ROCKETMQ_AUTHORIZE_LOGGER_NAME = "RocketmqAuthorize"; +++ public static final String DUPLICATION_LOGGER_NAME = "RocketmqDuplication"; +++ public static final String PROTECTION_LOGGER_NAME = "RocketmqProtection"; +++ public static final String WATER_MARK_LOGGER_NAME = "RocketmqWaterMark"; +++ public static final String FILTER_LOGGER_NAME = "RocketmqFilter"; +++ public static final String ROCKETMQ_POP_LOGGER_NAME = "RocketmqPop"; +++ public static final String FAILOVER_LOGGER_NAME = "RocketmqFailover"; +++ public static final String STDOUT_LOGGER_NAME = "STDOUT"; +++ public static final String PROXY_LOGGER_NAME = "RocketmqProxy"; +++ public static final String PROXY_WATER_MARK_LOGGER_NAME = "RocketmqProxyWatermark"; +++ public static final String ROCKETMQ_COLDCTR_LOGGER_NAME = "RocketmqColdCtr"; +++ public static final String ROCKSDB_LOGGER_NAME = "RocketmqRocksDB"; +++} ++-- ++2.32.0.windows.2 ++ +diff --git a/patch010-backport-add-some-fixes b/patch010-backport-add-some-fixes +new file mode 100644 +index 000000000..36d5e4a1f +--- /dev/null ++++ b/patch010-backport-add-some-fixes +@@ -0,0 +1,1286 @@ ++From b2deef179dbc6a9eb1a2b6dd7b652d95cb768295 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Thu, 10 Aug 2023 10:38:47 +0800 ++Subject: [PATCH 01/12] [ISSUE #7144] Accelerate the recovery speed of the ++ tiered storage module (#7145) ++ ++--- ++ .../tieredstore/TieredDispatcher.java | 3 + ++ .../tieredstore/TieredMessageStore.java | 2 +- ++ .../common/TieredStoreExecutor.java | 25 ++-- ++ .../tieredstore/file/CompositeFlatFile.java | 15 +- ++ .../file/CompositeQueueFlatFile.java | 20 ++- ++ .../tieredstore/file/TieredCommitLog.java | 24 +++- ++ .../tieredstore/file/TieredFlatFile.java | 42 +++--- ++ .../file/TieredFlatFileManager.java | 135 ++++++++++-------- ++ .../metadata/FileSegmentMetadata.java | 26 +++- ++ .../tieredstore/TieredDispatcherTest.java | 15 +- ++ .../tieredstore/TieredMessageFetcherTest.java | 2 +- ++ .../file/CompositeQueueFlatFileTest.java | 2 +- ++ .../file/TieredFlatFileManagerTest.java | 7 +- ++ 13 files changed, 194 insertions(+), 124 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++index bb58ea7dd..1746190cd 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++@@ -279,6 +279,9 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ long upperBound = Math.min(dispatchOffset + maxCount, maxOffsetInQueue); ++ ConsumeQueue consumeQueue = (ConsumeQueue) defaultStore.getConsumeQueue(topic, queueId); ++ +++ logger.debug("DispatchFlatFile race, topic={}, queueId={}, cq range={}-{}, dispatch offset={}-{}", +++ topic, queueId, minOffsetInQueue, maxOffsetInQueue, dispatchOffset, upperBound - 1); +++ ++ for (; dispatchOffset < upperBound; dispatchOffset++) { ++ // get consume queue ++ SelectMappedBufferResult cqItem = consumeQueue.getIndexBuffer(dispatchOffset); ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index 1f12410f2..ced1fb818 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -147,7 +147,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { ++ ++ if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) { ++- logger.debug("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); +++ logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ } ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++index 6eb3478b3..6dd0e8846 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++@@ -43,18 +43,9 @@ public class TieredStoreExecutor { ++ public static ExecutorService compactIndexFileExecutor; ++ ++ public static void init() { ++- dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++- dispatchExecutor = new ThreadPoolExecutor( ++- Math.max(2, Runtime.getRuntime().availableProcessors()), ++- Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++- 1000 * 60, ++- TimeUnit.MILLISECONDS, ++- dispatchThreadPoolQueue, ++- new ThreadFactoryImpl("TieredCommonExecutor_")); ++- ++ commonScheduledExecutor = new ScheduledThreadPoolExecutor( ++ Math.max(4, Runtime.getRuntime().availableProcessors()), ++- new ThreadFactoryImpl("TieredCommonScheduledExecutor_")); +++ new ThreadFactoryImpl("TieredCommonExecutor_")); ++ ++ commitExecutor = new ScheduledThreadPoolExecutor( ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++@@ -62,7 +53,17 @@ public class TieredStoreExecutor { ++ ++ cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor( ++ Math.max(4, Runtime.getRuntime().availableProcessors()), ++- new ThreadFactoryImpl("TieredCleanExpiredFileExecutor_")); +++ new ThreadFactoryImpl("TieredCleanFileExecutor_")); +++ +++ dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); +++ dispatchExecutor = new ThreadPoolExecutor( +++ Math.max(2, Runtime.getRuntime().availableProcessors()), +++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), +++ 1000 * 60, +++ TimeUnit.MILLISECONDS, +++ dispatchThreadPoolQueue, +++ new ThreadFactoryImpl("TieredDispatchExecutor_"), +++ new ThreadPoolExecutor.DiscardOldestPolicy()); ++ ++ fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++ fetchDataExecutor = new ThreadPoolExecutor( ++@@ -71,7 +72,7 @@ public class TieredStoreExecutor { ++ 1000 * 60, ++ TimeUnit.MILLISECONDS, ++ fetchDataThreadPoolQueue, ++- new ThreadFactoryImpl("TieredFetchDataExecutor_")); +++ new ThreadFactoryImpl("TieredFetchExecutor_")); ++ ++ compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++ compactIndexFileExecutor = new ThreadPoolExecutor( ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java ++index df4baf33f..5ad3a6ff3 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeFlatFile.java ++@@ -76,20 +76,15 @@ public class CompositeFlatFile implements CompositeAccess { ++ this.storeConfig = fileQueueFactory.getStoreConfig(); ++ this.readAheadFactor = this.storeConfig.getReadAheadMinFactor(); ++ this.metadataStore = TieredStoreUtil.getMetadataStore(this.storeConfig); ++- this.dispatchOffset = new AtomicLong(); ++ this.compositeFlatFileLock = new ReentrantLock(); ++ this.inFlightRequestMap = new ConcurrentHashMap<>(); ++ this.commitLog = new TieredCommitLog(fileQueueFactory, filePath); ++ this.consumeQueue = new TieredConsumeQueue(fileQueueFactory, filePath); +++ this.dispatchOffset = new AtomicLong( +++ this.consumeQueue.isInitialized() ? this.getConsumeQueueCommitOffset() : -1L); ++ this.groupOffsetCache = this.initOffsetCache(); ++ } ++ ++- protected void recoverMetadata() { ++- if (!consumeQueue.isInitialized() && this.dispatchOffset.get() != -1) { ++- consumeQueue.setBaseOffset(this.dispatchOffset.get() * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); ++- } ++- } ++- ++ private Cache initOffsetCache() { ++ return Caffeine.newBuilder() ++ .expireAfterWrite(2, TimeUnit.MINUTES) ++@@ -310,10 +305,12 @@ public class CompositeFlatFile implements CompositeAccess { ++ ++ @Override ++ public void initOffset(long offset) { ++- if (!consumeQueue.isInitialized()) { +++ if (consumeQueue.isInitialized()) { +++ dispatchOffset.set(this.getConsumeQueueCommitOffset()); +++ } else { ++ consumeQueue.setBaseOffset(offset * TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); +++ dispatchOffset.set(offset); ++ } ++- dispatchOffset.set(offset); ++ } ++ ++ @Override ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java ++index f6c0afed0..0a797f465 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFile.java ++@@ -36,8 +36,7 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { ++ public CompositeQueueFlatFile(TieredFileAllocator fileQueueFactory, MessageQueue messageQueue) { ++ super(fileQueueFactory, TieredStoreUtil.toPath(messageQueue)); ++ this.messageQueue = messageQueue; ++- this.recoverTopicMetadata(); ++- super.recoverMetadata(); +++ this.recoverQueueMetadata(); ++ this.indexFile = TieredFlatFileManager.getIndexFile(storeConfig); ++ } ++ ++@@ -46,11 +45,12 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { ++ if (!consumeQueue.isInitialized()) { ++ queueMetadata.setMinOffset(offset); ++ queueMetadata.setMaxOffset(offset); +++ metadataStore.updateQueue(queueMetadata); ++ } ++ super.initOffset(offset); ++ } ++ ++- public void recoverTopicMetadata() { +++ public void recoverQueueMetadata() { ++ TopicMetadata topicMetadata = this.metadataStore.getTopic(messageQueue.getTopic()); ++ if (topicMetadata == null) { ++ topicMetadata = this.metadataStore.addTopic(messageQueue.getTopic(), -1L); ++@@ -64,18 +64,16 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { ++ if (queueMetadata.getMaxOffset() < queueMetadata.getMinOffset()) { ++ queueMetadata.setMaxOffset(queueMetadata.getMinOffset()); ++ } ++- this.dispatchOffset.set(queueMetadata.getMaxOffset()); ++ } ++ ++- public void persistMetadata() { +++ public void flushMetadata() { ++ try { ++- if (consumeQueue.getCommitOffset() < queueMetadata.getMinOffset()) { ++- return; ++- } ++- queueMetadata.setMaxOffset(consumeQueue.getCommitOffset() / TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE); +++ queueMetadata.setMinOffset(super.getConsumeQueueMinOffset()); +++ queueMetadata.setMaxOffset(super.getConsumeQueueMaxOffset()); ++ metadataStore.updateQueue(queueMetadata); ++ } catch (Exception e) { ++- LOGGER.error("CompositeFlatFile#flushMetadata: update queue metadata failed: topic: {}, queue: {}", messageQueue.getTopic(), messageQueue.getQueueId(), e); +++ LOGGER.error("CompositeFlatFile#flushMetadata error, topic: {}, queue: {}", +++ messageQueue.getTopic(), messageQueue.getQueueId(), e); ++ } ++ } ++ ++@@ -114,7 +112,7 @@ public class CompositeQueueFlatFile extends CompositeFlatFile { ++ @Override ++ public void shutdown() { ++ super.shutdown(); ++- metadataStore.updateQueue(queueMetadata); +++ this.flushMetadata(); ++ } ++ ++ @Override ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java ++index 80e1bce50..0e5f79132 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredCommitLog.java ++@@ -50,7 +50,7 @@ public class TieredCommitLog { ++ this.storeConfig = fileQueueFactory.getStoreConfig(); ++ this.flatFile = fileQueueFactory.createFlatFileForCommitLog(filePath); ++ this.minConsumeQueueOffset = new AtomicLong(NOT_EXIST_MIN_OFFSET); ++- this.correctMinOffset(); +++ this.correctMinOffsetAsync(); ++ } ++ ++ @VisibleForTesting ++@@ -91,17 +91,26 @@ public class TieredCommitLog { ++ return flatFile.getFileToWrite().getMaxTimestamp(); ++ } ++ ++- public synchronized long correctMinOffset() { +++ public long correctMinOffset() { +++ try { +++ return correctMinOffsetAsync().get(); +++ } catch (Exception e) { +++ log.error("Correct min offset failed in clean expired file", e); +++ } +++ return NOT_EXIST_MIN_OFFSET; +++ } +++ +++ public synchronized CompletableFuture correctMinOffsetAsync() { ++ if (flatFile.getFileSegmentCount() == 0) { ++ this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET); ++- return NOT_EXIST_MIN_OFFSET; +++ return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET); ++ } ++ ++ // queue offset field length is 8 ++ int length = MessageBufferUtil.QUEUE_OFFSET_POSITION + 8; ++ if (flatFile.getCommitOffset() - flatFile.getMinOffset() < length) { ++ this.minConsumeQueueOffset.set(NOT_EXIST_MIN_OFFSET); ++- return NOT_EXIST_MIN_OFFSET; +++ return CompletableFuture.completedFuture(NOT_EXIST_MIN_OFFSET); ++ } ++ ++ try { ++@@ -109,7 +118,8 @@ public class TieredCommitLog { ++ .thenApply(buffer -> { ++ long offset = MessageBufferUtil.getQueueOffset(buffer); ++ minConsumeQueueOffset.set(offset); ++- log.info("Correct commitlog min cq offset success, filePath={}, min cq offset={}, range={}-{}", +++ log.debug("Correct commitlog min cq offset success, " + +++ "filePath={}, min cq offset={}, commitlog range={}-{}", ++ flatFile.getFilePath(), offset, flatFile.getMinOffset(), flatFile.getCommitOffset()); ++ return offset; ++ }) ++@@ -117,11 +127,11 @@ public class TieredCommitLog { ++ log.warn("Correct commitlog min cq offset error, filePath={}, range={}-{}", ++ flatFile.getFilePath(), flatFile.getMinOffset(), flatFile.getCommitOffset(), throwable); ++ return minConsumeQueueOffset.get(); ++- }).get(); +++ }); ++ } catch (Exception e) { ++ log.error("Correct commitlog min cq offset error, filePath={}", flatFile.getFilePath(), e); ++ } ++- return minConsumeQueueOffset.get(); +++ return CompletableFuture.completedFuture(minConsumeQueueOffset.get()); ++ } ++ ++ public AppendResult append(ByteBuffer byteBuf) { ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++index 75ce8d89f..426c4e09d 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ +++import com.alibaba.fastjson.JSON; ++ import com.google.common.annotations.VisibleForTesting; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; ++@@ -24,6 +25,7 @@ import java.util.Comparator; ++ import java.util.HashSet; ++ import java.util.LinkedList; ++ import java.util.List; +++import java.util.Objects; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.CopyOnWriteArrayList; ++@@ -178,32 +180,26 @@ public class TieredFlatFile { ++ private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) { ++ ++ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( ++- fileSegment.getPath(), fileSegment.getFileType(), fileSegment.getBaseOffset()); ++- ++- if (metadata != null) { ++- return metadata; ++- } +++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); ++ ++ // Note: file segment path may not the same as file base path, use base path here. ++- metadata = new FileSegmentMetadata( ++- this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); ++- ++- if (fileSegment.isClosed()) { ++- metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); +++ if (metadata == null) { +++ metadata = new FileSegmentMetadata( +++ this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); +++ metadata.setCreateTimestamp(fileSegment.getMinTimestamp()); +++ metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); +++ metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); +++ if (fileSegment.isClosed()) { +++ metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); +++ } +++ this.tieredMetadataStore.updateFileSegment(metadata); ++ } ++- ++- metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); ++- metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++- ++- // Submit to persist ++- this.tieredMetadataStore.updateFileSegment(metadata); ++ return metadata; ++ } ++ ++ /** ++ * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full ++ */ ++- @VisibleForTesting ++ public void updateFileSegment(TieredFileSegment fileSegment) { ++ FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment); ++ ++@@ -219,9 +215,16 @@ public class TieredFlatFile { ++ } ++ ++ segmentMetadata.setSize(fileSegment.getCommitPosition()); ++- segmentMetadata.setBeginTimestamp(fileSegment.getMinTimestamp()); ++ segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++- this.tieredMetadataStore.updateFileSegment(segmentMetadata); +++ +++ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( +++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); +++ +++ if (!Objects.equals(metadata, segmentMetadata)) { +++ this.tieredMetadataStore.updateFileSegment(segmentMetadata); +++ logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}", +++ segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata)); +++ } ++ } ++ ++ private void checkAndFixFileSize() { ++@@ -257,6 +260,7 @@ public class TieredFlatFile { ++ logger.warn("TieredFlatFile#checkAndFixFileSize: fix last file {} size: origin: {}, actual: {}", ++ lastFile.getPath(), lastFile.getCommitOffset() - lastFile.getBaseOffset(), lastFileSize); ++ lastFile.initPosition(lastFileSize); +++ this.updateFileSegment(lastFile); ++ } ++ } ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++index aeca44b8c..e9ae4a5a5 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++@@ -16,16 +16,19 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ +++import com.google.common.base.Stopwatch; ++ import com.google.common.collect.ImmutableList; ++ import java.util.ArrayList; ++ import java.util.List; ++ import java.util.Random; +++import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.Future; +++import java.util.concurrent.Semaphore; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicLong; ++ import javax.annotation.Nullable; +++import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++@@ -36,6 +39,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ ++ public class TieredFlatFileManager { ++ +++ private static final Logger BROKER_LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); ++ private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); ++ ++ private static volatile TieredFlatFileManager instance; ++@@ -44,7 +48,7 @@ public class TieredFlatFileManager { ++ private final TieredMetadataStore metadataStore; ++ private final TieredMessageStoreConfig storeConfig; ++ private final TieredFileAllocator tieredFileAllocator; ++- private final ConcurrentMap queueFlatFileMap; +++ private final ConcurrentMap flatFileConcurrentMap; ++ ++ public TieredFlatFileManager(TieredMessageStoreConfig storeConfig) ++ throws ClassNotFoundException, NoSuchMethodException { ++@@ -52,23 +56,20 @@ public class TieredFlatFileManager { ++ this.storeConfig = storeConfig; ++ this.metadataStore = TieredStoreUtil.getMetadataStore(storeConfig); ++ this.tieredFileAllocator = new TieredFileAllocator(storeConfig); ++- this.queueFlatFileMap = new ConcurrentHashMap<>(); +++ this.flatFileConcurrentMap = new ConcurrentHashMap<>(); ++ this.doScheduleTask(); ++ } ++ ++ public static TieredFlatFileManager getInstance(TieredMessageStoreConfig storeConfig) { ++- if (storeConfig == null) { +++ if (storeConfig == null || instance != null) { ++ return instance; ++ } ++- ++- if (instance == null) { ++- synchronized (TieredFlatFileManager.class) { ++- if (instance == null) { ++- try { ++- instance = new TieredFlatFileManager(storeConfig); ++- } catch (Exception e) { ++- logger.error("TieredFlatFileManager#getInstance: create flat file manager failed", e); ++- } +++ synchronized (TieredFlatFileManager.class) { +++ if (instance == null) { +++ try { +++ instance = new TieredFlatFileManager(storeConfig); +++ } catch (Exception e) { +++ logger.error("Construct FlatFileManager instance error", e); ++ } ++ } ++ } ++@@ -88,7 +89,7 @@ public class TieredFlatFileManager { ++ TieredStoreUtil.RMQ_SYS_TIERED_STORE_INDEX_TOPIC, storeConfig.getBrokerName(), 0)); ++ indexFile = new TieredIndexFile(new TieredFileAllocator(storeConfig), filePath); ++ } catch (Exception e) { ++- logger.error("TieredFlatFileManager#getIndexFile: create index file failed", e); +++ logger.error("Construct FlatFileManager indexFile error", e); ++ } ++ } ++ } ++@@ -105,7 +106,7 @@ public class TieredFlatFileManager { ++ flatFile.commitCommitLog(); ++ } catch (Throwable e) { ++ MessageQueue mq = flatFile.getMessageQueue(); ++- logger.error("commit commitLog periodically failed: topic: {}, queue: {}", +++ logger.error("Commit commitLog periodically failed: topic: {}, queue: {}", ++ mq.getTopic(), mq.getQueueId(), e); ++ } ++ }, delay, TimeUnit.MILLISECONDS); ++@@ -114,7 +115,7 @@ public class TieredFlatFileManager { ++ flatFile.commitConsumeQueue(); ++ } catch (Throwable e) { ++ MessageQueue mq = flatFile.getMessageQueue(); ++- logger.error("commit consumeQueue periodically failed: topic: {}, queue: {}", +++ logger.error("Commit consumeQueue periodically failed: topic: {}, queue: {}", ++ mq.getTopic(), mq.getQueueId(), e); ++ } ++ }, delay, TimeUnit.MILLISECONDS); ++@@ -125,7 +126,7 @@ public class TieredFlatFileManager { ++ indexFile.commit(true); ++ } ++ } catch (Throwable e) { ++- logger.error("commit indexFile periodically failed", e); +++ logger.error("Commit indexFile periodically failed", e); ++ } ++ }, 0, TimeUnit.MILLISECONDS); ++ } ++@@ -160,7 +161,7 @@ public class TieredFlatFileManager { ++ try { ++ doCommit(); ++ } catch (Throwable e) { ++- logger.error("commit flat file periodically failed: ", e); +++ logger.error("Commit flat file periodically failed: ", e); ++ } ++ }, 60, 60, TimeUnit.SECONDS); ++ ++@@ -168,49 +169,73 @@ public class TieredFlatFileManager { ++ try { ++ doCleanExpiredFile(); ++ } catch (Throwable e) { ++- logger.error("clean expired flat file failed: ", e); +++ logger.error("Clean expired flat file failed: ", e); ++ } ++ }, 30, 30, TimeUnit.SECONDS); ++ } ++ ++ public boolean load() { +++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ try { ++- AtomicLong topicSequenceNumber = new AtomicLong(); ++- List> futureList = new ArrayList<>(); ++- queueFlatFileMap.clear(); ++- metadataStore.iterateTopic(topicMetadata -> { +++ flatFileConcurrentMap.clear(); +++ this.recoverSequenceNumber(); +++ this.recoverTieredFlatFile(); +++ logger.info("Message store recover end, total cost={}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); +++ } catch (Exception e) { +++ long costTime = stopwatch.elapsed(TimeUnit.MILLISECONDS); +++ logger.info("Message store recover error, total cost={}ms", costTime); +++ BROKER_LOG.error("Message store recover error, total cost={}ms", costTime, e); +++ return false; +++ } +++ return true; +++ } +++ +++ public void recoverSequenceNumber() { +++ AtomicLong topicSequenceNumber = new AtomicLong(); +++ metadataStore.iterateTopic(topicMetadata -> { +++ if (topicMetadata != null && topicMetadata.getTopicId() > 0) { ++ topicSequenceNumber.set(Math.max(topicSequenceNumber.get(), topicMetadata.getTopicId())); ++- Future future = TieredStoreExecutor.dispatchExecutor.submit(() -> { ++- if (topicMetadata.getStatus() != 0) { ++- return; ++- } +++ } +++ }); +++ metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet()); +++ } +++ +++ public void recoverTieredFlatFile() { +++ Semaphore semaphore = new Semaphore((int) (TieredStoreExecutor.QUEUE_CAPACITY * 0.75)); +++ List> futures = new ArrayList<>(); +++ metadataStore.iterateTopic(topicMetadata -> { +++ try { +++ semaphore.acquire(); +++ CompletableFuture future = CompletableFuture.runAsync(() -> { ++ try { ++- metadataStore.iterateQueue(topicMetadata.getTopic(), ++- queueMetadata -> getOrCreateFlatFileIfAbsent( ++- new MessageQueue(topicMetadata.getTopic(), ++- storeConfig.getBrokerName(), ++- queueMetadata.getQueue().getQueueId()))); +++ Stopwatch subWatch = Stopwatch.createStarted(); +++ if (topicMetadata.getStatus() != 0) { +++ return; +++ } +++ AtomicLong queueCount = new AtomicLong(); +++ metadataStore.iterateQueue(topicMetadata.getTopic(), queueMetadata -> { +++ this.getOrCreateFlatFileIfAbsent(new MessageQueue(topicMetadata.getTopic(), +++ storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId())); +++ queueCount.incrementAndGet(); +++ }); +++ logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", +++ topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); ++ } catch (Exception e) { ++- logger.error("load mq composite flat file from metadata failed", e); +++ logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e); +++ } finally { +++ semaphore.release(); ++ } ++- }); ++- futureList.add(future); ++- }); ++- ++- // Wait for load all metadata done ++- for (Future future : futureList) { ++- future.get(); +++ }, TieredStoreExecutor.commitExecutor); +++ futures.add(future); +++ } catch (Exception e) { +++ throw new RuntimeException(e); ++ } ++- metadataStore.setTopicSequenceNumber(topicSequenceNumber.incrementAndGet()); ++- } catch (Exception e) { ++- logger.error("load mq composite flat file from metadata failed", e); ++- return false; ++- } ++- return true; +++ }); +++ CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); ++ } ++ ++ public void cleanup() { ++- queueFlatFileMap.clear(); +++ flatFileConcurrentMap.clear(); ++ cleanStaticReference(); ++ } ++ ++@@ -221,27 +246,25 @@ public class TieredFlatFileManager { ++ ++ @Nullable ++ public CompositeQueueFlatFile getOrCreateFlatFileIfAbsent(MessageQueue messageQueue) { ++- return queueFlatFileMap.computeIfAbsent(messageQueue, mq -> { +++ return flatFileConcurrentMap.computeIfAbsent(messageQueue, mq -> { ++ try { ++- logger.debug("TieredFlatFileManager#getOrCreateFlatFileIfAbsent: " + ++- "try to create new flat file: topic: {}, queueId: {}", +++ logger.debug("Create new TopicFlatFile, topic: {}, queueId: {}", ++ messageQueue.getTopic(), messageQueue.getQueueId()); ++ return new CompositeQueueFlatFile(tieredFileAllocator, mq); ++ } catch (Exception e) { ++- logger.error("TieredFlatFileManager#getOrCreateFlatFileIfAbsent: " + ++- "create new flat file: topic: {}, queueId: {}", +++ logger.debug("Create new TopicFlatFile failed, topic: {}, queueId: {}", ++ messageQueue.getTopic(), messageQueue.getQueueId(), e); ++- return null; ++ } +++ return null; ++ }); ++ } ++ ++ public CompositeQueueFlatFile getFlatFile(MessageQueue messageQueue) { ++- return queueFlatFileMap.get(messageQueue); +++ return flatFileConcurrentMap.get(messageQueue); ++ } ++ ++ public ImmutableList deepCopyFlatFileToList() { ++- return ImmutableList.copyOf(queueFlatFileMap.values()); +++ return ImmutableList.copyOf(flatFileConcurrentMap.values()); ++ } ++ ++ public void shutdown() { ++@@ -270,7 +293,7 @@ public class TieredFlatFileManager { ++ } ++ ++ // delete memory reference ++- CompositeQueueFlatFile flatFile = queueFlatFileMap.remove(mq); +++ CompositeQueueFlatFile flatFile = flatFileConcurrentMap.remove(mq); ++ if (flatFile != null) { ++ MessageQueue messageQueue = flatFile.getMessageQueue(); ++ logger.info("TieredFlatFileManager#destroyCompositeFile: " + ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java ++index 1b232fc75..2f0fd71de 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metadata/FileSegmentMetadata.java ++@@ -16,6 +16,8 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.metadata; ++ +++import java.util.Objects; +++ ++ public class FileSegmentMetadata { ++ ++ public static final int STATUS_NEW = 0; ++@@ -43,7 +45,6 @@ public class FileSegmentMetadata { ++ this.baseOffset = baseOffset; ++ this.type = type; ++ this.status = STATUS_NEW; ++- this.createTimestamp = System.currentTimeMillis(); ++ } ++ ++ public void markSealed() { ++@@ -122,4 +123,27 @@ public class FileSegmentMetadata { ++ public void setSealTimestamp(long sealTimestamp) { ++ this.sealTimestamp = sealTimestamp; ++ } +++ +++ @Override +++ public boolean equals(Object o) { +++ if (this == o) +++ return true; +++ if (o == null || getClass() != o.getClass()) +++ return false; +++ FileSegmentMetadata metadata = (FileSegmentMetadata) o; +++ return size == metadata.size +++ && baseOffset == metadata.baseOffset +++ && status == metadata.status +++ && path.equals(metadata.path) +++ && type == metadata.type +++ && createTimestamp == metadata.createTimestamp +++ && beginTimestamp == metadata.beginTimestamp +++ && endTimestamp == metadata.endTimestamp +++ && sealTimestamp == metadata.sealTimestamp; +++ } +++ +++ @Override +++ public int hashCode() { +++ return Objects.hash(type, path, baseOffset, status, size, createTimestamp, beginTimestamp, endTimestamp, sealTimestamp); +++ } ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java ++index e6adef1d1..5791dc9a4 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredDispatcherTest.java ++@@ -116,19 +116,20 @@ public class TieredDispatcherTest { ++ buffer3.putLong(MessageBufferUtil.QUEUE_OFFSET_POSITION, 9); ++ flatFile.appendCommitLog(buffer3); ++ flatFile.commitCommitLog(); ++- Assert.assertEquals(10, flatFile.getDispatchOffset()); +++ Assert.assertEquals(9 + 1, flatFile.getDispatchOffset()); +++ Assert.assertEquals(9, flatFile.getCommitLogDispatchCommitOffset()); ++ ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer1); ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer2); ++ dispatcher.buildConsumeQueueAndIndexFile(); ++ Assert.assertEquals(7, flatFile.getConsumeQueueMaxOffset()); ++- Assert.assertEquals(7, flatFile.getDispatchOffset()); ++ ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 7, 7, 0, 0, buffer1); ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 8, 8, 0, 0, buffer2); ++ dispatcher.doRedispatchRequestToWriteMap(AppendResult.SUCCESS, flatFile, 9, 9, 0, 0, buffer3); ++ dispatcher.buildConsumeQueueAndIndexFile(); ++- Assert.assertEquals(10, flatFile.getConsumeQueueMaxOffset()); +++ Assert.assertEquals(6, flatFile.getConsumeQueueMinOffset()); +++ Assert.assertEquals(9 + 1, flatFile.getConsumeQueueMaxOffset()); ++ } ++ ++ @Test ++@@ -142,6 +143,7 @@ public class TieredDispatcherTest { ++ Mockito.when(defaultStore.getMinOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(0L); ++ Mockito.when(defaultStore.getMaxOffsetInQueue(mq.getTopic(), mq.getQueueId())).thenReturn(9L); ++ +++ // mock cq item, position = 7 ++ ByteBuffer cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE); ++ cqItem.putLong(7); ++ cqItem.putInt(MessageBufferUtilTest.MSG_LEN); ++@@ -150,13 +152,13 @@ public class TieredDispatcherTest { ++ SelectMappedBufferResult mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null); ++ Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(6)).thenReturn(mockResult); ++ +++ // mock cq item, position = 8 ++ cqItem = ByteBuffer.allocate(ConsumeQueue.CQ_STORE_UNIT_SIZE); ++ cqItem.putLong(8); ++ cqItem.putInt(MessageBufferUtilTest.MSG_LEN); ++ cqItem.putLong(1); ++ cqItem.flip(); ++ mockResult = new SelectMappedBufferResult(0, cqItem, ConsumeQueue.CQ_STORE_UNIT_SIZE, null); ++- ++ Mockito.when(((ConsumeQueue) defaultStore.getConsumeQueue(mq.getTopic(), mq.getQueueId())).getIndexBuffer(7)).thenReturn(mockResult); ++ ++ mockResult = new SelectMappedBufferResult(0, MessageBufferUtilTest.buildMockedMessageBuffer(), MessageBufferUtilTest.MSG_LEN, null); ++@@ -167,7 +169,10 @@ public class TieredDispatcherTest { ++ mockResult = new SelectMappedBufferResult(0, msg, MessageBufferUtilTest.MSG_LEN, null); ++ Mockito.when(defaultStore.selectOneMessageByOffset(8, MessageBufferUtilTest.MSG_LEN)).thenReturn(mockResult); ++ ++- dispatcher.dispatchFlatFile(flatFileManager.getOrCreateFlatFileIfAbsent(mq)); +++ CompositeQueueFlatFile flatFile = flatFileManager.getOrCreateFlatFileIfAbsent(mq); +++ Assert.assertNotNull(flatFile); +++ flatFile.initOffset(7); +++ dispatcher.dispatchFlatFile(flatFile); ++ Assert.assertEquals(8, flatFileManager.getFlatFile(mq).getDispatchOffset()); ++ } ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java ++index d75b2f916..774c6cf64 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageFetcherTest.java ++@@ -23,6 +23,7 @@ import java.util.Objects; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.SystemUtils; ++ import org.apache.commons.lang3.tuple.Triple; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.store.DispatchRequest; ++ import org.apache.rocketmq.store.GetMessageResult; ++@@ -40,7 +41,6 @@ import org.apache.rocketmq.tieredstore.file.TieredIndexFile; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++-import org.apache.rocketmq.common.BoundaryType; ++ import org.awaitility.Awaitility; ++ import org.junit.After; ++ import org.junit.Assert; ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java ++index 27efe111e..2e028ada3 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/CompositeQueueFlatFileTest.java ++@@ -119,7 +119,7 @@ public class CompositeQueueFlatFileTest { ++ Assert.assertEquals(AppendResult.SUCCESS, result); ++ ++ file.commit(true); ++- file.persistMetadata(); +++ file.flushMetadata(); ++ ++ QueueMetadata queueMetadata = metadataStore.getQueue(mq); ++ Assert.assertEquals(53, queueMetadata.getMaxOffset()); ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java ++index b7528c5e4..20fe4dd70 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManagerTest.java ++@@ -72,10 +72,15 @@ public class TieredFlatFileManagerTest { ++ ++ CompositeFlatFile flatFile = flatFileManager.getFlatFile(mq); ++ Assert.assertNotNull(flatFile); ++- Assert.assertEquals(100, flatFile.getDispatchOffset()); +++ Assert.assertEquals(-1L, flatFile.getDispatchOffset()); +++ flatFile.initOffset(100L); +++ Assert.assertEquals(100L, flatFile.getDispatchOffset()); +++ flatFile.initOffset(200L); +++ Assert.assertEquals(100L, flatFile.getDispatchOffset()); ++ ++ CompositeFlatFile flatFile1 = flatFileManager.getFlatFile(mq1); ++ Assert.assertNotNull(flatFile1); +++ flatFile1.initOffset(200L); ++ Assert.assertEquals(200, flatFile1.getDispatchOffset()); ++ ++ flatFileManager.destroyCompositeFile(mq); ++-- ++2.32.0.windows.2 ++ ++ ++From 99b39a35f29e491862296d56b7938a995d153974 Mon Sep 17 00:00:00 2001 ++From: ShuangxiDing ++Date: Thu, 10 Aug 2023 11:28:39 +0800 ++Subject: [PATCH 02/12] [ISSUE #7115] Fix grpc response message NPE (#7116) ++ ++--- ++ .../apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java ++index 0b3c85ea6..efa879a9c 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/common/ResponseBuilder.java ++@@ -92,7 +92,7 @@ public class ResponseBuilder { ++ public Status buildStatus(Code code, String message) { ++ return Status.newBuilder() ++ .setCode(code) ++- .setMessage(message) +++ .setMessage(message != null ? message : code.name()) ++ .build(); ++ } ++ ++-- ++2.32.0.windows.2 ++ ++ ++From c0ba453f38183266cf9a69be754e620311e1923b Mon Sep 17 00:00:00 2001 ++From: caigy ++Date: Thu, 10 Aug 2023 14:08:17 +0800 ++Subject: [PATCH 03/12] [ISSUE #7129] Fix resource collisions in acl tests ++ (#7130) ++ ++* run acl tests sequencially to avoid collision ++ ++* disable reuseForks for acl like broker ++ ++* Revert "[ISSUE #7135] Temporarily ignoring plainAccessValidator test (#7135)" ++ ++This reverts commit 6bc2c8474a0ce1e2833c82dffea7b1d8f718fcd7. ++--- ++ acl/pom.xml | 13 +++++++++++++ ++ .../acl/plain/PlainAccessControlFlowTest.java | 5 ----- ++ .../acl/plain/PlainAccessValidatorTest.java | 3 --- ++ .../acl/plain/PlainPermissionManagerTest.java | 3 --- ++ 4 files changed, 13 insertions(+), 11 deletions(-) ++ ++diff --git a/acl/pom.xml b/acl/pom.xml ++index 67bfcb8d2..989c0cf77 100644 ++--- a/acl/pom.xml +++++ b/acl/pom.xml ++@@ -74,4 +74,17 @@ ++ test ++ ++ +++ +++ +++ +++ +++ maven-surefire-plugin +++ ${maven-surefire-plugin.version} +++ +++ 1 +++ false +++ +++ +++ +++ ++ ++diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java ++index e7fd0932f..519345714 100644 ++--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java +++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessControlFlowTest.java ++@@ -31,7 +31,6 @@ import org.apache.rocketmq.remoting.protocol.header.PullMessageRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; ++ import org.junit.Assert; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++ import java.io.File; ++@@ -44,7 +43,6 @@ import java.util.Collections; ++ import java.util.LinkedList; ++ import java.util.List; ++ ++- ++ /** ++ *

In this class, we'll test the following scenarios, each containing several consecutive operations on ACL, ++ *

like updating and deleting ACL, changing config files and checking validations. ++@@ -52,9 +50,6 @@ import java.util.List; ++ *

Case 2: Only conf/acl/plain_acl.yml exists; ++ *

Case 3: Both conf/plain_acl.yml and conf/acl/plain_acl.yml exists. ++ */ ++- ++-// Ignore this test case as it is currently unable to pass on ubuntu workflow ++-@Ignore ++ public class PlainAccessControlFlowTest { ++ public static final String DEFAULT_TOPIC = "topic-acl"; ++ ++diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java ++index a3a925758..ef0cffbdc 100644 ++--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java +++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainAccessValidatorTest.java ++@@ -56,11 +56,8 @@ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; ++ import org.junit.After; ++ import org.junit.Assert; ++ import org.junit.Before; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++-// Ignore this test case as it is currently unable to pass on ubuntu workflow ++-@Ignore ++ public class PlainAccessValidatorTest { ++ ++ private PlainAccessValidator plainAccessValidator; ++diff --git a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java ++index aa7539f3a..941d8c779 100644 ++--- a/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java +++++ b/acl/src/test/java/org/apache/rocketmq/acl/plain/PlainPermissionManagerTest.java ++@@ -29,7 +29,6 @@ import org.assertj.core.api.Assertions; ++ import org.assertj.core.util.Lists; ++ import org.junit.Assert; ++ import org.junit.Before; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++ import java.io.File; ++@@ -42,8 +41,6 @@ import java.util.List; ++ import java.util.Map; ++ import java.util.Set; ++ ++-// Ignore this test case as it is currently unable to pass on ubuntu workflow ++-@Ignore ++ public class PlainPermissionManagerTest { ++ ++ PlainPermissionManager plainPermissionManager; ++-- ++2.32.0.windows.2 ++ ++ ++From 8741ff8c9b3bdbfc97976285affa7ea35c81243c Mon Sep 17 00:00:00 2001 ++From: ShuangxiDing ++Date: Thu, 10 Aug 2023 17:41:15 +0800 ++Subject: [PATCH 04/12] [ISSUE #7153] Add switch for MIXED message type (#7154) ++ ++Add a switch for MIXED message type when creating a Topic in the Broker. ++--- ++ .../broker/processor/AdminBrokerProcessor.java | 8 ++++++++ ++ .../java/org/apache/rocketmq/common/BrokerConfig.java | 10 ++++++++++ ++ 2 files changed, 18 insertions(+) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++index a6ce03dc2..bbddcec2d 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.TopicConfig; ++ import org.apache.rocketmq.common.UnlockCallback; ++ import org.apache.rocketmq.common.UtilAll; ++ import org.apache.rocketmq.common.attribute.AttributeParser; +++import org.apache.rocketmq.common.attribute.TopicMessageType; ++ import org.apache.rocketmq.common.constant.ConsumeInitMode; ++ import org.apache.rocketmq.common.constant.FIleReadaheadMode; ++ import org.apache.rocketmq.common.constant.LoggerName; ++@@ -439,6 +440,13 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ String attributesModification = requestHeader.getAttributes(); ++ topicConfig.setAttributes(AttributeParser.parseToMap(attributesModification)); ++ +++ if (topicConfig.getTopicMessageType() == TopicMessageType.MIXED +++ && !brokerController.getBrokerConfig().isEnableMixedMessageType()) { +++ response.setCode(ResponseCode.SYSTEM_ERROR); +++ response.setRemark("MIXED message type is not supported."); +++ return response; +++ } +++ ++ try { ++ this.brokerController.getTopicConfigManager().updateTopicConfig(topicConfig); ++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++index a815636b1..99a5db5ad 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++@@ -393,6 +393,8 @@ public class BrokerConfig extends BrokerIdentity { ++ */ ++ private boolean enableSingleTopicRegister = false; ++ +++ private boolean enableMixedMessageType = false; +++ ++ public long getMaxPopPollingSize() { ++ return maxPopPollingSize; ++ } ++@@ -1712,4 +1714,12 @@ public class BrokerConfig extends BrokerIdentity { ++ public void setEnableSingleTopicRegister(boolean enableSingleTopicRegister) { ++ this.enableSingleTopicRegister = enableSingleTopicRegister; ++ } +++ +++ public boolean isEnableMixedMessageType() { +++ return enableMixedMessageType; +++ } +++ +++ public void setEnableMixedMessageType(boolean enableMixedMessageType) { +++ this.enableMixedMessageType = enableMixedMessageType; +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From f534501855f8edbcb58f5b856973bf1027b5cf3a Mon Sep 17 00:00:00 2001 ++From: Steven ++Date: Fri, 11 Aug 2023 10:25:48 +0800 ++Subject: [PATCH 05/12] [Feature 7155] add errlog when cmd err (#7157) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Co-authored-by: 十真 ++--- ++ .../src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java ++index b00bad3c5..5a8a7cd54 100644 ++--- a/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java +++++ b/srvutil/src/main/java/org/apache/rocketmq/srvutil/ServerUtil.java ++@@ -52,6 +52,7 @@ public class ServerUtil { ++ System.exit(0); ++ } ++ } catch (ParseException e) { +++ System.err.println(e.getMessage()); ++ hf.printHelp(appName, options, true); ++ System.exit(1); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From db58f00c0fe0f129611d654291f2177de55dc9ff Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Fri, 11 Aug 2023 19:18:30 +0800 ++Subject: [PATCH 06/12] [ISSUE #7169] Change metadataThreadPoolQueueCapacity to ++ 100000 (#7170) ++ ++--- ++ .../main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 4f57a7052..39caaa0d9 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -165,7 +165,7 @@ public class ProxyConfig implements ConfigFile { ++ private int subscriptionGroupConfigCacheExpiredInSeconds = 20; ++ private int subscriptionGroupConfigCacheMaxNum = 20000; ++ private int metadataThreadPoolNums = 3; ++- private int metadataThreadPoolQueueCapacity = 1000; +++ private int metadataThreadPoolQueueCapacity = 100000; ++ ++ private int transactionHeartbeatThreadPoolNums = 20; ++ private int transactionHeartbeatThreadPoolQueueCapacity = 200; ++-- ++2.32.0.windows.2 ++ ++ ++From 1f04e68a2e331ab035b791280c5a91b60fe0c85f Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Sat, 12 Aug 2023 21:12:22 +0800 ++Subject: [PATCH 07/12] [ISSUE #7172] Unified Chinese for Name Server (#7173) ++ ++--- ++ docs/cn/concept.md | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/docs/cn/concept.md b/docs/cn/concept.md ++index cb2c863bd..3d67e9371 100644 ++--- a/docs/cn/concept.md +++++ b/docs/cn/concept.md ++@@ -17,7 +17,7 @@ RocketMQ主要由 Producer、Broker、Consumer 三部分组成，其中Producer ++ 消息中转角色，负责存储消息、转发消息。代理服务器在RocketMQ系统中负责接收从生产者发送来的消息并存储、同时为消费者的拉取请求作准备。代理服务器也存储消息相关的元数据，包括消费者组、消费进度偏移和主题和队列消息等。 ++ ++ ## 6 名字服务（Name Server） ++- 名称服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。 +++名字服务充当路由消息的提供者。生产者或消费者能够通过名字服务查找各主题相应的Broker IP列表。多个Namesrv实例组成集群，但相互独立，没有信息交换。 ++ ++ ## 7 拉取式消费（Pull Consumer） ++ Consumer消费的一种类型，应用通常主动调用Consumer的拉消息方法从Broker服务器拉消息、主动权由应用控制。一旦获取了批量消息，应用就会启动消费过程。 ++-- ++2.32.0.windows.2 ++ ++ ++From 25005060bbace477eeaaf4c0142cece5213efbbf Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Sun, 13 Aug 2023 20:52:17 +0800 ++Subject: [PATCH 08/12] [ISSUE #7176] Correct mismatched logs (#7177) ++ ++--- ++ .../org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java ++index 0055a1cc8..f7a95f0a6 100644 ++--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java +++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/routeinfo/RouteInfoManager.java ++@@ -522,7 +522,7 @@ public class RouteInfoManager { ++ this.lock.writeLock().unlock(); ++ } ++ } catch (Exception e) { ++- log.error("wipeWritePermOfBrokerByLock Exception", e); +++ log.error("addWritePermOfBrokerByLock Exception", e); ++ } ++ return 0; ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From ac411daa27117e9115a8fc5e2d5753085f009ed9 Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Tue, 15 Aug 2023 08:31:00 +0800 ++Subject: [PATCH 09/12] [ISSUE #7183] Correct mismatched commandDesc (#7184) ++ ++--- ++ .../tools/command/topic/RemappingStaticTopicSubCommand.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java ++index 849f680d0..2a08fdb5b 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/RemappingStaticTopicSubCommand.java ++@@ -47,7 +47,7 @@ public class RemappingStaticTopicSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update or create static topic, which has fixed number of queues"; +++ return "Remapping static topic."; ++ } ++ ++ @Override ++-- ++2.32.0.windows.2 ++ ++ ++From 55e0cdb2af3ab75a6d892f919d60797f17a99fda Mon Sep 17 00:00:00 2001 ++From: redlsz ++Date: Tue, 15 Aug 2023 19:19:45 +0800 ++Subject: [PATCH 10/12] fix: IndexOutOfBoundsException when process pop ++ response (#7003) ++ ++--- ++ .../org/apache/rocketmq/client/impl/MQClientAPIImpl.java | 5 ++++- ++ .../rocketmq/proxy/service/message/LocalMessageService.java | 5 ++++- ++ .../rocketmq/remoting/protocol/header/ExtraInfoUtil.java | 4 ++++ ++ 3 files changed, 12 insertions(+), 2 deletions(-) ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 708a6acd1..5101ffc8e 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -1174,7 +1174,10 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ Long.parseLong(messageExt.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET))); ++ continue; ++ } ++- key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId()); +++ // Value of POP_CK is used to determine whether it is a pop retry, +++ // cause topic could be rewritten by broker. +++ key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), +++ messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId()); ++ if (!sortMap.containsKey(key)) { ++ sortMap.put(key, new ArrayList<>(4)); ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++index 115c140ff..eb2c4d9ee 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++@@ -249,7 +249,10 @@ public class LocalMessageService implements MessageService { ++ // ++ Map> sortMap = new HashMap<>(16); ++ for (MessageExt messageExt : messageExtList) { ++- String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), messageExt.getQueueId()); +++ // Value of POP_CK is used to determine whether it is a pop retry, +++ // cause topic could be rewritten by broker. +++ String key = ExtraInfoUtil.getStartOffsetInfoMapKey(messageExt.getTopic(), +++ messageExt.getProperty(MessageConst.PROPERTY_POP_CK), messageExt.getQueueId()); ++ if (!sortMap.containsKey(key)) { ++ sortMap.put(key, new ArrayList<>(4)); ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java ++index 9a5fa89ab..13094331e 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/header/ExtraInfoUtil.java ++@@ -282,6 +282,10 @@ public class ExtraInfoUtil { ++ return (topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) ? RETRY_TOPIC : NORMAL_TOPIC) + "@" + key; ++ } ++ +++ public static String getStartOffsetInfoMapKey(String topic, String popCk, long key) { +++ return ((topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) || popCk != null) ? RETRY_TOPIC : NORMAL_TOPIC) + "@" + key; +++ } +++ ++ public static String getQueueOffsetKeyValueKey(long queueId, long queueOffset) { ++ return QUEUE_OFFSET + queueId + "%" + queueOffset; ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From a9c0b43f7f6ce5acfc4f2f3069553071fa93dfee Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Wed, 16 Aug 2023 18:45:00 +0800 ++Subject: [PATCH 11/12] [ISSUE #7192] Correct typos (#7193) ++ ++--- ++ .../tools/command/consumer/ConsumerProgressSubCommand.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java ++index f51a24673..97125b854 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerProgressSubCommand.java ++@@ -54,7 +54,7 @@ public class ConsumerProgressSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query consumers's progress, speed"; +++ return "Query consumer's progress, speed."; ++ } ++ ++ @Override ++-- ++2.32.0.windows.2 ++ ++ ++From 5a3de926b816db5a121c1d788430072a3bc942ae Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Wed, 16 Aug 2023 20:52:53 +0800 ++Subject: [PATCH 12/12] Optimize updateSubscription check exist loop (#7190) ++ ++--- ++ .../broker/client/ConsumerGroupInfo.java | 17 ++++++----------- ++ 1 file changed, 6 insertions(+), 11 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java ++index 867b9c720..1ea58c125 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ConsumerGroupInfo.java ++@@ -18,6 +18,7 @@ package org.apache.rocketmq.broker.client; ++ ++ import io.netty.channel.Channel; ++ import java.util.ArrayList; +++import java.util.HashSet; ++ import java.util.Iterator; ++ import java.util.List; ++ import java.util.Map.Entry; ++@@ -172,7 +173,7 @@ public class ConsumerGroupInfo { ++ */ ++ public boolean updateSubscription(final Set subList) { ++ boolean updated = false; ++- +++ Set topicSet = new HashSet<>(); ++ for (SubscriptionData sub : subList) { ++ SubscriptionData old = this.subscriptionTable.get(sub.getTopic()); ++ if (old == null) { ++@@ -194,22 +195,16 @@ public class ConsumerGroupInfo { ++ ++ this.subscriptionTable.put(sub.getTopic(), sub); ++ } +++ // Add all new topics to the HashSet +++ topicSet.add(sub.getTopic()); ++ } ++ ++ Iterator> it = this.subscriptionTable.entrySet().iterator(); ++ while (it.hasNext()) { ++ Entry next = it.next(); ++ String oldTopic = next.getKey(); ++- ++- boolean exist = false; ++- for (SubscriptionData sub : subList) { ++- if (sub.getTopic().equals(oldTopic)) { ++- exist = true; ++- break; ++- } ++- } ++- ++- if (!exist) { +++ // Check HashSet with O(1) time complexity +++ if (!topicSet.contains(oldTopic)) { ++ log.warn("subscription changed, group: {} remove topic {} {}", ++ this.groupName, ++ oldTopic, ++-- ++2.32.0.windows.2 ++ +diff --git a/patch011-backport-optimize-config b/patch011-backport-optimize-config +new file mode 100644 +index 000000000..fc8e2eceb +--- /dev/null ++++ b/patch011-backport-optimize-config +@@ -0,0 +1,1390 @@ ++From 50d1050437ed8748f86ee50261b50a1e1f63162e Mon Sep 17 00:00:00 2001 ++From: Jixiang Jin ++Date: Wed, 16 Aug 2023 21:15:00 +0800 ++Subject: [PATCH 1/7] To config the cardinalityLimit for openTelemetry metrics ++ exporting and fix logging config for metrics (#7196) ++ ++--- ++ WORKSPACE | 14 +++--- ++ .../broker/metrics/BrokerMetricsManager.java | 47 ++++++++++++++----- ++ .../broker/metrics/PopMetricsManager.java | 11 +++-- ++ .../src/main/resources/rmq.broker.logback.xml | 17 ++++--- ++ .../apache/rocketmq/common/BrokerConfig.java | 9 ++++ ++ .../metrics/ControllerMetricsManager.java | 6 +-- ++ pom.xml | 4 +- ++ .../metrics/RemotingMetricsManager.java | 10 ++-- ++ .../rocketmq/store/DefaultMessageStore.java | 24 +++++----- ++ .../apache/rocketmq/store/MessageStore.java | 6 +-- ++ .../metrics/DefaultStoreMetricsManager.java | 4 +- ++ .../plugin/AbstractPluginMessageStore.java | 6 +-- ++ .../tieredstore/TieredMessageStore.java | 6 +-- ++ .../metrics/TieredStoreMetricsManager.java | 23 +++++---- ++ 14 files changed, 110 insertions(+), 77 deletions(-) ++ ++diff --git a/WORKSPACE b/WORKSPACE ++index a8a0aafe9..3126f2d1d 100644 ++--- a/WORKSPACE +++++ b/WORKSPACE ++@@ -88,14 +88,14 @@ maven_install( ++ "io.grpc:grpc-api:1.47.0", ++ "io.grpc:grpc-testing:1.47.0", ++ "org.springframework:spring-core:5.3.26", ++- "io.opentelemetry:opentelemetry-exporter-otlp:1.19.0", ++- "io.opentelemetry:opentelemetry-exporter-prometheus:1.19.0-alpha", ++- "io.opentelemetry:opentelemetry-exporter-logging:1.19.0", ++- "io.opentelemetry:opentelemetry-sdk:1.19.0", +++ "io.opentelemetry:opentelemetry-exporter-otlp:1.29.0", +++ "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha", +++ "io.opentelemetry:opentelemetry-exporter-logging:1.29.0", +++ "io.opentelemetry:opentelemetry-sdk:1.29.0", ++ "com.squareup.okio:okio-jvm:3.0.0", ++- "io.opentelemetry:opentelemetry-api:1.19.0", ++- "io.opentelemetry:opentelemetry-sdk-metrics:1.19.0", ++- "io.opentelemetry:opentelemetry-sdk-common:1.19.0", +++ "io.opentelemetry:opentelemetry-api:1.29.0", +++ "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0", +++ "io.opentelemetry:opentelemetry-sdk-common:1.29.0", ++ "io.github.aliyunmq:rocketmq-slf4j-api:1.0.0", ++ "io.github.aliyunmq:rocketmq-logback-classic:1.0.0", ++ "org.slf4j:jul-to-slf4j:2.0.6", ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++index f0b76107e..6af5afc14 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++@@ -34,8 +34,10 @@ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.SdkMeterProvider; ++ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; ++ import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import io.opentelemetry.sdk.metrics.data.AggregationTemporality; ++ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; +++import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; ++ import io.opentelemetry.sdk.resources.Resource; ++ import java.util.ArrayList; ++ import java.util.Arrays; ++@@ -361,22 +363,45 @@ public class BrokerMetricsManager { ++ .setType(InstrumentType.HISTOGRAM) ++ .setName(HISTOGRAM_MESSAGE_SIZE) ++ .build(); ++- View messageSizeView = View.builder() ++- .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets)) ++- .build(); ++- providerBuilder.registerView(messageSizeSelector, messageSizeView); ++- ++- for (Pair selectorViewPair : RemotingMetricsManager.getMetricsView()) { ++- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); +++ ViewBuilder messageSizeViewBuilder = View.builder() +++ .setAggregation(Aggregation.explicitBucketHistogram(messageSizeBuckets)); +++ // To config the cardinalityLimit for openTelemetry metrics exporting. +++ SdkMeterProviderUtil.setCardinalityLimit(messageSizeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(messageSizeSelector, messageSizeViewBuilder.build()); +++ +++ for (Pair selectorViewPair : RemotingMetricsManager.getMetricsView()) { +++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); +++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); ++ } ++ ++- for (Pair selectorViewPair : messageStore.getMetricsView()) { ++- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); +++ for (Pair selectorViewPair : messageStore.getMetricsView()) { +++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); +++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); ++ } ++ ++- for (Pair selectorViewPair : PopMetricsManager.getMetricsView()) { ++- providerBuilder.registerView(selectorViewPair.getObject1(), selectorViewPair.getObject2()); +++ for (Pair selectorViewPair : PopMetricsManager.getMetricsView()) { +++ ViewBuilder viewBuilder = selectorViewPair.getObject2(); +++ SdkMeterProviderUtil.setCardinalityLimit(viewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(selectorViewPair.getObject1(), viewBuilder.build()); ++ } +++ +++ // default view builder for all counter. +++ InstrumentSelector defaultCounterSelector = InstrumentSelector.builder() +++ .setType(InstrumentType.COUNTER) +++ .build(); +++ ViewBuilder defaultCounterViewBuilder = View.builder().setDescription("default view for counter."); +++ SdkMeterProviderUtil.setCardinalityLimit(defaultCounterViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(defaultCounterSelector, defaultCounterViewBuilder.build()); +++ +++ //default view builder for all observable gauge. +++ InstrumentSelector defaultGaugeSelector = InstrumentSelector.builder() +++ .setType(InstrumentType.OBSERVABLE_GAUGE) +++ .build(); +++ ViewBuilder defaultGaugeViewBuilder = View.builder().setDescription("default view for gauge."); +++ SdkMeterProviderUtil.setCardinalityLimit(defaultGaugeViewBuilder, brokerConfig.getMetricsOtelCardinalityLimit()); +++ providerBuilder.registerView(defaultGaugeSelector, defaultGaugeViewBuilder.build()); ++ } ++ ++ private void initStatsMetrics() { ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java ++index 463371d7e..2de220da1 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/PopMetricsManager.java ++@@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.time.Duration; ++ import java.util.Arrays; ++ import java.util.List; ++@@ -63,7 +64,7 @@ public class PopMetricsManager { ++ private static LongCounter popReviveGetTotal = new NopLongCounter(); ++ private static LongCounter popReviveRetryMessageTotal = new NopLongCounter(); ++ ++- public static List> getMetricsView() { +++ public static List> getMetricsView() { ++ List rpcCostTimeBuckets = Arrays.asList( ++ (double) Duration.ofMillis(1).toMillis(), ++ (double) Duration.ofMillis(10).toMillis(), ++@@ -76,10 +77,10 @@ public class PopMetricsManager { ++ .setType(InstrumentType.HISTOGRAM) ++ .setName(HISTOGRAM_POP_BUFFER_SCAN_TIME_CONSUME) ++ .build(); ++- View popBufferScanTimeConsumeView = View.builder() ++- .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)) ++- .build(); ++- return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeView)); +++ ViewBuilder popBufferScanTimeConsumeViewBuilder = View.builder() +++ .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)); +++ +++ return Lists.newArrayList(new Pair<>(popBufferScanTimeConsumeSelector, popBufferScanTimeConsumeViewBuilder)); ++ } ++ ++ public static void initMetrics(Meter meter, BrokerController brokerController, ++diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml ++index 7d49f6664..3c51e59d4 100644 ++--- a/broker/src/main/resources/rmq.broker.logback.xml +++++ b/broker/src/main/resources/rmq.broker.logback.xml ++@@ -559,27 +559,27 @@ ++ ++ ++ ++- +++ ++ ++ brokerContainerLogDir ++ ${file.separator} ++ ++ ++- ++ ++- ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metric.log +++ ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}broker_metrics.log ++ ++ true ++ ++ ++- ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metric.%i.log.gz +++ ${user.home}${file.separator}logs${file.separator}rocketmqlogs${brokerLogDir:-${file.separator}}${brokerContainerLogDir}${file.separator}otherdays${file.separator}broker_metrics.%i.log.gz ++ ++ 1 ++- 10 +++ 3 ++ ++ ++- 500MB +++ 512MB ++ ++ ++ %d{yyy-MM-dd HH:mm:ss,GMT+8} %p %t - %m%n ++@@ -588,6 +588,9 @@ ++ ++ ++ +++ +++ +++ ++ ++ ++ ++@@ -670,7 +673,7 @@ ++ ++ ++ ++- +++ ++ ++ ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++index 99a5db5ad..45d26b29c 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++@@ -350,6 +350,7 @@ public class BrokerConfig extends BrokerIdentity { ++ ++ private MetricsExporterType metricsExporterType = MetricsExporterType.DISABLE; ++ +++ private int metricsOtelCardinalityLimit = 50 * 1000; ++ private String metricsGrpcExporterTarget = ""; ++ private String metricsGrpcExporterHeader = ""; ++ private long metricGrpcExporterTimeOutInMills = 3 * 1000; ++@@ -1531,6 +1532,14 @@ public class BrokerConfig extends BrokerIdentity { ++ this.metricsExporterType = MetricsExporterType.valueOf(metricsExporterType); ++ } ++ +++ public int getMetricsOtelCardinalityLimit() { +++ return metricsOtelCardinalityLimit; +++ } +++ +++ public void setMetricsOtelCardinalityLimit(int metricsOtelCardinalityLimit) { +++ this.metricsOtelCardinalityLimit = metricsOtelCardinalityLimit; +++ } +++ ++ public String getMetricsGrpcExporterTarget() { ++ return metricsGrpcExporterTarget; ++ } ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++index 9b30a3b43..650740bcc 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++@@ -203,7 +203,7 @@ public class ControllerMetricsManager { ++ 10 * s ++ ); ++ ++- View latecyView = View.builder() +++ View latencyView = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(latencyBuckets)) ++ .build(); ++ ++@@ -217,8 +217,8 @@ public class ControllerMetricsManager { ++ .setName(HISTOGRAM_DLEDGER_OP_LATENCY) ++ .build(); ++ ++- providerBuilder.registerView(requestLatencySelector, latecyView); ++- providerBuilder.registerView(dLedgerOpLatencySelector, latecyView); +++ providerBuilder.registerView(requestLatencySelector, latencyView); +++ providerBuilder.registerView(dLedgerOpLatencySelector, latencyView); ++ } ++ ++ private void initMetric(Meter meter) { ++diff --git a/pom.xml b/pom.xml ++index 3a08d75f2..9f0b3eb96 100644 ++--- a/pom.xml +++++ b/pom.xml ++@@ -133,8 +133,8 @@ ++ 2.9.3 ++ 5.3.27 ++ 3.0.0 ++- 1.26.0 ++- 1.26.0-alpha +++ 1.29.0 +++ 1.29.0-alpha ++ 2.0.6 ++ 2.20.29 ++ 1.0.3 ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java b/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java ++index 34136f94f..2e0d70856 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/metrics/RemotingMetricsManager.java ++@@ -26,6 +26,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.time.Duration; ++ import java.util.Arrays; ++ import java.util.List; ++@@ -61,7 +62,7 @@ public class RemotingMetricsManager { ++ .build(); ++ } ++ ++- public static List> getMetricsView() { +++ public static List> getMetricsView() { ++ List rpcCostTimeBuckets = Arrays.asList( ++ (double) Duration.ofMillis(1).toMillis(), ++ (double) Duration.ofMillis(3).toMillis(), ++@@ -77,10 +78,9 @@ public class RemotingMetricsManager { ++ .setType(InstrumentType.HISTOGRAM) ++ .setName(HISTOGRAM_RPC_LATENCY) ++ .build(); ++- View view = View.builder() ++- .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)) ++- .build(); ++- return Lists.newArrayList(new Pair<>(selector, view)); +++ ViewBuilder viewBuilder = View.builder() +++ .setAggregation(Aggregation.explicitBucketHistogram(rpcCostTimeBuckets)); +++ return Lists.newArrayList(new Pair<>(selector, viewBuilder)); ++ } ++ ++ public static String getWriteAndFlushResult(Future future) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++index 25e4a166f..6115ead59 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++@@ -22,7 +22,7 @@ import io.openmessaging.storage.dledger.entry.DLedgerEntry; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.io.File; ++ import java.io.IOException; ++ import java.io.RandomAccessFile; ++@@ -42,23 +42,24 @@ import java.util.Map; ++ import java.util.Objects; ++ import java.util.Optional; ++ import java.util.Set; ++-import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.TimeUnit; ++-import java.util.concurrent.TimeoutException; ++-import java.util.concurrent.ExecutionException; ++-import java.util.concurrent.Executors; ++-import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.ThreadPoolExecutor; ++-import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.CompletableFuture; ++-import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentLinkedQueue; +++import java.util.concurrent.ConcurrentMap; +++import java.util.concurrent.ExecutionException; +++import java.util.concurrent.ExecutorService; +++import java.util.concurrent.Executors; +++import java.util.concurrent.LinkedBlockingQueue; +++import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ThreadPoolExecutor; +++import java.util.concurrent.TimeUnit; +++import java.util.concurrent.TimeoutException; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.concurrent.atomic.AtomicLong; ++ import java.util.function.Supplier; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.AbstractBrokerRunnable; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.BrokerConfig; ++ import org.apache.rocketmq.common.BrokerIdentity; ++ import org.apache.rocketmq.common.MixAll; ++@@ -82,7 +83,6 @@ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.common.utils.CleanupPolicyUtils; ++ import org.apache.rocketmq.common.utils.QueueTypeUtils; ++ import org.apache.rocketmq.common.utils.ServiceProvider; ++-import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; ++@@ -3268,7 +3268,7 @@ public class DefaultMessageStore implements MessageStore { ++ } ++ ++ @Override ++- public List> getMetricsView() { +++ public List> getMetricsView() { ++ return DefaultStoreMetricsManager.getMetricsView(); ++ } ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java ++index 31bbb907f..989cbbe31 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java ++@@ -19,8 +19,7 @@ package org.apache.rocketmq.store; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; ++- +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.nio.ByteBuffer; ++ import java.util.HashMap; ++ import java.util.LinkedList; ++@@ -28,7 +27,6 @@ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.function.Supplier; ++- ++ import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.SystemClock; ++@@ -964,7 +962,7 @@ public interface MessageStore { ++ * ++ * @return List of metrics selector and view pair ++ */ ++- List> getMetricsView(); +++ List> getMetricsView(); ++ ++ /** ++ * Init store metrics ++diff --git a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java ++index ff87f6369..45a6bbc68 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java +++++ b/store/src/main/java/org/apache/rocketmq/store/metrics/DefaultStoreMetricsManager.java ++@@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongCounter; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.api.metrics.ObservableLongGauge; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.io.File; ++ import java.util.List; ++ import java.util.function.Supplier; ++@@ -69,7 +69,7 @@ public class DefaultStoreMetricsManager { ++ public static LongCounter timerDequeueTotal = new NopLongCounter(); ++ public static LongCounter timerEnqueueTotal = new NopLongCounter(); ++ ++- public static List> getMetricsView() { +++ public static List> getMetricsView() { ++ return Lists.newArrayList(); ++ } ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java ++index 25e947512..ab9fc6da7 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java ++@@ -20,8 +20,7 @@ package org.apache.rocketmq.store.plugin; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; ++- +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.nio.ByteBuffer; ++ import java.util.HashMap; ++ import java.util.LinkedList; ++@@ -29,7 +28,6 @@ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.function.Supplier; ++- ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.SystemClock; ++ import org.apache.rocketmq.common.message.MessageExt; ++@@ -643,7 +641,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { ++ } ++ ++ @Override ++- public List> getMetricsView() { +++ public List> getMetricsView() { ++ return next.getMetricsView(); ++ } ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index ced1fb818..5240ac8e9 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -21,7 +21,7 @@ import io.opentelemetry.api.common.Attributes; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++-import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++@@ -352,8 +352,8 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ } ++ ++ @Override ++- public List> getMetricsView() { ++- List> res = super.getMetricsView(); +++ public List> getMetricsView() { +++ List> res = super.getMetricsView(); ++ res.addAll(TieredStoreMetricsManager.getMetricsView()); ++ return res; ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java ++index 3ca0fb614..d8a07f0a7 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/metrics/TieredStoreMetricsManager.java ++@@ -27,6 +27,7 @@ import io.opentelemetry.sdk.metrics.Aggregation; ++ import io.opentelemetry.sdk.metrics.InstrumentSelector; ++ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.View; +++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import java.util.ArrayList; ++ import java.util.Arrays; ++ import java.util.HashMap; ++@@ -101,8 +102,8 @@ public class TieredStoreMetricsManager { ++ public static ObservableLongGauge storageSize = new NopObservableLongGauge(); ++ public static ObservableLongGauge storageMessageReserveTime = new NopObservableLongGauge(); ++ ++- public static List> getMetricsView() { ++- ArrayList> res = new ArrayList<>(); +++ public static List> getMetricsView() { +++ ArrayList> res = new ArrayList<>(); ++ ++ InstrumentSelector providerRpcLatencySelector = InstrumentSelector.builder() ++ .setType(InstrumentType.HISTOGRAM) ++@@ -114,10 +115,9 @@ public class TieredStoreMetricsManager { ++ .setName(HISTOGRAM_API_LATENCY) ++ .build(); ++ ++- View rpcLatencyView = View.builder() +++ ViewBuilder rpcLatencyViewBuilder = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d, 3d, 5d, 7d, 10d, 100d, 200d, 400d, 600d, 800d, 1d * 1000, 1d * 1500, 1d * 3000))) ++- .setDescription("tiered_store_rpc_latency_view") ++- .build(); +++ .setDescription("tiered_store_rpc_latency_view"); ++ ++ InstrumentSelector uploadBufferSizeSelector = InstrumentSelector.builder() ++ .setType(InstrumentType.HISTOGRAM) ++@@ -129,15 +129,14 @@ public class TieredStoreMetricsManager { ++ .setName(HISTOGRAM_DOWNLOAD_BYTES) ++ .build(); ++ ++- View bufferSizeView = View.builder() +++ ViewBuilder bufferSizeViewBuilder = View.builder() ++ .setAggregation(Aggregation.explicitBucketHistogram(Arrays.asList(1d * TieredStoreUtil.KB, 10d * TieredStoreUtil.KB, 100d * TieredStoreUtil.KB, 1d * TieredStoreUtil.MB, 10d * TieredStoreUtil.MB, 32d * TieredStoreUtil.MB, 50d * TieredStoreUtil.MB, 100d * TieredStoreUtil.MB))) ++- .setDescription("tiered_store_buffer_size_view") ++- .build(); +++ .setDescription("tiered_store_buffer_size_view"); ++ ++- res.add(new Pair<>(rpcLatencySelector, rpcLatencyView)); ++- res.add(new Pair<>(providerRpcLatencySelector, rpcLatencyView)); ++- res.add(new Pair<>(uploadBufferSizeSelector, bufferSizeView)); ++- res.add(new Pair<>(downloadBufferSizeSelector, bufferSizeView)); +++ res.add(new Pair<>(rpcLatencySelector, rpcLatencyViewBuilder)); +++ res.add(new Pair<>(providerRpcLatencySelector, rpcLatencyViewBuilder)); +++ res.add(new Pair<>(uploadBufferSizeSelector, bufferSizeViewBuilder)); +++ res.add(new Pair<>(downloadBufferSizeSelector, bufferSizeViewBuilder)); ++ return res; ++ } ++ ++-- ++2.32.0.windows.2 ++ ++ ++From a4bcc2a74d8bec9c9d34565536e87df06e0b11c1 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Thu, 17 Aug 2023 13:53:48 +0800 ++Subject: [PATCH 2/7] [ISSUE #7178] refresh metadata after broker startup ++ ++Signed-off-by: Ziy1-Tan ++--- ++ .../rocketmq/broker/BrokerController.java | 24 +++++++++---------- ++ 1 file changed, 12 insertions(+), 12 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 30b1d2299..13f9d002b 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -663,7 +663,7 @@ public class BrokerController { ++ BrokerController.this.getSlaveSynchronize().syncAll(); ++ lastSyncTimeMs = System.currentTimeMillis(); ++ } ++- +++ ++ //timer checkpoint, latency-sensitive, so sync it more frequently ++ if (messageStoreConfig.isTimerWheelEnable()) { ++ BrokerController.this.getSlaveSynchronize().syncTimerCheckPoint(); ++@@ -698,17 +698,6 @@ public class BrokerController { ++ ++ initializeBrokerScheduledTasks(); ++ ++- this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { ++- @Override ++- public void run() { ++- try { ++- BrokerController.this.brokerOuterAPI.refreshMetadata(); ++- } catch (Exception e) { ++- LOG.error("ScheduledTask refresh metadata exception", e); ++- } ++- } ++- }, 10, 5, TimeUnit.SECONDS); ++- ++ if (this.brokerConfig.getNamesrvAddr() != null) { ++ this.updateNamesrvAddr(); ++ LOG.info("Set user specified name server address: {}", this.brokerConfig.getNamesrvAddr()); ++@@ -1682,6 +1671,17 @@ public class BrokerController { ++ if (brokerConfig.isSkipPreOnline()) { ++ startServiceWithoutCondition(); ++ } +++ +++ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { +++ @Override +++ public void run() { +++ try { +++ BrokerController.this.brokerOuterAPI.refreshMetadata(); +++ } catch (Exception e) { +++ LOG.error("ScheduledTask refresh metadata exception", e); +++ } +++ } +++ }, 10, 5, TimeUnit.SECONDS); ++ } ++ ++ protected void scheduleSendHeartbeat() { ++-- ++2.32.0.windows.2 ++ ++ ++From 3df1b9232af99944cb3d4d4d2d00c5a85cd3b57d Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Thu, 17 Aug 2023 13:59:04 +0800 ++Subject: [PATCH 3/7] [ISSUE #7201] Remove the DefaultMessageStore.class ++ dependency in TransientStorePool ++ ++Co-authored-by: guyinyou ++--- ++ .../rocketmq/store/AllocateMappedFileService.java | 6 +++--- ++ .../apache/rocketmq/store/DefaultMessageStore.java | 7 +++++-- ++ .../apache/rocketmq/store/TransientStorePool.java | 13 ++++--------- ++ 3 files changed, 12 insertions(+), 14 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java ++index dca7d5325..c8420fea1 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java +++++ b/store/src/main/java/org/apache/rocketmq/store/AllocateMappedFileService.java ++@@ -55,7 +55,7 @@ public class AllocateMappedFileService extends ServiceThread { ++ if (this.messageStore.isTransientStorePoolEnable()) { ++ if (this.messageStore.getMessageStoreConfig().isFastFailIfNoBufferInStorePool() ++ && BrokerRole.SLAVE != this.messageStore.getMessageStoreConfig().getBrokerRole()) { //if broker is slave, don't fast fail even no buffer in pool ++- canSubmitRequests = this.messageStore.getTransientStorePool().availableBufferNums() - this.requestQueue.size(); +++ canSubmitRequests = this.messageStore.remainTransientStoreBufferNumbs() - this.requestQueue.size(); ++ } ++ } ++ ++@@ -65,7 +65,7 @@ public class AllocateMappedFileService extends ServiceThread { ++ if (nextPutOK) { ++ if (canSubmitRequests <= 0) { ++ log.warn("[NOTIFYME]TransientStorePool is not enough, so create mapped file error, " + ++- "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums()); +++ "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs()); ++ this.requestTable.remove(nextFilePath); ++ return null; ++ } ++@@ -81,7 +81,7 @@ public class AllocateMappedFileService extends ServiceThread { ++ if (nextNextPutOK) { ++ if (canSubmitRequests <= 0) { ++ log.warn("[NOTIFYME]TransientStorePool is not enough, so skip preallocate mapped file, " + ++- "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.getTransientStorePool().availableBufferNums()); +++ "RequestQueueSize : {}, StorePoolSize: {}", this.requestQueue.size(), this.messageStore.remainTransientStoreBufferNumbs()); ++ this.requestTable.remove(nextNextFilePath); ++ } else { ++ boolean offerOK = this.requestQueue.offer(nextNextReq); ++diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++index 6115ead59..f2a54ddf6 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++@@ -250,7 +250,7 @@ public class DefaultMessageStore implements MessageStore { ++ this.reputMessageService = new ConcurrentReputMessageService(); ++ } ++ ++- this.transientStorePool = new TransientStorePool(this); +++ this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog()); ++ ++ this.scheduledExecutorService = ++ Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); ++@@ -1983,7 +1983,10 @@ public class DefaultMessageStore implements MessageStore { ++ } ++ ++ public int remainTransientStoreBufferNumbs() { ++- return this.transientStorePool.availableBufferNums(); +++ if (this.isTransientStorePoolEnable()) { +++ return this.transientStorePool.availableBufferNums(); +++ } +++ return Integer.MAX_VALUE; ++ } ++ ++ @Override ++diff --git a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java ++index 8c1a5338b..0d42ee69e 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java +++++ b/store/src/main/java/org/apache/rocketmq/store/TransientStorePool.java ++@@ -33,13 +33,11 @@ public class TransientStorePool { ++ private final int poolSize; ++ private final int fileSize; ++ private final Deque availableBuffers; ++- private final DefaultMessageStore messageStore; ++ private volatile boolean isRealCommit = true; ++ ++- public TransientStorePool(final DefaultMessageStore messageStore) { ++- this.messageStore = messageStore; ++- this.poolSize = messageStore.getMessageStoreConfig().getTransientStorePoolSize(); ++- this.fileSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); +++ public TransientStorePool(final int poolSize, final int fileSize) { +++ this.poolSize = poolSize; +++ this.fileSize = fileSize; ++ this.availableBuffers = new ConcurrentLinkedDeque<>(); ++ } ++ ++@@ -81,10 +79,7 @@ public class TransientStorePool { ++ } ++ ++ public int availableBufferNums() { ++- if (messageStore.isTransientStorePoolEnable()) { ++- return availableBuffers.size(); ++- } ++- return Integer.MAX_VALUE; +++ return availableBuffers.size(); ++ } ++ ++ public boolean isRealCommit() { ++-- ++2.32.0.windows.2 ++ ++ ++From 2b93e1e32fd458d9df2091e89ea259ddd4d54061 Mon Sep 17 00:00:00 2001 ++From: iamgd67 ++Date: Thu, 17 Aug 2023 15:31:14 +0800 ++Subject: [PATCH 4/7] Update mqbroker to use runbroker.sh instead of ++ runserver.sh when use --enable-proxy (#7150) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Update mqbroker to use runbroker.sh instead of runserver.sh when enabling `--enable-proxy` ++this allow JVM `heap` and `gc` configuration using broker's settings instead of other common serverices'(proxy,namenode, etc). ++our main purpose, like the filename `mqbroker` suggest, is to start broker (which embeds a proxy), so use broker's config is reasonable ++ ++chinese version ++mqbroker的--enable-proxy选项是启动内嵌了proxy的broker，而不是内嵌broker的proxy，而且broker的工作量和重要程度大于proxy，所以使用broker的gc和heap配置更合适 ++--- ++ distribution/bin/mqbroker | 4 ++-- ++ 1 file changed, 2 insertions(+), 2 deletions(-) ++ ++diff --git a/distribution/bin/mqbroker b/distribution/bin/mqbroker ++index 3758ed597..35eb93c44 100644 ++--- a/distribution/bin/mqbroker +++++ b/distribution/bin/mqbroker ++@@ -68,11 +68,11 @@ if [ "$enable_proxy" = true ]; then ++ if [ "$broker_config" != "" ]; then ++ args_for_proxy=${args_for_proxy}" -bc "${broker_config} ++ fi ++- sh ${ROCKETMQ_HOME}/bin/runserver.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy} +++ sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.proxy.logback.xml org.apache.rocketmq.proxy.ProxyStartup ${args_for_proxy} ++ else ++ args_for_broker=$other_args ++ if [ "$broker_config" != "" ]; then ++ args_for_broker=${args_for_broker}" -c "${broker_config} ++ fi ++ sh ${ROCKETMQ_HOME}/bin/runbroker.sh -Drmq.logback.configurationFile=$ROCKETMQ_HOME/conf/rmq.broker.logback.xml org.apache.rocketmq.broker.BrokerStartup ${args_for_broker} ++-fi ++\ No newline at end of file +++fi ++-- ++2.32.0.windows.2 ++ ++ ++From 05e7cde610255ed9410fffb0f153efe7c2c8a326 Mon Sep 17 00:00:00 2001 ++From: yao-wenbin ++Date: Fri, 18 Aug 2023 09:49:59 +0800 ++Subject: [PATCH 5/7] [ISSUE #7042] maven-compile job failed, Because TlsTest's ++ serverRejectsSSLClient test case will throw TooLongFrameException (#7179) ++ ++--- ++ .../remoting/netty/NettyRemotingServer.java | 2 +- ++ .../java/org/apache/rocketmq/remoting/TlsTest.java | 14 ++++++++++++-- ++ 2 files changed, 13 insertions(+), 3 deletions(-) ++ ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++index 90e358ce3..17f138f86 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++@@ -502,7 +502,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ case DISABLED: ++ ctx.close(); ++ log.warn("Clients intend to establish an SSL connection while this server is running in SSL disabled mode"); ++- break; +++ throw new UnsupportedOperationException("The NettyRemotingServer in SSL disabled mode doesn't support ssl client"); ++ case PERMISSIVE: ++ case ENFORCING: ++ if (null != sslContext) { ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java ++index de7edbbfb..a4890d73d 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/TlsTest.java ++@@ -144,8 +144,13 @@ public class TlsTest { ++ tlsClientKeyPath = ""; ++ tlsClientCertPath = ""; ++ clientConfig.setUseTLS(false); ++- } else if ("serverRejectsSSLClient".equals(name.getMethodName())) { +++ } else if ("disabledServerRejectsSSLClient".equals(name.getMethodName())) { ++ tlsMode = TlsMode.DISABLED; +++ } else if ("disabledServerAcceptUnAuthClient".equals(name.getMethodName())) { +++ tlsMode = TlsMode.DISABLED; +++ tlsClientKeyPath = ""; +++ tlsClientCertPath = ""; +++ clientConfig.setUseTLS(false); ++ } else if ("reloadSslContextForServer".equals(name.getMethodName())) { ++ tlsClientAuthServer = false; ++ tlsServerNeedClientAuth = "none"; ++@@ -211,7 +216,7 @@ public class TlsTest { ++ } ++ ++ @Test ++- public void serverRejectsSSLClient() throws Exception { +++ public void disabledServerRejectsSSLClient() throws Exception { ++ try { ++ RemotingCommand response = remotingClient.invokeSync(getServerAddress(), createRequest(), 1000 * 5); ++ failBecauseExceptionWasNotThrown(RemotingSendRequestException.class); ++@@ -219,6 +224,11 @@ public class TlsTest { ++ } ++ } ++ +++ @Test +++ public void disabledServerAcceptUnAuthClient() throws Exception { +++ requestThenAssertResponse(); +++ } +++ ++ /** ++ * Tests that a server configured to require client authentication refuses to accept connections ++ * from a client that has an untrusted certificate. ++-- ++2.32.0.windows.2 ++ ++ ++From 72d796f2b20b3ec6aebca8c004d9275d7c749a95 Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Fri, 18 Aug 2023 11:55:39 +0800 ++Subject: [PATCH 6/7] [ISSUE #7205] support batch ack for pop orderly (#7206) ++ ++--- ++ .../broker/processor/AckMessageProcessor.java | 99 ++++++----- ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 91 ++++++++-- ++ .../test/client/rmq/RMQPopClient.java | 22 +++ ++ .../client/consumer/pop/BasePopNormally.java | 6 + ++ .../test/client/consumer/pop/BatchAckIT.java | 159 ++++++++++++++++++ ++ 5 files changed, 322 insertions(+), 55 deletions(-) ++ create mode 100644 test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java ++index 687811409..244b459d6 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AckMessageProcessor.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.processor; ++ import com.alibaba.fastjson.JSON; ++ import io.netty.channel.Channel; ++ import io.netty.channel.ChannelHandlerContext; +++import java.util.BitSet; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.broker.metrics.PopMetricsManager; ++ import org.apache.rocketmq.common.KeyBuilder; ++@@ -186,46 +187,7 @@ public class AckMessageProcessor implements NettyRequestProcessor { ++ invisibleTime = ExtraInfoUtil.getInvisibleTime(extraInfo); ++ ++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { ++- // order ++- String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; ++- long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); ++- if (ackOffset < oldOffset) { ++- return; ++- } ++- while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { ++- } ++- try { ++- oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); ++- if (ackOffset < oldOffset) { ++- return; ++- } ++- long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( ++- topic, consumeGroup, ++- qId, ackOffset, ++- popTime); ++- if (nextOffset > -1) { ++- if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( ++- topic, consumeGroup, qId)) { ++- this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), ++- consumeGroup, topic, qId, nextOffset); ++- } ++- if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, ++- consumeGroup, qId, invisibleTime)) { ++- this.brokerController.getPopMessageProcessor().notifyMessageArriving( ++- topic, consumeGroup, qId); ++- } ++- } else if (nextOffset == -1) { ++- String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", ++- lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress()); ++- POP_LOGGER.warn(errorInfo); ++- response.setCode(ResponseCode.MESSAGE_ILLEGAL); ++- response.setRemark(errorInfo); ++- return; ++- } ++- } finally { ++- this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); ++- } ++- brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); +++ ackOrderly(topic, consumeGroup, qId, ackOffset, popTime, invisibleTime, channel, response); ++ return; ++ } ++ ++@@ -250,17 +212,22 @@ public class AckMessageProcessor implements NettyRequestProcessor { ++ } ++ ++ BatchAckMsg batchAckMsg = new BatchAckMsg(); ++- for (int i = 0; batchAck.getBitSet() != null && i < batchAck.getBitSet().length(); i++) { ++- if (!batchAck.getBitSet().get(i)) { ++- continue; +++ BitSet bitSet = batchAck.getBitSet(); +++ for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) { +++ if (i == Integer.MAX_VALUE) { +++ break; ++ } ++ long offset = startOffset + i; ++ if (offset < minOffset || offset > maxOffset) { ++ continue; ++ } ++- batchAckMsg.getAckOffsetList().add(offset); +++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE) { +++ ackOrderly(topic, consumeGroup, qId, offset, popTime, invisibleTime, channel, response); +++ } else { +++ batchAckMsg.getAckOffsetList().add(offset); +++ } ++ } ++- if (batchAckMsg.getAckOffsetList().isEmpty()) { +++ if (rqId == KeyBuilder.POP_ORDER_REVIVE_QUEUE || batchAckMsg.getAckOffsetList().isEmpty()) { ++ return; ++ } ++ ++@@ -311,4 +278,46 @@ public class AckMessageProcessor implements NettyRequestProcessor { ++ PopMetricsManager.incPopReviveAckPutCount(ackMsg, putMessageResult.getPutMessageStatus()); ++ brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, ackCount); ++ } +++ +++ protected void ackOrderly(String topic, String consumeGroup, int qId, long ackOffset, long popTime, long invisibleTime, Channel channel, RemotingCommand response) { +++ String lockKey = topic + PopAckConstants.SPLIT + consumeGroup + PopAckConstants.SPLIT + qId; +++ long oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); +++ if (ackOffset < oldOffset) { +++ return; +++ } +++ while (!this.brokerController.getPopMessageProcessor().getQueueLockManager().tryLock(lockKey)) { +++ } +++ try { +++ oldOffset = this.brokerController.getConsumerOffsetManager().queryOffset(consumeGroup, topic, qId); +++ if (ackOffset < oldOffset) { +++ return; +++ } +++ long nextOffset = brokerController.getConsumerOrderInfoManager().commitAndNext( +++ topic, consumeGroup, +++ qId, ackOffset, +++ popTime); +++ if (nextOffset > -1) { +++ if (!this.brokerController.getConsumerOffsetManager().hasOffsetReset( +++ topic, consumeGroup, qId)) { +++ this.brokerController.getConsumerOffsetManager().commitOffset(channel.remoteAddress().toString(), +++ consumeGroup, topic, qId, nextOffset); +++ } +++ if (!this.brokerController.getConsumerOrderInfoManager().checkBlock(null, topic, +++ consumeGroup, qId, invisibleTime)) { +++ this.brokerController.getPopMessageProcessor().notifyMessageArriving( +++ topic, consumeGroup, qId); +++ } +++ } else if (nextOffset == -1) { +++ String errorInfo = String.format("offset is illegal, key:%s, old:%d, commit:%d, next:%d, %s", +++ lockKey, oldOffset, ackOffset, nextOffset, channel.remoteAddress()); +++ POP_LOGGER.warn(errorInfo); +++ response.setCode(ResponseCode.MESSAGE_ILLEGAL); +++ response.setRemark(errorInfo); +++ return; +++ } +++ } finally { +++ this.brokerController.getPopMessageProcessor().getQueueLockManager().unLock(lockKey); +++ } +++ brokerController.getPopInflightMessageCounter().decrementInFlightMessageNum(topic, consumeGroup, popTime, qId, 1); +++ } ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 5101ffc8e..213c26fd6 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -21,6 +21,7 @@ import java.io.UnsupportedEncodingException; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; ++ import java.util.Arrays; +++import java.util.BitSet; ++ import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.Iterator; ++@@ -54,6 +55,7 @@ import org.apache.rocketmq.client.producer.SendCallback; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.client.producer.SendStatus; ++ import org.apache.rocketmq.common.AclConfig; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.MQVersion; ++ import org.apache.rocketmq.common.MixAll; ++ import org.apache.rocketmq.common.Pair; ++@@ -76,7 +78,8 @@ import org.apache.rocketmq.common.namesrv.NameServerUpdateCallback; ++ import org.apache.rocketmq.common.namesrv.TopAddressing; ++ import org.apache.rocketmq.common.sysflag.PullSysFlag; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++-import org.apache.rocketmq.common.BoundaryType; +++import org.apache.rocketmq.logging.org.slf4j.Logger; +++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.CommandCustomHeader; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.RPCHook; ++@@ -101,7 +104,10 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.apache.rocketmq.remoting.protocol.admin.ConsumeStats; ++ import org.apache.rocketmq.remoting.protocol.admin.TopicStatsTable; +++import org.apache.rocketmq.remoting.protocol.body.BatchAck; +++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; +++import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; ++ import org.apache.rocketmq.remoting.protocol.body.BrokerStatsData; ++ import org.apache.rocketmq.remoting.protocol.body.CheckClientRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.ClusterAclVersionInfo; ++@@ -114,7 +120,6 @@ import org.apache.rocketmq.remoting.protocol.body.EpochEntryCache; ++ import org.apache.rocketmq.remoting.protocol.body.GetConsumerStatusBody; ++ import org.apache.rocketmq.remoting.protocol.body.GroupList; ++ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; ++-import org.apache.rocketmq.remoting.protocol.body.BrokerReplicasInfo; ++ import org.apache.rocketmq.remoting.protocol.body.KVTable; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchResponseBody; ++@@ -196,6 +201,10 @@ import org.apache.rocketmq.remoting.protocol.header.UpdateGlobalWhiteAddrsConfig ++ import org.apache.rocketmq.remoting.protocol.header.UpdateGroupForbiddenRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.ViewBrokerStatsDataRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.ViewMessageRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.AddWritePermOfBrokerResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.DeleteKVConfigRequestHeader; ++@@ -207,10 +216,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.GetRouteInfoRequestH ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.PutKVConfigRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.WipeWritePermOfBrokerResponseHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.heartbeat.HeartbeatData; ++ import org.apache.rocketmq.remoting.protocol.heartbeat.MessageModel; ++ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; ++@@ -221,8 +226,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.GroupForbidden; ++ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; ++ import org.apache.rocketmq.remoting.rpchook.DynamicalExtFieldRPCHook; ++ import org.apache.rocketmq.remoting.rpchook.StreamTypeRPCHook; ++-import org.apache.rocketmq.logging.org.slf4j.Logger; ++-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ import static org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode.SUCCESS; ++ ++@@ -885,9 +888,77 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ final String addr, ++ final long timeOut, ++ final AckCallback ackCallback, ++- final AckMessageRequestHeader requestHeader // +++ final AckMessageRequestHeader requestHeader +++ ) throws RemotingException, MQBrokerException, InterruptedException { +++ ackMessageAsync(addr, timeOut, ackCallback, requestHeader, null); +++ } +++ +++ public void batchAckMessageAsync( +++ final String addr, +++ final long timeOut, +++ final AckCallback ackCallback, +++ final String topic, +++ final String consumerGroup, +++ final List extraInfoList +++ ) throws RemotingException, MQBrokerException, InterruptedException { +++ String brokerName = null; +++ Map batchAckMap = new HashMap<>(); +++ for (String extraInfo : extraInfoList) { +++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); +++ if (brokerName == null) { +++ brokerName = ExtraInfoUtil.getBrokerName(extraInfoData); +++ } +++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + +++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + +++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + +++ ExtraInfoUtil.getPopTime(extraInfoData); +++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { +++ BatchAck newBatchAck = new BatchAck(); +++ newBatchAck.setConsumerGroup(consumerGroup); +++ newBatchAck.setTopic(topic); +++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); +++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); +++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); +++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); +++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); +++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); +++ newBatchAck.setBitSet(new BitSet()); +++ return newBatchAck; +++ }); +++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); +++ } +++ +++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); +++ requestBody.setBrokerName(brokerName); +++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); +++ batchAckMessageAsync(addr, timeOut, ackCallback, requestBody); +++ } +++ +++ public void batchAckMessageAsync( +++ final String addr, +++ final long timeOut, +++ final AckCallback ackCallback, +++ final BatchAckMessageRequestBody requestBody ++ ) throws RemotingException, MQBrokerException, InterruptedException { ++- final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); +++ ackMessageAsync(addr, timeOut, ackCallback, null, requestBody); +++ } +++ +++ protected void ackMessageAsync( +++ final String addr, +++ final long timeOut, +++ final AckCallback ackCallback, +++ final AckMessageRequestHeader requestHeader, +++ final BatchAckMessageRequestBody requestBody +++ ) throws RemotingException, MQBrokerException, InterruptedException { +++ RemotingCommand request; +++ if (requestHeader != null) { +++ request = RemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); +++ } else { +++ request = RemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); +++ if (requestBody != null) { +++ request.setBody(requestBody.encode()); +++ } +++ } ++ this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) { ++ ++ @Override ++diff --git a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java ++index 496bd6da4..09c60c0b4 100644 ++--- a/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java +++++ b/test/src/main/java/org/apache/rocketmq/test/client/rmq/RMQPopClient.java ++@@ -17,6 +17,7 @@ ++ ++ package org.apache.rocketmq.test.client.rmq; ++ +++import java.util.List; ++ import java.util.concurrent.CompletableFuture; ++ import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.consumer.AckCallback; ++@@ -140,6 +141,27 @@ public class RMQPopClient implements MQConsumer { ++ return future; ++ } ++ +++ public CompletableFuture batchAckMessageAsync(String brokerAddr, String topic, String consumerGroup, +++ List extraInfoList) { +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ this.mqClientAPI.batchAckMessageAsync(brokerAddr, DEFAULT_TIMEOUT, new AckCallback() { +++ @Override +++ public void onSuccess(AckResult ackResult) { +++ future.complete(ackResult); +++ } +++ +++ @Override +++ public void onException(Throwable e) { +++ future.completeExceptionally(e); +++ } +++ }, topic, consumerGroup, extraInfoList); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ ++ public CompletableFuture changeInvisibleTimeAsync(String brokerAddr, String brokerName, String topic, ++ String consumerGroup, String extraInfo, long invisibleTime) { ++ String[] extraInfoStrs = ExtraInfoUtil.split(extraInfo); ++diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java ++index 952fbe3f5..2e29b95a5 100644 ++--- a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java +++++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BasePopNormally.java ++@@ -63,4 +63,10 @@ public class BasePopNormally extends BasePop { ++ brokerAddr, messageQueue, invisibleTime, maxNums, group, timeout, true, ++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); ++ } +++ +++ protected CompletableFuture popMessageAsync(long invisibleTime, int maxNums) { +++ return client.popMessageAsync( +++ brokerAddr, messageQueue, invisibleTime, maxNums, group, 3000, false, +++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); +++ } ++ } ++diff --git a/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java ++new file mode 100644 ++index 000000000..ec9153ccc ++--- /dev/null +++++ b/test/src/test/java/org/apache/rocketmq/test/client/consumer/pop/BatchAckIT.java ++@@ -0,0 +1,159 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.test.client.consumer.pop; +++ +++import java.time.Duration; +++import java.util.ArrayList; +++import java.util.List; +++import java.util.concurrent.CompletableFuture; +++import java.util.concurrent.TimeUnit; +++import java.util.concurrent.atomic.AtomicInteger; +++import java.util.function.Supplier; +++import org.apache.rocketmq.client.consumer.AckResult; +++import org.apache.rocketmq.client.consumer.AckStatus; +++import org.apache.rocketmq.client.consumer.PopResult; +++import org.apache.rocketmq.client.consumer.PopStatus; +++import org.apache.rocketmq.common.attribute.CQType; +++import org.apache.rocketmq.common.attribute.TopicMessageType; +++import org.apache.rocketmq.common.constant.ConsumeInitMode; +++import org.apache.rocketmq.common.filter.ExpressionType; +++import org.apache.rocketmq.common.message.MessageConst; +++import org.apache.rocketmq.common.message.MessageExt; +++import org.apache.rocketmq.common.message.MessageQueue; +++import org.apache.rocketmq.test.base.IntegrationTestBase; +++import org.apache.rocketmq.test.client.rmq.RMQNormalProducer; +++import org.apache.rocketmq.test.client.rmq.RMQPopClient; +++import org.apache.rocketmq.test.util.MQRandomUtils; +++import org.junit.After; +++import org.junit.Before; +++import org.junit.Test; +++ +++import static org.awaitility.Awaitility.await; +++import static org.junit.Assert.assertEquals; +++ +++public class BatchAckIT extends BasePop { +++ +++ protected String topic; +++ protected String group; +++ protected RMQNormalProducer producer = null; +++ protected RMQPopClient client = null; +++ protected String brokerAddr; +++ protected MessageQueue messageQueue; +++ +++ @Before +++ public void setUp() { +++ brokerAddr = brokerController1.getBrokerAddr(); +++ topic = MQRandomUtils.getRandomTopic(); +++ group = initConsumerGroup(); +++ IntegrationTestBase.initTopic(topic, NAMESRV_ADDR, BROKER1_NAME, 8, CQType.SimpleCQ, TopicMessageType.NORMAL); +++ producer = getProducer(NAMESRV_ADDR, topic); +++ client = getRMQPopClient(); +++ messageQueue = new MessageQueue(topic, BROKER1_NAME, -1); +++ } +++ +++ @After +++ public void tearDown() { +++ shutdown(); +++ } +++ +++ @Test +++ public void testBatchAckNormallyWithPopBuffer() throws Throwable { +++ brokerController1.getBrokerConfig().setEnablePopBufferMerge(true); +++ brokerController2.getBrokerConfig().setEnablePopBufferMerge(true); +++ +++ testBatchAck(() -> { +++ try { +++ return popMessageAsync().get(); +++ } catch (Exception e) { +++ throw new RuntimeException(e); +++ } +++ }); +++ } +++ +++ @Test +++ public void testBatchAckNormallyWithOutPopBuffer() throws Throwable { +++ brokerController1.getBrokerConfig().setEnablePopBufferMerge(false); +++ brokerController2.getBrokerConfig().setEnablePopBufferMerge(false); +++ +++ testBatchAck(() -> { +++ try { +++ return popMessageAsync().get(); +++ } catch (Exception e) { +++ throw new RuntimeException(e); +++ } +++ }); +++ } +++ +++ @Test +++ public void testBatchAckOrderly() throws Throwable { +++ testBatchAck(() -> { +++ try { +++ return popMessageOrderlyAsync().get(); +++ } catch (Exception e) { +++ throw new RuntimeException(e); +++ } +++ }); +++ } +++ +++ public void testBatchAck(Supplier popResultSupplier) throws Throwable { +++ // Send 10 messages but do not ack, let them enter the retry topic +++ producer.send(10); +++ AtomicInteger firstMsgRcvNum = new AtomicInteger(); +++ await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { +++ PopResult popResult = popResultSupplier.get(); +++ if (popResult.getPopStatus().equals(PopStatus.FOUND)) { +++ firstMsgRcvNum.addAndGet(popResult.getMsgFoundList().size()); +++ } +++ assertEquals(10, firstMsgRcvNum.get()); +++ }); +++ // sleep 6s, expect messages to enter the retry topic +++ TimeUnit.SECONDS.sleep(6); +++ +++ producer.send(20); +++ List extraInfoList = new ArrayList<>(); +++ await().atMost(Duration.ofSeconds(3)).untilAsserted(() -> { +++ PopResult popResult = popResultSupplier.get(); +++ if (popResult.getPopStatus().equals(PopStatus.FOUND)) { +++ for (MessageExt messageExt : popResult.getMsgFoundList()) { +++ extraInfoList.add(messageExt.getProperty(MessageConst.PROPERTY_POP_CK)); +++ } +++ } +++ assertEquals(30, extraInfoList.size()); +++ }); +++ +++ AckResult ackResult = client.batchAckMessageAsync(brokerAddr, topic, group, extraInfoList).get(); +++ assertEquals(AckStatus.OK, ackResult.getStatus()); +++ +++ // sleep 6s, expected that messages that have been acked will not be re-consumed +++ TimeUnit.SECONDS.sleep(6); +++ PopResult popResult = popResultSupplier.get(); +++ assertEquals(PopStatus.POLLING_NOT_FOUND, popResult.getPopStatus()); +++ } +++ +++ private CompletableFuture popMessageAsync() { +++ return client.popMessageAsync( +++ brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false, +++ ConsumeInitMode.MIN, false, ExpressionType.TAG, "*"); +++ } +++ +++ private CompletableFuture popMessageOrderlyAsync() { +++ return client.popMessageAsync( +++ brokerAddr, messageQueue, Duration.ofSeconds(3).toMillis(), 30, group, 3000, false, +++ ConsumeInitMode.MIN, true, ExpressionType.TAG, "*", null); +++ } +++} ++-- ++2.32.0.windows.2 ++ ++ ++From cc16a1b51216e1e80c22011b8b01e060bb4af8b3 Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Tue, 22 Aug 2023 10:42:25 +0800 ++Subject: [PATCH 7/7] Set table reference the same object for ++ setSubscriptionGroupTable method (#7204) ++ ++--- ++ .../broker/subscription/SubscriptionGroupManager.java | 5 +---- ++ 1 file changed, 1 insertion(+), 4 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java ++index 74e39c0fe..e63b93058 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/subscription/SubscriptionGroupManager.java ++@@ -341,10 +341,7 @@ public class SubscriptionGroupManager extends ConfigManager { ++ ++ ++ public void setSubscriptionGroupTable(ConcurrentMap subscriptionGroupTable) { ++- this.subscriptionGroupTable.clear(); ++- for (String key : subscriptionGroupTable.keySet()) { ++- putSubscriptionGroupConfig(subscriptionGroupTable.get(key)); ++- } +++ this.subscriptionGroupTable = subscriptionGroupTable; ++ } ++ ++ public boolean containsSubscriptionGroup(String group) { ++-- ++2.32.0.windows.2 ++ +diff --git a/patch011-backport-optimize-opentelemetry-metric-config.patch b/patch011-backport-optimize-opentelemetry-metric-config.patch +new file mode 100644 +index 000000000..b59d7e2c7 +--- /dev/null ++++ b/patch011-backport-optimize-opentelemetry-metric-config.patch +@@ -0,0 +1,2081 @@ ++From 744167bd01fab6821b4d5ae1794dc845153d5156 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Wed, 23 Aug 2023 08:32:17 +0800 ++Subject: [PATCH 1/7] [ISSUE #7142] Add command `RocksDBConfigToJson` to ++ inspect rocksdb content (#7180) ++ ++* feat: add command `RocksDBConfigToJson` to inspect rocksdb content ++ ++Signed-off-by: Ziy1-Tan ++ ++* refactor: fix style ++ ++--------- ++ ++Signed-off-by: Ziy1-Tan ++Co-authored-by: Ziy1-Tan ++--- ++ .../tools/command/MQAdminStartup.java | 2 + ++ .../metadata/RocksDBConfigToJsonCommand.java | 118 ++++++++++++++++++ ++ .../metadata/KvConfigToJsonCommandTest.java | 65 ++++++++++ ++ 3 files changed, 185 insertions(+) ++ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++ create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++index 890125ca0..324aa1856 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++@@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; ++ import org.apache.rocketmq.tools.command.message.SendMessageCommand; +++import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; ++ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; ++ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; ++ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; ++@@ -211,6 +212,7 @@ public class MQAdminStartup { ++ ++ initCommand(new ClusterListSubCommand()); ++ initCommand(new TopicListSubCommand()); +++ initCommand(new RocksDBConfigToJsonCommand()); ++ ++ initCommand(new UpdateKvConfigCommand()); ++ initCommand(new DeleteKvConfigCommand()); ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++new file mode 100644 ++index 000000000..3053f4684 ++--- /dev/null +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++@@ -0,0 +1,118 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.metadata; +++ +++import com.alibaba.fastjson.JSONObject; +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.Option; +++import org.apache.commons.cli.Options; +++import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.common.config.RocksDBConfigManager; +++import org.apache.rocketmq.common.utils.DataConverter; +++import org.apache.rocketmq.remoting.RPCHook; +++import org.apache.rocketmq.tools.command.SubCommand; +++import org.apache.rocketmq.tools.command.SubCommandException; +++ +++import java.io.File; +++import java.util.HashMap; +++import java.util.Map; +++ +++public class RocksDBConfigToJsonCommand implements SubCommand { +++ private static final String TOPICS_JSON_CONFIG = "topics"; +++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; +++ +++ @Override +++ public String commandName() { +++ return "rocksDBConfigToJson"; +++ } +++ +++ @Override +++ public String commandDesc() { +++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; +++ } +++ +++ @Override +++ public Options buildCommandlineOptions(Options options) { +++ Option pathOption = new Option("p", "path", true, +++ "Absolute path to the metadata directory"); +++ pathOption.setRequired(true); +++ options.addOption(pathOption); +++ +++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + +++ "topics/subscriptionGroups"); +++ configTypeOption.setRequired(true); +++ options.addOption(configTypeOption); +++ +++ return options; +++ } +++ +++ @Override +++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { +++ String path = commandLine.getOptionValue("path").trim(); +++ if (StringUtils.isEmpty(path) || !new File(path).exists()) { +++ System.out.print("Rocksdb path is invalid.\n"); +++ return; +++ } +++ +++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); +++ +++ final long memTableFlushInterval = 60 * 60 * 1000L; +++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); +++ try { +++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { +++ // for topics.json +++ final Map topicsJsonConfig = new HashMap<>(); +++ final Map topicConfigTable = new HashMap<>(); +++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { +++ final String topic = new String(key, DataConverter.charset); +++ final String topicConfig = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig); +++ topicConfigTable.put(topic, jsonObject); +++ }); +++ +++ if (isLoad) { +++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); +++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); +++ System.out.print(topicsJsonStr + "\n"); +++ return; +++ } +++ } +++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { +++ // for subscriptionGroup.json +++ final Map subscriptionGroupJsonConfig = new HashMap<>(); +++ final Map subscriptionGroupTable = new HashMap<>(); +++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { +++ final String subscriptionGroup = new String(key, DataConverter.charset); +++ final String subscriptionGroupConfig = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); +++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); +++ }); +++ +++ if (isLoad) { +++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", +++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); +++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); +++ System.out.print(subscriptionGroupJsonStr + "\n"); +++ return; +++ } +++ } +++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); +++ } finally { +++ kvConfigManager.stop(); +++ } +++ } +++} ++diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++new file mode 100644 ++index 000000000..b2f66c7b0 ++--- /dev/null +++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++@@ -0,0 +1,65 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.metadata; +++ +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.DefaultParser; +++import org.apache.commons.cli.Options; +++import org.apache.rocketmq.srvutil.ServerUtil; +++import org.apache.rocketmq.tools.command.SubCommandException; +++import org.junit.Test; +++ +++import java.io.File; +++ +++import static org.assertj.core.api.Assertions.assertThat; +++ +++public class KvConfigToJsonCommandTest { +++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; +++ +++ @Test +++ public void testExecute() throws SubCommandException { +++ { +++ String[] cases = new String[]{"topics", "subscriptionGroups"}; +++ for (String c : cases) { +++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ Options options = ServerUtil.buildCommandlineOptions(new Options()); +++ String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; +++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, +++ cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.execute(commandLine, options, null); +++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); +++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); +++ } +++ } +++ // invalid cases +++ { +++ String[][] cases = new String[][]{ +++ {"-p " + BASE_PATH + "tmpPath", "-t topics"}, +++ {"-p ", "-t topics"}, +++ {"-p " + BASE_PATH + "topics", "-t invalid_type"} +++ }; +++ +++ for (String[] c : cases) { +++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ Options options = ServerUtil.buildCommandlineOptions(new Options()); +++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, +++ cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.execute(commandLine, options, null); +++ } +++ } +++ } +++} ++-- ++2.32.0.windows.2 ++ ++ ++From bdede35db365a49b211cdc249c68b0f60a3df46d Mon Sep 17 00:00:00 2001 ++From: mxsm ++Date: Wed, 23 Aug 2023 08:34:56 +0800 ++Subject: [PATCH 2/7] [ISSUE #7124] Fix the typos in the code comments (#7125) ++ ++--- ++ .../apache/rocketmq/broker/processor/ReplyMessageProcessor.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++index b2db356c8..d3bb048f7 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++@@ -234,7 +234,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor { ++ } else { ++ response.setCode(ResponseCode.SUCCESS); ++ response.setRemark(null); ++- //set to zore to avoid client decoding exception +++ //set to zero to avoid client decoding exception ++ responseHeader.setMsgId("0"); ++ responseHeader.setQueueId(queueIdInt); ++ responseHeader.setQueueOffset(0L); ++-- ++2.32.0.windows.2 ++ ++ ++From 9bb73b9a38548b99ac5126c40380c3c2e7fc586e Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 23 Aug 2023 09:46:27 +0800 ++Subject: [PATCH 3/7] [#ISSUE 7222] Bug fix and refactoring of the Indexfile in ++ tiered storage (#7224) ++ ++--- ++ .../tieredstore/file/TieredIndexFile.java | 38 +++++++-- ++ .../tieredstore/file/TieredIndexFileTest.java | 84 +++++-------------- ++ 2 files changed, 52 insertions(+), 70 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++index 50beb01ae..eda5e0106 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ +++import com.google.common.annotations.VisibleForTesting; ++ import java.io.File; ++ import java.io.IOException; ++ import java.nio.ByteBuffer; ++@@ -99,7 +100,7 @@ public class TieredIndexFile { ++ this::doScheduleTask, 10, 10, TimeUnit.SECONDS); ++ } ++ ++- private void doScheduleTask() { +++ protected void doScheduleTask() { ++ try { ++ curFileLock.lock(); ++ try { ++@@ -145,6 +146,11 @@ public class TieredIndexFile { ++ } ++ } ++ +++ @VisibleForTesting +++ public MappedFile getPreMappedFile() { +++ return preMappedFile; +++ } +++ ++ private void initFile() throws IOException { ++ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); ++ initIndexFileHeader(curMappedFile); ++@@ -156,19 +162,26 @@ public class TieredIndexFile { ++ ++ if (isFileSealed(curMappedFile)) { ++ if (preFileExists) { ++- preFile.delete(); +++ if (preFile.delete()) { +++ logger.info("Pre IndexFile deleted success", preFilepath); +++ } else { +++ logger.error("Pre IndexFile deleted failed", preFilepath); +++ } ++ } ++ boolean rename = curMappedFile.renameTo(preFilepath); ++ if (rename) { ++ preMappedFile = curMappedFile; ++ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); +++ initIndexFileHeader(curMappedFile); ++ preFileExists = true; ++ } ++ } +++ ++ if (preFileExists) { ++ synchronized (TieredIndexFile.class) { ++ if (inflightCompactFuture.isDone()) { ++- inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null); +++ inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit( +++ new CompactTask(storeConfig, preMappedFile, flatFile), null); ++ } ++ } ++ } ++@@ -261,7 +274,8 @@ public class TieredIndexFile { ++ } ++ } ++ ++- public CompletableFuture>> queryAsync(String topic, String key, long beginTime, long endTime) { +++ public CompletableFuture>> queryAsync(String topic, String key, long beginTime, +++ long endTime) { ++ int hashCode = indexKeyHashMethod(buildKey(topic, key)); ++ int slotPosition = hashCode % maxHashSlotNum; ++ List fileSegmentList = flatFile.getFileListByTime(beginTime, endTime); ++@@ -355,7 +369,7 @@ public class TieredIndexFile { ++ private final int fileMaxSize; ++ private MappedFile originFile; ++ private TieredFlatFile fileQueue; ++- private final MappedFile compactFile; +++ private MappedFile compactFile; ++ ++ public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile, ++ TieredFlatFile fileQueue) throws IOException { ++@@ -381,6 +395,17 @@ public class TieredIndexFile { ++ } catch (Throwable throwable) { ++ logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable); ++ } +++ +++ try { +++ if (originFile != null) { +++ originFile.destroy(-1); +++ } +++ if (compactFile != null) { +++ compactFile.destroy(-1); +++ } +++ } catch (Throwable throwable) { +++ logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable); +++ } ++ } ++ ++ public void compact() { ++@@ -396,6 +421,8 @@ public class TieredIndexFile { ++ fileQueue.commit(true); ++ compactFile.destroy(-1); ++ originFile.destroy(-1); +++ compactFile = null; +++ originFile = null; ++ } ++ ++ private void buildCompactFile() { ++@@ -414,6 +441,7 @@ public class TieredIndexFile { ++ if (slotValue != -1) { ++ int indexTotalSize = 0; ++ int indexPosition = slotValue; +++ ++ while (indexPosition >= 0 && indexPosition < maxIndexNum) { ++ int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE ++ + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE; ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++index 7ef49578d..262d6645b 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++@@ -19,9 +19,8 @@ package org.apache.rocketmq.tieredstore.file; ++ import com.sun.jna.Platform; ++ import java.io.IOException; ++ import java.nio.ByteBuffer; +++import java.time.Duration; ++ import java.util.List; ++-import java.util.concurrent.TimeUnit; ++-import org.apache.commons.lang3.SystemUtils; ++ import org.apache.commons.lang3.tuple.Pair; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; ++@@ -31,9 +30,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ import org.awaitility.Awaitility; ++ import org.junit.After; ++ import org.junit.Assert; ++-import org.junit.Assume; ++ import org.junit.Before; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++ public class TieredIndexFileTest { ++@@ -45,11 +42,12 @@ public class TieredIndexFileTest { ++ @Before ++ public void setUp() { ++ storeConfig = new TieredMessageStoreConfig(); +++ storeConfig.setBrokerName("IndexFileBroker"); ++ storeConfig.setStorePathRootDir(storePath); ++- storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment"); ++- storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2); ++- storeConfig.setTieredStoreIndexFileMaxIndexNum(3); ++- mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1); +++ storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment"); +++ storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5); +++ storeConfig.setTieredStoreIndexFileMaxIndexNum(20); +++ mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1); ++ TieredStoreUtil.getMetadataStore(storeConfig); ++ TieredStoreExecutor.init(); ++ } ++@@ -61,77 +59,33 @@ public class TieredIndexFileTest { ++ TieredStoreExecutor.shutdown(); ++ } ++ ++- @Ignore ++ @Test ++ public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException { ++ if (Platform.isWindows()) { ++ return; ++ } ++ ++- // skip this test on windows ++- Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); ++- ++ TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig); ++ TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath); +++ ++ indexFile.append(mq, 0, "key3", 3, 300, 1000); ++ indexFile.append(mq, 0, "key2", 2, 200, 1100); ++ indexFile.append(mq, 0, "key1", 1, 100, 1200); ++ ++- Awaitility.waitAtMost(5, TimeUnit.SECONDS) ++- .until(() -> { ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++- if (indexList.size() != 1) { ++- return false; ++- } ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining()); ++- ++- Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++- ++- Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4)); ++- Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4)); ++- return true; ++- }); ++- ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- ++- Awaitility.waitAtMost(5, TimeUnit.SECONDS) ++- .until(() -> { ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join(); ++- if (indexList.size() != 1) { ++- return false; ++- } ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); ++- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++- return true; ++- }); ++- ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join(); +++ // do not do schedule task here +++ TieredStoreExecutor.shutdown(); +++ List> indexList = +++ indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++ Assert.assertEquals(0, indexList.size()); ++ ++- indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join(); ++- Assert.assertEquals(2, indexList.size()); ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); ++- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +++ // do compaction once +++ TieredStoreExecutor.init(); +++ storeConfig.setTieredStoreIndexFileRollingIdleInterval(0); +++ indexFile.doScheduleTask(); +++ Awaitility.await().atMost(Duration.ofSeconds(10)) +++ .until(() -> !indexFile.getPreMappedFile().getFile().exists()); ++ ++- indexBuffer = indexList.get(1).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining()); ++- Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); +++ Assert.assertEquals(1, indexList.size()); ++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 69c26d3d29cde7b4484ecd112ab9224f9f42bf45 Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Wed, 23 Aug 2023 10:27:52 +0800 ++Subject: [PATCH 4/7] [ISSUE #7228] Converge the use of some important ++ variables for some class ++ ++--- ++ .../apache/rocketmq/store/ConsumeQueue.java | 16 ++++++------ ++ .../rocketmq/store/MappedFileQueue.java | 26 +++++++++++-------- ++ .../store/MultiPathMappedFileQueue.java | 4 +-- ++ 3 files changed, 24 insertions(+), 22 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++index a0b886eb0..56bee2af3 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++@@ -145,7 +145,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ if (offset >= 0 && size > 0) { ++ mappedFileOffset = i + CQ_STORE_UNIT_SIZE; ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++ } ++@@ -409,7 +409,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ int logicFileSize = this.mappedFileSize; ++ ++- this.maxPhysicOffset = phyOffset; +++ this.setMaxPhysicOffset(phyOffset); ++ long maxExtAddr = 1; ++ boolean shouldDeleteFile = false; ++ while (true) { ++@@ -435,7 +435,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ mappedFile.setWrotePosition(pos); ++ mappedFile.setCommittedPosition(pos); ++ mappedFile.setFlushedPosition(pos); ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ // This maybe not take effect, when not every consume queue has extend file. ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++@@ -453,7 +453,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ mappedFile.setWrotePosition(pos); ++ mappedFile.setCommittedPosition(pos); ++ mappedFile.setFlushedPosition(pos); ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++ } ++@@ -881,8 +881,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, ++ final long cqOffset) { ++ ++- if (offset + size <= this.maxPhysicOffset) { ++- log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset); +++ if (offset + size <= this.getMaxPhysicOffset()) { +++ log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset); ++ return true; ++ } ++ ++@@ -926,7 +926,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ); ++ } ++ } ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ return mappedFile.appendMessage(this.byteBufferIndex.array()); ++ } ++ return false; ++@@ -1130,7 +1130,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ @Override ++ public void destroy() { ++- this.maxPhysicOffset = -1; +++ this.setMaxPhysicOffset(-1); ++ this.minLogicOffset = 0; ++ this.mappedFileQueue.destroy(); ++ if (isExtReadEnable()) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++index 0bc70642f..32b90d14f 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++@@ -285,7 +285,7 @@ public class MappedFileQueue implements Swappable { ++ if (this.mappedFiles.isEmpty()) ++ return 0; ++ ++- long committed = this.flushedWhere; +++ long committed = this.getFlushedWhere(); ++ if (committed != 0) { ++ MappedFile mappedFile = this.getLastMappedFile(0, false); ++ if (mappedFile != null) { ++@@ -442,11 +442,11 @@ public class MappedFileQueue implements Swappable { ++ } ++ ++ public long remainHowManyDataToCommit() { ++- return getMaxWrotePosition() - committedWhere; +++ return getMaxWrotePosition() - getCommittedWhere(); ++ } ++ ++ public long remainHowManyDataToFlush() { ++- return getMaxOffset() - flushedWhere; +++ return getMaxOffset() - this.getFlushedWhere(); ++ } ++ ++ public void deleteLastMappedFile() { ++@@ -616,15 +616,15 @@ public class MappedFileQueue implements Swappable { ++ ++ public boolean flush(final int flushLeastPages) { ++ boolean result = true; ++- MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0); +++ MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0); ++ if (mappedFile != null) { ++ long tmpTimeStamp = mappedFile.getStoreTimestamp(); ++ int offset = mappedFile.flush(flushLeastPages); ++ long where = mappedFile.getFileFromOffset() + offset; ++- result = where == this.flushedWhere; ++- this.flushedWhere = where; +++ result = where == this.getFlushedWhere(); +++ this.setFlushedWhere(where); ++ if (0 == flushLeastPages) { ++- this.storeTimestamp = tmpTimeStamp; +++ this.setStoreTimestamp(tmpTimeStamp); ++ } ++ } ++ ++@@ -633,12 +633,12 @@ public class MappedFileQueue implements Swappable { ++ ++ public synchronized boolean commit(final int commitLeastPages) { ++ boolean result = true; ++- MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); +++ MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0); ++ if (mappedFile != null) { ++ int offset = mappedFile.commit(commitLeastPages); ++ long where = mappedFile.getFileFromOffset() + offset; ++- result = where == this.committedWhere; ++- this.committedWhere = where; +++ result = where == this.getCommittedWhere(); +++ this.setCommittedWhere(where); ++ } ++ ++ return result; ++@@ -763,7 +763,7 @@ public class MappedFileQueue implements Swappable { ++ mf.destroy(1000 * 3); ++ } ++ this.mappedFiles.clear(); ++- this.flushedWhere = 0; +++ this.setFlushedWhere(0); ++ ++ // delete parent directory ++ File file = new File(storePath); ++@@ -848,6 +848,10 @@ public class MappedFileQueue implements Swappable { ++ return storeTimestamp; ++ } ++ +++ public void setStoreTimestamp(long storeTimestamp) { +++ this.storeTimestamp = storeTimestamp; +++ } +++ ++ public List getMappedFiles() { ++ return mappedFiles; ++ } ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++index 8f5af9438..8ff050dfe 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.store; ++ ++- ++ import java.util.Arrays; ++ import java.util.HashSet; ++ import java.util.Set; ++@@ -113,8 +112,7 @@ public class MultiPathMappedFileQueue extends MappedFileQueue { ++ mf.destroy(1000 * 3); ++ } ++ this.mappedFiles.clear(); ++- this.flushedWhere = 0; ++- +++ this.setFlushedWhere(0); ++ ++ Set storePathSet = getPaths(); ++ storePathSet.addAll(getReadonlyPaths()); ++-- ++2.32.0.windows.2 ++ ++ ++From 3884f595949462044c5cb3c236199bc1d7ad2341 Mon Sep 17 00:00:00 2001 ++From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= ++ ++Date: Wed, 23 Aug 2023 11:10:30 +0800 ++Subject: [PATCH 5/7] [ISSUE #7149] When creating and updating Topic, there ++ will be problems with permission settings (#7151) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings ++ ++* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings ++ ++* [issue#7249] ++ ++--------- ++ ++Co-authored-by: 十真 ++--- ++ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 3 ++- ++ 1 file changed, 2 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 13f9d002b..e8f943702 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1733,7 +1733,8 @@ public class BrokerController { ++ new TopicConfig(topicConfig.getTopicName(), ++ topicConfig.getReadQueueNums(), ++ topicConfig.getWriteQueueNums(), ++- this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); +++ topicConfig.getPerm() +++ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); ++ } else { ++ registerTopicConfig = new TopicConfig(topicConfig); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 017ad110475e8024585327b44f47e5e97aabc63b Mon Sep 17 00:00:00 2001 ++From: echooymxq ++Date: Wed, 23 Aug 2023 11:11:42 +0800 ++Subject: [PATCH 6/7] [ISSUE #7219] Fix Concurrent modify syncStateSet and Mark ++ synchronizing frequently when shrink. (#7220) ++ ++--- ++ .../broker/controller/ReplicasManager.java | 29 ++++++++++--------- ++ .../ha/autoswitch/AutoSwitchHAService.java | 21 ++++++++------ ++ 2 files changed, 28 insertions(+), 22 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++index abae7cdb0..37c82e434 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++@@ -542,7 +542,7 @@ public class ReplicasManager { ++ this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId()); ++ this.tempBrokerMetadata.clear(); ++ this.brokerControllerId = this.brokerMetadata.getBrokerId(); ++- this.haService.setBrokerControllerId(this.brokerControllerId); +++ this.haService.setLocalBrokerId(this.brokerControllerId); ++ return true; ++ } catch (Exception e) { ++ LOGGER.error("fail to create metadata file", e); ++@@ -594,7 +594,7 @@ public class ReplicasManager { ++ if (this.brokerMetadata.isLoaded()) { ++ this.registerState = RegisterState.CREATE_METADATA_FILE_DONE; ++ this.brokerControllerId = brokerMetadata.getBrokerId(); ++- this.haService.setBrokerControllerId(this.brokerControllerId); +++ this.haService.setLocalBrokerId(this.brokerControllerId); ++ return; ++ } ++ // 2. check if temp metadata exist ++@@ -735,23 +735,26 @@ public class ReplicasManager { ++ if (this.checkSyncStateSetTaskFuture != null) { ++ this.checkSyncStateSetTaskFuture.cancel(false); ++ } ++- this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> { ++- checkSyncStateSetAndDoReport(); ++- }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); +++ this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000, +++ this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); ++ } ++ ++ private void checkSyncStateSetAndDoReport() { ++- final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); ++- newSyncStateSet.add(this.brokerControllerId); ++- synchronized (this) { ++- if (this.syncStateSet != null) { ++- // Check if syncStateSet changed ++- if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { ++- return; +++ try { +++ final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); +++ newSyncStateSet.add(this.brokerControllerId); +++ synchronized (this) { +++ if (this.syncStateSet != null) { +++ // Check if syncStateSet changed +++ if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { +++ return; +++ } ++ } ++ } +++ doReportSyncStateSetChanged(newSyncStateSet); +++ } catch (Exception e) { +++ LOGGER.error("Check syncStateSet error", e); ++ } ++- doReportSyncStateSetChanged(newSyncStateSet); ++ } ++ ++ private void doReportSyncStateSetChanged(Set newSyncStateSet) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++index 6dc734e0c..d5393fdca 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++@@ -41,6 +41,7 @@ import java.nio.channels.SocketChannel; ++ import java.util.ArrayList; ++ import java.util.HashSet; ++ import java.util.List; +++import java.util.Iterator; ++ import java.util.Map; ++ import java.util.Objects; ++ import java.util.Set; ++@@ -73,7 +74,7 @@ public class AutoSwitchHAService extends DefaultHAService { ++ private EpochFileCache epochCache; ++ private AutoSwitchHAClient haClient; ++ ++- private Long brokerControllerId = null; +++ private Long localBrokerId = null; ++ ++ public AutoSwitchHAService() { ++ } ++@@ -287,9 +288,11 @@ public class AutoSwitchHAService extends DefaultHAService { ++ ++ // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable, ++ // it means that the broker has not connected. ++- for (Long slaveBrokerId : newSyncStateSet) { ++- if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { ++- newSyncStateSet.remove(slaveBrokerId); +++ Iterator iterator = newSyncStateSet.iterator(); +++ while (iterator.hasNext()) { +++ Long slaveBrokerId = iterator.next(); +++ if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { +++ iterator.remove(); ++ isSyncStateSetChanged = true; ++ } ++ } ++@@ -419,7 +422,7 @@ public class AutoSwitchHAService extends DefaultHAService { ++ // To avoid the syncStateSet is not consistent with connectionList. ++ // Fix issue: https://github.com/apache/rocketmq/issues/6662 ++ for (Long syncId : currentSyncStateSet) { ++- if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) { +++ if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) { ++ LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId); ++ // Without check and re-compute, return the confirmOffset's value directly. ++ return this.defaultMessageStore.getConfirmOffsetDirectly(); ++@@ -545,12 +548,12 @@ public class AutoSwitchHAService extends DefaultHAService { ++ return this.epochCache.getAllEntries(); ++ } ++ ++- public Long getBrokerControllerId() { ++- return brokerControllerId; +++ public Long getLocalBrokerId() { +++ return localBrokerId; ++ } ++ ++- public void setBrokerControllerId(Long brokerControllerId) { ++- this.brokerControllerId = brokerControllerId; +++ public void setLocalBrokerId(Long localBrokerId) { +++ this.localBrokerId = localBrokerId; ++ } ++ ++ class AutoSwitchAcceptSocketService extends AcceptSocketService { ++-- ++2.32.0.windows.2 ++ ++ ++From 77e8e54b37c3fc3ea0beffc1ace6f5bf20af10d9 Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Wed, 23 Aug 2023 15:56:39 +0800 ++Subject: [PATCH 7/7] [ISSUE #7223] Support batch ack for grpc client in proxy ++ (#7225) ++ ++--- ++ .../client/impl/mqclient/MQClientAPIExt.java | 26 +++ ++ .../rocketmq/proxy/config/ProxyConfig.java | 10 + ++ .../grpc/v2/consumer/AckMessageActivity.java | 136 ++++++++--- ++ .../proxy/processor/AbstractProcessor.java | 4 +- ++ .../proxy/processor/BatchAckResult.java | 53 +++++ ++ .../proxy/processor/ConsumerProcessor.java | 64 +++++ ++ .../processor/DefaultMessagingProcessor.java | 7 + ++ .../proxy/processor/MessagingProcessor.java | 18 ++ ++ .../message/ClusterMessageService.java | 16 +- ++ .../service/message/LocalMessageService.java | 58 +++++ ++ .../proxy/service/message/MessageService.java | 8 + ++ .../service/message/ReceiptHandleMessage.java | 39 ++++ ++ .../v2/consumer/AckMessageActivityTest.java | 221 +++++++++++++++--- ++ .../proxy/processor/BaseProcessorTest.java | 18 +- ++ .../processor/ConsumerProcessorTest.java | 115 +++++++++ ++ .../service/mqclient/MQClientAPIExtTest.java | 12 + ++ 16 files changed, 728 insertions(+), 77 deletions(-) ++ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++index fb8f8d11f..d7c8ef8d9 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++@@ -306,6 +306,32 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ return future; ++ } ++ +++ public CompletableFuture batchAckMessageAsync( +++ String brokerAddr, +++ String topic, +++ String consumerGroup, +++ List extraInfoList, +++ long timeoutMillis +++ ) { +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() { +++ @Override +++ public void onSuccess(AckResult ackResult) { +++ future.complete(ackResult); +++ } +++ +++ @Override +++ public void onException(Throwable t) { +++ future.completeExceptionally(t); +++ } +++ }, topic, consumerGroup, extraInfoList); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ ++ public CompletableFuture changeInvisibleTimeAsync( ++ String brokerAddr, ++ String brokerName, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 39caaa0d9..76a243919 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -250,6 +250,8 @@ public class ProxyConfig implements ConfigFile { ++ private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000; ++ private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000; ++ +++ private boolean enableBatchAck = false; +++ ++ @Override ++ public void initData() { ++ parseDelayLevel(); ++@@ -1379,4 +1381,12 @@ public class ProxyConfig implements ConfigFile { ++ public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) { ++ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; ++ } +++ +++ public boolean isEnableBatchAck() { +++ return enableBatchAck; +++ } +++ +++ public void setEnableBatchAck(boolean enableBatchAck) { +++ this.enableBatchAck = enableBatchAck; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++index 9a3a77201..97c716c8f 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++@@ -31,12 +31,15 @@ import org.apache.rocketmq.client.consumer.AckStatus; ++ import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; +++import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; ++ import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; ++ import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +++import org.apache.rocketmq.proxy.processor.BatchAckResult; ++ import org.apache.rocketmq.proxy.processor.MessagingProcessor; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ ++ public class AckMessageActivity extends AbstractMessingActivity { ++ ++@@ -50,60 +53,98 @@ public class AckMessageActivity extends AbstractMessingActivity { ++ ++ try { ++ validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); ++- ++- CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; ++- for (int i = 0; i < request.getEntriesCount(); i++) { ++- futures[i] = processAckMessage(ctx, request, request.getEntries(i)); +++ String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); +++ String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()); +++ if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) { +++ future = ackMessageInBatch(ctx, group, topic, request); +++ } else { +++ future = ackMessageOneByOne(ctx, group, topic, request); ++ } ++- CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { ++- if (throwable != null) { ++- future.completeExceptionally(throwable); ++- return; ++- } +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ +++ protected CompletableFuture ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) { +++ List handleMessageList = new ArrayList<>(request.getEntriesCount()); ++ +++ for (AckMessageEntry ackMessageEntry : request.getEntriesList()) { +++ String handleString = getHandleString(ctx, group, request, ackMessageEntry); +++ handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId())); +++ } +++ return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic) +++ .thenApply(batchAckResultList -> { +++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder(); ++ Set responseCodes = new HashSet<>(); ++- List entryList = new ArrayList<>(); ++- for (CompletableFuture entryFuture : futures) { ++- AckMessageResultEntry entryResult = entryFuture.join(); ++- responseCodes.add(entryResult.getStatus().getCode()); ++- entryList.add(entryResult); ++- } ++- AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() ++- .addAllEntries(entryList); ++- if (responseCodes.size() > 1) { ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); ++- } else if (responseCodes.size() == 1) { ++- Code code = responseCodes.stream().findAny().get(); ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); ++- } else { ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult); +++ responseBuilder.addEntries(entry); +++ responseCodes.add(entry.getStatus().getCode()); ++ } ++- future.complete(responseBuilder.build()); +++ setAckResponseStatus(responseBuilder, responseCodes); +++ return responseBuilder.build(); ++ }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); +++ } +++ +++ protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) { +++ ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage(); +++ AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder() +++ .setMessageId(handleMessage.getMessageId()) +++ .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle()); +++ if (batchAckResult.getProxyException() != null) { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException())); +++ } else { +++ AckResult ackResult = batchAckResult.getAckResult(); +++ if (AckStatus.OK.equals(ackResult.getStatus())) { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); +++ } else { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")); +++ } ++ } ++- return future; +++ return resultBuilder.build(); ++ } ++ ++- protected CompletableFuture processAckMessage(ProxyContext ctx, AckMessageRequest request, +++ protected CompletableFuture ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) { +++ CompletableFuture resultFuture = new CompletableFuture<>(); +++ CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; +++ for (int i = 0; i < request.getEntriesCount(); i++) { +++ futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i)); +++ } +++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +++ if (throwable != null) { +++ resultFuture.completeExceptionally(throwable); +++ return; +++ } +++ +++ Set responseCodes = new HashSet<>(); +++ List entryList = new ArrayList<>(); +++ for (CompletableFuture entryFuture : futures) { +++ AckMessageResultEntry entryResult = entryFuture.join(); +++ responseCodes.add(entryResult.getStatus().getCode()); +++ entryList.add(entryResult); +++ } +++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() +++ .addAllEntries(entryList); +++ setAckResponseStatus(responseBuilder, responseCodes); +++ resultFuture.complete(responseBuilder.build()); +++ }); +++ return resultFuture; +++ } +++ +++ protected CompletableFuture processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request, ++ AckMessageEntry ackMessageEntry) { ++ CompletableFuture future = new CompletableFuture<>(); ++ ++ try { ++- String handleString = ackMessageEntry.getReceiptHandle(); ++- ++- String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); ++- MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); ++- if (messageReceiptHandle != null) { ++- handleString = messageReceiptHandle.getReceiptHandleStr(); ++- } +++ String handleString = this.getHandleString(ctx, group, request, ackMessageEntry); ++ CompletableFuture ackResultFuture = this.messagingProcessor.ackMessage( ++ ctx, ++ ReceiptHandle.decode(handleString), ++ ackMessageEntry.getMessageId(), ++ group, ++- GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); +++ topic +++ ); ++ ackResultFuture.thenAccept(result -> { ++ future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result)); ++ }).exceptionally(t -> { ++@@ -139,4 +180,25 @@ public class AckMessageActivity extends AbstractMessingActivity { ++ .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")) ++ .build(); ++ } +++ +++ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set responseCodes) { +++ if (responseCodes.size() > 1) { +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); +++ } else if (responseCodes.size() == 1) { +++ Code code = responseCodes.stream().findAny().get(); +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); +++ } else { +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); +++ } +++ } +++ +++ protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) { +++ String handleString = ackMessageEntry.getReceiptHandle(); +++ +++ MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); +++ if (messageReceiptHandle != null) { +++ handleString = messageReceiptHandle.getReceiptHandleStr(); +++ } +++ return handleString; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++index b61c3df9e..c63212c23 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++@@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { ++ protected MessagingProcessor messagingProcessor; ++ protected ServiceManager serviceManager; ++ +++ protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); +++ ++ public AbstractProcessor(MessagingProcessor messagingProcessor, ++ ServiceManager serviceManager) { ++ this.messagingProcessor = messagingProcessor; ++@@ -35,7 +37,7 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { ++ ++ protected void validateReceiptHandle(ReceiptHandle handle) { ++ if (handle.isExpired()) { ++- throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); +++ throw EXPIRED_HANDLE_PROXY_EXCEPTION; ++ } ++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++new file mode 100644 ++index 000000000..dfb9c9b9e ++--- /dev/null +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++@@ -0,0 +1,53 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.proxy.processor; +++ +++import org.apache.rocketmq.client.consumer.AckResult; +++import org.apache.rocketmq.proxy.common.ProxyException; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; +++ +++public class BatchAckResult { +++ +++ private final ReceiptHandleMessage receiptHandleMessage; +++ private AckResult ackResult; +++ private ProxyException proxyException; +++ +++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, +++ AckResult ackResult) { +++ this.receiptHandleMessage = receiptHandleMessage; +++ this.ackResult = ackResult; +++ } +++ +++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, +++ ProxyException proxyException) { +++ this.receiptHandleMessage = receiptHandleMessage; +++ this.proxyException = proxyException; +++ } +++ +++ public ReceiptHandleMessage getReceiptHandleMessage() { +++ return receiptHandleMessage; +++ } +++ +++ public AckResult getAckResult() { +++ return ackResult; +++ } +++ +++ public ProxyException getProxyException() { +++ return proxyException; +++ } +++} ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++index 656a6339d..f3522b374 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++@@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.common.utils.ProxyUtils; ++ import org.apache.rocketmq.proxy.service.ServiceManager; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; ++@@ -241,6 +242,69 @@ public class ConsumerProcessor extends AbstractProcessor { ++ return FutureUtils.addExecutor(future, this.executor); ++ } ++ +++ public CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ) { +++ CompletableFuture> future = new CompletableFuture<>(); +++ try { +++ List batchAckResultList = new ArrayList<>(handleMessageList.size()); +++ Map> brokerHandleListMap = new HashMap<>(); +++ +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ if (handleMessage.getReceiptHandle().isExpired()) { +++ batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION)); +++ continue; +++ } +++ List brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>()); +++ brokerHandleList.add(handleMessage); +++ } +++ +++ if (brokerHandleListMap.isEmpty()) { +++ return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor); +++ } +++ Set>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet(); +++ CompletableFuture>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()]; +++ int futureIndex = 0; +++ for (Map.Entry> entry : brokerHandleListMapEntrySet) { +++ futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis); +++ } +++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +++ if (throwable != null) { +++ future.completeExceptionally(throwable); +++ } +++ for (CompletableFuture> resultFuture : futures) { +++ batchAckResultList.addAll(resultFuture.join()); +++ } +++ future.complete(batchAckResultList); +++ }); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return FutureUtils.addExecutor(future, this.executor); +++ } +++ +++ protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List handleMessageList, long timeoutMillis) { +++ return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis) +++ .thenApply(result -> { +++ List results = new ArrayList<>(); +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ results.add(new BatchAckResult(handleMessage, result)); +++ } +++ return results; +++ }) +++ .exceptionally(throwable -> { +++ List results = new ArrayList<>(); +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable))); +++ } +++ return results; +++ }); +++ } +++ ++ public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, ++ String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++index 188cb7b9b..ba150051b 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++@@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.service.ServiceManager; ++ import org.apache.rocketmq.proxy.service.ServiceManagerFactory; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; ++@@ -183,6 +184,12 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen ++ return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis); ++ } ++ +++ @Override +++ public CompletableFuture> batchAckMessage(ProxyContext ctx, +++ List handleMessageList, String consumerGroup, String topic, long timeoutMillis) { +++ return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis); +++ } +++ ++ @Override ++ public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, ++ String groupName, String topicName, long invisibleTime, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++index d86be0bd8..2ae7418ba 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++@@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.Address; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; ++@@ -155,6 +156,23 @@ public interface MessagingProcessor extends StartAndShutdown { ++ long timeoutMillis ++ ); ++ +++ default CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic +++ ) { +++ return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS); +++ } +++ +++ CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ); +++ ++ default CompletableFuture changeInvisibleTime( ++ ProxyContext ctx, ++ ReceiptHandle handle, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++index 9f163f1b9..70b72deae 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++@@ -20,9 +20,11 @@ import com.google.common.collect.Lists; ++ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; +++import java.util.stream.Collectors; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.PopResult; ++ import org.apache.rocketmq.client.consumer.PullResult; +++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ import org.apache.rocketmq.common.message.Message; ++@@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.TopicRouteService; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -137,6 +138,19 @@ public class ClusterMessageService implements MessageService { ++ ); ++ } ++ +++ @Override +++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, String consumerGroup, +++ String topic, long timeoutMillis) { +++ List extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList()); +++ return this.mqClientAPIFactory.getClient().batchAckMessageAsync( +++ this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()), +++ topic, +++ consumerGroup, +++ extraInfoList, +++ timeoutMillis +++ ); +++ } +++ ++ @Override ++ public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, ++ PullMessageRequestHeader requestHeader, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++index eb2c4d9ee..ca7dcc9eb 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.service.message; ++ import io.netty.channel.ChannelHandlerContext; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; +++import java.util.BitSet; ++ import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.List; ++@@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; +++import org.apache.rocketmq.remoting.protocol.body.BatchAck; +++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; ++@@ -364,6 +367,61 @@ public class LocalMessageService implements MessageService { ++ }); ++ } ++ +++ @Override +++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, +++ String consumerGroup, String topic, long timeoutMillis) { +++ SimpleChannel channel = channelManager.createChannel(ctx); +++ ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); +++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); +++ +++ Map batchAckMap = new HashMap<>(); +++ for (ReceiptHandleMessage receiptHandleMessage : handleList) { +++ String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle(); +++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); +++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + +++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + +++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + +++ ExtraInfoUtil.getPopTime(extraInfoData); +++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { +++ BatchAck newBatchAck = new BatchAck(); +++ newBatchAck.setConsumerGroup(consumerGroup); +++ newBatchAck.setTopic(topic); +++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); +++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); +++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); +++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); +++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); +++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); +++ newBatchAck.setBitSet(new BitSet()); +++ return newBatchAck; +++ }); +++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); +++ } +++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); +++ requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); +++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); +++ +++ command.setBody(requestBody.encode()); +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ RemotingCommand response = brokerController.getAckMessageProcessor() +++ .processRequest(channelHandlerContext, command); +++ future.complete(response); +++ } catch (Exception e) { +++ log.error("Fail to process batchAckMessage command", e); +++ future.completeExceptionally(e); +++ } +++ return future.thenApply(r -> { +++ AckResult ackResult = new AckResult(); +++ if (ResponseCode.SUCCESS == r.getCode()) { +++ ackResult.setStatus(AckStatus.OK); +++ } else { +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ } +++ return ackResult; +++ }); +++ } +++ ++ @Override ++ public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, ++ PullMessageRequestHeader requestHeader, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++index 15da17154..58a835adb 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++@@ -91,6 +91,14 @@ public interface MessageService { ++ long timeoutMillis ++ ); ++ +++ CompletableFuture batchAckMessage( +++ ProxyContext ctx, +++ List handleList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ); +++ ++ CompletableFuture pullMessage( ++ ProxyContext ctx, ++ AddressableMessageQueue messageQueue, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++new file mode 100644 ++index 000000000..ae63fed49 ++--- /dev/null +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++@@ -0,0 +1,39 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.proxy.service.message; +++ +++import org.apache.rocketmq.common.consumer.ReceiptHandle; +++ +++public class ReceiptHandleMessage { +++ +++ private final ReceiptHandle receiptHandle; +++ private final String messageId; +++ +++ public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) { +++ this.receiptHandle = receiptHandle; +++ this.messageId = messageId; +++ } +++ +++ public ReceiptHandle getReceiptHandle() { +++ return receiptHandle; +++ } +++ +++ public String getMessageId() { +++ return messageId; +++ } +++} ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++index 49fdfc6a8..3c4746105 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++@@ -20,21 +20,32 @@ package org.apache.rocketmq.proxy.grpc.v2.consumer; ++ import apache.rocketmq.v2.AckMessageEntry; ++ import apache.rocketmq.v2.AckMessageRequest; ++ import apache.rocketmq.v2.AckMessageResponse; +++import apache.rocketmq.v2.AckMessageResultEntry; ++ import apache.rocketmq.v2.Code; ++ import apache.rocketmq.v2.Resource; +++import java.util.ArrayList; +++import java.util.HashMap; +++import java.util.List; +++import java.util.Map; ++ import java.util.concurrent.CompletableFuture; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.AckStatus; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +++import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +++import org.apache.rocketmq.proxy.processor.BatchAckResult; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.junit.Before; ++ import org.junit.Test; +++import org.mockito.stubbing.Answer; ++ ++ import static org.junit.Assert.assertEquals; ++ import static org.mockito.ArgumentMatchers.any; +++import static org.mockito.ArgumentMatchers.anyList; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; +++import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.when; ++ ++ public class AckMessageActivityTest extends BaseActivityTest { ++@@ -52,43 +63,197 @@ public class AckMessageActivityTest extends BaseActivityTest { ++ ++ @Test ++ public void testAckMessage() throws Throwable { ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString())) +++ ConfigurationManager.getProxyConfig().setEnableBatchAck(false); +++ +++ String msg1 = "msg1"; +++ String msg2 = "msg2"; +++ String msg3 = "msg3"; +++ +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString())) ++ .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired")); ++ ++ AckResult msg2AckResult = new AckResult(); ++ msg2AckResult.setStatus(AckStatus.OK); ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString())) +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString())) ++ .thenReturn(CompletableFuture.completedFuture(msg2AckResult)); ++ ++ AckResult msg3AckResult = new AckResult(); ++ msg3AckResult.setStatus(AckStatus.NO_EXIST); ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString())) +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString())) ++ .thenReturn(CompletableFuture.completedFuture(msg3AckResult)); ++ ++- AckMessageResponse response = this.ackMessageActivity.ackMessage( ++- createContext(), ++- AckMessageRequest.newBuilder() ++- .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++- .setGroup(Resource.newBuilder().setName(GROUP).build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg1") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++- .build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg2") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++- .build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg3") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++- .build()) ++- .build() ++- ).get(); ++- ++- assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); ++- assertEquals(3, response.getEntriesCount()); ++- assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); ++- assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); ++- assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg1) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg2) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.OK, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg3) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg1) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg2) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg3) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ +++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +++ assertEquals(3, response.getEntriesCount()); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); +++ assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); +++ } +++ } +++ +++ @Test +++ public void testAckMessageInBatch() throws Throwable { +++ ConfigurationManager.getProxyConfig().setEnableBatchAck(true); +++ +++ String successMessageId = "msg1"; +++ String notOkMessageId = "msg2"; +++ String exceptionMessageId = "msg3"; +++ +++ doAnswer((Answer>>) invocation -> { +++ List receiptHandleMessageList = invocation.getArgument(1, List.class); +++ List batchAckResultList = new ArrayList<>(); +++ for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) { +++ BatchAckResult batchAckResult; +++ if (receiptHandleMessage.getMessageId().equals(successMessageId)) { +++ AckResult ackResult = new AckResult(); +++ ackResult.setStatus(AckStatus.OK); +++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); +++ } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) { +++ AckResult ackResult = new AckResult(); +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); +++ } else { +++ batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "")); +++ } +++ batchAckResultList.add(batchAckResult); +++ } +++ return CompletableFuture.completedFuture(batchAckResultList); +++ }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString()); +++ +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(successMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.OK, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(notOkMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(exceptionMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(successMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(notOkMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(exceptionMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ +++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +++ assertEquals(3, response.getEntriesCount()); +++ Map msgCode = new HashMap<>(); +++ for (AckMessageResultEntry entry : response.getEntriesList()) { +++ msgCode.put(entry.getMessageId(), entry.getStatus().getCode()); +++ } +++ assertEquals(Code.OK, msgCode.get(successMessageId)); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId)); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId)); +++ } ++ } ++ } ++\ No newline at end of file ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++index 5c1ea9627..072630e39 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++@@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest { ++ protected ProxyRelayService proxyRelayService; ++ @Mock ++ protected MetadataService metadataService; ++- @Mock ++- protected ProducerProcessor producerProcessor; ++- @Mock ++- protected ConsumerProcessor consumerProcessor; ++- @Mock ++- protected TransactionProcessor transactionProcessor; ++- @Mock ++- protected ClientProcessor clientProcessor; ++ ++ public void before() throws Throwable { ++ super.before(); ++@@ -92,6 +84,13 @@ public class BaseProcessorTest extends InitConfigTest { ++ } ++ ++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) { +++ return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(), +++ RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), +++ RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker"); +++ } +++ +++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime, +++ long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) { ++ MessageExt messageExt = new MessageExt(); ++ messageExt.setTopic(topic); ++ messageExt.setTags(tags); ++@@ -100,8 +99,7 @@ public class BaseProcessorTest extends InitConfigTest { ++ messageExt.setMsgId(MessageClientIDSetter.createUniqID()); ++ messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); ++ MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, ++- ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime, ++- RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); +++ ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset)); ++ return messageExt; ++ } ++ ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++index 717e86fc0..db268a06e 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++@@ -20,8 +20,11 @@ package org.apache.rocketmq.proxy.processor; ++ import com.google.common.collect.Sets; ++ import java.time.Duration; ++ import java.util.ArrayList; +++import java.util.Collections; +++import java.util.HashMap; ++ import java.util.HashSet; ++ import java.util.List; +++import java.util.Map; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.Executors; ++@@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageClientIDSetter; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.proxy.common.ProxyContext; +++import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +++import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.MessageQueueView; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -50,16 +56,22 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; ++ import org.junit.Before; ++ import org.junit.Test; ++ import org.mockito.ArgumentCaptor; +++import org.mockito.stubbing.Answer; ++ ++ import static org.assertj.core.api.Assertions.assertThat; ++ import static org.junit.Assert.assertEquals; ++ import static org.junit.Assert.assertNotNull; +++import static org.junit.Assert.assertNull; ++ import static org.junit.Assert.assertSame; ++ import static org.mockito.ArgumentMatchers.any; +++import static org.mockito.ArgumentMatchers.anyList; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; +++import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.mock; +++import static org.mockito.Mockito.never; +++import static org.mockito.Mockito.verify; ++ import static org.mockito.Mockito.when; ++ ++ public class ConsumerProcessorTest extends BaseProcessorTest { ++@@ -162,6 +174,109 @@ public class ConsumerProcessorTest extends BaseProcessorTest { ++ assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); ++ } ++ +++ @Test +++ public void testBatchAckExpireMessage() throws Throwable { +++ String brokerName1 = "brokerName1"; +++ +++ List receiptHandleMessageList = new ArrayList<>(); +++ for (int i = 0; i < 3; i++) { +++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, +++ 0, 0, 0, i, brokerName1); +++ ReceiptHandle expireHandle = create(expireMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); +++ } +++ +++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); +++ +++ verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); +++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ assertNull(batchAckResult.getAckResult()); +++ assertNotNull(batchAckResult.getProxyException()); +++ assertNotNull(batchAckResult.getReceiptHandleMessage()); +++ } +++ +++ } +++ +++ @Test +++ public void testBatchAckMessage() throws Throwable { +++ String brokerName1 = "brokerName1"; +++ String brokerName2 = "brokerName2"; +++ String errThrowBrokerName = "errThrowBrokerName"; +++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, +++ 0, 0, 0, 0, brokerName1); +++ ReceiptHandle expireHandle = create(expireMessage); +++ +++ List receiptHandleMessageList = new ArrayList<>(); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); +++ List broker1Msg = new ArrayList<>(); +++ List broker2Msg = new ArrayList<>(); +++ +++ long now = System.currentTimeMillis(); +++ int msgNum = 3; +++ for (int i = 0; i < msgNum; i++) { +++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, i + 1, brokerName1); +++ ReceiptHandle brokerHandle = create(brokerMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); +++ broker1Msg.add(brokerMessage.getMsgId()); +++ } +++ for (int i = 0; i < msgNum; i++) { +++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, i + 1, brokerName2); +++ ReceiptHandle brokerHandle = create(brokerMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); +++ broker2Msg.add(brokerMessage.getMsgId()); +++ } +++ +++ // for this message, will throw exception in batchAckMessage +++ MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, 0, errThrowBrokerName); +++ ReceiptHandle errThrowHandle = create(errThrowMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId())); +++ +++ Collections.shuffle(receiptHandleMessageList); +++ +++ doAnswer((Answer>) invocation -> { +++ List handleMessageList = invocation.getArgument(1, List.class); +++ AckResult ackResult = new AckResult(); +++ String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName(); +++ if (brokerName.equals(brokerName1)) { +++ ackResult.setStatus(AckStatus.OK); +++ } else if (brokerName.equals(brokerName2)) { +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ } else { +++ return FutureUtils.completeExceptionally(new RuntimeException()); +++ } +++ +++ return CompletableFuture.completedFuture(ackResult); +++ }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); +++ +++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); +++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); +++ +++ // check ackResult for each msg +++ Map msgBatchAckResult = new HashMap<>(); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult); +++ } +++ for (String msgId : broker1Msg) { +++ assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus()); +++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); +++ } +++ for (String msgId : broker2Msg) { +++ assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus()); +++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); +++ } +++ assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException()); +++ assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode()); +++ assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult()); +++ +++ assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException()); +++ assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode()); +++ assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult()); +++ } +++ ++ @Test ++ public void testChangeInvisibleTime() throws Throwable { ++ ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++index 77a119a29..3f3a4ae40 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++@@ -220,6 +220,18 @@ public class MQClientAPIExtTest { ++ assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get()); ++ } ++ +++ @Test +++ public void testBatchAckMessageAsync() throws Exception { +++ AckResult ackResult = new AckResult(); +++ doAnswer((Answer) mock -> { +++ AckCallback ackCallback = mock.getArgument(2); +++ ackCallback.onSuccess(ackResult); +++ return null; +++ }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any()); +++ +++ assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get()); +++ } +++ ++ @Test ++ public void testChangeInvisibleTimeAsync() throws Exception { ++ AckResult ackResult = new AckResult(); ++-- ++2.32.0.windows.2 ++ +diff --git a/patch012-backport-enhance-rockdbconfigtojson.patch b/patch012-backport-enhance-rockdbconfigtojson.patch +new file mode 100644 +index 000000000..cf6128b24 +--- /dev/null ++++ b/patch012-backport-enhance-rockdbconfigtojson.patch +@@ -0,0 +1,2920 @@ ++From fec141481496c53a0db398367006c34264662d18 Mon Sep 17 00:00:00 2001 ++From: yx9o ++Date: Wed, 23 Aug 2023 08:22:34 +0800 ++Subject: [PATCH 1/8] [ISSUE #7166] Optimize the display format of admin ++ (#7210) ++ ++--- ++ .../java/org/apache/rocketmq/tools/command/MQAdminStartup.java | 2 +- ++ .../command/acl/ClusterAclConfigVersionListSubCommand.java | 2 +- ++ .../tools/command/acl/DeleteAccessConfigSubCommand.java | 2 +- ++ .../rocketmq/tools/command/acl/GetAccessConfigSubCommand.java | 2 +- ++ .../tools/command/acl/UpdateAccessConfigSubCommand.java | 2 +- ++ .../tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java | 2 +- ++ .../tools/command/broker/BrokerConsumeStatsSubCommad.java | 2 +- ++ .../rocketmq/tools/command/broker/BrokerStatusSubCommand.java | 2 +- ++ .../tools/command/broker/CommitLogSetReadAheadSubCommand.java | 2 +- ++ .../tools/command/broker/DeleteExpiredCommitLogSubCommand.java | 2 +- ++ .../rocketmq/tools/command/broker/GetBrokerConfigCommand.java | 2 +- ++ .../rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java | 2 +- ++ .../tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java | 2 +- ++ .../broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java | 2 +- ++ .../tools/command/broker/ResetMasterFlushOffsetSubCommand.java | 2 +- ++ .../tools/command/broker/UpdateBrokerConfigSubCommand.java | 2 +- ++ .../broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java | 2 +- ++ .../rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java | 2 +- ++ .../rocketmq/tools/command/cluster/ClusterListSubCommand.java | 2 +- ++ .../tools/command/connection/ConsumerConnectionSubCommand.java | 2 +- ++ .../tools/command/connection/ProducerConnectionSubCommand.java | 2 +- ++ .../tools/command/consumer/ConsumerStatusSubCommand.java | 2 +- ++ .../tools/command/consumer/GetConsumerConfigSubCommand.java | 2 +- ++ .../tools/command/consumer/StartMonitoringSubCommand.java | 2 +- ++ .../tools/command/consumer/UpdateSubGroupSubCommand.java | 2 +- ++ .../rocketmq/tools/command/container/AddBrokerSubCommand.java | 2 +- ++ .../tools/command/container/RemoveBrokerSubCommand.java | 2 +- ++ .../command/controller/CleanControllerBrokerMetaSubCommand.java | 2 +- ++ .../command/controller/GetControllerMetaDataSubCommand.java | 2 +- ++ .../tools/command/controller/ReElectMasterSubCommand.java | 2 +- ++ .../rocketmq/tools/command/export/ExportConfigsCommand.java | 2 +- ++ .../rocketmq/tools/command/export/ExportMetadataCommand.java | 2 +- ++ .../rocketmq/tools/command/export/ExportMetricsCommand.java | 2 +- ++ .../rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java | 2 +- ++ .../apache/rocketmq/tools/command/ha/HAStatusSubCommand.java | 2 +- ++ .../rocketmq/tools/command/message/CheckMsgSendRTCommand.java | 2 +- ++ .../rocketmq/tools/command/message/ConsumeMessageCommand.java | 2 +- ++ .../tools/command/message/DumpCompactionLogCommand.java | 2 +- ++ .../tools/command/message/PrintMessageByQueueCommand.java | 2 +- ++ .../rocketmq/tools/command/message/PrintMessageSubCommand.java | 2 +- ++ .../rocketmq/tools/command/message/QueryMsgByIdSubCommand.java | 2 +- ++ .../rocketmq/tools/command/message/QueryMsgByKeySubCommand.java | 2 +- ++ .../tools/command/message/QueryMsgByOffsetSubCommand.java | 2 +- ++ .../tools/command/message/QueryMsgByUniqueKeySubCommand.java | 2 +- ++ .../tools/command/message/QueryMsgTraceByIdSubCommand.java | 2 +- ++ .../rocketmq/tools/command/message/SendMessageCommand.java | 2 +- ++ .../rocketmq/tools/command/namesrv/AddWritePermSubCommand.java | 2 +- ++ .../rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java | 2 +- ++ .../tools/command/offset/SkipAccumulationSubCommand.java | 2 +- ++ .../apache/rocketmq/tools/command/stats/StatsAllSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/AllocateMQSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/TopicClusterSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/TopicListSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/TopicRouteSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/TopicStatusSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/UpdateOrderConfCommand.java | 2 +- ++ .../tools/command/topic/UpdateStaticTopicSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java | 2 +- ++ .../rocketmq/tools/command/topic/UpdateTopicSubCommand.java | 2 +- ++ 59 files changed, 59 insertions(+), 59 deletions(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++index 0c2618e91..890125ca0 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++@@ -278,7 +278,7 @@ public class MQAdminStartup { ++ System.out.printf("The most commonly used mqadmin commands are:%n"); ++ ++ for (SubCommand cmd : SUB_COMMANDS) { ++- System.out.printf(" %-25s %s%n", cmd.commandName(), cmd.commandDesc()); +++ System.out.printf(" %-35s %s%n", cmd.commandName(), cmd.commandDesc()); ++ } ++ ++ System.out.printf("%nSee 'mqadmin help ' for more information on a specific command.%n"); ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java ++index f8a00b1e0..26ed028fb 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/ClusterAclConfigVersionListSubCommand.java ++@@ -47,7 +47,7 @@ public class ClusterAclConfigVersionListSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "List all of acl config version information in cluster"; +++ return "List all of acl config version information in cluster."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java ++index fd3a92fff..a7f3d295a 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/DeleteAccessConfigSubCommand.java ++@@ -42,7 +42,7 @@ public class DeleteAccessConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Delete Acl Config Account in broker"; +++ return "Delete Acl Config Account in broker."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java ++index 25844d6a1..f1c9a1496 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/GetAccessConfigSubCommand.java ++@@ -49,7 +49,7 @@ public class GetAccessConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "List all of acl config information in cluster"; +++ return "List all of acl config information in cluster."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java ++index 3be40daa1..d8a06f92d 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateAccessConfigSubCommand.java ++@@ -40,7 +40,7 @@ public class UpdateAccessConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update acl config yaml file in broker"; +++ return "Update acl config yaml file in broker."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java ++index ff662b506..9dacf1fae 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/acl/UpdateGlobalWhiteAddrSubCommand.java ++@@ -37,7 +37,7 @@ public class UpdateGlobalWhiteAddrSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update global white address for acl Config File in broker"; +++ return "Update global white address for acl Config File in broker."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java ++index 3f2f90673..7658a2139 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerConsumeStatsSubCommad.java ++@@ -61,7 +61,7 @@ public class BrokerConsumeStatsSubCommad implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch broker consume stats data"; +++ return "Fetch broker consume stats data."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java ++index 830ff3425..ce934f547 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/BrokerStatusSubCommand.java ++@@ -44,7 +44,7 @@ public class BrokerStatusSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch broker runtime status data"; +++ return "Fetch broker runtime status data."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java ++index b00c7f5f5..4fdabfdf8 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/CommitLogSetReadAheadSubCommand.java ++@@ -44,7 +44,7 @@ public class CommitLogSetReadAheadSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "set read ahead mode for all commitlog files"; +++ return "Set read ahead mode for all commitlog files."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java ++index a4b2a51ad..142bb7b3c 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/DeleteExpiredCommitLogSubCommand.java ++@@ -37,7 +37,7 @@ public class DeleteExpiredCommitLogSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Delete expired CommitLog files"; +++ return "Delete expired CommitLog files."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java ++index 5d86c10e4..c4762a296 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerConfigCommand.java ++@@ -45,7 +45,7 @@ public class GetBrokerConfigCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Get broker config by cluster or special broker"; +++ return "Get broker config by cluster or special broker."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java ++index abe8fc622..1a8961e04 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetBrokerEpochSubCommand.java ++@@ -38,7 +38,7 @@ public class GetBrokerEpochSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch broker epoch entries"; +++ return "Fetch broker epoch entries."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java ++index 7c54e650c..34b3ba7d3 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/GetColdDataFlowCtrInfoSubCommand.java ++@@ -47,7 +47,7 @@ public class GetColdDataFlowCtrInfoSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "get cold data flow ctr info"; +++ return "Get cold data flow ctr info."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java ++index b0477924f..f20407480 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/RemoveColdDataFlowCtrGroupConfigSubCommand.java ++@@ -36,7 +36,7 @@ public class RemoveColdDataFlowCtrGroupConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "remove consumer from cold ctr config"; +++ return "Remove consumer from cold ctr config."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java ++index b2ac48c84..90451b51f 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/ResetMasterFlushOffsetSubCommand.java ++@@ -33,7 +33,7 @@ public class ResetMasterFlushOffsetSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Reset master flush offset in slave"; +++ return "Reset master flush offset in slave."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java ++index 98abeb6ae..62816ef03 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateBrokerConfigSubCommand.java ++@@ -37,7 +37,7 @@ public class UpdateBrokerConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update broker's config"; +++ return "Update broker's config."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java ++index d06a24b57..8d1a00077 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/broker/UpdateColdDataFlowCtrGroupConfigSubCommand.java ++@@ -39,7 +39,7 @@ public class UpdateColdDataFlowCtrGroupConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "addOrUpdate cold data flow ctr group config"; +++ return "Add or update cold data flow ctr group config."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java ++index 7253970bd..d755e9e5d 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/CLusterSendMsgRTCommand.java ++@@ -48,7 +48,7 @@ public class CLusterSendMsgRTCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "List All clusters Message Send RT"; +++ return "List All clusters Message Send RT."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java ++index a7a840a44..ede0fa5cf 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/cluster/ClusterListSubCommand.java ++@@ -41,7 +41,7 @@ public class ClusterListSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "List cluster infos"; +++ return "List cluster infos."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java ++index 630961e31..35f73d8a0 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ConsumerConnectionSubCommand.java ++@@ -39,7 +39,7 @@ public class ConsumerConnectionSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query consumer's socket connection, client version and subscription"; +++ return "Query consumer's socket connection, client version and subscription."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java ++index 2533982c8..bde674ab2 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/connection/ProducerConnectionSubCommand.java ++@@ -36,7 +36,7 @@ public class ProducerConnectionSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query producer's socket connection and client version"; +++ return "Query producer's socket connection and client version."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java ++index 72b9c975e..d8f6f9aa9 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/ConsumerStatusSubCommand.java ++@@ -47,7 +47,7 @@ public class ConsumerStatusSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query consumer's internal data structure"; +++ return "Query consumer's internal data structure."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java ++index 6095e7668..4a8253a02 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/GetConsumerConfigSubCommand.java ++@@ -43,7 +43,7 @@ public class GetConsumerConfigSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Get consumer config by subscription group name"; +++ return "Get consumer config by subscription group name."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java ++index 2d08d0bd0..f5e140433 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/StartMonitoringSubCommand.java ++@@ -34,7 +34,7 @@ public class StartMonitoringSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Start Monitoring"; +++ return "Start Monitoring."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java ++index f87bafc93..b17da4de4 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/consumer/UpdateSubGroupSubCommand.java ++@@ -41,7 +41,7 @@ public class UpdateSubGroupSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update or create subscription group"; +++ return "Update or create subscription group."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java ++index e9e5be4a5..007d42ae6 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/AddBrokerSubCommand.java ++@@ -33,7 +33,7 @@ public class AddBrokerSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Add a broker to specified container"; +++ return "Add a broker to specified container."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java ++index 7c455f858..ab25d8ebe 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/container/RemoveBrokerSubCommand.java ++@@ -33,7 +33,7 @@ public class RemoveBrokerSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Remove a broker from specified container"; +++ return "Remove a broker from specified container."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java ++index 856e4b426..24ed02566 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/CleanControllerBrokerMetaSubCommand.java ++@@ -37,7 +37,7 @@ public class CleanControllerBrokerMetaSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Clean metadata of broker on controller"; +++ return "Clean metadata of broker on controller."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java ++index 70bd7f8e9..966443127 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/GetControllerMetaDataSubCommand.java ++@@ -34,7 +34,7 @@ public class GetControllerMetaDataSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Get controller cluster's metadata"; +++ return "Get controller cluster's metadata."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java ++index 1affe81f9..a522a903d 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/controller/ReElectMasterSubCommand.java ++@@ -37,7 +37,7 @@ public class ReElectMasterSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Re-elect the specified broker as master"; +++ return "Re-elect the specified broker as master."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++index b8191296d..03613b29c 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++@@ -42,7 +42,7 @@ public class ExportConfigsCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Export configs"; +++ return "Export configs."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java ++index 1f9cf7d96..748f7b16e 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataCommand.java ++@@ -46,7 +46,7 @@ public class ExportMetadataCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Export metadata"; +++ return "Export metadata."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java ++index a793b4b84..5d8bb37ba 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetricsCommand.java ++@@ -56,7 +56,7 @@ public class ExportMetricsCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Export metrics"; +++ return "Export metrics."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java ++index 44b3ec3e1..b6231e4f9 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/GetSyncStateSetSubCommand.java ++@@ -40,7 +40,7 @@ public class GetSyncStateSetSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch syncStateSet for target brokers"; +++ return "Fetch syncStateSet for target brokers."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java ++index b1795e046..931658a08 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/ha/HAStatusSubCommand.java ++@@ -41,7 +41,7 @@ public class HAStatusSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch ha runtime status data"; +++ return "Fetch ha runtime status data."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java ++index 4c6d5ffb6..b15b59d50 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/CheckMsgSendRTCommand.java ++@@ -40,7 +40,7 @@ public class CheckMsgSendRTCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Check message send response time"; +++ return "Check message send response time."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java ++index 8aed59ea4..02ff53269 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/ConsumeMessageCommand.java ++@@ -70,7 +70,7 @@ public class ConsumeMessageCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Consume message"; +++ return "Consume message."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java ++index ae6d9bdcf..eee8f3d4b 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/DumpCompactionLogCommand.java ++@@ -38,7 +38,7 @@ import java.nio.file.Paths; ++ public class DumpCompactionLogCommand implements SubCommand { ++ @Override ++ public String commandDesc() { ++- return "parse compaction log to message"; +++ return "Parse compaction log to message."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java ++index 654560167..0418e88a7 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageByQueueCommand.java ++@@ -108,7 +108,7 @@ public class PrintMessageByQueueCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Print Message Detail"; +++ return "Print Message Detail by queueId."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java ++index d01c36d42..bb82f5079 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/PrintMessageSubCommand.java ++@@ -62,7 +62,7 @@ public class PrintMessageSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Print Message Detail"; +++ return "Print Message Detail."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java ++index 2880477f1..b42612150 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByIdSubCommand.java ++@@ -186,7 +186,7 @@ public class QueryMsgByIdSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query Message by Id"; +++ return "Query Message by Id."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java ++index ba7b00c3b..64627fd19 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByKeySubCommand.java ++@@ -36,7 +36,7 @@ public class QueryMsgByKeySubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query Message by Key"; +++ return "Query Message by Key."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java ++index d27313af1..14d0625fd 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByOffsetSubCommand.java ++@@ -39,7 +39,7 @@ public class QueryMsgByOffsetSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query Message by offset"; +++ return "Query Message by offset."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java ++index 1b28f8be1..b71cee901 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgByUniqueKeySubCommand.java ++@@ -141,7 +141,7 @@ public class QueryMsgByUniqueKeySubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query Message by Unique key"; +++ return "Query Message by Unique key."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java ++index 2b982efef..2c546ec56 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/QueryMsgTraceByIdSubCommand.java ++@@ -65,7 +65,7 @@ public class QueryMsgTraceByIdSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Query a message trace"; +++ return "Query a message trace."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java ++index 836ee192b..970da6b16 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/message/SendMessageCommand.java ++@@ -41,7 +41,7 @@ public class SendMessageCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Send a message"; +++ return "Send a message."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java ++index 98542d065..0b0a075bd 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/AddWritePermSubCommand.java ++@@ -34,7 +34,7 @@ public class AddWritePermSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Add write perm of broker in all name server you defined in the -n param"; +++ return "Add write perm of broker in all name server you defined in the -n param."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java ++index 213931ed8..637dd52c8 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/namesrv/WipeWritePermSubCommand.java ++@@ -34,7 +34,7 @@ public class WipeWritePermSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Wipe write perm of broker in all name server you defined in the -n param"; +++ return "Wipe write perm of broker in all name server you defined in the -n param."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java ++index 139821f9c..b22491a59 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/offset/SkipAccumulationSubCommand.java ++@@ -41,7 +41,7 @@ public class SkipAccumulationSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Skip all messages that are accumulated (not consumed) currently"; +++ return "Skip all messages that are accumulated (not consumed) currently."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java ++index 1d49bbe11..96097a93e 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/stats/StatsAllSubCommand.java ++@@ -144,7 +144,7 @@ public class StatsAllSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Topic and Consumer tps stats"; +++ return "Topic and Consumer tps stats."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java ++index 3fa42f297..6a9b81eb8 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/AllocateMQSubCommand.java ++@@ -41,7 +41,7 @@ public class AllocateMQSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Allocate MQ"; +++ return "Allocate MQ."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java ++index 1dab693d9..098f34ff0 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicClusterSubCommand.java ++@@ -34,7 +34,7 @@ public class TopicClusterSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Get cluster info for topic"; +++ return "Get cluster info for topic."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java ++index 346bac704..d9a279f80 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicListSubCommand.java ++@@ -45,7 +45,7 @@ public class TopicListSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Fetch all topic list from name server"; +++ return "Fetch all topic list from name server."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java ++index f2dabec4e..70949d388 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicRouteSubCommand.java ++@@ -42,7 +42,7 @@ public class TopicRouteSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Examine topic route info"; +++ return "Examine topic route info."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java ++index fdb249fab..a1619eced 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/TopicStatusSubCommand.java ++@@ -40,7 +40,7 @@ public class TopicStatusSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Examine topic Status info"; +++ return "Examine topic Status info."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java ++index bebc646b4..3040d04c2 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateOrderConfCommand.java ++@@ -36,7 +36,7 @@ public class UpdateOrderConfCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Create or update or delete order conf"; +++ return "Create or update or delete order conf."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java ++index 85a18c654..3daeee86c 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateStaticTopicSubCommand.java ++@@ -48,7 +48,7 @@ public class UpdateStaticTopicSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update or create static topic, which has fixed number of queues"; +++ return "Update or create static topic, which has fixed number of queues."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java ++index aaa881538..d27cd1861 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicPermSubCommand.java ++@@ -44,7 +44,7 @@ public class UpdateTopicPermSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update topic perm"; +++ return "Update topic perm."; ++ } ++ ++ @Override ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java ++index b68463396..298914175 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/topic/UpdateTopicSubCommand.java ++@@ -42,7 +42,7 @@ public class UpdateTopicSubCommand implements SubCommand { ++ ++ @Override ++ public String commandDesc() { ++- return "Update or create topic"; +++ return "Update or create topic."; ++ } ++ ++ @Override ++-- ++2.32.0.windows.2 ++ ++ ++From 744167bd01fab6821b4d5ae1794dc845153d5156 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Wed, 23 Aug 2023 08:32:17 +0800 ++Subject: [PATCH 2/8] [ISSUE #7142] Add command `RocksDBConfigToJson` to ++ inspect rocksdb content (#7180) ++ ++* feat: add command `RocksDBConfigToJson` to inspect rocksdb content ++ ++Signed-off-by: Ziy1-Tan ++ ++* refactor: fix style ++ ++--------- ++ ++Signed-off-by: Ziy1-Tan ++Co-authored-by: Ziy1-Tan ++--- ++ .../tools/command/MQAdminStartup.java | 2 + ++ .../metadata/RocksDBConfigToJsonCommand.java | 118 ++++++++++++++++++ ++ .../metadata/KvConfigToJsonCommandTest.java | 65 ++++++++++ ++ 3 files changed, 185 insertions(+) ++ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++ create mode 100644 tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++index 890125ca0..324aa1856 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++@@ -80,6 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; ++ import org.apache.rocketmq.tools.command.message.SendMessageCommand; +++import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; ++ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; ++ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; ++ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; ++@@ -211,6 +212,7 @@ public class MQAdminStartup { ++ ++ initCommand(new ClusterListSubCommand()); ++ initCommand(new TopicListSubCommand()); +++ initCommand(new RocksDBConfigToJsonCommand()); ++ ++ initCommand(new UpdateKvConfigCommand()); ++ initCommand(new DeleteKvConfigCommand()); ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++new file mode 100644 ++index 000000000..3053f4684 ++--- /dev/null +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++@@ -0,0 +1,118 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.metadata; +++ +++import com.alibaba.fastjson.JSONObject; +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.Option; +++import org.apache.commons.cli.Options; +++import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.common.config.RocksDBConfigManager; +++import org.apache.rocketmq.common.utils.DataConverter; +++import org.apache.rocketmq.remoting.RPCHook; +++import org.apache.rocketmq.tools.command.SubCommand; +++import org.apache.rocketmq.tools.command.SubCommandException; +++ +++import java.io.File; +++import java.util.HashMap; +++import java.util.Map; +++ +++public class RocksDBConfigToJsonCommand implements SubCommand { +++ private static final String TOPICS_JSON_CONFIG = "topics"; +++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; +++ +++ @Override +++ public String commandName() { +++ return "rocksDBConfigToJson"; +++ } +++ +++ @Override +++ public String commandDesc() { +++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; +++ } +++ +++ @Override +++ public Options buildCommandlineOptions(Options options) { +++ Option pathOption = new Option("p", "path", true, +++ "Absolute path to the metadata directory"); +++ pathOption.setRequired(true); +++ options.addOption(pathOption); +++ +++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + +++ "topics/subscriptionGroups"); +++ configTypeOption.setRequired(true); +++ options.addOption(configTypeOption); +++ +++ return options; +++ } +++ +++ @Override +++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { +++ String path = commandLine.getOptionValue("path").trim(); +++ if (StringUtils.isEmpty(path) || !new File(path).exists()) { +++ System.out.print("Rocksdb path is invalid.\n"); +++ return; +++ } +++ +++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); +++ +++ final long memTableFlushInterval = 60 * 60 * 1000L; +++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); +++ try { +++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { +++ // for topics.json +++ final Map topicsJsonConfig = new HashMap<>(); +++ final Map topicConfigTable = new HashMap<>(); +++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { +++ final String topic = new String(key, DataConverter.charset); +++ final String topicConfig = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig); +++ topicConfigTable.put(topic, jsonObject); +++ }); +++ +++ if (isLoad) { +++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); +++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); +++ System.out.print(topicsJsonStr + "\n"); +++ return; +++ } +++ } +++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { +++ // for subscriptionGroup.json +++ final Map subscriptionGroupJsonConfig = new HashMap<>(); +++ final Map subscriptionGroupTable = new HashMap<>(); +++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { +++ final String subscriptionGroup = new String(key, DataConverter.charset); +++ final String subscriptionGroupConfig = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); +++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); +++ }); +++ +++ if (isLoad) { +++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", +++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); +++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); +++ System.out.print(subscriptionGroupJsonStr + "\n"); +++ return; +++ } +++ } +++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); +++ } finally { +++ kvConfigManager.stop(); +++ } +++ } +++} ++diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++new file mode 100644 ++index 000000000..b2f66c7b0 ++--- /dev/null +++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++@@ -0,0 +1,65 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.metadata; +++ +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.DefaultParser; +++import org.apache.commons.cli.Options; +++import org.apache.rocketmq.srvutil.ServerUtil; +++import org.apache.rocketmq.tools.command.SubCommandException; +++import org.junit.Test; +++ +++import java.io.File; +++ +++import static org.assertj.core.api.Assertions.assertThat; +++ +++public class KvConfigToJsonCommandTest { +++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; +++ +++ @Test +++ public void testExecute() throws SubCommandException { +++ { +++ String[] cases = new String[]{"topics", "subscriptionGroups"}; +++ for (String c : cases) { +++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ Options options = ServerUtil.buildCommandlineOptions(new Options()); +++ String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; +++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, +++ cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.execute(commandLine, options, null); +++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); +++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); +++ } +++ } +++ // invalid cases +++ { +++ String[][] cases = new String[][]{ +++ {"-p " + BASE_PATH + "tmpPath", "-t topics"}, +++ {"-p ", "-t topics"}, +++ {"-p " + BASE_PATH + "topics", "-t invalid_type"} +++ }; +++ +++ for (String[] c : cases) { +++ RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ Options options = ServerUtil.buildCommandlineOptions(new Options()); +++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, +++ cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.execute(commandLine, options, null); +++ } +++ } +++ } +++} ++-- ++2.32.0.windows.2 ++ ++ ++From bdede35db365a49b211cdc249c68b0f60a3df46d Mon Sep 17 00:00:00 2001 ++From: mxsm ++Date: Wed, 23 Aug 2023 08:34:56 +0800 ++Subject: [PATCH 3/8] [ISSUE #7124] Fix the typos in the code comments (#7125) ++ ++--- ++ .../apache/rocketmq/broker/processor/ReplyMessageProcessor.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++index b2db356c8..d3bb048f7 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/ReplyMessageProcessor.java ++@@ -234,7 +234,7 @@ public class ReplyMessageProcessor extends AbstractSendMessageProcessor { ++ } else { ++ response.setCode(ResponseCode.SUCCESS); ++ response.setRemark(null); ++- //set to zore to avoid client decoding exception +++ //set to zero to avoid client decoding exception ++ responseHeader.setMsgId("0"); ++ responseHeader.setQueueId(queueIdInt); ++ responseHeader.setQueueOffset(0L); ++-- ++2.32.0.windows.2 ++ ++ ++From 9bb73b9a38548b99ac5126c40380c3c2e7fc586e Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 23 Aug 2023 09:46:27 +0800 ++Subject: [PATCH 4/8] [#ISSUE 7222] Bug fix and refactoring of the Indexfile in ++ tiered storage (#7224) ++ ++--- ++ .../tieredstore/file/TieredIndexFile.java | 38 +++++++-- ++ .../tieredstore/file/TieredIndexFileTest.java | 84 +++++-------------- ++ 2 files changed, 52 insertions(+), 70 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++index 50beb01ae..eda5e0106 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredIndexFile.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ +++import com.google.common.annotations.VisibleForTesting; ++ import java.io.File; ++ import java.io.IOException; ++ import java.nio.ByteBuffer; ++@@ -99,7 +100,7 @@ public class TieredIndexFile { ++ this::doScheduleTask, 10, 10, TimeUnit.SECONDS); ++ } ++ ++- private void doScheduleTask() { +++ protected void doScheduleTask() { ++ try { ++ curFileLock.lock(); ++ try { ++@@ -145,6 +146,11 @@ public class TieredIndexFile { ++ } ++ } ++ +++ @VisibleForTesting +++ public MappedFile getPreMappedFile() { +++ return preMappedFile; +++ } +++ ++ private void initFile() throws IOException { ++ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); ++ initIndexFileHeader(curMappedFile); ++@@ -156,19 +162,26 @@ public class TieredIndexFile { ++ ++ if (isFileSealed(curMappedFile)) { ++ if (preFileExists) { ++- preFile.delete(); +++ if (preFile.delete()) { +++ logger.info("Pre IndexFile deleted success", preFilepath); +++ } else { +++ logger.error("Pre IndexFile deleted failed", preFilepath); +++ } ++ } ++ boolean rename = curMappedFile.renameTo(preFilepath); ++ if (rename) { ++ preMappedFile = curMappedFile; ++ curMappedFile = new DefaultMappedFile(curFilePath, fileMaxSize); +++ initIndexFileHeader(curMappedFile); ++ preFileExists = true; ++ } ++ } +++ ++ if (preFileExists) { ++ synchronized (TieredIndexFile.class) { ++ if (inflightCompactFuture.isDone()) { ++- inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit(new CompactTask(storeConfig, preMappedFile, flatFile), null); +++ inflightCompactFuture = TieredStoreExecutor.compactIndexFileExecutor.submit( +++ new CompactTask(storeConfig, preMappedFile, flatFile), null); ++ } ++ } ++ } ++@@ -261,7 +274,8 @@ public class TieredIndexFile { ++ } ++ } ++ ++- public CompletableFuture>> queryAsync(String topic, String key, long beginTime, long endTime) { +++ public CompletableFuture>> queryAsync(String topic, String key, long beginTime, +++ long endTime) { ++ int hashCode = indexKeyHashMethod(buildKey(topic, key)); ++ int slotPosition = hashCode % maxHashSlotNum; ++ List fileSegmentList = flatFile.getFileListByTime(beginTime, endTime); ++@@ -355,7 +369,7 @@ public class TieredIndexFile { ++ private final int fileMaxSize; ++ private MappedFile originFile; ++ private TieredFlatFile fileQueue; ++- private final MappedFile compactFile; +++ private MappedFile compactFile; ++ ++ public CompactTask(TieredMessageStoreConfig storeConfig, MappedFile originFile, ++ TieredFlatFile fileQueue) throws IOException { ++@@ -381,6 +395,17 @@ public class TieredIndexFile { ++ } catch (Throwable throwable) { ++ logger.error("TieredIndexFile#compactTask: compact index file failed:", throwable); ++ } +++ +++ try { +++ if (originFile != null) { +++ originFile.destroy(-1); +++ } +++ if (compactFile != null) { +++ compactFile.destroy(-1); +++ } +++ } catch (Throwable throwable) { +++ logger.error("TieredIndexFile#compactTask: destroy index file failed:", throwable); +++ } ++ } ++ ++ public void compact() { ++@@ -396,6 +421,8 @@ public class TieredIndexFile { ++ fileQueue.commit(true); ++ compactFile.destroy(-1); ++ originFile.destroy(-1); +++ compactFile = null; +++ originFile = null; ++ } ++ ++ private void buildCompactFile() { ++@@ -414,6 +441,7 @@ public class TieredIndexFile { ++ if (slotValue != -1) { ++ int indexTotalSize = 0; ++ int indexPosition = slotValue; +++ ++ while (indexPosition >= 0 && indexPosition < maxIndexNum) { ++ int indexOffset = INDEX_FILE_HEADER_SIZE + maxHashSlotNum * INDEX_FILE_HASH_SLOT_SIZE ++ + indexPosition * INDEX_FILE_HASH_ORIGIN_INDEX_SIZE; ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++index 7ef49578d..262d6645b 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++@@ -19,9 +19,8 @@ package org.apache.rocketmq.tieredstore.file; ++ import com.sun.jna.Platform; ++ import java.io.IOException; ++ import java.nio.ByteBuffer; +++import java.time.Duration; ++ import java.util.List; ++-import java.util.concurrent.TimeUnit; ++-import org.apache.commons.lang3.SystemUtils; ++ import org.apache.commons.lang3.tuple.Pair; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; ++@@ -31,9 +30,7 @@ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ import org.awaitility.Awaitility; ++ import org.junit.After; ++ import org.junit.Assert; ++-import org.junit.Assume; ++ import org.junit.Before; ++-import org.junit.Ignore; ++ import org.junit.Test; ++ ++ public class TieredIndexFileTest { ++@@ -45,11 +42,12 @@ public class TieredIndexFileTest { ++ @Before ++ public void setUp() { ++ storeConfig = new TieredMessageStoreConfig(); +++ storeConfig.setBrokerName("IndexFileBroker"); ++ storeConfig.setStorePathRootDir(storePath); ++- storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.memory.MemoryFileSegment"); ++- storeConfig.setTieredStoreIndexFileMaxHashSlotNum(2); ++- storeConfig.setTieredStoreIndexFileMaxIndexNum(3); ++- mq = new MessageQueue("TieredIndexFileTest", storeConfig.getBrokerName(), 1); +++ storeConfig.setTieredBackendServiceProvider("org.apache.rocketmq.tieredstore.provider.posix.PosixFileSegment"); +++ storeConfig.setTieredStoreIndexFileMaxHashSlotNum(5); +++ storeConfig.setTieredStoreIndexFileMaxIndexNum(20); +++ mq = new MessageQueue("IndexFileTest", storeConfig.getBrokerName(), 1); ++ TieredStoreUtil.getMetadataStore(storeConfig); ++ TieredStoreExecutor.init(); ++ } ++@@ -61,77 +59,33 @@ public class TieredIndexFileTest { ++ TieredStoreExecutor.shutdown(); ++ } ++ ++- @Ignore ++ @Test ++ public void testAppendAndQuery() throws IOException, ClassNotFoundException, NoSuchMethodException { ++ if (Platform.isWindows()) { ++ return; ++ } ++ ++- // skip this test on windows ++- Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS); ++- ++ TieredFileAllocator fileQueueFactory = new TieredFileAllocator(storeConfig); ++ TieredIndexFile indexFile = new TieredIndexFile(fileQueueFactory, storePath); +++ ++ indexFile.append(mq, 0, "key3", 3, 300, 1000); ++ indexFile.append(mq, 0, "key2", 2, 200, 1100); ++ indexFile.append(mq, 0, "key1", 1, 100, 1200); ++ ++- Awaitility.waitAtMost(5, TimeUnit.SECONDS) ++- .until(() -> { ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++- if (indexList.size() != 1) { ++- return false; ++- } ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 2, indexBuffer.remaining()); ++- ++- Assert.assertEquals(1, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++- ++- Assert.assertEquals(3, indexBuffer.getLong(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4)); ++- Assert.assertEquals(300, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE + 4 + 4 + 4 + 8 + 4)); ++- return true; ++- }); ++- ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- indexFile.append(mq, 0, "key4", 4, 400, 1300); ++- ++- Awaitility.waitAtMost(5, TimeUnit.SECONDS) ++- .until(() -> { ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1300, 1300).join(); ++- if (indexList.size() != 1) { ++- return false; ++- } ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); ++- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); ++- return true; ++- }); ++- ++- List> indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1300, 1300).join(); +++ // do not do schedule task here +++ TieredStoreExecutor.shutdown(); +++ List> indexList = +++ indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++ Assert.assertEquals(0, indexList.size()); ++ ++- indexList = indexFile.queryAsync(mq.getTopic(), "key4", 1200, 1300).join(); ++- Assert.assertEquals(2, indexList.size()); ++- ++- ByteBuffer indexBuffer = indexList.get(0).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE * 3, indexBuffer.remaining()); ++- Assert.assertEquals(4, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(400, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(0, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +++ // do compaction once +++ TieredStoreExecutor.init(); +++ storeConfig.setTieredStoreIndexFileRollingIdleInterval(0); +++ indexFile.doScheduleTask(); +++ Awaitility.await().atMost(Duration.ofSeconds(10)) +++ .until(() -> !indexFile.getPreMappedFile().getFile().exists()); ++ ++- indexBuffer = indexList.get(1).getValue(); ++- Assert.assertEquals(TieredIndexFile.INDEX_FILE_HASH_COMPACT_INDEX_SIZE, indexBuffer.remaining()); ++- Assert.assertEquals(2, indexBuffer.getLong(4 + 4 + 4)); ++- Assert.assertEquals(200, indexBuffer.getInt(4 + 4 + 4 + 8)); ++- Assert.assertEquals(100, indexBuffer.getInt(4 + 4 + 4 + 8 + 4)); +++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); +++ Assert.assertEquals(1, indexList.size()); ++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 69c26d3d29cde7b4484ecd112ab9224f9f42bf45 Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Wed, 23 Aug 2023 10:27:52 +0800 ++Subject: [PATCH 5/8] [ISSUE #7228] Converge the use of some important ++ variables for some class ++ ++--- ++ .../apache/rocketmq/store/ConsumeQueue.java | 16 ++++++------ ++ .../rocketmq/store/MappedFileQueue.java | 26 +++++++++++-------- ++ .../store/MultiPathMappedFileQueue.java | 4 +-- ++ 3 files changed, 24 insertions(+), 22 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++index a0b886eb0..56bee2af3 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++@@ -145,7 +145,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ if (offset >= 0 && size > 0) { ++ mappedFileOffset = i + CQ_STORE_UNIT_SIZE; ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++ } ++@@ -409,7 +409,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ int logicFileSize = this.mappedFileSize; ++ ++- this.maxPhysicOffset = phyOffset; +++ this.setMaxPhysicOffset(phyOffset); ++ long maxExtAddr = 1; ++ boolean shouldDeleteFile = false; ++ while (true) { ++@@ -435,7 +435,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ mappedFile.setWrotePosition(pos); ++ mappedFile.setCommittedPosition(pos); ++ mappedFile.setFlushedPosition(pos); ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ // This maybe not take effect, when not every consume queue has extend file. ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++@@ -453,7 +453,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ mappedFile.setWrotePosition(pos); ++ mappedFile.setCommittedPosition(pos); ++ mappedFile.setFlushedPosition(pos); ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ if (isExtAddr(tagsCode)) { ++ maxExtAddr = tagsCode; ++ } ++@@ -881,8 +881,8 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, ++ final long cqOffset) { ++ ++- if (offset + size <= this.maxPhysicOffset) { ++- log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", maxPhysicOffset, offset); +++ if (offset + size <= this.getMaxPhysicOffset()) { +++ log.warn("Maybe try to build consume queue repeatedly maxPhysicOffset={} phyOffset={}", this.getMaxPhysicOffset(), offset); ++ return true; ++ } ++ ++@@ -926,7 +926,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ); ++ } ++ } ++- this.maxPhysicOffset = offset + size; +++ this.setMaxPhysicOffset(offset + size); ++ return mappedFile.appendMessage(this.byteBufferIndex.array()); ++ } ++ return false; ++@@ -1130,7 +1130,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { ++ ++ @Override ++ public void destroy() { ++- this.maxPhysicOffset = -1; +++ this.setMaxPhysicOffset(-1); ++ this.minLogicOffset = 0; ++ this.mappedFileQueue.destroy(); ++ if (isExtReadEnable()) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++index 0bc70642f..32b90d14f 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++@@ -285,7 +285,7 @@ public class MappedFileQueue implements Swappable { ++ if (this.mappedFiles.isEmpty()) ++ return 0; ++ ++- long committed = this.flushedWhere; +++ long committed = this.getFlushedWhere(); ++ if (committed != 0) { ++ MappedFile mappedFile = this.getLastMappedFile(0, false); ++ if (mappedFile != null) { ++@@ -442,11 +442,11 @@ public class MappedFileQueue implements Swappable { ++ } ++ ++ public long remainHowManyDataToCommit() { ++- return getMaxWrotePosition() - committedWhere; +++ return getMaxWrotePosition() - getCommittedWhere(); ++ } ++ ++ public long remainHowManyDataToFlush() { ++- return getMaxOffset() - flushedWhere; +++ return getMaxOffset() - this.getFlushedWhere(); ++ } ++ ++ public void deleteLastMappedFile() { ++@@ -616,15 +616,15 @@ public class MappedFileQueue implements Swappable { ++ ++ public boolean flush(final int flushLeastPages) { ++ boolean result = true; ++- MappedFile mappedFile = this.findMappedFileByOffset(this.flushedWhere, this.flushedWhere == 0); +++ MappedFile mappedFile = this.findMappedFileByOffset(this.getFlushedWhere(), this.getFlushedWhere() == 0); ++ if (mappedFile != null) { ++ long tmpTimeStamp = mappedFile.getStoreTimestamp(); ++ int offset = mappedFile.flush(flushLeastPages); ++ long where = mappedFile.getFileFromOffset() + offset; ++- result = where == this.flushedWhere; ++- this.flushedWhere = where; +++ result = where == this.getFlushedWhere(); +++ this.setFlushedWhere(where); ++ if (0 == flushLeastPages) { ++- this.storeTimestamp = tmpTimeStamp; +++ this.setStoreTimestamp(tmpTimeStamp); ++ } ++ } ++ ++@@ -633,12 +633,12 @@ public class MappedFileQueue implements Swappable { ++ ++ public synchronized boolean commit(final int commitLeastPages) { ++ boolean result = true; ++- MappedFile mappedFile = this.findMappedFileByOffset(this.committedWhere, this.committedWhere == 0); +++ MappedFile mappedFile = this.findMappedFileByOffset(this.getCommittedWhere(), this.getCommittedWhere() == 0); ++ if (mappedFile != null) { ++ int offset = mappedFile.commit(commitLeastPages); ++ long where = mappedFile.getFileFromOffset() + offset; ++- result = where == this.committedWhere; ++- this.committedWhere = where; +++ result = where == this.getCommittedWhere(); +++ this.setCommittedWhere(where); ++ } ++ ++ return result; ++@@ -763,7 +763,7 @@ public class MappedFileQueue implements Swappable { ++ mf.destroy(1000 * 3); ++ } ++ this.mappedFiles.clear(); ++- this.flushedWhere = 0; +++ this.setFlushedWhere(0); ++ ++ // delete parent directory ++ File file = new File(storePath); ++@@ -848,6 +848,10 @@ public class MappedFileQueue implements Swappable { ++ return storeTimestamp; ++ } ++ +++ public void setStoreTimestamp(long storeTimestamp) { +++ this.storeTimestamp = storeTimestamp; +++ } +++ ++ public List getMappedFiles() { ++ return mappedFiles; ++ } ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++index 8f5af9438..8ff050dfe 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MultiPathMappedFileQueue.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.store; ++ ++- ++ import java.util.Arrays; ++ import java.util.HashSet; ++ import java.util.Set; ++@@ -113,8 +112,7 @@ public class MultiPathMappedFileQueue extends MappedFileQueue { ++ mf.destroy(1000 * 3); ++ } ++ this.mappedFiles.clear(); ++- this.flushedWhere = 0; ++- +++ this.setFlushedWhere(0); ++ ++ Set storePathSet = getPaths(); ++ storePathSet.addAll(getReadonlyPaths()); ++-- ++2.32.0.windows.2 ++ ++ ++From 3884f595949462044c5cb3c236199bc1d7ad2341 Mon Sep 17 00:00:00 2001 ++From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= ++ ++Date: Wed, 23 Aug 2023 11:10:30 +0800 ++Subject: [PATCH 6/8] [ISSUE #7149] When creating and updating Topic, there ++ will be problems with permission settings (#7151) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings ++ ++* [ISSUE #7149] fix bug : When creating and updating Topic, there will be problems with permission settings ++ ++* [issue#7249] ++ ++--------- ++ ++Co-authored-by: 十真 ++--- ++ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 3 ++- ++ 1 file changed, 2 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 13f9d002b..e8f943702 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1733,7 +1733,8 @@ public class BrokerController { ++ new TopicConfig(topicConfig.getTopicName(), ++ topicConfig.getReadQueueNums(), ++ topicConfig.getWriteQueueNums(), ++- this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); +++ topicConfig.getPerm() +++ & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); ++ } else { ++ registerTopicConfig = new TopicConfig(topicConfig); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 017ad110475e8024585327b44f47e5e97aabc63b Mon Sep 17 00:00:00 2001 ++From: echooymxq ++Date: Wed, 23 Aug 2023 11:11:42 +0800 ++Subject: [PATCH 7/8] [ISSUE #7219] Fix Concurrent modify syncStateSet and Mark ++ synchronizing frequently when shrink. (#7220) ++ ++--- ++ .../broker/controller/ReplicasManager.java | 29 ++++++++++--------- ++ .../ha/autoswitch/AutoSwitchHAService.java | 21 ++++++++------ ++ 2 files changed, 28 insertions(+), 22 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++index abae7cdb0..37c82e434 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++@@ -542,7 +542,7 @@ public class ReplicasManager { ++ this.brokerMetadata.updateAndPersist(brokerConfig.getBrokerClusterName(), brokerConfig.getBrokerName(), tempBrokerMetadata.getBrokerId()); ++ this.tempBrokerMetadata.clear(); ++ this.brokerControllerId = this.brokerMetadata.getBrokerId(); ++- this.haService.setBrokerControllerId(this.brokerControllerId); +++ this.haService.setLocalBrokerId(this.brokerControllerId); ++ return true; ++ } catch (Exception e) { ++ LOGGER.error("fail to create metadata file", e); ++@@ -594,7 +594,7 @@ public class ReplicasManager { ++ if (this.brokerMetadata.isLoaded()) { ++ this.registerState = RegisterState.CREATE_METADATA_FILE_DONE; ++ this.brokerControllerId = brokerMetadata.getBrokerId(); ++- this.haService.setBrokerControllerId(this.brokerControllerId); +++ this.haService.setLocalBrokerId(this.brokerControllerId); ++ return; ++ } ++ // 2. check if temp metadata exist ++@@ -735,23 +735,26 @@ public class ReplicasManager { ++ if (this.checkSyncStateSetTaskFuture != null) { ++ this.checkSyncStateSetTaskFuture.cancel(false); ++ } ++- this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(() -> { ++- checkSyncStateSetAndDoReport(); ++- }, 3 * 1000, this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); +++ this.checkSyncStateSetTaskFuture = this.scheduledService.scheduleAtFixedRate(this::checkSyncStateSetAndDoReport, 3 * 1000, +++ this.brokerConfig.getCheckSyncStateSetPeriod(), TimeUnit.MILLISECONDS); ++ } ++ ++ private void checkSyncStateSetAndDoReport() { ++- final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); ++- newSyncStateSet.add(this.brokerControllerId); ++- synchronized (this) { ++- if (this.syncStateSet != null) { ++- // Check if syncStateSet changed ++- if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { ++- return; +++ try { +++ final Set newSyncStateSet = this.haService.maybeShrinkSyncStateSet(); +++ newSyncStateSet.add(this.brokerControllerId); +++ synchronized (this) { +++ if (this.syncStateSet != null) { +++ // Check if syncStateSet changed +++ if (this.syncStateSet.size() == newSyncStateSet.size() && this.syncStateSet.containsAll(newSyncStateSet)) { +++ return; +++ } ++ } ++ } +++ doReportSyncStateSetChanged(newSyncStateSet); +++ } catch (Exception e) { +++ LOGGER.error("Check syncStateSet error", e); ++ } ++- doReportSyncStateSetChanged(newSyncStateSet); ++ } ++ ++ private void doReportSyncStateSetChanged(Set newSyncStateSet) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++index 6dc734e0c..d5393fdca 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++@@ -41,6 +41,7 @@ import java.nio.channels.SocketChannel; ++ import java.util.ArrayList; ++ import java.util.HashSet; ++ import java.util.List; +++import java.util.Iterator; ++ import java.util.Map; ++ import java.util.Objects; ++ import java.util.Set; ++@@ -73,7 +74,7 @@ public class AutoSwitchHAService extends DefaultHAService { ++ private EpochFileCache epochCache; ++ private AutoSwitchHAClient haClient; ++ ++- private Long brokerControllerId = null; +++ private Long localBrokerId = null; ++ ++ public AutoSwitchHAService() { ++ } ++@@ -287,9 +288,11 @@ public class AutoSwitchHAService extends DefaultHAService { ++ ++ // If the slaveBrokerId is in syncStateSet but not in connectionCaughtUpTimeTable, ++ // it means that the broker has not connected. ++- for (Long slaveBrokerId : newSyncStateSet) { ++- if (!this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { ++- newSyncStateSet.remove(slaveBrokerId); +++ Iterator iterator = newSyncStateSet.iterator(); +++ while (iterator.hasNext()) { +++ Long slaveBrokerId = iterator.next(); +++ if (!Objects.equals(slaveBrokerId, this.localBrokerId) && !this.connectionCaughtUpTimeTable.containsKey(slaveBrokerId)) { +++ iterator.remove(); ++ isSyncStateSetChanged = true; ++ } ++ } ++@@ -419,7 +422,7 @@ public class AutoSwitchHAService extends DefaultHAService { ++ // To avoid the syncStateSet is not consistent with connectionList. ++ // Fix issue: https://github.com/apache/rocketmq/issues/6662 ++ for (Long syncId : currentSyncStateSet) { ++- if (!idList.contains(syncId) && this.brokerControllerId != null && !Objects.equals(syncId, this.brokerControllerId)) { +++ if (!idList.contains(syncId) && this.localBrokerId != null && !Objects.equals(syncId, this.localBrokerId)) { ++ LOGGER.warn("Slave {} is still in syncStateSet, but has lost its connection. So new offset can't be compute.", syncId); ++ // Without check and re-compute, return the confirmOffset's value directly. ++ return this.defaultMessageStore.getConfirmOffsetDirectly(); ++@@ -545,12 +548,12 @@ public class AutoSwitchHAService extends DefaultHAService { ++ return this.epochCache.getAllEntries(); ++ } ++ ++- public Long getBrokerControllerId() { ++- return brokerControllerId; +++ public Long getLocalBrokerId() { +++ return localBrokerId; ++ } ++ ++- public void setBrokerControllerId(Long brokerControllerId) { ++- this.brokerControllerId = brokerControllerId; +++ public void setLocalBrokerId(Long localBrokerId) { +++ this.localBrokerId = localBrokerId; ++ } ++ ++ class AutoSwitchAcceptSocketService extends AcceptSocketService { ++-- ++2.32.0.windows.2 ++ ++ ++From 77e8e54b37c3fc3ea0beffc1ace6f5bf20af10d9 Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Wed, 23 Aug 2023 15:56:39 +0800 ++Subject: [PATCH 8/8] [ISSUE #7223] Support batch ack for grpc client in proxy ++ (#7225) ++ ++--- ++ .../client/impl/mqclient/MQClientAPIExt.java | 26 +++ ++ .../rocketmq/proxy/config/ProxyConfig.java | 10 + ++ .../grpc/v2/consumer/AckMessageActivity.java | 136 ++++++++--- ++ .../proxy/processor/AbstractProcessor.java | 4 +- ++ .../proxy/processor/BatchAckResult.java | 53 +++++ ++ .../proxy/processor/ConsumerProcessor.java | 64 +++++ ++ .../processor/DefaultMessagingProcessor.java | 7 + ++ .../proxy/processor/MessagingProcessor.java | 18 ++ ++ .../message/ClusterMessageService.java | 16 +- ++ .../service/message/LocalMessageService.java | 58 +++++ ++ .../proxy/service/message/MessageService.java | 8 + ++ .../service/message/ReceiptHandleMessage.java | 39 ++++ ++ .../v2/consumer/AckMessageActivityTest.java | 221 +++++++++++++++--- ++ .../proxy/processor/BaseProcessorTest.java | 18 +- ++ .../processor/ConsumerProcessorTest.java | 115 +++++++++ ++ .../service/mqclient/MQClientAPIExtTest.java | 12 + ++ 16 files changed, 728 insertions(+), 77 deletions(-) ++ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++ create mode 100644 proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++index fb8f8d11f..d7c8ef8d9 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++@@ -306,6 +306,32 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ return future; ++ } ++ +++ public CompletableFuture batchAckMessageAsync( +++ String brokerAddr, +++ String topic, +++ String consumerGroup, +++ List extraInfoList, +++ long timeoutMillis +++ ) { +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ this.batchAckMessageAsync(brokerAddr, timeoutMillis, new AckCallback() { +++ @Override +++ public void onSuccess(AckResult ackResult) { +++ future.complete(ackResult); +++ } +++ +++ @Override +++ public void onException(Throwable t) { +++ future.completeExceptionally(t); +++ } +++ }, topic, consumerGroup, extraInfoList); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ ++ public CompletableFuture changeInvisibleTimeAsync( ++ String brokerAddr, ++ String brokerName, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 39caaa0d9..76a243919 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -250,6 +250,8 @@ public class ProxyConfig implements ConfigFile { ++ private long remotingWaitTimeMillsInTopicRouteQueue = 3 * 1000; ++ private long remotingWaitTimeMillsInDefaultQueue = 3 * 1000; ++ +++ private boolean enableBatchAck = false; +++ ++ @Override ++ public void initData() { ++ parseDelayLevel(); ++@@ -1379,4 +1381,12 @@ public class ProxyConfig implements ConfigFile { ++ public void setRemotingWaitTimeMillsInDefaultQueue(long remotingWaitTimeMillsInDefaultQueue) { ++ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; ++ } +++ +++ public boolean isEnableBatchAck() { +++ return enableBatchAck; +++ } +++ +++ public void setEnableBatchAck(boolean enableBatchAck) { +++ this.enableBatchAck = enableBatchAck; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++index 9a3a77201..97c716c8f 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivity.java ++@@ -31,12 +31,15 @@ import org.apache.rocketmq.client.consumer.AckStatus; ++ import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; +++import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.grpc.v2.AbstractMessingActivity; ++ import org.apache.rocketmq.proxy.grpc.v2.channel.GrpcChannelManager; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcConverter; ++ import org.apache.rocketmq.proxy.grpc.v2.common.ResponseBuilder; +++import org.apache.rocketmq.proxy.processor.BatchAckResult; ++ import org.apache.rocketmq.proxy.processor.MessagingProcessor; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ ++ public class AckMessageActivity extends AbstractMessingActivity { ++ ++@@ -50,60 +53,98 @@ public class AckMessageActivity extends AbstractMessingActivity { ++ ++ try { ++ validateTopicAndConsumerGroup(request.getTopic(), request.getGroup()); ++- ++- CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; ++- for (int i = 0; i < request.getEntriesCount(); i++) { ++- futures[i] = processAckMessage(ctx, request, request.getEntries(i)); +++ String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); +++ String topic = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic()); +++ if (ConfigurationManager.getProxyConfig().isEnableBatchAck()) { +++ future = ackMessageInBatch(ctx, group, topic, request); +++ } else { +++ future = ackMessageOneByOne(ctx, group, topic, request); ++ } ++- CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { ++- if (throwable != null) { ++- future.completeExceptionally(throwable); ++- return; ++- } +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ +++ protected CompletableFuture ackMessageInBatch(ProxyContext ctx, String group, String topic, AckMessageRequest request) { +++ List handleMessageList = new ArrayList<>(request.getEntriesCount()); ++ +++ for (AckMessageEntry ackMessageEntry : request.getEntriesList()) { +++ String handleString = getHandleString(ctx, group, request, ackMessageEntry); +++ handleMessageList.add(new ReceiptHandleMessage(ReceiptHandle.decode(handleString), ackMessageEntry.getMessageId())); +++ } +++ return this.messagingProcessor.batchAckMessage(ctx, handleMessageList, group, topic) +++ .thenApply(batchAckResultList -> { +++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder(); ++ Set responseCodes = new HashSet<>(); ++- List entryList = new ArrayList<>(); ++- for (CompletableFuture entryFuture : futures) { ++- AckMessageResultEntry entryResult = entryFuture.join(); ++- responseCodes.add(entryResult.getStatus().getCode()); ++- entryList.add(entryResult); ++- } ++- AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() ++- .addAllEntries(entryList); ++- if (responseCodes.size() > 1) { ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); ++- } else if (responseCodes.size() == 1) { ++- Code code = responseCodes.stream().findAny().get(); ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); ++- } else { ++- responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ AckMessageResultEntry entry = convertToAckMessageResultEntry(batchAckResult); +++ responseBuilder.addEntries(entry); +++ responseCodes.add(entry.getStatus().getCode()); ++ } ++- future.complete(responseBuilder.build()); +++ setAckResponseStatus(responseBuilder, responseCodes); +++ return responseBuilder.build(); ++ }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); +++ } +++ +++ protected AckMessageResultEntry convertToAckMessageResultEntry(BatchAckResult batchAckResult) { +++ ReceiptHandleMessage handleMessage = batchAckResult.getReceiptHandleMessage(); +++ AckMessageResultEntry.Builder resultBuilder = AckMessageResultEntry.newBuilder() +++ .setMessageId(handleMessage.getMessageId()) +++ .setReceiptHandle(handleMessage.getReceiptHandle().getReceiptHandle()); +++ if (batchAckResult.getProxyException() != null) { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(batchAckResult.getProxyException())); +++ } else { +++ AckResult ackResult = batchAckResult.getAckResult(); +++ if (AckStatus.OK.equals(ackResult.getStatus())) { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.OK, Code.OK.name())); +++ } else { +++ resultBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")); +++ } ++ } ++- return future; +++ return resultBuilder.build(); ++ } ++ ++- protected CompletableFuture processAckMessage(ProxyContext ctx, AckMessageRequest request, +++ protected CompletableFuture ackMessageOneByOne(ProxyContext ctx, String group, String topic, AckMessageRequest request) { +++ CompletableFuture resultFuture = new CompletableFuture<>(); +++ CompletableFuture[] futures = new CompletableFuture[request.getEntriesCount()]; +++ for (int i = 0; i < request.getEntriesCount(); i++) { +++ futures[i] = processAckMessage(ctx, group, topic, request, request.getEntries(i)); +++ } +++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +++ if (throwable != null) { +++ resultFuture.completeExceptionally(throwable); +++ return; +++ } +++ +++ Set responseCodes = new HashSet<>(); +++ List entryList = new ArrayList<>(); +++ for (CompletableFuture entryFuture : futures) { +++ AckMessageResultEntry entryResult = entryFuture.join(); +++ responseCodes.add(entryResult.getStatus().getCode()); +++ entryList.add(entryResult); +++ } +++ AckMessageResponse.Builder responseBuilder = AckMessageResponse.newBuilder() +++ .addAllEntries(entryList); +++ setAckResponseStatus(responseBuilder, responseCodes); +++ resultFuture.complete(responseBuilder.build()); +++ }); +++ return resultFuture; +++ } +++ +++ protected CompletableFuture processAckMessage(ProxyContext ctx, String group, String topic, AckMessageRequest request, ++ AckMessageEntry ackMessageEntry) { ++ CompletableFuture future = new CompletableFuture<>(); ++ ++ try { ++- String handleString = ackMessageEntry.getReceiptHandle(); ++- ++- String group = GrpcConverter.getInstance().wrapResourceWithNamespace(request.getGroup()); ++- MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); ++- if (messageReceiptHandle != null) { ++- handleString = messageReceiptHandle.getReceiptHandleStr(); ++- } +++ String handleString = this.getHandleString(ctx, group, request, ackMessageEntry); ++ CompletableFuture ackResultFuture = this.messagingProcessor.ackMessage( ++ ctx, ++ ReceiptHandle.decode(handleString), ++ ackMessageEntry.getMessageId(), ++ group, ++- GrpcConverter.getInstance().wrapResourceWithNamespace(request.getTopic())); +++ topic +++ ); ++ ackResultFuture.thenAccept(result -> { ++ future.complete(convertToAckMessageResultEntry(ctx, ackMessageEntry, result)); ++ }).exceptionally(t -> { ++@@ -139,4 +180,25 @@ public class AckMessageActivity extends AbstractMessingActivity { ++ .setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack failed: status is abnormal")) ++ .build(); ++ } +++ +++ protected void setAckResponseStatus(AckMessageResponse.Builder responseBuilder, Set responseCodes) { +++ if (responseCodes.size() > 1) { +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.MULTIPLE_RESULTS, Code.MULTIPLE_RESULTS.name())); +++ } else if (responseCodes.size() == 1) { +++ Code code = responseCodes.stream().findAny().get(); +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(code, code.name())); +++ } else { +++ responseBuilder.setStatus(ResponseBuilder.getInstance().buildStatus(Code.INTERNAL_SERVER_ERROR, "ack message result is empty")); +++ } +++ } +++ +++ protected String getHandleString(ProxyContext ctx, String group, AckMessageRequest request, AckMessageEntry ackMessageEntry) { +++ String handleString = ackMessageEntry.getReceiptHandle(); +++ +++ MessageReceiptHandle messageReceiptHandle = messagingProcessor.removeReceiptHandle(ctx, grpcChannelManager.getChannel(ctx.getClientID()), group, ackMessageEntry.getMessageId(), ackMessageEntry.getReceiptHandle()); +++ if (messageReceiptHandle != null) { +++ handleString = messageReceiptHandle.getReceiptHandleStr(); +++ } +++ return handleString; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++index b61c3df9e..c63212c23 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/AbstractProcessor.java ++@@ -27,6 +27,8 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { ++ protected MessagingProcessor messagingProcessor; ++ protected ServiceManager serviceManager; ++ +++ protected static final ProxyException EXPIRED_HANDLE_PROXY_EXCEPTION = new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); +++ ++ public AbstractProcessor(MessagingProcessor messagingProcessor, ++ ServiceManager serviceManager) { ++ this.messagingProcessor = messagingProcessor; ++@@ -35,7 +37,7 @@ public abstract class AbstractProcessor extends AbstractStartAndShutdown { ++ ++ protected void validateReceiptHandle(ReceiptHandle handle) { ++ if (handle.isExpired()) { ++- throw new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired"); +++ throw EXPIRED_HANDLE_PROXY_EXCEPTION; ++ } ++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++new file mode 100644 ++index 000000000..dfb9c9b9e ++--- /dev/null +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/BatchAckResult.java ++@@ -0,0 +1,53 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.proxy.processor; +++ +++import org.apache.rocketmq.client.consumer.AckResult; +++import org.apache.rocketmq.proxy.common.ProxyException; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; +++ +++public class BatchAckResult { +++ +++ private final ReceiptHandleMessage receiptHandleMessage; +++ private AckResult ackResult; +++ private ProxyException proxyException; +++ +++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, +++ AckResult ackResult) { +++ this.receiptHandleMessage = receiptHandleMessage; +++ this.ackResult = ackResult; +++ } +++ +++ public BatchAckResult(ReceiptHandleMessage receiptHandleMessage, +++ ProxyException proxyException) { +++ this.receiptHandleMessage = receiptHandleMessage; +++ this.proxyException = proxyException; +++ } +++ +++ public ReceiptHandleMessage getReceiptHandleMessage() { +++ return receiptHandleMessage; +++ } +++ +++ public AckResult getAckResult() { +++ return ackResult; +++ } +++ +++ public ProxyException getProxyException() { +++ return proxyException; +++ } +++} ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++index 656a6339d..f3522b374 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ConsumerProcessor.java ++@@ -48,6 +48,7 @@ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.common.utils.ProxyUtils; ++ import org.apache.rocketmq.proxy.service.ServiceManager; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; ++@@ -241,6 +242,69 @@ public class ConsumerProcessor extends AbstractProcessor { ++ return FutureUtils.addExecutor(future, this.executor); ++ } ++ +++ public CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ) { +++ CompletableFuture> future = new CompletableFuture<>(); +++ try { +++ List batchAckResultList = new ArrayList<>(handleMessageList.size()); +++ Map> brokerHandleListMap = new HashMap<>(); +++ +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ if (handleMessage.getReceiptHandle().isExpired()) { +++ batchAckResultList.add(new BatchAckResult(handleMessage, EXPIRED_HANDLE_PROXY_EXCEPTION)); +++ continue; +++ } +++ List brokerHandleList = brokerHandleListMap.computeIfAbsent(handleMessage.getReceiptHandle().getBrokerName(), key -> new ArrayList<>()); +++ brokerHandleList.add(handleMessage); +++ } +++ +++ if (brokerHandleListMap.isEmpty()) { +++ return FutureUtils.addExecutor(CompletableFuture.completedFuture(batchAckResultList), this.executor); +++ } +++ Set>> brokerHandleListMapEntrySet = brokerHandleListMap.entrySet(); +++ CompletableFuture>[] futures = new CompletableFuture[brokerHandleListMapEntrySet.size()]; +++ int futureIndex = 0; +++ for (Map.Entry> entry : brokerHandleListMapEntrySet) { +++ futures[futureIndex++] = processBrokerHandle(ctx, consumerGroup, topic, entry.getValue(), timeoutMillis); +++ } +++ CompletableFuture.allOf(futures).whenComplete((val, throwable) -> { +++ if (throwable != null) { +++ future.completeExceptionally(throwable); +++ } +++ for (CompletableFuture> resultFuture : futures) { +++ batchAckResultList.addAll(resultFuture.join()); +++ } +++ future.complete(batchAckResultList); +++ }); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return FutureUtils.addExecutor(future, this.executor); +++ } +++ +++ protected CompletableFuture> processBrokerHandle(ProxyContext ctx, String consumerGroup, String topic, List handleMessageList, long timeoutMillis) { +++ return this.serviceManager.getMessageService().batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis) +++ .thenApply(result -> { +++ List results = new ArrayList<>(); +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ results.add(new BatchAckResult(handleMessage, result)); +++ } +++ return results; +++ }) +++ .exceptionally(throwable -> { +++ List results = new ArrayList<>(); +++ for (ReceiptHandleMessage handleMessage : handleMessageList) { +++ results.add(new BatchAckResult(handleMessage, new ProxyException(ProxyExceptionCode.INTERNAL_SERVER_ERROR, throwable.getMessage(), throwable))); +++ } +++ return results; +++ }); +++ } +++ ++ public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, ++ String messageId, String groupName, String topicName, long invisibleTime, long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++index 188cb7b9b..ba150051b 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/DefaultMessagingProcessor.java ++@@ -46,6 +46,7 @@ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.service.ServiceManager; ++ import org.apache.rocketmq.proxy.service.ServiceManagerFactory; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; ++@@ -183,6 +184,12 @@ public class DefaultMessagingProcessor extends AbstractStartAndShutdown implemen ++ return this.consumerProcessor.ackMessage(ctx, handle, messageId, consumerGroup, topic, timeoutMillis); ++ } ++ +++ @Override +++ public CompletableFuture> batchAckMessage(ProxyContext ctx, +++ List handleMessageList, String consumerGroup, String topic, long timeoutMillis) { +++ return this.consumerProcessor.batchAckMessage(ctx, handleMessageList, consumerGroup, topic, timeoutMillis); +++ } +++ ++ @Override ++ public CompletableFuture changeInvisibleTime(ProxyContext ctx, ReceiptHandle handle, String messageId, ++ String groupName, String topicName, long invisibleTime, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++index d86be0bd8..2ae7418ba 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/MessagingProcessor.java ++@@ -37,6 +37,7 @@ import org.apache.rocketmq.proxy.common.Address; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ProxyTopicRouteData; ++@@ -155,6 +156,23 @@ public interface MessagingProcessor extends StartAndShutdown { ++ long timeoutMillis ++ ); ++ +++ default CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic +++ ) { +++ return batchAckMessage(ctx, handleMessageList, consumerGroup, topic, DEFAULT_TIMEOUT_MILLS); +++ } +++ +++ CompletableFuture> batchAckMessage( +++ ProxyContext ctx, +++ List handleMessageList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ); +++ ++ default CompletableFuture changeInvisibleTime( ++ ProxyContext ctx, ++ ReceiptHandle handle, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++index 9f163f1b9..70b72deae 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ClusterMessageService.java ++@@ -20,9 +20,11 @@ import com.google.common.collect.Lists; ++ import java.util.List; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; +++import java.util.stream.Collectors; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.PopResult; ++ import org.apache.rocketmq.client.consumer.PullResult; +++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.common.consumer.ReceiptHandle; ++ import org.apache.rocketmq.common.message.Message; ++@@ -31,7 +33,6 @@ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.TopicRouteService; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -137,6 +138,19 @@ public class ClusterMessageService implements MessageService { ++ ); ++ } ++ +++ @Override +++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, String consumerGroup, +++ String topic, long timeoutMillis) { +++ List extraInfoList = handleList.stream().map(message -> message.getReceiptHandle().getReceiptHandle()).collect(Collectors.toList()); +++ return this.mqClientAPIFactory.getClient().batchAckMessageAsync( +++ this.resolveBrokerAddrInReceiptHandle(ctx, handleList.get(0).getReceiptHandle()), +++ topic, +++ consumerGroup, +++ extraInfoList, +++ timeoutMillis +++ ); +++ } +++ ++ @Override ++ public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, ++ PullMessageRequestHeader requestHeader, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++index eb2c4d9ee..ca7dcc9eb 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.service.message; ++ import io.netty.channel.ChannelHandlerContext; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; +++import java.util.BitSet; ++ import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.List; ++@@ -54,6 +55,8 @@ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; +++import org.apache.rocketmq.remoting.protocol.body.BatchAck; +++import org.apache.rocketmq.remoting.protocol.body.BatchAckMessageRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.LockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.body.UnlockBatchRequestBody; ++ import org.apache.rocketmq.remoting.protocol.header.AckMessageRequestHeader; ++@@ -364,6 +367,61 @@ public class LocalMessageService implements MessageService { ++ }); ++ } ++ +++ @Override +++ public CompletableFuture batchAckMessage(ProxyContext ctx, List handleList, +++ String consumerGroup, String topic, long timeoutMillis) { +++ SimpleChannel channel = channelManager.createChannel(ctx); +++ ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); +++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.BATCH_ACK_MESSAGE, null); +++ +++ Map batchAckMap = new HashMap<>(); +++ for (ReceiptHandleMessage receiptHandleMessage : handleList) { +++ String extraInfo = receiptHandleMessage.getReceiptHandle().getReceiptHandle(); +++ String[] extraInfoData = ExtraInfoUtil.split(extraInfo); +++ String mergeKey = ExtraInfoUtil.getRetry(extraInfoData) + "@" + +++ ExtraInfoUtil.getQueueId(extraInfoData) + "@" + +++ ExtraInfoUtil.getCkQueueOffset(extraInfoData) + "@" + +++ ExtraInfoUtil.getPopTime(extraInfoData); +++ BatchAck bAck = batchAckMap.computeIfAbsent(mergeKey, k -> { +++ BatchAck newBatchAck = new BatchAck(); +++ newBatchAck.setConsumerGroup(consumerGroup); +++ newBatchAck.setTopic(topic); +++ newBatchAck.setRetry(ExtraInfoUtil.getRetry(extraInfoData)); +++ newBatchAck.setStartOffset(ExtraInfoUtil.getCkQueueOffset(extraInfoData)); +++ newBatchAck.setQueueId(ExtraInfoUtil.getQueueId(extraInfoData)); +++ newBatchAck.setReviveQueueId(ExtraInfoUtil.getReviveQid(extraInfoData)); +++ newBatchAck.setPopTime(ExtraInfoUtil.getPopTime(extraInfoData)); +++ newBatchAck.setInvisibleTime(ExtraInfoUtil.getInvisibleTime(extraInfoData)); +++ newBatchAck.setBitSet(new BitSet()); +++ return newBatchAck; +++ }); +++ bAck.getBitSet().set((int) (ExtraInfoUtil.getQueueOffset(extraInfoData) - ExtraInfoUtil.getCkQueueOffset(extraInfoData))); +++ } +++ BatchAckMessageRequestBody requestBody = new BatchAckMessageRequestBody(); +++ requestBody.setBrokerName(brokerController.getBrokerConfig().getBrokerName()); +++ requestBody.setAcks(new ArrayList<>(batchAckMap.values())); +++ +++ command.setBody(requestBody.encode()); +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ RemotingCommand response = brokerController.getAckMessageProcessor() +++ .processRequest(channelHandlerContext, command); +++ future.complete(response); +++ } catch (Exception e) { +++ log.error("Fail to process batchAckMessage command", e); +++ future.completeExceptionally(e); +++ } +++ return future.thenApply(r -> { +++ AckResult ackResult = new AckResult(); +++ if (ResponseCode.SUCCESS == r.getCode()) { +++ ackResult.setStatus(AckStatus.OK); +++ } else { +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ } +++ return ackResult; +++ }); +++ } +++ ++ @Override ++ public CompletableFuture pullMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, ++ PullMessageRequestHeader requestHeader, long timeoutMillis) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++index 15da17154..58a835adb 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/MessageService.java ++@@ -91,6 +91,14 @@ public interface MessageService { ++ long timeoutMillis ++ ); ++ +++ CompletableFuture batchAckMessage( +++ ProxyContext ctx, +++ List handleList, +++ String consumerGroup, +++ String topic, +++ long timeoutMillis +++ ); +++ ++ CompletableFuture pullMessage( ++ ProxyContext ctx, ++ AddressableMessageQueue messageQueue, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++new file mode 100644 ++index 000000000..ae63fed49 ++--- /dev/null +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/ReceiptHandleMessage.java ++@@ -0,0 +1,39 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.proxy.service.message; +++ +++import org.apache.rocketmq.common.consumer.ReceiptHandle; +++ +++public class ReceiptHandleMessage { +++ +++ private final ReceiptHandle receiptHandle; +++ private final String messageId; +++ +++ public ReceiptHandleMessage(ReceiptHandle receiptHandle, String messageId) { +++ this.receiptHandle = receiptHandle; +++ this.messageId = messageId; +++ } +++ +++ public ReceiptHandle getReceiptHandle() { +++ return receiptHandle; +++ } +++ +++ public String getMessageId() { +++ return messageId; +++ } +++} ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++index 49fdfc6a8..3c4746105 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/AckMessageActivityTest.java ++@@ -20,21 +20,32 @@ package org.apache.rocketmq.proxy.grpc.v2.consumer; ++ import apache.rocketmq.v2.AckMessageEntry; ++ import apache.rocketmq.v2.AckMessageRequest; ++ import apache.rocketmq.v2.AckMessageResponse; +++import apache.rocketmq.v2.AckMessageResultEntry; ++ import apache.rocketmq.v2.Code; ++ import apache.rocketmq.v2.Resource; +++import java.util.ArrayList; +++import java.util.HashMap; +++import java.util.List; +++import java.util.Map; ++ import java.util.concurrent.CompletableFuture; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.AckStatus; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +++import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; +++import org.apache.rocketmq.proxy.processor.BatchAckResult; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.junit.Before; ++ import org.junit.Test; +++import org.mockito.stubbing.Answer; ++ ++ import static org.junit.Assert.assertEquals; ++ import static org.mockito.ArgumentMatchers.any; +++import static org.mockito.ArgumentMatchers.anyList; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; +++import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.when; ++ ++ public class AckMessageActivityTest extends BaseActivityTest { ++@@ -52,43 +63,197 @@ public class AckMessageActivityTest extends BaseActivityTest { ++ ++ @Test ++ public void testAckMessage() throws Throwable { ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg1"), anyString(), anyString())) +++ ConfigurationManager.getProxyConfig().setEnableBatchAck(false); +++ +++ String msg1 = "msg1"; +++ String msg2 = "msg2"; +++ String msg3 = "msg3"; +++ +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg1), anyString(), anyString())) ++ .thenThrow(new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "receipt handle is expired")); ++ ++ AckResult msg2AckResult = new AckResult(); ++ msg2AckResult.setStatus(AckStatus.OK); ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg2"), anyString(), anyString())) +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg2), anyString(), anyString())) ++ .thenReturn(CompletableFuture.completedFuture(msg2AckResult)); ++ ++ AckResult msg3AckResult = new AckResult(); ++ msg3AckResult.setStatus(AckStatus.NO_EXIST); ++- when(this.messagingProcessor.ackMessage(any(), any(), eq("msg3"), anyString(), anyString())) +++ when(this.messagingProcessor.ackMessage(any(), any(), eq(msg3), anyString(), anyString())) ++ .thenReturn(CompletableFuture.completedFuture(msg3AckResult)); ++ ++- AckMessageResponse response = this.ackMessageActivity.ackMessage( ++- createContext(), ++- AckMessageRequest.newBuilder() ++- .setTopic(Resource.newBuilder().setName(TOPIC).build()) ++- .setGroup(Resource.newBuilder().setName(GROUP).build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg1") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) ++- .build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg2") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++- .build()) ++- .addEntries(AckMessageEntry.newBuilder() ++- .setMessageId("msg3") ++- .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) ++- .build()) ++- .build() ++- ).get(); ++- ++- assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); ++- assertEquals(3, response.getEntriesCount()); ++- assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); ++- assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); ++- assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg1) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg2) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.OK, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg3) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg1) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis() - 10000, 1000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg2) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(msg3) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ +++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +++ assertEquals(3, response.getEntriesCount()); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getEntries(0).getStatus().getCode()); +++ assertEquals(Code.OK, response.getEntries(1).getStatus().getCode()); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getEntries(2).getStatus().getCode()); +++ } +++ } +++ +++ @Test +++ public void testAckMessageInBatch() throws Throwable { +++ ConfigurationManager.getProxyConfig().setEnableBatchAck(true); +++ +++ String successMessageId = "msg1"; +++ String notOkMessageId = "msg2"; +++ String exceptionMessageId = "msg3"; +++ +++ doAnswer((Answer>>) invocation -> { +++ List receiptHandleMessageList = invocation.getArgument(1, List.class); +++ List batchAckResultList = new ArrayList<>(); +++ for (ReceiptHandleMessage receiptHandleMessage : receiptHandleMessageList) { +++ BatchAckResult batchAckResult; +++ if (receiptHandleMessage.getMessageId().equals(successMessageId)) { +++ AckResult ackResult = new AckResult(); +++ ackResult.setStatus(AckStatus.OK); +++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); +++ } else if (receiptHandleMessage.getMessageId().equals(notOkMessageId)) { +++ AckResult ackResult = new AckResult(); +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ batchAckResult = new BatchAckResult(receiptHandleMessage, ackResult); +++ } else { +++ batchAckResult = new BatchAckResult(receiptHandleMessage, new ProxyException(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, "")); +++ } +++ batchAckResultList.add(batchAckResult); +++ } +++ return CompletableFuture.completedFuture(batchAckResultList); +++ }).when(this.messagingProcessor).batchAckMessage(any(), anyList(), anyString(), anyString()); +++ +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(successMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.OK, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(notOkMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(exceptionMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, response.getStatus().getCode()); +++ } +++ { +++ AckMessageResponse response = this.ackMessageActivity.ackMessage( +++ createContext(), +++ AckMessageRequest.newBuilder() +++ .setTopic(Resource.newBuilder().setName(TOPIC).build()) +++ .setGroup(Resource.newBuilder().setName(GROUP).build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(successMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(notOkMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .addEntries(AckMessageEntry.newBuilder() +++ .setMessageId(exceptionMessageId) +++ .setReceiptHandle(buildReceiptHandle(TOPIC, System.currentTimeMillis(), 3000)) +++ .build()) +++ .build() +++ ).get(); +++ +++ assertEquals(Code.MULTIPLE_RESULTS, response.getStatus().getCode()); +++ assertEquals(3, response.getEntriesCount()); +++ Map msgCode = new HashMap<>(); +++ for (AckMessageResultEntry entry : response.getEntriesList()) { +++ msgCode.put(entry.getMessageId(), entry.getStatus().getCode()); +++ } +++ assertEquals(Code.OK, msgCode.get(successMessageId)); +++ assertEquals(Code.INTERNAL_SERVER_ERROR, msgCode.get(notOkMessageId)); +++ assertEquals(Code.INVALID_RECEIPT_HANDLE, msgCode.get(exceptionMessageId)); +++ } ++ } ++ } ++\ No newline at end of file ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++index 5c1ea9627..072630e39 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/BaseProcessorTest.java ++@@ -66,14 +66,6 @@ public class BaseProcessorTest extends InitConfigTest { ++ protected ProxyRelayService proxyRelayService; ++ @Mock ++ protected MetadataService metadataService; ++- @Mock ++- protected ProducerProcessor producerProcessor; ++- @Mock ++- protected ConsumerProcessor consumerProcessor; ++- @Mock ++- protected TransactionProcessor transactionProcessor; ++- @Mock ++- protected ClientProcessor clientProcessor; ++ ++ public void before() throws Throwable { ++ super.before(); ++@@ -92,6 +84,13 @@ public class BaseProcessorTest extends InitConfigTest { ++ } ++ ++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime) { +++ return createMessageExt(topic, tags, reconsumeTimes, invisibleTime, System.currentTimeMillis(), +++ RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE), +++ RANDOM.nextInt(Integer.MAX_VALUE), "mockBroker"); +++ } +++ +++ protected static MessageExt createMessageExt(String topic, String tags, int reconsumeTimes, long invisibleTime, long popTime, +++ long startOffset, int reviveQid, int queueId, long queueOffset, String brokerName) { ++ MessageExt messageExt = new MessageExt(); ++ messageExt.setTopic(topic); ++ messageExt.setTags(tags); ++@@ -100,8 +99,7 @@ public class BaseProcessorTest extends InitConfigTest { ++ messageExt.setMsgId(MessageClientIDSetter.createUniqID()); ++ messageExt.setCommitLogOffset(RANDOM.nextInt(Integer.MAX_VALUE)); ++ MessageAccessor.putProperty(messageExt, MessageConst.PROPERTY_POP_CK, ++- ExtraInfoUtil.buildExtraInfo(RANDOM.nextInt(Integer.MAX_VALUE), System.currentTimeMillis(), invisibleTime, ++- RANDOM.nextInt(Integer.MAX_VALUE), topic, "mockBroker", RANDOM.nextInt(Integer.MAX_VALUE), RANDOM.nextInt(Integer.MAX_VALUE))); +++ ExtraInfoUtil.buildExtraInfo(startOffset, popTime, invisibleTime, reviveQid, topic, brokerName, queueId, queueOffset)); ++ return messageExt; ++ } ++ ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++index 717e86fc0..db268a06e 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/processor/ConsumerProcessorTest.java ++@@ -20,8 +20,11 @@ package org.apache.rocketmq.proxy.processor; ++ import com.google.common.collect.Sets; ++ import java.time.Duration; ++ import java.util.ArrayList; +++import java.util.Collections; +++import java.util.HashMap; ++ import java.util.HashSet; ++ import java.util.List; +++import java.util.Map; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.Executors; ++@@ -39,7 +42,10 @@ import org.apache.rocketmq.common.message.MessageClientIDSetter; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.proxy.common.ProxyContext; +++import org.apache.rocketmq.proxy.common.ProxyExceptionCode; +++import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.common.utils.ProxyUtils; +++import org.apache.rocketmq.proxy.service.message.ReceiptHandleMessage; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.MessageQueueView; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -50,16 +56,22 @@ import org.apache.rocketmq.remoting.protocol.header.PopMessageRequestHeader; ++ import org.junit.Before; ++ import org.junit.Test; ++ import org.mockito.ArgumentCaptor; +++import org.mockito.stubbing.Answer; ++ ++ import static org.assertj.core.api.Assertions.assertThat; ++ import static org.junit.Assert.assertEquals; ++ import static org.junit.Assert.assertNotNull; +++import static org.junit.Assert.assertNull; ++ import static org.junit.Assert.assertSame; ++ import static org.mockito.ArgumentMatchers.any; +++import static org.mockito.ArgumentMatchers.anyList; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; +++import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.mock; +++import static org.mockito.Mockito.never; +++import static org.mockito.Mockito.verify; ++ import static org.mockito.Mockito.when; ++ ++ public class ConsumerProcessorTest extends BaseProcessorTest { ++@@ -162,6 +174,109 @@ public class ConsumerProcessorTest extends BaseProcessorTest { ++ assertEquals(handle.getReceiptHandle(), requestHeaderArgumentCaptor.getValue().getExtraInfo()); ++ } ++ +++ @Test +++ public void testBatchAckExpireMessage() throws Throwable { +++ String brokerName1 = "brokerName1"; +++ +++ List receiptHandleMessageList = new ArrayList<>(); +++ for (int i = 0; i < 3; i++) { +++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, +++ 0, 0, 0, i, brokerName1); +++ ReceiptHandle expireHandle = create(expireMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); +++ } +++ +++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); +++ +++ verify(this.messageService, never()).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); +++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ assertNull(batchAckResult.getAckResult()); +++ assertNotNull(batchAckResult.getProxyException()); +++ assertNotNull(batchAckResult.getReceiptHandleMessage()); +++ } +++ +++ } +++ +++ @Test +++ public void testBatchAckMessage() throws Throwable { +++ String brokerName1 = "brokerName1"; +++ String brokerName2 = "brokerName2"; +++ String errThrowBrokerName = "errThrowBrokerName"; +++ MessageExt expireMessage = createMessageExt(TOPIC, "", 0, 3000, System.currentTimeMillis() - 10000, +++ 0, 0, 0, 0, brokerName1); +++ ReceiptHandle expireHandle = create(expireMessage); +++ +++ List receiptHandleMessageList = new ArrayList<>(); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(expireHandle, expireMessage.getMsgId())); +++ List broker1Msg = new ArrayList<>(); +++ List broker2Msg = new ArrayList<>(); +++ +++ long now = System.currentTimeMillis(); +++ int msgNum = 3; +++ for (int i = 0; i < msgNum; i++) { +++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, i + 1, brokerName1); +++ ReceiptHandle brokerHandle = create(brokerMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); +++ broker1Msg.add(brokerMessage.getMsgId()); +++ } +++ for (int i = 0; i < msgNum; i++) { +++ MessageExt brokerMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, i + 1, brokerName2); +++ ReceiptHandle brokerHandle = create(brokerMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(brokerHandle, brokerMessage.getMsgId())); +++ broker2Msg.add(brokerMessage.getMsgId()); +++ } +++ +++ // for this message, will throw exception in batchAckMessage +++ MessageExt errThrowMessage = createMessageExt(TOPIC, "", 0, 3000, now, +++ 0, 0, 0, 0, errThrowBrokerName); +++ ReceiptHandle errThrowHandle = create(errThrowMessage); +++ receiptHandleMessageList.add(new ReceiptHandleMessage(errThrowHandle, errThrowMessage.getMsgId())); +++ +++ Collections.shuffle(receiptHandleMessageList); +++ +++ doAnswer((Answer>) invocation -> { +++ List handleMessageList = invocation.getArgument(1, List.class); +++ AckResult ackResult = new AckResult(); +++ String brokerName = handleMessageList.get(0).getReceiptHandle().getBrokerName(); +++ if (brokerName.equals(brokerName1)) { +++ ackResult.setStatus(AckStatus.OK); +++ } else if (brokerName.equals(brokerName2)) { +++ ackResult.setStatus(AckStatus.NO_EXIST); +++ } else { +++ return FutureUtils.completeExceptionally(new RuntimeException()); +++ } +++ +++ return CompletableFuture.completedFuture(ackResult); +++ }).when(this.messageService).batchAckMessage(any(), anyList(), anyString(), anyString(), anyLong()); +++ +++ List batchAckResultList = this.consumerProcessor.batchAckMessage(createContext(), receiptHandleMessageList, CONSUMER_GROUP, TOPIC, 3000).get(); +++ assertEquals(receiptHandleMessageList.size(), batchAckResultList.size()); +++ +++ // check ackResult for each msg +++ Map msgBatchAckResult = new HashMap<>(); +++ for (BatchAckResult batchAckResult : batchAckResultList) { +++ msgBatchAckResult.put(batchAckResult.getReceiptHandleMessage().getMessageId(), batchAckResult); +++ } +++ for (String msgId : broker1Msg) { +++ assertEquals(AckStatus.OK, msgBatchAckResult.get(msgId).getAckResult().getStatus()); +++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); +++ } +++ for (String msgId : broker2Msg) { +++ assertEquals(AckStatus.NO_EXIST, msgBatchAckResult.get(msgId).getAckResult().getStatus()); +++ assertNull(msgBatchAckResult.get(msgId).getProxyException()); +++ } +++ assertNotNull(msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException()); +++ assertEquals(ProxyExceptionCode.INVALID_RECEIPT_HANDLE, msgBatchAckResult.get(expireMessage.getMsgId()).getProxyException().getCode()); +++ assertNull(msgBatchAckResult.get(expireMessage.getMsgId()).getAckResult()); +++ +++ assertNotNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException()); +++ assertEquals(ProxyExceptionCode.INTERNAL_SERVER_ERROR, msgBatchAckResult.get(errThrowMessage.getMsgId()).getProxyException().getCode()); +++ assertNull(msgBatchAckResult.get(errThrowMessage.getMsgId()).getAckResult()); +++ } +++ ++ @Test ++ public void testChangeInvisibleTime() throws Throwable { ++ ReceiptHandle handle = create(createMessageExt(MixAll.RETRY_GROUP_TOPIC_PREFIX + TOPIC, "", 0, 3000)); ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++index 77a119a29..3f3a4ae40 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++@@ -220,6 +220,18 @@ public class MQClientAPIExtTest { ++ assertSame(ackResult, mqClientAPI.ackMessageAsync(BROKER_ADDR, new AckMessageRequestHeader(), TIMEOUT).get()); ++ } ++ +++ @Test +++ public void testBatchAckMessageAsync() throws Exception { +++ AckResult ackResult = new AckResult(); +++ doAnswer((Answer) mock -> { +++ AckCallback ackCallback = mock.getArgument(2); +++ ackCallback.onSuccess(ackResult); +++ return null; +++ }).when(mqClientAPI).batchAckMessageAsync(anyString(), anyLong(), any(AckCallback.class), any()); +++ +++ assertSame(ackResult, mqClientAPI.batchAckMessageAsync(BROKER_ADDR, TOPIC, CONSUMER_GROUP, new ArrayList<>(), TIMEOUT).get()); +++ } +++ ++ @Test ++ public void testChangeInvisibleTimeAsync() throws Exception { ++ AckResult ackResult = new AckResult(); ++-- ++2.32.0.windows.2 ++ +diff --git a/patch013-backport-enhance-admin-output.patch b/patch013-backport-enhance-admin-output.patch +new file mode 100644 +index 000000000..3fa60916f +--- /dev/null ++++ b/patch013-backport-enhance-admin-output.patch +@@ -0,0 +1,892 @@ ++From 7e018520ef707a841c66c55d621f6560d03b631b Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Fri, 25 Aug 2023 09:49:22 +0800 ++Subject: [PATCH 1/6] Add expireAfterAccess for cache (#7247) ++ ++Add expireAfterAccess for cache ++--- ++ .../rocketmq/proxy/config/ProxyConfig.java | 59 ++++++++++++++----- ++ .../metadata/ClusterMetadataService.java | 6 +- ++ .../service/route/TopicRouteService.java | 14 +++-- ++ 3 files changed, 56 insertions(+), 23 deletions(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 76a243919..2994893d7 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -155,14 +155,17 @@ public class ProxyConfig implements ConfigFile { ++ private int consumerProcessorThreadPoolQueueCapacity = 10000; ++ ++ private boolean useEndpointPortFromRequest = false; ++- private int topicRouteServiceCacheExpiredInSeconds = 20; +++ +++ private int topicRouteServiceCacheExpiredSeconds = 300; +++ private int topicRouteServiceCacheRefreshSeconds = 20; ++ private int topicRouteServiceCacheMaxNum = 20000; ++ private int topicRouteServiceThreadPoolNums = PROCESSOR_NUMBER; ++ private int topicRouteServiceThreadPoolQueueCapacity = 5000; ++- ++- private int topicConfigCacheExpiredInSeconds = 20; +++ private int topicConfigCacheExpiredSeconds = 300; +++ private int topicConfigCacheRefreshSeconds = 20; ++ private int topicConfigCacheMaxNum = 20000; ++- private int subscriptionGroupConfigCacheExpiredInSeconds = 20; +++ private int subscriptionGroupConfigCacheExpiredSeconds = 300; +++ private int subscriptionGroupConfigCacheRefreshSeconds = 20; ++ private int subscriptionGroupConfigCacheMaxNum = 20000; ++ private int metadataThreadPoolNums = 3; ++ private int metadataThreadPoolQueueCapacity = 100000; ++@@ -794,12 +797,20 @@ public class ProxyConfig implements ConfigFile { ++ this.consumerProcessorThreadPoolQueueCapacity = consumerProcessorThreadPoolQueueCapacity; ++ } ++ ++- public int getTopicRouteServiceCacheExpiredInSeconds() { ++- return topicRouteServiceCacheExpiredInSeconds; +++ public int getTopicRouteServiceCacheExpiredSeconds() { +++ return topicRouteServiceCacheExpiredSeconds; +++ } +++ +++ public void setTopicRouteServiceCacheExpiredSeconds(int topicRouteServiceCacheExpiredSeconds) { +++ this.topicRouteServiceCacheExpiredSeconds = topicRouteServiceCacheExpiredSeconds; ++ } ++ ++- public void setTopicRouteServiceCacheExpiredInSeconds(int topicRouteServiceCacheExpiredInSeconds) { ++- this.topicRouteServiceCacheExpiredInSeconds = topicRouteServiceCacheExpiredInSeconds; +++ public int getTopicRouteServiceCacheRefreshSeconds() { +++ return topicRouteServiceCacheRefreshSeconds; +++ } +++ +++ public void setTopicRouteServiceCacheRefreshSeconds(int topicRouteServiceCacheRefreshSeconds) { +++ this.topicRouteServiceCacheRefreshSeconds = topicRouteServiceCacheRefreshSeconds; ++ } ++ ++ public int getTopicRouteServiceCacheMaxNum() { ++@@ -826,12 +837,20 @@ public class ProxyConfig implements ConfigFile { ++ this.topicRouteServiceThreadPoolQueueCapacity = topicRouteServiceThreadPoolQueueCapacity; ++ } ++ ++- public int getTopicConfigCacheExpiredInSeconds() { ++- return topicConfigCacheExpiredInSeconds; +++ public int getTopicConfigCacheRefreshSeconds() { +++ return topicConfigCacheRefreshSeconds; +++ } +++ +++ public void setTopicConfigCacheRefreshSeconds(int topicConfigCacheRefreshSeconds) { +++ this.topicConfigCacheRefreshSeconds = topicConfigCacheRefreshSeconds; +++ } +++ +++ public int getTopicConfigCacheExpiredSeconds() { +++ return topicConfigCacheExpiredSeconds; ++ } ++ ++- public void setTopicConfigCacheExpiredInSeconds(int topicConfigCacheExpiredInSeconds) { ++- this.topicConfigCacheExpiredInSeconds = topicConfigCacheExpiredInSeconds; +++ public void setTopicConfigCacheExpiredSeconds(int topicConfigCacheExpiredSeconds) { +++ this.topicConfigCacheExpiredSeconds = topicConfigCacheExpiredSeconds; ++ } ++ ++ public int getTopicConfigCacheMaxNum() { ++@@ -842,12 +861,20 @@ public class ProxyConfig implements ConfigFile { ++ this.topicConfigCacheMaxNum = topicConfigCacheMaxNum; ++ } ++ ++- public int getSubscriptionGroupConfigCacheExpiredInSeconds() { ++- return subscriptionGroupConfigCacheExpiredInSeconds; +++ public int getSubscriptionGroupConfigCacheRefreshSeconds() { +++ return subscriptionGroupConfigCacheRefreshSeconds; +++ } +++ +++ public void setSubscriptionGroupConfigCacheRefreshSeconds(int subscriptionGroupConfigCacheRefreshSeconds) { +++ this.subscriptionGroupConfigCacheRefreshSeconds = subscriptionGroupConfigCacheRefreshSeconds; +++ } +++ +++ public int getSubscriptionGroupConfigCacheExpiredSeconds() { +++ return subscriptionGroupConfigCacheExpiredSeconds; ++ } ++ ++- public void setSubscriptionGroupConfigCacheExpiredInSeconds(int subscriptionGroupConfigCacheExpiredInSeconds) { ++- this.subscriptionGroupConfigCacheExpiredInSeconds = subscriptionGroupConfigCacheExpiredInSeconds; +++ public void setSubscriptionGroupConfigCacheExpiredSeconds(int subscriptionGroupConfigCacheExpiredSeconds) { +++ this.subscriptionGroupConfigCacheExpiredSeconds = subscriptionGroupConfigCacheExpiredSeconds; ++ } ++ ++ public int getSubscriptionGroupConfigCacheMaxNum() { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java ++index bc9582ad8..d34a0efd9 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/metadata/ClusterMetadataService.java ++@@ -69,11 +69,13 @@ public class ClusterMetadataService extends AbstractStartAndShutdown implements ++ ); ++ this.topicConfigCache = CacheBuilder.newBuilder() ++ .maximumSize(config.getTopicConfigCacheMaxNum()) ++- .refreshAfterWrite(config.getTopicConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) +++ .expireAfterAccess(config.getTopicConfigCacheExpiredSeconds(), TimeUnit.SECONDS) +++ .refreshAfterWrite(config.getTopicConfigCacheRefreshSeconds(), TimeUnit.SECONDS) ++ .build(new ClusterTopicConfigCacheLoader()); ++ this.subscriptionGroupConfigCache = CacheBuilder.newBuilder() ++ .maximumSize(config.getSubscriptionGroupConfigCacheMaxNum()) ++- .refreshAfterWrite(config.getSubscriptionGroupConfigCacheExpiredInSeconds(), TimeUnit.SECONDS) +++ .expireAfterAccess(config.getSubscriptionGroupConfigCacheExpiredSeconds(), TimeUnit.SECONDS) +++ .refreshAfterWrite(config.getSubscriptionGroupConfigCacheRefreshSeconds(), TimeUnit.SECONDS) ++ .build(new ClusterSubscriptionGroupConfigCacheLoader()); ++ ++ this.init(); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++index e012a5465..84348adc3 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++@@ -68,10 +68,13 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ ); ++ this.mqClientAPIFactory = mqClientAPIFactory; ++ ++- this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()). ++- refreshAfterWrite(config.getTopicRouteServiceCacheExpiredInSeconds(), TimeUnit.SECONDS). ++- executor(cacheRefreshExecutor).build(new CacheLoader() { ++- @Override public @Nullable MessageQueueView load(String topic) throws Exception { +++ this.topicCache = Caffeine.newBuilder().maximumSize(config.getTopicRouteServiceCacheMaxNum()) +++ .expireAfterAccess(config.getTopicRouteServiceCacheExpiredSeconds(), TimeUnit.SECONDS) +++ .refreshAfterWrite(config.getTopicRouteServiceCacheRefreshSeconds(), TimeUnit.SECONDS) +++ .executor(cacheRefreshExecutor) +++ .build(new CacheLoader() { +++ @Override +++ public @Nullable MessageQueueView load(String topic) throws Exception { ++ try { ++ TopicRouteData topicRouteData = mqClientAPIFactory.getClient().getTopicRouteInfoFromNameServer(topic, Duration.ofSeconds(3).toMillis()); ++ return buildMessageQueueView(topic, topicRouteData); ++@@ -83,7 +86,8 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ } ++ } ++ ++- @Override public @Nullable MessageQueueView reload(@NonNull String key, +++ @Override +++ public @Nullable MessageQueueView reload(@NonNull String key, ++ @NonNull MessageQueueView oldValue) throws Exception { ++ try { ++ return load(key); ++-- ++2.32.0.windows.2 ++ ++ ++From 5f6dc90f9dab35809fcb0407d4d5cc2737d2335e Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Fri, 25 Aug 2023 11:17:23 +0800 ++Subject: [PATCH 2/6] [ISSUE #7250] Beautify command rocksDBConfigToJson output ++ ++Co-authored-by: Ziy1-Tan ++--- ++ .../metadata/RocksDBConfigToJsonCommand.java | 32 +++++++++++-------- ++ 1 file changed, 18 insertions(+), 14 deletions(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++index 3053f4684..3fc63e4dd 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++@@ -21,13 +21,13 @@ import org.apache.commons.cli.CommandLine; ++ import org.apache.commons.cli.Option; ++ import org.apache.commons.cli.Options; ++ import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.common.UtilAll; ++ import org.apache.rocketmq.common.config.RocksDBConfigManager; ++ import org.apache.rocketmq.common.utils.DataConverter; ++ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.tools.command.SubCommand; ++ import org.apache.rocketmq.tools.command.SubCommandException; ++ ++-import java.io.File; ++ import java.util.HashMap; ++ import java.util.Map; ++ ++@@ -48,7 +48,7 @@ public class RocksDBConfigToJsonCommand implements SubCommand { ++ @Override ++ public Options buildCommandlineOptions(Options options) { ++ Option pathOption = new Option("p", "path", true, ++- "Absolute path to the metadata directory"); +++ "Absolute path for the metadata directory"); ++ pathOption.setRequired(true); ++ options.addOption(pathOption); ++ ++@@ -63,15 +63,14 @@ public class RocksDBConfigToJsonCommand implements SubCommand { ++ @Override ++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++ String path = commandLine.getOptionValue("path").trim(); ++- if (StringUtils.isEmpty(path) || !new File(path).exists()) { +++ if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { ++ System.out.print("Rocksdb path is invalid.\n"); ++ return; ++ } ++ ++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++ ++- final long memTableFlushInterval = 60 * 60 * 1000L; ++- RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); +++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(60 * 60 * 1000L); ++ try { ++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for topics.json ++@@ -84,13 +83,16 @@ public class RocksDBConfigToJsonCommand implements SubCommand { ++ topicConfigTable.put(topic, jsonObject); ++ }); ++ ++- if (isLoad) { ++- topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++- final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++- System.out.print(topicsJsonStr + "\n"); +++ if (!isLoad) { +++ System.out.print("RocksDB load error, path=" + path); ++ return; ++ } +++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); +++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); +++ System.out.print(topicsJsonStr + "\n"); +++ return; ++ } +++ ++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for subscriptionGroup.json ++ final Map subscriptionGroupJsonConfig = new HashMap<>(); ++@@ -102,13 +104,15 @@ public class RocksDBConfigToJsonCommand implements SubCommand { ++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); ++ }); ++ ++- if (isLoad) { ++- subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++- (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++- final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++- System.out.print(subscriptionGroupJsonStr + "\n"); +++ if (!isLoad) { +++ System.out.print("RocksDB load error, path=" + path); ++ return; ++ } +++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", +++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); +++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); +++ System.out.print(subscriptionGroupJsonStr + "\n"); +++ return; ++ } ++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); ++ } finally { ++-- ++2.32.0.windows.2 ++ ++ ++From b4f73e2aabc1b141cec98431899e4090340adf0f Mon Sep 17 00:00:00 2001 ++From: mxsm ++Date: Sun, 27 Aug 2023 20:58:58 +0800 ++Subject: [PATCH 3/6] [ISSUE #7271] Optimize the configuration for setting the ++ quantity of TimerDequeuePutMessageService (#7272) ++ ++--- ++ .../java/org/apache/rocketmq/store/timer/TimerMessageStore.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++index 690f4863e..181f7087a 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++@@ -222,7 +222,7 @@ public class TimerMessageStore { ++ dequeueGetMessageServices[i] = new TimerDequeueGetMessageService(); ++ } ++ ++- int putThreadNum = Math.max(storeConfig.getTimerGetMessageThreadNum(), 1); +++ int putThreadNum = Math.max(storeConfig.getTimerPutMessageThreadNum(), 1); ++ dequeuePutMessageServices = new TimerDequeuePutMessageService[putThreadNum]; ++ for (int i = 0; i < dequeuePutMessageServices.length; i++) { ++ dequeuePutMessageServices[i] = new TimerDequeuePutMessageService(); ++-- ++2.32.0.windows.2 ++ ++ ++From 3e100103af68588528bf32f3752a85e8023f46f8 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Tue, 29 Aug 2023 13:48:51 +0800 ++Subject: [PATCH 4/6] [ISSUE #7277] Enhance rocksDBConfigToJson to support ++ metadata counting (#7276) ++ ++--- ++ .../common/config/AbstractRocksDBStorage.java | 4 +- ++ .../common/config/ConfigRocksDBStorage.java | 6 + ++ .../tools/command/MQAdminStartup.java | 4 +- ++ .../ExportMetadataInRocksDBCommand.java | 138 ++++++++++++++++++ ++ .../metadata/RocksDBConfigToJsonCommand.java | 122 ---------------- ++ ...> ExportMetadataInRocksDBCommandTest.java} | 38 +++-- ++ 6 files changed, 173 insertions(+), 139 deletions(-) ++ create mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java ++ delete mode 100644 tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++ rename tools/src/test/java/org/apache/rocketmq/tools/command/metadata/{KvConfigToJsonCommandTest.java => ExportMetadataInRocksDBCommandTest.java} (62%) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++index e3673baad..a720a5be3 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++@@ -385,8 +385,10 @@ public abstract class AbstractRocksDBStorage { ++ this.options.close(); ++ } ++ //4. close db. ++- if (db != null) { +++ if (db != null && !this.readOnly) { ++ this.db.syncWal(); +++ } +++ if (db != null) { ++ this.db.closeE(); ++ } ++ //5. help gc. ++diff --git a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java ++index 9d05ed282..463bd8fed 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java +++++ b/common/src/main/java/org/apache/rocketmq/common/config/ConfigRocksDBStorage.java ++@@ -60,6 +60,12 @@ public class ConfigRocksDBStorage extends AbstractRocksDBStorage { ++ this.readOnly = false; ++ } ++ +++ public ConfigRocksDBStorage(final String dbPath, boolean readOnly) { +++ super(); +++ this.dbPath = dbPath; +++ this.readOnly = readOnly; +++ } +++ ++ private void initOptions() { ++ this.options = createConfigDBOptions(); ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++index 324aa1856..788fa83c2 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/MQAdminStartup.java ++@@ -80,7 +80,7 @@ import org.apache.rocketmq.tools.command.message.QueryMsgByOffsetSubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgByUniqueKeySubCommand; ++ import org.apache.rocketmq.tools.command.message.QueryMsgTraceByIdSubCommand; ++ import org.apache.rocketmq.tools.command.message.SendMessageCommand; ++-import org.apache.rocketmq.tools.command.metadata.RocksDBConfigToJsonCommand; +++import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand; ++ import org.apache.rocketmq.tools.command.namesrv.AddWritePermSubCommand; ++ import org.apache.rocketmq.tools.command.namesrv.DeleteKvConfigCommand; ++ import org.apache.rocketmq.tools.command.namesrv.GetNamesrvConfigCommand; ++@@ -212,7 +212,6 @@ public class MQAdminStartup { ++ ++ initCommand(new ClusterListSubCommand()); ++ initCommand(new TopicListSubCommand()); ++- initCommand(new RocksDBConfigToJsonCommand()); ++ ++ initCommand(new UpdateKvConfigCommand()); ++ initCommand(new DeleteKvConfigCommand()); ++@@ -257,6 +256,7 @@ public class MQAdminStartup { ++ initCommand(new ExportMetadataCommand()); ++ initCommand(new ExportConfigsCommand()); ++ initCommand(new ExportMetricsCommand()); +++ initCommand(new ExportMetadataInRocksDBCommand()); ++ ++ initCommand(new HAStatusSubCommand()); ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java ++new file mode 100644 ++index 000000000..2a7d3fba4 ++--- /dev/null +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java ++@@ -0,0 +1,138 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tools.command.export; +++ +++import com.alibaba.fastjson.JSONObject; +++import org.apache.commons.cli.CommandLine; +++import org.apache.commons.cli.Option; +++import org.apache.commons.cli.Options; +++import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.common.UtilAll; +++import org.apache.rocketmq.common.config.ConfigRocksDBStorage; +++import org.apache.rocketmq.common.utils.DataConverter; +++import org.apache.rocketmq.remoting.RPCHook; +++import org.apache.rocketmq.tools.command.SubCommand; +++import org.apache.rocketmq.tools.command.SubCommandException; +++import org.rocksdb.RocksIterator; +++ +++import java.util.HashMap; +++import java.util.Map; +++import java.util.concurrent.atomic.AtomicLong; +++import java.util.function.BiConsumer; +++ +++public class ExportMetadataInRocksDBCommand implements SubCommand { +++ private static final String TOPICS_JSON_CONFIG = "topics"; +++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; +++ +++ @Override +++ public String commandName() { +++ return "exportMetadataInRocksDB"; +++ } +++ +++ @Override +++ public String commandDesc() { +++ return "export RocksDB kv config (topics/subscriptionGroups)"; +++ } +++ +++ @Override +++ public Options buildCommandlineOptions(Options options) { +++ Option pathOption = new Option("p", "path", true, +++ "Absolute path for the metadata directory"); +++ pathOption.setRequired(true); +++ options.addOption(pathOption); +++ +++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + +++ "topics/subscriptionGroups"); +++ configTypeOption.setRequired(true); +++ options.addOption(configTypeOption); +++ +++ Option jsonEnableOption = new Option("j", "jsonEnable", true, +++ "Json format enable, Default: false"); +++ jsonEnableOption.setRequired(false); +++ options.addOption(jsonEnableOption); +++ +++ return options; +++ } +++ +++ @Override +++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { +++ String path = commandLine.getOptionValue("path").trim(); +++ if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { +++ System.out.print("RocksDB path is invalid.\n"); +++ return; +++ } +++ +++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); +++ +++ boolean jsonEnable = false; +++ if (commandLine.hasOption("jsonEnable")) { +++ jsonEnable = Boolean.parseBoolean(commandLine.getOptionValue("jsonEnable").trim()); +++ } +++ +++ +++ ConfigRocksDBStorage kvStore = new ConfigRocksDBStorage(path, true /* readOnly */); +++ if (!kvStore.start()) { +++ System.out.print("RocksDB load error, path=" + path + "\n"); +++ return; +++ } +++ +++ try { +++ if (TOPICS_JSON_CONFIG.equalsIgnoreCase(configType) || SUBSCRIPTION_GROUP_JSON_CONFIG.equalsIgnoreCase(configType)) { +++ handleExportMetadata(kvStore, configType, jsonEnable); +++ } else { +++ System.out.printf("Invalid config type=%s, Options: topics,subscriptionGroups\n", configType); +++ } +++ } finally { +++ kvStore.shutdown(); +++ } +++ } +++ +++ private static void handleExportMetadata(ConfigRocksDBStorage kvStore, String configType, boolean jsonEnable) { +++ if (jsonEnable) { +++ final Map jsonConfig = new HashMap<>(); +++ final Map configTable = new HashMap<>(); +++ iterateKvStore(kvStore, (key, value) -> { +++ final String configKey = new String(key, DataConverter.charset); +++ final String configValue = new String(value, DataConverter.charset); +++ final JSONObject jsonObject = JSONObject.parseObject(configValue); +++ configTable.put(configKey, jsonObject); +++ } +++ ); +++ +++ jsonConfig.put(configType.equalsIgnoreCase(TOPICS_JSON_CONFIG) ? "topicConfigTable" : "subscriptionGroupTable", +++ (JSONObject) JSONObject.toJSON(configTable)); +++ final String jsonConfigStr = JSONObject.toJSONString(jsonConfig, true); +++ System.out.print(jsonConfigStr + "\n"); +++ } else { +++ AtomicLong count = new AtomicLong(0); +++ iterateKvStore(kvStore, (key, value) -> { +++ final String configKey = new String(key, DataConverter.charset); +++ final String configValue = new String(value, DataConverter.charset); +++ System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue); +++ }); +++ } +++ } +++ +++ private static void iterateKvStore(ConfigRocksDBStorage kvStore, BiConsumer biConsumer) { +++ try (RocksIterator iterator = kvStore.iterator()) { +++ iterator.seekToFirst(); +++ for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { +++ biConsumer.accept(iterator.key(), iterator.value()); +++ } +++ } +++ } +++} ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java ++deleted file mode 100644 ++index 3fc63e4dd..000000000 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +++++ /dev/null ++@@ -1,122 +0,0 @@ ++-/* ++- * Licensed to the Apache Software Foundation (ASF) under one or more ++- * contributor license agreements. See the NOTICE file distributed with ++- * this work for additional information regarding copyright ownership. ++- * The ASF licenses this file to You under the Apache License, Version 2.0 ++- * (the "License"); you may not use this file except in compliance with ++- * the License. You may obtain a copy of the License at ++- * ++- * http://www.apache.org/licenses/LICENSE-2.0 ++- * ++- * Unless required by applicable law or agreed to in writing, software ++- * distributed under the License is distributed on an "AS IS" BASIS, ++- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++- * See the License for the specific language governing permissions and ++- * limitations under the License. ++- */ ++-package org.apache.rocketmq.tools.command.metadata; ++- ++-import com.alibaba.fastjson.JSONObject; ++-import org.apache.commons.cli.CommandLine; ++-import org.apache.commons.cli.Option; ++-import org.apache.commons.cli.Options; ++-import org.apache.commons.lang3.StringUtils; ++-import org.apache.rocketmq.common.UtilAll; ++-import org.apache.rocketmq.common.config.RocksDBConfigManager; ++-import org.apache.rocketmq.common.utils.DataConverter; ++-import org.apache.rocketmq.remoting.RPCHook; ++-import org.apache.rocketmq.tools.command.SubCommand; ++-import org.apache.rocketmq.tools.command.SubCommandException; ++- ++-import java.util.HashMap; ++-import java.util.Map; ++- ++-public class RocksDBConfigToJsonCommand implements SubCommand { ++- private static final String TOPICS_JSON_CONFIG = "topics"; ++- private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; ++- ++- @Override ++- public String commandName() { ++- return "rocksDBConfigToJson"; ++- } ++- ++- @Override ++- public String commandDesc() { ++- return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; ++- } ++- ++- @Override ++- public Options buildCommandlineOptions(Options options) { ++- Option pathOption = new Option("p", "path", true, ++- "Absolute path for the metadata directory"); ++- pathOption.setRequired(true); ++- options.addOption(pathOption); ++- ++- Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + ++- "topics/subscriptionGroups"); ++- configTypeOption.setRequired(true); ++- options.addOption(configTypeOption); ++- ++- return options; ++- } ++- ++- @Override ++- public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++- String path = commandLine.getOptionValue("path").trim(); ++- if (StringUtils.isEmpty(path) || !UtilAll.isPathExists(path)) { ++- System.out.print("Rocksdb path is invalid.\n"); ++- return; ++- } ++- ++- String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++- ++- RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(60 * 60 * 1000L); ++- try { ++- if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { ++- // for topics.json ++- final Map topicsJsonConfig = new HashMap<>(); ++- final Map topicConfigTable = new HashMap<>(); ++- boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++- final String topic = new String(key, DataConverter.charset); ++- final String topicConfig = new String(value, DataConverter.charset); ++- final JSONObject jsonObject = JSONObject.parseObject(topicConfig); ++- topicConfigTable.put(topic, jsonObject); ++- }); ++- ++- if (!isLoad) { ++- System.out.print("RocksDB load error, path=" + path); ++- return; ++- } ++- topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++- final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++- System.out.print(topicsJsonStr + "\n"); ++- return; ++- } ++- ++- if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { ++- // for subscriptionGroup.json ++- final Map subscriptionGroupJsonConfig = new HashMap<>(); ++- final Map subscriptionGroupTable = new HashMap<>(); ++- boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++- final String subscriptionGroup = new String(key, DataConverter.charset); ++- final String subscriptionGroupConfig = new String(value, DataConverter.charset); ++- final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); ++- subscriptionGroupTable.put(subscriptionGroup, jsonObject); ++- }); ++- ++- if (!isLoad) { ++- System.out.print("RocksDB load error, path=" + path); ++- return; ++- } ++- subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++- (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++- final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++- System.out.print(subscriptionGroupJsonStr + "\n"); ++- return; ++- } ++- System.out.print("Config type was not recognized, configType=" + configType + "\n"); ++- } finally { ++- kvConfigManager.stop(); ++- } ++- } ++-} ++diff --git a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java ++similarity index 62% ++rename from tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java ++rename to tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java ++index b2f66c7b0..2b938c90f 100644 ++--- a/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/KvConfigToJsonCommandTest.java +++++ b/tools/src/test/java/org/apache/rocketmq/tools/command/metadata/ExportMetadataInRocksDBCommandTest.java ++@@ -21,43 +21,53 @@ import org.apache.commons.cli.DefaultParser; ++ import org.apache.commons.cli.Options; ++ import org.apache.rocketmq.srvutil.ServerUtil; ++ import org.apache.rocketmq.tools.command.SubCommandException; +++import org.apache.rocketmq.tools.command.export.ExportMetadataInRocksDBCommand; ++ import org.junit.Test; ++ ++ import java.io.File; ++ ++ import static org.assertj.core.api.Assertions.assertThat; ++ ++-public class KvConfigToJsonCommandTest { +++public class ExportMetadataInRocksDBCommandTest { ++ private static final String BASE_PATH = System.getProperty("user.home") + File.separator + "store/config/"; ++ ++ @Test ++ public void testExecute() throws SubCommandException { ++ { ++- String[] cases = new String[]{"topics", "subscriptionGroups"}; ++- for (String c : cases) { ++- RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ String[][] cases = new String[][] { +++ {"topics", "false"}, +++ {"topics", "false1"}, +++ {"topics", "true"}, +++ {"subscriptionGroups", "false"}, +++ {"subscriptionGroups", "false2"}, +++ {"subscriptionGroups", "true"} +++ }; +++ +++ for (String[] c : cases) { +++ ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand(); ++ Options options = ServerUtil.buildCommandlineOptions(new Options()); ++- String[] subargs = new String[]{"-p " + BASE_PATH + c, "-t " + c}; +++ String[] subargs = new String[] {"-p " + BASE_PATH + c[0], "-t " + c[0], "-j " + c[1]}; ++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), subargs, ++- cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.execute(commandLine, options, null); ++- assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c); ++- assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c); +++ assertThat(commandLine.getOptionValue("p").trim()).isEqualTo(BASE_PATH + c[0]); +++ assertThat(commandLine.getOptionValue("t").trim()).isEqualTo(c[0]); +++ assertThat(commandLine.getOptionValue("j").trim()).isEqualTo(c[1]); ++ } ++ } ++ // invalid cases ++ { ++- String[][] cases = new String[][]{ ++- {"-p " + BASE_PATH + "tmpPath", "-t topics"}, ++- {"-p ", "-t topics"}, ++- {"-p " + BASE_PATH + "topics", "-t invalid_type"} +++ String[][] cases = new String[][] { +++ {"-p " + BASE_PATH + "tmpPath", "-t topics", "-j true"}, +++ {"-p ", "-t topics", "-j true"}, +++ {"-p " + BASE_PATH + "topics", "-t invalid_type", "-j true"} ++ }; ++ ++ for (String[] c : cases) { ++- RocksDBConfigToJsonCommand cmd = new RocksDBConfigToJsonCommand(); +++ ExportMetadataInRocksDBCommand cmd = new ExportMetadataInRocksDBCommand(); ++ Options options = ServerUtil.buildCommandlineOptions(new Options()); ++ final CommandLine commandLine = ServerUtil.parseCmdLine("mqadmin " + cmd.commandName(), c, ++- cmd.buildCommandlineOptions(options), new DefaultParser()); +++ cmd.buildCommandlineOptions(options), new DefaultParser()); ++ cmd.execute(commandLine, options, null); ++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From fa549154370cb866a90e37c13a90d2c598d6b1f6 Mon Sep 17 00:00:00 2001 ++From: yuz10 <845238369@qq.com> ++Date: Tue, 29 Aug 2023 15:22:09 +0800 ++Subject: [PATCH 5/6] [ISSUE #7261] Slave high CPU usage when ++ enableScheduleAsyncDeliver=true (#7262) ++ ++* [ISSUE #6390] Add break to the exception of WHEEL_TIMER_NOT_ENABLE. ++ ++* fix broker start fail if mapped file size is 0 ++ ++* log ++ ++* only delete the last empty file ++ ++* change dataReadAheadEnable default to true ++ ++* fix endless loop when master change to slave. ++--- ++ .../rocketmq/broker/schedule/ScheduleMessageService.java | 7 ++++++- ++ 1 file changed, 6 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++index aed0ee19f..297b14207 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++@@ -566,7 +566,8 @@ public class ScheduleMessageService extends ConfigManager { ++ pendingQueue.remove(); ++ break; ++ case RUNNING: ++- break; +++ scheduleNextTask(); +++ return; ++ case EXCEPTION: ++ if (!isStarted()) { ++ log.warn("HandlePutResultTask shutdown, info={}", putResultProcess.toString()); ++@@ -586,6 +587,10 @@ public class ScheduleMessageService extends ConfigManager { ++ } ++ } ++ +++ scheduleNextTask(); +++ } +++ +++ private void scheduleNextTask() { ++ if (isStarted()) { ++ ScheduleMessageService.this.handleExecutorService ++ .schedule(new HandlePutResultTask(this.delayLevel), DELAY_FOR_A_SLEEP, TimeUnit.MILLISECONDS); ++-- ++2.32.0.windows.2 ++ ++ ++From 9f34f55e1dac495730c9cd5469f2ab3225b8f0b9 Mon Sep 17 00:00:00 2001 ++From: ShuangxiDing ++Date: Tue, 29 Aug 2023 15:48:46 +0800 ++Subject: [PATCH 6/6] [ISSUE #7226] Filter tlvs in ppv2 which contents not are ++ spec-compliant ASCII characters and space (#7227) ++ ++Filter tlvs in ppv2 which not are spec-compliant ASCII characters and space ++--- ++ .../rocketmq/common/utils/BinaryUtil.java | 17 +++++++++++++++++ ++ .../grpc/ProxyAndTlsProtocolNegotiator.java | 8 +++++++- ++ .../remoting/netty/NettyRemotingServer.java | 8 +++++++- ++ 3 files changed, 31 insertions(+), 2 deletions(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java ++index 421adaca4..7b4b24819 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java +++++ b/common/src/main/java/org/apache/rocketmq/common/utils/BinaryUtil.java ++@@ -43,4 +43,21 @@ public class BinaryUtil { ++ byte[] bytes = calculateMd5(content); ++ return Hex.encodeHexString(bytes, false); ++ } +++ +++ /** +++ * Returns true if subject contains only bytes that are spec-compliant ASCII characters. +++ * @param subject +++ * @return +++ */ +++ public static boolean isAscii(byte[] subject) { +++ if (subject == null) { +++ return false; +++ } +++ for (byte b : subject) { +++ if ((b & 0x80) != 0) { +++ return false; +++ } +++ } +++ return true; +++ } ++ } ++\ No newline at end of file ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java ++index ee167bd7b..b584ddfbd 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/ProxyAndTlsProtocolNegotiator.java ++@@ -24,6 +24,7 @@ import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiator; ++ import io.grpc.netty.shaded.io.grpc.netty.InternalProtocolNegotiators; ++ import io.grpc.netty.shaded.io.grpc.netty.ProtocolNegotiationEvent; ++ import io.grpc.netty.shaded.io.netty.buffer.ByteBuf; +++import io.grpc.netty.shaded.io.netty.buffer.ByteBufUtil; ++ import io.grpc.netty.shaded.io.netty.channel.ChannelHandler; ++ import io.grpc.netty.shaded.io.netty.channel.ChannelHandlerContext; ++ import io.grpc.netty.shaded.io.netty.channel.ChannelInboundHandlerAdapter; ++@@ -44,6 +45,7 @@ import org.apache.commons.collections.CollectionUtils; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.constant.HAProxyConstants; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.BinaryUtil; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++@@ -191,9 +193,13 @@ public class ProxyAndTlsProtocolNegotiator implements InternalProtocolNegotiator ++ } ++ if (CollectionUtils.isNotEmpty(msg.tlvs())) { ++ msg.tlvs().forEach(tlv -> { +++ byte[] valueBytes = ByteBufUtil.getBytes(tlv.content()); +++ if (!BinaryUtil.isAscii(valueBytes)) { +++ return; +++ } ++ Attributes.Key key = AttributeKeys.valueOf( ++ HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); ++- String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8)); +++ String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8)); ++ builder.set(key, value); ++ }); ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++index 17f138f86..e626260c9 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty; ++ ++ import io.netty.bootstrap.ServerBootstrap; ++ import io.netty.buffer.ByteBuf; +++import io.netty.buffer.ByteBufUtil; ++ import io.netty.buffer.PooledByteBufAllocator; ++ import io.netty.channel.Channel; ++ import io.netty.channel.ChannelDuplexHandler; ++@@ -58,6 +59,7 @@ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.HAProxyConstants; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.BinaryUtil; ++ import org.apache.rocketmq.common.utils.NetworkUtil; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++@@ -787,9 +789,13 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ } ++ if (CollectionUtils.isNotEmpty(msg.tlvs())) { ++ msg.tlvs().forEach(tlv -> { +++ byte[] valueBytes = ByteBufUtil.getBytes(tlv.content()); +++ if (!BinaryUtil.isAscii(valueBytes)) { +++ return; +++ } ++ AttributeKey key = AttributeKeys.valueOf( ++ HAProxyConstants.PROXY_PROTOCOL_TLV_PREFIX + String.format("%02x", tlv.typeByteValue())); ++- String value = StringUtils.trim(tlv.content().toString(CharsetUtil.UTF_8)); +++ String value = StringUtils.trim(new String(valueBytes, CharsetUtil.UTF_8)); ++ channel.attr(key).set(value); ++ }); ++ } ++-- ++2.32.0.windows.2 ++ +diff --git a/patch014-backport-Queue-Selection-Strategy-Optimization.patch b/patch014-backport-Queue-Selection-Strategy-Optimization.patch +new file mode 100644 +index 000000000..90d3553aa +--- /dev/null ++++ b/patch014-backport-Queue-Selection-Strategy-Optimization.patch +@@ -0,0 +1,2023 @@ ++From b028277018946868838a82a08211071bc231a175 Mon Sep 17 00:00:00 2001 ++From: Ji Juntao ++Date: Tue, 29 Aug 2023 16:13:38 +0800 ++Subject: [PATCH] [ISSUE #6567] [RIP-63] Queue Selection Strategy Optimization ++ (#6568) ++ ++Optimize the proxy's and client's selection strategy for brokers when sending messages, and use multiple selection strategies as a pipeline to filter suitable queues. ++--- ++ .../apache/rocketmq/client/ClientConfig.java | 54 +++++ ++ .../client/common/ThreadLocalIndex.java | 8 + ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 12 +- ++ .../client/impl/factory/MQClientInstance.java | 7 + ++ .../impl/producer/DefaultMQProducerImpl.java | 87 ++++++-- ++ .../impl/producer/TopicPublishInfo.java | 40 ++++ ++ .../client/latency/LatencyFaultTolerance.java | 66 +++++- ++ .../latency/LatencyFaultToleranceImpl.java | 189 ++++++++++++++---- ++ .../client/latency/MQFaultStrategy.java | 155 ++++++++++---- ++ .../rocketmq/client/latency/Resolver.java | 17 +- ++ .../client/latency/ServiceDetector.java | 30 +++ ++ .../LatencyFaultToleranceImplTest.java | 36 +++- ++ .../processor/DefaultRequestProcessor.java | 24 --- ++ .../rocketmq/proxy/config/ProxyConfig.java | 46 +++++ ++ .../grpc/v2/producer/SendMessageActivity.java | 2 +- ++ .../proxy/processor/ProducerProcessor.java | 18 +- ++ .../service/route/LocalTopicRouteService.java | 2 +- ++ .../service/route/MessageQueueSelector.java | 95 ++++++++- ++ .../proxy/service/route/MessageQueueView.java | 18 +- ++ .../service/route/TopicRouteService.java | 80 +++++++- ++ .../consumer/ReceiveMessageActivityTest.java | 5 +- ++ .../v2/producer/SendMessageActivityTest.java | 82 +++++++- ++ .../proxy/service/BaseServiceTest.java | 4 +- ++ .../route/MessageQueueSelectorTest.java | 8 +- ++ .../sysmessage/HeartbeatSyncerTest.java | 2 +- ++ .../ClusterTransactionServiceTest.java | 8 +- ++ 26 files changed, 919 insertions(+), 176 deletions(-) ++ rename remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java => client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java (65%) ++ create mode 100644 client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++index f87450f66..bb0fe3522 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++@@ -38,6 +38,8 @@ public class ClientConfig { ++ public static final String SOCKS_PROXY_CONFIG = "com.rocketmq.socks.proxy.config"; ++ public static final String DECODE_READ_BODY = "com.rocketmq.read.body"; ++ public static final String DECODE_DECOMPRESS_BODY = "com.rocketmq.decompress.body"; +++ public static final String SEND_LATENCY_ENABLE = "com.rocketmq.sendLatencyEnable"; +++ public static final String START_DETECTOR_ENABLE = "com.rocketmq.startDetectorEnable"; ++ public static final String HEART_BEAT_V2 = "com.rocketmq.heartbeat.v2"; ++ private String namesrvAddr = NameServerAddressUtils.getNameServerAddresses(); ++ private String clientIP = NetworkUtil.getLocalAddress(); ++@@ -72,6 +74,8 @@ public class ClientConfig { ++ private String socksProxyConfig = System.getProperty(SOCKS_PROXY_CONFIG, "{}"); ++ ++ private int mqClientApiTimeout = 3 * 1000; +++ private int detectTimeout = 200; +++ private int detectInterval = 2 * 1000; ++ ++ private LanguageCode language = LanguageCode.JAVA; ++ ++@@ -81,6 +85,15 @@ public class ClientConfig { ++ */ ++ protected boolean enableStreamRequestType = false; ++ +++ /** +++ * Enable the fault tolerance mechanism of the client sending process. +++ * DO NOT OPEN when ORDER messages are required. +++ * Turning on will interfere with the queue selection functionality, +++ * possibly conflicting with the order message. +++ */ +++ private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false")); +++ private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false")); +++ ++ public String buildMQClientId() { ++ StringBuilder sb = new StringBuilder(); ++ sb.append(this.getClientIP()); ++@@ -186,6 +199,10 @@ public class ClientConfig { ++ this.decodeDecompressBody = cc.decodeDecompressBody; ++ this.enableStreamRequestType = cc.enableStreamRequestType; ++ this.useHeartbeatV2 = cc.useHeartbeatV2; +++ this.startDetectorEnable = cc.startDetectorEnable; +++ this.sendLatencyEnable = cc.sendLatencyEnable; +++ this.detectInterval = cc.detectInterval; +++ this.detectTimeout = cc.detectTimeout; ++ } ++ ++ public ClientConfig cloneClientConfig() { ++@@ -210,6 +227,10 @@ public class ClientConfig { ++ cc.decodeDecompressBody = decodeDecompressBody; ++ cc.enableStreamRequestType = enableStreamRequestType; ++ cc.useHeartbeatV2 = useHeartbeatV2; +++ cc.startDetectorEnable = startDetectorEnable; +++ cc.sendLatencyEnable = sendLatencyEnable; +++ cc.detectInterval = detectInterval; +++ cc.detectTimeout = detectTimeout; ++ return cc; ++ } ++ ++@@ -381,6 +402,38 @@ public class ClientConfig { ++ this.enableStreamRequestType = enableStreamRequestType; ++ } ++ +++ public boolean isSendLatencyEnable() { +++ return sendLatencyEnable; +++ } +++ +++ public void setSendLatencyEnable(boolean sendLatencyEnable) { +++ this.sendLatencyEnable = sendLatencyEnable; +++ } +++ +++ public boolean isStartDetectorEnable() { +++ return startDetectorEnable; +++ } +++ +++ public void setStartDetectorEnable(boolean startDetectorEnable) { +++ this.startDetectorEnable = startDetectorEnable; +++ } +++ +++ public int getDetectTimeout() { +++ return this.detectTimeout; +++ } +++ +++ public void setDetectTimeout(int detectTimeout) { +++ this.detectTimeout = detectTimeout; +++ } +++ +++ public int getDetectInterval() { +++ return this.detectInterval; +++ } +++ +++ public void setDetectInterval(int detectInterval) { +++ this.detectInterval = detectInterval; +++ } +++ ++ public boolean isUseHeartbeatV2() { ++ return useHeartbeatV2; ++ } ++@@ -403,6 +456,7 @@ public class ClientConfig { ++ + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name() ++ + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout ++ + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody +++ + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable ++ + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]"; ++ } ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java ++index 4a3d90135..3a086c13d 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java +++++ b/client/src/main/java/org/apache/rocketmq/client/common/ThreadLocalIndex.java ++@@ -33,6 +33,14 @@ public class ThreadLocalIndex { ++ return index & POSITIVE_MASK; ++ } ++ +++ public void reset() { +++ int index = Math.abs(random.nextInt(Integer.MAX_VALUE)); +++ if (index < 0) { +++ index = 0; +++ } +++ this.threadLocalIndex.set(index); +++ } +++ ++ @Override ++ public String toString() { ++ return "ThreadLocalIndex{" + ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 213c26fd6..3201a493f 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -666,7 +666,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ } catch (Throwable e) { ++ } ++ ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); ++ return; ++ } ++ ++@@ -684,14 +684,14 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ } catch (Throwable e) { ++ } ++ ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); ++ } catch (Exception e) { ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++ retryTimesWhenSendFailed, times, e, context, false, producer); ++ } ++ } else { ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); ++ if (!responseFuture.isSendRequestOK()) { ++ MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++@@ -711,7 +711,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ }); ++ } catch (Exception ex) { ++ long cost = System.currentTimeMillis() - beginStartTime; ++- producer.updateFaultItem(brokerName, cost, true); +++ producer.updateFaultItem(brokerName, cost, true, false); ++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++ retryTimesWhenSendFailed, times, ex, context, true, producer); ++ } ++@@ -735,7 +735,7 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ if (needRetry && tmp <= timesTotal) { ++ String retryBrokerName = brokerName;//by default, it will send to the same broker ++ if (topicPublishInfo != null) { //select one message queue accordingly, in order to determine which broker to send ++- MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName); +++ MessageQueue mqChosen = producer.selectOneMessageQueue(topicPublishInfo, brokerName, false); ++ retryBrokerName = instance.getBrokerNameFromMessageQueue(mqChosen); ++ } ++ String addr = instance.findBrokerAddressInPublish(retryBrokerName); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++index 8851bc815..9484b26f8 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++@@ -29,6 +29,7 @@ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ThreadFactory; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicLong; ++ import java.util.concurrent.locks.Lock; ++@@ -125,6 +126,12 @@ public class MQClientInstance { ++ private final Set brokerSupportV2HeartbeatSet = new HashSet(); ++ private final ConcurrentMap brokerAddrHeartbeatFingerprintTable = new ConcurrentHashMap(); ++ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(r -> new Thread(r, "MQClientFactoryScheduledThread")); +++ private final ScheduledExecutorService fetchRemoteConfigExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { +++ @Override +++ public Thread newThread(Runnable r) { +++ return new Thread(r, "MQClientFactoryFetchRemoteConfigScheduledThread"); +++ } +++ }); ++ private final PullMessageService pullMessageService; ++ private final RebalanceService rebalanceService; ++ private final DefaultMQProducer defaultMQProducer; ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++index 3f4c6e5f7..bbbb17b07 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++@@ -33,6 +33,8 @@ import java.util.concurrent.RejectedExecutionException; ++ import java.util.concurrent.Semaphore; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; +++ +++import com.google.common.base.Optional; ++ import org.apache.rocketmq.client.QueryResult; ++ import org.apache.rocketmq.client.Validators; ++ import org.apache.rocketmq.client.common.ClientErrorCode; ++@@ -49,6 +51,8 @@ import org.apache.rocketmq.client.impl.CommunicationMode; ++ import org.apache.rocketmq.client.impl.MQClientManager; ++ import org.apache.rocketmq.client.impl.factory.MQClientInstance; ++ import org.apache.rocketmq.client.latency.MQFaultStrategy; +++import org.apache.rocketmq.client.latency.Resolver; +++import org.apache.rocketmq.client.latency.ServiceDetector; ++ import org.apache.rocketmq.client.producer.DefaultMQProducer; ++ import org.apache.rocketmq.client.producer.LocalTransactionExecuter; ++ import org.apache.rocketmq.client.producer.LocalTransactionState; ++@@ -112,7 +116,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ private ServiceState serviceState = ServiceState.CREATE_JUST; ++ private MQClientInstance mQClientFactory; ++ private ArrayList checkForbiddenHookList = new ArrayList<>(); ++- private MQFaultStrategy mqFaultStrategy = new MQFaultStrategy(); +++ private MQFaultStrategy mqFaultStrategy; ++ private ExecutorService asyncSenderExecutor; ++ ++ // compression related ++@@ -153,8 +157,38 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ semaphoreAsyncSendSize = new Semaphore(1024 * 1024, true); ++ log.info("semaphoreAsyncSendSize can not be smaller than 1M."); ++ } ++- } ++ +++ ServiceDetector serviceDetector = new ServiceDetector() { +++ @Override +++ public boolean detect(String endpoint, long timeoutMillis) { +++ Optional candidateTopic = pickTopic(); +++ if (!candidateTopic.isPresent()) { +++ return false; +++ } +++ try { +++ MessageQueue mq = new MessageQueue(candidateTopic.get(), null, 0); +++ mQClientFactory.getMQClientAPIImpl() +++ .getMaxOffset(endpoint, mq, timeoutMillis); +++ return true; +++ } catch (Exception e) { +++ return false; +++ } +++ } +++ }; +++ +++ this.mqFaultStrategy = new MQFaultStrategy(defaultMQProducer.cloneClientConfig(), new Resolver() { +++ @Override +++ public String resolve(String name) { +++ return DefaultMQProducerImpl.this.mQClientFactory.findBrokerAddressInPublish(name); +++ } +++ }, serviceDetector); +++ } +++ private Optional pickTopic() { +++ if (topicPublishInfoTable.isEmpty()) { +++ return Optional.absent(); +++ } +++ return Optional.of(topicPublishInfoTable.keySet().iterator().next()); +++ } ++ public void registerCheckForbiddenHook(CheckForbiddenHook checkForbiddenHook) { ++ this.checkForbiddenHookList.add(checkForbiddenHook); ++ log.info("register a new checkForbiddenHook. hookName={}, allHookSize={}", checkForbiddenHook.hookName(), ++@@ -229,6 +263,10 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ mQClientFactory.start(); ++ } ++ +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ this.mqFaultStrategy.startDetector(); +++ } +++ ++ log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), ++ this.defaultMQProducer.isSendMessageWithVIPChannel()); ++ this.serviceState = ServiceState.RUNNING; ++@@ -273,6 +311,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ if (shutdownFactory) { ++ this.mQClientFactory.shutdown(); ++ } +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ this.mqFaultStrategy.shutdown(); +++ } ++ RequestFutureHolder.getInstance().shutdown(this); ++ log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); ++ this.serviceState = ServiceState.SHUTDOWN_ALREADY; ++@@ -574,7 +615,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ } ++ ++ public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg, ++- final long timeout) throws MQClientException, RemotingTooMuchRequestException { +++ final long timeout) throws MQClientException, RemotingTooMuchRequestException { ++ long beginStartTime = System.currentTimeMillis(); ++ this.makeSureStateOK(); ++ Validators.checkMessage(msg, this.defaultMQProducer); ++@@ -584,7 +625,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ MessageQueue mq = null; ++ try { ++ List messageQueueList = ++- mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); +++ mQClientFactory.getMQAdminImpl().parsePublishMessageQueues(topicPublishInfo.getMessageQueueList()); ++ Message userMessage = MessageAccessor.cloneMessage(msg); ++ String userTopic = NamespaceUtil.withoutNamespace(userMessage.getTopic(), mQClientFactory.getClientConfig().getNamespace()); ++ userMessage.setTopic(userTopic); ++@@ -609,12 +650,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ throw new MQClientException("No route info for this topic, " + msg.getTopic(), null); ++ } ++ ++- public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { ++- return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName); +++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { +++ return this.mqFaultStrategy.selectOneMessageQueue(tpInfo, lastBrokerName, resetIndex); ++ } ++ ++- public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { ++- this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation); +++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, +++ boolean reachable) { +++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); ++ } ++ ++ private void validateNameServerSetting() throws MQClientException { ++@@ -647,9 +689,13 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ int timesTotal = communicationMode == CommunicationMode.SYNC ? 1 + this.defaultMQProducer.getRetryTimesWhenSendFailed() : 1; ++ int times = 0; ++ String[] brokersSent = new String[timesTotal]; +++ boolean resetIndex = false; ++ for (; times < timesTotal; times++) { ++ String lastBrokerName = null == mq ? null : mq.getBrokerName(); ++- MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName); +++ if (times > 0) { +++ resetIndex = true; +++ } +++ MessageQueue mqSelected = this.selectOneMessageQueue(topicPublishInfo, lastBrokerName, resetIndex); ++ if (mqSelected != null) { ++ mq = mqSelected; ++ brokersSent[times] = mq.getBrokerName(); ++@@ -667,7 +713,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ ++ sendResult = this.sendKernelImpl(msg, mq, communicationMode, sendCallback, topicPublishInfo, timeout - costTime); ++ endTimestamp = System.currentTimeMillis(); ++- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); ++ switch (communicationMode) { ++ case ASYNC: ++ return null; ++@@ -684,9 +730,22 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ default: ++ break; ++ } ++- } catch (RemotingException | MQClientException e) { +++ } catch (MQClientException e) { ++ endTimestamp = System.currentTimeMillis(); ++- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); +++ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); +++ log.warn(msg.toString()); +++ exception = e; +++ continue; +++ } catch (RemotingException e) { +++ endTimestamp = System.currentTimeMillis(); +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ // Set this broker unreachable when detecting schedule task is running for RemotingException. +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false); +++ } else { +++ // Otherwise, isolate this broker. +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, true); +++ } ++ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); ++ if (log.isDebugEnabled()) { ++ log.debug(msg.toString()); ++@@ -695,7 +754,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ continue; ++ } catch (MQBrokerException e) { ++ endTimestamp = System.currentTimeMillis(); ++- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true); +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, true, false); ++ log.warn("sendKernelImpl exception, resend at once, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); ++ if (log.isDebugEnabled()) { ++ log.debug(msg.toString()); ++@@ -712,7 +771,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ } ++ } catch (InterruptedException e) { ++ endTimestamp = System.currentTimeMillis(); ++- this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false); +++ this.updateFaultItem(mq.getBrokerName(), endTimestamp - beginTimestampPrev, false, true); ++ log.warn("sendKernelImpl exception, throw exception, InvokeID: %s, RT: %sms, Broker: %s", invokeID, endTimestamp - beginTimestampPrev, mq, e); ++ if (log.isDebugEnabled()) { ++ log.debug(msg.toString()); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java ++index 275ada7ac..37b1f3252 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/TopicPublishInfo.java ++@@ -18,6 +18,8 @@ package org.apache.rocketmq.client.impl.producer; ++ ++ import java.util.ArrayList; ++ import java.util.List; +++ +++import com.google.common.base.Preconditions; ++ import org.apache.rocketmq.client.common.ThreadLocalIndex; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.remoting.protocol.route.QueueData; ++@@ -30,6 +32,10 @@ public class TopicPublishInfo { ++ private volatile ThreadLocalIndex sendWhichQueue = new ThreadLocalIndex(); ++ private TopicRouteData topicRouteData; ++ +++ public interface QueueFilter { +++ boolean filter(MessageQueue mq); +++ } +++ ++ public boolean isOrderTopic() { ++ return orderTopic; ++ } ++@@ -66,6 +72,40 @@ public class TopicPublishInfo { ++ this.haveTopicRouterInfo = haveTopicRouterInfo; ++ } ++ +++ public MessageQueue selectOneMessageQueue(QueueFilter ...filter) { +++ return selectOneMessageQueue(this.messageQueueList, this.sendWhichQueue, filter); +++ } +++ +++ private MessageQueue selectOneMessageQueue(List messageQueueList, ThreadLocalIndex sendQueue, QueueFilter ...filter) { +++ if (messageQueueList == null || messageQueueList.isEmpty()) { +++ return null; +++ } +++ +++ if (filter != null && filter.length != 0) { +++ for (int i = 0; i < messageQueueList.size(); i++) { +++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); +++ MessageQueue mq = messageQueueList.get(index); +++ boolean filterResult = true; +++ for (QueueFilter f: filter) { +++ Preconditions.checkNotNull(f); +++ filterResult &= f.filter(mq); +++ } +++ if (filterResult) { +++ return mq; +++ } +++ } +++ +++ return null; +++ } +++ +++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); +++ return messageQueueList.get(index); +++ } +++ +++ public void resetIndex() { +++ this.sendWhichQueue.reset(); +++ } +++ ++ public MessageQueue selectOneMessageQueue(final String lastBrokerName) { ++ if (lastBrokerName == null) { ++ return selectOneMessageQueue(); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++index 09a8aa461..72d2f3450 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++@@ -18,11 +18,75 @@ ++ package org.apache.rocketmq.client.latency; ++ ++ public interface LatencyFaultTolerance { ++- void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration); +++ /** +++ * Update brokers' states, to decide if they are good or not. +++ * +++ * @param name Broker's name. +++ * @param currentLatency Current message sending process's latency. +++ * @param notAvailableDuration Corresponding not available time, ms. The broker will be not available until it +++ * spends such time. +++ * @param reachable To decide if this broker is reachable or not. +++ */ +++ void updateFaultItem(final T name, final long currentLatency, final long notAvailableDuration, +++ final boolean reachable); ++ +++ /** +++ * To check if this broker is available. +++ * +++ * @param name Broker's name. +++ * @return boolean variable, if this is true, then the broker is available. +++ */ ++ boolean isAvailable(final T name); ++ +++ /** +++ * To check if this broker is reachable. +++ * +++ * @param name Broker's name. +++ * @return boolean variable, if this is true, then the broker is reachable. +++ */ +++ boolean isReachable(final T name); +++ +++ /** +++ * Remove the broker in this fault item table. +++ * +++ * @param name broker's name. +++ */ ++ void remove(final T name); ++ +++ /** +++ * The worst situation, no broker can be available. Then choose random one. +++ * +++ * @return A random mq will be returned. +++ */ ++ T pickOneAtLeast(); +++ +++ /** +++ * Start a new thread, to detect the broker's reachable tag. +++ */ +++ void startDetector(); +++ +++ /** +++ * Shutdown threads that started by LatencyFaultTolerance. +++ */ +++ void shutdown(); +++ +++ /** +++ * A function reserved, just detect by once, won't create a new thread. +++ */ +++ void detectByOneRound(); +++ +++ /** +++ * Use it to set the detect timeout bound. +++ * +++ * @param detectTimeout timeout bound +++ */ +++ void setDetectTimeout(final int detectTimeout); +++ +++ /** +++ * Use it to set the detector's detector interval for each broker (each broker will be detected once during this +++ * time) +++ * +++ * @param detectInterval each broker's detecting interval +++ */ +++ void setDetectInterval(final int detectInterval); ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++index 93795d957..8af629574 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++@@ -21,30 +21,97 @@ import java.util.Collections; ++ import java.util.Enumeration; ++ import java.util.LinkedList; ++ import java.util.List; +++import java.util.Map; ++ import java.util.concurrent.ConcurrentHashMap; +++import java.util.concurrent.Executors; +++import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ThreadFactory; +++import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.client.common.ThreadLocalIndex; +++import org.apache.rocketmq.logging.org.slf4j.Logger; +++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance { ++- private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap<>(16); +++ private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class); +++ private final ConcurrentHashMap faultItemTable = new ConcurrentHashMap(16); +++ private int detectTimeout = 200; +++ private int detectInterval = 2000; +++ private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex(); +++ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { +++ @Override +++ public Thread newThread(Runnable r) { +++ return new Thread(r, "LatencyFaultToleranceScheduledThread"); +++ } +++ }); ++ ++- private final ThreadLocalIndex randomItem = new ThreadLocalIndex(); +++ private final Resolver resolver; +++ +++ private final ServiceDetector serviceDetector; +++ +++ public LatencyFaultToleranceImpl(Resolver resolver, ServiceDetector serviceDetector) { +++ this.resolver = resolver; +++ this.serviceDetector = serviceDetector; +++ } +++ +++ public void detectByOneRound() { +++ for (Map.Entry item : this.faultItemTable.entrySet()) { +++ FaultItem brokerItem = item.getValue(); +++ if (System.currentTimeMillis() - brokerItem.checkStamp >= 0) { +++ brokerItem.checkStamp = System.currentTimeMillis() + this.detectInterval; +++ String brokerAddr = resolver.resolve(brokerItem.getName()); +++ if (brokerAddr == null) { +++ faultItemTable.remove(item.getKey()); +++ continue; +++ } +++ if (null == serviceDetector) { +++ continue; +++ } +++ boolean serviceOK = serviceDetector.detect(brokerAddr, detectTimeout); +++ if (serviceOK && !brokerItem.reachableFlag) { +++ log.info(brokerItem.name + " is reachable now, then it can be used."); +++ brokerItem.reachableFlag = true; +++ } +++ } +++ } +++ } +++ +++ public void startDetector() { +++ this.scheduledExecutorService.scheduleAtFixedRate(new Runnable() { +++ @Override +++ public void run() { +++ try { +++ detectByOneRound(); +++ } catch (Exception e) { +++ log.warn("Unexpected exception raised while detecting service reachability", e); +++ } +++ } +++ }, 3, 3, TimeUnit.SECONDS); +++ } +++ +++ public void shutdown() { +++ this.scheduledExecutorService.shutdown(); +++ } ++ ++ @Override ++- public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration) { +++ public void updateFaultItem(final String name, final long currentLatency, final long notAvailableDuration, +++ final boolean reachable) { ++ FaultItem old = this.faultItemTable.get(name); ++ if (null == old) { ++ final FaultItem faultItem = new FaultItem(name); ++ faultItem.setCurrentLatency(currentLatency); ++- faultItem.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); ++- +++ faultItem.updateNotAvailableDuration(notAvailableDuration); +++ faultItem.setReachable(reachable); ++ old = this.faultItemTable.putIfAbsent(name, faultItem); ++- if (old != null) { ++- old.setCurrentLatency(currentLatency); ++- old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); ++- } ++- } else { +++ } +++ +++ if (null != old) { ++ old.setCurrentLatency(currentLatency); ++- old.setStartTimestamp(System.currentTimeMillis() + notAvailableDuration); +++ old.updateNotAvailableDuration(notAvailableDuration); +++ old.setReachable(reachable); +++ } +++ +++ if (!reachable) { +++ log.info(name + " is unreachable, it will not be used until it's reachable"); ++ } ++ } ++ ++@@ -57,6 +124,14 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ return true; ++ } ++ +++ public boolean isReachable(final String name) { +++ final FaultItem faultItem = this.faultItemTable.get(name); +++ if (faultItem != null) { +++ return faultItem.isReachable(); +++ } +++ return true; +++ } +++ ++ @Override ++ public void remove(final String name) { ++ this.faultItemTable.remove(name); ++@@ -65,68 +140,98 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ @Override ++ public String pickOneAtLeast() { ++ final Enumeration elements = this.faultItemTable.elements(); ++- List tmpList = new LinkedList<>(); +++ List tmpList = new LinkedList(); ++ while (elements.hasMoreElements()) { ++ final FaultItem faultItem = elements.nextElement(); ++ tmpList.add(faultItem); ++ } +++ ++ if (!tmpList.isEmpty()) { ++- Collections.sort(tmpList); ++- final int half = tmpList.size() / 2; ++- if (half <= 0) { ++- return tmpList.get(0).getName(); ++- } else { ++- final int i = this.randomItem.incrementAndGet() % half; ++- return tmpList.get(i).getName(); +++ Collections.shuffle(tmpList); +++ for (FaultItem faultItem : tmpList) { +++ if (faultItem.reachableFlag) { +++ return faultItem.name; +++ } ++ } ++ } +++ ++ return null; ++ } ++ ++ @Override ++ public String toString() { ++ return "LatencyFaultToleranceImpl{" + ++- "faultItemTable=" + faultItemTable + ++- ", whichItemWorst=" + randomItem + ++- '}'; +++ "faultItemTable=" + faultItemTable + +++ ", whichItemWorst=" + whichItemWorst + +++ '}'; +++ } +++ +++ public void setDetectTimeout(final int detectTimeout) { +++ this.detectTimeout = detectTimeout; ++ } ++ ++- class FaultItem implements Comparable { +++ public void setDetectInterval(final int detectInterval) { +++ this.detectInterval = detectInterval; +++ } +++ +++ public class FaultItem implements Comparable { ++ private final String name; ++ private volatile long currentLatency; ++ private volatile long startTimestamp; +++ private volatile long checkStamp; +++ private volatile boolean reachableFlag; ++ ++ public FaultItem(final String name) { ++ this.name = name; ++ } ++ +++ public void updateNotAvailableDuration(long notAvailableDuration) { +++ if (notAvailableDuration > 0 && System.currentTimeMillis() + notAvailableDuration > this.startTimestamp) { +++ this.startTimestamp = System.currentTimeMillis() + notAvailableDuration; +++ log.info(name + " will be isolated for " + notAvailableDuration + " ms."); +++ } +++ } +++ ++ @Override ++ public int compareTo(final FaultItem other) { ++ if (this.isAvailable() != other.isAvailable()) { ++- if (this.isAvailable()) +++ if (this.isAvailable()) { ++ return -1; +++ } ++ ++- if (other.isAvailable()) +++ if (other.isAvailable()) { ++ return 1; +++ } ++ } ++ ++- if (this.currentLatency < other.currentLatency) +++ if (this.currentLatency < other.currentLatency) { ++ return -1; ++- else if (this.currentLatency > other.currentLatency) { +++ } else if (this.currentLatency > other.currentLatency) { ++ return 1; ++ } ++ ++- if (this.startTimestamp < other.startTimestamp) +++ if (this.startTimestamp < other.startTimestamp) { ++ return -1; ++- else if (this.startTimestamp > other.startTimestamp) { +++ } else if (this.startTimestamp > other.startTimestamp) { ++ return 1; ++ } ++- ++ return 0; ++ } ++ +++ public void setReachable(boolean reachableFlag) { +++ this.reachableFlag = reachableFlag; +++ } +++ +++ public void setCheckStamp(long checkStamp) { +++ this.checkStamp = checkStamp; +++ } +++ ++ public boolean isAvailable() { ++- return (System.currentTimeMillis() - startTimestamp) >= 0; +++ return reachableFlag && System.currentTimeMillis() >= startTimestamp; +++ } +++ +++ public boolean isReachable() { +++ return reachableFlag; ++ } ++ ++ @Override ++@@ -139,28 +244,32 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ ++ @Override ++ public boolean equals(final Object o) { ++- if (this == o) +++ if (this == o) { ++ return true; ++- if (!(o instanceof FaultItem)) +++ } +++ if (!(o instanceof FaultItem)) { ++ return false; +++ } ++ ++ final FaultItem faultItem = (FaultItem) o; ++ ++- if (getCurrentLatency() != faultItem.getCurrentLatency()) +++ if (getCurrentLatency() != faultItem.getCurrentLatency()) { ++ return false; ++- if (getStartTimestamp() != faultItem.getStartTimestamp()) +++ } +++ if (getStartTimestamp() != faultItem.getStartTimestamp()) { ++ return false; +++ } ++ return getName() != null ? getName().equals(faultItem.getName()) : faultItem.getName() == null; ++- ++ } ++ ++ @Override ++ public String toString() { ++ return "FaultItem{" + ++- "name='" + name + '\'' + ++- ", currentLatency=" + currentLatency + ++- ", startTimestamp=" + startTimestamp + ++- '}'; +++ "name='" + name + '\'' + +++ ", currentLatency=" + currentLatency + +++ ", startTimestamp=" + startTimestamp + +++ ", reachableFlag=" + reachableFlag + +++ '}'; ++ } ++ ++ public String getName() { ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++index 1e1953fad..c01490784 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++@@ -17,25 +17,86 @@ ++ ++ package org.apache.rocketmq.client.latency; ++ ++-import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +++import org.apache.rocketmq.client.impl.producer.TopicPublishInfo.QueueFilter; ++ import org.apache.rocketmq.common.message.MessageQueue; ++-import org.apache.rocketmq.logging.org.slf4j.Logger; ++-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ public class MQFaultStrategy { ++- private final static Logger log = LoggerFactory.getLogger(MQFaultStrategy.class); ++- private final LatencyFaultTolerance latencyFaultTolerance = new LatencyFaultToleranceImpl(); +++ private LatencyFaultTolerance latencyFaultTolerance; +++ private boolean sendLatencyFaultEnable; +++ private boolean startDetectorEnable; +++ private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L}; +++ private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L}; ++ ++- private boolean sendLatencyFaultEnable = false; +++ public static class BrokerFilter implements QueueFilter { +++ private String lastBrokerName; +++ +++ public void setLastBrokerName(String lastBrokerName) { +++ this.lastBrokerName = lastBrokerName; +++ } +++ +++ @Override public boolean filter(MessageQueue mq) { +++ if (lastBrokerName != null) { +++ return !mq.getBrokerName().equals(lastBrokerName); +++ } +++ return true; +++ } +++ } +++ +++ private ThreadLocal threadBrokerFilter = new ThreadLocal() { +++ @Override protected BrokerFilter initialValue() { +++ return new BrokerFilter(); +++ } +++ }; +++ +++ private QueueFilter reachableFilter = new QueueFilter() { +++ @Override public boolean filter(MessageQueue mq) { +++ return latencyFaultTolerance.isReachable(mq.getBrokerName()); +++ } +++ }; +++ +++ private QueueFilter availableFilter = new QueueFilter() { +++ @Override public boolean filter(MessageQueue mq) { +++ return latencyFaultTolerance.isAvailable(mq.getBrokerName()); +++ } +++ }; +++ +++ +++ public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) { +++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); +++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); +++ this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector); +++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); +++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); +++ } +++ +++ // For unit test. +++ public MQFaultStrategy(ClientConfig cc, LatencyFaultTolerance tolerance) { +++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); +++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); +++ this.latencyFaultTolerance = tolerance; +++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); +++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); +++ } ++ ++- private long[] latencyMax = {50L, 100L, 550L, 1000L, 2000L, 3000L, 15000L}; ++- private long[] notAvailableDuration = {0L, 0L, 30000L, 60000L, 120000L, 180000L, 600000L}; ++ ++ public long[] getNotAvailableDuration() { ++ return notAvailableDuration; ++ } ++ +++ public QueueFilter getAvailableFilter() { +++ return availableFilter; +++ } +++ +++ public QueueFilter getReachableFilter() { +++ return reachableFilter; +++ } +++ +++ public ThreadLocal getThreadBrokerFilter() { +++ return threadBrokerFilter; +++ } +++ ++ public void setNotAvailableDuration(final long[] notAvailableDuration) { ++ this.notAvailableDuration = notAvailableDuration; ++ } ++@@ -56,51 +117,69 @@ public class MQFaultStrategy { ++ this.sendLatencyFaultEnable = sendLatencyFaultEnable; ++ } ++ ++- public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName) { +++ public boolean isStartDetectorEnable() { +++ return startDetectorEnable; +++ } +++ +++ public void setStartDetectorEnable(boolean startDetectorEnable) { +++ this.startDetectorEnable = startDetectorEnable; +++ } +++ +++ public void startDetector() { +++ // user should start the detector +++ // and the thread should not be in running state. +++ if (this.sendLatencyFaultEnable && this.startDetectorEnable) { +++ // start the detector. +++ this.latencyFaultTolerance.startDetector(); +++ } +++ } +++ +++ public void shutdown() { +++ if (this.sendLatencyFaultEnable && this.startDetectorEnable) { +++ this.latencyFaultTolerance.shutdown(); +++ } +++ } +++ +++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { +++ BrokerFilter brokerFilter = threadBrokerFilter.get(); +++ brokerFilter.setLastBrokerName(lastBrokerName); ++ if (this.sendLatencyFaultEnable) { ++- try { ++- int index = tpInfo.getSendWhichQueue().incrementAndGet(); ++- for (int i = 0; i < tpInfo.getMessageQueueList().size(); i++) { ++- int pos = index++ % tpInfo.getMessageQueueList().size(); ++- MessageQueue mq = tpInfo.getMessageQueueList().get(pos); ++- if (!StringUtils.equals(lastBrokerName, mq.getBrokerName()) && latencyFaultTolerance.isAvailable(mq.getBrokerName())) { ++- return mq; ++- } ++- } ++- ++- final String notBestBroker = latencyFaultTolerance.pickOneAtLeast(); ++- int writeQueueNums = tpInfo.getWriteQueueNumsByBroker(notBestBroker); ++- if (writeQueueNums > 0) { ++- final MessageQueue mq = tpInfo.selectOneMessageQueue(); ++- if (notBestBroker != null) { ++- mq.setBrokerName(notBestBroker); ++- mq.setQueueId(tpInfo.getSendWhichQueue().incrementAndGet() % writeQueueNums); ++- } ++- return mq; ++- } else { ++- latencyFaultTolerance.remove(notBestBroker); ++- } ++- } catch (Exception e) { ++- log.error("Error occurred when selecting message queue", e); +++ if (resetIndex) { +++ tpInfo.resetIndex(); +++ } +++ MessageQueue mq = tpInfo.selectOneMessageQueue(availableFilter, brokerFilter); +++ if (mq != null) { +++ return mq; +++ } +++ +++ mq = tpInfo.selectOneMessageQueue(reachableFilter, brokerFilter); +++ if (mq != null) { +++ return mq; ++ } ++ ++ return tpInfo.selectOneMessageQueue(); ++ } ++ ++- return tpInfo.selectOneMessageQueue(lastBrokerName); +++ MessageQueue mq = tpInfo.selectOneMessageQueue(brokerFilter); +++ if (mq != null) { +++ return mq; +++ } +++ return tpInfo.selectOneMessageQueue(); ++ } ++ ++- public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation) { +++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, +++ final boolean reachable) { ++ if (this.sendLatencyFaultEnable) { ++- long duration = computeNotAvailableDuration(isolation ? 30000 : currentLatency); ++- this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration); +++ long duration = computeNotAvailableDuration(isolation ? 10000 : currentLatency); +++ this.latencyFaultTolerance.updateFaultItem(brokerName, currentLatency, duration, reachable); ++ } ++ } ++ ++ private long computeNotAvailableDuration(final long currentLatency) { ++ for (int i = latencyMax.length - 1; i >= 0; i--) { ++- if (currentLatency >= latencyMax[i]) +++ if (currentLatency >= latencyMax[i]) { ++ return this.notAvailableDuration[i]; +++ } ++ } ++ ++ return 0; ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java ++similarity index 65% ++rename from remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java ++rename to client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java ++index 6aa547047..1c29ba334 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/body/GetRemoteClientConfigBody.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/Resolver.java ++@@ -14,20 +14,9 @@ ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++-package org.apache.rocketmq.remoting.protocol.body; +++package org.apache.rocketmq.client.latency; ++ ++-import java.util.ArrayList; ++-import java.util.List; ++-import org.apache.rocketmq.remoting.protocol.RemotingSerializable; +++public interface Resolver { ++ ++-public class GetRemoteClientConfigBody extends RemotingSerializable { ++- private List keys = new ArrayList<>(); ++- ++- public List getKeys() { ++- return keys; ++- } ++- ++- public void setKeys(List keys) { ++- this.keys = keys; ++- } +++ String resolve(String name); ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java ++new file mode 100644 ++index 000000000..c6ffbad1c ++--- /dev/null +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/ServiceDetector.java ++@@ -0,0 +1,30 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.client.latency; +++ +++/** +++ * Detect whether the remote service state is normal. +++ */ +++public interface ServiceDetector { +++ +++ /** +++ * Check if the remote service is normal. +++ * @param endpoint Service endpoint to check against +++ * @return true if the service is back to normal; false otherwise. +++ */ +++ boolean detect(String endpoint, long timeoutMillis); +++} ++diff --git a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java ++index 86690e40b..42ccdae5a 100644 ++--- a/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java +++++ b/client/src/test/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImplTest.java ++@@ -16,11 +16,14 @@ ++ */ ++ package org.apache.rocketmq.client.latency; ++ ++-import java.util.concurrent.TimeUnit; +++import org.awaitility.core.ThrowingRunnable; ++ import org.junit.Before; ++ import org.junit.Test; ++ +++import java.util.concurrent.TimeUnit; +++ ++ import static org.assertj.core.api.Assertions.assertThat; +++import static org.awaitility.Awaitility.await; ++ ++ public class LatencyFaultToleranceImplTest { ++ private LatencyFaultTolerance latencyFaultTolerance; ++@@ -29,28 +32,31 @@ public class LatencyFaultToleranceImplTest { ++ ++ @Before ++ public void init() { ++- latencyFaultTolerance = new LatencyFaultToleranceImpl(); +++ latencyFaultTolerance = new LatencyFaultToleranceImpl(null, null); ++ } ++ ++ @Test ++ public void testUpdateFaultItem() throws Exception { ++- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000); +++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true); ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); ++ assertThat(latencyFaultTolerance.isAvailable(anotherBrokerName)).isTrue(); ++ } ++ ++ @Test ++ public void testIsAvailable() throws Exception { ++- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50); +++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 50, true); ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); ++ ++- TimeUnit.MILLISECONDS.sleep(70); ++- assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); +++ await().atMost(500, TimeUnit.MILLISECONDS).untilAsserted(new ThrowingRunnable() { +++ @Override public void run() throws Throwable { +++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); +++ } +++ }); ++ } ++ ++ @Test ++ public void testRemove() throws Exception { ++- latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000); +++ latencyFaultTolerance.updateFaultItem(brokerName, 3000, 3000, true); ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isFalse(); ++ latencyFaultTolerance.remove(brokerName); ++ assertThat(latencyFaultTolerance.isAvailable(brokerName)).isTrue(); ++@@ -58,10 +64,20 @@ public class LatencyFaultToleranceImplTest { ++ ++ @Test ++ public void testPickOneAtLeast() throws Exception { ++- latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000); +++ latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true); ++ assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); ++ ++- latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000); ++- assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); +++ // Bad case, since pickOneAtLeast's behavior becomes random +++ // latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, "127.0.0.1:12011", true); +++ // assertThat(latencyFaultTolerance.pickOneAtLeast()).isEqualTo(brokerName); +++ } +++ +++ @Test +++ public void testIsReachable() throws Exception { +++ latencyFaultTolerance.updateFaultItem(brokerName, 1000, 3000, true); +++ assertThat(latencyFaultTolerance.isReachable(brokerName)).isEqualTo(true); +++ +++ latencyFaultTolerance.updateFaultItem(anotherBrokerName, 1001, 3000, false); +++ assertThat(latencyFaultTolerance.isReachable(anotherBrokerName)).isEqualTo(false); ++ } ++ } ++\ No newline at end of file ++diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java ++index fada0efd7..485b95c42 100644 ++--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java +++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/processor/DefaultRequestProcessor.java ++@@ -41,7 +41,6 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.apache.rocketmq.remoting.protocol.body.BrokerMemberGroup; ++ import org.apache.rocketmq.remoting.protocol.body.GetBrokerMemberGroupResponseBody; ++-import org.apache.rocketmq.remoting.protocol.body.GetRemoteClientConfigBody; ++ import org.apache.rocketmq.remoting.protocol.body.RegisterBrokerBody; ++ import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; ++ import org.apache.rocketmq.remoting.protocol.body.TopicList; ++@@ -132,8 +131,6 @@ public class DefaultRequestProcessor implements NettyRequestProcessor { ++ return this.updateConfig(ctx, request); ++ case RequestCode.GET_NAMESRV_CONFIG: ++ return this.getConfig(ctx, request); ++- case RequestCode.GET_CLIENT_CONFIG: ++- return this.getClientConfigs(ctx, request); ++ default: ++ String error = " request type " + request.getCode() + " not supported"; ++ return RemotingCommand.createResponseCommand(RemotingSysResponseCode.REQUEST_CODE_NOT_SUPPORTED, error); ++@@ -661,25 +658,4 @@ public class DefaultRequestProcessor implements NettyRequestProcessor { ++ return response; ++ } ++ ++- private RemotingCommand getClientConfigs(ChannelHandlerContext ctx, RemotingCommand request) { ++- final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- final GetRemoteClientConfigBody body = GetRemoteClientConfigBody.decode(request.getBody(), GetRemoteClientConfigBody.class); ++- ++- String content = this.namesrvController.getConfiguration().getClientConfigsFormatString(body.getKeys()); ++- if (StringUtils.isNotBlank(content)) { ++- try { ++- response.setBody(content.getBytes(MixAll.DEFAULT_CHARSET)); ++- } catch (UnsupportedEncodingException e) { ++- log.error("getConfig error, ", e); ++- response.setCode(ResponseCode.SYSTEM_ERROR); ++- response.setRemark("UnsupportedEncodingException " + e); ++- return response; ++- } ++- } ++- ++- response.setCode(ResponseCode.SUCCESS); ++- response.setRemark(null); ++- return response; ++- } ++- ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index 2994893d7..b2478fec3 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -232,6 +232,12 @@ public class ProxyConfig implements ConfigFile { ++ private String remotingAccessAddr = ""; ++ private int remotingListenPort = 8080; ++ +++ // related to proxy's send strategy in cluster mode. +++ private boolean sendLatencyEnable = false; +++ private boolean startDetectorEnable = false; +++ private int detectTimeout = 200; +++ private int detectInterval = 2 * 1000; +++ ++ private int remotingHeartbeatThreadPoolNums = 2 * PROCESSOR_NUMBER; ++ private int remotingTopicRouteThreadPoolNums = 2 * PROCESSOR_NUMBER; ++ private int remotingSendMessageThreadPoolNums = 4 * PROCESSOR_NUMBER; ++@@ -1409,6 +1415,46 @@ public class ProxyConfig implements ConfigFile { ++ this.remotingWaitTimeMillsInDefaultQueue = remotingWaitTimeMillsInDefaultQueue; ++ } ++ +++ public boolean isSendLatencyEnable() { +++ return sendLatencyEnable; +++ } +++ +++ public boolean isStartDetectorEnable() { +++ return startDetectorEnable; +++ } +++ +++ public void setStartDetectorEnable(boolean startDetectorEnable) { +++ this.startDetectorEnable = startDetectorEnable; +++ } +++ +++ public void setSendLatencyEnable(boolean sendLatencyEnable) { +++ this.sendLatencyEnable = sendLatencyEnable; +++ } +++ +++ public boolean getStartDetectorEnable() { +++ return this.startDetectorEnable; +++ } +++ +++ public boolean getSendLatencyEnable() { +++ return this.sendLatencyEnable; +++ } +++ +++ public int getDetectTimeout() { +++ return detectTimeout; +++ } +++ +++ public void setDetectTimeout(int detectTimeout) { +++ this.detectTimeout = detectTimeout; +++ } +++ +++ public int getDetectInterval() { +++ return detectInterval; +++ } +++ +++ public void setDetectInterval(int detectInterval) { +++ this.detectInterval = detectInterval; +++ } +++ ++ public boolean isEnableBatchAck() { ++ return enableBatchAck; ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java ++index 6146c80cd..f670df205 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivity.java ++@@ -382,7 +382,7 @@ public class SendMessageActivity extends AbstractMessingActivity { ++ int bucket = Hashing.consistentHash(shardingKey.hashCode(), writeQueues.size()); ++ targetMessageQueue = writeQueues.get(bucket); ++ } else { ++- targetMessageQueue = messageQueueView.getWriteSelector().selectOne(false); +++ targetMessageQueue = messageQueueView.getWriteSelector().selectOneByPipeline(false); ++ } ++ return targetMessageQueue; ++ } catch (Exception e) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java ++index 0d0c62168..a80f6df0b 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/processor/ProducerProcessor.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.proxy.processor; ++ import java.util.List; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutorService; +++ ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.client.producer.SendStatus; ++@@ -66,6 +67,8 @@ public class ProducerProcessor extends AbstractProcessor { ++ public CompletableFuture> sendMessage(ProxyContext ctx, QueueSelector queueSelector, ++ String producerGroup, int sysFlag, List messageList, long timeoutMillis) { ++ CompletableFuture> future = new CompletableFuture<>(); +++ long beginTimestampFirst = System.currentTimeMillis(); +++ AddressableMessageQueue messageQueue = null; ++ try { ++ Message message = messageList.get(0); ++ String topic = message.getTopic(); ++@@ -79,7 +82,7 @@ public class ProducerProcessor extends AbstractProcessor { ++ } ++ } ++ } ++- AddressableMessageQueue messageQueue = queueSelector.select(ctx, +++ messageQueue = queueSelector.select(ctx, ++ this.serviceManager.getTopicRouteService().getCurrentMessageQueueView(ctx, topic)); ++ if (messageQueue == null) { ++ throw new ProxyException(ProxyExceptionCode.FORBIDDEN, "no writable queue"); ++@@ -90,6 +93,7 @@ public class ProducerProcessor extends AbstractProcessor { ++ } ++ SendMessageRequestHeader requestHeader = buildSendMessageRequestHeader(messageList, producerGroup, sysFlag, messageQueue.getQueueId()); ++ +++ AddressableMessageQueue finalMessageQueue = messageQueue; ++ future = this.serviceManager.getMessageService().sendMessage( ++ ctx, ++ messageQueue, ++@@ -102,11 +106,19 @@ public class ProducerProcessor extends AbstractProcessor { ++ if (SendStatus.SEND_OK.equals(sendResult.getSendStatus()) && ++ tranType == MessageSysFlag.TRANSACTION_PREPARED_TYPE && ++ StringUtils.isNotBlank(sendResult.getTransactionId())) { ++- fillTransactionData(ctx, producerGroup, messageQueue, sendResult, messageList); +++ fillTransactionData(ctx, producerGroup, finalMessageQueue, sendResult, messageList); ++ } ++ } ++ return sendResultList; ++- }, this.executor); +++ }, this.executor) +++ .whenComplete((result, exception) -> { +++ long endTimestamp = System.currentTimeMillis(); +++ if (exception != null) { +++ this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(), endTimestamp - beginTimestampFirst, true, false); +++ } else { +++ this.serviceManager.getTopicRouteService().updateFaultItem(finalMessageQueue.getBrokerName(),endTimestamp - beginTimestampFirst, false, true); +++ } +++ }); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java ++index d67b68f38..aced15cee 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/LocalTopicRouteService.java ++@@ -54,7 +54,7 @@ public class LocalTopicRouteService extends TopicRouteService { ++ @Override ++ public MessageQueueView getCurrentMessageQueueView(ProxyContext ctx, String topic) throws Exception { ++ TopicConfig topicConfig = this.brokerController.getTopicConfigManager().getTopicConfigTable().get(topic); ++- return new MessageQueueView(topic, toTopicRouteData(topicConfig)); +++ return new MessageQueueView(topic, toTopicRouteData(topicConfig), null); ++ } ++ ++ @Override ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java ++index 85cd18d45..f25fb907e 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelector.java ++@@ -17,6 +17,7 @@ ++ package org.apache.rocketmq.proxy.service.route; ++ ++ import com.google.common.base.MoreObjects; +++import com.google.common.base.Preconditions; ++ import com.google.common.math.IntMath; ++ import java.util.ArrayList; ++ import java.util.Collections; ++@@ -30,6 +31,8 @@ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.stream.Collectors; ++ import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.client.impl.producer.TopicPublishInfo; +++import org.apache.rocketmq.client.latency.MQFaultStrategy; ++ import org.apache.rocketmq.common.constant.PermName; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.remoting.protocol.route.QueueData; ++@@ -44,8 +47,9 @@ public class MessageQueueSelector { ++ private final Map brokerNameQueueMap = new ConcurrentHashMap<>(); ++ private final AtomicInteger queueIndex; ++ private final AtomicInteger brokerIndex; +++ private MQFaultStrategy mqFaultStrategy; ++ ++- public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, boolean read) { +++ public MessageQueueSelector(TopicRouteWrapper topicRouteWrapper, MQFaultStrategy mqFaultStrategy, boolean read) { ++ if (read) { ++ this.queues.addAll(buildRead(topicRouteWrapper)); ++ } else { ++@@ -55,6 +59,7 @@ public class MessageQueueSelector { ++ Random random = new Random(); ++ this.queueIndex = new AtomicInteger(random.nextInt()); ++ this.brokerIndex = new AtomicInteger(random.nextInt()); +++ this.mqFaultStrategy = mqFaultStrategy; ++ } ++ ++ private static List buildRead(TopicRouteWrapper topicRoute) { ++@@ -154,6 +159,86 @@ public class MessageQueueSelector { ++ return selectOneByIndex(nextIndex, onlyBroker); ++ } ++ +++ public AddressableMessageQueue selectOneByPipeline(boolean onlyBroker) { +++ if (mqFaultStrategy != null && mqFaultStrategy.isSendLatencyFaultEnable()) { +++ List messageQueueList = null; +++ MessageQueue messageQueue = null; +++ if (onlyBroker) { +++ messageQueueList = transferAddressableQueues(brokerActingQueues); +++ } else { +++ messageQueueList = transferAddressableQueues(queues); +++ } +++ AddressableMessageQueue addressableMessageQueue = null; +++ +++ // use both available filter. +++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, +++ mqFaultStrategy.getAvailableFilter(), mqFaultStrategy.getReachableFilter()); +++ addressableMessageQueue = transferQueue2Addressable(messageQueue); +++ if (addressableMessageQueue != null) { +++ return addressableMessageQueue; +++ } +++ +++ // use available filter. +++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, +++ mqFaultStrategy.getAvailableFilter()); +++ addressableMessageQueue = transferQueue2Addressable(messageQueue); +++ if (addressableMessageQueue != null) { +++ return addressableMessageQueue; +++ } +++ +++ // no available filter, then use reachable filter. +++ messageQueue = selectOneMessageQueue(messageQueueList, onlyBroker ? brokerIndex : queueIndex, +++ mqFaultStrategy.getReachableFilter()); +++ addressableMessageQueue = transferQueue2Addressable(messageQueue); +++ if (addressableMessageQueue != null) { +++ return addressableMessageQueue; +++ } +++ } +++ +++ // SendLatency is not enabled, or no queue is selected, then select by index. +++ return selectOne(onlyBroker); +++ } +++ +++ private MessageQueue selectOneMessageQueue(List messageQueueList, AtomicInteger sendQueue, TopicPublishInfo.QueueFilter...filter) { +++ if (messageQueueList == null || messageQueueList.isEmpty()) { +++ return null; +++ } +++ if (filter != null && filter.length != 0) { +++ for (int i = 0; i < messageQueueList.size(); i++) { +++ int index = Math.abs(sendQueue.incrementAndGet() % messageQueueList.size()); +++ MessageQueue mq = messageQueueList.get(index); +++ boolean filterResult = true; +++ for (TopicPublishInfo.QueueFilter f: filter) { +++ Preconditions.checkNotNull(f); +++ filterResult &= f.filter(mq); +++ } +++ if (filterResult) { +++ return mq; +++ } +++ } +++ } +++ return null; +++ } +++ +++ public List transferAddressableQueues(List addressableMessageQueueList) { +++ if (addressableMessageQueueList == null) { +++ return null; +++ } +++ +++ return addressableMessageQueueList.stream() +++ .map(AddressableMessageQueue::getMessageQueue) +++ .collect(Collectors.toList()); +++ } +++ +++ private AddressableMessageQueue transferQueue2Addressable(MessageQueue messageQueue) { +++ for (AddressableMessageQueue amq: queues) { +++ if (amq.getMessageQueue().equals(messageQueue)) { +++ return amq; +++ } +++ } +++ return null; +++ } +++ ++ public AddressableMessageQueue selectNextOne(AddressableMessageQueue last) { ++ boolean onlyBroker = last.getQueueId() < 0; ++ AddressableMessageQueue newOne = last; ++@@ -190,6 +275,14 @@ public class MessageQueueSelector { ++ return brokerActingQueues; ++ } ++ +++ public MQFaultStrategy getMQFaultStrategy() { +++ return mqFaultStrategy; +++ } +++ +++ public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { +++ this.mqFaultStrategy = mqFaultStrategy; +++ } +++ ++ @Override ++ public boolean equals(Object o) { ++ if (this == o) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++index fe5387cfd..8b3c2f7c8 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++@@ -17,20 +17,22 @@ ++ package org.apache.rocketmq.proxy.service.route; ++ ++ import com.google.common.base.MoreObjects; +++import org.apache.rocketmq.client.latency.MQFaultStrategy; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++ ++ public class MessageQueueView { ++- public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData()); +++ public static final MessageQueueView WRAPPED_EMPTY_QUEUE = new MessageQueueView("", new TopicRouteData(), null); ++ ++ private final MessageQueueSelector readSelector; ++ private final MessageQueueSelector writeSelector; ++ private final TopicRouteWrapper topicRouteWrapper; +++ private MQFaultStrategy mqFaultStrategy; ++ ++- public MessageQueueView(String topic, TopicRouteData topicRouteData) { +++ public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) { ++ this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic); ++ ++- this.readSelector = new MessageQueueSelector(topicRouteWrapper, true); ++- this.writeSelector = new MessageQueueSelector(topicRouteWrapper, false); +++ this.readSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, true); +++ this.writeSelector = new MessageQueueSelector(topicRouteWrapper, mqFaultStrategy, false); ++ } ++ ++ public TopicRouteData getTopicRouteData() { ++@@ -65,4 +67,12 @@ public class MessageQueueView { ++ .add("topicRouteWrapper", topicRouteWrapper) ++ .toString(); ++ } +++ +++ public MQFaultStrategy getMQFaultStrategy() { +++ return mqFaultStrategy; +++ } +++ +++ public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { +++ this.mqFaultStrategy = mqFaultStrategy; +++ } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++index 84348adc3..74769a423 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++@@ -25,7 +25,13 @@ import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; +++ +++import com.google.common.base.Optional; +++import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.exception.MQClientException; +++import org.apache.rocketmq.client.latency.MQFaultStrategy; +++import org.apache.rocketmq.client.latency.Resolver; +++import org.apache.rocketmq.client.latency.ServiceDetector; ++ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++@@ -39,6 +45,7 @@ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; +++import org.apache.rocketmq.remoting.protocol.header.GetMaxOffsetRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++ import org.checkerframework.checker.nullness.qual.NonNull; ++ import org.checkerframework.checker.nullness.qual.Nullable; ++@@ -47,6 +54,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); ++ ++ private final MQClientAPIFactory mqClientAPIFactory; +++ private MQFaultStrategy mqFaultStrategy; ++ ++ protected final LoadingCache topicCache; ++ protected final ScheduledExecutorService scheduledExecutorService; ++@@ -97,15 +105,83 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ } ++ } ++ }); ++- +++ ServiceDetector serviceDetector = new ServiceDetector() { +++ @Override +++ public boolean detect(String endpoint, long timeoutMillis) { +++ Optional candidateTopic = pickTopic(); +++ if (!candidateTopic.isPresent()) { +++ return false; +++ } +++ try { +++ GetMaxOffsetRequestHeader requestHeader = new GetMaxOffsetRequestHeader(); +++ requestHeader.setTopic(candidateTopic.get()); +++ requestHeader.setQueueId(0); +++ Long maxOffset = mqClientAPIFactory.getClient().getMaxOffset(endpoint, requestHeader, timeoutMillis).get(); +++ return true; +++ } catch (Exception e) { +++ return false; +++ } +++ } +++ }; +++ mqFaultStrategy = new MQFaultStrategy(extractClientConfigFromProxyConfig(config), new Resolver() { +++ @Override +++ public String resolve(String name) { +++ try { +++ String brokerAddr = getBrokerAddr(null, name); +++ return brokerAddr; +++ } catch (Exception e) { +++ return null; +++ } +++ } +++ }, serviceDetector); ++ this.init(); ++ } ++ +++ // pickup one topic in the topic cache +++ private Optional pickTopic() { +++ if (topicCache.asMap().isEmpty()) { +++ return Optional.absent(); +++ } +++ return Optional.of(topicCache.asMap().keySet().iterator().next()); +++ } +++ ++ protected void init() { ++ this.appendShutdown(this.scheduledExecutorService::shutdown); ++ this.appendStartAndShutdown(this.mqClientAPIFactory); ++ } ++ +++ @Override +++ public void shutdown() throws Exception { +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ mqFaultStrategy.shutdown(); +++ } +++ } +++ +++ @Override +++ public void start() throws Exception { +++ if (this.mqFaultStrategy.isStartDetectorEnable()) { +++ this.mqFaultStrategy.startDetector(); +++ } +++ } +++ +++ public ClientConfig extractClientConfigFromProxyConfig(ProxyConfig proxyConfig) { +++ ClientConfig tempClientConfig = new ClientConfig(); +++ tempClientConfig.setSendLatencyEnable(proxyConfig.getSendLatencyEnable()); +++ tempClientConfig.setStartDetectorEnable(proxyConfig.getStartDetectorEnable()); +++ tempClientConfig.setDetectTimeout(proxyConfig.getDetectTimeout()); +++ tempClientConfig.setDetectInterval(proxyConfig.getDetectInterval()); +++ return tempClientConfig; +++ } +++ +++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, +++ boolean reachable) { +++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); +++ } +++ +++ public MQFaultStrategy getMqFaultStrategy() { +++ return this.mqFaultStrategy; +++ } +++ ++ public MessageQueueView getAllMessageQueueView(ProxyContext ctx, String topicName) throws Exception { ++ return getCacheMessageQueueWrapper(this.topicCache, topicName); ++ } ++@@ -136,7 +212,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ ++ protected MessageQueueView buildMessageQueueView(String topic, TopicRouteData topicRouteData) { ++ if (isTopicRouteValid(topicRouteData)) { ++- MessageQueueView tmp = new MessageQueueView(topic, topicRouteData); +++ MessageQueueView tmp = new MessageQueueView(topic, topicRouteData, TopicRouteService.this.getMqFaultStrategy()); ++ log.debug("load topic route from namesrv. topic: {}, queue: {}", topic, tmp); ++ return tmp; ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java ++index 7fd9a9ffd..77ae5e4d1 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/consumer/ReceiveMessageActivityTest.java ++@@ -93,7 +93,6 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { ++ pollTimeCaptor.capture(), anyInt(), any(), anyBoolean(), any(), isNull(), anyLong())) ++ .thenReturn(CompletableFuture.completedFuture(new PopResult(PopStatus.NO_NEW_MSG, Collections.emptyList()))); ++ ++- ++ ProxyContext context = createContext(); ++ context.setRemainingMs(1L); ++ this.receiveMessageActivity.receiveMessage( ++@@ -274,7 +273,7 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { ++ } ++ ++ @Test ++- public void testReceiveMessageQueueSelector() { +++ public void testReceiveMessageQueueSelector() throws Exception { ++ TopicRouteData topicRouteData = new TopicRouteData(); ++ List queueDatas = new ArrayList<>(); ++ for (int i = 0; i < 2; i++) { ++@@ -298,7 +297,7 @@ public class ReceiveMessageActivityTest extends BaseActivityTest { ++ } ++ topicRouteData.setBrokerDatas(brokerDatas); ++ ++- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); ++ ReceiveMessageActivity.ReceiveMessageQueueSelector selector = new ReceiveMessageActivity.ReceiveMessageQueueSelector(""); ++ ++ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java ++index 588423bb9..4882a5ed8 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/grpc/v2/producer/SendMessageActivityTest.java ++@@ -35,6 +35,8 @@ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.client.ClientConfig; +++import org.apache.rocketmq.client.latency.MQFaultStrategy; ++ import org.apache.rocketmq.client.producer.SendResult; ++ import org.apache.rocketmq.client.producer.SendStatus; ++ import org.apache.rocketmq.common.MixAll; ++@@ -49,6 +51,7 @@ import org.apache.rocketmq.proxy.grpc.v2.BaseActivityTest; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcProxyException; ++ import org.apache.rocketmq.proxy.service.route.AddressableMessageQueue; ++ import org.apache.rocketmq.proxy.service.route.MessageQueueView; +++import org.apache.rocketmq.proxy.service.route.TopicRouteService; ++ import org.apache.rocketmq.remoting.protocol.route.BrokerData; ++ import org.apache.rocketmq.remoting.protocol.route.QueueData; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++@@ -62,15 +65,19 @@ import static org.junit.Assert.assertThrows; ++ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyInt; ++ import static org.mockito.ArgumentMatchers.anyString; +++import static org.mockito.Mockito.mock; ++ import static org.mockito.Mockito.when; ++ ++ public class SendMessageActivityTest extends BaseActivityTest { ++ ++ protected static final String BROKER_NAME = "broker"; +++ protected static final String BROKER_NAME2 = "broker2"; ++ protected static final String CLUSTER_NAME = "cluster"; ++ protected static final String BROKER_ADDR = "127.0.0.1:10911"; +++ protected static final String BROKER_ADDR2 = "127.0.0.1:10912"; ++ private static final String TOPIC = "topic"; ++ private static final String CONSUMER_GROUP = "consumerGroup"; +++ MQFaultStrategy mqFaultStrategy; ++ ++ private SendMessageActivity sendMessageActivity; ++ ++@@ -262,7 +269,7 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ } ++ ++ @Test ++- public void testSendOrderMessageQueueSelector() { +++ public void testSendOrderMessageQueueSelector() throws Exception { ++ TopicRouteData topicRouteData = new TopicRouteData(); ++ QueueData queueData = new QueueData(); ++ BrokerData brokerData = new BrokerData(); ++@@ -277,7 +284,7 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ brokerData.setBrokerAddrs(brokerAddrs); ++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); ++ ++- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); ++ SendMessageActivity.SendMessageQueueSelector selector1 = new SendMessageActivity.SendMessageQueueSelector( ++ SendMessageRequest.newBuilder() ++ .addMessages(Message.newBuilder() ++@@ -288,6 +295,12 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ .build() ++ ); ++ +++ TopicRouteService topicRouteService = mock(TopicRouteService.class); +++ MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class); +++ when(topicRouteService.getAllMessageQueueView(any(), any())).thenReturn(messageQueueView); +++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); +++ when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false); +++ ++ SendMessageActivity.SendMessageQueueSelector selector2 = new SendMessageActivity.SendMessageQueueSelector( ++ SendMessageRequest.newBuilder() ++ .addMessages(Message.newBuilder() ++@@ -328,12 +341,17 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ brokerData.setBrokerAddrs(brokerAddrs); ++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); ++ ++- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); +++ ++ SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( ++ SendMessageRequest.newBuilder() ++ .addMessages(Message.newBuilder().build()) ++ .build() ++ ); +++ TopicRouteService topicRouteService = mock(TopicRouteService.class); +++ MQFaultStrategy mqFaultStrategy = mock(MQFaultStrategy.class); +++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); +++ when(mqFaultStrategy.isSendLatencyFaultEnable()).thenReturn(false); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy()); ++ ++ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); ++ AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); ++@@ -343,6 +361,45 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ assertNotEquals(firstSelect, secondSelect); ++ } ++ +++ @Test +++ public void testSendNormalMessageQueueSelectorPipeLine() throws Exception { +++ TopicRouteData topicRouteData = new TopicRouteData(); +++ int queueNums = 2; +++ +++ QueueData queueData = createQueueData(BROKER_NAME, queueNums); +++ QueueData queueData2 = createQueueData(BROKER_NAME2, queueNums); +++ topicRouteData.setQueueDatas(Lists.newArrayList(queueData,queueData2)); +++ +++ +++ BrokerData brokerData = createBrokerData(CLUSTER_NAME, BROKER_NAME, BROKER_ADDR); +++ BrokerData brokerData2 = createBrokerData(CLUSTER_NAME, BROKER_NAME2, BROKER_ADDR2); +++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData, brokerData2)); +++ +++ SendMessageActivity.SendMessageQueueSelector selector = new SendMessageActivity.SendMessageQueueSelector( +++ SendMessageRequest.newBuilder() +++ .addMessages(Message.newBuilder().build()) +++ .build() +++ ); +++ +++ ClientConfig cc = new ClientConfig(); +++ this.mqFaultStrategy = new MQFaultStrategy(cc, null, null); +++ mqFaultStrategy.setSendLatencyFaultEnable(true); +++ mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, true); +++ mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, false); +++ +++ TopicRouteService topicRouteService = mock(TopicRouteService.class); +++ when(topicRouteService.getMqFaultStrategy()).thenReturn(mqFaultStrategy); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, topicRouteService.getMqFaultStrategy()); +++ +++ +++ AddressableMessageQueue firstSelect = selector.select(ProxyContext.create(), messageQueueView); +++ assertEquals(firstSelect.getBrokerName(), BROKER_NAME2); +++ +++ mqFaultStrategy.updateFaultItem(BROKER_NAME2, 1000, true, false); +++ mqFaultStrategy.updateFaultItem(BROKER_NAME, 1000, true, true); +++ AddressableMessageQueue secondSelect = selector.select(ProxyContext.create(), messageQueueView); +++ assertEquals(secondSelect.getBrokerName(), BROKER_NAME); +++ } ++ @Test ++ public void testParameterValidate() { ++ // too large message body ++@@ -850,4 +907,23 @@ public class SendMessageActivityTest extends BaseActivityTest { ++ } ++ return sb.toString(); ++ } +++ +++ private static QueueData createQueueData(String brokerName, int writeQueueNums) { +++ QueueData queueData = new QueueData(); +++ queueData.setBrokerName(brokerName); +++ queueData.setWriteQueueNums(writeQueueNums); +++ queueData.setPerm(PermName.PERM_WRITE); +++ return queueData; +++ } +++ +++ private static BrokerData createBrokerData(String clusterName, String brokerName, String brokerAddrs) { +++ BrokerData brokerData = new BrokerData(); +++ brokerData.setCluster(clusterName); +++ brokerData.setBrokerName(brokerName); +++ HashMap brokerAddrsMap = new HashMap<>(); +++ brokerAddrsMap.put(MixAll.MASTER_ID, brokerAddrs); +++ brokerData.setBrokerAddrs(brokerAddrsMap); +++ +++ return brokerData; +++ } ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java ++index c97bd5a72..ca6fe909e 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/BaseServiceTest.java ++@@ -78,7 +78,7 @@ public class BaseServiceTest extends InitConfigTest { ++ topicRouteData.setBrokerDatas(Lists.newArrayList(brokerData)); ++ ++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(ERR_TOPIC))).thenThrow(new MQClientException(ResponseCode.TOPIC_NOT_EXIST, "")); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, topicRouteData, null)); ++ } ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java ++index e44ed28f4..d150f87c4 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/route/MessageQueueSelectorTest.java ++@@ -30,12 +30,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest { ++ public void testReadMessageQueue() { ++ queueData.setPerm(PermName.PERM_READ); ++ queueData.setReadQueueNums(0); ++- MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); +++ MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true); ++ assertTrue(messageQueueSelector.getQueues().isEmpty()); ++ ++ queueData.setPerm(PermName.PERM_READ); ++ queueData.setReadQueueNums(3); ++- messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), true); +++ messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, true); ++ assertEquals(3, messageQueueSelector.getQueues().size()); ++ assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); ++ for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { ++@@ -58,12 +58,12 @@ public class MessageQueueSelectorTest extends BaseServiceTest { ++ public void testWriteMessageQueue() { ++ queueData.setPerm(PermName.PERM_WRITE); ++ queueData.setReadQueueNums(0); ++- MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); +++ MessageQueueSelector messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false); ++ assertTrue(messageQueueSelector.getQueues().isEmpty()); ++ ++ queueData.setPerm(PermName.PERM_WRITE); ++ queueData.setWriteQueueNums(3); ++- messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), false); +++ messageQueueSelector = new MessageQueueSelector(new TopicRouteWrapper(topicRouteData, TOPIC), null, false); ++ assertEquals(3, messageQueueSelector.getQueues().size()); ++ assertEquals(1, messageQueueSelector.getBrokerActingQueues().size()); ++ for (int i = 0; i < messageQueueSelector.getQueues().size(); i++) { ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++index c67f4953d..43fba3d03 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++@@ -132,7 +132,7 @@ public class HeartbeatSyncerTest extends InitConfigTest { ++ brokerAddr.put(0L, "127.0.0.1:10911"); ++ brokerData.setBrokerAddrs(brokerAddr); ++ topicRouteData.getBrokerDatas().add(brokerData); ++- MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData); +++ MessageQueueView messageQueueView = new MessageQueueView("foo", topicRouteData, null); ++ when(this.topicRouteService.getAllMessageQueueView(any(), anyString())).thenReturn(messageQueueView); ++ } ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java ++index a0063544e..91af74cbe 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/transaction/ClusterTransactionServiceTest.java ++@@ -64,7 +64,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { ++ this.clusterTransactionService = new ClusterTransactionService(this.topicRouteService, this.producerManager, ++ this.mqClientAPIFactory); ++ ++- MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData); +++ MessageQueueView messageQueueView = new MessageQueueView(TOPIC, topicRouteData, null); ++ when(this.topicRouteService.getAllMessageQueueView(any(), anyString())) ++ .thenReturn(messageQueueView); ++ ++@@ -127,7 +127,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { ++ brokerData.setBrokerAddrs(brokerAddrs); ++ topicRouteData.getQueueDatas().add(queueData); ++ topicRouteData.getBrokerDatas().add(brokerData); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(TOPIC))).thenReturn(new MessageQueueView(TOPIC, topicRouteData, null)); ++ ++ TopicRouteData clusterTopicRouteData = new TopicRouteData(); ++ QueueData clusterQueueData = new QueueData(); ++@@ -141,7 +141,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { ++ brokerAddrs.put(MixAll.MASTER_ID, BROKER_ADDR); ++ clusterBrokerData.setBrokerAddrs(brokerAddrs); ++ clusterTopicRouteData.setBrokerDatas(Lists.newArrayList(clusterBrokerData)); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(CLUSTER_NAME))).thenReturn(new MessageQueueView(CLUSTER_NAME, clusterTopicRouteData, null)); ++ ++ TopicRouteData clusterTopicRouteData2 = new TopicRouteData(); ++ QueueData clusterQueueData2 = new QueueData(); ++@@ -155,7 +155,7 @@ public class ClusterTransactionServiceTest extends BaseServiceTest { ++ brokerAddrs.put(MixAll.MASTER_ID, brokerAddr2); ++ clusterBrokerData2.setBrokerAddrs(brokerAddrs); ++ clusterTopicRouteData2.setBrokerDatas(Lists.newArrayList(clusterBrokerData2)); ++- when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2)); +++ when(this.topicRouteService.getAllMessageQueueView(any(), eq(clusterName2))).thenReturn(new MessageQueueView(clusterName2, clusterTopicRouteData2, null)); ++ ++ ConfigurationManager.getProxyConfig().setTransactionHeartbeatBatchNum(2); ++ this.clusterTransactionService.start(); ++-- ++2.32.0.windows.2 ++ +diff --git a/patch015-backport-fix-some-bugs.patch b/patch015-backport-fix-some-bugs.patch +new file mode 100644 +index 000000000..11c10a6bd +--- /dev/null ++++ b/patch015-backport-fix-some-bugs.patch +@@ -0,0 +1,1894 @@ ++From bd0e9c09db9748f7f74a0c707579142dccf30afc Mon Sep 17 00:00:00 2001 ++From: PiteXChen <44110731+RapperCL@users.noreply.github.com> ++Date: Tue, 29 Aug 2023 19:39:27 +0800 ++Subject: [PATCH 1/7] [ISSUE #7111] Remove responseFuture from the ++ responseTable when exception occurs (#7112) ++ ++* remove responseFuture when exception ++* Empty-Commit ++ ++--------- ++Co-authored-by: chenyong152 ++--- ++ .../apache/rocketmq/remoting/netty/NettyRemotingAbstract.java | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++index 44d6a3df4..fce2de267 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++@@ -529,6 +529,7 @@ public abstract class NettyRemotingAbstract { ++ log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); ++ }); ++ } catch (Exception e) { +++ responseTable.remove(opaque); ++ responseFuture.release(); ++ log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); ++ throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); ++-- ++2.32.0.windows.2 ++ ++ ++From c78061bf6ca5f35452510ec4107c46735c51c316 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 30 Aug 2023 22:29:51 +0800 ++Subject: [PATCH 2/7] [ISSUE#7280] Fix and refactor handle commit exception in ++ tiered storage (#7281) ++ ++* refactor handle commit exception ++ ++* refactor handle commit exception ++ ++* fix handle commit exception ++--- ++ .../tieredstore/TieredDispatcher.java | 3 +- ++ .../tieredstore/TieredMessageFetcher.java | 57 ++-- ++ .../tieredstore/TieredMessageStore.java | 26 +- ++ .../provider/TieredFileSegment.java | 291 ++++++++++-------- ++ .../provider/TieredStoreProvider.java | 8 +- ++ .../provider/posix/PosixFileSegment.java | 4 +- ++ .../CommitLogInputStream.java} | 30 +- ++ .../FileSegmentInputStream.java} | 49 ++- ++ .../FileSegmentInputStreamFactory.java} | 26 +- ++ .../tieredstore/TieredMessageStoreTest.java | 14 +- ++ .../tieredstore/file/TieredFlatFileTest.java | 2 + ++ .../tieredstore/file/TieredIndexFileTest.java | 2 + ++ ...m.java => MockFileSegmentInputStream.java} | 8 +- ++ .../TieredFileSegmentInputStreamTest.java | 24 +- ++ .../provider/TieredFileSegmentTest.java | 89 +++++- ++ .../provider/memory/MemoryFileSegment.java | 27 +- ++ .../memory/MemoryFileSegmentWithoutCheck.java | 4 +- ++ 17 files changed, 427 insertions(+), 237 deletions(-) ++ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredCommitLogInputStream.java => stream/CommitLogInputStream.java} (88%) ++ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredFileSegmentInputStream.java => stream/FileSegmentInputStream.java} (77%) ++ rename tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/{inputstream/TieredFileSegmentInputStreamFactory.java => stream/FileSegmentInputStreamFactory.java} (54%) ++ rename tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/{MockTieredFileSegmentInputStream.java => MockFileSegmentInputStream.java} (82%) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++index 1746190cd..430c2b62e 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++@@ -318,8 +318,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ continue; ++ case FILE_CLOSED: ++ tieredFlatFileManager.destroyCompositeFile(flatFile.getMessageQueue()); ++- logger.info("TieredDispatcher#dispatchFlatFile: file has been close and destroy, " + ++- "topic: {}, queueId: {}", topic, queueId); +++ logger.info("File has been closed and destroy, topic: {}, queueId: {}", topic, queueId); ++ return; ++ default: ++ dispatchOffset--; ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++index 9a9a3e5a5..766ff64f6 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++@@ -273,15 +273,17 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ TieredStoreMetricsManager.cacheHit.add(resultWrapperList.size(), attributes); ++ } ++ ++- // if no cached message found and there is currently an inflight request, wait for the request to end before continuing +++ // If there are no messages in the cache and there are currently requests being pulled. +++ // We need to wait for the request to return before continuing. ++ if (resultWrapperList.isEmpty() && waitInflightRequest) { ++- CompletableFuture future = flatFile.getInflightRequest(group, queueOffset, maxCount) ++- .getFuture(queueOffset); +++ CompletableFuture future = +++ flatFile.getInflightRequest(group, queueOffset, maxCount).getFuture(queueOffset); ++ if (!future.isDone()) { ++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ // to prevent starvation issues, only allow waiting for inflight request once ++ return future.thenCompose(v -> { ++- LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: wait for inflight request cost: {}ms", stopwatch.elapsed(TimeUnit.MILLISECONDS)); +++ LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: wait for response cost: {}ms", +++ stopwatch.elapsed(TimeUnit.MILLISECONDS)); ++ return getMessageFromCacheAsync(flatFile, group, queueOffset, maxCount, false); ++ }); ++ } ++@@ -302,7 +304,8 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ ++ // if cache hit, result will be returned immediately and asynchronously prefetch messages for later requests ++ if (!resultWrapperList.isEmpty()) { ++- LOGGER.debug("TieredMessageFetcher#getMessageFromCacheAsync: cache hit: topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}", +++ LOGGER.debug("MessageFetcher#getMessageFromCacheAsync: cache hit: " + +++ "topic: {}, queue: {}, queue offset: {}, max message num: {}, cache hit num: {}", ++ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount, resultWrapperList.size()); ++ prefetchMessage(flatFile, group, maxCount, lastGetOffset + 1); ++ ++@@ -316,8 +319,10 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ } ++ ++ // if cache is miss, immediately pull messages ++- LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: topic: {}, queue: {}, queue offset: {}, max message num: {}", +++ LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + +++ "topic: {}, queue: {}, queue offset: {}, max message num: {}", ++ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount); +++ ++ CompletableFuture resultFuture; ++ synchronized (flatFile) { ++ int batchSize = maxCount * storeConfig.getReadAheadMinFactor(); ++@@ -453,42 +458,42 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ public CompletableFuture getMessageAsync( ++ String group, String topic, int queueId, long queueOffset, int maxCount, final MessageFilter messageFilter) { ++ +++ GetMessageResult result = new GetMessageResult(); ++ CompositeQueueFlatFile flatFile = flatFileManager.getFlatFile(new MessageQueue(topic, brokerName, queueId)); +++ ++ if (flatFile == null) { ++- GetMessageResult result = new GetMessageResult(); ++ result.setNextBeginOffset(queueOffset); ++ result.setStatus(GetMessageStatus.NO_MATCHED_LOGIC_QUEUE); ++ return CompletableFuture.completedFuture(result); ++ } ++ ++- GetMessageResult result = new GetMessageResult(); ++- long minQueueOffset = flatFile.getConsumeQueueMinOffset(); ++- long maxQueueOffset = flatFile.getConsumeQueueCommitOffset(); ++- result.setMinOffset(minQueueOffset); ++- result.setMaxOffset(maxQueueOffset); +++ // Max queue offset means next message put position +++ result.setMinOffset(flatFile.getConsumeQueueMinOffset()); +++ result.setMaxOffset(flatFile.getConsumeQueueCommitOffset()); +++ +++ // Fill result according file offset. +++ // Offset range | Result | Fix to +++ // (-oo, 0] | no message | current offset +++ // (0, min) | too small | min offset +++ // [min, max) | correct | +++ // [max, max] | overflow one | max offset +++ // (max, +oo) | overflow badly | max offset ++ ++- if (flatFile.getConsumeQueueCommitOffset() <= 0) { +++ if (result.getMaxOffset() <= 0) { ++ result.setStatus(GetMessageStatus.NO_MESSAGE_IN_QUEUE); ++ result.setNextBeginOffset(queueOffset); ++ return CompletableFuture.completedFuture(result); ++- } ++- ++- // request range | result ++- // (0, min) | too small ++- // [min, max) | correct ++- // [max, max] | overflow one ++- // (max, +oo) | overflow badly ++- if (queueOffset < minQueueOffset) { +++ } else if (queueOffset < result.getMinOffset()) { ++ result.setStatus(GetMessageStatus.OFFSET_TOO_SMALL); ++- result.setNextBeginOffset(flatFile.getConsumeQueueMinOffset()); +++ result.setNextBeginOffset(result.getMinOffset()); ++ return CompletableFuture.completedFuture(result); ++- } else if (queueOffset == maxQueueOffset) { +++ } else if (queueOffset == result.getMaxOffset()) { ++ result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_ONE); ++- result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset()); +++ result.setNextBeginOffset(result.getMaxOffset()); ++ return CompletableFuture.completedFuture(result); ++- } else if (queueOffset > maxQueueOffset) { +++ } else if (queueOffset > result.getMaxOffset()) { ++ result.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY); ++- result.setNextBeginOffset(flatFile.getConsumeQueueCommitOffset()); +++ result.setNextBeginOffset(result.getMaxOffset()); ++ return CompletableFuture.completedFuture(result); ++ } ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index 5240ac8e9..78e855f36 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -99,11 +99,11 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ return storeConfig; ++ } ++ ++- public boolean viaTieredStorage(String topic, int queueId, long offset) { ++- return viaTieredStorage(topic, queueId, offset, 1); +++ public boolean fetchFromCurrentStore(String topic, int queueId, long offset) { +++ return fetchFromCurrentStore(topic, queueId, offset, 1); ++ } ++ ++- public boolean viaTieredStorage(String topic, int queueId, long offset, int batchSize) { +++ public boolean fetchFromCurrentStore(String topic, int queueId, long offset, int batchSize) { ++ TieredMessageStoreConfig.TieredStorageLevel deepStorageLevel = storeConfig.getTieredStorageLevel(); ++ ++ if (deepStorageLevel.check(TieredMessageStoreConfig.TieredStorageLevel.FORCE)) { ++@@ -146,8 +146,10 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ public CompletableFuture getMessageAsync(String group, String topic, ++ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { ++ ++- if (!viaTieredStorage(topic, queueId, offset, maxMsgNums)) { ++- logger.trace("GetMessageAsync from next store topic: {}, queue: {}, offset: {}", topic, queueId, offset); +++ if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { +++ logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); +++ } else { +++ logger.trace("GetMessageAsync from next store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ } ++ ++@@ -168,14 +170,14 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ ++ if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) { ++ TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes); ++- logger.debug("GetMessageAsync not found then try back to next store, result: {}, " + +++ logger.debug("GetMessageAsync not found, then back to next store, result: {}, " + ++ "topic: {}, queue: {}, queue offset: {}, offset range: {}-{}", ++ result.getStatus(), topic, queueId, offset, result.getMinOffset(), result.getMaxOffset()); ++ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ } ++ } ++ ++- // system topic +++ // Fetch system topic data from the broker when using the force level. ++ if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { ++ if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { ++ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); ++@@ -198,7 +200,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ TieredStoreMetricsManager.messagesOutTotal.add(result.getMessageCount(), messagesOutAttributes); ++ } ++ ++- // fix min or max offset according next store +++ // Fix min or max offset according next store at last ++ long minOffsetInQueue = next.getMinOffsetInQueue(topic, queueId); ++ if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) { ++ result.setMinOffset(minOffsetInQueue); ++@@ -209,7 +211,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ } ++ return result; ++ }).exceptionally(e -> { ++- logger.error("GetMessageAsync from tiered store failed: ", e); +++ logger.error("GetMessageAsync from tiered store failed", e); ++ return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); ++ }); ++ } ++@@ -251,7 +253,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ .build(); ++ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); ++ if (time < 0) { ++- logger.debug("TieredMessageStore#getEarliestMessageTimeAsync: get earliest message time failed, try to get earliest message time from next store: topic: {}, queue: {}", +++ logger.debug("GetEarliestMessageTimeAsync failed, try to get earliest message time from next store: topic: {}, queue: {}", ++ topic, queueId); ++ return finalNextEarliestMessageTime != Long.MAX_VALUE ? finalNextEarliestMessageTime : -1; ++ } ++@@ -262,7 +264,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ @Override ++ public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, ++ long consumeQueueOffset) { ++- if (viaTieredStorage(topic, queueId, consumeQueueOffset)) { +++ if (fetchFromCurrentStore(topic, queueId, consumeQueueOffset)) { ++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ return fetcher.getMessageStoreTimeStampAsync(topic, queueId, consumeQueueOffset) ++ .thenApply(time -> { ++@@ -272,7 +274,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ .build(); ++ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); ++ if (time == -1) { ++- logger.debug("TieredMessageStore#getMessageStoreTimeStampAsync: get message time failed, try to get message time from next store: topic: {}, queue: {}, queue offset: {}", +++ logger.debug("GetEarliestMessageTimeAsync failed, try to get message time from next store, topic: {}, queue: {}, queue offset: {}", ++ topic, queueId, consumeQueueOffset); ++ return next.getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset); ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java ++index 5062c7d9e..32911a6e8 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegment.java ++@@ -16,14 +16,11 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.provider; ++ ++-import com.google.common.annotations.VisibleForTesting; ++-import com.google.common.base.Stopwatch; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; ++ import java.util.List; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.Semaphore; ++-import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.locks.ReentrantLock; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++@@ -35,8 +32,8 @@ import org.apache.rocketmq.tieredstore.exception.TieredStoreException; ++ import org.apache.rocketmq.tieredstore.file.TieredCommitLog; ++ import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue; ++ import org.apache.rocketmq.tieredstore.file.TieredIndexFile; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ ++@@ -50,22 +47,23 @@ public abstract class TieredFileSegment implements Comparable ++ protected final TieredMessageStoreConfig storeConfig; ++ ++ private final long maxSize; ++- private final ReentrantLock bufferLock; ++- private final Semaphore commitLock; +++ private final ReentrantLock bufferLock = new ReentrantLock(); +++ private final Semaphore commitLock = new Semaphore(1); ++ ++- private volatile boolean full; ++- private volatile boolean closed; +++ private volatile boolean full = false; +++ private volatile boolean closed = false; ++ ++- private volatile long minTimestamp; ++- private volatile long maxTimestamp; ++- private volatile long commitPosition; ++- private volatile long appendPosition; +++ private volatile long minTimestamp = Long.MAX_VALUE; +++ private volatile long maxTimestamp = Long.MAX_VALUE; +++ private volatile long commitPosition = 0L; +++ private volatile long appendPosition = 0L; ++ ++ // only used in commitLog ++- private volatile long dispatchCommitOffset = 0; +++ private volatile long dispatchCommitOffset = 0L; ++ ++ private ByteBuffer codaBuffer; ++- private List uploadBufferList = new ArrayList<>(); +++ private List bufferList = new ArrayList<>(); +++ private FileSegmentInputStream fileSegmentInputStream; ++ private CompletableFuture flightCommitRequest = CompletableFuture.completedFuture(false); ++ ++ public TieredFileSegment(TieredMessageStoreConfig storeConfig, ++@@ -75,21 +73,13 @@ public abstract class TieredFileSegment implements Comparable ++ this.fileType = fileType; ++ this.filePath = filePath; ++ this.baseOffset = baseOffset; ++- ++- this.closed = false; ++- this.bufferLock = new ReentrantLock(); ++- this.commitLock = new Semaphore(1); ++- ++- this.commitPosition = 0L; ++- this.appendPosition = 0L; ++- this.minTimestamp = Long.MAX_VALUE; ++- this.maxTimestamp = Long.MAX_VALUE; ++- ++- // The max segment size of a file is determined by the file type ++- this.maxSize = getMaxSizeAccordingFileType(storeConfig); +++ this.maxSize = getMaxSizeByFileType(); ++ } ++ ++- private long getMaxSizeAccordingFileType(TieredMessageStoreConfig storeConfig) { +++ /** +++ * The max segment size of a file is determined by the file type +++ */ +++ protected long getMaxSizeByFileType() { ++ switch (fileType) { ++ case COMMIT_LOG: ++ return storeConfig.getTieredStoreCommitLogMaxSize(); ++@@ -184,39 +174,23 @@ public abstract class TieredFileSegment implements Comparable ++ this.appendPosition = pos; ++ } ++ ++- private List rollingUploadBuffer() { +++ private List borrowBuffer() { ++ bufferLock.lock(); ++ try { ++- List tmp = uploadBufferList; ++- uploadBufferList = new ArrayList<>(); +++ List tmp = bufferList; +++ bufferList = new ArrayList<>(); ++ return tmp; ++ } finally { ++ bufferLock.unlock(); ++ } ++ } ++ ++- private void sendBackBuffer(TieredFileSegmentInputStream inputStream) { ++- bufferLock.lock(); ++- try { ++- List tmpBufferList = inputStream.getUploadBufferList(); ++- for (ByteBuffer buffer : tmpBufferList) { ++- buffer.rewind(); ++- } ++- tmpBufferList.addAll(uploadBufferList); ++- uploadBufferList = tmpBufferList; ++- if (inputStream.getCodaBuffer() != null) { ++- codaBuffer.rewind(); ++- } ++- } finally { ++- bufferLock.unlock(); ++- } ++- } ++- ++ @SuppressWarnings("NonAtomicOperationOnVolatileField") ++- public AppendResult append(ByteBuffer byteBuf, long timeStamp) { +++ public AppendResult append(ByteBuffer byteBuf, long timestamp) { ++ if (closed) { ++ return AppendResult.FILE_CLOSED; ++ } +++ ++ bufferLock.lock(); ++ try { ++ if (full || codaBuffer != null) { ++@@ -227,7 +201,8 @@ public abstract class TieredFileSegment implements Comparable ++ minTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_BEGIN_TIME_STAMP_POSITION); ++ maxTimestamp = byteBuf.getLong(TieredIndexFile.INDEX_FILE_HEADER_END_TIME_STAMP_POSITION); ++ appendPosition += byteBuf.remaining(); ++- uploadBufferList.add(byteBuf); +++ // IndexFile is large and not change after compaction, no need deep copy +++ bufferList.add(byteBuf); ++ setFull(); ++ return AppendResult.SUCCESS; ++ } ++@@ -236,23 +211,34 @@ public abstract class TieredFileSegment implements Comparable ++ setFull(); ++ return AppendResult.FILE_FULL; ++ } ++- if (uploadBufferList.size() > storeConfig.getTieredStoreGroupCommitCount() +++ +++ if (bufferList.size() > storeConfig.getTieredStoreGroupCommitCount() ++ || appendPosition - commitPosition > storeConfig.getTieredStoreGroupCommitSize()) { ++ commitAsync(); ++ } ++- if (uploadBufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) { ++- logger.debug("TieredFileSegment#append: buffer full: file: {}, upload buffer size: {}", ++- getPath(), uploadBufferList.size()); +++ +++ if (bufferList.size() > storeConfig.getTieredStoreMaxGroupCommitCount()) { +++ logger.debug("File segment append buffer full, file: {}, buffer size: {}, pending bytes: {}", +++ getPath(), bufferList.size(), appendPosition - commitPosition); ++ return AppendResult.BUFFER_FULL; ++ } ++- if (timeStamp != Long.MAX_VALUE) { ++- maxTimestamp = timeStamp; +++ +++ if (timestamp != Long.MAX_VALUE) { +++ maxTimestamp = timestamp; ++ if (minTimestamp == Long.MAX_VALUE) { ++- minTimestamp = timeStamp; +++ minTimestamp = timestamp; ++ } ++ } +++ ++ appendPosition += byteBuf.remaining(); ++- uploadBufferList.add(byteBuf); +++ +++ // deep copy buffer +++ ByteBuffer byteBuffer = ByteBuffer.allocateDirect(byteBuf.remaining()); +++ byteBuffer.put(byteBuf); +++ byteBuffer.flip(); +++ byteBuf.rewind(); +++ +++ bufferList.add(byteBuffer); ++ return AppendResult.SUCCESS; ++ } finally { ++ bufferLock.unlock(); ++@@ -267,7 +253,6 @@ public abstract class TieredFileSegment implements Comparable ++ return appendPosition; ++ } ++ ++- @VisibleForTesting ++ public void setAppendPosition(long appendPosition) { ++ this.appendPosition = appendPosition; ++ } ++@@ -333,6 +318,8 @@ public abstract class TieredFileSegment implements Comparable ++ if (closed) { ++ return false; ++ } +++ // result is false when we send real commit request +++ // use join for wait flight request done ++ Boolean result = commitAsync().join(); ++ if (!result) { ++ result = flightCommitRequest.join(); ++@@ -340,92 +327,156 @@ public abstract class TieredFileSegment implements Comparable ++ return result; ++ } ++ +++ private void releaseCommitLock() { +++ if (commitLock.availablePermits() == 0) { +++ commitLock.release(); +++ } else { +++ logger.error("[Bug] FileSegmentCommitAsync, lock is already released: available permits: {}", +++ commitLock.availablePermits()); +++ } +++ } +++ +++ private void updateDispatchCommitOffset(List bufferList) { +++ if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) { +++ dispatchCommitOffset = +++ MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1)); +++ } +++ } +++ +++ /** +++ * @return false: commit, true: no commit operation +++ */ ++ @SuppressWarnings("NonAtomicOperationOnVolatileField") ++ public CompletableFuture commitAsync() { ++ if (closed) { ++ return CompletableFuture.completedFuture(false); ++ } ++- Stopwatch stopwatch = Stopwatch.createStarted(); +++ ++ if (!needCommit()) { ++ return CompletableFuture.completedFuture(true); ++ } ++- try { ++- int permits = commitLock.drainPermits(); ++- if (permits <= 0) { ++- return CompletableFuture.completedFuture(false); ++- } ++- } catch (Exception e) { +++ +++ if (commitLock.drainPermits() <= 0) { ++ return CompletableFuture.completedFuture(false); ++ } ++- List bufferList = rollingUploadBuffer(); ++- int bufferSize = 0; ++- for (ByteBuffer buffer : bufferList) { ++- bufferSize += buffer.remaining(); ++- } ++- if (codaBuffer != null) { ++- bufferSize += codaBuffer.remaining(); ++- } ++- if (bufferSize == 0) { ++- return CompletableFuture.completedFuture(true); ++- } ++- TieredFileSegmentInputStream inputStream = TieredFileSegmentInputStreamFactory.build( ++- fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize); ++- int finalBufferSize = bufferSize; +++ ++ try { ++- flightCommitRequest = commit0(inputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX) +++ if (fileSegmentInputStream != null) { +++ long fileSize = this.getSize(); +++ if (fileSize == -1L) { +++ logger.error("Get commit position error before commit, Commit: %d, Expect: %d, Current Max: %d, FileName: %s", +++ commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath()); +++ releaseCommitLock(); +++ return CompletableFuture.completedFuture(false); +++ } else { +++ if (correctPosition(fileSize, null)) { +++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); +++ fileSegmentInputStream = null; +++ } +++ } +++ } +++ +++ int bufferSize; +++ if (fileSegmentInputStream != null) { +++ bufferSize = fileSegmentInputStream.available(); +++ } else { +++ List bufferList = borrowBuffer(); +++ bufferSize = bufferList.stream().mapToInt(ByteBuffer::remaining).sum() +++ + (codaBuffer != null ? codaBuffer.remaining() : 0); +++ if (bufferSize == 0) { +++ releaseCommitLock(); +++ return CompletableFuture.completedFuture(true); +++ } +++ fileSegmentInputStream = FileSegmentInputStreamFactory.build( +++ fileType, baseOffset + commitPosition, bufferList, codaBuffer, bufferSize); +++ } +++ +++ return flightCommitRequest = this +++ .commit0(fileSegmentInputStream, commitPosition, bufferSize, fileType != FileSegmentType.INDEX) ++ .thenApply(result -> { ++ if (result) { ++- if (fileType == FileSegmentType.COMMIT_LOG && bufferList.size() > 0) { ++- dispatchCommitOffset = MessageBufferUtil.getQueueOffset(bufferList.get(bufferList.size() - 1)); ++- } ++- commitPosition += finalBufferSize; +++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); +++ commitPosition += bufferSize; +++ fileSegmentInputStream = null; ++ return true; ++- } ++- sendBackBuffer(inputStream); ++- return false; ++- }) ++- .exceptionally(e -> handleCommitException(inputStream, e)) ++- .whenComplete((result, e) -> { ++- if (commitLock.availablePermits() == 0) { ++- logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize); ++- commitLock.release(); ++ } else { ++- logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits()); +++ fileSegmentInputStream.rewind(); +++ return false; ++ } ++- }); ++- return flightCommitRequest; +++ }) +++ .exceptionally(this::handleCommitException) +++ .whenComplete((result, e) -> releaseCommitLock()); +++ ++ } catch (Exception e) { ++- handleCommitException(inputStream, e); ++- if (commitLock.availablePermits() == 0) { ++- logger.debug("TieredFileSegment#commitAsync: commit cost: {}ms, file: {}, item count: {}, buffer size: {}", stopwatch.elapsed(TimeUnit.MILLISECONDS), getPath(), bufferList.size(), finalBufferSize); ++- commitLock.release(); ++- } else { ++- logger.error("[Bug]TieredFileSegment#commitAsync: commit lock is already released: available permits: {}", commitLock.availablePermits()); ++- } +++ handleCommitException(e); +++ releaseCommitLock(); ++ } ++ return CompletableFuture.completedFuture(false); ++ } ++ ++- private boolean handleCommitException(TieredFileSegmentInputStream inputStream, Throwable e) { +++ private long getCorrectFileSize(Throwable throwable) { +++ if (throwable instanceof TieredStoreException) { +++ long fileSize = ((TieredStoreException) throwable).getPosition(); +++ if (fileSize > 0) { +++ return fileSize; +++ } +++ } +++ return getSize(); +++ } +++ +++ private boolean handleCommitException(Throwable e) { +++ // Get root cause here ++ Throwable cause = e.getCause() != null ? e.getCause() : e; ++- sendBackBuffer(inputStream); ++- long realSize = 0; ++- if (cause instanceof TieredStoreException && ((TieredStoreException) cause).getPosition() > 0) { ++- realSize = ((TieredStoreException) cause).getPosition(); +++ long fileSize = this.getCorrectFileSize(cause); +++ +++ if (fileSize == -1L) { +++ logger.error("Get commit position error, Commit: %d, Expect: %d, Current Max: %d, FileName: %s", +++ commitPosition, commitPosition + fileSegmentInputStream.getContentLength(), appendPosition, getPath()); +++ fileSegmentInputStream.rewind(); +++ return false; +++ } +++ +++ if (correctPosition(fileSize, cause)) { +++ updateDispatchCommitOffset(fileSegmentInputStream.getBufferList()); +++ fileSegmentInputStream = null; +++ return true; +++ } else { +++ fileSegmentInputStream.rewind(); +++ return false; ++ } ++- if (realSize <= 0) { ++- realSize = getSize(); +++ } +++ +++ /** +++ * return true to clear buffer +++ */ +++ private boolean correctPosition(long fileSize, Throwable throwable) { +++ +++ // Current we have three offsets here: commit offset, expect offset, file size. +++ // We guarantee that the commit offset is less than or equal to the expect offset. +++ // Max offset will increase because we can continuously put in new buffers +++ String handleInfo = throwable == null ? "before commit" : "after commit"; +++ long expectPosition = commitPosition + fileSegmentInputStream.getContentLength(); +++ +++ String offsetInfo = String.format("Correct Commit Position, %s, result=[{}], " + +++ "Commit: %d, Expect: %d, Current Max: %d, FileSize: %d, FileName: %s", +++ handleInfo, commitPosition, expectPosition, appendPosition, fileSize, this.getPath()); +++ +++ // We are believing that the file size returned by the server is correct, +++ // can reset the commit offset to the file size reported by the storage system. +++ if (fileSize == expectPosition) { +++ logger.info(offsetInfo, "Success", throwable); +++ commitPosition = fileSize; +++ return true; ++ } ++- if (realSize > 0 && realSize > commitPosition) { ++- logger.error("TieredFileSegment#handleCommitException: commit failed: file: {}, try to fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause); ++- // TODO check if this diff part is uploaded to backend storage ++- long diff = appendPosition - commitPosition; ++- commitPosition = realSize; ++- appendPosition = realSize + diff; ++- // TODO check if appendPosition is large than maxOffset ++- } else if (realSize < commitPosition) { ++- logger.error("[Bug]TieredFileSegment#handleCommitException: commit failed: file: {}, can not fix position: origin: {}, real: {}", getPath(), commitPosition, realSize, cause); +++ +++ if (fileSize < commitPosition) { +++ logger.error(offsetInfo, "FileSizeIncorrect", throwable); +++ } else if (fileSize == commitPosition) { +++ logger.warn(offsetInfo, "CommitFailed", throwable); +++ } else if (fileSize > commitPosition) { +++ logger.warn(offsetInfo, "PartialSuccess", throwable); ++ } +++ commitPosition = fileSize; ++ return false; ++ } ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java ++index 5a0ca25f5..0db3eaf8f 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreProvider.java ++@@ -18,7 +18,7 @@ package org.apache.rocketmq.tieredstore.provider; ++ ++ import java.nio.ByteBuffer; ++ import java.util.concurrent.CompletableFuture; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ ++ public interface TieredStoreProvider { ++ ++@@ -30,7 +30,9 @@ public interface TieredStoreProvider { ++ String getPath(); ++ ++ /** ++- * Get file size in backend file system +++ * Get the real length of the file. +++ * Return 0 if the file does not exist, +++ * Return -1 if system get size failed. ++ * ++ * @return file real size ++ */ ++@@ -71,5 +73,5 @@ public interface TieredStoreProvider { ++ * @param append try to append or create a new file ++ * @return put result, true if data successfully write; false otherwise ++ */ ++- CompletableFuture commit0(TieredFileSegmentInputStream inputStream,long position, int length, boolean append); +++ CompletableFuture commit0(FileSegmentInputStream inputStream,long position, int length, boolean append); ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java ++index 52be90b1d..7e949cb28 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/posix/PosixFileSegment.java ++@@ -36,7 +36,7 @@ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; ++ import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor; ++ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; ++ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ ++ import static org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant.LABEL_FILE_TYPE; ++@@ -184,7 +184,7 @@ public class PosixFileSegment extends TieredFileSegment { ++ ++ @Override ++ public CompletableFuture commit0( ++- TieredFileSegmentInputStream inputStream, long position, int length, boolean append) { +++ FileSegmentInputStream inputStream, long position, int length, boolean append) { ++ ++ Stopwatch stopwatch = Stopwatch.createStarted(); ++ AttributesBuilder attributesBuilder = newAttributesBuilder() ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java ++similarity index 88% ++rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java ++rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java ++index c70bb7656..13b6e0ef9 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredCommitLogInputStream.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/CommitLogInputStream.java ++@@ -15,7 +15,7 @@ ++ * limitations under the License. ++ */ ++ ++-package org.apache.rocketmq.tieredstore.provider.inputstream; +++package org.apache.rocketmq.tieredstore.provider.stream; ++ ++ import java.io.IOException; ++ import java.nio.ByteBuffer; ++@@ -23,20 +23,23 @@ import java.util.List; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ ++-public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { +++public class CommitLogInputStream extends FileSegmentInputStream { ++ ++ /** ++ * commitLogOffset is the real physical offset of the commitLog buffer which is being read ++ */ +++ private final long startCommitLogOffset; +++ ++ private long commitLogOffset; ++ ++ private final ByteBuffer codaBuffer; ++ ++ private long markCommitLogOffset = -1; ++ ++- public TieredCommitLogInputStream(FileSegmentType fileType, long startOffset, +++ public CommitLogInputStream(FileSegmentType fileType, long startOffset, ++ List uploadBufferList, ByteBuffer codaBuffer, int contentLength) { ++ super(fileType, uploadBufferList, contentLength); +++ this.startCommitLogOffset = startOffset; ++ this.commitLogOffset = startOffset; ++ this.codaBuffer = codaBuffer; ++ } ++@@ -53,6 +56,15 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++ this.commitLogOffset = markCommitLogOffset; ++ } ++ +++ @Override +++ public synchronized void rewind() { +++ super.rewind(); +++ this.commitLogOffset = this.startCommitLogOffset; +++ if (this.codaBuffer != null) { +++ this.codaBuffer.rewind(); +++ } +++ } +++ ++ @Override ++ public ByteBuffer getCodaBuffer() { ++ return this.codaBuffer; ++@@ -64,17 +76,17 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++ return -1; ++ } ++ readPosition++; ++- if (curReadBufferIndex >= uploadBufferList.size()) { +++ if (curReadBufferIndex >= bufferList.size()) { ++ return readCoda(); ++ } ++ int res; ++ if (readPosInCurBuffer >= curBuffer.remaining()) { ++ curReadBufferIndex++; ++- if (curReadBufferIndex >= uploadBufferList.size()) { +++ if (curReadBufferIndex >= bufferList.size()) { ++ readPosInCurBuffer = 0; ++ return readCoda(); ++ } ++- curBuffer = uploadBufferList.get(curReadBufferIndex); +++ curBuffer = bufferList.get(curReadBufferIndex); ++ commitLogOffset += readPosInCurBuffer; ++ readPosInCurBuffer = 0; ++ } ++@@ -119,9 +131,9 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++ int posInCurBuffer = readPosInCurBuffer; ++ long curCommitLogOffset = commitLogOffset; ++ ByteBuffer curBuf = curBuffer; ++- while (needRead > 0 && bufIndex <= uploadBufferList.size()) { +++ while (needRead > 0 && bufIndex <= bufferList.size()) { ++ int readLen, remaining, realReadLen = 0; ++- if (bufIndex == uploadBufferList.size()) { +++ if (bufIndex == bufferList.size()) { ++ // read from coda buffer ++ remaining = codaBuffer.remaining() - posInCurBuffer; ++ readLen = Math.min(remaining, needRead); ++@@ -137,7 +149,7 @@ public class TieredCommitLogInputStream extends TieredFileSegmentInputStream { ++ } ++ remaining = curBuf.remaining() - posInCurBuffer; ++ readLen = Math.min(remaining, needRead); ++- curBuf = uploadBufferList.get(bufIndex); +++ curBuf = bufferList.get(bufIndex); ++ if (posInCurBuffer < MessageBufferUtil.PHYSICAL_OFFSET_POSITION) { ++ realReadLen = Math.min(MessageBufferUtil.PHYSICAL_OFFSET_POSITION - posInCurBuffer, readLen); ++ // read from commitLog buffer ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java ++similarity index 77% ++rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java ++rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java ++index e1758ca93..9e9d5135c 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStream.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStream.java ++@@ -15,15 +15,16 @@ ++ * limitations under the License. ++ */ ++ ++-package org.apache.rocketmq.tieredstore.provider.inputstream; +++package org.apache.rocketmq.tieredstore.provider.stream; ++ ++ import java.io.IOException; ++ import java.io.InputStream; ++ import java.nio.ByteBuffer; ++ import java.util.List; +++import org.apache.commons.collections.CollectionUtils; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ ++-public class TieredFileSegmentInputStream extends InputStream { +++public class FileSegmentInputStream extends InputStream { ++ ++ /** ++ * file type, can be commitlog, consume queue or indexfile now ++@@ -33,7 +34,7 @@ public class TieredFileSegmentInputStream extends InputStream { ++ /** ++ * hold bytebuffer ++ */ ++- protected final List uploadBufferList; +++ protected final List bufferList; ++ ++ /** ++ * total remaining of bytebuffer list ++@@ -65,13 +66,13 @@ public class TieredFileSegmentInputStream extends InputStream { ++ ++ private int markReadPosInCurBuffer = -1; ++ ++- public TieredFileSegmentInputStream(FileSegmentType fileType, List uploadBufferList, ++- int contentLength) { +++ public FileSegmentInputStream( +++ FileSegmentType fileType, List bufferList, int contentLength) { ++ this.fileType = fileType; ++ this.contentLength = contentLength; ++- this.uploadBufferList = uploadBufferList; ++- if (uploadBufferList != null && uploadBufferList.size() > 0) { ++- this.curBuffer = uploadBufferList.get(curReadBufferIndex); +++ this.bufferList = bufferList; +++ if (bufferList != null && bufferList.size() > 0) { +++ this.curBuffer = bufferList.get(curReadBufferIndex); ++ } ++ } ++ ++@@ -95,18 +96,34 @@ public class TieredFileSegmentInputStream extends InputStream { ++ this.readPosition = markReadPosition; ++ this.curReadBufferIndex = markCurReadBufferIndex; ++ this.readPosInCurBuffer = markReadPosInCurBuffer; ++- if (this.curReadBufferIndex < uploadBufferList.size()) { ++- this.curBuffer = uploadBufferList.get(curReadBufferIndex); +++ if (this.curReadBufferIndex < bufferList.size()) { +++ this.curBuffer = bufferList.get(curReadBufferIndex); ++ } ++ } ++ +++ public synchronized void rewind() { +++ this.readPosition = 0; +++ this.curReadBufferIndex = 0; +++ this.readPosInCurBuffer = 0; +++ if (CollectionUtils.isNotEmpty(bufferList)) { +++ this.curBuffer = bufferList.get(0); +++ for (ByteBuffer buffer : bufferList) { +++ buffer.rewind(); +++ } +++ } +++ } +++ +++ public int getContentLength() { +++ return contentLength; +++ } +++ ++ @Override ++ public int available() { ++ return contentLength - readPosition; ++ } ++ ++- public List getUploadBufferList() { ++- return uploadBufferList; +++ public List getBufferList() { +++ return bufferList; ++ } ++ ++ public ByteBuffer getCodaBuffer() { ++@@ -121,10 +138,10 @@ public class TieredFileSegmentInputStream extends InputStream { ++ readPosition++; ++ if (readPosInCurBuffer >= curBuffer.remaining()) { ++ curReadBufferIndex++; ++- if (curReadBufferIndex >= uploadBufferList.size()) { +++ if (curReadBufferIndex >= bufferList.size()) { ++ return -1; ++ } ++- curBuffer = uploadBufferList.get(curReadBufferIndex); +++ curBuffer = bufferList.get(curReadBufferIndex); ++ readPosInCurBuffer = 0; ++ } ++ return curBuffer.get(readPosInCurBuffer++) & 0xff; ++@@ -153,8 +170,8 @@ public class TieredFileSegmentInputStream extends InputStream { ++ int bufIndex = curReadBufferIndex; ++ int posInCurBuffer = readPosInCurBuffer; ++ ByteBuffer curBuf = curBuffer; ++- while (needRead > 0 && bufIndex < uploadBufferList.size()) { ++- curBuf = uploadBufferList.get(bufIndex); +++ while (needRead > 0 && bufIndex < bufferList.size()) { +++ curBuf = bufferList.get(bufIndex); ++ int remaining = curBuf.remaining() - posInCurBuffer; ++ int readLen = Math.min(remaining, needRead); ++ // read from curBuf ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java ++similarity index 54% ++rename from tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java ++rename to tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java ++index d0c983fd4..a90baff3a 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/inputstream/TieredFileSegmentInputStreamFactory.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/stream/FileSegmentInputStreamFactory.java ++@@ -15,30 +15,34 @@ ++ * limitations under the License. ++ */ ++ ++-package org.apache.rocketmq.tieredstore.provider.inputstream; +++package org.apache.rocketmq.tieredstore.provider.stream; ++ ++ import java.nio.ByteBuffer; ++ import java.util.List; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ ++-public class TieredFileSegmentInputStreamFactory { +++public class FileSegmentInputStreamFactory { ++ ++- public static TieredFileSegmentInputStream build(FileSegmentType fileType, ++- long startOffset, List uploadBufferList, ByteBuffer codaBuffer, int contentLength) { +++ public static FileSegmentInputStream build( +++ FileSegmentType fileType, long offset, List bufferList, ByteBuffer byteBuffer, int length) { +++ +++ if (bufferList == null) { +++ throw new IllegalArgumentException("bufferList is null"); +++ } ++ ++ switch (fileType) { ++ case COMMIT_LOG: ++- return new TieredCommitLogInputStream( ++- fileType, startOffset, uploadBufferList, codaBuffer, contentLength); +++ return new CommitLogInputStream( +++ fileType, offset, bufferList, byteBuffer, length); ++ case CONSUME_QUEUE: ++- return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength); +++ return new FileSegmentInputStream(fileType, bufferList, length); ++ case INDEX: ++- if (uploadBufferList.size() != 1) { ++- throw new IllegalArgumentException("uploadBufferList size in INDEX type input stream must be 1"); +++ if (bufferList.size() != 1) { +++ throw new IllegalArgumentException("buffer block size must be 1 when file type is IndexFile"); ++ } ++- return new TieredFileSegmentInputStream(fileType, uploadBufferList, contentLength); +++ return new FileSegmentInputStream(fileType, bufferList, length); ++ default: ++- throw new IllegalArgumentException("fileType is not supported"); +++ throw new IllegalArgumentException("file type is not supported"); ++ } ++ } ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++index 8601392e7..2451199c2 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++@@ -130,36 +130,36 @@ public class TieredMessageStoreTest { ++ // TieredStorageLevel.DISABLE ++ properties.setProperty("tieredStorageLevel", "0"); ++ configuration.update(properties); ++- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ // TieredStorageLevel.NOT_IN_DISK ++ properties.setProperty("tieredStorageLevel", "1"); ++ configuration.update(properties); ++ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false); ++- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); ++- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ // TieredStorageLevel.NOT_IN_MEM ++ properties.setProperty("tieredStorageLevel", "2"); ++ configuration.update(properties); ++ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(false); ++ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true); ++- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); ++ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(false); ++- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ Mockito.when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); ++ Mockito.when(nextStore.checkInMemByConsumeOffset(anyString(), anyInt(), anyLong(), anyInt())).thenReturn(true); ++- Assert.assertFalse(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertFalse(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ ++ // TieredStorageLevel.FORCE ++ properties.setProperty("tieredStorageLevel", "3"); ++ configuration.update(properties); ++- Assert.assertTrue(store.viaTieredStorage(mq.getTopic(), mq.getQueueId(), 0)); +++ Assert.assertTrue(store.fetchFromCurrentStore(mq.getTopic(), mq.getQueueId(), 0)); ++ } ++ ++ @Test ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++index cc39cfbfc..7a4d05969 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++@@ -24,6 +24,7 @@ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; +++import org.apache.rocketmq.tieredstore.common.TieredStoreExecutor; ++ import org.apache.rocketmq.tieredstore.metadata.FileSegmentMetadata; ++ import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore; ++ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; ++@@ -55,6 +56,7 @@ public class TieredFlatFileTest { ++ public void tearDown() throws IOException { ++ TieredStoreTestUtil.destroyMetadataStore(); ++ TieredStoreTestUtil.destroyTempDir(storePath); +++ TieredStoreExecutor.shutdown(); ++ } ++ ++ private List getSegmentMetadataList(TieredMetadataStore metadataStore) { ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++index 262d6645b..2da72bc7a 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredIndexFileTest.java ++@@ -87,5 +87,7 @@ public class TieredIndexFileTest { ++ ++ indexList = indexFile.queryAsync(mq.getTopic(), "key1", 1000, 1200).join(); ++ Assert.assertEquals(1, indexList.size()); +++ +++ indexFile.destroy(); ++ } ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java ++similarity index 82% ++rename from tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java ++rename to tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java ++index a6566b7de..3bbe41dd4 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockTieredFileSegmentInputStream.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/MockFileSegmentInputStream.java ++@@ -20,13 +20,13 @@ package org.apache.rocketmq.tieredstore.provider; ++ import java.io.InputStream; ++ import java.nio.ByteBuffer; ++ import java.util.List; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ ++-public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStream { +++public class MockFileSegmentInputStream extends FileSegmentInputStream { ++ ++ private final InputStream inputStream; ++ ++- public MockTieredFileSegmentInputStream(InputStream inputStream) { +++ public MockFileSegmentInputStream(InputStream inputStream) { ++ super(null, null, Integer.MAX_VALUE); ++ this.inputStream = inputStream; ++ } ++@@ -43,7 +43,7 @@ public class MockTieredFileSegmentInputStream extends TieredFileSegmentInputStre ++ } ++ ++ @Override ++- public List getUploadBufferList() { +++ public List getBufferList() { ++ return null; ++ } ++ ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java ++index a2554ba3d..743d9182c 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentInputStreamTest.java ++@@ -28,8 +28,8 @@ import java.util.Random; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.file.TieredCommitLog; ++ import org.apache.rocketmq.tieredstore.file.TieredConsumeQueue; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStreamFactory; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStreamFactory; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtilTest; ++ import org.junit.Assert; ++@@ -57,7 +57,7 @@ public class TieredFileSegmentInputStreamTest { ++ bufferSize += byteBuffer.remaining(); ++ } ++ ++- // build expected byte buffer for verifying the TieredFileSegmentInputStream +++ // build expected byte buffer for verifying the FileSegmentInputStream ++ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); ++ for (ByteBuffer byteBuffer : uploadBufferList) { ++ expectedByteBuffer.put(byteBuffer); ++@@ -74,7 +74,7 @@ public class TieredFileSegmentInputStreamTest { ++ int[] batchReadSizeTestSet = { ++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION - 1, MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtil.PHYSICAL_OFFSET_POSITION + 1, MSG_LEN - 1, MSG_LEN, MSG_LEN + 1 ++ }; ++- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( +++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( ++ FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), finalBufferSize, batchReadSizeTestSet); ++ ++ } ++@@ -98,7 +98,7 @@ public class TieredFileSegmentInputStreamTest { ++ int codaBufferSize = codaBuffer.remaining(); ++ bufferSize += codaBufferSize; ++ ++- // build expected byte buffer for verifying the TieredFileSegmentInputStream +++ // build expected byte buffer for verifying the FileSegmentInputStream ++ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); ++ for (ByteBuffer byteBuffer : uploadBufferList) { ++ expectedByteBuffer.put(byteBuffer); ++@@ -119,7 +119,7 @@ public class TieredFileSegmentInputStreamTest { ++ MSG_LEN - 1, MSG_LEN, MSG_LEN + 1, ++ bufferSize - 1, bufferSize, bufferSize + 1 ++ }; ++- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( +++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( ++ FileSegmentType.COMMIT_LOG, COMMIT_LOG_START_OFFSET, uploadBufferList, codaBuffer, finalBufferSize), finalBufferSize, batchReadSizeTestSet); ++ ++ } ++@@ -134,7 +134,7 @@ public class TieredFileSegmentInputStreamTest { ++ bufferSize += byteBuffer.remaining(); ++ } ++ ++- // build expected byte buffer for verifying the TieredFileSegmentInputStream +++ // build expected byte buffer for verifying the FileSegmentInputStream ++ ByteBuffer expectedByteBuffer = ByteBuffer.allocate(bufferSize); ++ for (ByteBuffer byteBuffer : uploadBufferList) { ++ expectedByteBuffer.put(byteBuffer); ++@@ -143,7 +143,7 @@ public class TieredFileSegmentInputStreamTest { ++ ++ int finalBufferSize = bufferSize; ++ int[] batchReadSizeTestSet = {TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE - 1, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE, TieredConsumeQueue.CONSUME_QUEUE_STORE_UNIT_SIZE + 1}; ++- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( +++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( ++ FileSegmentType.CONSUME_QUEUE, COMMIT_LOG_START_OFFSET, uploadBufferList, null, finalBufferSize), bufferSize, batchReadSizeTestSet); ++ } ++ ++@@ -156,16 +156,16 @@ public class TieredFileSegmentInputStreamTest { ++ byteBuffer.flip(); ++ List uploadBufferList = Arrays.asList(byteBuffer); ++ ++- // build expected byte buffer for verifying the TieredFileSegmentInputStream +++ // build expected byte buffer for verifying the FileSegmentInputStream ++ ByteBuffer expectedByteBuffer = byteBuffer.slice(); ++ ++- verifyReadAndReset(expectedByteBuffer, () -> TieredFileSegmentInputStreamFactory.build( +++ verifyReadAndReset(expectedByteBuffer, () -> FileSegmentInputStreamFactory.build( ++ FileSegmentType.INDEX, COMMIT_LOG_START_OFFSET, uploadBufferList, null, byteBuffer.limit()), byteBuffer.limit(), new int[] {23, 24, 25}); ++ } ++ ++- private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier constructor, +++ private void verifyReadAndReset(ByteBuffer expectedByteBuffer, Supplier constructor, ++ int bufferSize, int[] readBatchSizeTestSet) { ++- TieredFileSegmentInputStream inputStream = constructor.get(); +++ FileSegmentInputStream inputStream = constructor.get(); ++ ++ // verify ++ verifyInputStream(inputStream, expectedByteBuffer); ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java ++index 4cd83e0d2..a655710a5 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredFileSegmentTest.java ++@@ -116,13 +116,22 @@ public class TieredFileSegmentTest { ++ } ++ ++ @Test ++- public void testCommitFailed() { +++ public void testCommitFailedThenSuccess() { ++ long startTime = System.currentTimeMillis(); ++ MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG); ++ long lastSize = segment.getSize(); ++- segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0); ++- segment.append(MessageBufferUtilTest.buildMockedMessageBuffer(), 0); ++- +++ segment.setCheckSize(false); +++ segment.initPosition(lastSize); +++ segment.setSize((int) lastSize); +++ +++ ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( +++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize); +++ ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( +++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN); +++ segment.append(buffer1, 0); +++ segment.append(buffer2, 0); +++ +++ // Mock new message arrive ++ segment.blocker = new CompletableFuture<>(); ++ new Thread(() -> { ++ try { ++@@ -131,20 +140,88 @@ public class TieredFileSegmentTest { ++ Assert.fail(e.getMessage()); ++ } ++ ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer(); +++ buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2); ++ buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime); ++ segment.append(buffer, 0); ++ segment.blocker.complete(false); ++ }).start(); ++ +++ // Commit failed ++ segment.commit(); ++ segment.blocker.join(); +++ segment.blocker = null; +++ +++ // Copy data and assume commit success +++ segment.getMemStore().put(buffer1); +++ segment.getMemStore().put(buffer2); +++ segment.setSize((int) (lastSize + MessageBufferUtilTest.MSG_LEN * 2)); ++ ++- segment.blocker = new CompletableFuture<>(); ++- segment.blocker.complete(true); ++ segment.commit(); +++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); +++ +++ ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN); +++ Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1)); +++ +++ ByteBuffer msg2 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtilTest.MSG_LEN); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN, MessageBufferUtil.getCommitLogOffset(msg2)); +++ +++ ByteBuffer msg3 = segment.read(lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtilTest.MSG_LEN); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, MessageBufferUtil.getCommitLogOffset(msg3)); +++ } +++ +++ @Test +++ public void testCommitFailed3Times() { +++ long startTime = System.currentTimeMillis(); +++ MemoryFileSegment segment = (MemoryFileSegment) createFileSegment(FileSegmentType.COMMIT_LOG); +++ long lastSize = segment.getSize(); +++ segment.setCheckSize(false); +++ segment.initPosition(lastSize); +++ segment.setSize((int) lastSize); +++ +++ ByteBuffer buffer1 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( +++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize); +++ ByteBuffer buffer2 = MessageBufferUtilTest.buildMockedMessageBuffer().putLong( +++ MessageBufferUtil.PHYSICAL_OFFSET_POSITION, baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN); +++ segment.append(buffer1, 0); +++ segment.append(buffer2, 0); +++ +++ // Mock new message arrive +++ segment.blocker = new CompletableFuture<>(); +++ new Thread(() -> { +++ try { +++ Thread.sleep(3000); +++ } catch (InterruptedException e) { +++ Assert.fail(e.getMessage()); +++ } +++ ByteBuffer buffer = MessageBufferUtilTest.buildMockedMessageBuffer(); +++ buffer.putLong(MessageBufferUtil.PHYSICAL_OFFSET_POSITION, MessageBufferUtilTest.MSG_LEN * 2); +++ buffer.putLong(MessageBufferUtil.STORE_TIMESTAMP_POSITION, startTime); +++ segment.append(buffer, 0); +++ segment.blocker.complete(false); +++ }).start(); +++ +++ for (int i = 0; i < 3; i++) { +++ segment.commit(); +++ } +++ +++ Assert.assertEquals(lastSize, segment.getCommitPosition()); +++ Assert.assertEquals(baseOffset + lastSize, segment.getCommitOffset()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); +++ +++ segment.blocker.join(); +++ segment.blocker = null; ++ +++ segment.commit(); +++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitPosition()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 2, segment.getCommitOffset()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); +++ +++ segment.commit(); +++ Assert.assertEquals(lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitPosition()); ++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getCommitOffset()); +++ Assert.assertEquals(baseOffset + lastSize + MessageBufferUtilTest.MSG_LEN * 3, segment.getMaxOffset()); ++ ++ ByteBuffer msg1 = segment.read(lastSize, MessageBufferUtilTest.MSG_LEN); ++ Assert.assertEquals(baseOffset + lastSize, MessageBufferUtil.getCommitLogOffset(msg1)); ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java ++index cb155cf8f..80ad41f68 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegment.java ++@@ -23,7 +23,7 @@ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; ++ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ import org.junit.Assert; ++ ++@@ -33,6 +33,8 @@ public class MemoryFileSegment extends TieredFileSegment { ++ ++ public CompletableFuture blocker; ++ +++ protected int size = 0; +++ ++ protected boolean checkSize = true; ++ ++ public MemoryFileSegment(FileSegmentType fileType, MessageQueue messageQueue, long baseOffset, ++@@ -56,6 +58,18 @@ public class MemoryFileSegment extends TieredFileSegment { ++ memStore.position((int) getSize()); ++ } ++ +++ public boolean isCheckSize() { +++ return checkSize; +++ } +++ +++ public void setCheckSize(boolean checkSize) { +++ this.checkSize = checkSize; +++ } +++ +++ public ByteBuffer getMemStore() { +++ return memStore; +++ } +++ ++ @Override ++ public String getPath() { ++ return filePath; ++@@ -66,7 +80,11 @@ public class MemoryFileSegment extends TieredFileSegment { ++ if (checkSize) { ++ return 1000; ++ } ++- return 0; +++ return size; +++ } +++ +++ public void setSize(int size) { +++ this.size = size; ++ } ++ ++ @Override ++@@ -85,11 +103,11 @@ public class MemoryFileSegment extends TieredFileSegment { ++ ++ @Override ++ public CompletableFuture commit0( ++- TieredFileSegmentInputStream inputStream, long position, int length, boolean append) { +++ FileSegmentInputStream inputStream, long position, int length, boolean append) { ++ ++ try { ++ if (blocker != null && !blocker.get()) { ++- throw new IllegalStateException(); +++ throw new IllegalStateException("Commit Exception for Memory Test"); ++ } ++ } catch (InterruptedException | ExecutionException e) { ++ Assert.fail(e.getMessage()); ++@@ -98,7 +116,6 @@ public class MemoryFileSegment extends TieredFileSegment { ++ Assert.assertTrue(!checkSize || position >= getSize()); ++ ++ byte[] buffer = new byte[1024]; ++- ++ int startPos = memStore.position(); ++ try { ++ int len; ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java ++index 8ac330b37..630fd2223 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/memory/MemoryFileSegmentWithoutCheck.java ++@@ -22,7 +22,7 @@ import java.util.concurrent.ExecutionException; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++ import org.apache.rocketmq.tieredstore.common.TieredMessageStoreConfig; ++-import org.apache.rocketmq.tieredstore.provider.inputstream.TieredFileSegmentInputStream; +++import org.apache.rocketmq.tieredstore.provider.stream.FileSegmentInputStream; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++ import org.junit.Assert; ++ ++@@ -46,7 +46,7 @@ public class MemoryFileSegmentWithoutCheck extends MemoryFileSegment { ++ } ++ ++ @Override ++- public CompletableFuture commit0(TieredFileSegmentInputStream inputStream, long position, int length, +++ public CompletableFuture commit0(FileSegmentInputStream inputStream, long position, int length, ++ boolean append) { ++ try { ++ if (blocker != null && !blocker.get()) { ++-- ++2.32.0.windows.2 ++ ++ ++From d000ef947d7c99918ceba0fa451c1e29fd84ba07 Mon Sep 17 00:00:00 2001 ++From: yuz10 <845238369@qq.com> ++Date: Thu, 31 Aug 2023 09:41:33 +0800 ++Subject: [PATCH 3/7] [ISSUE #7283] Incorrect dledger commitlog min offset ++ after mappedFile re delete failed (#7284) ++ ++--- ++ .../apache/rocketmq/store/dledger/DLedgerCommitLog.java | 7 ++++++- ++ 1 file changed, 6 insertions(+), 1 deletion(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java ++index ec5e86d70..d5f6acdc0 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +++++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java ++@@ -162,7 +162,12 @@ public class DLedgerCommitLog extends CommitLog { ++ if (!mappedFileQueue.getMappedFiles().isEmpty()) { ++ return mappedFileQueue.getMinOffset(); ++ } ++- return dLedgerFileList.getMinOffset(); +++ for (MmapFile file : dLedgerFileList.getMappedFiles()) { +++ if (file.isAvailable()) { +++ return file.getFileFromOffset() + file.getStartPosition(); +++ } +++ } +++ return 0; ++ } ++ ++ @Override ++-- ++2.32.0.windows.2 ++ ++ ++From f82718ae3b77a16b553c03f672dc971a2d5d48fa Mon Sep 17 00:00:00 2001 ++From: cnScarb ++Date: Thu, 31 Aug 2023 15:50:10 +0800 ++Subject: [PATCH 4/7] [ISSUE #7208] fix: when deleting topic also delete its ++ pop retry topic (#7209) ++ ++--- ++ .../processor/AdminBrokerProcessor.java | 24 ++++++++++--- ++ .../processor/AdminBrokerProcessorTest.java | 36 +++++++++++++++++++ ++ 2 files changed, 55 insertions(+), 5 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++index bbddcec2d..8fbcd3c94 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++@@ -51,6 +51,7 @@ import org.apache.rocketmq.broker.subscription.SubscriptionGroupManager; ++ import org.apache.rocketmq.broker.transaction.queue.TransactionalMessageUtil; ++ import org.apache.rocketmq.common.AclConfig; ++ import org.apache.rocketmq.common.BrokerConfig; +++import org.apache.rocketmq.common.KeyBuilder; ++ import org.apache.rocketmq.common.LockCallback; ++ import org.apache.rocketmq.common.MQVersion; ++ import org.apache.rocketmq.common.MixAll; ++@@ -542,16 +543,29 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ } ++ } ++ ++- this.brokerController.getTopicConfigManager().deleteTopicConfig(requestHeader.getTopic()); ++- this.brokerController.getTopicQueueMappingManager().delete(requestHeader.getTopic()); ++- this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(requestHeader.getTopic()); ++- this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(requestHeader.getTopic()); ++- this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(requestHeader.getTopic())); +++ final Set groups = this.brokerController.getConsumerOffsetManager().whichGroupByTopic(topic); +++ // delete pop retry topics first +++ for (String group : groups) { +++ final String popRetryTopic = KeyBuilder.buildPopRetryTopic(topic, group); +++ if (brokerController.getTopicConfigManager().selectTopicConfig(popRetryTopic) != null) { +++ deleteTopicInBroker(popRetryTopic); +++ } +++ } +++ // delete topic +++ deleteTopicInBroker(topic); ++ response.setCode(ResponseCode.SUCCESS); ++ response.setRemark(null); ++ return response; ++ } ++ +++ private void deleteTopicInBroker(String topic) { +++ this.brokerController.getTopicConfigManager().deleteTopicConfig(topic); +++ this.brokerController.getTopicQueueMappingManager().delete(topic); +++ this.brokerController.getConsumerOffsetManager().cleanOffsetByTopic(topic); +++ this.brokerController.getPopInflightMessageCounter().clearInFlightMessageNumByTopicName(topic); +++ this.brokerController.getMessageStore().deleteTopics(Sets.newHashSet(topic)); +++ } +++ ++ private synchronized RemotingCommand updateAndCreateAccessConfig(ChannelHandlerContext ctx, ++ RemotingCommand request) throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++index d33a217f7..9d17011b6 100644 ++--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++@@ -29,6 +29,7 @@ import java.util.HashMap; ++ import java.util.Map; ++ import java.util.Properties; ++ import java.util.Set; +++import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.atomic.LongAdder; ++ import org.apache.rocketmq.broker.BrokerController; ++@@ -41,6 +42,7 @@ import org.apache.rocketmq.broker.topic.RocksDBTopicConfigManager; ++ import org.apache.rocketmq.broker.topic.TopicConfigManager; ++ import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.BrokerConfig; +++import org.apache.rocketmq.common.KeyBuilder; ++ import org.apache.rocketmq.common.MixAll; ++ import org.apache.rocketmq.common.TopicConfig; ++ import org.apache.rocketmq.common.TopicFilterType; ++@@ -90,8 +92,11 @@ import static org.assertj.core.api.Assertions.assertThat; ++ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyInt; ++ import static org.mockito.ArgumentMatchers.anyLong; +++import static org.mockito.ArgumentMatchers.anySet; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.Mockito.mock; +++import static org.mockito.Mockito.times; +++import static org.mockito.Mockito.verify; ++ import static org.mockito.Mockito.when; ++ ++ @RunWith(MockitoJUnitRunner.class) ++@@ -321,6 +326,37 @@ public class AdminBrokerProcessorTest { ++ "please execute it from master broker."); ++ } ++ +++ @Test +++ public void testDeleteWithPopRetryTopic() throws Exception { +++ String topic = "topicA"; +++ String anotherTopic = "another_topicA"; +++ +++ topicConfigManager = mock(TopicConfigManager.class); +++ when(brokerController.getTopicConfigManager()).thenReturn(topicConfigManager); +++ final ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); +++ topicConfigTable.put(topic, new TopicConfig()); +++ topicConfigTable.put(KeyBuilder.buildPopRetryTopic(topic, "cid1"), new TopicConfig()); +++ +++ topicConfigTable.put(anotherTopic, new TopicConfig()); +++ topicConfigTable.put(KeyBuilder.buildPopRetryTopic(anotherTopic, "cid2"), new TopicConfig()); +++ when(topicConfigManager.getTopicConfigTable()).thenReturn(topicConfigTable); +++ when(topicConfigManager.selectTopicConfig(anyString())).thenAnswer(invocation -> { +++ final String selectTopic = invocation.getArgument(0); +++ return topicConfigManager.getTopicConfigTable().get(selectTopic); +++ }); +++ +++ when(brokerController.getConsumerOffsetManager()).thenReturn(consumerOffsetManager); +++ when(consumerOffsetManager.whichGroupByTopic(topic)).thenReturn(Sets.newHashSet("cid1")); +++ +++ RemotingCommand request = buildDeleteTopicRequest(topic); +++ RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); +++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); +++ +++ verify(topicConfigManager).deleteTopicConfig(topic); +++ verify(topicConfigManager).deleteTopicConfig(KeyBuilder.buildPopRetryTopic(topic, "cid1")); +++ verify(messageStore, times(2)).deleteTopics(anySet()); +++ } +++ ++ @Test ++ public void testGetAllTopicConfigInRocksdb() throws Exception { ++ if (notToBeExecuted()) { ++-- ++2.32.0.windows.2 ++ ++ ++From 31d10385d1616445478104ce9ef463a8c4852ba2 Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Mon, 4 Sep 2023 14:09:32 +0800 ++Subject: [PATCH 5/7] [ISSUE #7289] Fixed asynchronous send backpressure ++ capability ++ ++Co-authored-by: guyinyou ++--- ++ .../impl/producer/DefaultMQProducerImpl.java | 77 +++++++++++++------ ++ 1 file changed, 53 insertions(+), 24 deletions(-) ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++index bbbb17b07..2d6b83ac2 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++@@ -547,6 +547,8 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ @Deprecated ++ public void send(final Message msg, final SendCallback sendCallback, final long timeout) ++ throws MQClientException, RemotingException, InterruptedException { +++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); +++ ++ final long beginStartTime = System.currentTimeMillis(); ++ Runnable runnable = new Runnable() { ++ @Override ++@@ -554,20 +556,53 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ if (timeout > costTime) { ++ try { ++- sendDefaultImpl(msg, CommunicationMode.ASYNC, sendCallback, timeout - costTime); +++ sendDefaultImpl(msg, CommunicationMode.ASYNC, newCallBack, timeout - costTime); ++ } catch (Exception e) { ++- sendCallback.onException(e); +++ newCallBack.onException(e); ++ } ++ } else { ++- sendCallback.onException( +++ newCallBack.onException( ++ new RemotingTooMuchRequestException("DEFAULT ASYNC send call timeout")); ++ } ++ } ++ }; ++- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); +++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); ++ } ++ ++- public void executeAsyncMessageSend(Runnable runnable, final Message msg, final SendCallback sendCallback, +++ class BackpressureSendCallBack implements SendCallback { +++ public boolean isSemaphoreAsyncSizeAquired = false; +++ public boolean isSemaphoreAsyncNumAquired = false; +++ public int msgLen; +++ private final SendCallback sendCallback; +++ +++ public BackpressureSendCallBack(final SendCallback sendCallback) { +++ this.sendCallback = sendCallback; +++ } +++ +++ @Override +++ public void onSuccess(SendResult sendResult) { +++ if (isSemaphoreAsyncSizeAquired) { +++ semaphoreAsyncSendSize.release(msgLen); +++ } +++ if (isSemaphoreAsyncNumAquired) { +++ semaphoreAsyncSendNum.release(); +++ } +++ sendCallback.onSuccess(sendResult); +++ } +++ +++ @Override +++ public void onException(Throwable e) { +++ if (isSemaphoreAsyncSizeAquired) { +++ semaphoreAsyncSendSize.release(msgLen); +++ } +++ if (isSemaphoreAsyncNumAquired) { +++ semaphoreAsyncSendNum.release(); +++ } +++ sendCallback.onException(e); +++ } +++ } +++ +++ public void executeAsyncMessageSend(Runnable runnable, final Message msg, final BackpressureSendCallBack sendCallback, ++ final long timeout, final long beginStartTime) ++ throws MQClientException, InterruptedException { ++ ExecutorService executor = this.getAsyncSenderExecutor(); ++@@ -595,7 +630,9 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ return; ++ } ++ } ++- +++ sendCallback.isSemaphoreAsyncSizeAquired = isSemaphoreAsyncSizeAquired; +++ sendCallback.isSemaphoreAsyncNumAquired = isSemaphoreAsyncNumAquired; +++ sendCallback.msgLen = msgLen; ++ executor.submit(runnable); ++ } catch (RejectedExecutionException e) { ++ if (isEnableBackpressureForAsyncMode) { ++@@ -603,15 +640,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ } else { ++ throw new MQClientException("executor rejected ", e); ++ } ++- } finally { ++- if (isSemaphoreAsyncSizeAquired) { ++- semaphoreAsyncSendSize.release(msgLen); ++- } ++- if (isSemaphoreAsyncNumAquired) { ++- semaphoreAsyncSendNum.release(); ++- } ++ } ++- ++ } ++ ++ public MessageQueue invokeMessageQueueSelector(Message msg, MessageQueueSelector selector, Object arg, ++@@ -1188,7 +1217,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ @Deprecated ++ public void send(final Message msg, final MessageQueue mq, final SendCallback sendCallback, final long timeout) ++ throws MQClientException, RemotingException, InterruptedException { ++- +++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); ++ final long beginStartTime = System.currentTimeMillis(); ++ Runnable runnable = new Runnable() { ++ @Override ++@@ -1203,22 +1232,22 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ if (timeout > costTime) { ++ try { ++- sendKernelImpl(msg, mq, CommunicationMode.ASYNC, sendCallback, null, +++ sendKernelImpl(msg, mq, CommunicationMode.ASYNC, newCallBack, null, ++ timeout - costTime); ++ } catch (MQBrokerException e) { ++ throw new MQClientException("unknown exception", e); ++ } ++ } else { ++- sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); +++ newCallBack.onException(new RemotingTooMuchRequestException("call timeout")); ++ } ++ } catch (Exception e) { ++- sendCallback.onException(e); +++ newCallBack.onException(e); ++ } ++ } ++ ++ }; ++ ++- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); +++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); ++ } ++ ++ /** ++@@ -1315,7 +1344,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ public void send(final Message msg, final MessageQueueSelector selector, final Object arg, ++ final SendCallback sendCallback, final long timeout) ++ throws MQClientException, RemotingException, InterruptedException { ++- +++ BackpressureSendCallBack newCallBack = new BackpressureSendCallBack(sendCallback); ++ final long beginStartTime = System.currentTimeMillis(); ++ Runnable runnable = new Runnable() { ++ @Override ++@@ -1324,21 +1353,21 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ if (timeout > costTime) { ++ try { ++ try { ++- sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, sendCallback, +++ sendSelectImpl(msg, selector, arg, CommunicationMode.ASYNC, newCallBack, ++ timeout - costTime); ++ } catch (MQBrokerException e) { ++ throw new MQClientException("unknown exception", e); ++ } ++ } catch (Exception e) { ++- sendCallback.onException(e); +++ newCallBack.onException(e); ++ } ++ } else { ++- sendCallback.onException(new RemotingTooMuchRequestException("call timeout")); +++ newCallBack.onException(new RemotingTooMuchRequestException("call timeout")); ++ } ++ } ++ ++ }; ++- executeAsyncMessageSend(runnable, msg, sendCallback, timeout, beginStartTime); +++ executeAsyncMessageSend(runnable, msg, newCallBack, timeout, beginStartTime); ++ } ++ ++ /** ++-- ++2.32.0.windows.2 ++ ++ ++From d67b9d64cbd53824798af57ba18770e0fcefa37a Mon Sep 17 00:00:00 2001 ++From: yuz10 <845238369@qq.com> ++Date: Wed, 6 Sep 2023 14:07:23 +0800 ++Subject: [PATCH 6/7] [ISSUE #7302] Fix singleTopicRegister code deleted in ++ merge ++ ++--- ++ .../apache/rocketmq/broker/topic/TopicConfigManager.java | 6 +++++- ++ 1 file changed, 5 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++index 1c3b9711f..4e3c1736c 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++@@ -330,7 +330,11 @@ public class TopicConfigManager extends ConfigManager { ++ log.error("createTopicIfAbsent ", e); ++ } ++ if (createNew && register) { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ return getTopicConfig(topicConfig.getTopicName()); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 37017dbaec5c521fd529ef4aecf3658092884f84 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 6 Sep 2023 15:23:15 +0800 ++Subject: [PATCH 7/7] [ISSUE #7305] Fix metrics and transactional module not ++ shutdown while broker offline cause coredump(#7307) ++ ++--- ++ .../java/org/apache/rocketmq/broker/BrokerController.java | 8 ++++++++ ++ .../queue/TransactionalMessageServiceImpl.java | 4 +++- ++ 2 files changed, 11 insertions(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index e8f943702..6aba70cb2 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1302,6 +1302,10 @@ public class BrokerController { ++ this.fastRemotingServer.shutdown(); ++ } ++ +++ if (this.brokerMetricsManager != null) { +++ this.brokerMetricsManager.shutdown(); +++ } +++ ++ if (this.brokerStatsManager != null) { ++ this.brokerStatsManager.shutdown(); ++ } ++@@ -1324,6 +1328,10 @@ public class BrokerController { ++ this.ackMessageProcessor.shutdownPopReviveService(); ++ } ++ +++ if (this.transactionalMessageService != null) { +++ this.transactionalMessageService.close(); +++ } +++ ++ if (this.notificationProcessor != null) { ++ this.notificationProcessor.getPopLongPollingService().shutdown(); ++ } ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java ++index 93fa725a9..48db828e0 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/queue/TransactionalMessageServiceImpl.java ++@@ -629,7 +629,9 @@ public class TransactionalMessageServiceImpl implements TransactionalMessageServ ++ ++ @Override ++ public void close() { ++- +++ if (this.transactionalOpBatchService != null) { +++ this.transactionalOpBatchService.shutdown(); +++ } ++ } ++ ++ public Message getOpMessage(int queueId, String moreData) { ++-- ++2.32.0.windows.2 ++ +diff --git a/patch016-backport-Optimize-fault-tolerant-mechanism.patch b/patch016-backport-Optimize-fault-tolerant-mechanism.patch +new file mode 100644 +index 000000000..9d8ef20a1 +--- /dev/null ++++ b/patch016-backport-Optimize-fault-tolerant-mechanism.patch +@@ -0,0 +1,520 @@ ++From e11e29419f6e2d1d9673d0329e57b824ebf3da47 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Wed, 6 Sep 2023 20:42:24 +0800 ++Subject: [PATCH 1/3] [ISSUE #7308] Adding topic blacklist and filter in tiered ++ storage module (#7310) ++ ++--- ++ .../tieredstore/TieredDispatcher.java | 21 +++++++-- ++ .../tieredstore/TieredMessageStore.java | 1 + ++ .../file/TieredFlatFileManager.java | 17 ++++--- ++ .../TieredStoreTopicBlackListFilter.java | 45 +++++++++++++++++++ ++ .../provider/TieredStoreTopicFilter.java | 25 +++++++++++ ++ .../TieredStoreTopicBlackListFilterTest.java | 36 +++++++++++++++ ++ 6 files changed, 136 insertions(+), 9 deletions(-) ++ create mode 100644 tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++ create mode 100644 tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++ create mode 100644 tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++index 430c2b62e..766c559e9 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredDispatcher.java ++@@ -48,6 +48,8 @@ import org.apache.rocketmq.tieredstore.file.CompositeQueueFlatFile; ++ import org.apache.rocketmq.tieredstore.file.TieredFlatFileManager; ++ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant; ++ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; +++import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicBlackListFilter; +++import org.apache.rocketmq.tieredstore.provider.TieredStoreTopicFilter; ++ import org.apache.rocketmq.tieredstore.util.CQItemBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.MessageBufferUtil; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++@@ -56,6 +58,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ ++ private static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); ++ +++ private TieredStoreTopicFilter topicFilter; ++ private final String brokerName; ++ private final MessageStore defaultStore; ++ private final TieredMessageStoreConfig storeConfig; ++@@ -70,15 +73,15 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ this.defaultStore = defaultStore; ++ this.storeConfig = storeConfig; ++ this.brokerName = storeConfig.getBrokerName(); +++ this.topicFilter = new TieredStoreTopicBlackListFilter(); ++ this.tieredFlatFileManager = TieredFlatFileManager.getInstance(storeConfig); ++ this.dispatchRequestReadMap = new ConcurrentHashMap<>(); ++ this.dispatchRequestWriteMap = new ConcurrentHashMap<>(); ++ this.dispatchTaskLock = new ReentrantLock(); ++ this.dispatchWriteLock = new ReentrantLock(); ++- this.initScheduleTask(); ++ } ++ ++- private void initScheduleTask() { +++ protected void initScheduleTask() { ++ TieredStoreExecutor.commonScheduledExecutor.scheduleWithFixedDelay(() -> ++ tieredFlatFileManager.deepCopyFlatFileToList().forEach(flatFile -> { ++ if (!flatFile.getCompositeFlatFileLock().isLocked()) { ++@@ -87,6 +90,14 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ }), 30, 10, TimeUnit.SECONDS); ++ } ++ +++ public TieredStoreTopicFilter getTopicFilter() { +++ return topicFilter; +++ } +++ +++ public void setTopicFilter(TieredStoreTopicFilter topicFilter) { +++ this.topicFilter = topicFilter; +++ } +++ ++ @Override ++ public void dispatch(DispatchRequest request) { ++ if (stopped) { ++@@ -94,7 +105,7 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ } ++ ++ String topic = request.getTopic(); ++- if (TieredStoreUtil.isSystemTopic(topic)) { +++ if (topicFilter != null && topicFilter.filterTopic(topic)) { ++ return; ++ } ++ ++@@ -219,6 +230,10 @@ public class TieredDispatcher extends ServiceThread implements CommitLogDispatch ++ return; ++ } ++ +++ if (topicFilter != null && topicFilter.filterTopic(flatFile.getMessageQueue().getTopic())) { +++ return; +++ } +++ ++ if (flatFile.getDispatchOffset() == -1L) { ++ return; ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index 78e855f36..9fb1b2f01 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -90,6 +90,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ boolean loadNextStore = next.load(); ++ boolean result = loadFlatFile && loadNextStore; ++ if (result) { +++ dispatcher.initScheduleTask(); ++ dispatcher.start(); ++ } ++ return result; ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++index e9ae4a5a5..7c744af3b 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++@@ -134,21 +134,21 @@ public class TieredFlatFileManager { ++ public void doCleanExpiredFile() { ++ long expiredTimeStamp = System.currentTimeMillis() - ++ TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); ++- Random random = new Random(); ++ for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) { ++- int delay = random.nextInt(storeConfig.getMaxCommitJitter()); ++- TieredStoreExecutor.cleanExpiredFileExecutor.schedule(() -> { +++ TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> { ++ flatFile.getCompositeFlatFileLock().lock(); ++ try { ++ flatFile.cleanExpiredFile(expiredTimeStamp); ++ flatFile.destroyExpiredFile(); ++ if (flatFile.getConsumeQueueBaseOffset() == -1) { +++ logger.info("Clean flatFile because file not initialized, topic={}, queueId={}", +++ flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId()); ++ destroyCompositeFile(flatFile.getMessageQueue()); ++ } ++ } finally { ++ flatFile.getCompositeFlatFileLock().unlock(); ++ } ++- }, delay, TimeUnit.MILLISECONDS); +++ }); ++ } ++ if (indexFile != null) { ++ indexFile.cleanExpiredFile(expiredTimeStamp); ++@@ -218,8 +218,13 @@ public class TieredFlatFileManager { ++ storeConfig.getBrokerName(), queueMetadata.getQueue().getQueueId())); ++ queueCount.incrementAndGet(); ++ }); ++- logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", ++- topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); +++ +++ if (queueCount.get() == 0L) { +++ metadataStore.deleteTopic(topicMetadata.getTopic()); +++ } else { +++ logger.info("Recover TopicFlatFile, topic: {}, queueCount: {}, cost: {}ms", +++ topicMetadata.getTopic(), queueCount.get(), subWatch.elapsed(TimeUnit.MILLISECONDS)); +++ } ++ } catch (Exception e) { ++ logger.error("Recover TopicFlatFile error, topic: {}", topicMetadata.getTopic(), e); ++ } finally { ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++new file mode 100644 ++index 000000000..50adbb713 ++--- /dev/null +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++@@ -0,0 +1,45 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.tieredstore.provider; +++ +++import java.util.HashSet; +++import java.util.Set; +++import org.apache.commons.lang3.StringUtils; +++import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; +++ +++public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter { +++ +++ private final Set topicBlackSet; +++ +++ public TieredStoreTopicBlackListFilter() { +++ this.topicBlackSet = new HashSet<>(); +++ } +++ +++ @Override +++ public boolean filterTopic(String topicName) { +++ if (StringUtils.isBlank(topicName)) { +++ return true; +++ } +++ return TieredStoreUtil.isSystemTopic(topicName) || topicBlackSet.contains(topicName); +++ } +++ +++ @Override +++ public void addTopicToWhiteList(String topicName) { +++ this.topicBlackSet.add(topicName); +++ } +++} ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++new file mode 100644 ++index 000000000..3f26b8b02 ++--- /dev/null +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++@@ -0,0 +1,25 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.tieredstore.provider; +++ +++public interface TieredStoreTopicFilter { +++ +++ boolean filterTopic(String topicName); +++ +++ void addTopicToWhiteList(String topicName); +++} ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++new file mode 100644 ++index 000000000..2bf48173c ++--- /dev/null +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++@@ -0,0 +1,36 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++package org.apache.rocketmq.tieredstore.provider; +++ +++import org.apache.rocketmq.common.topic.TopicValidator; +++import org.junit.Assert; +++import org.junit.Test; +++ +++public class TieredStoreTopicBlackListFilterTest { +++ +++ @Test +++ public void filterTopicTest() { +++ TieredStoreTopicFilter topicFilter = new TieredStoreTopicBlackListFilter(); +++ Assert.assertTrue(topicFilter.filterTopic("")); +++ Assert.assertTrue(topicFilter.filterTopic(TopicValidator.SYSTEM_TOPIC_PREFIX + "_Topic")); +++ +++ String topicName = "WhiteTopic"; +++ Assert.assertFalse(topicFilter.filterTopic(topicName)); +++ topicFilter.addTopicToWhiteList(topicName); +++ Assert.assertTrue(topicFilter.filterTopic(topicName)); +++ } +++} ++\ No newline at end of file ++-- ++2.32.0.windows.2 ++ ++ ++From 628020537fa7035226bc8dcde9fa33d9d5df30ff Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Thu, 7 Sep 2023 16:17:47 +0800 ++Subject: [PATCH 2/3] [ISSUE #7293] Fix NPE when alter sync state set ++ ++--- ++ .../rocketmq/controller/impl/manager/ReplicasInfoManager.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java ++index b0a67531d..d83a690f9 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/manager/ReplicasInfoManager.java ++@@ -104,7 +104,7 @@ public class ReplicasInfoManager { ++ } ++ ++ // Check master ++- if (!syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) { +++ if (syncStateInfo.getMasterBrokerId() == null || !syncStateInfo.getMasterBrokerId().equals(request.getMasterBrokerId())) { ++ String err = String.format("Rejecting alter syncStateSet request because the current leader is:{%s}, not {%s}", ++ syncStateInfo.getMasterBrokerId(), request.getMasterBrokerId()); ++ LOGGER.error("{}", err); ++-- ++2.32.0.windows.2 ++ ++ ++From 6fd0073d6475c539e8f4c30dc4f104a56a21d724 Mon Sep 17 00:00:00 2001 ++From: Ji Juntao ++Date: Thu, 7 Sep 2023 20:21:16 +0800 ++Subject: [PATCH 3/3] [ISSUE #7319] Optimize fault-tolerant mechanism for ++ sending messages and hot update switch (#7320) ++ ++--- ++ .../impl/producer/DefaultMQProducerImpl.java | 8 ++------ ++ .../client/latency/LatencyFaultTolerance.java | 14 +++++++++++++ ++ .../latency/LatencyFaultToleranceImpl.java | 13 +++++++++++- ++ .../client/latency/MQFaultStrategy.java | 20 +++++++------------ ++ .../proxy/service/route/MessageQueueView.java | 9 --------- ++ .../service/route/TopicRouteService.java | 10 +++++++++- ++ 6 files changed, 44 insertions(+), 30 deletions(-) ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++index 2d6b83ac2..b0c212e46 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/producer/DefaultMQProducerImpl.java ++@@ -263,9 +263,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ mQClientFactory.start(); ++ } ++ ++- if (this.mqFaultStrategy.isStartDetectorEnable()) { ++- this.mqFaultStrategy.startDetector(); ++- } +++ this.mqFaultStrategy.startDetector(); ++ ++ log.info("the producer [{}] start OK. sendMessageWithVIPChannel={}", this.defaultMQProducer.getProducerGroup(), ++ this.defaultMQProducer.isSendMessageWithVIPChannel()); ++@@ -311,9 +309,7 @@ public class DefaultMQProducerImpl implements MQProducerInner { ++ if (shutdownFactory) { ++ this.mQClientFactory.shutdown(); ++ } ++- if (this.mqFaultStrategy.isStartDetectorEnable()) { ++- this.mqFaultStrategy.shutdown(); ++- } +++ this.mqFaultStrategy.shutdown(); ++ RequestFutureHolder.getInstance().shutdown(this); ++ log.info("the producer [{}] shutdown OK", this.defaultMQProducer.getProducerGroup()); ++ this.serviceState = ServiceState.SHUTDOWN_ALREADY; ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++index 72d2f3450..17aaa266a 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultTolerance.java ++@@ -89,4 +89,18 @@ public interface LatencyFaultTolerance { ++ * @param detectInterval each broker's detecting interval ++ */ ++ void setDetectInterval(final int detectInterval); +++ +++ /** +++ * Use it to set the detector work or not. +++ * +++ * @param startDetectorEnable set the detector's work status +++ */ +++ void setStartDetectorEnable(final boolean startDetectorEnable); +++ +++ /** +++ * Use it to judge if the detector enabled. +++ * +++ * @return is the detector should be started. +++ */ +++ boolean isStartDetectorEnable(); ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++index 8af629574..d3ff7eb45 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/LatencyFaultToleranceImpl.java ++@@ -37,6 +37,8 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ private int detectTimeout = 200; ++ private int detectInterval = 2000; ++ private final ThreadLocalIndex whichItemWorst = new ThreadLocalIndex(); +++ +++ private volatile boolean startDetectorEnable = false; ++ private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() { ++ @Override ++ public Thread newThread(Runnable r) { ++@@ -80,7 +82,9 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ @Override ++ public void run() { ++ try { ++- detectByOneRound(); +++ if (startDetectorEnable) { +++ detectByOneRound(); +++ } ++ } catch (Exception e) { ++ log.warn("Unexpected exception raised while detecting service reachability", e); ++ } ++@@ -137,6 +141,13 @@ public class LatencyFaultToleranceImpl implements LatencyFaultTolerance ++ this.faultItemTable.remove(name); ++ } ++ +++ public boolean isStartDetectorEnable() { +++ return startDetectorEnable; +++ } +++ +++ public void setStartDetectorEnable(boolean startDetectorEnable) { +++ this.startDetectorEnable = startDetectorEnable; +++ } ++ @Override ++ public String pickOneAtLeast() { ++ final Enumeration elements = this.faultItemTable.elements(); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++index c01490784..69fb533e5 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java +++++ b/client/src/main/java/org/apache/rocketmq/client/latency/MQFaultStrategy.java ++@@ -24,8 +24,8 @@ import org.apache.rocketmq.common.message.MessageQueue; ++ ++ public class MQFaultStrategy { ++ private LatencyFaultTolerance latencyFaultTolerance; ++- private boolean sendLatencyFaultEnable; ++- private boolean startDetectorEnable; +++ private volatile boolean sendLatencyFaultEnable; +++ private volatile boolean startDetectorEnable; ++ private long[] latencyMax = {50L, 100L, 550L, 1800L, 3000L, 5000L, 15000L}; ++ private long[] notAvailableDuration = {0L, 0L, 2000L, 5000L, 6000L, 10000L, 30000L}; ++ ++@@ -64,11 +64,11 @@ public class MQFaultStrategy { ++ ++ ++ public MQFaultStrategy(ClientConfig cc, Resolver fetcher, ServiceDetector serviceDetector) { ++- this.setStartDetectorEnable(cc.isStartDetectorEnable()); ++- this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); ++ this.latencyFaultTolerance = new LatencyFaultToleranceImpl(fetcher, serviceDetector); ++ this.latencyFaultTolerance.setDetectInterval(cc.getDetectInterval()); ++ this.latencyFaultTolerance.setDetectTimeout(cc.getDetectTimeout()); +++ this.setStartDetectorEnable(cc.isStartDetectorEnable()); +++ this.setSendLatencyFaultEnable(cc.isSendLatencyEnable()); ++ } ++ ++ // For unit test. ++@@ -123,21 +123,15 @@ public class MQFaultStrategy { ++ ++ public void setStartDetectorEnable(boolean startDetectorEnable) { ++ this.startDetectorEnable = startDetectorEnable; +++ this.latencyFaultTolerance.setStartDetectorEnable(startDetectorEnable); ++ } ++ ++ public void startDetector() { ++- // user should start the detector ++- // and the thread should not be in running state. ++- if (this.sendLatencyFaultEnable && this.startDetectorEnable) { ++- // start the detector. ++- this.latencyFaultTolerance.startDetector(); ++- } +++ this.latencyFaultTolerance.startDetector(); ++ } ++ ++ public void shutdown() { ++- if (this.sendLatencyFaultEnable && this.startDetectorEnable) { ++- this.latencyFaultTolerance.shutdown(); ++- } +++ this.latencyFaultTolerance.shutdown(); ++ } ++ ++ public MessageQueue selectOneMessageQueue(final TopicPublishInfo tpInfo, final String lastBrokerName, final boolean resetIndex) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++index 8b3c2f7c8..898e529f8 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/MessageQueueView.java ++@@ -26,7 +26,6 @@ public class MessageQueueView { ++ private final MessageQueueSelector readSelector; ++ private final MessageQueueSelector writeSelector; ++ private final TopicRouteWrapper topicRouteWrapper; ++- private MQFaultStrategy mqFaultStrategy; ++ ++ public MessageQueueView(String topic, TopicRouteData topicRouteData, MQFaultStrategy mqFaultStrategy) { ++ this.topicRouteWrapper = new TopicRouteWrapper(topicRouteData, topic); ++@@ -67,12 +66,4 @@ public class MessageQueueView { ++ .add("topicRouteWrapper", topicRouteWrapper) ++ .toString(); ++ } ++- ++- public MQFaultStrategy getMQFaultStrategy() { ++- return mqFaultStrategy; ++- } ++- ++- public void setMQFaultStrategy(MQFaultStrategy mqFaultStrategy) { ++- this.mqFaultStrategy = mqFaultStrategy; ++- } ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++index 74769a423..caf62a1e0 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++@@ -127,7 +127,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ @Override ++ public String resolve(String name) { ++ try { ++- String brokerAddr = getBrokerAddr(null, name); +++ String brokerAddr = getBrokerAddr(ProxyContext.createForInner("MQFaultStrategy"), name); ++ return brokerAddr; ++ } catch (Exception e) { ++ return null; ++@@ -175,9 +175,17 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ ++ public void updateFaultItem(final String brokerName, final long currentLatency, boolean isolation, ++ boolean reachable) { +++ checkSendFaultToleranceEnable(); ++ this.mqFaultStrategy.updateFaultItem(brokerName, currentLatency, isolation, reachable); ++ } ++ +++ public void checkSendFaultToleranceEnable() { +++ boolean hotLatencySwitch = ConfigurationManager.getProxyConfig().isSendLatencyEnable(); +++ boolean hotDetectorSwitch = ConfigurationManager.getProxyConfig().isStartDetectorEnable(); +++ this.mqFaultStrategy.setSendLatencyFaultEnable(hotLatencySwitch); +++ this.mqFaultStrategy.setStartDetectorEnable(hotDetectorSwitch); +++ } +++ ++ public MQFaultStrategy getMqFaultStrategy() { ++ return this.mqFaultStrategy; ++ } ++-- ++2.32.0.windows.2 ++ +diff --git a/patch017-backport-Convergent-thread-pool-creation.patch b/patch017-backport-Convergent-thread-pool-creation.patch +new file mode 100644 +index 000000000..92d0bd2e9 +--- /dev/null ++++ b/patch017-backport-Convergent-thread-pool-creation.patch +@@ -0,0 +1,2243 @@ ++From c100d815d754d7cb330bc63e145bafd2d9b59cb1 Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Mon, 11 Sep 2023 10:13:56 +0800 ++Subject: [PATCH 1/6] [ISSUE #7328] Convergent thread pool creation (#7329) ++ ++* Convergence thread pool creation to facilitate subsequent iteration management ++ ++* Convergence thread pool creation in ThreadPoolMonitor.java ++ ++* fix unit test ++ ++* Convergence ThreadPool constructor ++ ++* Convergence ScheduledThreadPool constructor ++ ++* remove unused import ++ ++* Convergence ScheduledThreadPool constructor ++ ++* remove unused import ++ ++--------- ++--- ++ .../rocketmq/broker/BrokerController.java | 39 +++++----- ++ .../client/ClientHousekeepingService.java | 4 +- ++ .../DefaultConsumerIdsChangeListener.java | 3 +- ++ .../broker/controller/ReplicasManager.java | 9 +-- ++ .../dledger/DLedgerRoleChangeHandler.java | 4 +- ++ .../broker/failover/EscapeBridge.java | 4 +- ++ .../broker/latency/BrokerFastFailure.java | 5 +- ++ .../BrokerFixedThreadPoolExecutor.java | 57 -------------- ++ .../broker/latency/FutureTaskExt.java | 39 ---------- ++ .../rocketmq/broker/out/BrokerOuterAPI.java | 7 +- ++ .../schedule/ScheduleMessageService.java | 7 +- ++ .../broker/topic/TopicRouteInfoManager.java | 4 +- ++ ...ractTransactionalMessageCheckListener.java | 4 +- ++ .../rocketmq/broker/BrokerControllerTest.java | 2 +- ++ .../broker/latency/BrokerFastFailureTest.java | 1 + ++ .../common/config/AbstractRocksDBStorage.java | 6 +- ++ .../FutureTaskExtThreadPoolExecutor.java | 3 +- ++ .../common/thread/ThreadPoolMonitor.java | 6 +- ++ .../rocketmq/common/utils/ThreadUtils.java | 74 ++++++++++++++++--- ++ .../rocketmq/container/BrokerContainer.java | 6 +- ++ .../controller/ControllerManager.java | 14 +--- ++ .../controller/impl/DLedgerController.java | 10 +-- ++ .../DefaultBrokerHeartbeatManager.java | 3 +- ++ .../rocketmq/namesrv/NamesrvController.java | 22 ++---- ++ .../grpc/v2/channel/GrpcChannelManager.java | 6 +- ++ .../remoting/RemotingProtocolServer.java | 4 +- ++ .../proxy/service/ClusterServiceManager.java | 12 +-- ++ .../proxy/service/LocalServiceManager.java | 4 +- ++ .../receipt/DefaultReceiptHandleManager.java | 8 +- ++ .../service/route/TopicRouteService.java | 9 +-- ++ .../remoting/netty/NettyRemotingClient.java | 4 +- ++ .../remoting/netty/NettyRemotingServer.java | 4 +- ++ .../rocketmq/store/DefaultMessageStore.java | 8 +- ++ .../ha/autoswitch/AutoSwitchHAService.java | 38 +++++----- ++ .../rocketmq/store/kv/CompactionStore.java | 21 +++--- ++ .../store/queue/ConsumeQueueStore.java | 4 +- ++ .../store/stats/BrokerStatsManager.java | 14 ++-- ++ .../store/timer/TimerMessageStore.java | 6 +- ++ .../apache/rocketmq/test/util/StatUtil.java | 1 - ++ .../common/TieredStoreExecutor.java | 14 ++-- ++ .../tools/admin/DefaultMQAdminExtImpl.java | 3 +- ++ 41 files changed, 215 insertions(+), 278 deletions(-) ++ delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java ++ delete mode 100644 broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 6aba70cb2..275b64b1a 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -34,7 +34,6 @@ import org.apache.rocketmq.broker.failover.EscapeBridge; ++ import org.apache.rocketmq.broker.filter.CommitLogDispatcherCalcBitMap; ++ import org.apache.rocketmq.broker.filter.ConsumerFilterManager; ++ import org.apache.rocketmq.broker.latency.BrokerFastFailure; ++-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; ++ import org.apache.rocketmq.broker.longpolling.LmqPullRequestHoldService; ++ import org.apache.rocketmq.broker.longpolling.NotifyMessageArrivingListener; ++ import org.apache.rocketmq.broker.longpolling.PullRequestHoldService; ++@@ -98,6 +97,7 @@ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++ import org.apache.rocketmq.common.stats.MomentStatsItem; ++ import org.apache.rocketmq.common.utils.ServiceProvider; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.Configuration; ++@@ -160,7 +160,6 @@ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ScheduledFuture; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.locks.Lock; ++ import java.util.concurrent.locks.ReentrantLock; ++@@ -455,10 +454,10 @@ public class BrokerController { ++ * Initialize resources including remoting server and thread executors. ++ */ ++ protected void initializeResources() { ++- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("BrokerControllerScheduledThread", true, getBrokerIdentity())); ++ ++- this.sendMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.sendMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getSendMessageThreadPoolNums(), ++ this.brokerConfig.getSendMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -466,7 +465,7 @@ public class BrokerController { ++ this.sendThreadPoolQueue, ++ new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity())); ++ ++- this.pullMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.pullMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getPullMessageThreadPoolNums(), ++ this.brokerConfig.getPullMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -474,7 +473,7 @@ public class BrokerController { ++ this.pullThreadPoolQueue, ++ new ThreadFactoryImpl("PullMessageThread_", getBrokerIdentity())); ++ ++- this.litePullMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.litePullMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getLitePullMessageThreadPoolNums(), ++ this.brokerConfig.getLitePullMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -482,7 +481,7 @@ public class BrokerController { ++ this.litePullThreadPoolQueue, ++ new ThreadFactoryImpl("LitePullMessageThread_", getBrokerIdentity())); ++ ++- this.putMessageFutureExecutor = new BrokerFixedThreadPoolExecutor( +++ this.putMessageFutureExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getPutMessageFutureThreadPoolNums(), ++ this.brokerConfig.getPutMessageFutureThreadPoolNums(), ++ 1000 * 60, ++@@ -490,7 +489,7 @@ public class BrokerController { ++ this.putThreadPoolQueue, ++ new ThreadFactoryImpl("SendMessageThread_", getBrokerIdentity())); ++ ++- this.ackMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.ackMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getAckMessageThreadPoolNums(), ++ this.brokerConfig.getAckMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -498,7 +497,7 @@ public class BrokerController { ++ this.ackThreadPoolQueue, ++ new ThreadFactoryImpl("AckMessageThread_", getBrokerIdentity())); ++ ++- this.queryMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.queryMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getQueryMessageThreadPoolNums(), ++ this.brokerConfig.getQueryMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -506,7 +505,7 @@ public class BrokerController { ++ this.queryThreadPoolQueue, ++ new ThreadFactoryImpl("QueryMessageThread_", getBrokerIdentity())); ++ ++- this.adminBrokerExecutor = new BrokerFixedThreadPoolExecutor( +++ this.adminBrokerExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getAdminBrokerThreadPoolNums(), ++ this.brokerConfig.getAdminBrokerThreadPoolNums(), ++ 1000 * 60, ++@@ -514,7 +513,7 @@ public class BrokerController { ++ this.adminBrokerThreadPoolQueue, ++ new ThreadFactoryImpl("AdminBrokerThread_", getBrokerIdentity())); ++ ++- this.clientManageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.clientManageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getClientManageThreadPoolNums(), ++ this.brokerConfig.getClientManageThreadPoolNums(), ++ 1000 * 60, ++@@ -522,7 +521,7 @@ public class BrokerController { ++ this.clientManagerThreadPoolQueue, ++ new ThreadFactoryImpl("ClientManageThread_", getBrokerIdentity())); ++ ++- this.heartbeatExecutor = new BrokerFixedThreadPoolExecutor( +++ this.heartbeatExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getHeartbeatThreadPoolNums(), ++ this.brokerConfig.getHeartbeatThreadPoolNums(), ++ 1000 * 60, ++@@ -530,7 +529,7 @@ public class BrokerController { ++ this.heartbeatThreadPoolQueue, ++ new ThreadFactoryImpl("HeartbeatThread_", true, getBrokerIdentity())); ++ ++- this.consumerManageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.consumerManageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getConsumerManageThreadPoolNums(), ++ this.brokerConfig.getConsumerManageThreadPoolNums(), ++ 1000 * 60, ++@@ -538,7 +537,7 @@ public class BrokerController { ++ this.consumerManagerThreadPoolQueue, ++ new ThreadFactoryImpl("ConsumerManageThread_", true, getBrokerIdentity())); ++ ++- this.replyMessageExecutor = new BrokerFixedThreadPoolExecutor( +++ this.replyMessageExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getProcessReplyMessageThreadPoolNums(), ++ this.brokerConfig.getProcessReplyMessageThreadPoolNums(), ++ 1000 * 60, ++@@ -546,7 +545,7 @@ public class BrokerController { ++ this.replyThreadPoolQueue, ++ new ThreadFactoryImpl("ProcessReplyMessageThread_", getBrokerIdentity())); ++ ++- this.endTransactionExecutor = new BrokerFixedThreadPoolExecutor( +++ this.endTransactionExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getEndTransactionThreadPoolNums(), ++ this.brokerConfig.getEndTransactionThreadPoolNums(), ++ 1000 * 60, ++@@ -554,7 +553,7 @@ public class BrokerController { ++ this.endTransactionThreadPoolQueue, ++ new ThreadFactoryImpl("EndTransactionThread_", getBrokerIdentity())); ++ ++- this.loadBalanceExecutor = new BrokerFixedThreadPoolExecutor( +++ this.loadBalanceExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), ++ this.brokerConfig.getLoadBalanceProcessorThreadPoolNums(), ++ 1000 * 60, ++@@ -562,9 +561,9 @@ public class BrokerController { ++ this.loadBalanceThreadPoolQueue, ++ new ThreadFactoryImpl("LoadBalanceProcessorThread_", getBrokerIdentity())); ++ ++- this.syncBrokerMemberGroupExecutorService = new ScheduledThreadPoolExecutor(1, +++ this.syncBrokerMemberGroupExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("BrokerControllerSyncBrokerScheduledThread", getBrokerIdentity())); ++- this.brokerHeartbeatExecutorService = new ScheduledThreadPoolExecutor(1, +++ this.brokerHeartbeatExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("BrokerControllerHeartbeatScheduledThread", getBrokerIdentity())); ++ ++ this.topicQueueMappingCleanService = new TopicQueueMappingCleanService(this); ++@@ -828,8 +827,6 @@ public class BrokerController { ++ ++ initializeResources(); ++ ++- registerProcessor(); ++- ++ initializeScheduledTasks(); ++ ++ initialTransaction(); ++@@ -1690,6 +1687,8 @@ public class BrokerController { ++ } ++ } ++ }, 10, 5, TimeUnit.SECONDS); +++ +++ registerProcessor(); ++ } ++ ++ protected void scheduleSendHeartbeat() { ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java ++index 98e5f450f..cbb81f632 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/ClientHousekeepingService.java ++@@ -18,11 +18,11 @@ package org.apache.rocketmq.broker.client; ++ ++ import io.netty.channel.Channel; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.ChannelEventListener; ++@@ -35,7 +35,7 @@ public class ClientHousekeepingService implements ChannelEventListener { ++ ++ public ClientHousekeepingService(final BrokerController brokerController) { ++ this.brokerController = brokerController; ++- scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("ClientHousekeepingScheduledThread", brokerController.getBrokerIdentity())); ++ } ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java ++index 2ce036a0f..d17a2a547 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/client/DefaultConsumerIdsChangeListener.java ++@@ -22,7 +22,6 @@ import java.util.List; ++ import java.util.Map; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.common.AbstractBrokerRunnable; ++@@ -37,7 +36,7 @@ public class DefaultConsumerIdsChangeListener implements ConsumerIdsChangeListen ++ private final BrokerController brokerController; ++ private final int cacheSize = 8096; ++ ++- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ ThreadUtils.newGenericThreadFactory("DefaultConsumerIdsChangeListener", true)); ++ ++ private ConcurrentHashMap> consumerChannelMap = new ConcurrentHashMap<>(cacheSize); ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++index 37c82e434..a989e6e68 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/controller/ReplicasManager.java ++@@ -27,10 +27,8 @@ import java.util.concurrent.ArrayBlockingQueue; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ScheduledFuture; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ ++ import org.apache.commons.lang3.StringUtils; ++@@ -42,6 +40,7 @@ import org.apache.rocketmq.common.MixAll; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.EpochEntry; ++@@ -107,9 +106,9 @@ public class ReplicasManager { ++ public ReplicasManager(final BrokerController brokerController) { ++ this.brokerController = brokerController; ++ this.brokerOuterAPI = brokerController.getBrokerOuterAPI(); ++- this.scheduledService = Executors.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity())); ++- this.executorService = Executors.newFixedThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity())); ++- this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, +++ this.scheduledService = ThreadUtils.newScheduledThreadPool(3, new ThreadFactoryImpl("ReplicasManager_ScheduledService_", brokerController.getBrokerIdentity())); +++ this.executorService = ThreadUtils.newThreadPoolExecutor(3, new ThreadFactoryImpl("ReplicasManager_ExecutorService_", brokerController.getBrokerIdentity())); +++ this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, ++ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("ReplicasManager_scan_thread_", brokerController.getBrokerIdentity())); ++ this.haService = (AutoSwitchHAService) brokerController.getMessageStore().getHaService(); ++ this.brokerConfig = brokerController.getBrokerConfig(); ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java ++index 75023ee1b..e6cb97640 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/dledger/DLedgerRoleChangeHandler.java ++@@ -21,12 +21,12 @@ import io.openmessaging.storage.dledger.DLedgerServer; ++ import io.openmessaging.storage.dledger.MemberState; ++ import io.openmessaging.storage.dledger.utils.DLedgerUtils; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.Future; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.store.DefaultMessageStore; ++@@ -49,7 +49,7 @@ public class DLedgerRoleChangeHandler implements DLedgerLeaderElector.RoleChange ++ this.messageStore = messageStore; ++ this.dLedgerCommitLog = (DLedgerCommitLog) messageStore.getCommitLog(); ++ this.dLegerServer = dLedgerCommitLog.getdLedgerServer(); ++- this.executorService = Executors.newSingleThreadExecutor( +++ this.executorService = ThreadUtils.newSingleThreadExecutor( ++ new ThreadFactoryImpl("DLegerRoleChangeHandler_", brokerController.getBrokerIdentity())); ++ } ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java ++index 7c350fc1d..6a0817480 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/failover/EscapeBridge.java ++@@ -24,7 +24,6 @@ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.broker.BrokerController; ++@@ -43,6 +42,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++ import org.apache.rocketmq.common.message.MessageQueue; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++@@ -72,7 +72,7 @@ public class EscapeBridge { ++ public void start() throws Exception { ++ if (brokerController.getBrokerConfig().isEnableSlaveActingMaster() && brokerController.getBrokerConfig().isEnableRemoteEscape()) { ++ final BlockingQueue asyncSenderThreadPoolQueue = new LinkedBlockingQueue<>(50000); ++- this.defaultAsyncSenderExecutor = new ThreadPoolExecutor( +++ this.defaultAsyncSenderExecutor = ThreadUtils.newThreadPoolExecutor( ++ Runtime.getRuntime().availableProcessors(), ++ Runtime.getRuntime().availableProcessors(), ++ 1000 * 60, ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java ++index d3d0bc8ba..3b6e9dc67 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFastFailure.java ++@@ -18,13 +18,14 @@ package org.apache.rocketmq.broker.latency; ++ ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.common.AbstractBrokerRunnable; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.UtilAll; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.future.FutureTaskExt; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.netty.RequestTask; ++@@ -43,7 +44,7 @@ public class BrokerFastFailure { ++ ++ public BrokerFastFailure(final BrokerController brokerController) { ++ this.brokerController = brokerController; ++- this.scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("BrokerFastFailureScheduledThread", true, ++ brokerController == null ? null : brokerController.getBrokerConfig())); ++ } ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java ++deleted file mode 100644 ++index d2d1143a3..000000000 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/BrokerFixedThreadPoolExecutor.java +++++ /dev/null ++@@ -1,57 +0,0 @@ ++-/* ++- * Licensed to the Apache Software Foundation (ASF) under one or more ++- * contributor license agreements. See the NOTICE file distributed with ++- * this work for additional information regarding copyright ownership. ++- * The ASF licenses this file to You under the Apache License, Version 2.0 ++- * (the "License"); you may not use this file except in compliance with ++- * the License. You may obtain a copy of the License at ++- * ++- * http://www.apache.org/licenses/LICENSE-2.0 ++- * ++- * Unless required by applicable law or agreed to in writing, software ++- * distributed under the License is distributed on an "AS IS" BASIS, ++- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++- * See the License for the specific language governing permissions and ++- * limitations under the License. ++- */ ++- ++-package org.apache.rocketmq.broker.latency; ++- ++-import java.util.concurrent.BlockingQueue; ++-import java.util.concurrent.RejectedExecutionHandler; ++-import java.util.concurrent.RunnableFuture; ++-import java.util.concurrent.ThreadFactory; ++-import java.util.concurrent.ThreadPoolExecutor; ++-import java.util.concurrent.TimeUnit; ++- ++-public class BrokerFixedThreadPoolExecutor extends ThreadPoolExecutor { ++- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, ++- final TimeUnit unit, ++- final BlockingQueue workQueue) { ++- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); ++- } ++- ++- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, ++- final TimeUnit unit, ++- final BlockingQueue workQueue, final ThreadFactory threadFactory) { ++- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); ++- } ++- ++- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, ++- final TimeUnit unit, ++- final BlockingQueue workQueue, final RejectedExecutionHandler handler) { ++- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler); ++- } ++- ++- public BrokerFixedThreadPoolExecutor(final int corePoolSize, final int maximumPoolSize, final long keepAliveTime, ++- final TimeUnit unit, ++- final BlockingQueue workQueue, final ThreadFactory threadFactory, ++- final RejectedExecutionHandler handler) { ++- super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); ++- } ++- ++- @Override ++- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { ++- return new FutureTaskExt<>(runnable, value); ++- } ++-} ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java b/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java ++deleted file mode 100644 ++index f132efaeb..000000000 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/latency/FutureTaskExt.java +++++ /dev/null ++@@ -1,39 +0,0 @@ ++-/* ++- * Licensed to the Apache Software Foundation (ASF) under one or more ++- * contributor license agreements. See the NOTICE file distributed with ++- * this work for additional information regarding copyright ownership. ++- * The ASF licenses this file to You under the Apache License, Version 2.0 ++- * (the "License"); you may not use this file except in compliance with ++- * the License. You may obtain a copy of the License at ++- * ++- * http://www.apache.org/licenses/LICENSE-2.0 ++- * ++- * Unless required by applicable law or agreed to in writing, software ++- * distributed under the License is distributed on an "AS IS" BASIS, ++- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++- * See the License for the specific language governing permissions and ++- * limitations under the License. ++- */ ++- ++-package org.apache.rocketmq.broker.latency; ++- ++-import java.util.concurrent.Callable; ++-import java.util.concurrent.FutureTask; ++- ++-public class FutureTaskExt extends FutureTask { ++- private final Runnable runnable; ++- ++- public FutureTaskExt(final Callable callable) { ++- super(callable); ++- this.runnable = null; ++- } ++- ++- public FutureTaskExt(final Runnable runnable, final V result) { ++- super(runnable, result); ++- this.runnable = runnable; ++- } ++- ++- public Runnable getRunnable() { ++- return runnable; ++- } ++-} ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++index ae81e8b11..9dfb8127d 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++@@ -27,9 +27,9 @@ import java.util.concurrent.ArrayBlockingQueue; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.CopyOnWriteArrayList; ++ import java.util.concurrent.CountDownLatch; +++import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.StringUtils; ++-import org.apache.rocketmq.broker.latency.BrokerFixedThreadPoolExecutor; ++ import org.apache.rocketmq.client.consumer.PullResult; ++ import org.apache.rocketmq.client.consumer.PullStatus; ++ import org.apache.rocketmq.client.exception.MQBrokerException; ++@@ -59,6 +59,7 @@ import org.apache.rocketmq.common.namesrv.DefaultTopAddressing; ++ import org.apache.rocketmq.common.namesrv.TopAddressing; ++ import org.apache.rocketmq.common.sysflag.PullSysFlag; ++ import org.apache.rocketmq.common.topic.TopicValidator; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++@@ -144,7 +145,7 @@ public class BrokerOuterAPI { ++ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); ++ private final RemotingClient remotingClient; ++ private final TopAddressing topAddressing = new DefaultTopAddressing(MixAll.getWSAddr()); ++- private final BrokerFixedThreadPoolExecutor brokerOuterExecutor = new BrokerFixedThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, +++ private final ExecutorService brokerOuterExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 1, TimeUnit.MINUTES, ++ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("brokerOutApi_thread_", true)); ++ private final ClientMetadata clientMetadata; ++ private final RpcClient rpcClient; ++@@ -1092,7 +1093,7 @@ public class BrokerOuterAPI { ++ throw new MQBrokerException(response.getCode(), response.getRemark()); ++ } ++ ++- public BrokerFixedThreadPoolExecutor getBrokerOuterExecutor() { +++ public ExecutorService getBrokerOuterExecutor() { ++ return brokerOuterExecutor; ++ } ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++index 297b14207..0c2e6507b 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/schedule/ScheduleMessageService.java ++@@ -26,7 +26,6 @@ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicBoolean; ++ import java.util.concurrent.atomic.AtomicInteger; ++@@ -91,7 +90,7 @@ public class ScheduleMessageService extends ConfigManager { ++ public ScheduleMessageService(final BrokerController brokerController) { ++ this.brokerController = brokerController; ++ this.enableAsyncDeliver = brokerController.getMessageStoreConfig().isEnableScheduleAsyncDeliver(); ++- scheduledPersistService = new ScheduledThreadPoolExecutor(1, +++ scheduledPersistService = ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("ScheduleMessageServicePersistThread", true, brokerController.getBrokerConfig())); ++ } ++ ++@@ -134,9 +133,9 @@ public class ScheduleMessageService extends ConfigManager { ++ public void start() { ++ if (started.compareAndSet(false, true)) { ++ this.load(); ++- this.deliverExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_")); +++ this.deliverExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageTimerThread_")); ++ if (this.enableAsyncDeliver) { ++- this.handleExecutorService = new ScheduledThreadPoolExecutor(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_")); +++ this.handleExecutorService = ThreadUtils.newScheduledThreadPool(this.maxDelayLevel, new ThreadFactoryImpl("ScheduleMessageExecutorHandleThread_")); ++ } ++ for (Map.Entry entry : this.delayLevelTable.entrySet()) { ++ Integer level = entry.getKey(); ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java ++index b35564725..11bde5f5f 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicRouteInfoManager.java ++@@ -23,7 +23,6 @@ import java.util.Objects; ++ import java.util.Set; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.locks.Lock; ++@@ -36,6 +35,7 @@ import org.apache.rocketmq.common.MixAll; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.message.MessageQueue; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++@@ -66,7 +66,7 @@ public class TopicRouteInfoManager { ++ } ++ ++ public void start() { ++- this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread")); +++ this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("TopicRouteInfoManagerScheduledThread")); ++ ++ this.scheduledExecutorService.scheduleAtFixedRate(() -> { ++ try { ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java ++index 771d84300..982355d78 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/transaction/AbstractTransactionalMessageCheckListener.java ++@@ -19,7 +19,6 @@ package org.apache.rocketmq.broker.transaction; ++ import io.netty.channel.Channel; ++ import java.util.concurrent.ArrayBlockingQueue; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.ThreadPoolExecutor.CallerRunsPolicy; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++@@ -27,6 +26,7 @@ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.message.MessageConst; ++ import org.apache.rocketmq.common.message.MessageExt; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.header.CheckTransactionStateRequestHeader; ++@@ -97,7 +97,7 @@ public abstract class AbstractTransactionalMessageCheckListener { ++ ++ public synchronized void initExecutorService() { ++ if (executorService == null) { ++- executorService = new ThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), +++ executorService = ThreadUtils.newThreadPoolExecutor(2, 5, 100, TimeUnit.SECONDS, new ArrayBlockingQueue<>(2000), ++ new ThreadFactoryImpl("Transaction-msg-check-thread", brokerController.getBrokerIdentity()), new CallerRunsPolicy()); ++ } ++ } ++diff --git a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java ++index 75ad961ce..6035a20ac 100644 ++--- a/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java +++++ b/broker/src/test/java/org/apache/rocketmq/broker/BrokerControllerTest.java ++@@ -23,9 +23,9 @@ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.TimeUnit; ++ ++-import org.apache.rocketmq.broker.latency.FutureTaskExt; ++ import org.apache.rocketmq.common.BrokerConfig; ++ import org.apache.rocketmq.common.UtilAll; +++import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.remoting.netty.NettyClientConfig; ++ import org.apache.rocketmq.remoting.netty.NettyServerConfig; ++ import org.apache.rocketmq.remoting.netty.RequestTask; ++diff --git a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java ++index 5d0f7f9d7..31b547cf1 100644 ++--- a/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java +++++ b/broker/src/test/java/org/apache/rocketmq/broker/latency/BrokerFastFailureTest.java ++@@ -19,6 +19,7 @@ package org.apache.rocketmq.broker.latency; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.TimeUnit; +++import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.remoting.netty.RequestTask; ++ import org.junit.Test; ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++index a720a5be3..6f19a9815 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java +++++ b/common/src/main/java/org/apache/rocketmq/common/config/AbstractRocksDBStorage.java ++@@ -23,7 +23,6 @@ import java.util.List; ++ import java.util.Map; ++ import java.util.concurrent.ArrayBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.Semaphore; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++@@ -33,6 +32,7 @@ import com.google.common.collect.Maps; ++ ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.rocksdb.ColumnFamilyDescriptor; ++@@ -82,8 +82,8 @@ public abstract class AbstractRocksDBStorage { ++ private volatile boolean closed; ++ ++ private final Semaphore reloadPermit = new Semaphore(1); ++- private final ScheduledExecutorService reloadScheduler = new ScheduledThreadPoolExecutor(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); ++- private final ThreadPoolExecutor manualCompactionThread = new ThreadPoolExecutor( +++ private final ScheduledExecutorService reloadScheduler = ThreadUtils.newScheduledThreadPool(1, new ThreadFactoryImpl("RocksDBStorageReloadService_")); +++ private final ThreadPoolExecutor manualCompactionThread = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( ++ 1, 1, 1000 * 60, TimeUnit.MILLISECONDS, ++ new ArrayBlockingQueue(1), ++ new ThreadFactoryImpl("RocksDBManualCompactionService_"), ++diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java ++index 411da9221..7b68873a9 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java +++++ b/common/src/main/java/org/apache/rocketmq/common/thread/FutureTaskExtThreadPoolExecutor.java ++@@ -29,7 +29,8 @@ public class FutureTaskExtThreadPoolExecutor extends ThreadPoolExecutor { ++ ++ public FutureTaskExtThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, ++ TimeUnit unit, ++- BlockingQueue workQueue, ThreadFactory threadFactory, +++ BlockingQueue workQueue, +++ ThreadFactory threadFactory, ++ RejectedExecutionHandler handler) { ++ super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); ++ } ++diff --git a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java ++index 49d97a5d7..1bfabbffe 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java +++++ b/common/src/main/java/org/apache/rocketmq/common/thread/ThreadPoolMonitor.java ++@@ -22,12 +22,12 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder; ++ import java.util.Collections; ++ import java.util.List; ++ import java.util.concurrent.CopyOnWriteArrayList; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.UtilAll; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++@@ -36,7 +36,7 @@ public class ThreadPoolMonitor { ++ private static Logger waterMarkLogger = LoggerFactory.getLogger(ThreadPoolMonitor.class); ++ ++ private static final List MONITOR_EXECUTOR = new CopyOnWriteArrayList<>(); ++- private static final ScheduledExecutorService MONITOR_SCHEDULED = Executors.newSingleThreadScheduledExecutor( +++ private static final ScheduledExecutorService MONITOR_SCHEDULED = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryBuilder().setNameFormat("ThreadPoolMonitor-%d").build() ++ ); ++ ++@@ -81,7 +81,7 @@ public class ThreadPoolMonitor { ++ String name, ++ int queueCapacity, ++ List threadPoolStatusMonitors) { ++- ThreadPoolExecutor executor = new FutureTaskExtThreadPoolExecutor( +++ ThreadPoolExecutor executor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor( ++ corePoolSize, ++ maximumPoolSize, ++ keepAliveTime, ++diff --git a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java ++index 4b366d4e3..1644c6360 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java +++++ b/common/src/main/java/org/apache/rocketmq/common/utils/ThreadUtils.java ++@@ -20,38 +20,94 @@ package org.apache.rocketmq.common.utils; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; +++import java.util.concurrent.LinkedBlockingQueue; +++import java.util.concurrent.RejectedExecutionHandler; ++ import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.ThreadFactory; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.thread.FutureTaskExtThreadPoolExecutor; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ public final class ThreadUtils { ++ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.TOOLS_LOGGER_NAME); ++ ++- public static ExecutorService newThreadPoolExecutor(int corePoolSize, int maximumPoolSize, long keepAliveTime, ++- TimeUnit unit, BlockingQueue workQueue, String processName, boolean isDaemon) { ++- return new ThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon)); +++ public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) { +++ return ThreadUtils.newSingleThreadExecutor(newThreadFactory(processName, isDaemon)); ++ } ++ ++- public static ExecutorService newSingleThreadExecutor(String processName, boolean isDaemon) { ++- return Executors.newSingleThreadExecutor(newThreadFactory(processName, isDaemon)); +++ public static ExecutorService newSingleThreadExecutor(ThreadFactory threadFactory) { +++ return ThreadUtils.newThreadPoolExecutor(1, threadFactory); +++ } +++ +++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, ThreadFactory threadFactory) { +++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, corePoolSize, +++ 0L, TimeUnit.MILLISECONDS, +++ new LinkedBlockingQueue<>(), +++ threadFactory); +++ } +++ +++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, +++ int maximumPoolSize, +++ long keepAliveTime, +++ TimeUnit unit, BlockingQueue workQueue, +++ String processName, +++ boolean isDaemon) { +++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, newThreadFactory(processName, isDaemon)); +++ } +++ +++ public static ExecutorService newThreadPoolExecutor(final int corePoolSize, +++ final int maximumPoolSize, +++ final long keepAliveTime, +++ final TimeUnit unit, +++ final BlockingQueue workQueue, +++ final ThreadFactory threadFactory) { +++ return ThreadUtils.newThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, new ThreadPoolExecutor.AbortPolicy()); +++ } +++ +++ public static ExecutorService newThreadPoolExecutor(int corePoolSize, +++ int maximumPoolSize, +++ long keepAliveTime, +++ TimeUnit unit, +++ BlockingQueue workQueue, +++ ThreadFactory threadFactory, +++ RejectedExecutionHandler handler) { +++ return new FutureTaskExtThreadPoolExecutor(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); ++ } ++ ++ public static ScheduledExecutorService newSingleThreadScheduledExecutor(String processName, boolean isDaemon) { ++- return Executors.newSingleThreadScheduledExecutor(newThreadFactory(processName, isDaemon)); +++ return ThreadUtils.newScheduledThreadPool(1, processName, isDaemon); +++ } +++ +++ public static ScheduledExecutorService newSingleThreadScheduledExecutor(ThreadFactory threadFactory) { +++ return ThreadUtils.newScheduledThreadPool(1, threadFactory); +++ } +++ +++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize) { +++ return ThreadUtils.newScheduledThreadPool(corePoolSize, Executors.defaultThreadFactory()); ++ } ++ ++- public static ScheduledExecutorService newFixedThreadScheduledPool(int nThreads, String processName, +++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, String processName, ++ boolean isDaemon) { ++- return Executors.newScheduledThreadPool(nThreads, newThreadFactory(processName, isDaemon)); +++ return ThreadUtils.newScheduledThreadPool(corePoolSize, newThreadFactory(processName, isDaemon)); +++ } +++ +++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, ThreadFactory threadFactory) { +++ return ThreadUtils.newScheduledThreadPool(corePoolSize, threadFactory, new ThreadPoolExecutor.AbortPolicy()); +++ } +++ +++ public static ScheduledExecutorService newScheduledThreadPool(int corePoolSize, +++ ThreadFactory threadFactory, +++ RejectedExecutionHandler handler) { +++ return new ScheduledThreadPoolExecutor(corePoolSize, threadFactory, handler); ++ } ++ ++ public static ThreadFactory newThreadFactory(String processName, boolean isDaemon) { ++- return newGenericThreadFactory("Remoting-" + processName, isDaemon); +++ return newGenericThreadFactory("ThreadUtils-" + processName, isDaemon); ++ } ++ ++ public static ThreadFactory newGenericThreadFactory(String processName) { ++diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java ++index c6446f058..5b712bc30 100644 ++--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java +++++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainer.java ++@@ -47,14 +47,12 @@ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ ++ public class BrokerContainer implements IBrokerContainer { ++ private static final Logger LOG = LoggerFactory.getLogger(LoggerName.BROKER_LOGGER_NAME); ++ ++- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new BasicThreadFactory.Builder() ++ .namingPattern("BrokerContainerScheduledThread") ++ .daemon(true) ++@@ -143,7 +141,7 @@ public class BrokerContainer implements IBrokerContainer { ++ this.remotingServer = new NettyRemotingServer(this.nettyServerConfig, this.containerClientHouseKeepingService); ++ this.fastRemotingServer = this.remotingServer.newRemotingServer(this.nettyServerConfig.getListenPort() - 2); ++ ++- this.brokerContainerExecutor = new ThreadPoolExecutor( +++ this.brokerContainerExecutor = ThreadUtils.newThreadPoolExecutor( ++ 1, ++ 1, ++ 1000 * 60, ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java ++index 7c91e70da..3e6b0eba5 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerManager.java ++@@ -25,8 +25,6 @@ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; ++ import java.util.concurrent.Future; ++ import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.RunnableFuture; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ ++ import org.apache.commons.lang3.StringUtils; ++@@ -34,8 +32,8 @@ import org.apache.rocketmq.common.ControllerConfig; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++-import org.apache.rocketmq.common.future.FutureTaskExt; ++ +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy; ++ import org.apache.rocketmq.controller.impl.DLedgerController; ++ import org.apache.rocketmq.controller.impl.heartbeat.DefaultBrokerHeartbeatManager; ++@@ -93,18 +91,14 @@ public class ControllerManager { ++ ++ public boolean initialize() { ++ this.controllerRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.controllerConfig.getControllerRequestThreadPoolQueueCapacity()); ++- this.controllerRequestExecutor = new ThreadPoolExecutor( +++ this.controllerRequestExecutor = ThreadUtils.newThreadPoolExecutor( ++ this.controllerConfig.getControllerThreadPoolNums(), ++ this.controllerConfig.getControllerThreadPoolNums(), ++ 1000 * 60, ++ TimeUnit.MILLISECONDS, ++ this.controllerRequestThreadPoolQueue, ++- new ThreadFactoryImpl("ControllerRequestExecutorThread_")) { ++- @Override ++- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { ++- return new FutureTaskExt(runnable, value); ++- } ++- }; +++ new ThreadFactoryImpl("ControllerRequestExecutorThread_")); +++ ++ this.notifyService.initialize(); ++ if (StringUtils.isEmpty(this.controllerConfig.getControllerDLegerPeers())) { ++ throw new IllegalArgumentException("Attribute value controllerDLegerPeers of ControllerConfig is null or empty"); ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java ++index fa91f288e..33e4406e4 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/DLedgerController.java ++@@ -32,7 +32,6 @@ import java.util.Map; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ScheduledFuture; ++@@ -44,6 +43,7 @@ import org.apache.rocketmq.common.ControllerConfig; ++ import org.apache.rocketmq.common.ServiceThread; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.controller.Controller; ++ import org.apache.rocketmq.controller.elect.ElectPolicy; ++ import org.apache.rocketmq.controller.elect.impl.DefaultElectPolicy; ++@@ -66,11 +66,11 @@ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.apache.rocketmq.remoting.protocol.body.SyncStateSet; ++ import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.admin.CleanControllerBrokerDataRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.register.ApplyBrokerIdRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.register.GetNextBrokerIdRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.register.RegisterBrokerToControllerRequestHeader; ++@@ -136,7 +136,7 @@ public class DLedgerController implements Controller { ++ this.dLedgerServer = new DLedgerServer(dLedgerConfig, nettyServerConfig, nettyClientConfig, channelEventListener); ++ this.dLedgerServer.registerStateMachine(this.statemachine); ++ this.dLedgerServer.getDLedgerLeaderElector().addRoleChangeHandler(this.roleHandler); ++- this.scanInactiveMasterService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_")); +++ this.scanInactiveMasterService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DLedgerController_scanInactiveService_")); ++ this.brokerLifecycleListeners = new ArrayList<>(); ++ } ++ ++@@ -513,7 +513,7 @@ public class DLedgerController implements Controller { ++ class RoleChangeHandler implements DLedgerLeaderElector.RoleChangeHandler { ++ ++ private final String selfId; ++- private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_")); +++ private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("DLedgerControllerRoleChangeHandler_")); ++ private volatile MemberState.Role currentRole = MemberState.Role.FOLLOWER; ++ ++ public RoleChangeHandler(final String selfId) { ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java ++index 2fbddb9cd..6ebb2c994 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/impl/heartbeat/DefaultBrokerHeartbeatManager.java ++@@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ControllerConfig; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.controller.BrokerHeartbeatManager; ++ import org.apache.rocketmq.controller.helper.BrokerLifecycleListener; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++@@ -66,7 +67,7 @@ public class DefaultBrokerHeartbeatManager implements BrokerHeartbeatManager { ++ ++ @Override ++ public void initialize() { ++- this.scheduledService = Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_")); +++ this.scheduledService = ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_scheduledService_")); ++ this.executor = Executors.newFixedThreadPool(2, new ThreadFactoryImpl("DefaultBrokerHeartbeatManager_executorService_")); ++ } ++ ++diff --git a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java ++index 15c65ebec..be327cffa 100644 ++--- a/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java +++++ b/namesrv/src/main/java/org/apache/rocketmq/namesrv/NamesrvController.java ++@@ -20,10 +20,7 @@ import java.util.Collections; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.RunnableFuture; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.commons.lang3.concurrent.BasicThreadFactory; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++@@ -31,6 +28,7 @@ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.common.namesrv.NamesrvConfig; ++ import org.apache.rocketmq.common.utils.NetworkUtil; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.namesrv.kvconfig.KVConfigManager; ++@@ -62,10 +60,10 @@ public class NamesrvController { ++ private final NettyServerConfig nettyServerConfig; ++ private final NettyClientConfig nettyClientConfig; ++ ++- private final ScheduledExecutorService scheduledExecutorService = new ScheduledThreadPoolExecutor(1, +++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new BasicThreadFactory.Builder().namingPattern("NSScheduledThread").daemon(true).build()); ++ ++- private final ScheduledExecutorService scanExecutorService = new ScheduledThreadPoolExecutor(1, +++ private final ScheduledExecutorService scanExecutorService = ThreadUtils.newScheduledThreadPool(1, ++ new BasicThreadFactory.Builder().namingPattern("NSScanScheduledThread").daemon(true).build()); ++ ++ private final KVConfigManager kvConfigManager; ++@@ -138,20 +136,10 @@ public class NamesrvController { ++ ++ private void initiateThreadExecutors() { ++ this.defaultThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getDefaultThreadPoolQueueCapacity()); ++- this.defaultExecutor = new ThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")) { ++- @Override ++- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { ++- return new FutureTaskExt<>(runnable, value); ++- } ++- }; +++ this.defaultExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getDefaultThreadPoolNums(), this.namesrvConfig.getDefaultThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.defaultThreadPoolQueue, new ThreadFactoryImpl("RemotingExecutorThread_")); ++ ++ this.clientRequestThreadPoolQueue = new LinkedBlockingQueue<>(this.namesrvConfig.getClientRequestThreadPoolQueueCapacity()); ++- this.clientRequestExecutor = new ThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")) { ++- @Override ++- protected RunnableFuture newTaskFor(final Runnable runnable, final T value) { ++- return new FutureTaskExt<>(runnable, value); ++- } ++- }; +++ this.clientRequestExecutor = ThreadUtils.newThreadPoolExecutor(this.namesrvConfig.getClientRequestThreadPoolNums(), this.namesrvConfig.getClientRequestThreadPoolNums(), 1000 * 60, TimeUnit.MILLISECONDS, this.clientRequestThreadPoolQueue, new ThreadFactoryImpl("ClientRequestExecutorThread_")); ++ } ++ ++ private void initiateSslContext() { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java ++index 14330dd8d..a18cf7600 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/v2/channel/GrpcChannelManager.java ++@@ -21,13 +21,13 @@ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicLong; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++-import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; +++import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.grpc.v2.common.GrpcClientSettingsManager; ++@@ -43,7 +43,7 @@ public class GrpcChannelManager implements StartAndShutdown { ++ protected final AtomicLong nonceIdGenerator = new AtomicLong(0); ++ protected final ConcurrentMap resultNonceFutureMap = new ConcurrentHashMap<>(); ++ ++- protected final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( +++ protected final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("GrpcChannelManager_") ++ ); ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++index bcc9edd09..fe07090d5 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++@@ -22,7 +22,6 @@ import io.netty.channel.Channel; ++ import java.util.List; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.CompletableFuture; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++@@ -33,6 +32,7 @@ import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; ++ import org.apache.rocketmq.common.thread.ThreadPoolStatusMonitor; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++@@ -178,7 +178,7 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu ++ new ThreadPoolHeadSlowTimeMillsMonitor(config.getRemotingWaitTimeMillsInDefaultQueue()) ++ ); ++ ++- this.timerExecutor = Executors.newSingleThreadScheduledExecutor( +++ this.timerExecutor = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryBuilder().setNameFormat("RemotingServerScheduler-%d").build() ++ ); ++ this.timerExecutor.scheduleAtFixedRate(this::cleanExpireRequest, 10, 10, TimeUnit.SECONDS); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java ++index d2ddfc352..9786cec55 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/ClusterServiceManager.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.proxy.service; ++ ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.client.ClientChannelInfo; ++@@ -27,23 +26,24 @@ import org.apache.rocketmq.broker.client.ProducerChangeListener; ++ import org.apache.rocketmq.broker.client.ProducerGroupEvent; ++ import org.apache.rocketmq.broker.client.ProducerManager; ++ import org.apache.rocketmq.client.common.NameserverAccessConfig; +++import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor; +++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++-import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; ++ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.service.admin.AdminService; ++ import org.apache.rocketmq.proxy.service.admin.DefaultAdminService; ++ import org.apache.rocketmq.proxy.service.client.ClusterConsumerManager; +++import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor; ++ import org.apache.rocketmq.proxy.service.message.ClusterMessageService; ++ import org.apache.rocketmq.proxy.service.message.MessageService; ++ import org.apache.rocketmq.proxy.service.metadata.ClusterMetadataService; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++-import org.apache.rocketmq.client.impl.mqclient.DoNothingClientRemotingProcessor; ++-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++-import org.apache.rocketmq.proxy.service.client.ProxyClientRemotingProcessor; ++ import org.apache.rocketmq.proxy.service.relay.ClusterProxyRelayService; ++ import org.apache.rocketmq.proxy.service.relay.ProxyRelayService; ++ import org.apache.rocketmq.proxy.service.route.ClusterTopicRouteService; ++@@ -73,7 +73,7 @@ public class ClusterServiceManager extends AbstractStartAndShutdown implements S ++ ProxyConfig proxyConfig = ConfigurationManager.getProxyConfig(); ++ NameserverAccessConfig nameserverAccessConfig = new NameserverAccessConfig(proxyConfig.getNamesrvAddr(), ++ proxyConfig.getNamesrvDomain(), proxyConfig.getNamesrvDomainSubgroup()); ++- this.scheduledExecutorService = Executors.newScheduledThreadPool(3); +++ this.scheduledExecutorService = ThreadUtils.newScheduledThreadPool(3); ++ ++ this.messagingClientAPIFactory = new MQClientAPIFactory( ++ nameserverAccessConfig, ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java ++index 4d1ca7b66..59cd92685 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/LocalServiceManager.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.proxy.service; ++ ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.broker.BrokerController; ++@@ -28,6 +27,7 @@ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.proxy.service.admin.AdminService; ++@@ -58,7 +58,7 @@ public class LocalServiceManager extends AbstractStartAndShutdown implements Ser ++ private final MQClientAPIFactory mqClientAPIFactory; ++ private final ChannelManager channelManager; ++ ++- private final ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( +++ private final ScheduledExecutorService scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("LocalServiceManagerScheduledThread")); ++ ++ public LocalServiceManager(BrokerController brokerController, RPCHook rpcHook) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java ++index 69f44344a..207603fe8 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/receipt/DefaultReceiptHandleManager.java ++@@ -24,7 +24,6 @@ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++@@ -42,20 +41,21 @@ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; ++ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; ++ import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++-import org.apache.rocketmq.proxy.common.RenewEvent; ++ import org.apache.rocketmq.proxy.common.MessageReceiptHandle; ++ import org.apache.rocketmq.proxy.common.ProxyContext; ++ import org.apache.rocketmq.proxy.common.ProxyException; ++ import org.apache.rocketmq.proxy.common.ProxyExceptionCode; ++ import org.apache.rocketmq.proxy.common.ReceiptHandleGroup; +++import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey; +++import org.apache.rocketmq.proxy.common.RenewEvent; ++ import org.apache.rocketmq.proxy.common.RenewStrategyPolicy; ++ import org.apache.rocketmq.proxy.common.channel.ChannelHelper; ++ import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.config.ProxyConfig; ++-import org.apache.rocketmq.proxy.common.ReceiptHandleGroupKey; ++ import org.apache.rocketmq.proxy.service.metadata.MetadataService; ++ import org.apache.rocketmq.remoting.protocol.subscription.RetryPolicy; ++ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfig; ++@@ -68,7 +68,7 @@ public class DefaultReceiptHandleManager extends AbstractStartAndShutdown implem ++ protected final StateEventListener eventListener; ++ protected final static RetryPolicy RENEW_POLICY = new RenewStrategyPolicy(); ++ protected final ScheduledExecutorService scheduledExecutorService = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("RenewalScheduledThread_")); ++ protected final ThreadPoolExecutor renewalWorkerService; ++ ++ public DefaultReceiptHandleManager(MetadataService metadataService, ConsumerManager consumerManager, StateEventListener eventListener) { ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++index caf62a1e0..ccf094c03 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/route/TopicRouteService.java ++@@ -19,25 +19,24 @@ package org.apache.rocketmq.proxy.service.route; ++ import com.github.benmanes.caffeine.cache.CacheLoader; ++ import com.github.benmanes.caffeine.cache.Caffeine; ++ import com.github.benmanes.caffeine.cache.LoadingCache; +++import com.google.common.base.Optional; ++ import java.time.Duration; ++ import java.util.List; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++- ++-import com.google.common.base.Optional; ++ import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.exception.MQClientException; +++import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.client.latency.MQFaultStrategy; ++ import org.apache.rocketmq.client.latency.Resolver; ++ import org.apache.rocketmq.client.latency.ServiceDetector; ++-import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; ++ import org.apache.rocketmq.common.utils.AbstractStartAndShutdown; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.proxy.common.Address; ++@@ -63,7 +62,7 @@ public abstract class TopicRouteService extends AbstractStartAndShutdown { ++ public TopicRouteService(MQClientAPIFactory mqClientAPIFactory) { ++ ProxyConfig config = ConfigurationManager.getProxyConfig(); ++ ++- this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( +++ this.scheduledExecutorService = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("TopicRouteService_") ++ ); ++ this.cacheRefreshExecutor = ThreadPoolMonitor.createAndMonitor( ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++index 8491f4354..64621dd6c 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++@@ -61,7 +61,6 @@ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.concurrent.atomic.AtomicReference; ++@@ -71,6 +70,7 @@ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.ChannelEventListener; ++@@ -142,7 +142,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ ++ this.publicExecutor = Executors.newFixedThreadPool(publicThreadNums, new ThreadFactoryImpl("NettyClientPublicExecutor_")); ++ ++- this.scanExecutor = new ThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, +++ this.scanExecutor = ThreadUtils.newThreadPoolExecutor(4, 10, 60, TimeUnit.SECONDS, ++ new ArrayBlockingQueue<>(32), new ThreadFactoryImpl("NettyClientScan_thread_")); ++ ++ if (eventLoopGroup != null) { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++index e626260c9..aa0d46542 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++@@ -61,6 +61,7 @@ import org.apache.rocketmq.common.constant.HAProxyConstants; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.utils.BinaryUtil; ++ import org.apache.rocketmq.common.utils.NetworkUtil; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.ChannelEventListener; ++@@ -83,7 +84,6 @@ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ ++@@ -171,7 +171,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ } ++ ++ private ScheduledExecutorService buildScheduleExecutor() { ++- return new ScheduledThreadPoolExecutor(1, +++ return ThreadUtils.newScheduledThreadPool(1, ++ new ThreadFactoryImpl("NettyServerScheduler_", true), ++ new ThreadPoolExecutor.DiscardOldestPolicy()); ++ } ++diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++index f2a54ddf6..02ea47f13 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++@@ -48,7 +48,6 @@ import java.util.concurrent.ConcurrentLinkedQueue; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++@@ -83,6 +82,7 @@ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.common.utils.CleanupPolicyUtils; ++ import org.apache.rocketmq.common.utils.QueueTypeUtils; ++ import org.apache.rocketmq.common.utils.ServiceProvider; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; ++@@ -205,7 +205,7 @@ public class DefaultMessageStore implements MessageStore { ++ private ConcurrentMap topicConfigTable; ++ ++ private final ScheduledExecutorService scheduledCleanQueueExecutorService = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreCleanQueueScheduledThread")); ++ ++ public DefaultMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, ++ final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws IOException { ++@@ -253,7 +253,7 @@ public class DefaultMessageStore implements MessageStore { ++ this.transientStorePool = new TransientStorePool(messageStoreConfig.getTransientStorePoolSize(), messageStoreConfig.getMappedFileSizeCommitLog()); ++ ++ this.scheduledExecutorService = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("StoreScheduledThread", getBrokerIdentity())); ++ ++ this.dispatcherList = new LinkedList<>(); ++ this.dispatcherList.addLast(new CommitLogDispatcherBuildConsumeQueue()); ++@@ -2915,7 +2915,7 @@ public class DefaultMessageStore implements MessageStore { ++ private final ExecutorService batchDispatchRequestExecutor; ++ ++ public MainBatchDispatchRequestService() { ++- batchDispatchRequestExecutor = new ThreadPoolExecutor( +++ batchDispatchRequestExecutor = ThreadUtils.newThreadPoolExecutor( ++ DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), ++ DefaultMessageStore.this.getMessageStoreConfig().getBatchDispatchRequestThreadPoolNums(), ++ 1000 * 60, ++diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++index d5393fdca..f20bc3e28 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++@@ -17,10 +17,26 @@ ++ ++ package org.apache.rocketmq.store.ha.autoswitch; ++ ++- +++import java.io.IOException; +++import java.nio.channels.SocketChannel; +++import java.util.ArrayList; +++import java.util.HashSet; +++import java.util.Iterator; +++import java.util.List; +++import java.util.Map; +++import java.util.Objects; +++import java.util.Set; +++import java.util.concurrent.ConcurrentHashMap; +++import java.util.concurrent.ExecutorService; +++import java.util.concurrent.locks.Lock; +++import java.util.concurrent.locks.ReadWriteLock; +++import java.util.concurrent.locks.ReentrantReadWriteLock; +++import java.util.function.Consumer; +++import java.util.stream.Collectors; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.utils.ConcurrentHashMapUtils; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.EpochEntry; ++@@ -36,30 +52,12 @@ import org.apache.rocketmq.store.ha.HAClient; ++ import org.apache.rocketmq.store.ha.HAConnection; ++ import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService; ++ ++-import java.io.IOException; ++-import java.nio.channels.SocketChannel; ++-import java.util.ArrayList; ++-import java.util.HashSet; ++-import java.util.List; ++-import java.util.Iterator; ++-import java.util.Map; ++-import java.util.Objects; ++-import java.util.Set; ++-import java.util.concurrent.ConcurrentHashMap; ++-import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++-import java.util.concurrent.locks.Lock; ++-import java.util.concurrent.locks.ReadWriteLock; ++-import java.util.concurrent.locks.ReentrantReadWriteLock; ++-import java.util.function.Consumer; ++-import java.util.stream.Collectors; ++- ++ /** ++ * SwitchAble ha service, support switch role to master or slave. ++ */ ++ public class AutoSwitchHAService extends DefaultHAService { ++ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++- private final ExecutorService executorService = Executors.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_")); +++ private final ExecutorService executorService = ThreadUtils.newSingleThreadExecutor(new ThreadFactoryImpl("AutoSwitchHAService_Executor_")); ++ private final ConcurrentHashMap connectionCaughtUpTimeTable = new ConcurrentHashMap<>(); ++ private final List>> syncStateSetChangedListeners = new ArrayList<>(); ++ private final Set syncStateSet = new HashSet<>(); ++diff --git a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java ++index b37c90726..639084fa2 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/kv/CompactionStore.java ++@@ -16,17 +16,25 @@ ++ */ ++ package org.apache.rocketmq.store.kv; ++ ++-import java.util.Random; +++import java.io.File; +++import java.io.IOException; +++import java.nio.file.Files; +++import java.nio.file.Paths; ++ import java.util.Iterator; ++ import java.util.List; ++ import java.util.Map; ++ import java.util.Objects; ++ import java.util.Optional; +++import java.util.Random; +++import java.util.concurrent.ConcurrentHashMap; +++import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.TopicConfig; ++ import org.apache.rocketmq.common.attribute.CleanupPolicy; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.utils.CleanupPolicyUtils; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.store.DefaultMessageStore; ++@@ -35,15 +43,6 @@ import org.apache.rocketmq.store.GetMessageResult; ++ import org.apache.rocketmq.store.SelectMappedBufferResult; ++ import org.apache.rocketmq.store.config.MessageStoreConfig; ++ ++-import java.io.File; ++-import java.io.IOException; ++-import java.nio.file.Files; ++-import java.nio.file.Paths; ++-import java.util.concurrent.ConcurrentHashMap; ++-import java.util.concurrent.Executors; ++-import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.TimeUnit; ++- ++ public class CompactionStore { ++ ++ public static final String COMPACTION_DIR = "compaction"; ++@@ -76,7 +75,7 @@ public class CompactionStore { ++ this.positionMgr = new CompactionPositionMgr(compactionPath); ++ this.compactionThreadNum = Math.min(Runtime.getRuntime().availableProcessors(), Math.max(1, config.getCompactionThreadNum())); ++ ++- this.compactionSchedule = Executors.newScheduledThreadPool(this.compactionThreadNum, +++ this.compactionSchedule = ThreadUtils.newScheduledThreadPool(this.compactionThreadNum, ++ new ThreadFactoryImpl("compactionSchedule_")); ++ this.offsetMapSize = config.getMaxOffsetMapSize() / compactionThreadNum; ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java ++index 8d38503b3..d03d15d65 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java ++@@ -24,7 +24,6 @@ import java.util.concurrent.CountDownLatch; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.FutureTask; ++ import java.util.concurrent.LinkedBlockingQueue; ++-import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.TopicConfig; ++@@ -34,6 +33,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.common.utils.QueueTypeUtils; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.store.CommitLog; ++@@ -175,7 +175,7 @@ public class ConsumeQueueStore { ++ } ++ ++ private ExecutorService buildExecutorService(BlockingQueue blockingQueue, String threadNamePrefix) { ++- return new ThreadPoolExecutor( +++ return ThreadUtils.newThreadPoolExecutor( ++ this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(), ++ this.messageStore.getBrokerConfig().getRecoverThreadPoolNums(), ++ 1000 * 60, ++diff --git a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java ++index 2dd3fc5b5..489d7b4fb 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java +++++ b/store/src/main/java/org/apache/rocketmq/store/stats/BrokerStatsManager.java ++@@ -17,7 +17,6 @@ ++ package org.apache.rocketmq.store.stats; ++ ++ import java.util.HashMap; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.ScheduledExecutorService; ++ import org.apache.commons.lang3.tuple.Pair; ++ import org.apache.rocketmq.common.BrokerConfig; ++@@ -32,13 +31,14 @@ import org.apache.rocketmq.common.statistics.StatisticsItemScheduledPrinter; ++ import org.apache.rocketmq.common.statistics.StatisticsItemStateGetter; ++ import org.apache.rocketmq.common.statistics.StatisticsKindMeta; ++ import org.apache.rocketmq.common.statistics.StatisticsManager; +++import org.apache.rocketmq.common.stats.MomentStatsItemSet; ++ import org.apache.rocketmq.common.stats.Stats; +++import org.apache.rocketmq.common.stats.StatsItem; +++import org.apache.rocketmq.common.stats.StatsItemSet; ++ import org.apache.rocketmq.common.topic.TopicValidator; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++-import org.apache.rocketmq.common.stats.MomentStatsItemSet; ++-import org.apache.rocketmq.common.stats.StatsItem; ++-import org.apache.rocketmq.common.stats.StatsItemSet; ++ ++ public class BrokerStatsManager { ++ ++@@ -281,11 +281,11 @@ public class BrokerStatsManager { ++ ++ private void initScheduleService() { ++ this.scheduledExecutorService = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig)); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("BrokerStatsThread", true, brokerConfig)); ++ this.commercialExecutor = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig)); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("CommercialStatsThread", true, brokerConfig)); ++ this.accountExecutor = ++- Executors.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig)); +++ ThreadUtils.newSingleThreadScheduledExecutor(new ThreadFactoryImpl("AccountStatsThread", true, brokerConfig)); ++ } ++ ++ public MomentStatsItemSet getMomentStatsItemSetFallSize() { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++index 181f7087a..0d50de65a 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++@@ -35,7 +35,6 @@ import java.util.Set; ++ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ConcurrentSkipListSet; ++ import java.util.concurrent.CountDownLatch; ++-import java.util.concurrent.Executors; ++ import java.util.concurrent.LinkedBlockingDeque; ++ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++@@ -54,6 +53,7 @@ import org.apache.rocketmq.common.message.MessageDecoder; ++ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++ import org.apache.rocketmq.common.topic.TopicValidator; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.store.ConsumeQueue; ++@@ -174,11 +174,11 @@ public class TimerMessageStore { ++ this.lastBrokerRole = storeConfig.getBrokerRole(); ++ ++ if (messageStore instanceof DefaultMessageStore) { ++- scheduler = Executors.newSingleThreadScheduledExecutor( +++ scheduler = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("TimerScheduledThread", ++ ((DefaultMessageStore) messageStore).getBrokerIdentity())); ++ } else { ++- scheduler = Executors.newSingleThreadScheduledExecutor( +++ scheduler = ThreadUtils.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("TimerScheduledThread")); ++ } ++ ++diff --git a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java ++index f3d105bc6..080b7e385 100644 ++--- a/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java +++++ b/test/src/main/java/org/apache/rocketmq/test/util/StatUtil.java ++@@ -28,7 +28,6 @@ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.concurrent.atomic.AtomicLong; ++- ++ import javax.annotation.Generated; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++index 6dd0e8846..65d586f43 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/common/TieredStoreExecutor.java ++@@ -20,10 +20,10 @@ import java.util.concurrent.BlockingQueue; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ScheduledThreadPoolExecutor; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ ++ public class TieredStoreExecutor { ++ ++@@ -43,20 +43,20 @@ public class TieredStoreExecutor { ++ public static ExecutorService compactIndexFileExecutor; ++ ++ public static void init() { ++- commonScheduledExecutor = new ScheduledThreadPoolExecutor( +++ commonScheduledExecutor = ThreadUtils.newScheduledThreadPool( ++ Math.max(4, Runtime.getRuntime().availableProcessors()), ++ new ThreadFactoryImpl("TieredCommonExecutor_")); ++ ++- commitExecutor = new ScheduledThreadPoolExecutor( +++ commitExecutor = ThreadUtils.newScheduledThreadPool( ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++ new ThreadFactoryImpl("TieredCommitExecutor_")); ++ ++- cleanExpiredFileExecutor = new ScheduledThreadPoolExecutor( +++ cleanExpiredFileExecutor = ThreadUtils.newScheduledThreadPool( ++ Math.max(4, Runtime.getRuntime().availableProcessors()), ++ new ThreadFactoryImpl("TieredCleanFileExecutor_")); ++ ++ dispatchThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++- dispatchExecutor = new ThreadPoolExecutor( +++ dispatchExecutor = ThreadUtils.newThreadPoolExecutor( ++ Math.max(2, Runtime.getRuntime().availableProcessors()), ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++ 1000 * 60, ++@@ -66,7 +66,7 @@ public class TieredStoreExecutor { ++ new ThreadPoolExecutor.DiscardOldestPolicy()); ++ ++ fetchDataThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++- fetchDataExecutor = new ThreadPoolExecutor( +++ fetchDataExecutor = ThreadUtils.newThreadPoolExecutor( ++ Math.max(16, Runtime.getRuntime().availableProcessors() * 4), ++ Math.max(64, Runtime.getRuntime().availableProcessors() * 8), ++ 1000 * 60, ++@@ -75,7 +75,7 @@ public class TieredStoreExecutor { ++ new ThreadFactoryImpl("TieredFetchExecutor_")); ++ ++ compactIndexFileThreadPoolQueue = new LinkedBlockingQueue<>(QUEUE_CAPACITY); ++- compactIndexFileExecutor = new ThreadPoolExecutor( +++ compactIndexFileExecutor = ThreadUtils.newThreadPoolExecutor( ++ 1, ++ 1, ++ 1000 * 60, ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java ++index fa3596d51..1ebff6d8a 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/admin/DefaultMQAdminExtImpl.java ++@@ -66,6 +66,7 @@ import org.apache.rocketmq.common.namesrv.NamesrvUtil; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.common.utils.NetworkUtil; ++ import org.apache.rocketmq.common.BoundaryType; +++import org.apache.rocketmq.common.utils.ThreadUtils; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.RPCHook; ++@@ -193,7 +194,7 @@ public class DefaultMQAdminExtImpl implements MQAdminExt, MQAdminExtInner { ++ ++ int threadPoolCoreSize = Integer.parseInt(System.getProperty("rocketmq.admin.threadpool.coresize", "20")); ++ ++- this.threadPoolExecutor = new ThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_")); +++ this.threadPoolExecutor = (ThreadPoolExecutor) ThreadUtils.newThreadPoolExecutor(threadPoolCoreSize, 100, 5, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadFactoryImpl("DefaultMQAdminExtImpl_")); ++ ++ break; ++ case RUNNING: ++-- ++2.32.0.windows.2 ++ ++ ++From dad6b4dadfec7a58e78a6715ec16c2eb6b17ff27 Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Mon, 11 Sep 2023 14:34:10 +0800 ++Subject: [PATCH 2/6] [ISSUE #7334] `registerIncrementBrokerData` for single ++ topic update (#7335) ++ ++Signed-off-by: Ziy1-Tan ++--- ++ .../broker/topic/TopicConfigManager.java | 30 +++++++++++++++---- ++ 1 file changed, 25 insertions(+), 5 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++index 4e3c1736c..754605438 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++@@ -290,7 +290,11 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ ++ return topicConfig; ++@@ -394,7 +398,11 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ ++ return topicConfig; ++@@ -435,7 +443,11 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ ++ return topicConfig; ++@@ -461,7 +473,11 @@ public class TopicConfigManager extends ConfigManager { ++ dataVersion.nextVersion(stateMachineVersion); ++ ++ this.persist(); ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ } ++ ++@@ -484,7 +500,11 @@ public class TopicConfigManager extends ConfigManager { ++ dataVersion.nextVersion(stateMachineVersion); ++ ++ this.persist(); ++- this.brokerController.registerBrokerAll(false, true, true); +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } ++ } ++ } ++ ++-- ++2.32.0.windows.2 ++ ++ ++From 0dbd0772b99f618f757d42cd64542b83e2100e4f Mon Sep 17 00:00:00 2001 ++From: Ziyi Tan ++Date: Mon, 11 Sep 2023 15:48:07 +0800 ++Subject: [PATCH 3/6] [ISSUE #7326] Split the request to register to the ++ nameserver (#7325) ++ ++Signed-off-by: Ziy1-Tan ++--- ++ .../rocketmq/broker/BrokerController.java | 41 +++++++++++-------- ++ .../broker/topic/TopicConfigManager.java | 21 ++++++++++ ++ .../apache/rocketmq/common/BrokerConfig.java | 24 +++++++++++ ++ .../test/route/CreateAndUpdateTopicIT.java | 31 ++++++++++++++ ++ 4 files changed, 99 insertions(+), 18 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 275b64b1a..9e49f636d 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1765,29 +1765,34 @@ public class BrokerController { ++ } ++ ++ public synchronized void registerBrokerAll(final boolean checkOrderConfig, boolean oneway, boolean forceRegister) { +++ ConcurrentMap topicConfigMap = this.getTopicConfigManager().getTopicConfigTable(); +++ ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); ++ ++- TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); ++- ++- topicConfigWrapper.setDataVersion(this.getTopicConfigManager().getDataVersion()); ++- topicConfigWrapper.setTopicConfigTable(this.getTopicConfigManager().getTopicConfigTable()); ++- ++- topicConfigWrapper.setTopicQueueMappingInfoMap(this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream().map( ++- entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue())) ++- ).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue))); ++- ++- if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) ++- || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { ++- ConcurrentHashMap topicConfigTable = new ConcurrentHashMap<>(); ++- for (TopicConfig topicConfig : topicConfigWrapper.getTopicConfigTable().values()) { ++- TopicConfig tmp = +++ for (TopicConfig topicConfig : topicConfigMap.values()) { +++ if (!PermName.isWriteable(this.getBrokerConfig().getBrokerPermission()) +++ || !PermName.isReadable(this.getBrokerConfig().getBrokerPermission())) { +++ topicConfigTable.put(topicConfig.getTopicName(), ++ new TopicConfig(topicConfig.getTopicName(), topicConfig.getReadQueueNums(), topicConfig.getWriteQueueNums(), ++- topicConfig.getPerm() & this.brokerConfig.getBrokerPermission(), topicConfig.getTopicSysFlag()); ++- topicConfigTable.put(topicConfig.getTopicName(), tmp); +++ topicConfig.getPerm() & getBrokerConfig().getBrokerPermission())); +++ } else { +++ topicConfigTable.put(topicConfig.getTopicName(), topicConfig); +++ } +++ +++ if (this.brokerConfig.isEnableSplitRegistration() +++ && topicConfigTable.size() >= this.brokerConfig.getSplitRegistrationSize()) { +++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager().buildSerializeWrapper(topicConfigTable); +++ doRegisterBrokerAll(checkOrderConfig, oneway, topicConfigWrapper); +++ topicConfigTable.clear(); ++ } ++- topicConfigWrapper.setTopicConfigTable(topicConfigTable); ++ } ++ ++- if (forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), +++ Map topicQueueMappingInfoMap = this.getTopicQueueMappingManager().getTopicQueueMappingTable().entrySet().stream() +++ .map(entry -> new AbstractMap.SimpleImmutableEntry<>(entry.getKey(), TopicQueueMappingDetail.cloneAsMappingInfo(entry.getValue()))) +++ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); +++ +++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = this.getTopicConfigManager(). +++ buildSerializeWrapper(topicConfigTable, topicQueueMappingInfoMap); +++ if (this.brokerConfig.isEnableSplitRegistration() || forceRegister || needRegister(this.brokerConfig.getBrokerClusterName(), ++ this.getBrokerAddr(), ++ this.brokerConfig.getBrokerName(), ++ this.brokerConfig.getBrokerId(), ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++index 754605438..8537929be 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++@@ -29,6 +29,7 @@ import java.util.concurrent.locks.ReentrantLock; ++ ++ import com.google.common.collect.ImmutableMap; ++ +++import com.google.common.collect.Maps; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.broker.BrokerController; ++ import org.apache.rocketmq.broker.BrokerPathConfigHelper; ++@@ -47,7 +48,9 @@ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.protocol.DataVersion; ++ import org.apache.rocketmq.remoting.protocol.body.KVTable; +++import org.apache.rocketmq.remoting.protocol.body.TopicConfigAndMappingSerializeWrapper; ++ import org.apache.rocketmq.remoting.protocol.body.TopicConfigSerializeWrapper; +++import org.apache.rocketmq.remoting.protocol.statictopic.TopicQueueMappingInfo; ++ ++ import static com.google.common.base.Preconditions.checkNotNull; ++ ++@@ -609,6 +612,24 @@ public class TopicConfigManager extends ConfigManager { ++ return topicConfigSerializeWrapper; ++ } ++ +++ public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper(final ConcurrentMap topicConfigTable) { +++ return buildSerializeWrapper(topicConfigTable, Maps.newHashMap()); +++ } +++ +++ public TopicConfigAndMappingSerializeWrapper buildSerializeWrapper( +++ final ConcurrentMap topicConfigTable, +++ final Map topicQueueMappingInfoMap +++ ) { +++ TopicConfigAndMappingSerializeWrapper topicConfigWrapper = new TopicConfigAndMappingSerializeWrapper(); +++ topicConfigWrapper.setTopicConfigTable(topicConfigTable); +++ topicConfigWrapper.setTopicQueueMappingInfoMap(topicQueueMappingInfoMap); +++ topicConfigWrapper.setDataVersion(this.getDataVersion()); +++ if (this.brokerController.getBrokerConfig().isEnableSplitRegistration()) { +++ this.getDataVersion().nextVersion(); +++ } +++ return topicConfigWrapper; +++ } +++ ++ @Override ++ public String encode() { ++ return encode(false); ++diff --git a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++index 45d26b29c..0d248c4e1 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java +++++ b/common/src/main/java/org/apache/rocketmq/common/BrokerConfig.java ++@@ -396,6 +396,14 @@ public class BrokerConfig extends BrokerIdentity { ++ ++ private boolean enableMixedMessageType = false; ++ +++ /** +++ * This flag and deleteTopicWithBrokerRegistration flag in the NameServer cannot be set to true at the same time, +++ * otherwise there will be a loss of routing +++ */ +++ private boolean enableSplitRegistration = false; +++ +++ private int splitRegistrationSize = 800; +++ ++ public long getMaxPopPollingSize() { ++ return maxPopPollingSize; ++ } ++@@ -1731,4 +1739,20 @@ public class BrokerConfig extends BrokerIdentity { ++ public void setEnableMixedMessageType(boolean enableMixedMessageType) { ++ this.enableMixedMessageType = enableMixedMessageType; ++ } +++ +++ public boolean isEnableSplitRegistration() { +++ return enableSplitRegistration; +++ } +++ +++ public void setEnableSplitRegistration(boolean enableSplitRegistration) { +++ this.enableSplitRegistration = enableSplitRegistration; +++ } +++ +++ public int getSplitRegistrationSize() { +++ return splitRegistrationSize; +++ } +++ +++ public void setSplitRegistrationSize(int splitRegistrationSize) { +++ this.splitRegistrationSize = splitRegistrationSize; +++ } ++ } ++diff --git a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java ++index 7e3c7b871..2370e68c0 100644 ++--- a/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java +++++ b/test/src/test/java/org/apache/rocketmq/test/route/CreateAndUpdateTopicIT.java ++@@ -17,6 +17,7 @@ ++ ++ package org.apache.rocketmq.test.route; ++ +++import org.apache.rocketmq.common.TopicConfig; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++ import org.apache.rocketmq.test.base.BaseConf; ++ import org.apache.rocketmq.test.util.MQAdminTestUtils; ++@@ -111,4 +112,34 @@ public class CreateAndUpdateTopicIT extends BaseConf { ++ brokerController3.getBrokerConfig().setEnableSingleTopicRegister(false); ++ namesrvController.getNamesrvConfig().setDeleteTopicWithBrokerRegistration(false); ++ } +++ +++ @Test +++ public void testCreateOrUpdateTopic_EnableSplitRegistration() { +++ brokerController1.getBrokerConfig().setEnableSplitRegistration(true); +++ brokerController2.getBrokerConfig().setEnableSplitRegistration(true); +++ brokerController3.getBrokerConfig().setEnableSplitRegistration(true); +++ +++ String testTopic = "test-topic-"; +++ +++ for (int i = 0; i < 1000; i++) { +++ TopicConfig topicConfig = new TopicConfig(testTopic + i, 8, 8); +++ brokerController1.getTopicConfigManager().updateTopicConfig(topicConfig); +++ brokerController2.getTopicConfigManager().updateTopicConfig(topicConfig); +++ brokerController3.getTopicConfigManager().updateTopicConfig(topicConfig); +++ } +++ +++ brokerController1.registerBrokerAll(false, true, true); +++ brokerController2.registerBrokerAll(false, true, true); +++ brokerController3.registerBrokerAll(false, true, true); +++ +++ for (int i = 0; i < 1000; i++) { +++ TopicRouteData route = MQAdminTestUtils.examineTopicRouteInfo(NAMESRV_ADDR, testTopic + i); +++ assertThat(route.getBrokerDatas()).hasSize(3); +++ assertThat(route.getQueueDatas()).hasSize(3); +++ } +++ +++ brokerController1.getBrokerConfig().setEnableSplitRegistration(false); +++ brokerController2.getBrokerConfig().setEnableSplitRegistration(false); +++ brokerController3.getBrokerConfig().setEnableSplitRegistration(false); +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From a9e353285cea762b0c5eab567bdfa8e5c8c2d279 Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Mon, 11 Sep 2023 15:55:18 +0800 ++Subject: [PATCH 4/6] Add the configuration of topicQueueLock number to better ++ support different scenarios (#7317) ++ ++--- ++ .../main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- ++ .../java/org/apache/rocketmq/store/TopicQueueLock.java | 8 ++++++++ ++ .../rocketmq/store/config/MessageStoreConfig.java | 10 ++++++++++ ++ 3 files changed, 19 insertions(+), 1 deletion(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++index e6ee3bacc..456bf2b86 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++@@ -122,7 +122,7 @@ public class CommitLog implements Swappable { ++ ++ this.flushDiskWatcher = new FlushDiskWatcher(); ++ ++- this.topicQueueLock = new TopicQueueLock(); +++ this.topicQueueLock = new TopicQueueLock(messageStore.getMessageStoreConfig().getTopicQueueLockNum()); ++ ++ this.commitLogSize = messageStore.getMessageStoreConfig().getMappedFileSizeCommitLog(); ++ } ++diff --git a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java ++index a78eeed23..5a131b5c3 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java +++++ b/store/src/main/java/org/apache/rocketmq/store/TopicQueueLock.java ++@@ -34,6 +34,14 @@ public class TopicQueueLock { ++ } ++ } ++ +++ public TopicQueueLock(int size) { +++ this.size = size; +++ this.lockList = new ArrayList<>(size); +++ for (int i = 0; i < this.size; i++) { +++ this.lockList.add(new ReentrantLock()); +++ } +++ } +++ ++ public void lock(String topicQueueKey) { ++ Lock lock = this.lockList.get((topicQueueKey.hashCode() & 0x7fffffff) % this.size); ++ lock.lock(); ++diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java ++index efb728ac0..9fa448043 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +++++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java ++@@ -401,6 +401,8 @@ public class MessageStoreConfig { ++ private long memTableFlushInterval = 60 * 60 * 1000L; ++ private boolean enableRocksDBLog = false; ++ +++ private int topicQueueLockNum = 32; +++ ++ public boolean isDebugLockEnable() { ++ return debugLockEnable; ++ } ++@@ -1751,4 +1753,12 @@ public class MessageStoreConfig { ++ public void setEnableRocksDBLog(boolean enableRocksDBLog) { ++ this.enableRocksDBLog = enableRocksDBLog; ++ } +++ +++ public int getTopicQueueLockNum() { +++ return topicQueueLockNum; +++ } +++ +++ public void setTopicQueueLockNum(int topicQueueLockNum) { +++ this.topicQueueLockNum = topicQueueLockNum; +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 57f04c95d3a2ba6b91583058a6e4eda209f72d6e Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Mon, 11 Sep 2023 18:23:25 +0800 ++Subject: [PATCH 5/6] [ISSUE #7343] Rollback modifications to registerProcessor ++ ++Co-authored-by: guyinyou ++--- ++ .../java/org/apache/rocketmq/broker/BrokerController.java | 4 ++-- ++ 1 file changed, 2 insertions(+), 2 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 9e49f636d..13a3feb4e 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -827,6 +827,8 @@ public class BrokerController { ++ ++ initializeResources(); ++ +++ registerProcessor(); +++ ++ initializeScheduledTasks(); ++ ++ initialTransaction(); ++@@ -1687,8 +1689,6 @@ public class BrokerController { ++ } ++ } ++ }, 10, 5, TimeUnit.SECONDS); ++- ++- registerProcessor(); ++ } ++ ++ protected void scheduleSendHeartbeat() { ++-- ++2.32.0.windows.2 ++ ++ ++From dad6ad09d13dadc36b6342671c77f619bbb8c522 Mon Sep 17 00:00:00 2001 ++From: Ao Qiao ++Date: Tue, 12 Sep 2023 08:28:45 +0800 ++Subject: [PATCH 6/6] [ISSUE #7340] Abstract Duplicate code into a method in ++ `TopicConfigManager` (#7341) ++ ++--- ++ .../broker/topic/TopicConfigManager.java | 44 ++++++------------- ++ 1 file changed, 14 insertions(+), 30 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++index 8537929be..511d29e12 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/topic/TopicConfigManager.java ++@@ -293,11 +293,7 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ ++ return topicConfig; ++@@ -337,11 +333,7 @@ public class TopicConfigManager extends ConfigManager { ++ log.error("createTopicIfAbsent ", e); ++ } ++ if (createNew && register) { ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ return getTopicConfig(topicConfig.getTopicName()); ++ } ++@@ -401,11 +393,7 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ ++ return topicConfig; ++@@ -446,11 +434,7 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ ++ if (createNew) { ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ ++ return topicConfig; ++@@ -476,11 +460,7 @@ public class TopicConfigManager extends ConfigManager { ++ dataVersion.nextVersion(stateMachineVersion); ++ ++ this.persist(); ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ } ++ ++@@ -503,11 +483,7 @@ public class TopicConfigManager extends ConfigManager { ++ dataVersion.nextVersion(stateMachineVersion); ++ ++ this.persist(); ++- if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { ++- this.brokerController.registerSingleTopicAll(topicConfig); ++- } else { ++- this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); ++- } +++ registerBrokerData(topicConfig); ++ } ++ } ++ ++@@ -699,6 +675,14 @@ public class TopicConfigManager extends ConfigManager { ++ } ++ } ++ +++ private void registerBrokerData(TopicConfig topicConfig) { +++ if (brokerController.getBrokerConfig().isEnableSingleTopicRegister()) { +++ this.brokerController.registerSingleTopicAll(topicConfig); +++ } else { +++ this.brokerController.registerIncrementBrokerData(topicConfig, dataVersion); +++ } +++ } +++ ++ public boolean containsTopic(String topic) { ++ return topicConfigTable.containsKey(topic); ++ } ++-- ++2.32.0.windows.2 ++ +diff --git a/patch018-backport-enhancement-of-tiered-storage.patch b/patch018-backport-enhancement-of-tiered-storage.patch +new file mode 100644 +index 000000000..ed79df9c7 +--- /dev/null ++++ b/patch018-backport-enhancement-of-tiered-storage.patch +@@ -0,0 +1,601 @@ ++From 1a8e7cb17cb29ed33b0196b52e452a6e76ade781 Mon Sep 17 00:00:00 2001 ++From: yuz10 <845238369@qq.com> ++Date: Tue, 12 Sep 2023 19:33:41 +0800 ++Subject: [PATCH 1/5] [ISSUE #7345] Fix wrong result of searchOffset in tiered ++ storage ++ ++--- ++ .../tieredstore/file/TieredFlatFile.java | 5 +- ++ .../tieredstore/file/TieredFlatFileTest.java | 46 +++++++++++++++++-- ++ 2 files changed, 46 insertions(+), 5 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++index 426c4e09d..d973179ee 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++@@ -365,7 +365,10 @@ public class TieredFlatFile { ++ if (!segmentList.isEmpty()) { ++ return boundaryType == BoundaryType.UPPER ? segmentList.get(0) : segmentList.get(segmentList.size() - 1); ++ } ++- return fileSegmentList.isEmpty() ? null : fileSegmentList.get(fileSegmentList.size() - 1); +++ if (fileSegmentList.isEmpty()) { +++ return null; +++ } +++ return boundaryType == BoundaryType.UPPER ? fileSegmentList.get(fileSegmentList.size() - 1) : fileSegmentList.get(0); ++ } finally { ++ fileSegmentLock.readLock().unlock(); ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++index 7a4d05969..7e2fbf201 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileTest.java ++@@ -16,10 +16,7 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ ++-import java.io.IOException; ++-import java.nio.ByteBuffer; ++-import java.util.ArrayList; ++-import java.util.List; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.tieredstore.TieredStoreTestUtil; ++ import org.apache.rocketmq.tieredstore.common.FileSegmentType; ++@@ -35,6 +32,11 @@ import org.junit.Assert; ++ import org.junit.Before; ++ import org.junit.Test; ++ +++import java.io.IOException; +++import java.nio.ByteBuffer; +++import java.util.ArrayList; +++import java.util.List; +++ ++ public class TieredFlatFileTest { ++ ++ private final String storePath = TieredStoreTestUtil.getRandomStorePath(); ++@@ -301,4 +303,40 @@ public class TieredFlatFileTest { ++ fileQueue.rollingNewFile(); ++ Assert.assertEquals(2, fileQueue.getFileSegmentCount()); ++ } +++ +++ @Test +++ public void testGetFileByTime() { +++ String filePath = TieredStoreUtil.toPath(queue); +++ TieredFlatFile tieredFlatFile = fileQueueFactory.createFlatFileForCommitLog(filePath); +++ TieredFileSegment fileSegment1 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig); +++ fileSegment1.setMinTimestamp(100); +++ fileSegment1.setMaxTimestamp(200); +++ +++ TieredFileSegment fileSegment2 = new MemoryFileSegment(FileSegmentType.CONSUME_QUEUE, queue, 1100, storeConfig); +++ fileSegment2.setMinTimestamp(200); +++ fileSegment2.setMaxTimestamp(300); +++ +++ tieredFlatFile.getFileSegmentList().add(fileSegment1); +++ tieredFlatFile.getFileSegmentList().add(fileSegment2); +++ +++ TieredFileSegment segmentUpper = tieredFlatFile.getFileByTime(400, BoundaryType.UPPER); +++ Assert.assertEquals(fileSegment2, segmentUpper); +++ +++ TieredFileSegment segmentLower = tieredFlatFile.getFileByTime(400, BoundaryType.LOWER); +++ Assert.assertEquals(fileSegment2, segmentLower); +++ +++ +++ TieredFileSegment segmentUpper2 = tieredFlatFile.getFileByTime(0, BoundaryType.UPPER); +++ Assert.assertEquals(fileSegment1, segmentUpper2); +++ +++ TieredFileSegment segmentLower2 = tieredFlatFile.getFileByTime(0, BoundaryType.LOWER); +++ Assert.assertEquals(fileSegment1, segmentLower2); +++ +++ +++ TieredFileSegment segmentUpper3 = tieredFlatFile.getFileByTime(200, BoundaryType.UPPER); +++ Assert.assertEquals(fileSegment1, segmentUpper3); +++ +++ TieredFileSegment segmentLower3 = tieredFlatFile.getFileByTime(200, BoundaryType.LOWER); +++ Assert.assertEquals(fileSegment2, segmentLower3); +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From fd32dae2ab59f86dd215eeec405bf4fa6212bcb3 Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Tue, 12 Sep 2023 19:58:08 +0800 ++Subject: [PATCH 2/5] [ISSUE #6633] Not clear uninitialized files and fix ++ metadata recover (#7342) ++ ++--- ++ .../tieredstore/file/TieredFlatFile.java | 53 +++++++------------ ++ .../file/TieredFlatFileManager.java | 10 ++-- ++ 2 files changed, 22 insertions(+), 41 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++index d973179ee..d96eb6e8f 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFile.java ++@@ -16,7 +16,6 @@ ++ */ ++ package org.apache.rocketmq.tieredstore.file; ++ ++-import com.alibaba.fastjson.JSON; ++ import com.google.common.annotations.VisibleForTesting; ++ import java.nio.ByteBuffer; ++ import java.util.ArrayList; ++@@ -25,13 +24,13 @@ import java.util.Comparator; ++ import java.util.HashSet; ++ import java.util.LinkedList; ++ import java.util.List; ++-import java.util.Objects; ++ import java.util.Set; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.CopyOnWriteArrayList; ++ import java.util.concurrent.locks.ReentrantReadWriteLock; ++ import java.util.stream.Collectors; ++ import javax.annotation.Nullable; +++import org.apache.rocketmq.common.BoundaryType; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.tieredstore.common.AppendResult; ++@@ -43,7 +42,6 @@ import org.apache.rocketmq.tieredstore.metadata.TieredMetadataStore; ++ import org.apache.rocketmq.tieredstore.provider.FileSegmentAllocator; ++ import org.apache.rocketmq.tieredstore.provider.TieredFileSegment; ++ import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; ++-import org.apache.rocketmq.common.BoundaryType; ++ ++ public class TieredFlatFile { ++ ++@@ -177,7 +175,10 @@ public class TieredFlatFile { ++ } ++ } ++ ++- private FileSegmentMetadata getOrCreateFileSegmentMetadata(TieredFileSegment fileSegment) { +++ /** +++ * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full +++ */ +++ public void updateFileSegment(TieredFileSegment fileSegment) { ++ ++ FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( ++ this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); ++@@ -186,45 +187,24 @@ public class TieredFlatFile { ++ if (metadata == null) { ++ metadata = new FileSegmentMetadata( ++ this.filePath, fileSegment.getBaseOffset(), fileSegment.getFileType().getType()); ++- metadata.setCreateTimestamp(fileSegment.getMinTimestamp()); ++- metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); ++- metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++- if (fileSegment.isClosed()) { ++- metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); ++- } ++- this.tieredMetadataStore.updateFileSegment(metadata); +++ metadata.setCreateTimestamp(System.currentTimeMillis()); ++ } ++- return metadata; ++- } ++- ++- /** ++- * FileQueue Status: Sealed | Sealed | Sealed | Not sealed, Allow appended && Not Full ++- */ ++- public void updateFileSegment(TieredFileSegment fileSegment) { ++- FileSegmentMetadata segmentMetadata = getOrCreateFileSegmentMetadata(fileSegment); ++ ++- if (segmentMetadata.getStatus() == FileSegmentMetadata.STATUS_NEW ++- && fileSegment.isFull() ++- && !fileSegment.needCommit()) { +++ metadata.setSize(fileSegment.getCommitPosition()); +++ metadata.setBeginTimestamp(fileSegment.getMinTimestamp()); +++ metadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++ ++- segmentMetadata.markSealed(); +++ if (fileSegment.isFull() && !fileSegment.needCommit()) { +++ if (metadata.getStatus() == FileSegmentMetadata.STATUS_NEW) { +++ metadata.markSealed(); +++ } ++ } ++ ++ if (fileSegment.isClosed()) { ++- segmentMetadata.setStatus(FileSegmentMetadata.STATUS_DELETED); +++ metadata.setStatus(FileSegmentMetadata.STATUS_DELETED); ++ } ++ ++- segmentMetadata.setSize(fileSegment.getCommitPosition()); ++- segmentMetadata.setEndTimestamp(fileSegment.getMaxTimestamp()); ++- ++- FileSegmentMetadata metadata = tieredMetadataStore.getFileSegment( ++- this.filePath, fileSegment.getFileType(), fileSegment.getBaseOffset()); ++- ++- if (!Objects.equals(metadata, segmentMetadata)) { ++- this.tieredMetadataStore.updateFileSegment(segmentMetadata); ++- logger.debug("TieredFlatFile#UpdateSegmentMetadata, filePath: {}, content: {}", ++- segmentMetadata.getPath(), JSON.toJSONString(segmentMetadata)); ++- } +++ this.tieredMetadataStore.updateFileSegment(metadata); ++ } ++ ++ private void checkAndFixFileSize() { ++@@ -598,6 +578,9 @@ public class TieredFlatFile { ++ logger.error("TieredFlatFile#destroy: mark file segment: {} is deleted failed", fileSegment.getPath(), e); ++ } ++ fileSegment.destroyFile(); +++ if (!fileSegment.exists()) { +++ tieredMetadataStore.deleteFileSegment(filePath, fileType, fileSegment.getBaseOffset()); +++ } ++ } ++ fileSegmentList.clear(); ++ needCommitFileSegmentList.clear(); ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++index 7c744af3b..087ea8c9c 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/file/TieredFlatFileManager.java ++@@ -136,15 +136,13 @@ public class TieredFlatFileManager { ++ TimeUnit.HOURS.toMillis(storeConfig.getTieredStoreFileReservedTime()); ++ for (CompositeQueueFlatFile flatFile : deepCopyFlatFileToList()) { ++ TieredStoreExecutor.cleanExpiredFileExecutor.submit(() -> { ++- flatFile.getCompositeFlatFileLock().lock(); ++ try { +++ flatFile.getCompositeFlatFileLock().lock(); ++ flatFile.cleanExpiredFile(expiredTimeStamp); ++ flatFile.destroyExpiredFile(); ++- if (flatFile.getConsumeQueueBaseOffset() == -1) { ++- logger.info("Clean flatFile because file not initialized, topic={}, queueId={}", ++- flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId()); ++- destroyCompositeFile(flatFile.getMessageQueue()); ++- } +++ } catch (Throwable t) { +++ logger.error("Do Clean expired file error, topic={}, queueId={}", +++ flatFile.getMessageQueue().getTopic(), flatFile.getMessageQueue().getQueueId(), t); ++ } finally { ++ flatFile.getCompositeFlatFileLock().unlock(); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 4a8e0d5b851d1f9573cda79b7d2e42ee498809da Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Wed, 13 Sep 2023 16:08:03 +0800 ++Subject: [PATCH 3/5] [ISSUE #7351] Allow mqadmin to operate slave nodes ++ ++Co-authored-by: guyinyou ++--- ++ .../processor/AdminBrokerProcessor.java | 12 -- ++ .../processor/AdminBrokerProcessorTest.java | 106 ------------------ ++ 2 files changed, 118 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++index 8fbcd3c94..9e48431be 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++@@ -406,9 +406,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ private synchronized RemotingCommand updateAndCreateTopic(ChannelHandlerContext ctx, ++ RemotingCommand request) throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- if (validateSlave(response)) { ++- return response; ++- } ++ final CreateTopicRequestHeader requestHeader = ++ (CreateTopicRequestHeader) request.decodeCommandCustomHeader(CreateTopicRequestHeader.class); ++ ++@@ -519,9 +516,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ private synchronized RemotingCommand deleteTopic(ChannelHandlerContext ctx, ++ RemotingCommand request) throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- if (validateSlave(response)) { ++- return response; ++- } ++ DeleteTopicRequestHeader requestHeader = ++ (DeleteTopicRequestHeader) request.decodeCommandCustomHeader(DeleteTopicRequestHeader.class); ++ ++@@ -1413,9 +1407,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ private RemotingCommand updateAndCreateSubscriptionGroup(ChannelHandlerContext ctx, RemotingCommand request) ++ throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- if (validateSlave(response)) { ++- return response; ++- } ++ ++ LOGGER.info("AdminBrokerProcessor#updateAndCreateSubscriptionGroup called by {}", ++ RemotingHelper.parseChannelRemoteAddr(ctx.channel())); ++@@ -1480,9 +1471,6 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ private RemotingCommand deleteSubscriptionGroup(ChannelHandlerContext ctx, ++ RemotingCommand request) throws RemotingCommandException { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- if (validateSlave(response)) { ++- return response; ++- } ++ DeleteSubscriptionGroupRequestHeader requestHeader = ++ (DeleteSubscriptionGroupRequestHeader) request.decodeCommandCustomHeader(DeleteSubscriptionGroupRequestHeader.class); ++ ++diff --git a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++index 9d17011b6..ec252cece 100644 ++--- a/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java +++++ b/broker/src/test/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessorTest.java ++@@ -76,7 +76,6 @@ import org.apache.rocketmq.remoting.protocol.subscription.SubscriptionGroupConfi ++ import org.apache.rocketmq.store.DefaultMessageStore; ++ import org.apache.rocketmq.store.MessageStore; ++ import org.apache.rocketmq.store.SelectMappedBufferResult; ++-import org.apache.rocketmq.store.config.BrokerRole; ++ import org.apache.rocketmq.store.config.MessageStoreConfig; ++ import org.apache.rocketmq.store.logfile.DefaultMappedFile; ++ import org.apache.rocketmq.store.stats.BrokerStats; ++@@ -250,32 +249,6 @@ public class AdminBrokerProcessorTest { ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ } ++ ++- @Test ++- public void testUpdateAndCreateTopicOnSlaveInRocksdb() throws Exception { ++- if (notToBeExecuted()) { ++- return; ++- } ++- initRocksdbTopicManager(); ++- testUpdateAndCreateTopicOnSlave(); ++- } ++- ++- @Test ++- public void testUpdateAndCreateTopicOnSlave() throws Exception { ++- // setup ++- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); ++- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); ++- defaultMessageStore = mock(DefaultMessageStore.class); ++- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); ++- ++- // test on slave ++- String topic = "TEST_CREATE_TOPIC"; ++- RemotingCommand request = buildCreateTopicRequest(topic); ++- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); ++- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + ++- "please execute it from master broker."); ++- } ++- ++ @Test ++ public void testDeleteTopicInRocksdb() throws Exception { ++ if (notToBeExecuted()) { ++@@ -301,31 +274,6 @@ public class AdminBrokerProcessorTest { ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ } ++ ++- @Test ++- public void testDeleteTopicOnSlaveInRocksdb() throws Exception { ++- if (notToBeExecuted()) { ++- return; ++- } ++- initRocksdbTopicManager(); ++- testDeleteTopicOnSlave(); ++- } ++- ++- @Test ++- public void testDeleteTopicOnSlave() throws Exception { ++- // setup ++- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); ++- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); ++- defaultMessageStore = mock(DefaultMessageStore.class); ++- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); ++- ++- String topic = "TEST_DELETE_TOPIC"; ++- RemotingCommand request = buildDeleteTopicRequest(topic); ++- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); ++- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + ++- "please execute it from master broker."); ++- } ++- ++ @Test ++ public void testDeleteWithPopRetryTopic() throws Exception { ++ String topic = "topicA"; ++@@ -538,36 +486,6 @@ public class AdminBrokerProcessorTest { ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ } ++ ++- @Test ++- public void testUpdateAndCreateSubscriptionGroupOnSlaveInRocksdb() throws Exception { ++- initRocksdbSubscriptionManager(); ++- testUpdateAndCreateSubscriptionGroupOnSlave(); ++- } ++- ++- @Test ++- public void testUpdateAndCreateSubscriptionGroupOnSlave() throws RemotingCommandException { ++- // Setup ++- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); ++- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); ++- defaultMessageStore = mock(DefaultMessageStore.class); ++- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); ++- ++- // Test ++- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.UPDATE_AND_CREATE_SUBSCRIPTIONGROUP, null); ++- SubscriptionGroupConfig subscriptionGroupConfig = new SubscriptionGroupConfig(); ++- subscriptionGroupConfig.setBrokerId(1); ++- subscriptionGroupConfig.setGroupName("groupId"); ++- subscriptionGroupConfig.setConsumeEnable(Boolean.TRUE); ++- subscriptionGroupConfig.setConsumeBroadcastEnable(Boolean.TRUE); ++- subscriptionGroupConfig.setRetryMaxTimes(111); ++- subscriptionGroupConfig.setConsumeFromMinEnable(Boolean.TRUE); ++- request.setBody(JSON.toJSON(subscriptionGroupConfig).toString().getBytes()); ++- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); ++- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + ++- "please execute it from master broker."); ++- } ++- ++ @Test ++ public void testGetAllSubscriptionGroupInRocksdb() throws Exception { ++ initRocksdbSubscriptionManager(); ++@@ -596,30 +514,6 @@ public class AdminBrokerProcessorTest { ++ assertThat(response.getCode()).isEqualTo(ResponseCode.SUCCESS); ++ } ++ ++- @Test ++- public void testDeleteSubscriptionGroupOnSlaveInRocksdb() throws Exception { ++- initRocksdbSubscriptionManager(); ++- testDeleteSubscriptionGroupOnSlave(); ++- } ++- ++- @Test ++- public void testDeleteSubscriptionGroupOnSlave() throws RemotingCommandException { ++- // Setup ++- MessageStoreConfig messageStoreConfig = mock(MessageStoreConfig.class); ++- when(messageStoreConfig.getBrokerRole()).thenReturn(BrokerRole.SLAVE); ++- defaultMessageStore = mock(DefaultMessageStore.class); ++- when(brokerController.getMessageStoreConfig()).thenReturn(messageStoreConfig); ++- ++- // Test ++- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.DELETE_SUBSCRIPTIONGROUP, null); ++- request.addExtField("groupName", "GID-Group-Name"); ++- request.addExtField("removeOffset", "true"); ++- RemotingCommand response = adminBrokerProcessor.processRequest(handlerContext, request); ++- assertThat(response.getCode()).isEqualTo(ResponseCode.SYSTEM_ERROR); ++- assertThat(response.getRemark()).isEqualTo("Can't modify topic or subscription group from slave broker, " + ++- "please execute it from master broker."); ++- } ++- ++ @Test ++ public void testGetTopicStatsInfo() throws RemotingCommandException { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.GET_TOPIC_STATS_INFO, null); ++-- ++2.32.0.windows.2 ++ ++ ++From 831fcc76cd7cd362bb6c136c287c624bb7eaf40a Mon Sep 17 00:00:00 2001 ++From: lizhimins <707364882@qq.com> ++Date: Tue, 19 Sep 2023 10:04:04 +0800 ++Subject: [PATCH 4/5] [ISSUE #7363] Fix get message from tiered storage return ++ incorrect next pull offset (#7365) ++ ++--- ++ .../tieredstore/TieredMessageFetcher.java | 2 +- ++ .../tieredstore/TieredMessageStore.java | 29 ++++++++++--------- ++ .../tieredstore/TieredMessageStoreTest.java | 5 ++-- ++ 3 files changed, 20 insertions(+), 16 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++index 766ff64f6..c948fa3fa 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageFetcher.java ++@@ -319,7 +319,7 @@ public class TieredMessageFetcher implements MessageStoreFetcher { ++ } ++ ++ // if cache is miss, immediately pull messages ++- LOGGER.warn("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + +++ LOGGER.info("TieredMessageFetcher#getMessageFromCacheAsync: cache miss: " + ++ "topic: {}, queue: {}, queue offset: {}, max message num: {}", ++ mq.getTopic(), mq.getQueueId(), queueOffset, maxCount); ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++index 9fb1b2f01..d7d13d61e 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++@@ -147,6 +147,11 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ public CompletableFuture getMessageAsync(String group, String topic, ++ int queueId, long offset, int maxMsgNums, MessageFilter messageFilter) { ++ +++ // For system topic, force reading from local store +++ if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { +++ return next.getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter); +++ } +++ ++ if (fetchFromCurrentStore(topic, queueId, offset, maxMsgNums)) { ++ logger.trace("GetMessageAsync from current store, topic: {}, queue: {}, offset: {}", topic, queueId, offset); ++ } else { ++@@ -158,6 +163,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ return fetcher ++ .getMessageAsync(group, topic, queueId, offset, maxMsgNums, messageFilter) ++ .thenApply(result -> { +++ ++ Attributes latencyAttributes = TieredStoreMetricsManager.newAttributesBuilder() ++ .put(TieredStoreMetricsConstant.LABEL_OPERATION, TieredStoreMetricsConstant.OPERATION_API_GET_MESSAGE) ++ .put(TieredStoreMetricsConstant.LABEL_TOPIC, topic) ++@@ -166,8 +172,7 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ TieredStoreMetricsManager.apiLatency.record(stopwatch.elapsed(TimeUnit.MILLISECONDS), latencyAttributes); ++ ++ if (result.getStatus() == GetMessageStatus.OFFSET_FOUND_NULL || ++- result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_ONE || ++- result.getStatus() == GetMessageStatus.OFFSET_OVERFLOW_BADLY) { +++ result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { ++ ++ if (next.checkInStoreByConsumeOffset(topic, queueId, offset)) { ++ TieredStoreMetricsManager.fallbackTotal.add(1, latencyAttributes); ++@@ -178,14 +183,8 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ } ++ } ++ ++- // Fetch system topic data from the broker when using the force level. ++- if (result.getStatus() == GetMessageStatus.NO_MATCHED_LOGIC_QUEUE) { ++- if (TieredStoreUtil.isSystemTopic(topic) || PopAckConstants.isStartWithRevivePrefix(topic)) { ++- return next.getMessage(group, topic, queueId, offset, maxMsgNums, messageFilter); ++- } ++- } ++- ++ if (result.getStatus() != GetMessageStatus.FOUND && +++ result.getStatus() != GetMessageStatus.NO_MATCHED_LOGIC_QUEUE && ++ result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_ONE && ++ result.getStatus() != GetMessageStatus.OFFSET_OVERFLOW_BADLY) { ++ logger.warn("GetMessageAsync not found and message is not in next store, result: {}, " + ++@@ -206,10 +205,14 @@ public class TieredMessageStore extends AbstractPluginMessageStore { ++ if (minOffsetInQueue >= 0 && minOffsetInQueue < result.getMinOffset()) { ++ result.setMinOffset(minOffsetInQueue); ++ } ++- long maxOffsetInQueue = next.getMaxOffsetInQueue(topic, queueId); ++- if (maxOffsetInQueue >= 0 && maxOffsetInQueue > result.getMaxOffset()) { ++- result.setMaxOffset(maxOffsetInQueue); ++- } +++ +++ // In general, the local cq offset is slightly greater than the commit offset in read message, +++ // so there is no need to update the maximum offset to the local cq offset here, +++ // otherwise it will cause repeated consumption after next begin offset over commit offset. +++ +++ logger.trace("GetMessageAsync result, group: {}, topic: {}, queueId: {}, offset: {}, count:{}, {}", +++ group, topic, queueId, offset, maxMsgNums, result); +++ ++ return result; ++ }).exceptionally(e -> { ++ logger.error("GetMessageAsync from tiered store failed", e); ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++index 2451199c2..07af1fc8b 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/TieredMessageStoreTest.java ++@@ -168,7 +168,7 @@ public class TieredMessageStoreTest { ++ GetMessageResult result1 = new GetMessageResult(); ++ result1.setStatus(GetMessageStatus.FOUND); ++ GetMessageResult result2 = new GetMessageResult(); ++- result2.setStatus(GetMessageStatus.MESSAGE_WAS_REMOVING); +++ result2.setStatus(GetMessageStatus.OFFSET_OVERFLOW_BADLY); ++ ++ when(fetcher.getMessageAsync(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(CompletableFuture.completedFuture(result1)); ++ when(nextStore.getMessage(anyString(), anyString(), anyInt(), anyLong(), anyInt(), any())).thenReturn(result2); ++@@ -188,7 +188,8 @@ public class TieredMessageStoreTest { ++ properties.setProperty("tieredStorageLevel", "3"); ++ configuration.update(properties); ++ when(nextStore.checkInStoreByConsumeOffset(anyString(), anyInt(), anyLong())).thenReturn(true); ++- Assert.assertSame(result2, store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null)); +++ Assert.assertEquals(result2.getStatus(), +++ store.getMessage("group", mq.getTopic(), mq.getQueueId(), 0, 0, null).getStatus()); ++ } ++ ++ @Test ++-- ++2.32.0.windows.2 ++ ++ ++From f05a8da760dfade411ad56ef874f477988479cf9 Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Wed, 20 Sep 2023 15:06:21 +0800 ++Subject: [PATCH 5/5] Print admin queue watermark in log (#7372) ++ ++--- ++ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 13a3feb4e..53e2e1b62 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1182,6 +1182,7 @@ public class BrokerController { ++ LOG_WATER_MARK.info("[WATERMARK] ClientManager Queue Size: {} SlowTimeMills: {}", this.clientManagerThreadPoolQueue.size(), this.headSlowTimeMills(this.clientManagerThreadPoolQueue)); ++ LOG_WATER_MARK.info("[WATERMARK] Heartbeat Queue Size: {} SlowTimeMills: {}", this.heartbeatThreadPoolQueue.size(), this.headSlowTimeMills(this.heartbeatThreadPoolQueue)); ++ LOG_WATER_MARK.info("[WATERMARK] Ack Queue Size: {} SlowTimeMills: {}", this.ackThreadPoolQueue.size(), headSlowTimeMills(this.ackThreadPoolQueue)); +++ LOG_WATER_MARK.info("[WATERMARK] Admin Queue Size: {} SlowTimeMills: {}", this.adminBrokerThreadPoolQueue.size(), headSlowTimeMills(this.adminBrokerThreadPoolQueue)); ++ } ++ ++ public MessageStore getMessageStore() { ++-- ++2.32.0.windows.2 ++ +diff --git a/patch019-backport-some-bugfix.patch b/patch019-backport-some-bugfix.patch +new file mode 100644 +index 000000000..d85f01c98 +--- /dev/null ++++ b/patch019-backport-some-bugfix.patch +@@ -0,0 +1,1499 @@ ++From 42fcd278ca84f6988d48a7d11271fc81b921d59a Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Wed, 20 Sep 2023 15:41:23 +0800 ++Subject: [PATCH 01/12] [ISSUE #7374] Prepare to release Apache RocketMQ 5.1.4 ++ (#7375) ++ ++--- ++ common/src/main/java/org/apache/rocketmq/common/MQVersion.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java ++index bfd07a895..4f1990ff8 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/MQVersion.java +++++ b/common/src/main/java/org/apache/rocketmq/common/MQVersion.java ++@@ -18,7 +18,7 @@ package org.apache.rocketmq.common; ++ ++ public class MQVersion { ++ ++- public static final int CURRENT_VERSION = Version.V5_1_3.ordinal(); +++ public static final int CURRENT_VERSION = Version.V5_1_4.ordinal(); ++ ++ public static String getVersionDesc(int value) { ++ int length = Version.values().length; ++-- ++2.32.0.windows.2 ++ ++ ++From b8610d87bb55de1f4413460c05da529dab60c1c1 Mon Sep 17 00:00:00 2001 ++From: Jixiang Jin ++Date: Thu, 21 Sep 2023 16:21:44 +0800 ++Subject: [PATCH 02/12] Replace loggingMetricExporter with ++ OtlpJsonLoggingMetricExporter. (#7373) ++ ++* Replace loggingMetricExporter with OtlpJsonLoggingMetricExporter. ++ ++* Fix bazel workspace ++ ++* Move OtlpJsonLoggingMetricExporter to proxy and controller. ++ ++* Fix Bazel imports. ++--- ++ WORKSPACE | 1 + ++ broker/BUILD.bazel | 1 + ++ .../rocketmq/broker/metrics/BrokerMetricsManager.java | 9 +++++---- ++ broker/src/main/resources/rmq.broker.logback.xml | 5 +++++ ++ common/BUILD.bazel | 1 + ++ common/pom.xml | 4 ++++ ++ controller/BUILD.bazel | 1 + ++ .../controller/metrics/ControllerMetricsManager.java | 9 +++++---- ++ pom.xml | 5 +++++ ++ proxy/BUILD.bazel | 1 + ++ .../rocketmq/proxy/metrics/ProxyMetricsManager.java | 11 ++++++----- ++ proxy/src/main/resources/rmq.proxy.logback.xml | 5 +++++ ++ tieredstore/BUILD.bazel | 1 + ++ 13 files changed, 41 insertions(+), 13 deletions(-) ++ ++diff --git a/WORKSPACE b/WORKSPACE ++index 3126f2d1d..8640485ba 100644 ++--- a/WORKSPACE +++++ b/WORKSPACE ++@@ -92,6 +92,7 @@ maven_install( ++ "io.opentelemetry:opentelemetry-exporter-prometheus:1.29.0-alpha", ++ "io.opentelemetry:opentelemetry-exporter-logging:1.29.0", ++ "io.opentelemetry:opentelemetry-sdk:1.29.0", +++ "io.opentelemetry:opentelemetry-exporter-logging-otlp:1.29.0", ++ "com.squareup.okio:okio-jvm:3.0.0", ++ "io.opentelemetry:opentelemetry-api:1.29.0", ++ "io.opentelemetry:opentelemetry-sdk-metrics:1.29.0", ++diff --git a/broker/BUILD.bazel b/broker/BUILD.bazel ++index 6adcdc7b9..64cb1b341 100644 ++--- a/broker/BUILD.bazel +++++ b/broker/BUILD.bazel ++@@ -44,6 +44,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:io_opentelemetry_opentelemetry_sdk", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++index 6af5afc14..39af18b9f 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/metrics/BrokerMetricsManager.java ++@@ -23,7 +23,7 @@ import io.opentelemetry.api.metrics.LongCounter; ++ import io.opentelemetry.api.metrics.LongHistogram; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.api.metrics.ObservableLongGauge; ++-import io.opentelemetry.exporter.logging.LoggingMetricExporter; +++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; ++ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; ++@@ -36,6 +36,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; ++ import io.opentelemetry.sdk.metrics.View; ++ import io.opentelemetry.sdk.metrics.ViewBuilder; ++ import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +++import io.opentelemetry.sdk.metrics.export.MetricExporter; ++ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; ++ import io.opentelemetry.sdk.metrics.internal.SdkMeterProviderUtil; ++ import io.opentelemetry.sdk.resources.Resource; ++@@ -113,7 +114,7 @@ public class BrokerMetricsManager { ++ private OtlpGrpcMetricExporter metricExporter; ++ private PeriodicMetricReader periodicMetricReader; ++ private PrometheusHttpServer prometheusHttpServer; ++- private LoggingMetricExporter loggingMetricExporter; +++ private MetricExporter loggingMetricExporter; ++ private Meter brokerMeter; ++ ++ public static Supplier attributesBuilderSupplier = Attributes::builder; ++@@ -327,8 +328,8 @@ public class BrokerMetricsManager { ++ if (metricsExporterType == MetricsExporterType.LOG) { ++ SLF4JBridgeHandler.removeHandlersForRootLogger(); ++ SLF4JBridgeHandler.install(); ++- loggingMetricExporter = LoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); +++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(brokerConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) ++ .setInterval(brokerConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) ++ .build(); ++diff --git a/broker/src/main/resources/rmq.broker.logback.xml b/broker/src/main/resources/rmq.broker.logback.xml ++index 3c51e59d4..32dc29736 100644 ++--- a/broker/src/main/resources/rmq.broker.logback.xml +++++ b/broker/src/main/resources/rmq.broker.logback.xml ++@@ -672,6 +672,11 @@ ++ ++ ++ +++ +++ +++ +++ +++ ++ ++ ++ ++diff --git a/common/BUILD.bazel b/common/BUILD.bazel ++index a95a19ccd..e6701d0fc 100644 ++--- a/common/BUILD.bazel +++++ b/common/BUILD.bazel ++@@ -35,6 +35,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_sdk", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:org_apache_commons_commons_lang3", ++ "@maven//:org_lz4_lz4_java", ++ "@maven//:io_github_aliyunmq_rocketmq_slf4j_api", ++diff --git a/common/pom.xml b/common/pom.xml ++index 31eb0f087..accc7f0a8 100644 ++--- a/common/pom.xml +++++ b/common/pom.xml ++@@ -80,6 +80,10 @@ ++ io.opentelemetry ++ opentelemetry-sdk ++ +++ +++ io.opentelemetry +++ opentelemetry-exporter-logging-otlp +++ ++ ++ io.grpc ++ grpc-stub ++diff --git a/controller/BUILD.bazel b/controller/BUILD.bazel ++index 843d9dc77..b2b743eb2 100644 ++--- a/controller/BUILD.bazel +++++ b/controller/BUILD.bazel ++@@ -49,6 +49,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:org_slf4j_jul_to_slf4j", ++ ], ++ ) ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++index 650740bcc..be9e77eea 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/metrics/ControllerMetricsManager.java ++@@ -26,7 +26,7 @@ import io.opentelemetry.api.metrics.LongHistogram; ++ import io.opentelemetry.api.metrics.LongUpDownCounter; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.api.metrics.ObservableLongGauge; ++-import io.opentelemetry.exporter.logging.LoggingMetricExporter; +++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; ++ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; ++@@ -38,6 +38,7 @@ import io.opentelemetry.sdk.metrics.SdkMeterProvider; ++ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; ++ import io.opentelemetry.sdk.metrics.View; ++ import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +++import io.opentelemetry.sdk.metrics.export.MetricExporter; ++ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; ++ import io.opentelemetry.sdk.resources.Resource; ++ import java.io.File; ++@@ -121,7 +122,7 @@ public class ControllerMetricsManager { ++ ++ private PrometheusHttpServer prometheusHttpServer; ++ ++- private LoggingMetricExporter loggingMetricExporter; +++ private MetricExporter loggingMetricExporter; ++ ++ public static ControllerMetricsManager getInstance(ControllerManager controllerManager) { ++ if (instance == null) { ++@@ -364,8 +365,8 @@ public class ControllerMetricsManager { ++ if (type == MetricsExporterType.LOG) { ++ SLF4JBridgeHandler.removeHandlersForRootLogger(); ++ SLF4JBridgeHandler.install(); ++- loggingMetricExporter = LoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); +++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(config.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) ++ .setInterval(config.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) ++ .build(); ++diff --git a/pom.xml b/pom.xml ++index 9f0b3eb96..4b382c6da 100644 ++--- a/pom.xml +++++ b/pom.xml ++@@ -974,6 +974,11 @@ ++ opentelemetry-sdk ++ ${opentelemetry.version} ++ +++ +++ io.opentelemetry +++ opentelemetry-exporter-logging-otlp +++ ${opentelemetry.version} +++ ++ ++ org.slf4j ++ jul-to-slf4j ++diff --git a/proxy/BUILD.bazel b/proxy/BUILD.bazel ++index b4f3c16e2..cb7af9254 100644 ++--- a/proxy/BUILD.bazel +++++ b/proxy/BUILD.bazel ++@@ -52,6 +52,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_exporter_otlp", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_prometheus", ++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:io_opentelemetry_opentelemetry_sdk", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java ++index f5050858f..2b8dac5d8 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/metrics/ProxyMetricsManager.java ++@@ -21,15 +21,16 @@ import io.opentelemetry.api.common.Attributes; ++ import io.opentelemetry.api.common.AttributesBuilder; ++ import io.opentelemetry.api.metrics.Meter; ++ import io.opentelemetry.api.metrics.ObservableLongGauge; +++import io.opentelemetry.exporter.logging.otlp.OtlpJsonLoggingMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporter; ++ import io.opentelemetry.exporter.otlp.metrics.OtlpGrpcMetricExporterBuilder; ++ import io.opentelemetry.exporter.prometheus.PrometheusHttpServer; ++-import io.opentelemetry.exporter.logging.LoggingMetricExporter; ++ import io.opentelemetry.sdk.OpenTelemetrySdk; ++ import io.opentelemetry.sdk.metrics.InstrumentType; ++ import io.opentelemetry.sdk.metrics.SdkMeterProvider; ++ import io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder; ++ import io.opentelemetry.sdk.metrics.data.AggregationTemporality; +++import io.opentelemetry.sdk.metrics.export.MetricExporter; ++ import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader; ++ import io.opentelemetry.sdk.resources.Resource; ++ import java.util.HashMap; ++@@ -42,9 +43,9 @@ import org.apache.rocketmq.broker.metrics.BrokerMetricsManager; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.metrics.MetricsExporterType; ++ import org.apache.rocketmq.common.utils.StartAndShutdown; ++-import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +++import org.apache.rocketmq.proxy.config.ProxyConfig; ++ import org.slf4j.bridge.SLF4JBridgeHandler; ++ ++ import static org.apache.rocketmq.broker.metrics.BrokerMetricsConstant.AGGREGATION_DELTA; ++@@ -67,7 +68,7 @@ public class ProxyMetricsManager implements StartAndShutdown { ++ private OtlpGrpcMetricExporter metricExporter; ++ private PeriodicMetricReader periodicMetricReader; ++ private PrometheusHttpServer prometheusHttpServer; ++- private LoggingMetricExporter loggingMetricExporter; +++ private MetricExporter loggingMetricExporter; ++ ++ public static ObservableLongGauge proxyUp = null; ++ ++@@ -221,8 +222,8 @@ public class ProxyMetricsManager implements StartAndShutdown { ++ if (metricsExporterType == MetricsExporterType.LOG) { ++ SLF4JBridgeHandler.removeHandlersForRootLogger(); ++ SLF4JBridgeHandler.install(); ++- loggingMetricExporter = LoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); ++- java.util.logging.Logger.getLogger(LoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); +++ loggingMetricExporter = OtlpJsonLoggingMetricExporter.create(proxyConfig.isMetricsInDelta() ? AggregationTemporality.DELTA : AggregationTemporality.CUMULATIVE); +++ java.util.logging.Logger.getLogger(OtlpJsonLoggingMetricExporter.class.getName()).setLevel(java.util.logging.Level.FINEST); ++ periodicMetricReader = PeriodicMetricReader.builder(loggingMetricExporter) ++ .setInterval(proxyConfig.getMetricLoggingExporterIntervalInMills(), TimeUnit.MILLISECONDS) ++ .build(); ++diff --git a/proxy/src/main/resources/rmq.proxy.logback.xml b/proxy/src/main/resources/rmq.proxy.logback.xml ++index d38827f92..f968a45e6 100644 ++--- a/proxy/src/main/resources/rmq.proxy.logback.xml +++++ b/proxy/src/main/resources/rmq.proxy.logback.xml ++@@ -418,6 +418,11 @@ ++ ++ ++ +++ +++ +++ +++ +++ ++ ++ ++ ++diff --git a/tieredstore/BUILD.bazel b/tieredstore/BUILD.bazel ++index 5b3885a4e..dea2c561b 100644 ++--- a/tieredstore/BUILD.bazel +++++ b/tieredstore/BUILD.bazel ++@@ -36,6 +36,7 @@ java_library( ++ "@maven//:io_opentelemetry_opentelemetry_sdk", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_common", ++ "@maven//:io_opentelemetry_opentelemetry_sdk_metrics", +++ "@maven//:io_opentelemetry_opentelemetry_exporter_logging_otlp", ++ "@maven//:org_apache_commons_commons_lang3", ++ "@maven//:org_apache_tomcat_annotations_api", ++ "@maven//:com_alibaba_fastjson", ++-- ++2.32.0.windows.2 ++ ++ ++From 1a681bdf9b5c5ab0be446d6394c0cac8768f45d9 Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Thu, 21 Sep 2023 19:58:29 +0800 ++Subject: [PATCH 03/12] [maven-release-plugin] prepare release ++ rocketmq-all-5.1.4 (#7377) ++ ++--- ++ acl/pom.xml | 2 +- ++ broker/pom.xml | 2 +- ++ client/pom.xml | 2 +- ++ common/pom.xml | 2 +- ++ container/pom.xml | 2 +- ++ controller/pom.xml | 2 +- ++ distribution/pom.xml | 2 +- ++ example/pom.xml | 2 +- ++ filter/pom.xml | 2 +- ++ namesrv/pom.xml | 2 +- ++ openmessaging/pom.xml | 2 +- ++ pom.xml | 4 ++-- ++ proxy/pom.xml | 2 +- ++ remoting/pom.xml | 2 +- ++ srvutil/pom.xml | 2 +- ++ store/pom.xml | 2 +- ++ test/pom.xml | 2 +- ++ tieredstore/pom.xml | 2 +- ++ tools/pom.xml | 2 +- ++ 19 files changed, 20 insertions(+), 20 deletions(-) ++ ++diff --git a/acl/pom.xml b/acl/pom.xml ++index 989c0cf77..9f6838b00 100644 ++--- a/acl/pom.xml +++++ b/acl/pom.xml ++@@ -13,7 +13,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ rocketmq-acl ++ rocketmq-acl ${project.version} ++diff --git a/broker/pom.xml b/broker/pom.xml ++index 16e026276..d483e67ba 100644 ++--- a/broker/pom.xml +++++ b/broker/pom.xml ++@@ -13,7 +13,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/client/pom.xml b/client/pom.xml ++index c59a43889..4febedc6d 100644 ++--- a/client/pom.xml +++++ b/client/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/common/pom.xml b/common/pom.xml ++index accc7f0a8..b70873dfa 100644 ++--- a/common/pom.xml +++++ b/common/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/container/pom.xml b/container/pom.xml ++index c8499f127..e6c1f4b4d 100644 ++--- a/container/pom.xml +++++ b/container/pom.xml ++@@ -18,7 +18,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/controller/pom.xml b/controller/pom.xml ++index 3346c7c82..46a3834c6 100644 ++--- a/controller/pom.xml +++++ b/controller/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ 4.0.0 ++ jar ++diff --git a/distribution/pom.xml b/distribution/pom.xml ++index dbde2d9d4..346c4de35 100644 ++--- a/distribution/pom.xml +++++ b/distribution/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ rocketmq-distribution ++ rocketmq-distribution ${project.version} ++diff --git a/example/pom.xml b/example/pom.xml ++index 862fc3169..9e7db43f8 100644 ++--- a/example/pom.xml +++++ b/example/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/filter/pom.xml b/filter/pom.xml ++index 3fe51ceae..84189066d 100644 ++--- a/filter/pom.xml +++++ b/filter/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/namesrv/pom.xml b/namesrv/pom.xml ++index 684b2683c..7c218078a 100644 ++--- a/namesrv/pom.xml +++++ b/namesrv/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml ++index aaa4c896c..fd499e3de 100644 ++--- a/openmessaging/pom.xml +++++ b/openmessaging/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/pom.xml b/pom.xml ++index 4b382c6da..0e1d04f15 100644 ++--- a/pom.xml +++++ b/pom.xml ++@@ -28,7 +28,7 @@ ++ 2012 ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ pom ++ Apache RocketMQ ${project.version} ++ http://rocketmq.apache.org/ ++@@ -37,7 +37,7 @@ ++ git@github.com:apache/rocketmq.git ++ scm:git:git@github.com:apache/rocketmq.git ++ scm:git:git@github.com:apache/rocketmq.git ++- HEAD +++ rocketmq-all-5.1.4 ++ ++ ++ ++diff --git a/proxy/pom.xml b/proxy/pom.xml ++index 3fbea107a..abf242eee 100644 ++--- a/proxy/pom.xml +++++ b/proxy/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/remoting/pom.xml b/remoting/pom.xml ++index 8a43c5c30..fc70cb62e 100644 ++--- a/remoting/pom.xml +++++ b/remoting/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/srvutil/pom.xml b/srvutil/pom.xml ++index fa54ad019..d7f946cce 100644 ++--- a/srvutil/pom.xml +++++ b/srvutil/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/store/pom.xml b/store/pom.xml ++index 38f04009d..6d6983c5d 100644 ++--- a/store/pom.xml +++++ b/store/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/test/pom.xml b/test/pom.xml ++index 8f25c35c9..39090e426 100644 ++--- a/test/pom.xml +++++ b/test/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml ++index c476040ba..7b209751f 100644 ++--- a/tieredstore/pom.xml +++++ b/tieredstore/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++diff --git a/tools/pom.xml b/tools/pom.xml ++index 1c3b431bc..806787ec9 100644 ++--- a/tools/pom.xml +++++ b/tools/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4-SNAPSHOT +++ 5.1.4 ++ ++ ++ 4.0.0 ++-- ++2.32.0.windows.2 ++ ++ ++From 73b3fde83765e066541e3455cd1e6604292a9b7c Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Fri, 22 Sep 2023 10:08:59 +0800 ++Subject: [PATCH 04/12] [maven-release-plugin] prepare for next development ++ iteration (#7379) ++ ++--- ++ acl/pom.xml | 2 +- ++ broker/pom.xml | 2 +- ++ client/pom.xml | 2 +- ++ common/pom.xml | 2 +- ++ container/pom.xml | 2 +- ++ controller/pom.xml | 2 +- ++ distribution/pom.xml | 2 +- ++ example/pom.xml | 2 +- ++ filter/pom.xml | 2 +- ++ namesrv/pom.xml | 2 +- ++ openmessaging/pom.xml | 2 +- ++ pom.xml | 4 ++-- ++ proxy/pom.xml | 2 +- ++ remoting/pom.xml | 2 +- ++ srvutil/pom.xml | 2 +- ++ store/pom.xml | 2 +- ++ test/pom.xml | 2 +- ++ tieredstore/pom.xml | 2 +- ++ tools/pom.xml | 2 +- ++ 19 files changed, 20 insertions(+), 20 deletions(-) ++ ++diff --git a/acl/pom.xml b/acl/pom.xml ++index 9f6838b00..8a296e5ae 100644 ++--- a/acl/pom.xml +++++ b/acl/pom.xml ++@@ -13,7 +13,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ rocketmq-acl ++ rocketmq-acl ${project.version} ++diff --git a/broker/pom.xml b/broker/pom.xml ++index d483e67ba..add83045d 100644 ++--- a/broker/pom.xml +++++ b/broker/pom.xml ++@@ -13,7 +13,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/client/pom.xml b/client/pom.xml ++index 4febedc6d..d6fb3889b 100644 ++--- a/client/pom.xml +++++ b/client/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/common/pom.xml b/common/pom.xml ++index b70873dfa..6104c3ac6 100644 ++--- a/common/pom.xml +++++ b/common/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/container/pom.xml b/container/pom.xml ++index e6c1f4b4d..8af231e01 100644 ++--- a/container/pom.xml +++++ b/container/pom.xml ++@@ -18,7 +18,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/controller/pom.xml b/controller/pom.xml ++index 46a3834c6..8432b220b 100644 ++--- a/controller/pom.xml +++++ b/controller/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ 4.0.0 ++ jar ++diff --git a/distribution/pom.xml b/distribution/pom.xml ++index 346c4de35..73474d34a 100644 ++--- a/distribution/pom.xml +++++ b/distribution/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ rocketmq-distribution ++ rocketmq-distribution ${project.version} ++diff --git a/example/pom.xml b/example/pom.xml ++index 9e7db43f8..a8c7f5382 100644 ++--- a/example/pom.xml +++++ b/example/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/filter/pom.xml b/filter/pom.xml ++index 84189066d..892f46e9d 100644 ++--- a/filter/pom.xml +++++ b/filter/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/namesrv/pom.xml b/namesrv/pom.xml ++index 7c218078a..e320ed573 100644 ++--- a/namesrv/pom.xml +++++ b/namesrv/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/openmessaging/pom.xml b/openmessaging/pom.xml ++index fd499e3de..f10c8af6f 100644 ++--- a/openmessaging/pom.xml +++++ b/openmessaging/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/pom.xml b/pom.xml ++index 0e1d04f15..4202d4095 100644 ++--- a/pom.xml +++++ b/pom.xml ++@@ -28,7 +28,7 @@ ++ 2012 ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ pom ++ Apache RocketMQ ${project.version} ++ http://rocketmq.apache.org/ ++@@ -37,7 +37,7 @@ ++ git@github.com:apache/rocketmq.git ++ scm:git:git@github.com:apache/rocketmq.git ++ scm:git:git@github.com:apache/rocketmq.git ++- rocketmq-all-5.1.4 +++ HEAD ++ ++ ++ ++diff --git a/proxy/pom.xml b/proxy/pom.xml ++index abf242eee..5c5349a8c 100644 ++--- a/proxy/pom.xml +++++ b/proxy/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/remoting/pom.xml b/remoting/pom.xml ++index fc70cb62e..f78480680 100644 ++--- a/remoting/pom.xml +++++ b/remoting/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/srvutil/pom.xml b/srvutil/pom.xml ++index d7f946cce..894e9cc6f 100644 ++--- a/srvutil/pom.xml +++++ b/srvutil/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/store/pom.xml b/store/pom.xml ++index 6d6983c5d..e979030e8 100644 ++--- a/store/pom.xml +++++ b/store/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/test/pom.xml b/test/pom.xml ++index 39090e426..168cbab0b 100644 ++--- a/test/pom.xml +++++ b/test/pom.xml ++@@ -20,7 +20,7 @@ ++ ++ rocketmq-all ++ org.apache.rocketmq ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/tieredstore/pom.xml b/tieredstore/pom.xml ++index 7b209751f..b2ea40bf3 100644 ++--- a/tieredstore/pom.xml +++++ b/tieredstore/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++diff --git a/tools/pom.xml b/tools/pom.xml ++index 806787ec9..e1daa57a6 100644 ++--- a/tools/pom.xml +++++ b/tools/pom.xml ++@@ -19,7 +19,7 @@ ++ ++ org.apache.rocketmq ++ rocketmq-all ++- 5.1.4 +++ 5.1.5-SNAPSHOT ++ ++ ++ 4.0.0 ++-- ++2.32.0.windows.2 ++ ++ ++From 88a9d939ce110381b3b418370d4711c0c214dc7f Mon Sep 17 00:00:00 2001 ++From: Ji Juntao ++Date: Sat, 23 Sep 2023 17:38:27 +0800 ++Subject: [PATCH 05/12] [ISSUE #7381] Fix the problem of inaccurate timer ++ message metric (#7382) ++ ++* correct the timerMetrics' result. ++ ++* for further extension. ++ ++* checkstyle. ++ ++* use toLong. ++--- ++ .../store/timer/TimerMessageStore.java | 20 +++++++++++++++---- ++ .../rocketmq/store/timer/TimerMetrics.java | 5 ++++- ++ .../rocketmq/store/timer/TimerRequest.java | 7 +++++-- ++ .../store/timer/TimerMetricsTest.java | 10 ++++++++-- ++ 4 files changed, 33 insertions(+), 9 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++index 0d50de65a..ac4c61cd6 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++@@ -41,6 +41,7 @@ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.function.Function; ++ import org.apache.commons.collections.CollectionUtils; +++import org.apache.commons.lang3.math.NumberUtils; ++ import org.apache.rocketmq.common.ServiceThread; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++ import org.apache.rocketmq.common.TopicFilterType; ++@@ -599,7 +600,12 @@ public class TimerMessageStore { ++ if (null == msg || null == msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC)) { ++ return; ++ } ++- timerMetrics.addAndGet(msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC), value); +++ if (msg.getProperty(TIMER_ENQUEUE_MS) != null +++ && NumberUtils.toLong(msg.getProperty(TIMER_ENQUEUE_MS)) == Long.MAX_VALUE) { +++ return; +++ } +++ // pass msg into addAndGet, for further more judgement extension. +++ timerMetrics.addAndGet(msg, value); ++ } catch (Throwable t) { ++ if (frequency.incrementAndGet() % 1000 == 0) { ++ LOGGER.error("error in adding metric", t); ++@@ -1323,6 +1329,7 @@ public class TimerMessageStore { ++ perfCounterTicks.startTick(ENQUEUE_PUT); ++ DefaultStoreMetricsManager.incTimerEnqueueCount(getRealTopic(req.getMsg())); ++ if (shouldRunningDequeue && req.getDelayTime() < currWriteTimeMs) { +++ req.setEnqueueTime(Long.MAX_VALUE); ++ dequeuePutQueue.put(req); ++ } else { ++ boolean doEnqueueRes = doEnqueue( ++@@ -1452,9 +1459,14 @@ public class TimerMessageStore { ++ } ++ try { ++ perfCounterTicks.startTick(DEQUEUE_PUT); ++- DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(tr.getMsg())); ++- addMetric(tr.getMsg(), -1); ++- MessageExtBrokerInner msg = convert(tr.getMsg(), tr.getEnqueueTime(), needRoll(tr.getMagic())); +++ MessageExt msgExt = tr.getMsg(); +++ DefaultStoreMetricsManager.incTimerDequeueCount(getRealTopic(msgExt)); +++ if (tr.getEnqueueTime() == Long.MAX_VALUE) { +++ // never enqueue, mark it. +++ MessageAccessor.putProperty(msgExt, TIMER_ENQUEUE_MS, String.valueOf(Long.MAX_VALUE)); +++ } +++ addMetric(msgExt, -1); +++ MessageExtBrokerInner msg = convert(msgExt, tr.getEnqueueTime(), needRoll(tr.getMagic())); ++ doRes = PUT_NEED_RETRY != doPut(msg, needRoll(tr.getMagic())); ++ while (!doRes && !isStopped()) { ++ if (!isRunningDequeue()) { ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java ++index e7b00cc07..7f8fedd8a 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMetrics.java ++@@ -38,6 +38,8 @@ import java.util.concurrent.locks.Lock; ++ import java.util.concurrent.locks.ReentrantLock; ++ import org.apache.rocketmq.common.ConfigManager; ++ import org.apache.rocketmq.common.constant.LoggerName; +++import org.apache.rocketmq.common.message.MessageConst; +++import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++@@ -78,7 +80,8 @@ public class TimerMetrics extends ConfigManager { ++ return distPair.getCount().addAndGet(value); ++ } ++ ++- public long addAndGet(String topic, int value) { +++ public long addAndGet(MessageExt msg, int value) { +++ String topic = msg.getProperty(MessageConst.PROPERTY_REAL_TOPIC); ++ Metric pair = getTopicPair(topic); ++ getDataVersion().nextVersion(); ++ pair.setTimeStamp(System.currentTimeMillis()); ++diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java ++index 1dd64f759..1b25d355c 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java +++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerRequest.java ++@@ -27,8 +27,9 @@ public class TimerRequest { ++ private final int sizePy; ++ private final long delayTime; ++ ++- private final long enqueueTime; ++ private final int magic; +++ +++ private long enqueueTime; ++ private MessageExt msg; ++ ++ ++@@ -94,7 +95,9 @@ public class TimerRequest { ++ public void setLatch(CountDownLatch latch) { ++ this.latch = latch; ++ } ++- +++ public void setEnqueueTime(long enqueueTime) { +++ this.enqueueTime = enqueueTime; +++ } ++ public void idempotentRelease() { ++ idempotentRelease(true); ++ } ++diff --git a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java ++index b7392cc45..3c7b9b67f 100644 ++--- a/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java +++++ b/store/src/test/java/org/apache/rocketmq/store/timer/TimerMetricsTest.java ++@@ -16,6 +16,9 @@ ++ */ ++ package org.apache.rocketmq.store.timer; ++ +++import org.apache.rocketmq.common.message.MessageAccessor; +++import org.apache.rocketmq.common.message.MessageConst; +++import org.apache.rocketmq.common.message.MessageExt; ++ import org.junit.Assert; ++ import org.junit.Test; ++ ++@@ -31,8 +34,11 @@ public class TimerMetricsTest { ++ ++ TimerMetrics first = new TimerMetrics(baseDir); ++ Assert.assertTrue(first.load()); ++- first.addAndGet("AAA", 1000); ++- first.addAndGet("BBB", 2000); +++ MessageExt msg = new MessageExt(); +++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "AAA"); +++ first.addAndGet(msg, 1000); +++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_REAL_TOPIC, "BBB"); +++ first.addAndGet(msg, 2000); ++ Assert.assertEquals(1000, first.getTimingCount("AAA")); ++ Assert.assertEquals(2000, first.getTimingCount("BBB")); ++ long curr = System.currentTimeMillis(); ++-- ++2.32.0.windows.2 ++ ++ ++From d7e5c4d1a4e048cd97f0b29a96a0fc575927a03e Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Mon, 25 Sep 2023 13:37:36 +0800 ++Subject: [PATCH 06/12] [ISSUE #7389] Fix the problem that getLastMappedFile ++ function affects performance ++ ++Co-authored-by: guyinyou ++--- ++ .../apache/rocketmq/store/MappedFileQueue.java | 15 +++++++++++++-- ++ 1 file changed, 13 insertions(+), 2 deletions(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++index 32b90d14f..9a0824829 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java +++++ b/store/src/main/java/org/apache/rocketmq/store/MappedFileQueue.java ++@@ -377,8 +377,19 @@ public class MappedFileQueue implements Swappable { ++ } ++ ++ public MappedFile getLastMappedFile() { ++- MappedFile[] mappedFiles = this.mappedFiles.toArray(new MappedFile[0]); ++- return mappedFiles.length == 0 ? null : mappedFiles[mappedFiles.length - 1]; +++ MappedFile mappedFileLast = null; +++ while (!this.mappedFiles.isEmpty()) { +++ try { +++ mappedFileLast = this.mappedFiles.get(this.mappedFiles.size() - 1); +++ break; +++ } catch (IndexOutOfBoundsException e) { +++ //continue; +++ } catch (Exception e) { +++ log.error("getLastMappedFile has exception.", e); +++ break; +++ } +++ } +++ return mappedFileLast; ++ } ++ ++ public boolean resetOffset(long offset) { ++-- ++2.32.0.windows.2 ++ ++ ++From 3fd43353fdf880deb5d63ba3ad50cc6e3259dc3a Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Tue, 26 Sep 2023 13:53:51 +0800 ++Subject: [PATCH 07/12] [ISSUE #7393] Add timeout configuration for grpc server ++ (#7394) ++ ++* Add timeout configuration for grpc server ++ ++* Add proxyConfig ++--- ++ .../java/org/apache/rocketmq/proxy/ProxyStartup.java | 1 + ++ .../apache/rocketmq/proxy/config/ProxyConfig.java | 9 +++++++++ ++ .../org/apache/rocketmq/proxy/grpc/GrpcServer.java | 10 ++++++++-- ++ .../rocketmq/proxy/grpc/GrpcServerBuilder.java | 12 +++++++++++- ++ 4 files changed, 29 insertions(+), 3 deletions(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java ++index 06d5f4525..3b2ca99bf 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/ProxyStartup.java ++@@ -85,6 +85,7 @@ public class ProxyStartup { ++ .addService(ChannelzService.newInstance(100)) ++ .addService(ProtoReflectionService.newInstance()) ++ .configInterceptor(accessValidators) +++ .shutdownTime(ConfigurationManager.getProxyConfig().getGrpcShutdownTimeSeconds(), TimeUnit.SECONDS) ++ .build(); ++ PROXY_START_AND_SHUTDOWN.appendStartAndShutdown(grpcServer); ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++index b2478fec3..c0d00d864 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/config/ProxyConfig.java ++@@ -87,6 +87,7 @@ public class ProxyConfig implements ConfigFile { ++ */ ++ private String proxyMode = ProxyMode.CLUSTER.name(); ++ private Integer grpcServerPort = 8081; +++ private long grpcShutdownTimeSeconds = 30; ++ private int grpcBossLoopNum = 1; ++ private int grpcWorkerLoopNum = PROCESSOR_NUMBER * 2; ++ private boolean enableGrpcEpoll = false; ++@@ -443,6 +444,14 @@ public class ProxyConfig implements ConfigFile { ++ this.grpcServerPort = grpcServerPort; ++ } ++ +++ public long getGrpcShutdownTimeSeconds() { +++ return grpcShutdownTimeSeconds; +++ } +++ +++ public void setGrpcShutdownTimeSeconds(long grpcShutdownTimeSeconds) { +++ this.grpcShutdownTimeSeconds = grpcShutdownTimeSeconds; +++ } +++ ++ public boolean isUseEndpointPortFromRequest() { ++ return useEndpointPortFromRequest; ++ } ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java ++index 1bffa3c0b..d5b896fe1 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServer.java ++@@ -29,8 +29,14 @@ public class GrpcServer implements StartAndShutdown { ++ ++ private final Server server; ++ ++- protected GrpcServer(Server server) { +++ private final long timeout; +++ +++ private final TimeUnit unit; +++ +++ protected GrpcServer(Server server, long timeout, TimeUnit unit) { ++ this.server = server; +++ this.timeout = timeout; +++ this.unit = unit; ++ } ++ ++ public void start() throws Exception { ++@@ -40,7 +46,7 @@ public class GrpcServer implements StartAndShutdown { ++ ++ public void shutdown() { ++ try { ++- this.server.shutdown().awaitTermination(30, TimeUnit.SECONDS); +++ this.server.shutdown().awaitTermination(timeout, unit); ++ log.info("grpc server shutdown successfully."); ++ } catch (Exception e) { ++ e.printStackTrace(); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java ++index 9cddd3013..0e79006f6 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/grpc/GrpcServerBuilder.java ++@@ -41,6 +41,10 @@ public class GrpcServerBuilder { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.PROXY_LOGGER_NAME); ++ protected NettyServerBuilder serverBuilder; ++ +++ protected long time = 30; +++ +++ protected TimeUnit unit = TimeUnit.SECONDS; +++ ++ public static GrpcServerBuilder newBuilder(ThreadPoolExecutor executor, int port) { ++ return new GrpcServerBuilder(executor, port); ++ } ++@@ -77,6 +81,12 @@ public class GrpcServerBuilder { ++ port, bossLoopNum, workerLoopNum, maxInboundMessageSize); ++ } ++ +++ public GrpcServerBuilder shutdownTime(long time, TimeUnit unit) { +++ this.time = time; +++ this.unit = unit; +++ return this; +++ } +++ ++ public GrpcServerBuilder addService(BindableService service) { ++ this.serverBuilder.addService(service); ++ return this; ++@@ -93,7 +103,7 @@ public class GrpcServerBuilder { ++ } ++ ++ public GrpcServer build() { ++- return new GrpcServer(this.serverBuilder.build()); +++ return new GrpcServer(this.serverBuilder.build(), time, unit); ++ } ++ ++ public GrpcServerBuilder configInterceptor(List accessValidators) { ++-- ++2.32.0.windows.2 ++ ++ ++From c3b86cd1e3c068bc5847671c899a284e49a2ecdc Mon Sep 17 00:00:00 2001 ++From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= ++ ++Date: Tue, 26 Sep 2023 16:07:13 +0800 ++Subject: [PATCH 08/12] [ISSUE #7398] Fix ExportConfigsCommand NPE (#7399) ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Co-authored-by: 石臻臻 ++--- ++ .../command/export/ExportConfigsCommand.java | 42 ++++++++++++------- ++ 1 file changed, 26 insertions(+), 16 deletions(-) ++ ++diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++index 03613b29c..c3f96d597 100644 ++--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java +++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportConfigsCommand.java ++@@ -20,6 +20,7 @@ import java.util.HashMap; ++ import java.util.List; ++ import java.util.Map; ++ import java.util.Map.Entry; +++import java.util.Arrays; ++ import java.util.Properties; ++ ++ import com.alibaba.fastjson.JSON; ++@@ -106,24 +107,33 @@ public class ExportConfigsCommand implements SubCommand { ++ } ++ } ++ +++ ++ private Properties needBrokerProprties(Properties properties) { +++ List propertyKeys = Arrays.asList( +++ "brokerClusterName", +++ "brokerId", +++ "brokerName", +++ "brokerRole", +++ "fileReservedTime", +++ "filterServerNums", +++ "flushDiskType", +++ "maxMessageSize", +++ "messageDelayLevel", +++ "msgTraceTopicName", +++ "slaveReadEnable", +++ "traceOn", +++ "traceTopicEnable", +++ "useTLS", +++ "autoCreateTopicEnable", +++ "autoCreateSubscriptionGroup" +++ ); +++ ++ Properties newProperties = new Properties(); ++- newProperties.setProperty("brokerClusterName", properties.getProperty("brokerClusterName")); ++- newProperties.setProperty("brokerId", properties.getProperty("brokerId")); ++- newProperties.setProperty("brokerName", properties.getProperty("brokerName")); ++- newProperties.setProperty("brokerRole", properties.getProperty("brokerRole")); ++- newProperties.setProperty("fileReservedTime", properties.getProperty("fileReservedTime")); ++- newProperties.setProperty("filterServerNums", properties.getProperty("filterServerNums")); ++- newProperties.setProperty("flushDiskType", properties.getProperty("flushDiskType")); ++- newProperties.setProperty("maxMessageSize", properties.getProperty("maxMessageSize")); ++- newProperties.setProperty("messageDelayLevel", properties.getProperty("messageDelayLevel")); ++- newProperties.setProperty("msgTraceTopicName", properties.getProperty("msgTraceTopicName")); ++- newProperties.setProperty("slaveReadEnable", properties.getProperty("slaveReadEnable")); ++- newProperties.setProperty("traceOn", properties.getProperty("traceOn")); ++- newProperties.setProperty("traceTopicEnable", properties.getProperty("traceTopicEnable")); ++- newProperties.setProperty("useTLS", properties.getProperty("useTLS")); ++- newProperties.setProperty("autoCreateTopicEnable", properties.getProperty("autoCreateTopicEnable")); ++- newProperties.setProperty("autoCreateSubscriptionGroup", properties.getProperty("autoCreateSubscriptionGroup")); +++ propertyKeys.stream() +++ .filter(key -> properties.getProperty(key) != null) +++ .forEach(key -> newProperties.setProperty(key, properties.getProperty(key))); +++ ++ return newProperties; ++ } +++ ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 959a98120cea8022498557a308aff35e3d8def2a Mon Sep 17 00:00:00 2001 ++From: =?UTF-8?q?=E7=9F=B3=E8=87=BB=E8=87=BB=28Steven=20shi=29?= ++ ++Date: Wed, 27 Sep 2023 01:59:58 +0800 ++Subject: [PATCH 09/12] [ISSUE #7400] Fix getBrokerEpochSubCommand NPE ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Co-authored-by: 石臻臻 ++--- ++ .../broker/processor/AdminBrokerProcessor.java | 10 ++++++++-- ++ 1 file changed, 8 insertions(+), 2 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++index 9e48431be..e77120e15 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/AdminBrokerProcessor.java ++@@ -2736,10 +2736,16 @@ public class AdminBrokerProcessor implements NettyRequestProcessor { ++ final ReplicasManager replicasManager = this.brokerController.getReplicasManager(); ++ assert replicasManager != null; ++ final BrokerConfig brokerConfig = this.brokerController.getBrokerConfig(); +++ final RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ +++ if (!brokerConfig.isEnableControllerMode()) { +++ response.setCode(ResponseCode.SYSTEM_ERROR); +++ response.setRemark("this request only for controllerMode "); +++ return response; +++ } ++ final EpochEntryCache entryCache = new EpochEntryCache(brokerConfig.getBrokerClusterName(), ++- brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset()); +++ brokerConfig.getBrokerName(), brokerConfig.getBrokerId(), replicasManager.getEpochEntries(), this.brokerController.getMessageStore().getMaxPhyOffset()); ++ ++- final RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setBody(entryCache.encode()); ++ response.setCode(ResponseCode.SUCCESS); ++ response.setRemark(null); ++-- ++2.32.0.windows.2 ++ ++ ++From 0a6ae4605fef4eaab6262fbd370aba887d8ae58b Mon Sep 17 00:00:00 2001 ++From: tiger lee <1026203200@qq.com> ++Date: Wed, 27 Sep 2023 14:43:15 +0800 ++Subject: [PATCH 10/12] [ISSUE #7396] Fix wrong word in ++ BrokerController#doResterBrokerAll (#7397) ++ ++--- ++ .../main/java/org/apache/rocketmq/broker/BrokerController.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++index 53e2e1b62..d4bded600 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/BrokerController.java ++@@ -1807,7 +1807,7 @@ public class BrokerController { ++ TopicConfigSerializeWrapper topicConfigWrapper) { ++ ++ if (shutdown) { ++- BrokerController.LOG.info("BrokerController#doResterBrokerAll: broker has shutdown, no need to register any more."); +++ BrokerController.LOG.info("BrokerController#doRegisterBrokerAll: broker has shutdown, no need to register any more."); ++ return; ++ } ++ List registerBrokerResultList = this.brokerOuterAPI.registerBrokerAll( ++-- ++2.32.0.windows.2 ++ ++ ++From 4f1b42a7c5557bcadd6b9982a0c9bd876622c69e Mon Sep 17 00:00:00 2001 ++From: ShuangxiDing ++Date: Thu, 28 Sep 2023 16:52:02 +0800 ++Subject: [PATCH 11/12] [ISSUE #7410] Handle the Exception when the Proxy ++ requests the client ++MIME-Version: 1.0 ++Content-Type: text/plain; charset=UTF-8 ++Content-Transfer-Encoding: 8bit ++ ++Co-authored-by: 徒钟 ++--- ++ .../remoting/channel/RemotingChannel.java | 23 ++++++++++++++----- ++ 1 file changed, 17 insertions(+), 6 deletions(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java ++index 40946cabf..d755fdcc4 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/channel/RemotingChannel.java ++@@ -34,6 +34,7 @@ import org.apache.rocketmq.common.utils.NetworkUtil; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.proxy.common.channel.ChannelHelper; +++import org.apache.rocketmq.proxy.common.utils.ExceptionUtils; ++ import org.apache.rocketmq.proxy.common.utils.FutureUtils; ++ import org.apache.rocketmq.proxy.config.ConfigurationManager; ++ import org.apache.rocketmq.proxy.processor.channel.ChannelExtendAttributeGetter; ++@@ -158,10 +159,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ ConsumerRunningInfo consumerRunningInfo = ConsumerRunningInfo.decode(response.getBody(), ConsumerRunningInfo.class); ++ responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", consumerRunningInfo)); +++ } else { +++ String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark()); +++ RuntimeException e = new RuntimeException(errMsg); +++ responseFuture.completeExceptionally(e); ++ } ++- String errMsg = String.format("get consumer running info failed, code:%s remark:%s", response.getCode(), response.getRemark()); ++- RuntimeException e = new RuntimeException(errMsg); ++- responseFuture.completeExceptionally(e); +++ }) +++ .exceptionally(t -> { +++ responseFuture.completeExceptionally(ExceptionUtils.getRealException(t)); +++ return null; ++ }); ++ return CompletableFuture.completedFuture(null); ++ } catch (Throwable t) { ++@@ -183,10 +189,15 @@ public class RemotingChannel extends ProxyChannel implements RemoteChannelConver ++ if (response.getCode() == ResponseCode.SUCCESS) { ++ ConsumeMessageDirectlyResult result = ConsumeMessageDirectlyResult.decode(response.getBody(), ConsumeMessageDirectlyResult.class); ++ responseFuture.complete(new ProxyRelayResult<>(ResponseCode.SUCCESS, "", result)); +++ } else { +++ String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark()); +++ RuntimeException e = new RuntimeException(errMsg); +++ responseFuture.completeExceptionally(e); ++ } ++- String errMsg = String.format("consume message directly failed, code:%s remark:%s", response.getCode(), response.getRemark()); ++- RuntimeException e = new RuntimeException(errMsg); ++- responseFuture.completeExceptionally(e); +++ }) +++ .exceptionally(t -> { +++ responseFuture.completeExceptionally(ExceptionUtils.getRealException(t)); +++ return null; ++ }); ++ return CompletableFuture.completedFuture(null); ++ } catch (Throwable t) { ++-- ++2.32.0.windows.2 ++ ++ ++From c36bb78e850129b9db40adc5b0e1b9bfd5c8fd2e Mon Sep 17 00:00:00 2001 ++From: shriVATSA54 <116296557+shriVATSA54@users.noreply.github.com> ++Date: Sat, 7 Oct 2023 12:22:39 +0530 ++Subject: [PATCH 12/12] [ISSUE 7313] Enhancement Optimization Method name ++ (#7420) ++ ++* Enhancment/method_name/#7313/ ++ ++* Enhancment/method_name/#7313/ ++ ++* Enhancment/method_name/#7313/ ++--- ++ .../tieredstore/provider/TieredStoreTopicBlackListFilter.java | 2 +- ++ .../rocketmq/tieredstore/provider/TieredStoreTopicFilter.java | 2 +- ++ .../provider/TieredStoreTopicBlackListFilterTest.java | 2 +- ++ 3 files changed, 3 insertions(+), 3 deletions(-) ++ ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++index 50adbb713..f8bf165bc 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilter.java ++@@ -39,7 +39,7 @@ public class TieredStoreTopicBlackListFilter implements TieredStoreTopicFilter { ++ } ++ ++ @Override ++- public void addTopicToWhiteList(String topicName) { +++ public void addTopicToBlackList(String topicName) { ++ this.topicBlackSet.add(topicName); ++ } ++ } ++diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++index 3f26b8b02..f983ed6e9 100644 ++--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java +++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicFilter.java ++@@ -21,5 +21,5 @@ public interface TieredStoreTopicFilter { ++ ++ boolean filterTopic(String topicName); ++ ++- void addTopicToWhiteList(String topicName); +++ void addTopicToBlackList(String topicName); ++ } ++diff --git a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++index 2bf48173c..fbaafa1b4 100644 ++--- a/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java +++++ b/tieredstore/src/test/java/org/apache/rocketmq/tieredstore/provider/TieredStoreTopicBlackListFilterTest.java ++@@ -30,7 +30,7 @@ public class TieredStoreTopicBlackListFilterTest { ++ ++ String topicName = "WhiteTopic"; ++ Assert.assertFalse(topicFilter.filterTopic(topicName)); ++- topicFilter.addTopicToWhiteList(topicName); +++ topicFilter.addTopicToBlackList(topicName); ++ Assert.assertTrue(topicFilter.filterTopic(topicName)); ++ } ++ } ++\ No newline at end of file ++-- ++2.32.0.windows.2 ++ +diff --git a/patch020-backport-add-goaway-mechanism.patch b/patch020-backport-add-goaway-mechanism.patch +new file mode 100644 +index 000000000..122db0325 +--- /dev/null ++++ b/patch020-backport-add-goaway-mechanism.patch +@@ -0,0 +1,3696 @@ ++From 84156084a4c5228e1d2fe21e068fff330bbc40d1 Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Sun, 8 Oct 2023 11:13:25 +0800 ++Subject: [PATCH 1/7] [ISSUE #7321] Refector NettyRemotingAbstract with unify ++ future implementation (#7322) ++ ++* Refector NettyRemotingAbstract ++ ++* Add invoke with future method ++ ++* Deprecate InvokeCallback#operationComplete ++ ++* Add operationSuccess and operationException for InvokeCallback ++ ++* fix unit test ++ ++* fix unit test ++ ++* Keep InvokeCallback#operationComplete ++ ++* Optimize invokeAsyncImpl operationComplete ++ ++* Add unit test for NettyRemotingClient ++ ++* fix checkstyle ++--- ++ .../rocketmq/broker/out/BrokerOuterAPI.java | 147 +++++---- ++ .../rocketmq/client/impl/MQAdminImpl.java | 71 ++-- ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 239 +++++++------- ++ .../client/impl/mqclient/MQClientAPIExt.java | 309 ++++++++---------- ++ .../client/impl/MQClientAPIImplTest.java | 12 +- ++ .../remoting/RemotingProtocolServer.java | 22 +- ++ .../service/mqclient/MQClientAPIExtTest.java | 97 +++--- ++ .../rocketmq/remoting/InvokeCallback.java | 15 + ++ .../rocketmq/remoting/RemotingClient.java | 27 +- ++ .../remoting/netty/NettyRemotingAbstract.java | 123 ++++--- ++ .../remoting/netty/NettyRemotingClient.java | 33 +- ++ .../remoting/netty/ResponseFuture.java | 15 + ++ .../rocketmq/remoting/rpc/RpcClientImpl.java | 29 +- ++ .../rocketmq/remoting/RemotingServerTest.java | 22 +- ++ .../rocketmq/remoting/netty/MockChannel.java | 21 +- ++ .../remoting/netty/MockChannelPromise.java | 191 +++++++++++ ++ .../netty/NettyRemotingAbstractTest.java | 54 ++- ++ .../netty/NettyRemotingClientTest.java | 185 ++++++++++- ++ 18 files changed, 1029 insertions(+), 583 deletions(-) ++ rename client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java => remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java (57%) ++ create mode 100644 remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++index 9dfb8127d..6fde48dd9 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/out/BrokerOuterAPI.java ++@@ -73,6 +73,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.netty.NettyClientConfig; ++ import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +++import org.apache.rocketmq.remoting.netty.ResponseFuture; ++ import org.apache.rocketmq.remoting.protocol.BrokerSyncInfo; ++ import org.apache.rocketmq.remoting.protocol.DataVersion; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++@@ -107,6 +108,8 @@ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.SendMessageRequestHeaderV2; ++ import org.apache.rocketmq.remoting.protocol.header.SendMessageResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.AlterSyncStateSetRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; +++import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetMetaDataResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.controller.GetReplicaInfoResponseHeader; ++@@ -124,8 +127,6 @@ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerReques ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterBrokerResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.RegisterTopicRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.namesrv.UnRegisterBrokerRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterRequestHeader; ++-import org.apache.rocketmq.remoting.protocol.header.controller.ElectMasterResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.heartbeat.SubscriptionData; ++ import org.apache.rocketmq.remoting.protocol.namesrv.RegisterBrokerResult; ++ import org.apache.rocketmq.remoting.protocol.route.BrokerData; ++@@ -151,7 +152,6 @@ public class BrokerOuterAPI { ++ private final RpcClient rpcClient; ++ private String nameSrvAddr = null; ++ ++- ++ public BrokerOuterAPI(final NettyClientConfig nettyClientConfig) { ++ this(nettyClientConfig, new DynamicalExtFieldRPCHook(), new ClientMetadata()); ++ } ++@@ -459,7 +459,7 @@ public class BrokerOuterAPI { ++ * @param filterServerList ++ * @param oneway ++ * @param timeoutMills ++- * @param compressed default false +++ * @param compressed default false ++ * @return ++ */ ++ public List registerBrokerAll( ++@@ -643,7 +643,6 @@ public class BrokerOuterAPI { ++ queueDatas.add(queueData); ++ final byte[] topicRouteBody = topicRouteData.encode(); ++ ++- ++ List nameServerAddressList = this.remotingClient.getNameServerAddressList(); ++ final CountDownLatch countDownLatch = new CountDownLatch(nameServerAddressList.size()); ++ for (final String namesrvAddr : nameServerAddressList) { ++@@ -910,25 +909,33 @@ public class BrokerOuterAPI { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); ++ ++ request.setBody(requestBody.encode()); ++- this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { ++- if (callback == null) { ++- return; +++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ ++ } ++ ++- try { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), ++- LockBatchResponseBody.class); ++- Set messageQueues = responseBody.getLockOKMQSet(); ++- callback.onSuccess(messageQueues); ++- } else { ++- callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); ++- } +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ if (callback == null) { +++ return; ++ } ++- } catch (Throwable ignored) { +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), +++ LockBatchResponseBody.class); +++ Set messageQueues = responseBody.getLockOKMQSet(); +++ callback.onSuccess(messageQueues); +++ } else { +++ callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ } ++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ if (callback == null) { +++ return; +++ } +++ callback.onException(throwable); ++ } ++ }); ++ } ++@@ -942,22 +949,30 @@ public class BrokerOuterAPI { ++ ++ request.setBody(requestBody.encode()); ++ ++- this.remotingClient.invokeAsync(addr, request, timeoutMillis, responseFuture -> { ++- if (callback == null) { ++- return; +++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ ++ } ++ ++- try { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- callback.onSuccess(); ++- } else { ++- callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); ++- } +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ if (callback == null) { +++ return; ++ } ++- } catch (Throwable ignored) { +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ callback.onSuccess(); +++ } else { +++ callback.onException(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ } ++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ if (callback == null) { +++ return; +++ } +++ callback.onException(throwable); ++ } ++ }); ++ } ++@@ -983,21 +998,27 @@ public class BrokerOuterAPI { ++ CompletableFuture cf = new CompletableFuture<>(); ++ final String msgId = msg.getMsgId(); ++ try { ++- this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (null != response) { ++- SendResult sendResult = null; +++ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ try { ++- sendResult = this.processSendResponse(brokerName, msg, response); +++ SendResult sendResult = processSendResponse(brokerName, msg, response); ++ cf.complete(sendResult); ++ } catch (MQBrokerException | RemotingCommandException e) { ++ LOGGER.error("processSendResponse in sendMessageToSpecificBrokerAsync failed, msgId=" + msgId, e); ++ cf.completeExceptionally(e); ++ } ++- } else { ++- cf.complete(null); ++ } ++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ cf.completeExceptionally(throwable); +++ } ++ }); ++ } catch (Throwable t) { ++ LOGGER.error("invokeAsync failed in sendMessageToSpecificBrokerAsync, msgId=" + msgId, t); ++@@ -1057,7 +1078,7 @@ public class BrokerOuterAPI { ++ } ++ if (sendStatus != null) { ++ SendMessageResponseHeader responseHeader = ++- (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); +++ (SendMessageResponseHeader) response.decodeCommandCustomHeader(SendMessageResponseHeader.class); ++ ++ //If namespace not null , reset Topic without namespace. ++ String topic = msg.getTopic(); ++@@ -1073,8 +1094,8 @@ public class BrokerOuterAPI { ++ uniqMsgId = sb.toString(); ++ } ++ SendResult sendResult = new SendResult(sendStatus, ++- uniqMsgId, ++- responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); +++ uniqMsgId, +++ responseHeader.getMsgId(), messageQueue, responseHeader.getQueueOffset()); ++ sendResult.setTransactionId(responseHeader.getTransactionId()); ++ String regionId = response.getExtFields().get(MessageConst.PROPERTY_MSG_REGION); ++ String traceOn = response.getExtFields().get(MessageConst.PROPERTY_TRACE_SWITCH); ++@@ -1218,8 +1239,9 @@ public class BrokerOuterAPI { ++ /** ++ * Broker try to elect itself as a master in broker set ++ */ ++- public Pair> brokerElect(String controllerAddress, String clusterName, String brokerName, ++- Long brokerId) throws Exception { +++ public Pair> brokerElect(String controllerAddress, String clusterName, +++ String brokerName, +++ Long brokerId) throws Exception { ++ ++ final ElectMasterRequestHeader requestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_ELECT_MASTER, requestHeader); ++@@ -1237,7 +1259,8 @@ public class BrokerOuterAPI { ++ throw new MQBrokerException(response.getCode(), response.getRemark()); ++ } ++ ++- public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, final String controllerAddress) throws Exception { +++ public GetNextBrokerIdResponseHeader getNextBrokerId(final String clusterName, final String brokerName, +++ final String controllerAddress) throws Exception { ++ final GetNextBrokerIdRequestHeader requestHeader = new GetNextBrokerIdRequestHeader(clusterName, brokerName); ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_GET_NEXT_BROKER_ID, requestHeader); ++ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); ++@@ -1248,7 +1271,8 @@ public class BrokerOuterAPI { ++ throw new MQBrokerException(response.getCode(), response.getRemark()); ++ } ++ ++- public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { +++ public ApplyBrokerIdResponseHeader applyBrokerId(final String clusterName, final String brokerName, +++ final Long brokerId, final String registerCheckCode, final String controllerAddress) throws Exception { ++ final ApplyBrokerIdRequestHeader requestHeader = new ApplyBrokerIdRequestHeader(clusterName, brokerName, brokerId, registerCheckCode); ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_APPLY_BROKER_ID, requestHeader); ++ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); ++@@ -1259,7 +1283,9 @@ public class BrokerOuterAPI { ++ throw new MQBrokerException(response.getCode(), response.getRemark()); ++ } ++ ++- public Pair> registerBrokerToController(final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, final String controllerAddress) throws Exception { +++ public Pair> registerBrokerToController( +++ final String clusterName, final String brokerName, final Long brokerId, final String brokerAddress, +++ final String controllerAddress) throws Exception { ++ final RegisterBrokerToControllerRequestHeader requestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, brokerId, brokerAddress); ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONTROLLER_REGISTER_BROKER, requestHeader); ++ final RemotingCommand response = this.remotingClient.invokeSync(controllerAddress, request, 3000); ++@@ -1355,16 +1381,25 @@ public class BrokerOuterAPI { ++ ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, requestHeader); ++ CompletableFuture pullResultFuture = new CompletableFuture<>(); ++- this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- if (responseFuture.getCause() != null) { ++- pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); ++- return; +++ this.remotingClient.invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ ++ } ++- try { ++- PullResultExt pullResultExt = this.processPullResponse(responseFuture.getResponseCommand(), brokerAddr); ++- this.processPullResult(pullResultExt, brokerName, queueId); ++- pullResultFuture.complete(pullResultExt); ++- } catch (Exception e) { +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ try { +++ PullResultExt pullResultExt = processPullResponse(response, brokerAddr); +++ processPullResult(pullResultExt, brokerName, queueId); +++ pullResultFuture.complete(pullResultExt); +++ } catch (Exception e) { +++ pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); +++ } +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { ++ pullResultFuture.complete(new PullResult(PullStatus.NO_MATCHED_MSG, -1, -1, -1, new ArrayList<>())); ++ } ++ }); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java ++index 1ef3a9483..83835bd3d 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQAdminImpl.java ++@@ -44,6 +44,8 @@ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageId; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.common.utils.NetworkUtil; +++import org.apache.rocketmq.logging.org.slf4j.Logger; +++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.exception.RemotingCommandException; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++@@ -55,8 +57,6 @@ import org.apache.rocketmq.remoting.protocol.header.QueryMessageRequestHeader; ++ import org.apache.rocketmq.remoting.protocol.header.QueryMessageResponseHeader; ++ import org.apache.rocketmq.remoting.protocol.route.BrokerData; ++ import org.apache.rocketmq.remoting.protocol.route.TopicRouteData; ++-import org.apache.rocketmq.logging.org.slf4j.Logger; ++-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++ ++ public class MQAdminImpl { ++ ++@@ -357,44 +357,51 @@ public class MQAdminImpl { ++ new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ try { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- switch (response.getCode()) { ++- case ResponseCode.SUCCESS: { ++- QueryMessageResponseHeader responseHeader = null; ++- try { ++- responseHeader = ++- (QueryMessageResponseHeader) response ++- .decodeCommandCustomHeader(QueryMessageResponseHeader.class); ++- } catch (RemotingCommandException e) { ++- log.error("decodeCommandCustomHeader exception", e); ++- return; ++- } ++- ++- List wrappers = ++- MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); ++- ++- QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); ++- try { ++- lock.writeLock().lock(); ++- queryResultList.add(qr); ++- } finally { ++- lock.writeLock().unlock(); ++- } ++- break; +++ switch (response.getCode()) { +++ case ResponseCode.SUCCESS: { +++ QueryMessageResponseHeader responseHeader = null; +++ try { +++ responseHeader = +++ (QueryMessageResponseHeader) response +++ .decodeCommandCustomHeader(QueryMessageResponseHeader.class); +++ } catch (RemotingCommandException e) { +++ log.error("decodeCommandCustomHeader exception", e); +++ return; ++ } ++- default: ++- log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); ++- break; +++ +++ List wrappers = +++ MessageDecoder.decodes(ByteBuffer.wrap(response.getBody()), true); +++ +++ QueryResult qr = new QueryResult(responseHeader.getIndexLastUpdateTimestamp(), wrappers); +++ try { +++ lock.writeLock().lock(); +++ queryResultList.add(qr); +++ } finally { +++ lock.writeLock().unlock(); +++ } +++ break; ++ } ++- } else { ++- log.warn("getResponseCommand return null"); +++ default: +++ log.warn("getResponseCommand failed, {} {}", response.getCode(), response.getRemark()); +++ break; ++ } +++ ++ } finally { ++ countDownLatch.countDown(); ++ } ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ log.error("queryMessage error, requestHeader={}", requestHeader); +++ countDownLatch.countDown(); +++ } ++ }, isUniqKey); ++ } catch (Exception e) { ++ log.warn("queryMessage exception", e); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 3201a493f..2407e5737 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicInteger; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.client.ClientConfig; ++ import org.apache.rocketmq.client.Validators; ++-import org.apache.rocketmq.client.common.ClientErrorCode; ++ import org.apache.rocketmq.client.consumer.AckCallback; ++ import org.apache.rocketmq.client.consumer.AckResult; ++ import org.apache.rocketmq.client.consumer.AckStatus; ++@@ -653,10 +652,13 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++- long cost = System.currentTimeMillis() - beginStartTime; ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (null == sendCallback && response != null) { ++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ long cost = System.currentTimeMillis() - beginStartTime; +++ if (null == sendCallback) { ++ try { ++ SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); ++ if (context != null && sendResult != null) { ++@@ -666,46 +668,47 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ } catch (Throwable e) { ++ } ++ ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); ++ return; ++ } ++ ++- if (response != null) { +++ try { +++ SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); +++ assert sendResult != null; +++ if (context != null) { +++ context.setSendResult(sendResult); +++ context.getProducer().executeSendMessageHookAfter(context); +++ } +++ ++ try { ++- SendResult sendResult = MQClientAPIImpl.this.processSendResponse(brokerName, msg, response, addr); ++- assert sendResult != null; ++- if (context != null) { ++- context.setSendResult(sendResult); ++- context.getProducer().executeSendMessageHookAfter(context); ++- } +++ sendCallback.onSuccess(sendResult); +++ } catch (Throwable e) { +++ } ++ ++- try { ++- sendCallback.onSuccess(sendResult); ++- } catch (Throwable e) { ++- } +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, false, true); +++ } catch (Exception e) { +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); +++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +++ retryTimesWhenSendFailed, times, e, context, false, producer); +++ } +++ } ++ ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), false, true); ++- } catch (Exception e) { ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); ++- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++- retryTimesWhenSendFailed, times, e, context, false, producer); ++- } +++ @Override +++ public void operationFail(Throwable throwable) { +++ producer.updateFaultItem(brokerName, System.currentTimeMillis() - beginStartTime, true, true); +++ long cost = System.currentTimeMillis() - beginStartTime; +++ if (throwable instanceof RemotingSendRequestException) { +++ MQClientException ex = new MQClientException("send request failed", throwable); +++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +++ retryTimesWhenSendFailed, times, ex, context, true, producer); +++ } else if (throwable instanceof RemotingTimeoutException) { +++ MQClientException ex = new MQClientException("wait response timeout, cost=" + cost, throwable); +++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +++ retryTimesWhenSendFailed, times, ex, context, true, producer); ++ } else { ++- producer.updateFaultItem(brokerName, System.currentTimeMillis() - responseFuture.getBeginTimestamp(), true, true); ++- if (!responseFuture.isSendRequestOK()) { ++- MQClientException ex = new MQClientException("send request failed", responseFuture.getCause()); ++- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++- retryTimesWhenSendFailed, times, ex, context, true, producer); ++- } else if (responseFuture.isTimeout()) { ++- MQClientException ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", ++- responseFuture.getCause()); ++- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++- retryTimesWhenSendFailed, times, ex, context, true, producer); ++- } else { ++- MQClientException ex = new MQClientException("unknow reseaon", responseFuture.getCause()); ++- onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, ++- retryTimesWhenSendFailed, times, ex, context, true, producer); ++- } +++ MQClientException ex = new MQClientException("unknow reseaon", throwable); +++ onExceptionImpl(brokerName, msg, timeoutMillis - cost, request, sendCallback, topicPublishInfo, instance, +++ retryTimesWhenSendFailed, times, ex, context, true, producer); ++ } ++ } ++ }); ++@@ -857,30 +860,25 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ final long timeoutMillis, final PopCallback popCallback ++ ) throws RemotingException, InterruptedException { ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader); ++- this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { +++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++- public void onComplete(ResponseFuture responseFuture) { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- PopResult ++- popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); ++- assert popResult != null; ++- popCallback.onSuccess(popResult); ++- } catch (Exception e) { ++- popCallback.onException(e); ++- } ++- } else { ++- if (!responseFuture.isSendRequestOK()) { ++- popCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- popCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, ++- responseFuture.getCause())); ++- } else { ++- popCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); ++- } +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ try { +++ PopResult popResult = MQClientAPIImpl.this.processPopResponse(brokerName, response, requestHeader.getTopic(), requestHeader); +++ popCallback.onSuccess(popResult); +++ } catch (Exception e) { +++ popCallback.onException(e); ++ } ++ } +++ @Override +++ public void operationFail(Throwable throwable) { +++ popCallback.onException(throwable); +++ } ++ }); ++ } ++ ++@@ -959,34 +957,26 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ request.setBody(requestBody.encode()); ++ } ++ } ++- this.remotingClient.invokeAsync(addr, request, timeOut, new BaseInvokeCallback(MQClientAPIImpl.this) { +++ this.remotingClient.invokeAsync(addr, request, timeOut, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } ++ ++ @Override ++- public void onComplete(ResponseFuture responseFuture) { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- AckResult ackResult = new AckResult(); ++- if (ResponseCode.SUCCESS == response.getCode()) { ++- ackResult.setStatus(AckStatus.OK); ++- } else { ++- ackResult.setStatus(AckStatus.NO_EXIST); ++- } ++- ackCallback.onSuccess(ackResult); ++- } catch (Exception e) { ++- ackCallback.onException(e); ++- } +++ public void operationSucceed(RemotingCommand response) { +++ AckResult ackResult = new AckResult(); +++ if (ResponseCode.SUCCESS == response.getCode()) { +++ ackResult.setStatus(AckStatus.OK); ++ } else { ++- if (!responseFuture.isSendRequestOK()) { ++- ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, ++- responseFuture.getCause())); ++- } else { ++- ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeOut + ". Request: " + request, responseFuture.getCause())); ++- } +++ ackResult.setStatus(AckStatus.NO_EXIST); ++ } +++ ackCallback.onSuccess(ackResult); +++ } ++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ ackCallback.onException(throwable); ++ } ++ }); ++ } ++@@ -999,39 +989,37 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ final AckCallback ackCallback ++ ) throws RemotingException, MQBrokerException, InterruptedException { ++ final RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader); ++- this.remotingClient.invokeAsync(addr, request, timeoutMillis, new BaseInvokeCallback(MQClientAPIImpl.this) { +++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++- public void onComplete(ResponseFuture responseFuture) { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); ++- AckResult ackResult = new AckResult(); ++- if (ResponseCode.SUCCESS == response.getCode()) { ++- ackResult.setStatus(AckStatus.OK); ++- ackResult.setPopTime(responseHeader.getPopTime()); ++- ackResult.setExtraInfo(ExtraInfoUtil ++- .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), ++- responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR ++- + requestHeader.getOffset()); ++- } else { ++- ackResult.setStatus(AckStatus.NO_EXIST); ++- } ++- ackCallback.onSuccess(ackResult); ++- } catch (Exception e) { ++- ackCallback.onException(e); ++- } ++- } else { ++- if (!responseFuture.isSendRequestOK()) { ++- ackCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- ackCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, ++- responseFuture.getCause())); +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ try { +++ ChangeInvisibleTimeResponseHeader responseHeader = (ChangeInvisibleTimeResponseHeader) response.decodeCommandCustomHeader(ChangeInvisibleTimeResponseHeader.class); +++ AckResult ackResult = new AckResult(); +++ if (ResponseCode.SUCCESS == response.getCode()) { +++ ackResult.setStatus(AckStatus.OK); +++ ackResult.setPopTime(responseHeader.getPopTime()); +++ ackResult.setExtraInfo(ExtraInfoUtil +++ .buildExtraInfo(requestHeader.getOffset(), responseHeader.getPopTime(), responseHeader.getInvisibleTime(), +++ responseHeader.getReviveQid(), requestHeader.getTopic(), brokerName, requestHeader.getQueueId()) + MessageConst.KEY_SEPARATOR +++ + requestHeader.getOffset()); ++ } else { ++- ackCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); +++ ackResult.setStatus(AckStatus.NO_EXIST); ++ } +++ ackCallback.onSuccess(ackResult); +++ } catch (Exception e) { +++ ackCallback.onException(e); ++ } ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ ackCallback.onException(throwable); +++ } ++ }); ++ } ++ ++@@ -1044,26 +1032,23 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ this.remotingClient.invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); ++- assert pullResult != null; ++- pullCallback.onSuccess(pullResult); ++- } catch (Exception e) { ++- pullCallback.onException(e); ++- } ++- } else { ++- if (!responseFuture.isSendRequestOK()) { ++- pullCallback.onException(new MQClientException(ClientErrorCode.CONNECT_BROKER_EXCEPTION, "send request failed to " + addr + ". Request: " + request, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- pullCallback.onException(new MQClientException(ClientErrorCode.ACCESS_BROKER_TIMEOUT, "wait response from " + addr + " timeout :" + responseFuture.getTimeoutMillis() + "ms" + ". Request: " + request, ++- responseFuture.getCause())); ++- } else { ++- pullCallback.onException(new MQClientException("unknown reason. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + request, responseFuture.getCause())); ++- } +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ try { +++ PullResult pullResult = MQClientAPIImpl.this.processPullResponse(response, addr); +++ pullCallback.onSuccess(pullResult); +++ } catch (Exception e) { +++ pullCallback.onException(e); ++ } ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ pullCallback.onException(throwable); +++ } ++ }); ++ } ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++index d7c8ef8d9..f3102e175 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/mqclient/MQClientAPIExt.java ++@@ -30,7 +30,6 @@ import org.apache.rocketmq.client.consumer.PullCallback; ++ import org.apache.rocketmq.client.consumer.PullResult; ++ import org.apache.rocketmq.client.consumer.PullStatus; ++ import org.apache.rocketmq.client.exception.MQBrokerException; ++-import org.apache.rocketmq.client.exception.MQClientException; ++ import org.apache.rocketmq.client.exception.OffsetNotFoundException; ++ import org.apache.rocketmq.client.impl.ClientRemotingProcessor; ++ import org.apache.rocketmq.client.impl.CommunicationMode; ++@@ -47,6 +46,7 @@ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +++import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.remoting.exception.RemotingCommandException; ++ import org.apache.rocketmq.remoting.netty.NettyClientConfig; ++@@ -106,19 +106,6 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ return false; ++ } ++ ++- protected static MQClientException processNullResponseErr(ResponseFuture responseFuture) { ++- MQClientException ex; ++- if (!responseFuture.isSendRequestOK()) { ++- ex = new MQClientException("send request failed", responseFuture.getCause()); ++- } else if (responseFuture.isTimeout()) { ++- ex = new MQClientException("wait response timeout " + responseFuture.getTimeoutMillis() + "ms", ++- responseFuture.getCause()); ++- } else { ++- ex = new MQClientException("unknown reason", responseFuture.getCause()); ++- } ++- return ex; ++- } ++- ++ public CompletableFuture sendHeartbeatOneway( ++ String brokerAddr, ++ HeartbeatData heartbeatData, ++@@ -146,24 +133,15 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ request.setLanguage(clientConfig.getLanguage()); ++ request.setBody(heartbeatData.encode()); ++ ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (ResponseCode.SUCCESS == response.getCode()) { ++- future.complete(response.getVersion()); ++- } else { ++- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); ++- } ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); ++- } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ if (ResponseCode.SUCCESS == response.getCode()) { +++ future0.complete(response.getVersion()); +++ } else { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark(), brokerAddr)); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture sendMessageAsync( ++@@ -177,24 +155,15 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE_V2, requestHeaderV2); ++ request.setBody(msg.getBody()); ++ ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- future.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); ++- } catch (Exception e) { ++- future.completeExceptionally(e); ++- } ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); ++- } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ try { +++ future0.complete(this.processSendResponse(brokerName, msg, response, brokerAddr)); +++ } catch (Exception e) { +++ future0.completeExceptionally(e); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture sendMessageAsync( ++@@ -216,17 +185,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ msgBatch.setBody(body); ++ ++ request.setBody(body); ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- try { ++- future.complete(this.processSendResponse(brokerName, msgBatch, response, brokerAddr)); ++- } catch (Exception e) { ++- future.completeExceptionally(e); ++- } ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ try { +++ future0.complete(processSendResponse(brokerName, msgBatch, response, brokerAddr)); +++ } catch (Exception e) { +++ future0.completeExceptionally(e); ++ } +++ return future0; ++ }); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++@@ -240,21 +206,7 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ long timeoutMillis ++ ) { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); ++- ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- future.complete(response); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); ++- } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis); ++ } ++ ++ public CompletableFuture popMessageAsync( ++@@ -402,38 +354,31 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ QueryConsumerOffsetRequestHeader requestHeader, ++ long timeoutMillis ++ ) { ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- switch (response.getCode()) { ++- case ResponseCode.SUCCESS: { ++- try { ++- QueryConsumerOffsetResponseHeader responseHeader = ++- (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); ++- future.complete(responseHeader.getOffset()); ++- } catch (RemotingCommandException e) { ++- future.completeExceptionally(e); ++- } ++- break; ++- } ++- case ResponseCode.QUERY_NOT_FOUND: { ++- future.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); ++- break; ++- } ++- default: ++- break; +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.QUERY_CONSUMER_OFFSET, requestHeader); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ switch (response.getCode()) { +++ case ResponseCode.SUCCESS: { +++ try { +++ QueryConsumerOffsetResponseHeader responseHeader = +++ (QueryConsumerOffsetResponseHeader) response.decodeCommandCustomHeader(QueryConsumerOffsetResponseHeader.class); +++ future0.complete(responseHeader.getOffset()); +++ } catch (RemotingCommandException e) { +++ future0.completeExceptionally(e); ++ } ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ break; ++ } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ case ResponseCode.QUERY_NOT_FOUND: { +++ future0.completeExceptionally(new OffsetNotFoundException(response.getCode(), response.getRemark(), brokerAddr)); +++ break; +++ } +++ default: { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ break; +++ } +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture updateConsumerOffsetOneWay( ++@@ -461,9 +406,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ ++ CompletableFuture> future = new CompletableFuture<>(); ++ try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { +++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ switch (response.getCode()) { ++ case ResponseCode.SUCCESS: { ++ if (response.getBody() != null) { ++@@ -485,8 +435,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ break; ++ } ++ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++ }); ++ } catch (Throwable t) { ++@@ -501,9 +454,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { +++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ if (ResponseCode.SUCCESS == response.getCode()) { ++ try { ++ GetMaxOffsetResponseHeader responseHeader = (GetMaxOffsetResponseHeader) response.decodeCommandCustomHeader(GetMaxOffsetResponseHeader.class); ++@@ -513,8 +471,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ } ++ } ++ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++ }); ++ } catch (Throwable t) { ++@@ -529,9 +490,14 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { +++ this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ if (ResponseCode.SUCCESS == response.getCode()) { ++ try { ++ GetMinOffsetResponseHeader responseHeader = (GetMinOffsetResponseHeader) response.decodeCommandCustomHeader(GetMinOffsetResponseHeader.class); ++@@ -541,8 +507,11 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ } ++ } ++ future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++ }); ++ } catch (Throwable t) { ++@@ -555,57 +524,41 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ long timeoutMillis) { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.SEARCH_OFFSET_BY_TIMESTAMP, requestHeader); ++ ++- CompletableFuture future = new CompletableFuture<>(); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- try { ++- SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); ++- future.complete(responseHeader.getOffset()); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- } ++- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ try { +++ SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.decodeCommandCustomHeader(SearchOffsetResponseHeader.class); +++ future0.complete(responseHeader.getOffset()); +++ } catch (Throwable t) { +++ future0.completeExceptionally(t); ++ } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ } else { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture> lockBatchMQWithFuture(String brokerAddr, ++ LockBatchRequestBody requestBody, long timeoutMillis) { ++- CompletableFuture> future = new CompletableFuture<>(); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.LOCK_BATCH_MQ, null); ++ request.setBody(requestBody.encode()); ++- try { ++- this.getRemotingClient().invokeAsync(brokerAddr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- try { ++- LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); ++- Set messageQueues = responseBody.getLockOKMQSet(); ++- future.complete(messageQueues); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- } ++- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); ++- } else { ++- future.completeExceptionally(processNullResponseErr(responseFuture)); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture> future0 = new CompletableFuture<>(); +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ try { +++ LockBatchResponseBody responseBody = LockBatchResponseBody.decode(response.getBody(), LockBatchResponseBody.class); +++ Set messageQueues = responseBody.getLockOKMQSet(); +++ future0.complete(messageQueues); +++ } catch (Throwable t) { +++ future0.completeExceptionally(t); ++ } ++- }); ++- } catch (Exception e) { ++- future.completeExceptionally(e); ++- } ++- return future; +++ } else { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture unlockBatchMQOneway(String brokerAddr, ++@@ -624,25 +577,21 @@ public class MQClientAPIExt extends MQClientAPIImpl { ++ ++ public CompletableFuture notification(String brokerAddr, NotificationRequestHeader requestHeader, ++ long timeoutMillis) { ++- CompletableFuture future = new CompletableFuture<>(); ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.NOTIFICATION, requestHeader); ++- try { ++- this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenAccept(response -> { ++- if (response.getCode() == ResponseCode.SUCCESS) { ++- try { ++- NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); ++- future.complete(responseHeader.isHasMsg()); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- } else { ++- future.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ return this.getRemotingClient().invoke(brokerAddr, request, timeoutMillis).thenCompose(response -> { +++ CompletableFuture future0 = new CompletableFuture<>(); +++ if (response.getCode() == ResponseCode.SUCCESS) { +++ try { +++ NotificationResponseHeader responseHeader = (NotificationResponseHeader) response.decodeCommandCustomHeader(NotificationResponseHeader.class); +++ future0.complete(responseHeader.isHasMsg()); +++ } catch (Throwable t) { +++ future0.completeExceptionally(t); ++ } ++- }); ++- } catch (Throwable t) { ++- future.completeExceptionally(t); ++- } ++- return future; +++ } else { +++ future0.completeExceptionally(new MQBrokerException(response.getCode(), response.getRemark())); +++ } +++ return future0; +++ }); ++ } ++ ++ public CompletableFuture invoke(String brokerAddr, RemotingCommand request, long timeoutMillis) { ++diff --git a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java ++index d13f2cfe4..c152d38ea 100644 ++--- a/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java +++++ b/client/src/test/java/org/apache/rocketmq/client/impl/MQClientAPIImplTest.java ++@@ -212,7 +212,7 @@ public class MQClientAPIImplTest { ++ RemotingCommand request = mock.getArgument(1); ++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++ responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++@@ -386,7 +386,7 @@ public class MQClientAPIImplTest { ++ RemotingCommand request = mock.getArgument(1); ++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++ responseFuture.setResponseCommand(createSendMessageSuccessResponse(request)); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(Matchers.anyString(), Matchers.any(RemotingCommand.class), Matchers.anyLong(), Matchers.any(InvokeCallback.class)); ++@@ -472,7 +472,7 @@ public class MQClientAPIImplTest { ++ message.putUserProperty("key", "value"); ++ response.setBody(MessageDecoder.encode(message, false)); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++@@ -543,7 +543,7 @@ public class MQClientAPIImplTest { ++ message.getProperties().put(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, String.valueOf(0)); ++ response.setBody(MessageDecoder.encode(message, false)); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++@@ -585,7 +585,7 @@ public class MQClientAPIImplTest { ++ response.setOpaque(request.getOpaque()); ++ response.setCode(ResponseCode.SUCCESS); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++@@ -622,7 +622,7 @@ public class MQClientAPIImplTest { ++ responseHeader.setPopTime(System.currentTimeMillis()); ++ responseHeader.setInvisibleTime(10 * 1000L); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ } ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++index fe07090d5..3227d1e1c 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/remoting/RemotingProtocolServer.java ++@@ -26,7 +26,6 @@ import java.util.concurrent.ScheduledExecutorService; ++ import java.util.concurrent.ThreadPoolExecutor; ++ import java.util.concurrent.TimeUnit; ++ import org.apache.rocketmq.acl.AccessValidator; ++-import org.apache.rocketmq.client.exception.MQClientException; ++ import org.apache.rocketmq.common.constant.LoggerName; ++ import org.apache.rocketmq.common.future.FutureTaskExt; ++ import org.apache.rocketmq.common.thread.ThreadPoolMonitor; ++@@ -51,10 +50,12 @@ import org.apache.rocketmq.proxy.remoting.channel.RemotingChannelManager; ++ import org.apache.rocketmq.proxy.remoting.pipeline.AuthenticationPipeline; ++ import org.apache.rocketmq.proxy.remoting.pipeline.RequestPipeline; ++ import org.apache.rocketmq.remoting.ChannelEventListener; +++import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.RemotingServer; ++ import org.apache.rocketmq.remoting.netty.NettyRemotingServer; ++ import org.apache.rocketmq.remoting.netty.NettyServerConfig; ++ import org.apache.rocketmq.remoting.netty.RequestTask; +++import org.apache.rocketmq.remoting.netty.ResponseFuture; ++ import org.apache.rocketmq.remoting.netty.TlsSystemConfig; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.RequestCode; ++@@ -239,12 +240,21 @@ public class RemotingProtocolServer implements StartAndShutdown, RemotingProxyOu ++ long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++- this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, responseFuture -> { ++- if (responseFuture.getResponseCommand() == null) { ++- future.completeExceptionally(new MQClientException("response is null after send request to client", responseFuture.getCause())); ++- return; +++ this.defaultRemotingServer.invokeAsync(channel, request, timeoutMillis, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ future.complete(response); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++- future.complete(responseFuture.getResponseCommand()); ++ }); ++ } catch (Throwable t) { ++ future.completeExceptionally(t); ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++index 3f3a4ae40..e2d05b0f5 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/mqclient/MQClientAPIExtTest.java ++@@ -24,6 +24,7 @@ import java.util.ArrayList; ++ import java.util.List; ++ import java.util.Set; ++ import java.util.UUID; +++import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ThreadLocalRandom; ++ import java.util.concurrent.atomic.AtomicReference; ++ import java.util.stream.Collectors; ++@@ -85,6 +86,7 @@ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.Mockito.doAnswer; +++import static org.mockito.Mockito.doReturn; ++ ++ @RunWith(MockitoJUnitRunner.class) ++ public class MQClientAPIExtTest { ++@@ -109,13 +111,9 @@ public class MQClientAPIExtTest { ++ ++ @Test ++ public void testSendHeartbeatAsync() throws Exception { ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ assertNotNull(mqClientAPI.sendHeartbeatAsync(BROKER_ADDR, new HeartbeatData(), TIMEOUT).get()); ++ } ++@@ -123,20 +121,16 @@ public class MQClientAPIExtTest { ++ @Test ++ public void testSendMessageAsync() throws Exception { ++ AtomicReference msgIdRef = new AtomicReference<>(); ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); ++- SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); ++- sendMessageResponseHeader.setMsgId(msgIdRef.get()); ++- sendMessageResponseHeader.setQueueId(0); ++- sendMessageResponseHeader.setQueueOffset(1L); ++- response.setCode(ResponseCode.SUCCESS); ++- response.makeCustomHeaderToNet(); ++- responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); +++ SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); +++ sendMessageResponseHeader.setMsgId(msgIdRef.get()); +++ sendMessageResponseHeader.setQueueId(0); +++ sendMessageResponseHeader.setQueueOffset(1L); +++ response.setCode(ResponseCode.SUCCESS); +++ response.makeCustomHeaderToNet(); +++ future.complete(response); +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ MessageExt messageExt = createMessage(); ++ msgIdRef.set(MessageClientIDSetter.getUniqID(messageExt)); ++@@ -150,20 +144,16 @@ public class MQClientAPIExtTest { ++ ++ @Test ++ public void testSendMessageListAsync() throws Exception { ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); ++- SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); ++- sendMessageResponseHeader.setMsgId(""); ++- sendMessageResponseHeader.setQueueId(0); ++- sendMessageResponseHeader.setQueueOffset(1L); ++- response.setCode(ResponseCode.SUCCESS); ++- response.makeCustomHeaderToNet(); ++- responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ RemotingCommand response = RemotingCommand.createResponseCommand(SendMessageResponseHeader.class); +++ SendMessageResponseHeader sendMessageResponseHeader = (SendMessageResponseHeader) response.readCustomHeader(); +++ sendMessageResponseHeader.setMsgId(""); +++ sendMessageResponseHeader.setQueueId(0); +++ sendMessageResponseHeader.setQueueOffset(1L); +++ response.setCode(ResponseCode.SUCCESS); +++ response.makeCustomHeaderToNet(); +++ future.complete(response); +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ List messageExtList = new ArrayList<>(); ++ StringBuilder sb = new StringBuilder(); ++@@ -182,13 +172,9 @@ public class MQClientAPIExtTest { ++ ++ @Test ++ public void testSendMessageBackAsync() throws Exception { ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- responseFuture.putResponse(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(RemotingCommand.createResponseCommand(ResponseCode.SUCCESS, "")); +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ RemotingCommand remotingCommand = mqClientAPI.sendMessageBackAsync(BROKER_ADDR, new ConsumerSendMsgBackRequestHeader(), TIMEOUT) ++ .get(); ++@@ -285,7 +271,7 @@ public class MQClientAPIExtTest { ++ body.setConsumerIdList(clientIds); ++ response.setBody(body.encode()); ++ responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); +++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ ++@@ -302,7 +288,7 @@ public class MQClientAPIExtTest { ++ response.setCode(ResponseCode.SYSTEM_ERROR); ++ response.makeCustomHeaderToNet(); ++ responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); +++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ ++@@ -322,7 +308,7 @@ public class MQClientAPIExtTest { ++ response.setCode(ResponseCode.SUCCESS); ++ response.makeCustomHeaderToNet(); ++ responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); +++ invokeCallback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); ++ ++@@ -335,18 +321,15 @@ public class MQClientAPIExtTest { ++ @Test ++ public void testSearchOffsetAsync() throws Exception { ++ long offset = ThreadLocalRandom.current().nextLong(); ++- doAnswer((Answer) mock -> { ++- InvokeCallback invokeCallback = mock.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, 0, 3000, invokeCallback, null); ++- RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); ++- SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); ++- responseHeader.setOffset(offset); ++- response.setCode(ResponseCode.SUCCESS); ++- response.makeCustomHeaderToNet(); ++- responseFuture.putResponse(response); ++- invokeCallback.operationComplete(responseFuture); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any()); +++ CompletableFuture future = new CompletableFuture<>(); +++ RemotingCommand response = RemotingCommand.createResponseCommand(SearchOffsetResponseHeader.class); +++ SearchOffsetResponseHeader responseHeader = (SearchOffsetResponseHeader) response.readCustomHeader(); +++ responseHeader.setOffset(offset); +++ response.setCode(ResponseCode.SUCCESS); +++ response.makeCustomHeaderToNet(); +++ future.complete(response); +++ +++ doReturn(future).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ SearchOffsetRequestHeader requestHeader = new SearchOffsetRequestHeader(); ++ requestHeader.setTopic(TOPIC); ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java ++index ce78fa923..6be491745 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/InvokeCallback.java ++@@ -17,7 +17,22 @@ ++ package org.apache.rocketmq.remoting; ++ ++ import org.apache.rocketmq.remoting.netty.ResponseFuture; +++import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ ++ public interface InvokeCallback { +++ /** +++ * This method is expected to be invoked after {@link #operationSucceed(RemotingCommand)} +++ * or {@link #operationFail(Throwable)} +++ * +++ * @param responseFuture the returned object contains response or exception +++ */ ++ void operationComplete(final ResponseFuture responseFuture); +++ +++ default void operationSucceed(final RemotingCommand response) { +++ +++ } +++ +++ default void operationFail(final Throwable throwable) { +++ +++ } ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java ++index ff0b3df95..c8389eedb 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/RemotingClient.java ++@@ -20,11 +20,11 @@ import java.util.List; ++ import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ExecutorService; ++ import org.apache.rocketmq.remoting.exception.RemotingConnectException; ++-import org.apache.rocketmq.remoting.exception.RemotingException; ++ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +++import org.apache.rocketmq.remoting.netty.ResponseFuture; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ ++ public interface RemotingClient extends RemotingService { ++@@ -51,18 +51,21 @@ public interface RemotingClient extends RemotingService { ++ final long timeoutMillis) { ++ CompletableFuture future = new CompletableFuture<>(); ++ try { ++- invokeAsync(addr, request, timeoutMillis, responseFuture -> { ++- RemotingCommand response = responseFuture.getResponseCommand(); ++- if (response != null) { +++ invokeAsync(addr, request, timeoutMillis, new InvokeCallback() { +++ +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ future.complete(response); ++- } else { ++- if (!responseFuture.isSendRequestOK()) { ++- future.completeExceptionally(new RemotingSendRequestException(addr, responseFuture.getCause())); ++- } else if (responseFuture.isTimeout()) { ++- future.completeExceptionally(new RemotingTimeoutException(addr, timeoutMillis, responseFuture.getCause())); ++- } else { ++- future.completeExceptionally(new RemotingException(request.toString(), responseFuture.getCause())); ++- } +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); ++ } ++ }); ++ } catch (Throwable t) { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++index fce2de267..12e66f913 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++@@ -23,20 +23,23 @@ import io.netty.handler.ssl.SslContext; ++ import io.netty.handler.ssl.SslHandler; ++ import io.netty.util.concurrent.Future; ++ import io.opentelemetry.api.common.AttributesBuilder; ++-import java.net.SocketAddress; ++ import java.util.ArrayList; ++ import java.util.HashMap; ++ import java.util.Iterator; ++ import java.util.LinkedList; ++ import java.util.List; ++ import java.util.Map.Entry; +++import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; +++import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.LinkedBlockingQueue; ++ import java.util.concurrent.RejectedExecutionException; ++ import java.util.concurrent.Semaphore; ++ import java.util.concurrent.TimeUnit; +++import java.util.concurrent.TimeoutException; +++import java.util.concurrent.atomic.AtomicReference; ++ import java.util.function.Consumer; ++ import javax.annotation.Nullable; ++ import org.apache.rocketmq.common.AbortProcessException; ++@@ -125,7 +128,7 @@ public abstract class NettyRemotingAbstract { ++ * Constructor, specifying capacity of one-way and asynchronous semaphores. ++ * ++ * @param permitsOneway Number of permits for one-way requests. ++- * @param permitsAsync Number of permits for asynchronous requests. +++ * @param permitsAsync Number of permits for asynchronous requests. ++ */ ++ public NettyRemotingAbstract(final int permitsOneway, final int permitsAsync) { ++ this.semaphoreOneway = new Semaphore(permitsOneway, true); ++@@ -367,8 +370,7 @@ public abstract class NettyRemotingAbstract { ++ responseFuture.release(); ++ } ++ } else { ++- log.warn("receive response, but not matched any request, " + RemotingHelper.parseChannelRemoteAddr(ctx.channel())); ++- log.warn(cmd.toString()); +++ log.warn("receive response, cmd={}, but not matched any request, address={}", cmd, RemotingHelper.parseChannelRemoteAddr(ctx.channel())); ++ } ++ } ++ ++@@ -467,57 +469,68 @@ public abstract class NettyRemotingAbstract { ++ public RemotingCommand invokeSyncImpl(final Channel channel, final RemotingCommand request, ++ final long timeoutMillis) ++ throws InterruptedException, RemotingSendRequestException, RemotingTimeoutException { ++- //get the request id ++- final int opaque = request.getOpaque(); ++- ++ try { ++- final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis, null, null); ++- this.responseTable.put(opaque, responseFuture); ++- final SocketAddress addr = channel.remoteAddress(); ++- channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { ++- if (f.isSuccess()) { ++- responseFuture.setSendRequestOK(true); ++- return; ++- } ++- ++- responseFuture.setSendRequestOK(false); ++- responseTable.remove(opaque); ++- responseFuture.setCause(f.cause()); ++- responseFuture.putResponse(null); ++- log.warn("Failed to write a request command to {}, caused by underlying I/O operation failure", addr); ++- }); +++ return invokeImpl(channel, request, timeoutMillis).thenApply(ResponseFuture::getResponseCommand) +++ .get(timeoutMillis, TimeUnit.MILLISECONDS); +++ } catch (ExecutionException e) { +++ throw new RemotingSendRequestException(channel.remoteAddress().toString(), e.getCause()); +++ } catch (TimeoutException e) { +++ throw new RemotingTimeoutException(channel.remoteAddress().toString(), timeoutMillis, e.getCause()); +++ } +++ } ++ ++- RemotingCommand responseCommand = responseFuture.waitResponse(timeoutMillis); ++- if (null == responseCommand) { ++- if (responseFuture.isSendRequestOK()) { ++- throw new RemotingTimeoutException(RemotingHelper.parseSocketAddressAddr(addr), timeoutMillis, ++- responseFuture.getCause()); ++- } else { ++- throw new RemotingSendRequestException(RemotingHelper.parseSocketAddressAddr(addr), responseFuture.getCause()); ++- } +++ public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand request, +++ final long timeoutMillis) { +++ String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); +++ doBeforeRpcHooks(channelRemoteAddr, request); +++ return invoke0(channel, request, timeoutMillis).whenComplete((v, t) -> { +++ if (t == null) { +++ doAfterRpcHooks(channelRemoteAddr, request, v.getResponseCommand()); ++ } ++- ++- return responseCommand; ++- } finally { ++- this.responseTable.remove(opaque); ++- } +++ }); ++ } ++ ++- public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, ++- final InvokeCallback invokeCallback) ++- throws InterruptedException, RemotingTooMuchRequestException, RemotingTimeoutException, RemotingSendRequestException { +++ protected CompletableFuture invoke0(final Channel channel, final RemotingCommand request, +++ final long timeoutMillis) { +++ CompletableFuture future = new CompletableFuture<>(); ++ long beginStartTime = System.currentTimeMillis(); ++ final int opaque = request.getOpaque(); ++- boolean acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); +++ +++ boolean acquired; +++ try { +++ acquired = this.semaphoreAsync.tryAcquire(timeoutMillis, TimeUnit.MILLISECONDS); +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ return future; +++ } ++ if (acquired) { ++ final SemaphoreReleaseOnlyOnce once = new SemaphoreReleaseOnlyOnce(this.semaphoreAsync); ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ if (timeoutMillis < costTime) { ++ once.release(); ++- throw new RemotingTimeoutException("invokeAsyncImpl call timeout"); +++ future.completeExceptionally(new RemotingTimeoutException("invokeAsyncImpl call timeout")); +++ return future; ++ } ++ ++- final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, timeoutMillis - costTime, invokeCallback, once); +++ AtomicReference responseFutureReference = new AtomicReference<>(); +++ final ResponseFuture responseFuture = new ResponseFuture(channel, opaque, request, timeoutMillis - costTime, +++ new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ future.complete(responseFutureReference.get()); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ future.completeExceptionally(throwable); +++ } +++ }, once); +++ responseFutureReference.set(responseFuture); ++ this.responseTable.put(opaque, responseFuture); ++ try { ++ channel.writeAndFlush(request).addListener((ChannelFutureListener) f -> { ++@@ -528,15 +541,17 @@ public abstract class NettyRemotingAbstract { ++ requestFail(opaque); ++ log.warn("send a request command to channel <{}> failed.", RemotingHelper.parseChannelRemoteAddr(channel)); ++ }); +++ return future; ++ } catch (Exception e) { ++ responseTable.remove(opaque); ++ responseFuture.release(); ++ log.warn("send a request command to channel <" + RemotingHelper.parseChannelRemoteAddr(channel) + "> Exception", e); ++- throw new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e); +++ future.completeExceptionally(new RemotingSendRequestException(RemotingHelper.parseChannelRemoteAddr(channel), e)); +++ return future; ++ } ++ } else { ++ if (timeoutMillis <= 0) { ++- throw new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast"); +++ future.completeExceptionally(new RemotingTooMuchRequestException("invokeAsyncImpl invoke too fast")); ++ } else { ++ String info = ++ String.format("invokeAsyncImpl tryAcquire semaphore timeout, %dms, waiting thread nums: %d semaphoreAsyncValue: %d", ++@@ -545,11 +560,31 @@ public abstract class NettyRemotingAbstract { ++ this.semaphoreAsync.availablePermits() ++ ); ++ log.warn(info); ++- throw new RemotingTimeoutException(info); +++ future.completeExceptionally(new RemotingTimeoutException(info)); ++ } +++ return future; ++ } ++ } ++ +++ public void invokeAsyncImpl(final Channel channel, final RemotingCommand request, final long timeoutMillis, +++ final InvokeCallback invokeCallback) { +++ invokeImpl(channel, request, timeoutMillis) +++ .whenComplete((v, t) -> { +++ if (t == null) { +++ invokeCallback.operationComplete(v); +++ } else { +++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, timeoutMillis, null, null); +++ responseFuture.setCause(t); +++ invokeCallback.operationComplete(responseFuture); +++ } +++ }) +++ .thenAccept(responseFuture -> invokeCallback.operationSucceed(responseFuture.getResponseCommand())) +++ .exceptionally(t -> { +++ invokeCallback.operationFail(t); +++ return null; +++ }); +++ } +++ ++ private void requestFail(final int opaque) { ++ ResponseFuture responseFuture = responseTable.remove(opaque); ++ if (responseFuture != null) { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++index 64621dd6c..d784351a5 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++@@ -527,15 +527,13 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ if (channel != null && channel.isActive()) { ++ long left = timeoutMillis; ++ try { ++- doBeforeRpcHooks(channelRemoteAddr, request); ++ long costTime = System.currentTimeMillis() - beginStartTime; ++ left -= costTime; ++ if (left <= 0) { ++ throw new RemotingTimeoutException("invokeSync call the addr[" + channelRemoteAddr + "] timeout"); ++ } ++ RemotingCommand response = this.invokeSyncImpl(channel, request, left); ++- doAfterRpcHooks(channelRemoteAddr, request, response); ++- this.updateChannelLastResponseTime(addr); +++ updateChannelLastResponseTime(addr); ++ return response; ++ } catch (RemotingSendRequestException e) { ++ LOGGER.warn("invokeSync: send request exception, so close the channel[{}]", channelRemoteAddr); ++@@ -727,18 +725,11 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ final Channel channel = this.getAndCreateChannel(addr); ++ String channelRemoteAddr = RemotingHelper.parseChannelRemoteAddr(channel); ++ if (channel != null && channel.isActive()) { ++- try { ++- doBeforeRpcHooks(channelRemoteAddr, request); ++- long costTime = System.currentTimeMillis() - beginStartTime; ++- if (timeoutMillis < costTime) { ++- throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); ++- } ++- this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); ++- } catch (RemotingSendRequestException e) { ++- LOGGER.warn("invokeAsync: send request exception, so close the channel[{}]", channelRemoteAddr); ++- this.closeChannel(addr, channel); ++- throw e; +++ long costTime = System.currentTimeMillis() - beginStartTime; +++ if (timeoutMillis < costTime) { +++ throw new RemotingTooMuchRequestException("invokeAsync call the addr[" + channelRemoteAddr + "] timeout"); ++ } +++ this.invokeAsyncImpl(channel, request, timeoutMillis - costTime, new InvokeCallbackWrapper(invokeCallback, addr)); ++ } else { ++ this.closeChannel(addr, channel); ++ throw new RemotingConnectException(addr); ++@@ -931,11 +922,19 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++- if (responseFuture != null && responseFuture.isSendRequestOK() && responseFuture.getResponseCommand() != null) { ++- NettyRemotingClient.this.updateChannelLastResponseTime(addr); ++- } ++ this.invokeCallback.operationComplete(responseFuture); ++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ updateChannelLastResponseTime(addr); +++ this.invokeCallback.operationSucceed(response); +++ } +++ +++ @Override +++ public void operationFail(final Throwable throwable) { +++ this.invokeCallback.operationFail(throwable); +++ } ++ } ++ ++ class NettyClientHandler extends SimpleChannelInboundHandler { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java ++index 19f705d74..0882818fe 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/ResponseFuture.java ++@@ -22,6 +22,9 @@ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.atomic.AtomicBoolean; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; +++import org.apache.rocketmq.remoting.exception.RemotingException; +++import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; +++import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ ++ public class ResponseFuture { ++@@ -59,6 +62,18 @@ public class ResponseFuture { ++ public void executeInvokeCallback() { ++ if (invokeCallback != null) { ++ if (this.executeCallbackOnlyOnce.compareAndSet(false, true)) { +++ RemotingCommand response = getResponseCommand(); +++ if (response != null) { +++ invokeCallback.operationSucceed(response); +++ } else { +++ if (!isSendRequestOK()) { +++ invokeCallback.operationFail(new RemotingSendRequestException(channel.remoteAddress().toString(), getCause())); +++ } else if (isTimeout()) { +++ invokeCallback.operationFail(new RemotingTimeoutException(channel.remoteAddress().toString(), getTimeoutMillis(), getCause())); +++ } else { +++ invokeCallback.operationFail(new RemotingException(getRequestCommand().toString(), getCause())); +++ } +++ } ++ invokeCallback.operationComplete(this); ++ } ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java ++index 133e0ed31..5328e8845 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/rpc/RpcClientImpl.java ++@@ -160,31 +160,38 @@ public class RpcClientImpl implements RpcClient { ++ InvokeCallback callback = new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { ++- RemotingCommand responseCommand = responseFuture.getResponseCommand(); ++- if (responseCommand == null) { ++- processFailedResponse(addr, requestCommand, responseFuture, rpcResponsePromise); ++- return; ++- } +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ try { ++- switch (responseCommand.getCode()) { +++ switch (response.getCode()) { ++ case ResponseCode.SUCCESS: ++ case ResponseCode.PULL_NOT_FOUND: ++ case ResponseCode.PULL_RETRY_IMMEDIATELY: ++ case ResponseCode.PULL_OFFSET_MOVED: ++ PullMessageResponseHeader responseHeader = ++- (PullMessageResponseHeader) responseCommand.decodeCommandCustomHeader(PullMessageResponseHeader.class); ++- rpcResponsePromise.setSuccess(new RpcResponse(responseCommand.getCode(), responseHeader, responseCommand.getBody())); +++ (PullMessageResponseHeader) response.decodeCommandCustomHeader(PullMessageResponseHeader.class); +++ rpcResponsePromise.setSuccess(new RpcResponse(response.getCode(), responseHeader, response.getBody())); ++ default: ++- RpcResponse rpcResponse = new RpcResponse(new RpcException(responseCommand.getCode(), "unexpected remote response code")); +++ RpcResponse rpcResponse = new RpcResponse(new RpcException(response.getCode(), "unexpected remote response code")); ++ rpcResponsePromise.setSuccess(rpcResponse); ++ ++ } ++ } catch (Exception e) { ++- String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + responseFuture.getTimeoutMillis() + ". Request: " + requestCommand; ++- RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); +++ String errorMessage = "process failed. addr: " + addr + ", timeoutMillis: " + timeoutMillis + ". Request: " + requestCommand; +++ RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, e)); ++ rpcResponsePromise.setSuccess(rpcResponse); ++ } ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ String errorMessage = "process failed. addr: " + addr + ". Request: " + requestCommand; +++ RpcResponse rpcResponse = new RpcResponse(new RpcException(ResponseCode.RPC_UNKNOWN, errorMessage, throwable)); +++ rpcResponsePromise.setSuccess(rpcResponse); +++ } ++ }; ++ ++ this.remotingClient.invokeAsync(addr, requestCommand, timeoutMillis, callback); ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java ++index 90072960b..d0da0eb2e 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/RemotingServerTest.java ++@@ -26,12 +26,12 @@ import org.apache.rocketmq.remoting.exception.RemotingConnectException; ++ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++-import org.apache.rocketmq.remoting.netty.ResponseFuture; ++-import org.apache.rocketmq.remoting.netty.NettyServerConfig; ++ import org.apache.rocketmq.remoting.netty.NettyClientConfig; ++-import org.apache.rocketmq.remoting.netty.NettyRemotingServer; ++ import org.apache.rocketmq.remoting.netty.NettyRemotingClient; +++import org.apache.rocketmq.remoting.netty.NettyRemotingServer; ++ import org.apache.rocketmq.remoting.netty.NettyRequestProcessor; +++import org.apache.rocketmq.remoting.netty.NettyServerConfig; +++import org.apache.rocketmq.remoting.netty.ResponseFuture; ++ import org.apache.rocketmq.remoting.protocol.LanguageCode; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.junit.AfterClass; ++@@ -40,7 +40,6 @@ import org.junit.Test; ++ ++ import static org.assertj.core.api.Assertions.assertThat; ++ import static org.junit.Assert.assertNotNull; ++-import static org.junit.Assert.assertTrue; ++ ++ public class RemotingServerTest { ++ private static RemotingServer remotingServer; ++@@ -122,10 +121,19 @@ public class RemotingServerTest { ++ remotingClient.invokeAsync("localhost:" + remotingServer.localListenPort(), request, 1000 * 3, new InvokeCallback() { ++ @Override ++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ latch.countDown(); ++- assertTrue(responseFuture != null); ++- assertThat(responseFuture.getResponseCommand().getLanguage()).isEqualTo(LanguageCode.JAVA); ++- assertThat(responseFuture.getResponseCommand().getExtFields()).hasSize(2); +++ assertThat(response.getLanguage()).isEqualTo(LanguageCode.JAVA); +++ assertThat(response.getExtFields()).hasSize(2); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ ++ } ++ }); ++ latch.await(); ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java ++similarity index 57% ++rename from client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java ++rename to remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java ++index 80188832e..8ddcdf35d 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/BaseInvokeCallback.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannel.java ++@@ -15,23 +15,14 @@ ++ * limitations under the License. ++ */ ++ ++-package org.apache.rocketmq.client.impl; +++package org.apache.rocketmq.remoting.netty; ++ ++-import org.apache.rocketmq.remoting.InvokeCallback; ++-import org.apache.rocketmq.remoting.netty.ResponseFuture; ++- ++-public abstract class BaseInvokeCallback implements InvokeCallback { ++- private final MQClientAPIImpl mqClientAPI; ++- ++- public BaseInvokeCallback(MQClientAPIImpl mqClientAPI) { ++- this.mqClientAPI = mqClientAPI; ++- } +++import io.netty.channel.ChannelFuture; +++import io.netty.channel.local.LocalChannel; ++ +++public class MockChannel extends LocalChannel { ++ @Override ++- public void operationComplete(final ResponseFuture responseFuture) { ++- mqClientAPI.execRpcHooksAfterRequest(responseFuture); ++- onComplete(responseFuture); +++ public ChannelFuture writeAndFlush(Object msg) { +++ return new MockChannelPromise(MockChannel.this); ++ } ++- ++- public abstract void onComplete(final ResponseFuture responseFuture); ++ } ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java ++new file mode 100644 ++index 000000000..9c3a35487 ++--- /dev/null +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/MockChannelPromise.java ++@@ -0,0 +1,191 @@ +++/* +++ * Licensed to the Apache Software Foundation (ASF) under one or more +++ * contributor license agreements. See the NOTICE file distributed with +++ * this work for additional information regarding copyright ownership. +++ * The ASF licenses this file to You under the Apache License, Version 2.0 +++ * (the "License"); you may not use this file except in compliance with +++ * the License. You may obtain a copy of the License at +++ * +++ * http://www.apache.org/licenses/LICENSE-2.0 +++ * +++ * Unless required by applicable law or agreed to in writing, software +++ * distributed under the License is distributed on an "AS IS" BASIS, +++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +++ * See the License for the specific language governing permissions and +++ * limitations under the License. +++ */ +++ +++package org.apache.rocketmq.remoting.netty; +++ +++import io.netty.channel.Channel; +++import io.netty.channel.ChannelPromise; +++import io.netty.util.concurrent.Future; +++import io.netty.util.concurrent.GenericFutureListener; +++import java.util.concurrent.ExecutionException; +++import java.util.concurrent.TimeUnit; +++import java.util.concurrent.TimeoutException; +++import org.jetbrains.annotations.NotNull; +++ +++public class MockChannelPromise implements ChannelPromise { +++ protected Channel channel; +++ +++ public MockChannelPromise(Channel channel) { +++ this.channel = channel; +++ } +++ +++ @Override +++ public Channel channel() { +++ return channel; +++ } +++ +++ @Override +++ public ChannelPromise setSuccess(Void result) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise setSuccess() { +++ return this; +++ } +++ +++ @Override +++ public boolean trySuccess() { +++ return false; +++ } +++ +++ @Override +++ public ChannelPromise setFailure(Throwable cause) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise addListener(GenericFutureListener> listener) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise addListeners(GenericFutureListener>... listeners) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise removeListener(GenericFutureListener> listener) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise removeListeners(GenericFutureListener>... listeners) { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise sync() throws InterruptedException { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise syncUninterruptibly() { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise await() throws InterruptedException { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise awaitUninterruptibly() { +++ return this; +++ } +++ +++ @Override +++ public ChannelPromise unvoid() { +++ return this; +++ } +++ +++ @Override +++ public boolean isVoid() { +++ return false; +++ } +++ +++ @Override +++ public boolean trySuccess(Void result) { +++ return false; +++ } +++ +++ @Override +++ public boolean tryFailure(Throwable cause) { +++ return false; +++ } +++ +++ @Override +++ public boolean setUncancellable() { +++ return false; +++ } +++ +++ @Override +++ public boolean isSuccess() { +++ return false; +++ } +++ +++ @Override +++ public boolean isCancellable() { +++ return false; +++ } +++ +++ @Override +++ public Throwable cause() { +++ return null; +++ } +++ +++ @Override +++ public boolean await(long timeout, TimeUnit unit) throws InterruptedException { +++ return false; +++ } +++ +++ @Override +++ public boolean await(long timeoutMillis) throws InterruptedException { +++ return false; +++ } +++ +++ @Override +++ public boolean awaitUninterruptibly(long timeout, TimeUnit unit) { +++ return false; +++ } +++ +++ @Override +++ public boolean awaitUninterruptibly(long timeoutMillis) { +++ return false; +++ } +++ +++ @Override +++ public Void getNow() { +++ return null; +++ } +++ +++ @Override +++ public boolean cancel(boolean mayInterruptIfRunning) { +++ return false; +++ } +++ +++ @Override +++ public boolean isCancelled() { +++ return false; +++ } +++ +++ @Override +++ public boolean isDone() { +++ return false; +++ } +++ +++ @Override +++ public Void get() throws InterruptedException, ExecutionException { +++ return null; +++ } +++ +++ @Override +++ public Void get(long timeout, +++ @NotNull java.util.concurrent.TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { +++ return null; +++ } +++} ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java ++index 8381c132b..dbbea86ea 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstractTest.java ++@@ -39,9 +39,19 @@ public class NettyRemotingAbstractTest { ++ final Semaphore semaphore = new Semaphore(0); ++ ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { ++ @Override ++- public void operationComplete(final ResponseFuture responseFuture) { +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ assertThat(semaphore.availablePermits()).isEqualTo(0); ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ +++ } ++ }, new SemaphoreReleaseOnlyOnce(semaphore)); ++ ++ remotingAbstract.responseTable.putIfAbsent(1, responseFuture); ++@@ -75,9 +85,19 @@ public class NettyRemotingAbstractTest { ++ final Semaphore semaphore = new Semaphore(0); ++ ResponseFuture responseFuture = new ResponseFuture(null, 1, 3000, new InvokeCallback() { ++ @Override ++- public void operationComplete(final ResponseFuture responseFuture) { +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { ++ assertThat(semaphore.availablePermits()).isEqualTo(0); ++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ +++ } ++ }, new SemaphoreReleaseOnlyOnce(semaphore)); ++ ++ remotingAbstract.responseTable.putIfAbsent(1, responseFuture); ++@@ -98,7 +118,18 @@ public class NettyRemotingAbstractTest { ++ // mock timeout ++ ResponseFuture responseFuture = new ResponseFuture(null, dummyId, -1000, new InvokeCallback() { ++ @Override ++- public void operationComplete(final ResponseFuture responseFuture) { +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ ++ } ++ }, null); ++ remotingAbstract.responseTable.putIfAbsent(dummyId, responseFuture); ++@@ -111,7 +142,22 @@ public class NettyRemotingAbstractTest { ++ final Semaphore semaphore = new Semaphore(0); ++ RemotingCommand request = RemotingCommand.createRequestCommand(1, null); ++ ResponseFuture responseFuture = new ResponseFuture(null, 1, request, 3000, ++- responseFuture1 -> assertThat(semaphore.availablePermits()).isEqualTo(0), new SemaphoreReleaseOnlyOnce(semaphore)); +++ new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ +++ @Override +++ public void operationSucceed(RemotingCommand response) { +++ assertThat(semaphore.availablePermits()).isEqualTo(0); +++ } +++ +++ @Override +++ public void operationFail(Throwable throwable) { +++ +++ } +++ }, new SemaphoreReleaseOnlyOnce(semaphore)); ++ ++ remotingAbstract.responseTable.putIfAbsent(1, responseFuture); ++ RemotingCommand response = RemotingCommand.createResponseCommand(0, "Foo"); ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++index 8fabbb21d..e72e7bd53 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++@@ -16,10 +16,17 @@ ++ */ ++ package org.apache.rocketmq.remoting.netty; ++ +++import io.netty.channel.Channel; +++import io.netty.channel.ChannelFuture; +++import io.netty.channel.local.LocalChannel; ++ import java.util.concurrent.CompletableFuture; +++import java.util.concurrent.ExecutionException; ++ import java.util.concurrent.ExecutorService; ++ import java.util.concurrent.Executors; +++import java.util.concurrent.Semaphore; ++ import org.apache.rocketmq.remoting.InvokeCallback; +++import org.apache.rocketmq.remoting.RPCHook; +++import org.apache.rocketmq.remoting.common.SemaphoreReleaseOnlyOnce; ++ import org.apache.rocketmq.remoting.exception.RemotingConnectException; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++@@ -29,23 +36,33 @@ import org.apache.rocketmq.remoting.protocol.RequestCode; ++ import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.junit.Test; ++ import org.junit.runner.RunWith; +++import org.mockito.Mock; ++ import org.mockito.Spy; ++ import org.mockito.junit.MockitoJUnitRunner; ++ ++ import static org.assertj.core.api.Assertions.assertThat; +++import static org.assertj.core.api.Assertions.assertThatThrownBy; ++ import static org.assertj.core.api.AssertionsForClassTypes.catchThrowable; ++ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; +++import static org.mockito.ArgumentMatchers.eq; ++ import static org.mockito.Mockito.doAnswer; +++import static org.mockito.Mockito.doReturn; +++import static org.mockito.Mockito.mock; +++import static org.mockito.Mockito.never; +++import static org.mockito.Mockito.times; +++import static org.mockito.Mockito.verify; ++ ++ @RunWith(MockitoJUnitRunner.class) ++ public class NettyRemotingClientTest { ++ @Spy ++ private NettyRemotingClient remotingClient = new NettyRemotingClient(new NettyClientConfig()); +++ @Mock +++ private RPCHook rpcHookMock; ++ ++ @Test ++- public void testSetCallbackExecutor() throws NoSuchFieldException, IllegalAccessException { +++ public void testSetCallbackExecutor() { ++ ExecutorService customized = Executors.newCachedThreadPool(); ++ remotingClient.setCallbackExecutor(customized); ++ assertThat(remotingClient.getCallbackExecutor()).isEqualTo(customized); ++@@ -61,7 +78,7 @@ public class NettyRemotingClientTest { ++ InvokeCallback callback = invocation.getArgument(3); ++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++ responseFuture.setResponseCommand(response); ++- callback.operationComplete(responseFuture); +++ callback.operationSucceed(responseFuture.getResponseCommand()); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ++@@ -78,9 +95,7 @@ public class NettyRemotingClientTest { ++ response.setCode(ResponseCode.SUCCESS); ++ doAnswer(invocation -> { ++ InvokeCallback callback = invocation.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++- responseFuture.setSendRequestOK(false); ++- callback.operationComplete(responseFuture); +++ callback.operationFail(new RemotingSendRequestException(null)); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ++@@ -97,8 +112,7 @@ public class NettyRemotingClientTest { ++ response.setCode(ResponseCode.SUCCESS); ++ doAnswer(invocation -> { ++ InvokeCallback callback = invocation.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), -1L, null, null); ++- callback.operationComplete(responseFuture); +++ callback.operationFail(new RemotingTimeoutException("")); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ++@@ -115,8 +129,7 @@ public class NettyRemotingClientTest { ++ response.setCode(ResponseCode.SUCCESS); ++ doAnswer(invocation -> { ++ InvokeCallback callback = invocation.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++- callback.operationComplete(responseFuture); +++ callback.operationFail(new RemotingException(null)); ++ return null; ++ }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); ++ ++@@ -134,4 +147,158 @@ public class NettyRemotingClientTest { ++ assertThat(e.getMessage()).contains(addr); ++ } ++ } +++ +++ @Test +++ public void testInvoke0() throws ExecutionException, InterruptedException { +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ Channel channel = new MockChannel() { +++ @Override +++ public ChannelFuture writeAndFlush(Object msg) { +++ ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); +++ responseFuture.setResponseCommand(response); +++ responseFuture.executeInvokeCallback(); +++ return super.writeAndFlush(msg); +++ } +++ }; +++ CompletableFuture future = remotingClient.invoke0(channel, request, 1000L); +++ assertThat(future.get().getResponseCommand()).isEqualTo(response); +++ } +++ +++ @Test +++ public void testInvoke0WithException() { +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ Channel channel = new MockChannel() { +++ @Override +++ public ChannelFuture writeAndFlush(Object msg) { +++ ResponseFuture responseFuture = remotingClient.responseTable.get(request.getOpaque()); +++ responseFuture.executeInvokeCallback(); +++ return super.writeAndFlush(msg); +++ } +++ }; +++ CompletableFuture future = remotingClient.invoke0(channel, request, 1000L); +++ assertThatThrownBy(future::get).getCause().isInstanceOf(RemotingException.class); +++ } +++ +++ @Test +++ public void testInvokeSync() throws RemotingSendRequestException, RemotingTimeoutException, InterruptedException { +++ remotingClient.registerRPCHook(rpcHookMock); +++ +++ Channel channel = new LocalChannel(); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); +++ responseFuture.setResponseCommand(response); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(responseFuture); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ RemotingCommand actual = remotingClient.invokeSyncImpl(channel, request, 1000); +++ assertThat(actual).isEqualTo(response); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); +++ } +++ +++ @Test +++ public void testInvokeAsync() { +++ remotingClient.registerRPCHook(rpcHookMock); +++ Channel channel = new LocalChannel(); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); +++ responseFuture.setResponseCommand(response); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(responseFuture); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ +++ InvokeCallback callback = mock(InvokeCallback.class); +++ remotingClient.invokeAsyncImpl(channel, request, 1000, callback); +++ verify(callback, times(1)).operationSucceed(eq(response)); +++ verify(callback, times(1)).operationComplete(eq(responseFuture)); +++ verify(callback, never()).operationFail(any()); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); +++ } +++ +++ @Test +++ public void testInvokeAsyncFail() { +++ remotingClient.registerRPCHook(rpcHookMock); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ +++ Channel channel = new LocalChannel(); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.completeExceptionally(new RemotingException(null)); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ +++ InvokeCallback callback = mock(InvokeCallback.class); +++ remotingClient.invokeAsyncImpl(channel, request, 1000, callback); +++ verify(callback, never()).operationSucceed(any()); +++ verify(callback, times(1)).operationComplete(any()); +++ verify(callback, times(1)).operationFail(any()); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); +++ } +++ +++ @Test +++ public void testInvokeImpl() throws ExecutionException, InterruptedException { +++ remotingClient.registerRPCHook(rpcHookMock); +++ Channel channel = new LocalChannel(); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ RemotingCommand response = RemotingCommand.createResponseCommand(null); +++ response.setCode(ResponseCode.SUCCESS); +++ ResponseFuture responseFuture = new ResponseFuture(channel, request.getOpaque(), request, 1000, new InvokeCallback() { +++ @Override +++ public void operationComplete(ResponseFuture responseFuture) { +++ +++ } +++ }, new SemaphoreReleaseOnlyOnce(new Semaphore(1))); +++ responseFuture.setResponseCommand(response); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.complete(responseFuture); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ +++ CompletableFuture future0 = remotingClient.invokeImpl(channel, request, 1000); +++ assertThat(future0.get()).isEqualTo(responseFuture); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock).doAfterResponse(anyString(), eq(request), eq(response)); +++ } +++ +++ @Test +++ public void testInvokeImplFail() { +++ remotingClient.registerRPCHook(rpcHookMock); +++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); +++ +++ Channel channel = new LocalChannel(); +++ CompletableFuture future = new CompletableFuture<>(); +++ future.completeExceptionally(new RemotingException(null)); +++ +++ doReturn(future).when(remotingClient).invoke0(any(Channel.class), any(RemotingCommand.class), anyLong()); +++ +++ assertThatThrownBy(() -> remotingClient.invokeImpl(channel, request, 1000).get()).getCause().isInstanceOf(RemotingException.class); +++ +++ verify(rpcHookMock).doBeforeRequest(anyString(), eq(request)); +++ verify(rpcHookMock, never()).doAfterResponse(anyString(), eq(request), any()); +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From b9ffe0f9576f68b8a37cf3e2f68051658ae5a9a2 Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Sun, 8 Oct 2023 16:33:44 +0800 ++Subject: [PATCH 2/7] [ISSUE #7296] Add ChannelEventListener for ++ MQClientAPIImpl (#7324) ++ ++* Add ChannelEventListener for MQClientAPIImpl ++ ++* add heartbeat when channel connect ++ ++* remove log ++ ++* Add enableHeartbeatChannelEventListener for ClientConfig ++--- ++ .../apache/rocketmq/client/ClientConfig.java | 55 ++++++++++++++----- ++ .../rocketmq/client/impl/MQClientAPIImpl.java | 9 ++- ++ .../client/impl/factory/MQClientInstance.java | 35 +++++++++++- ++ .../remoting/netty/NettyRemotingClient.java | 2 + ++ 4 files changed, 85 insertions(+), 16 deletions(-) ++ ++diff --git a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++index bb0fe3522..f9843cc02 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java +++++ b/client/src/main/java/org/apache/rocketmq/client/ClientConfig.java ++@@ -94,6 +94,8 @@ public class ClientConfig { ++ private boolean sendLatencyEnable = Boolean.parseBoolean(System.getProperty(SEND_LATENCY_ENABLE, "false")); ++ private boolean startDetectorEnable = Boolean.parseBoolean(System.getProperty(START_DETECTOR_ENABLE, "false")); ++ +++ private boolean enableHeartbeatChannelEventListener = true; +++ ++ public String buildMQClientId() { ++ StringBuilder sb = new StringBuilder(); ++ sb.append(this.getClientIP()); ++@@ -201,6 +203,7 @@ public class ClientConfig { ++ this.useHeartbeatV2 = cc.useHeartbeatV2; ++ this.startDetectorEnable = cc.startDetectorEnable; ++ this.sendLatencyEnable = cc.sendLatencyEnable; +++ this.enableHeartbeatChannelEventListener = cc.enableHeartbeatChannelEventListener; ++ this.detectInterval = cc.detectInterval; ++ this.detectTimeout = cc.detectTimeout; ++ } ++@@ -228,6 +231,7 @@ public class ClientConfig { ++ cc.enableStreamRequestType = enableStreamRequestType; ++ cc.useHeartbeatV2 = useHeartbeatV2; ++ cc.startDetectorEnable = startDetectorEnable; +++ cc.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; ++ cc.sendLatencyEnable = sendLatencyEnable; ++ cc.detectInterval = detectInterval; ++ cc.detectTimeout = detectTimeout; ++@@ -418,6 +422,14 @@ public class ClientConfig { ++ this.startDetectorEnable = startDetectorEnable; ++ } ++ +++ public boolean isEnableHeartbeatChannelEventListener() { +++ return enableHeartbeatChannelEventListener; +++ } +++ +++ public void setEnableHeartbeatChannelEventListener(boolean enableHeartbeatChannelEventListener) { +++ this.enableHeartbeatChannelEventListener = enableHeartbeatChannelEventListener; +++ } +++ ++ public int getDetectTimeout() { ++ return this.detectTimeout; ++ } ++@@ -444,19 +456,34 @@ public class ClientConfig { ++ ++ @Override ++ public String toString() { ++- return "ClientConfig [namesrvAddr=" + namesrvAddr ++- + ", clientIP=" + clientIP + ", instanceName=" + instanceName ++- + ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads ++- + ", pollNameServerInterval=" + pollNameServerInterval ++- + ", heartbeatBrokerInterval=" + heartbeatBrokerInterval ++- + ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval ++- + ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException ++- + ", unitMode=" + unitMode + ", unitName=" + unitName ++- + ", vipChannelEnabled=" + vipChannelEnabled + ", useTLS=" + useTLS ++- + ", socksProxyConfig=" + socksProxyConfig + ", language=" + language.name() ++- + ", namespace=" + namespace + ", mqClientApiTimeout=" + mqClientApiTimeout ++- + ", decodeReadBody=" + decodeReadBody + ", decodeDecompressBody=" + decodeDecompressBody ++- + ", sendLatencyEnable=" + sendLatencyEnable + ", startDetectorEnable=" + startDetectorEnable ++- + ", enableStreamRequestType=" + enableStreamRequestType + ", useHeartbeatV2=" + useHeartbeatV2 + "]"; +++ return "ClientConfig{" + +++ "namesrvAddr='" + namesrvAddr + '\'' + +++ ", clientIP='" + clientIP + '\'' + +++ ", instanceName='" + instanceName + '\'' + +++ ", clientCallbackExecutorThreads=" + clientCallbackExecutorThreads + +++ ", namespace='" + namespace + '\'' + +++ ", namespaceInitialized=" + namespaceInitialized + +++ ", accessChannel=" + accessChannel + +++ ", pollNameServerInterval=" + pollNameServerInterval + +++ ", heartbeatBrokerInterval=" + heartbeatBrokerInterval + +++ ", persistConsumerOffsetInterval=" + persistConsumerOffsetInterval + +++ ", pullTimeDelayMillsWhenException=" + pullTimeDelayMillsWhenException + +++ ", unitMode=" + unitMode + +++ ", unitName='" + unitName + '\'' + +++ ", decodeReadBody=" + decodeReadBody + +++ ", decodeDecompressBody=" + decodeDecompressBody + +++ ", vipChannelEnabled=" + vipChannelEnabled + +++ ", useHeartbeatV2=" + useHeartbeatV2 + +++ ", useTLS=" + useTLS + +++ ", socksProxyConfig='" + socksProxyConfig + '\'' + +++ ", mqClientApiTimeout=" + mqClientApiTimeout + +++ ", detectTimeout=" + detectTimeout + +++ ", detectInterval=" + detectInterval + +++ ", language=" + language + +++ ", enableStreamRequestType=" + enableStreamRequestType + +++ ", sendLatencyEnable=" + sendLatencyEnable + +++ ", startDetectorEnable=" + startDetectorEnable + +++ ", enableHeartbeatChannelEventListener=" + enableHeartbeatChannelEventListener + +++ '}'; ++ } ++ } ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++index 2407e5737..e152be811 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/MQClientAPIImpl.java ++@@ -79,6 +79,7 @@ import org.apache.rocketmq.common.sysflag.PullSysFlag; ++ import org.apache.rocketmq.common.topic.TopicValidator; ++ import org.apache.rocketmq.logging.org.slf4j.Logger; ++ import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +++import org.apache.rocketmq.remoting.ChannelEventListener; ++ import org.apache.rocketmq.remoting.CommandCustomHeader; ++ import org.apache.rocketmq.remoting.InvokeCallback; ++ import org.apache.rocketmq.remoting.RPCHook; ++@@ -246,10 +247,16 @@ public class MQClientAPIImpl implements NameServerUpdateCallback { ++ public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, ++ final ClientRemotingProcessor clientRemotingProcessor, ++ RPCHook rpcHook, final ClientConfig clientConfig) { +++ this(nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, null); +++ } +++ +++ public MQClientAPIImpl(final NettyClientConfig nettyClientConfig, +++ final ClientRemotingProcessor clientRemotingProcessor, +++ RPCHook rpcHook, final ClientConfig clientConfig, final ChannelEventListener channelEventListener) { ++ this.clientConfig = clientConfig; ++ topAddressing = new DefaultTopAddressing(MixAll.getWSAddr(), clientConfig.getUnitName()); ++ topAddressing.registerChangeCallBack(this); ++- this.remotingClient = new NettyRemotingClient(nettyClientConfig, null); +++ this.remotingClient = new NettyRemotingClient(nettyClientConfig, channelEventListener); ++ this.clientRemotingProcessor = clientRemotingProcessor; ++ ++ // Inject stream rpc hook first to make reserve field signature ++diff --git a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++index 9484b26f8..09534a176 100644 ++--- a/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java +++++ b/client/src/main/java/org/apache/rocketmq/client/impl/factory/MQClientInstance.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.client.impl.factory; ++ +++import io.netty.channel.Channel; ++ import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.HashSet; ++@@ -65,6 +66,7 @@ import org.apache.rocketmq.common.message.MessageExt; ++ import org.apache.rocketmq.common.message.MessageQueue; ++ import org.apache.rocketmq.common.message.MessageQueueAssignment; ++ import org.apache.rocketmq.common.topic.TopicValidator; +++import org.apache.rocketmq.remoting.ChannelEventListener; ++ import org.apache.rocketmq.remoting.RPCHook; ++ import org.apache.rocketmq.remoting.common.HeartbeatV2Result; ++ import org.apache.rocketmq.remoting.exception.RemotingException; ++@@ -151,7 +153,38 @@ public class MQClientInstance { ++ this.nettyClientConfig.setUseTLS(clientConfig.isUseTLS()); ++ this.nettyClientConfig.setSocksProxyConfig(clientConfig.getSocksProxyConfig()); ++ ClientRemotingProcessor clientRemotingProcessor = new ClientRemotingProcessor(this); ++- this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig); +++ ChannelEventListener channelEventListener; +++ if (clientConfig.isEnableHeartbeatChannelEventListener()) { +++ channelEventListener = new ChannelEventListener() { +++ private final ConcurrentMap> brokerAddrTable = MQClientInstance.this.brokerAddrTable; +++ @Override +++ public void onChannelConnect(String remoteAddr, Channel channel) { +++ for (Map.Entry> addressEntry : brokerAddrTable.entrySet()) { +++ for (String address : addressEntry.getValue().values()) { +++ if (address.equals(remoteAddr)) { +++ sendHeartbeatToAllBrokerWithLockV2(false); +++ break; +++ } +++ } +++ } +++ } +++ +++ @Override +++ public void onChannelClose(String remoteAddr, Channel channel) { +++ } +++ +++ @Override +++ public void onChannelException(String remoteAddr, Channel channel) { +++ } +++ +++ @Override +++ public void onChannelIdle(String remoteAddr, Channel channel) { +++ } +++ }; +++ } else { +++ channelEventListener = null; +++ } +++ this.mQClientAPIImpl = new MQClientAPIImpl(this.nettyClientConfig, clientRemotingProcessor, rpcHook, clientConfig, channelEventListener); ++ ++ if (this.clientConfig.getNamesrvAddr() != null) { ++ this.mQClientAPIImpl.updateNameServerAddressList(this.clientConfig.getNamesrvAddr()); ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++index d784351a5..8631d0447 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++@@ -229,6 +229,8 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ handler.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT); ++ } ++ +++ nettyEventExecutor.start(); +++ ++ TimerTask timerTaskScanResponseTable = new TimerTask() { ++ @Override ++ public void run(Timeout timeout) { ++-- ++2.32.0.windows.2 ++ ++ ++From 3808387e1389278edbe4ef023d200ecb3015622b Mon Sep 17 00:00:00 2001 ++From: lk ++Date: Mon, 9 Oct 2023 16:07:56 +0800 ++Subject: [PATCH 3/7] [ISSUE #7429] clean channel map when CLIENT_UNREGISTER in ++ proxy ++ ++--- ++ .../service/sysmessage/HeartbeatSyncer.java | 31 ++++++--- ++ .../sysmessage/HeartbeatSyncerTest.java | 68 +++++++++++++++++++ ++ 2 files changed, 88 insertions(+), 11 deletions(-) ++ ++diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java ++index f70c06b8f..fee3ea87d 100644 ++--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java +++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncer.java ++@@ -18,6 +18,7 @@ ++ package org.apache.rocketmq.proxy.service.sysmessage; ++ ++ import com.alibaba.fastjson.JSON; +++import io.netty.channel.Channel; ++ import java.nio.charset.StandardCharsets; ++ import java.util.List; ++ import java.util.Map; ++@@ -73,16 +74,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ++ ); ++ this.consumerManager.appendConsumerIdsChangeListener(new ConsumerIdsChangeListener() { ++ @Override ++- public void handle(ConsumerGroupEvent event, String s, Object... args) { ++- if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { ++- if (args == null || args.length < 1) { ++- return; ++- } ++- if (args[0] instanceof ClientChannelInfo) { ++- ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; ++- remoteChannelMap.remove(clientChannelInfo.getChannel().id().asLongText()); ++- } ++- } +++ public void handle(ConsumerGroupEvent event, String group, Object... args) { +++ processConsumerGroupEvent(event, group, args); ++ } ++ ++ @Override ++@@ -98,6 +91,18 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ++ super.shutdown(); ++ } ++ +++ protected void processConsumerGroupEvent(ConsumerGroupEvent event, String group, Object... args) { +++ if (event == ConsumerGroupEvent.CLIENT_UNREGISTER) { +++ if (args == null || args.length < 1) { +++ return; +++ } +++ if (args[0] instanceof ClientChannelInfo) { +++ ClientChannelInfo clientChannelInfo = (ClientChannelInfo) args[0]; +++ remoteChannelMap.remove(buildKey(group, clientChannelInfo.getChannel())); +++ } +++ } +++ } +++ ++ public void onConsumerRegister(String consumerGroup, ClientChannelInfo clientChannelInfo, ++ ConsumeType consumeType, MessageModel messageModel, ConsumeFromWhere consumeFromWhere, ++ Set subList) { ++@@ -189,7 +194,7 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ++ } ++ ++ RemoteChannel decodedChannel = RemoteChannel.decode(data.getChannelData()); ++- RemoteChannel channel = remoteChannelMap.computeIfAbsent(data.getGroup() + "@" + decodedChannel.id().asLongText(), key -> decodedChannel); +++ RemoteChannel channel = remoteChannelMap.computeIfAbsent(buildKey(data.getGroup(), decodedChannel), key -> decodedChannel); ++ channel.setExtendAttribute(decodedChannel.getChannelExtendAttribute()); ++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo( ++ channel, ++@@ -228,4 +233,8 @@ public class HeartbeatSyncer extends AbstractSystemMessageSyncer { ++ // use local address, remoting port and grpc port to build unique local proxy Id ++ return proxyConfig.getLocalServeAddr() + "%" + proxyConfig.getRemotingListenPort() + "%" + proxyConfig.getGrpcServerPort(); ++ } +++ +++ private static String buildKey(String group, Channel channel) { +++ return group + "@" + channel.id().asLongText(); +++ } ++ } ++diff --git a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++index 43fba3d03..9a2c5e343 100644 ++--- a/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java +++++ b/proxy/src/test/java/org/apache/rocketmq/proxy/service/sysmessage/HeartbeatSyncerTest.java ++@@ -27,6 +27,7 @@ import com.google.common.collect.Sets; ++ import io.netty.channel.Channel; ++ import io.netty.channel.ChannelId; ++ import java.time.Duration; +++import java.util.Collections; ++ import java.util.HashMap; ++ import java.util.HashSet; ++ import java.util.List; ++@@ -35,6 +36,7 @@ import java.util.concurrent.CompletableFuture; ++ import java.util.stream.Collectors; ++ import org.apache.commons.lang3.RandomStringUtils; ++ import org.apache.rocketmq.broker.client.ClientChannelInfo; +++import org.apache.rocketmq.broker.client.ConsumerGroupEvent; ++ import org.apache.rocketmq.broker.client.ConsumerManager; ++ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIExt; ++ import org.apache.rocketmq.client.impl.mqclient.MQClientAPIFactory; ++@@ -320,6 +322,72 @@ public class HeartbeatSyncerTest extends InitConfigTest { ++ } ++ } ++ +++ @Test +++ public void testProcessConsumerGroupEventForRemoting() { +++ String consumerGroup = "consumerGroup"; +++ Channel channel = createMockChannel(); +++ RemotingProxyOutClient remotingProxyOutClient = mock(RemotingProxyOutClient.class); +++ RemotingChannel remotingChannel = new RemotingChannel(remotingProxyOutClient, proxyRelayService, channel, clientId, Collections.emptySet()); +++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo( +++ remotingChannel, +++ clientId, +++ LanguageCode.JAVA, +++ 4 +++ ); +++ +++ testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); +++ } +++ +++ @Test +++ public void testProcessConsumerGroupEventForGrpcV2() { +++ String consumerGroup = "consumerGroup"; +++ GrpcClientSettingsManager grpcClientSettingsManager = mock(GrpcClientSettingsManager.class); +++ GrpcChannelManager grpcChannelManager = mock(GrpcChannelManager.class); +++ GrpcClientChannel grpcClientChannel = new GrpcClientChannel( +++ proxyRelayService, grpcClientSettingsManager, grpcChannelManager, +++ ProxyContext.create().setRemoteAddress(remoteAddress).setLocalAddress(localAddress), +++ clientId); +++ ClientChannelInfo clientChannelInfo = new ClientChannelInfo( +++ grpcClientChannel, +++ clientId, +++ LanguageCode.JAVA, +++ 5 +++ ); +++ +++ testProcessConsumerGroupEvent(consumerGroup, clientChannelInfo); +++ } +++ +++ private void testProcessConsumerGroupEvent(String consumerGroup, ClientChannelInfo clientChannelInfo) { +++ HeartbeatSyncer heartbeatSyncer = new HeartbeatSyncer(topicRouteService, adminService, consumerManager, mqClientAPIFactory, null); +++ SendResult okSendResult = new SendResult(); +++ okSendResult.setSendStatus(SendStatus.SEND_OK); +++ +++ ArgumentCaptor messageArgumentCaptor = ArgumentCaptor.forClass(Message.class); +++ doReturn(CompletableFuture.completedFuture(okSendResult)).when(this.mqClientAPIExt) +++ .sendMessageAsync(anyString(), anyString(), messageArgumentCaptor.capture(), any(), anyLong()); +++ +++ heartbeatSyncer.onConsumerRegister( +++ consumerGroup, +++ clientChannelInfo, +++ ConsumeType.CONSUME_PASSIVELY, +++ MessageModel.CLUSTERING, +++ ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET, +++ Collections.emptySet() +++ ); +++ await().atMost(Duration.ofSeconds(3)).until(() -> messageArgumentCaptor.getAllValues().size() == 1); +++ +++ // change local serve addr, to simulate other proxy receive messages +++ heartbeatSyncer.localProxyId = RandomStringUtils.randomAlphabetic(10); +++ ArgumentCaptor channelInfoArgumentCaptor = ArgumentCaptor.forClass(ClientChannelInfo.class); +++ doReturn(true).when(consumerManager).registerConsumer(anyString(), channelInfoArgumentCaptor.capture(), any(), any(), any(), any(), anyBoolean()); +++ +++ heartbeatSyncer.consumeMessage(convertFromMessage(messageArgumentCaptor.getAllValues()), null); +++ assertEquals(1, heartbeatSyncer.remoteChannelMap.size()); +++ +++ heartbeatSyncer.processConsumerGroupEvent(ConsumerGroupEvent.CLIENT_UNREGISTER, consumerGroup, channelInfoArgumentCaptor.getValue()); +++ assertTrue(heartbeatSyncer.remoteChannelMap.isEmpty()); +++ } +++ ++ private MessageExt convertFromMessage(Message message) { ++ MessageExt messageExt = new MessageExt(); ++ messageExt.setTopic(message.getTopic()); ++-- ++2.32.0.windows.2 ++ ++ ++From 0027a1486d4f2d6f7dce3010751167e883783945 Mon Sep 17 00:00:00 2001 ++From: redlsz ++Date: Mon, 9 Oct 2023 16:52:10 +0800 ++Subject: [PATCH 4/7] [ISSUE #7412] Fix pop revive message error when reput ++ checkpoint ++ ++--- ++ .../org/apache/rocketmq/broker/processor/PopReviveService.java | 1 + ++ 1 file changed, 1 insertion(+) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java ++index 93167db37..d5174d3d1 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java ++@@ -595,6 +595,7 @@ public class PopReviveService extends ServiceThread { ++ newCk.setCId(oldCK.getCId()); ++ newCk.setTopic(oldCK.getTopic()); ++ newCk.setQueueId(oldCK.getQueueId()); +++ newCk.setBrokerName(oldCK.getBrokerName()); ++ newCk.addDiff(0); ++ MessageExtBrokerInner ckMsg = brokerController.getPopMessageProcessor().buildCkMsg(newCk, queueId); ++ brokerController.getMessageStore().putMessage(ckMsg); ++-- ++2.32.0.windows.2 ++ ++ ++From b18e564addbcff50165a5e1d9d4ab7db789d901b Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Mon, 9 Oct 2023 21:43:01 +0800 ++Subject: [PATCH 5/7] [ISSUE #7431] Fix flaky test of ++ DLedgerControllerTest#testBrokerLifecycleListener (#7432) ++ ++* Fix flaky test of DLedgerControllerTest#testBrokerLifecycleListener ++--- ++ .../impl/DLedgerControllerTest.java | 26 ++++++++++++------- ++ 1 file changed, 17 insertions(+), 9 deletions(-) ++ ++diff --git a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java ++index 595a5cb65..d6e5449c5 100644 ++--- a/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java +++++ b/controller/src/test/java/org/apache/rocketmq/controller/impl/DLedgerControllerTest.java ++@@ -63,7 +63,8 @@ public class DLedgerControllerTest { ++ private List baseDirs; ++ private List controllers; ++ ++- public DLedgerController launchController(final String group, final String peers, final String selfId, final boolean isEnableElectUncleanMaster) { +++ public DLedgerController launchController(final String group, final String peers, final String selfId, +++ final boolean isEnableElectUncleanMaster) { ++ String tmpdir = System.getProperty("java.io.tmpdir"); ++ final String path = (StringUtils.endsWith(tmpdir, File.separator) ? tmpdir : tmpdir + File.separator) + group + File.separator + selfId; ++ baseDirs.add(path); ++@@ -121,11 +122,11 @@ public class DLedgerControllerTest { ++ final RegisterBrokerToControllerRequestHeader registerBrokerToControllerRequestHeader = new RegisterBrokerToControllerRequestHeader(clusterName, brokerName, nextBrokerId, brokerAddress); ++ RemotingCommand remotingCommand2 = leader.registerBroker(registerBrokerToControllerRequestHeader).get(2, TimeUnit.SECONDS); ++ ++- ++ assertEquals(ResponseCode.SUCCESS, remotingCommand2.getCode()); ++ } ++ ++- public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, Long brokerId, +++ public void brokerTryElectMaster(Controller leader, String clusterName, String brokerName, String brokerAddress, +++ Long brokerId, ++ boolean exceptSuccess) throws Exception { ++ final ElectMasterRequestHeader electMasterRequestHeader = ElectMasterRequestHeader.ofBrokerTrigger(clusterName, brokerName, brokerId); ++ RemotingCommand command = leader.electMaster(electMasterRequestHeader).get(2, TimeUnit.SECONDS); ++@@ -186,9 +187,9 @@ public class DLedgerControllerTest { ++ registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L); ++ registerNewBroker(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L); ++ // try elect ++- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L,true); ++- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); ++- brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L,false); +++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[0], 1L, true); +++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[1], 2L, false); +++ brokerTryElectMaster(leader, DEFAULT_CLUSTER_NAME, DEFAULT_BROKER_NAME, DEFAULT_IP[2], 3L, false); ++ final RemotingCommand getInfoResponse = leader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)).get(10, TimeUnit.SECONDS); ++ final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) getInfoResponse.readCustomHeader(); ++ assertEquals(1, replicaInfo.getMasterEpoch().intValue()); ++@@ -239,6 +240,8 @@ public class DLedgerControllerTest { ++ @Test ++ public void testBrokerLifecycleListener() throws Exception { ++ final DLedgerController leader = mockMetaData(false); +++ +++ assertTrue(leader.isLeaderState()); ++ // Mock that master broker has been inactive, and try to elect a new master from sync-state-set ++ // But we shut down two controller, so the ElectMasterEvent will be appended to DLedger failed. ++ // So the statemachine still keep the stale master's information ++@@ -247,15 +250,20 @@ public class DLedgerControllerTest { ++ dLedgerController.shutdown(); ++ controllers.remove(dLedgerController); ++ } +++ ++ final ElectMasterRequestHeader request = ElectMasterRequestHeader.ofControllerTrigger(DEFAULT_BROKER_NAME); ++ setBrokerElectPolicy(leader, 1L); ++ Exception exception = null; +++ RemotingCommand remotingCommand = null; ++ try { ++- leader.electMaster(request).get(5, TimeUnit.SECONDS); +++ remotingCommand = leader.electMaster(request).get(5, TimeUnit.SECONDS); ++ } catch (Exception e) { ++ exception = e; ++ } ++- assertNotNull(exception); +++ +++ assertTrue(exception != null || +++ remotingCommand != null && remotingCommand.getCode() == ResponseCode.CONTROLLER_NOT_LEADER); +++ ++ // Shut down leader controller ++ leader.shutdown(); ++ controllers.remove(leader); ++@@ -272,7 +280,7 @@ public class DLedgerControllerTest { ++ setBrokerAlivePredicate(newLeader, 1L); ++ // Check if the statemachine is stale ++ final RemotingCommand resp = newLeader.getReplicaInfo(new GetReplicaInfoRequestHeader(DEFAULT_BROKER_NAME)). ++- get(10, TimeUnit.SECONDS); +++ get(10, TimeUnit.SECONDS); ++ final GetReplicaInfoResponseHeader replicaInfo = (GetReplicaInfoResponseHeader) resp.readCustomHeader(); ++ assertEquals(1, replicaInfo.getMasterBrokerId().longValue()); ++ assertEquals(1, replicaInfo.getMasterEpoch().intValue()); ++-- ++2.32.0.windows.2 ++ ++ ++From 38d3d5d95d371ac89f7d491a4c8719b4a22c60e1 Mon Sep 17 00:00:00 2001 ++From: mxsm ++Date: Tue, 10 Oct 2023 09:37:04 +0800 ++Subject: [PATCH 6/7] [ISSUE #7433]Update the version in the README.md document ++ to 5.1.4 (#7434) ++ ++--- ++ README.md | 8 ++++---- ++ 1 file changed, 4 insertions(+), 4 deletions(-) ++ ++diff --git a/README.md b/README.md ++index 56d253ce1..5aaa2ba73 100644 ++--- a/README.md +++++ b/README.md ++@@ -49,21 +49,21 @@ $ java -version ++ java version "1.8.0_121" ++ ``` ++ ++-For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip) to download the 5.1.3 RocketMQ binary release, +++For Windows users, click [here](https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip) to download the 5.1.4 RocketMQ binary release, ++ unpack it to your local disk, such as `D:\rocketmq`. ++ For macOS and Linux users, execute following commands: ++ ++ ```shell ++ # Download release from the Apache mirror ++-$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.3/rocketmq-all-5.1.3-bin-release.zip +++$ wget https://dist.apache.org/repos/dist/release/rocketmq/5.1.4/rocketmq-all-5.1.4-bin-release.zip ++ ++ # Unpack the release ++-$ unzip rocketmq-all-5.1.3-bin-release.zip +++$ unzip rocketmq-all-5.1.4-bin-release.zip ++ ``` ++ ++ Prepare a terminal and change to the extracted `bin` directory: ++ ```shell ++-$ cd rocketmq-all-5.1.3-bin-release/bin +++$ cd rocketmq-all-5.1.4-bin-release/bin ++ ``` ++ ++ **1) Start NameServer** ++-- ++2.32.0.windows.2 ++ ++ ++From 4acb43ecee03e429d036e3ff4c28bd402d1b30c7 Mon Sep 17 00:00:00 2001 ++From: Zhouxiang Zhan ++Date: Tue, 10 Oct 2023 13:54:01 +0800 ++Subject: [PATCH 7/7] [ISSUE #7330] Add goaway and reconnection mechanism ++ (#7331) ++ ++* Add shutdown wait for NettyRemotingServer ++ ++* Add goaway and reconnection mechanism ++ ++* Add client version check ++ ++* Add enableTransparentRetry for NettyClientConfig ++ ++* Add enableReconnectForGoAway for NettyClientConfig ++ ++* fix unit test ++ ++* fix client version check ++--- ++ .../remoting/netty/NettyClientConfig.java | 30 ++++ ++ .../remoting/netty/NettyRemotingAbstract.java | 15 ++ ++ .../remoting/netty/NettyRemotingClient.java | 153 ++++++++++++++++-- ++ .../remoting/netty/NettyRemotingServer.java | 31 ++-- ++ .../remoting/netty/NettyServerConfig.java | 19 +++ ++ .../remoting/protocol/ResponseCode.java | 2 + ++ .../netty/NettyRemotingClientTest.java | 39 ++--- ++ 7 files changed, 239 insertions(+), 50 deletions(-) ++ ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java ++index b2e7df754..c28288786 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyClientConfig.java ++@@ -53,6 +53,12 @@ public class NettyClientConfig { ++ private boolean disableCallbackExecutor = false; ++ private boolean disableNettyWorkerGroup = false; ++ +++ private long maxReconnectIntervalTimeSeconds = 60; +++ +++ private boolean enableReconnectForGoAway = true; +++ +++ private boolean enableTransparentRetry = true; +++ ++ public boolean isClientCloseSocketIfTimeout() { ++ return clientCloseSocketIfTimeout; ++ } ++@@ -181,6 +187,30 @@ public class NettyClientConfig { ++ this.disableNettyWorkerGroup = disableNettyWorkerGroup; ++ } ++ +++ public long getMaxReconnectIntervalTimeSeconds() { +++ return maxReconnectIntervalTimeSeconds; +++ } +++ +++ public void setMaxReconnectIntervalTimeSeconds(long maxReconnectIntervalTimeSeconds) { +++ this.maxReconnectIntervalTimeSeconds = maxReconnectIntervalTimeSeconds; +++ } +++ +++ public boolean isEnableReconnectForGoAway() { +++ return enableReconnectForGoAway; +++ } +++ +++ public void setEnableReconnectForGoAway(boolean enableReconnectForGoAway) { +++ this.enableReconnectForGoAway = enableReconnectForGoAway; +++ } +++ +++ public boolean isEnableTransparentRetry() { +++ return enableTransparentRetry; +++ } +++ +++ public void setEnableTransparentRetry(boolean enableTransparentRetry) { +++ this.enableTransparentRetry = enableTransparentRetry; +++ } +++ ++ public String getSocksProxyConfig() { ++ return socksProxyConfig; ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++index 12e66f913..07ace28ea 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingAbstract.java ++@@ -40,9 +40,11 @@ import java.util.concurrent.Semaphore; ++ import java.util.concurrent.TimeUnit; ++ import java.util.concurrent.TimeoutException; ++ import java.util.concurrent.atomic.AtomicReference; +++import java.util.concurrent.atomic.AtomicBoolean; ++ import java.util.function.Consumer; ++ import javax.annotation.Nullable; ++ import org.apache.rocketmq.common.AbortProcessException; +++import org.apache.rocketmq.common.MQVersion; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ServiceThread; ++ import org.apache.rocketmq.common.UtilAll; ++@@ -60,6 +62,7 @@ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.metrics.RemotingMetricsManager; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ import org.apache.rocketmq.remoting.protocol.RemotingSysResponseCode; +++import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ ++ import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_IS_LONG_POLLING; ++ import static org.apache.rocketmq.remoting.metrics.RemotingMetricsConstant.LABEL_REQUEST_CODE; ++@@ -120,6 +123,8 @@ public abstract class NettyRemotingAbstract { ++ */ ++ protected List rpcHooks = new ArrayList<>(); ++ +++ protected AtomicBoolean isShuttingDown = new AtomicBoolean(false); +++ ++ static { ++ NettyLogger.initNettyLogger(); ++ } ++@@ -264,6 +269,16 @@ public abstract class NettyRemotingAbstract { ++ ++ Runnable run = buildProcessRequestHandler(ctx, cmd, pair, opaque); ++ +++ if (isShuttingDown.get()) { +++ if (cmd.getVersion() > MQVersion.Version.V5_1_4.ordinal()) { +++ final RemotingCommand response = RemotingCommand.createResponseCommand(ResponseCode.GO_AWAY, +++ "please go away"); +++ response.setOpaque(opaque); +++ writeResponse(ctx.channel(), cmd, response); +++ return; +++ } +++ } +++ ++ if (pair.getObject1().rejectRequest()) { ++ final RemotingCommand response = RemotingCommand.createResponseCommand(RemotingSysResponseCode.SYSTEM_BUSY, ++ "[REJECTREQUEST]system busy, start flow control for a while"); ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++index 8631d0447..4bc51bd83 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++@@ -18,6 +18,7 @@ package org.apache.rocketmq.remoting.netty; ++ ++ import com.alibaba.fastjson.JSON; ++ import com.alibaba.fastjson.TypeReference; +++import com.google.common.base.Stopwatch; ++ import io.netty.bootstrap.Bootstrap; ++ import io.netty.buffer.PooledByteBufAllocator; ++ import io.netty.channel.Channel; ++@@ -48,6 +49,7 @@ import java.io.IOException; ++ import java.net.InetSocketAddress; ++ import java.net.SocketAddress; ++ import java.security.cert.CertificateException; +++import java.time.Duration; ++ import java.util.ArrayList; ++ import java.util.Collections; ++ import java.util.HashMap; ++@@ -57,6 +59,7 @@ import java.util.Map; ++ import java.util.Random; ++ import java.util.Set; ++ import java.util.concurrent.ArrayBlockingQueue; +++import java.util.concurrent.CompletableFuture; ++ import java.util.concurrent.ConcurrentHashMap; ++ import java.util.concurrent.ConcurrentMap; ++ import java.util.concurrent.ExecutorService; ++@@ -66,6 +69,7 @@ import java.util.concurrent.atomic.AtomicInteger; ++ import java.util.concurrent.atomic.AtomicReference; ++ import java.util.concurrent.locks.Lock; ++ import java.util.concurrent.locks.ReentrantLock; +++import java.util.concurrent.locks.ReentrantReadWriteLock; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.Pair; ++ import org.apache.rocketmq.common.ThreadFactoryImpl; ++@@ -82,6 +86,7 @@ import org.apache.rocketmq.remoting.exception.RemotingSendRequestException; ++ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; +++import org.apache.rocketmq.remoting.protocol.ResponseCode; ++ import org.apache.rocketmq.remoting.proxy.SocksProxyConfig; ++ ++ public class NettyRemotingClient extends NettyRemotingAbstract implements RemotingClient { ++@@ -97,6 +102,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ private final Map proxyMap = new HashMap<>(); ++ private final ConcurrentHashMap bootstrapMap = new ConcurrentHashMap<>(); ++ private final ConcurrentMap channelTables = new ConcurrentHashMap<>(); +++ private final ConcurrentMap channelWrapperTables = new ConcurrentHashMap<>(); ++ ++ private final HashedWheelTimer timer = new HashedWheelTimer(r -> new Thread(r, "ClientHouseKeepingService")); ++ ++@@ -356,9 +362,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ this.timer.stop(); ++ ++ for (String addr : this.channelTables.keySet()) { ++- this.closeChannel(addr, this.channelTables.get(addr).getChannel()); +++ this.channelTables.get(addr).close(); ++ } ++ +++ this.channelWrapperTables.clear(); ++ this.channelTables.clear(); ++ ++ this.eventLoopGroupWorker.shutdownGracefully(); ++@@ -416,7 +423,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ } ++ ++ if (removeItemFromTable) { ++- this.channelTables.remove(addrRemote); +++ ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); +++ if (channelWrapper != null && channelWrapper.tryClose(channel)) { +++ this.channelTables.remove(addrRemote); +++ } ++ LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); ++ } ++ ++@@ -463,7 +473,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ } ++ ++ if (removeItemFromTable) { ++- this.channelTables.remove(addrRemote); +++ ChannelWrapper channelWrapper = this.channelWrapperTables.remove(channel); +++ if (channelWrapper != null && channelWrapper.tryClose(channel)) { +++ this.channelTables.remove(addrRemote); +++ } ++ LOGGER.info("closeChannel: the channel[{}] was removed from channel table", addrRemote); ++ RemotingHelper.closeChannel(channel); ++ } ++@@ -511,7 +524,7 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ if (addr.contains(namesrvAddr)) { ++ ChannelWrapper channelWrapper = this.channelTables.get(addr); ++ if (channelWrapper != null) { ++- closeChannel(channelWrapper.getChannel()); +++ channelWrapper.close(); ++ } ++ } ++ } ++@@ -689,8 +702,9 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ ChannelFuture channelFuture = fetchBootstrap(addr) ++ .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); ++ LOGGER.info("createChannel: begin to connect remote host[{}] asynchronously", addr); ++- cw = new ChannelWrapper(channelFuture); +++ cw = new ChannelWrapper(addr, channelFuture); ++ this.channelTables.put(addr, cw); +++ this.channelWrapperTables.put(channelFuture.channel(), cw); ++ } ++ } catch (Exception e) { ++ LOGGER.error("createChannel: create channel exception", e); ++@@ -758,6 +772,64 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ } ++ } ++ +++ @Override +++ public CompletableFuture invoke(String addr, RemotingCommand request, +++ long timeoutMillis) { +++ CompletableFuture future = new CompletableFuture<>(); +++ try { +++ final Channel channel = this.getAndCreateChannel(addr); +++ if (channel != null && channel.isActive()) { +++ return invokeImpl(channel, request, timeoutMillis).whenComplete((v, t) -> { +++ if (t == null) { +++ updateChannelLastResponseTime(addr); +++ } +++ }).thenApply(ResponseFuture::getResponseCommand); +++ } else { +++ this.closeChannel(addr, channel); +++ future.completeExceptionally(new RemotingConnectException(addr)); +++ } +++ } catch (Throwable t) { +++ future.completeExceptionally(t); +++ } +++ return future; +++ } +++ +++ @Override +++ public CompletableFuture invokeImpl(final Channel channel, final RemotingCommand request, +++ final long timeoutMillis) { +++ Stopwatch stopwatch = Stopwatch.createStarted(); +++ return super.invokeImpl(channel, request, timeoutMillis).thenCompose(responseFuture -> { +++ RemotingCommand response = responseFuture.getResponseCommand(); +++ if (response.getCode() == ResponseCode.GO_AWAY) { +++ if (nettyClientConfig.isEnableReconnectForGoAway()) { +++ ChannelWrapper channelWrapper = channelWrapperTables.computeIfPresent(channel, (channel0, channelWrapper0) -> { +++ try { +++ if (channelWrapper0.reconnect()) { +++ LOGGER.info("Receive go away from channel {}, recreate the channel", channel0); +++ channelWrapperTables.put(channelWrapper0.getChannel(), channelWrapper0); +++ } +++ } catch (Throwable t) { +++ LOGGER.error("Channel {} reconnect error", channelWrapper0, t); +++ } +++ return channelWrapper0; +++ }); +++ if (channelWrapper != null) { +++ if (nettyClientConfig.isEnableTransparentRetry()) { +++ long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); +++ stopwatch.stop(); +++ RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); +++ Channel retryChannel = channelWrapper.getChannel(); +++ if (channel != retryChannel) { +++ return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration); +++ } +++ } +++ } +++ } +++ } +++ return CompletableFuture.completedFuture(responseFuture); +++ }); +++ } +++ ++ @Override ++ public void registerProcessor(int requestCode, NettyRequestProcessor processor, ExecutorService executor) { ++ ExecutorService executorThis = executor; ++@@ -877,30 +949,41 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ } ++ } ++ ++- static class ChannelWrapper { ++- private final ChannelFuture channelFuture; +++ class ChannelWrapper { +++ private final ReentrantReadWriteLock lock; +++ private ChannelFuture channelFuture; ++ // only affected by sync or async request, oneway is not included. +++ private ChannelFuture channelToClose; ++ private long lastResponseTime; +++ private volatile long lastReconnectTimestamp = 0L; +++ private final String channelAddress; ++ ++- public ChannelWrapper(ChannelFuture channelFuture) { +++ public ChannelWrapper(String address, ChannelFuture channelFuture) { +++ this.lock = new ReentrantReadWriteLock(); ++ this.channelFuture = channelFuture; ++ this.lastResponseTime = System.currentTimeMillis(); +++ this.channelAddress = address; ++ } ++ ++ public boolean isOK() { ++- return this.channelFuture.channel() != null && this.channelFuture.channel().isActive(); +++ return getChannel() != null && getChannel().isActive(); ++ } ++ ++ public boolean isWritable() { ++- return this.channelFuture.channel().isWritable(); +++ return getChannel().isWritable(); ++ } ++ ++ private Channel getChannel() { ++- return this.channelFuture.channel(); +++ return getChannelFuture().channel(); ++ } ++ ++ public ChannelFuture getChannelFuture() { ++- return channelFuture; +++ lock.readLock().lock(); +++ try { +++ return this.channelFuture; +++ } finally { +++ lock.readLock().unlock(); +++ } ++ } ++ ++ public long getLastResponseTime() { ++@@ -910,6 +993,52 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti ++ public void updateLastResponseTime() { ++ this.lastResponseTime = System.currentTimeMillis(); ++ } +++ +++ public boolean reconnect() { +++ if (lock.writeLock().tryLock()) { +++ try { +++ if (lastReconnectTimestamp == 0L || System.currentTimeMillis() - lastReconnectTimestamp > Duration.ofSeconds(nettyClientConfig.getMaxReconnectIntervalTimeSeconds()).toMillis()) { +++ channelToClose = channelFuture; +++ String[] hostAndPort = getHostAndPort(channelAddress); +++ channelFuture = fetchBootstrap(channelAddress) +++ .connect(hostAndPort[0], Integer.parseInt(hostAndPort[1])); +++ lastReconnectTimestamp = System.currentTimeMillis(); +++ return true; +++ } +++ } finally { +++ lock.writeLock().unlock(); +++ } +++ } +++ return false; +++ } +++ +++ public boolean tryClose(Channel channel) { +++ try { +++ lock.readLock().lock(); +++ if (channelFuture != null) { +++ if (channelFuture.channel().equals(channel)) { +++ return true; +++ } +++ } +++ } finally { +++ lock.readLock().unlock(); +++ } +++ return false; +++ } +++ +++ public void close() { +++ try { +++ lock.writeLock().lock(); +++ if (channelFuture != null) { +++ closeChannel(channelFuture.channel()); +++ } +++ if (channelToClose != null) { +++ closeChannel(channelToClose.channel()); +++ } +++ } finally { +++ lock.writeLock().unlock(); +++ } +++ } ++ } ++ ++ class InvokeCallbackWrapper implements InvokeCallback { ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++index aa0d46542..735d36168 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingServer.java ++@@ -53,6 +53,19 @@ import io.netty.util.HashedWheelTimer; ++ import io.netty.util.Timeout; ++ import io.netty.util.TimerTask; ++ import io.netty.util.concurrent.DefaultEventExecutorGroup; +++import java.io.IOException; +++import java.net.InetSocketAddress; +++import java.security.cert.CertificateException; +++import java.time.Duration; +++import java.util.List; +++import java.util.NoSuchElementException; +++import java.util.concurrent.ConcurrentHashMap; +++import java.util.concurrent.ConcurrentMap; +++import java.util.concurrent.ExecutorService; +++import java.util.concurrent.Executors; +++import java.util.concurrent.ScheduledExecutorService; +++import java.util.concurrent.ThreadPoolExecutor; +++import java.util.concurrent.TimeUnit; ++ import org.apache.commons.collections.CollectionUtils; ++ import org.apache.commons.lang3.StringUtils; ++ import org.apache.rocketmq.common.Pair; ++@@ -74,19 +87,6 @@ import org.apache.rocketmq.remoting.exception.RemotingTimeoutException; ++ import org.apache.rocketmq.remoting.exception.RemotingTooMuchRequestException; ++ import org.apache.rocketmq.remoting.protocol.RemotingCommand; ++ ++-import java.io.IOException; ++-import java.net.InetSocketAddress; ++-import java.security.cert.CertificateException; ++-import java.util.List; ++-import java.util.NoSuchElementException; ++-import java.util.concurrent.ConcurrentHashMap; ++-import java.util.concurrent.ConcurrentMap; ++-import java.util.concurrent.ExecutorService; ++-import java.util.concurrent.Executors; ++-import java.util.concurrent.ScheduledExecutorService; ++-import java.util.concurrent.ThreadPoolExecutor; ++-import java.util.concurrent.TimeUnit; ++- ++ @SuppressWarnings("NullableProblems") ++ public class NettyRemotingServer extends NettyRemotingAbstract implements RemotingServer { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.ROCKETMQ_REMOTING_NAME); ++@@ -305,6 +305,10 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ @Override ++ public void shutdown() { ++ try { +++ if (nettyServerConfig.isEnableShutdownGracefully() && isShuttingDown.compareAndSet(false, true)) { +++ Thread.sleep(Duration.ofSeconds(nettyServerConfig.getShutdownWaitTimeSeconds()).toMillis()); +++ } +++ ++ this.timer.stop(); ++ ++ this.eventLoopGroupBoss.shutdownGracefully(); ++@@ -736,6 +740,7 @@ public class NettyRemotingServer extends NettyRemotingAbstract implements Remoti ++ ++ @Override ++ public void shutdown() { +++ isShuttingDown.set(true); ++ if (this.serverChannel != null) { ++ try { ++ this.serverChannel.close().await(5, TimeUnit.SECONDS); ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java ++index 59ef2c84f..756661f62 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyServerConfig.java ++@@ -38,6 +38,9 @@ public class NettyServerConfig implements Cloneable { ++ private int serverSocketBacklog = NettySystemConfig.socketBacklog; ++ private boolean serverPooledByteBufAllocatorEnable = true; ++ +++ private boolean enableShutdownGracefully = false; +++ private int shutdownWaitTimeSeconds = 30; +++ ++ /** ++ * make install ++ * ++@@ -171,4 +174,20 @@ public class NettyServerConfig implements Cloneable { ++ public void setWriteBufferHighWaterMark(int writeBufferHighWaterMark) { ++ this.writeBufferHighWaterMark = writeBufferHighWaterMark; ++ } +++ +++ public boolean isEnableShutdownGracefully() { +++ return enableShutdownGracefully; +++ } +++ +++ public void setEnableShutdownGracefully(boolean enableShutdownGracefully) { +++ this.enableShutdownGracefully = enableShutdownGracefully; +++ } +++ +++ public int getShutdownWaitTimeSeconds() { +++ return shutdownWaitTimeSeconds; +++ } +++ +++ public void setShutdownWaitTimeSeconds(int shutdownWaitTimeSeconds) { +++ this.shutdownWaitTimeSeconds = shutdownWaitTimeSeconds; +++ } ++ } ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java ++index e81dadf2e..be945c48f 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/ResponseCode.java ++@@ -99,6 +99,8 @@ public class ResponseCode extends RemotingSysResponseCode { ++ public static final int RPC_SEND_TO_CHANNEL_FAILED = -1004; ++ public static final int RPC_TIME_OUT = -1006; ++ +++ public static final int GO_AWAY = 1500; +++ ++ /** ++ * Controller response code ++ */ ++diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++index e72e7bd53..1cc6b4f46 100644 ++--- a/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java +++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/netty/NettyRemotingClientTest.java ++@@ -47,7 +47,6 @@ import static org.mockito.ArgumentMatchers.any; ++ import static org.mockito.ArgumentMatchers.anyLong; ++ import static org.mockito.ArgumentMatchers.anyString; ++ import static org.mockito.ArgumentMatchers.eq; ++-import static org.mockito.Mockito.doAnswer; ++ import static org.mockito.Mockito.doReturn; ++ import static org.mockito.Mockito.mock; ++ import static org.mockito.Mockito.never; ++@@ -74,13 +73,11 @@ public class NettyRemotingClientTest { ++ ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++- doAnswer(invocation -> { ++- InvokeCallback callback = invocation.getArgument(3); ++- ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); ++- responseFuture.setResponseCommand(response); ++- callback.operationSucceed(responseFuture.getResponseCommand()); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +++ ResponseFuture responseFuture = new ResponseFuture(null, request.getOpaque(), 3 * 1000, null, null); +++ responseFuture.setResponseCommand(response); +++ CompletableFuture future0 = new CompletableFuture<>(); +++ future0.complete(responseFuture.getResponseCommand()); +++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); ++ RemotingCommand actual = future.get(); ++@@ -93,11 +90,9 @@ public class NettyRemotingClientTest { ++ ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++- doAnswer(invocation -> { ++- InvokeCallback callback = invocation.getArgument(3); ++- callback.operationFail(new RemotingSendRequestException(null)); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +++ CompletableFuture future0 = new CompletableFuture<>(); +++ future0.completeExceptionally(new RemotingSendRequestException(null)); +++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); ++ Throwable thrown = catchThrowable(future::get); ++@@ -110,11 +105,9 @@ public class NettyRemotingClientTest { ++ ++ RemotingCommand response = RemotingCommand.createResponseCommand(null); ++ response.setCode(ResponseCode.SUCCESS); ++- doAnswer(invocation -> { ++- InvokeCallback callback = invocation.getArgument(3); ++- callback.operationFail(new RemotingTimeoutException("")); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +++ CompletableFuture future0 = new CompletableFuture<>(); +++ future0.completeExceptionally(new RemotingTimeoutException("")); +++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); ++ Throwable thrown = catchThrowable(future::get); ++@@ -125,13 +118,9 @@ public class NettyRemotingClientTest { ++ public void testRemotingException() throws Exception { ++ RemotingCommand request = RemotingCommand.createRequestCommand(RequestCode.PULL_MESSAGE, null); ++ ++- RemotingCommand response = RemotingCommand.createResponseCommand(null); ++- response.setCode(ResponseCode.SUCCESS); ++- doAnswer(invocation -> { ++- InvokeCallback callback = invocation.getArgument(3); ++- callback.operationFail(new RemotingException(null)); ++- return null; ++- }).when(remotingClient).invokeAsync(anyString(), any(RemotingCommand.class), anyLong(), any(InvokeCallback.class)); +++ CompletableFuture future0 = new CompletableFuture<>(); +++ future0.completeExceptionally(new RemotingException("")); +++ doReturn(future0).when(remotingClient).invoke(anyString(), any(RemotingCommand.class), anyLong()); ++ ++ CompletableFuture future = remotingClient.invoke("0.0.0.0", request, 1000); ++ Throwable thrown = catchThrowable(future::get); ++-- ++2.32.0.windows.2 ++ +diff --git a/patch021-backport-some-enhancements.patch b/patch021-backport-some-enhancements.patch +new file mode 100644 +index 000000000..839b65fea +--- /dev/null ++++ b/patch021-backport-some-enhancements.patch +@@ -0,0 +1,344 @@ ++From dc3f22ffe9eb83ace991b68921076093c7c0da5f Mon Sep 17 00:00:00 2001 ++From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> ++Date: Tue, 10 Oct 2023 17:39:23 +0800 ++Subject: [PATCH 1/6] add getter for class Message ,fix json serialize bug ++ (#7439) ++ ++Co-authored-by: LetLetMe ++--- ++ .../rocketmq/common/message/Message.java | 24 ++++++++++++++++++- ++ 1 file changed, 23 insertions(+), 1 deletion(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/message/Message.java b/common/src/main/java/org/apache/rocketmq/common/message/Message.java ++index e02b526a1..c7997c473 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/message/Message.java +++++ b/common/src/main/java/org/apache/rocketmq/common/message/Message.java ++@@ -218,14 +218,36 @@ public class Message implements Serializable { ++ public void setDelayTimeSec(long sec) { ++ this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC, String.valueOf(sec)); ++ } +++ +++ public long getDelayTimeSec() { +++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_SEC); +++ if (t != null) { +++ return Long.parseLong(t); +++ } +++ return 0; +++ } +++ ++ public void setDelayTimeMs(long timeMs) { ++ this.putProperty(MessageConst.PROPERTY_TIMER_DELAY_MS, String.valueOf(timeMs)); ++ } +++ +++ public long getDelayTimeMs() { +++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELAY_MS); +++ if (t != null) { +++ return Long.parseLong(t); +++ } +++ return 0; +++ } +++ ++ public void setDeliverTimeMs(long timeMs) { ++ this.putProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS, String.valueOf(timeMs)); ++ } ++ ++ public long getDeliverTimeMs() { ++- return Long.parseLong(this.getUserProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS)); +++ String t = this.getProperty(MessageConst.PROPERTY_TIMER_DELIVER_MS); +++ if (t != null) { +++ return Long.parseLong(t); +++ } +++ return 0; ++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 7e4879a3bc120d6289aabc8354a2811f349ac8a6 Mon Sep 17 00:00:00 2001 ++From: fujian-zfj <2573259572@qq.com> ++Date: Wed, 11 Oct 2023 14:45:07 +0800 ++Subject: [PATCH 2/6] [ISSUE #7441] Fix log "Init the confirmOffset" keep ++ printing error in controller mode (#7442) ++ ++* typo int readme[ecosystem] ++ ++* fix keep printing log problem ++--- ++ store/src/main/java/org/apache/rocketmq/store/CommitLog.java | 2 +- ++ 1 file changed, 1 insertion(+), 1 deletion(-) ++ ++diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++index 456bf2b86..f98e9a284 100644 ++--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++@@ -580,7 +580,7 @@ public class CommitLog implements Swappable { ++ return this.defaultMessageStore.getMaxPhyOffset(); ++ } ++ // First time it will compute the confirmOffset. ++- if (this.confirmOffset <= 0) { +++ if (this.confirmOffset < 0) { ++ setConfirmOffset(((AutoSwitchHAService) this.defaultMessageStore.getHaService()).computeConfirmOffset()); ++ log.info("Init the confirmOffset to {}.", this.confirmOffset); ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 5d492c338258d07613103e6ae16df4c6fa5b3838 Mon Sep 17 00:00:00 2001 ++From: rongtong ++Date: Fri, 13 Oct 2023 11:23:30 +0800 ++Subject: [PATCH 3/6] [ISSUE #7444] Fix testCalculateFileSizeInPath test can ++ not rerun in same environment (#7445) ++ ++* Fix testCalculateFileSizeInPath test can not rerun in same environment ++ ++* Ensure that files are always deleted ++--- ++ .../apache/rocketmq/common/UtilAllTest.java | 83 +++++++++++-------- ++ 1 file changed, 48 insertions(+), 35 deletions(-) ++ ++diff --git a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java ++index f568a65f4..a0653d7fc 100644 ++--- a/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java +++++ b/common/src/test/java/org/apache/rocketmq/common/UtilAllTest.java ++@@ -238,41 +238,54 @@ public class UtilAllTest { ++ */ ++ String basePath = System.getProperty("java.io.tmpdir") + File.separator + "testCalculateFileSizeInPath"; ++ File baseFile = new File(basePath); ++- // test empty path ++- assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile)); ++- ++- // create baseDir ++- assertTrue(baseFile.mkdirs()); ++- ++- File file0 = new File(baseFile, "file_0"); ++- assertTrue(file0.createNewFile()); ++- writeFixedBytesToFile(file0, 1313); ++- ++- assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile)); ++- ++- // build a file tree like above ++- File dir1 = new File(baseFile, "dir_1"); ++- dir1.mkdirs(); ++- File file10 = new File(dir1, "file_1_0"); ++- File file11 = new File(dir1, "file_1_1"); ++- File dir12 = new File(dir1, "dir_1_2"); ++- dir12.mkdirs(); ++- File file120 = new File(dir12, "file_1_2_0"); ++- File dir2 = new File(baseFile, "dir_2"); ++- dir2.mkdirs(); ++- ++- // write all file with 1313 bytes data ++- assertTrue(file10.createNewFile()); ++- writeFixedBytesToFile(file10, 1313); ++- assertTrue(file11.createNewFile()); ++- writeFixedBytesToFile(file11, 1313); ++- assertTrue(file120.createNewFile()); ++- writeFixedBytesToFile(file120, 1313); ++- ++- assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile)); ++- ++- // clear all file ++- baseFile.deleteOnExit(); +++ try { +++ // test empty path +++ assertEquals(0, UtilAll.calculateFileSizeInPath(baseFile)); +++ +++ // create baseDir +++ assertTrue(baseFile.mkdirs()); +++ +++ File file0 = new File(baseFile, "file_0"); +++ assertTrue(file0.createNewFile()); +++ writeFixedBytesToFile(file0, 1313); +++ +++ assertEquals(1313, UtilAll.calculateFileSizeInPath(baseFile)); +++ +++ // build a file tree like above +++ File dir1 = new File(baseFile, "dir_1"); +++ dir1.mkdirs(); +++ File file10 = new File(dir1, "file_1_0"); +++ File file11 = new File(dir1, "file_1_1"); +++ File dir12 = new File(dir1, "dir_1_2"); +++ dir12.mkdirs(); +++ File file120 = new File(dir12, "file_1_2_0"); +++ File dir2 = new File(baseFile, "dir_2"); +++ dir2.mkdirs(); +++ +++ // write all file with 1313 bytes data +++ assertTrue(file10.createNewFile()); +++ writeFixedBytesToFile(file10, 1313); +++ assertTrue(file11.createNewFile()); +++ writeFixedBytesToFile(file11, 1313); +++ assertTrue(file120.createNewFile()); +++ writeFixedBytesToFile(file120, 1313); +++ +++ assertEquals(1313 * 4, UtilAll.calculateFileSizeInPath(baseFile)); +++ } finally { +++ deleteFolder(baseFile); +++ } +++ } +++ +++ public static void deleteFolder(File folder) { +++ if (folder.isDirectory()) { +++ File[] files = folder.listFiles(); +++ if (files != null) { +++ for (File file : files) { +++ deleteFolder(file); +++ } +++ } +++ } +++ folder.delete(); ++ } ++ ++ private void writeFixedBytesToFile(File file, int size) throws Exception { ++-- ++2.32.0.windows.2 ++ ++ ++From 28427d40129e3aa0c6f951535617e5cac0a8211b Mon Sep 17 00:00:00 2001 ++From: Lei Sun ++Date: Fri, 13 Oct 2023 13:42:27 +0800 ++Subject: [PATCH 4/6] [ISSUE #7425] Add RoccketmqControllerConsole log to fix ++ bug (#7458) ++ ++--- ++ .../org/apache/rocketmq/common/constant/LoggerName.java | 1 + ++ .../org/apache/rocketmq/controller/ControllerStartup.java | 7 ++++--- ++ controller/src/main/resources/rmq.controller.logback.xml | 4 ++++ ++ 3 files changed, 9 insertions(+), 3 deletions(-) ++ ++diff --git a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++index cb04b00b3..61310893f 100644 ++--- a/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java +++++ b/common/src/main/java/org/apache/rocketmq/common/constant/LoggerName.java ++@@ -21,6 +21,7 @@ public class LoggerName { ++ public static final String NAMESRV_LOGGER_NAME = "RocketmqNamesrv"; ++ public static final String NAMESRV_CONSOLE_LOGGER_NAME = "RocketmqNamesrvConsole"; ++ public static final String CONTROLLER_LOGGER_NAME = "RocketmqController"; +++ public static final String CONTROLLER_CONSOLE_NAME = "RocketmqControllerConsole"; ++ public static final String NAMESRV_WATER_MARK_LOGGER_NAME = "RocketmqNamesrvWaterMark"; ++ public static final String BROKER_LOGGER_NAME = "RocketmqBroker"; ++ public static final String BROKER_CONSOLE_NAME = "RocketmqConsole"; ++diff --git a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java ++index 401720d05..9e96a704d 100644 ++--- a/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java +++++ b/controller/src/main/java/org/apache/rocketmq/controller/ControllerStartup.java ++@@ -94,9 +94,10 @@ public class ControllerStartup { ++ } ++ ++ if (commandLine.hasOption('p')) { ++- MixAll.printObjectProperties(null, controllerConfig); ++- MixAll.printObjectProperties(null, nettyServerConfig); ++- MixAll.printObjectProperties(null, nettyClientConfig); +++ Logger console = LoggerFactory.getLogger(LoggerName.CONTROLLER_CONSOLE_NAME); +++ MixAll.printObjectProperties(console, controllerConfig); +++ MixAll.printObjectProperties(console, nettyServerConfig); +++ MixAll.printObjectProperties(console, nettyClientConfig); ++ System.exit(0); ++ } ++ ++diff --git a/controller/src/main/resources/rmq.controller.logback.xml b/controller/src/main/resources/rmq.controller.logback.xml ++index bb158213a..18083e8f9 100644 ++--- a/controller/src/main/resources/rmq.controller.logback.xml +++++ b/controller/src/main/resources/rmq.controller.logback.xml ++@@ -116,6 +116,10 @@ ++ ++ ++ +++ +++ +++ +++ ++ ++ ++ ++-- ++2.32.0.windows.2 ++ ++ ++From dc62d7f2e1ce4f99364599f8e23d65fd88eb1cd4 Mon Sep 17 00:00:00 2001 ++From: LetLetMe <43874697+LetLetMe@users.noreply.github.com> ++Date: Fri, 13 Oct 2023 13:45:48 +0800 ++Subject: [PATCH 5/6] [ISSUE #7451] Override toString for ++ TopicConfigAndQueueMapping ++ ++--- ++ .../statictopic/TopicConfigAndQueueMapping.java | 10 ++++++++++ ++ 1 file changed, 10 insertions(+) ++ ++diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java ++index c937fec23..d13692735 100644 ++--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java +++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/statictopic/TopicConfigAndQueueMapping.java ++@@ -16,6 +16,7 @@ ++ */ ++ package org.apache.rocketmq.remoting.protocol.statictopic; ++ +++import org.apache.commons.lang3.StringUtils; ++ import org.apache.commons.lang3.builder.EqualsBuilder; ++ import org.apache.commons.lang3.builder.HashCodeBuilder; ++ import org.apache.rocketmq.common.TopicConfig; ++@@ -60,4 +61,13 @@ public class TopicConfigAndQueueMapping extends TopicConfig { ++ .append(mappingDetail) ++ .toHashCode(); ++ } +++ +++ @Override +++ public String toString() { +++ String string = super.toString(); +++ if (StringUtils.isNotBlank(string)) { +++ string = string.substring(0, string.length() - 1) + ", mappingDetail=" + mappingDetail + "]"; +++ } +++ return string; +++ } ++ } ++-- ++2.32.0.windows.2 ++ ++ ++From 2113fa371b9c2bf7c512f8ad234e51c616f1362c Mon Sep 17 00:00:00 2001 ++From: guyinyou <36399867+guyinyou@users.noreply.github.com> ++Date: Fri, 13 Oct 2023 13:47:09 +0800 ++Subject: [PATCH 6/6] [ISSUE #7453] Fix the problem in constructing the ++ GetMessageResult (#7456) ++ ++* Fix the problem in constructing the GetMessageResult ++ ++* Optimize the initialization size of GetMessageResult ++--- ++ .../apache/rocketmq/broker/processor/PeekMessageProcessor.java | 3 +-- ++ .../apache/rocketmq/broker/processor/PopMessageProcessor.java | 3 +-- ++ 2 files changed, 2 insertions(+), 4 deletions(-) ++ ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java ++index a8358c4ff..e1e0e13e5 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PeekMessageProcessor.java ++@@ -129,8 +129,7 @@ public class PeekMessageProcessor implements NettyRequestProcessor { ++ } ++ int randomQ = random.nextInt(100); ++ int reviveQid = randomQ % this.brokerController.getBrokerConfig().getReviveQueueNum(); ++- int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg(); ++- GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg); +++ GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums()); ++ boolean needRetry = randomQ % 5 == 0; ++ long popTime = System.currentTimeMillis(); ++ long restNum = 0; ++diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java ++index 441f7de08..0d9bdf143 100644 ++--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java ++@@ -347,8 +347,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { ++ reviveQid = (int) Math.abs(ckMessageNumber.getAndIncrement() % this.brokerController.getBrokerConfig().getReviveQueueNum()); ++ } ++ ++- int commercialSizePerMsg = this.brokerController.getBrokerConfig().getCommercialSizePerMsg(); ++- GetMessageResult getMessageResult = new GetMessageResult(commercialSizePerMsg); +++ GetMessageResult getMessageResult = new GetMessageResult(requestHeader.getMaxMsgNums()); ++ ExpressionMessageFilter finalMessageFilter = messageFilter; ++ StringBuilder finalOrderCountInfo = orderCountInfo; ++ ++-- ++2.32.0.windows.2 ++ +diff --git a/pom.xml b/pom.xml +index 4202d4095..a3f7c2270 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -137,7 +137,7 @@ + 1.29.0-alpha + 2.0.6 + 2.20.29 +- 1.0.3 ++ 1.0.2 + 2.13.4.2 + + +@@ -713,7 +713,7 @@ + ${slf4j-api.version} + + +- io.github.aliyunmq ++ org.apache.rocketmq + rocketmq-rocksdb + ${rocksdb.version} + +diff --git a/store/pom.xml b/store/pom.xml +index e979030e8..e1e616123 100644 +--- a/store/pom.xml ++++ b/store/pom.xml +@@ -58,6 +58,10 @@ + com.google.guava + guava + ++ ++ commons-io ++ commons-io ++ + + + org.slf4j +diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +index f98e9a284..93102799b 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/CommitLog.java ++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLog.java +@@ -60,6 +60,8 @@ import org.apache.rocketmq.store.ha.HAService; + import org.apache.rocketmq.store.ha.autoswitch.AutoSwitchHAService; + import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.util.LibC; ++import org.rocksdb.RocksDBException; ++ + import sun.nio.ch.DirectBuffer; + + /** +@@ -299,8 +301,9 @@ public class CommitLog implements Swappable { + + /** + * When the normal exit, data recovery, all memory data have been flush ++ * @throws RocksDBException only in rocksdb mode + */ +- public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { ++ public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); + boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); + final List mappedFiles = this.mappedFileQueue.getMappedFiles(); +@@ -369,21 +372,22 @@ public class CommitLog implements Swappable { + this.setConfirmOffset(lastValidMsgPhyOffset); + } + +- this.mappedFileQueue.setFlushedWhere(processOffset); +- this.mappedFileQueue.setCommittedWhere(processOffset); +- this.mappedFileQueue.truncateDirtyFiles(processOffset); +- + // Clear ConsumeQueue redundant data + if (maxPhyOffsetOfConsumeQueue >= processOffset) { + log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); + this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); + } ++ ++ this.mappedFileQueue.setFlushedWhere(processOffset); ++ this.mappedFileQueue.setCommittedWhere(processOffset); ++ this.mappedFileQueue.truncateDirtyFiles(processOffset); + } else { + // Commitlog case files are deleted + log.warn("The commitlog files are deleted, and delete the consume queue files"); + this.mappedFileQueue.setFlushedWhere(0); + this.mappedFileQueue.setCommittedWhere(0); +- this.defaultMessageStore.destroyLogics(); ++ this.defaultMessageStore.getQueueStore().destroy(); ++ this.defaultMessageStore.getQueueStore().loadAfterDestroy(); + } + } + +@@ -626,8 +630,10 @@ public class CommitLog implements Swappable { + return -1; + } + +- @Deprecated +- public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { ++ /** ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + // recover by the minimum time stamp + boolean checkCRCOnRecover = this.defaultMessageStore.getMessageStoreConfig().isCheckCRCOnRecover(); + boolean checkDupInfo = this.defaultMessageStore.getMessageStoreConfig().isDuplicationEnable(); +@@ -705,6 +711,9 @@ public class CommitLog implements Swappable { + } + } + ++ // only for rocksdb mode ++ this.getMessageStore().finishCommitLogDispatch(); ++ + processOffset += mappedFileOffset; + if (this.defaultMessageStore.getBrokerConfig().isEnableControllerMode()) { + if (this.defaultMessageStore.getConfirmOffset() < this.defaultMessageStore.getMinPhyOffset()) { +@@ -717,22 +726,24 @@ public class CommitLog implements Swappable { + } else { + this.setConfirmOffset(lastValidMsgPhyOffset); + } +- this.mappedFileQueue.setFlushedWhere(processOffset); +- this.mappedFileQueue.setCommittedWhere(processOffset); +- this.mappedFileQueue.truncateDirtyFiles(processOffset); + + // Clear ConsumeQueue redundant data + if (maxPhyOffsetOfConsumeQueue >= processOffset) { + log.warn("maxPhyOffsetOfConsumeQueue({}) >= processOffset({}), truncate dirty logic files", maxPhyOffsetOfConsumeQueue, processOffset); + this.defaultMessageStore.truncateDirtyLogicFiles(processOffset); + } ++ ++ this.mappedFileQueue.setFlushedWhere(processOffset); ++ this.mappedFileQueue.setCommittedWhere(processOffset); ++ this.mappedFileQueue.truncateDirtyFiles(processOffset); + } + // Commitlog case files are deleted + else { + log.warn("The commitlog files are deleted, and delete the consume queue files"); + this.mappedFileQueue.setFlushedWhere(0); + this.mappedFileQueue.setCommittedWhere(0); +- this.defaultMessageStore.destroyLogics(); ++ this.defaultMessageStore.getQueueStore().destroy(); ++ this.defaultMessageStore.getQueueStore().loadAfterDestroy(); + } + } + +@@ -755,7 +766,7 @@ public class CommitLog implements Swappable { + this.getMessageStore().onCommitLogAppend(msg, result, commitLogFile); + } + +- private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) { ++ private boolean isMappedFileMatchedRecover(final MappedFile mappedFile) throws RocksDBException { + ByteBuffer byteBuffer = mappedFile.sliceByteBuffer(); + + int magicCode = byteBuffer.getInt(MessageDecoder.MESSAGE_MAGIC_CODE_POSITION); +@@ -763,28 +774,37 @@ public class CommitLog implements Swappable { + return false; + } + +- int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); +- int bornhostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; +- int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornhostLength; +- long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); +- if (0 == storeTimestamp) { +- return false; +- } +- +- if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() +- && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { +- if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { +- log.info("find check timestamp, {} {}", +- storeTimestamp, +- UtilAll.timeMillisToHumanString(storeTimestamp)); ++ if (this.defaultMessageStore.getMessageStoreConfig().isEnableRocksDBStore()) { ++ final long maxPhyOffsetInConsumeQueue = this.defaultMessageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(); ++ long phyOffset = byteBuffer.getLong(MessageDecoder.MESSAGE_PHYSIC_OFFSET_POSITION); ++ if (phyOffset <= maxPhyOffsetInConsumeQueue) { ++ log.info("find check. beginPhyOffset: {}, maxPhyOffsetInConsumeQueue: {}", phyOffset, maxPhyOffsetInConsumeQueue); + return true; + } + } else { +- if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { +- log.info("find check timestamp, {} {}", +- storeTimestamp, +- UtilAll.timeMillisToHumanString(storeTimestamp)); +- return true; ++ int sysFlag = byteBuffer.getInt(MessageDecoder.SYSFLAG_POSITION); ++ int bornHostLength = (sysFlag & MessageSysFlag.BORNHOST_V6_FLAG) == 0 ? 8 : 20; ++ int msgStoreTimePos = 4 + 4 + 4 + 4 + 4 + 8 + 8 + 4 + 8 + bornHostLength; ++ long storeTimestamp = byteBuffer.getLong(msgStoreTimePos); ++ if (0 == storeTimestamp) { ++ return false; ++ } ++ ++ if (this.defaultMessageStore.getMessageStoreConfig().isMessageIndexEnable() ++ && this.defaultMessageStore.getMessageStoreConfig().isMessageIndexSafe()) { ++ if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestampIndex()) { ++ log.info("find check timestamp, {} {}", ++ storeTimestamp, ++ UtilAll.timeMillisToHumanString(storeTimestamp)); ++ return true; ++ } ++ } else { ++ if (storeTimestamp <= this.defaultMessageStore.getStoreCheckpoint().getMinTimestamp()) { ++ log.info("find check timestamp, {} {}", ++ storeTimestamp, ++ UtilAll.timeMillisToHumanString(storeTimestamp)); ++ return true; ++ } + } + } + +@@ -958,8 +978,6 @@ public class CommitLog implements Swappable { + beginTimeInLock = 0; + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.MESSAGE_ILLEGAL, result)); + case UNKNOWN_ERROR: +- beginTimeInLock = 0; +- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); + default: + beginTimeInLock = 0; + return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); +@@ -974,6 +992,8 @@ public class CommitLog implements Swappable { + if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) { + this.defaultMessageStore.increaseOffset(msg, getMessageNum(msg)); + } ++ } catch (RocksDBException e) { ++ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); + } finally { + topicQueueLock.unlock(topicQueueKey); + } +@@ -997,7 +1017,7 @@ public class CommitLog implements Swappable { + + public CompletableFuture asyncPutMessages(final MessageExtBatch messageExtBatch) { + messageExtBatch.setStoreTimestamp(System.currentTimeMillis()); +- AppendMessageResult result; ++ AppendMessageResult result = null; + + StoreStatsService storeStatsService = this.defaultMessageStore.getStoreStatsService(); + +@@ -1133,7 +1153,9 @@ public class CommitLog implements Swappable { + if (AppendMessageStatus.PUT_OK.equals(result.getStatus())) { + this.defaultMessageStore.increaseOffset(messageExtBatch, (short) putMessageContext.getBatchSize()); + } +- } finally { ++ } catch (RocksDBException e) { ++ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, result)); ++ } finally { + topicQueueLock.unlock(topicQueueKey); + } + +diff --git a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java +index 9d6fa6ad9..f3a7b7c5c 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java ++++ b/store/src/main/java/org/apache/rocketmq/store/CommitLogDispatcher.java +@@ -17,6 +17,8 @@ + + package org.apache.rocketmq.store; + ++import org.rocksdb.RocksDBException; ++ + /** + * Dispatcher of commit log. + */ +@@ -25,6 +27,7 @@ public interface CommitLogDispatcher { + /** + * Dispatch messages from store to build consume queues, indexes, and filter data + * @param request dispatch message request ++ * @throws RocksDBException only in rocksdb mode + */ +- void dispatch(final DispatchRequest request); ++ void dispatch(final DispatchRequest request) throws RocksDBException; + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +index 56bee2af3..623509c8b 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ConsumeQueue.java +@@ -24,13 +24,12 @@ import java.util.Map; + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.MixAll; ++import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.attribute.CQType; + import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.message.MessageAccessor; + import org.apache.rocketmq.common.message.MessageConst; +-import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; +-import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.store.config.BrokerRole; +@@ -39,9 +38,9 @@ import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.queue.ConsumeQueueInterface; + import org.apache.rocketmq.store.queue.CqUnit; + import org.apache.rocketmq.store.queue.FileQueueLifeCycle; ++import org.apache.rocketmq.store.queue.MultiDispatch; + import org.apache.rocketmq.store.queue.QueueOffsetOperator; + import org.apache.rocketmq.store.queue.ReferredIterator; +-import org.apache.rocketmq.store.timer.TimerMessageStore; + + public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); +@@ -703,7 +702,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(request.getStoreTimestamp()); + } + this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(request.getStoreTimestamp()); +- if (checkMultiDispatchQueue(request)) { ++ if (MultiDispatch.checkMultiDispatchQueue(this.messageStore.getMessageStoreConfig(), request)) { + multiDispatchLmqQueue(request, maxRetries); + } + return; +@@ -725,25 +724,6 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + this.messageStore.getRunningFlags().makeLogicsQueueError(); + } + +- private boolean checkMultiDispatchQueue(DispatchRequest dispatchRequest) { +- if (!this.messageStore.getMessageStoreConfig().isEnableMultiDispatch() +- || dispatchRequest.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) +- || dispatchRequest.getTopic().equals(TimerMessageStore.TIMER_TOPIC) +- || dispatchRequest.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC)) { +- return false; +- } +- Map prop = dispatchRequest.getPropertiesMap(); +- if (prop == null || prop.isEmpty()) { +- return false; +- } +- String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); +- String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); +- if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { +- return false; +- } +- return true; +- } +- + private void multiDispatchLmqQueue(DispatchRequest request, int maxRetries) { + Map prop = request.getPropertiesMap(); + String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); +@@ -765,9 +745,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + queueId = 0; + } + doDispatchLmqQueue(request, maxRetries, queueName, queueOffset, queueId); +- + } +- return; + } + + private void doDispatchLmqQueue(DispatchRequest request, int maxRetries, String queueName, long queueOffset, +@@ -802,7 +780,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), + // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. +- if (!isNeedHandleMultiDispatch(msg)) { ++ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { + return; + } + String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); +@@ -812,14 +790,14 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + Long[] queueOffsets = new Long[queues.length]; + for (int i = 0; i < queues.length; i++) { +- String key = queueKey(queues[i], msg); +- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { ++ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { ++ String key = MultiDispatch.lmqQueueKey(queues[i]); + queueOffsets[i] = queueOffsetOperator.getLmqOffset(key); + } + } + MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, + StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); +- removeWaitStorePropertyString(msg); ++ msg.removeWaitStorePropertyString(); + } + + @Override +@@ -830,7 +808,7 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + + // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), + // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. +- if (!isNeedHandleMultiDispatch(msg)) { ++ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { + return; + } + String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); +@@ -839,45 +817,13 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + } + String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); + for (int i = 0; i < queues.length; i++) { +- String key = queueKey(queues[i], msg); +- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(key)) { ++ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { ++ String key = MultiDispatch.lmqQueueKey(queues[i]); + queueOffsetOperator.increaseLmqOffset(key, (short) 1); + } + } + } + +- public boolean isNeedHandleMultiDispatch(MessageExtBrokerInner msg) { +- return messageStore.getMessageStoreConfig().isEnableMultiDispatch() +- && !msg.getTopic().startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) +- && !msg.getTopic().equals(TimerMessageStore.TIMER_TOPIC) +- && !msg.getTopic().equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); +- } +- +- public String queueKey(String queueName, MessageExtBrokerInner msgInner) { +- StringBuilder keyBuilder = new StringBuilder(); +- keyBuilder.append(queueName); +- keyBuilder.append('-'); +- int queueId = msgInner.getQueueId(); +- if (messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queueName)) { +- queueId = 0; +- } +- keyBuilder.append(queueId); +- return keyBuilder.toString(); +- } +- +- private void removeWaitStorePropertyString(MessageExtBrokerInner msgInner) { +- if (msgInner.getProperties().containsKey(MessageConst.PROPERTY_WAIT_STORE_MSG_OK)) { +- // There is no need to store "WAIT=true", remove it from propertiesString to save 9 bytes for each message. +- // It works for most case. In some cases msgInner.setPropertiesString invoked later and replace it. +- String waitStoreMsgOKValue = msgInner.getProperties().remove(MessageConst.PROPERTY_WAIT_STORE_MSG_OK); +- msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); +- // Reput to properties, since msgInner.isWaitStoreMsgOK() will be invoked later +- msgInner.getProperties().put(MessageConst.PROPERTY_WAIT_STORE_MSG_OK, waitStoreMsgOKValue); +- } else { +- msgInner.setPropertiesString(MessageDecoder.messageProperties2String(msgInner.getProperties())); +- } +- } +- + private boolean putMessagePositionInfo(final long offset, final int size, final long tagsCode, + final long cqOffset) { + +@@ -965,6 +911,11 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + return new ConsumeQueueIterator(sbr); + } + ++ @Override ++ public ReferredIterator iterateFrom(long startIndex, int count) { ++ return iterateFrom(startIndex); ++ } ++ + @Override + public CqUnit get(long offset) { + ReferredIterator it = iterateFrom(offset); +@@ -974,6 +925,20 @@ public class ConsumeQueue implements ConsumeQueueInterface, FileQueueLifeCycle { + return it.nextAndRelease(); + } + ++ @Override ++ public Pair getCqUnitAndStoreTime(long index) { ++ CqUnit cqUnit = get(index); ++ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); ++ return new Pair<>(cqUnit, messageStoreTime); ++ } ++ ++ @Override ++ public Pair getEarliestUnitAndStoreTime() { ++ CqUnit cqUnit = getEarliestUnit(); ++ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); ++ return new Pair<>(cqUnit, messageStoreTime); ++ } ++ + @Override + public CqUnit getEarliestUnit() { + /** +diff --git a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +index 02ea47f13..99a54e2d7 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/DefaultMessageStore.java +@@ -105,32 +105,35 @@ import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; + import org.apache.rocketmq.store.queue.ConsumeQueueInterface; + import org.apache.rocketmq.store.queue.ConsumeQueueStore; ++import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; + import org.apache.rocketmq.store.queue.CqUnit; + import org.apache.rocketmq.store.queue.ReferredIterator; + import org.apache.rocketmq.store.stats.BrokerStatsManager; + import org.apache.rocketmq.store.timer.TimerMessageStore; + import org.apache.rocketmq.store.util.PerfCounter; ++import org.rocksdb.RocksDBException; + + public class DefaultMessageStore implements MessageStore { +- private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ protected static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ protected static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); + + public final PerfCounter.Ticks perfs = new PerfCounter.Ticks(LOGGER); + + private final MessageStoreConfig messageStoreConfig; + // CommitLog +- private final CommitLog commitLog; ++ protected final CommitLog commitLog; + +- private final ConsumeQueueStore consumeQueueStore; ++ protected final ConsumeQueueStoreInterface consumeQueueStore; + + private final FlushConsumeQueueService flushConsumeQueueService; + +- private final CleanCommitLogService cleanCommitLogService; ++ protected final CleanCommitLogService cleanCommitLogService; + + private final CleanConsumeQueueService cleanConsumeQueueService; + + private final CorrectLogicOffsetService correctLogicOffsetService; + +- private final IndexService indexService; ++ protected final IndexService indexService; + + private final AllocateMappedFileService allocateMappedFileService; + +@@ -147,7 +150,7 @@ public class DefaultMessageStore implements MessageStore { + + private final TransientStorePool transientStorePool; + +- private final RunningFlags runningFlags = new RunningFlags(); ++ protected final RunningFlags runningFlags = new RunningFlags(); + private final SystemClock systemClock = new SystemClock(); + + private final ScheduledExecutorService scheduledExecutorService; +@@ -156,6 +159,7 @@ public class DefaultMessageStore implements MessageStore { + private final BrokerConfig brokerConfig; + + private volatile boolean shutdown = true; ++ protected boolean notifyMessageArriveInBatch = false; + + private StoreCheckpoint storeCheckpoint; + private TimerMessageStore timerMessageStore; +@@ -182,7 +186,7 @@ public class DefaultMessageStore implements MessageStore { + + private volatile long brokerInitMaxOffset = -1L; + +- protected List putMessageHookList = new ArrayList<>(); ++ private List putMessageHookList = new ArrayList<>(); + + private SendMessageBackHook sendMessageBackHook; + +@@ -222,12 +226,12 @@ public class DefaultMessageStore implements MessageStore { + this.commitLog = new CommitLog(this); + } + +- this.consumeQueueStore = new ConsumeQueueStore(this, this.messageStoreConfig); ++ this.consumeQueueStore = createConsumeQueueStore(); + +- this.flushConsumeQueueService = new FlushConsumeQueueService(); ++ this.flushConsumeQueueService = createFlushConsumeQueueService(); + this.cleanCommitLogService = new CleanCommitLogService(); +- this.cleanConsumeQueueService = new CleanConsumeQueueService(); +- this.correctLogicOffsetService = new CorrectLogicOffsetService(); ++ this.cleanConsumeQueueService = createCleanConsumeQueueService(); ++ this.correctLogicOffsetService = createCorrectLogicOffsetService(); + this.storeStatsService = new StoreStatsService(getBrokerIdentity()); + this.indexService = new IndexService(this); + +@@ -273,6 +277,22 @@ public class DefaultMessageStore implements MessageStore { + parseDelayLevel(); + } + ++ public ConsumeQueueStoreInterface createConsumeQueueStore() { ++ return new ConsumeQueueStore(this); ++ } ++ ++ public CleanConsumeQueueService createCleanConsumeQueueService() { ++ return new CleanConsumeQueueService(); ++ } ++ ++ public FlushConsumeQueueService createFlushConsumeQueueService() { ++ return new FlushConsumeQueueService(); ++ } ++ ++ public CorrectLogicOffsetService createCorrectLogicOffsetService() { ++ return new CorrectLogicOffsetService(); ++ } ++ + public boolean parseDelayLevel() { + HashMap timeUnitTable = new HashMap<>(); + timeUnitTable.put("s", 1000L); +@@ -305,7 +325,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public void truncateDirtyLogicFiles(long phyOffset) { ++ public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException { + this.consumeQueueStore.truncateDirty(phyOffset); + } + +@@ -393,6 +413,7 @@ public class DefaultMessageStore implements MessageStore { + + this.flushConsumeQueueService.start(); + this.commitLog.start(); ++ this.consumeQueueStore.start(); + this.storeStatsService.start(); + + if (this.haService != null) { +@@ -481,6 +502,7 @@ public class DefaultMessageStore implements MessageStore { + this.storeStatsService.shutdown(); + this.commitLog.shutdown(); + this.reputMessageService.shutdown(); ++ this.consumeQueueStore.shutdown(); + // dispatch-related services must be shut down after reputMessageService + this.indexService.shutdown(); + if (this.compactionService != null) { +@@ -515,7 +537,7 @@ public class DefaultMessageStore implements MessageStore { + + @Override + public void destroy() { +- this.destroyLogics(); ++ this.consumeQueueStore.destroy(); + this.commitLog.destroy(); + this.indexService.destroy(); + this.deleteFile(StorePathConfigHelper.getAbortFile(this.messageStoreConfig.getStorePathRootDir())); +@@ -541,11 +563,6 @@ public class DefaultMessageStore implements MessageStore { + return commitLogSize + consumeQueueSize + indexFileSize; + } + +- @Override +- public void destroyLogics() { +- this.consumeQueueStore.destroy(); +- } +- + @Override + public CompletableFuture asyncPutMessage(MessageExtBrokerInner msg) { + +@@ -687,7 +704,7 @@ public class DefaultMessageStore implements MessageStore { + return commitLog; + } + +- public void truncateDirtyFiles(long offsetToTruncate) { ++ public void truncateDirtyFiles(long offsetToTruncate) throws RocksDBException { + + LOGGER.info("truncate dirty files to {}", offsetToTruncate); + +@@ -700,12 +717,12 @@ public class DefaultMessageStore implements MessageStore { + + long oldReputFromOffset = this.reputMessageService.getReputFromOffset(); + +- // truncate commitLog +- this.commitLog.truncateDirtyFiles(offsetToTruncate); +- + // truncate consume queue + this.truncateDirtyLogicFiles(offsetToTruncate); + ++ // truncate commitLog ++ this.commitLog.truncateDirtyFiles(offsetToTruncate); ++ + this.recoverTopicQueueTable(); + + if (!messageStoreConfig.isEnableBuildConsumeQueueConcurrently()) { +@@ -723,7 +740,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public boolean truncateFiles(long offsetToTruncate) { ++ public boolean truncateFiles(long offsetToTruncate) throws RocksDBException { + if (offsetToTruncate >= this.getMaxPhyOffset()) { + LOGGER.info("no need to truncate files, truncate offset is {}, max physical offset is {}", offsetToTruncate, this.getMaxPhyOffset()); + return true; +@@ -825,17 +842,19 @@ public class DefaultMessageStore implements MessageStore { + while (getResult.getBufferTotalSize() <= 0 + && nextBeginOffset < maxOffset + && cqFileNum++ < this.messageStoreConfig.getTravelCqFileNumWhenGetMessage()) { +- ReferredIterator bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset); +- +- if (bufferConsumeQueue == null) { +- status = GetMessageStatus.OFFSET_FOUND_NULL; +- nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset)); +- LOGGER.warn("consumer request topic: " + topic + "offset: " + offset + " minOffset: " + minOffset + " maxOffset: " +- + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset); +- break; +- } ++ ReferredIterator bufferConsumeQueue = null; + + try { ++ bufferConsumeQueue = consumeQueue.iterateFrom(nextBeginOffset, maxMsgNums); ++ ++ if (bufferConsumeQueue == null) { ++ status = GetMessageStatus.OFFSET_FOUND_NULL; ++ nextBeginOffset = nextOffsetCorrection(nextBeginOffset, this.consumeQueueStore.rollNextFile(consumeQueue, nextBeginOffset)); ++ LOGGER.warn("consumer request topic: " + topic + ", offset: " + offset + ", minOffset: " + minOffset + ", maxOffset: " ++ + maxOffset + ", but access logic queue failed. Correct nextBeginOffset to " + nextBeginOffset); ++ break; ++ } ++ + long nextPhyFileStartOffset = Long.MIN_VALUE; + while (bufferConsumeQueue.hasNext() + && nextBeginOffset < maxOffset) { +@@ -905,8 +924,13 @@ public class DefaultMessageStore implements MessageStore { + status = GetMessageStatus.FOUND; + nextPhyFileStartOffset = Long.MIN_VALUE; + } ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getMessage Failed. cid: {}, topic: {}, queueId: {}, offset: {}, minOffset: {}, maxOffset: {}, {}", ++ group, topic, queueId, offset, minOffset, maxOffset, e.getMessage()); + } finally { +- bufferConsumeQueue.release(); ++ if (bufferConsumeQueue != null) { ++ bufferConsumeQueue.release(); ++ } + } + } + +@@ -975,12 +999,12 @@ public class DefaultMessageStore implements MessageStore { + + @Override + public long getMinOffsetInQueue(String topic, int queueId) { +- ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); +- if (logic != null) { +- return logic.getMinOffsetInQueue(); ++ try { ++ return this.consumeQueueStore.getMinOffsetInQueue(topic, queueId); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getMinOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e); ++ return -1; + } +- +- return -1; + } + + @Override +@@ -997,38 +1021,27 @@ public class DefaultMessageStore implements MessageStore { + public long getCommitLogOffsetInQueue(String topic, int queueId, long consumeQueueOffset) { + ConsumeQueueInterface consumeQueue = findConsumeQueue(topic, queueId); + if (consumeQueue != null) { +- +- ReferredIterator bufferConsumeQueue = consumeQueue.iterateFrom(consumeQueueOffset); +- if (bufferConsumeQueue != null) { +- try { +- if (bufferConsumeQueue.hasNext()) { +- long offsetPy = bufferConsumeQueue.next().getPos(); +- return offsetPy; +- } +- } finally { +- bufferConsumeQueue.release(); +- } ++ CqUnit cqUnit = consumeQueue.get(consumeQueueOffset); ++ if (cqUnit != null) { ++ return cqUnit.getPos(); + } + } +- + return 0; + } + + @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp) { +- return getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER); ++ return this.getOffsetInQueueByTime(topic, queueId, timestamp, BoundaryType.LOWER); + } + ++ @Override + public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) { +- ConsumeQueueInterface logic = this.findConsumeQueue(topic, queueId); +- if (logic != null) { +- long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType); +- // Make sure the result offset is in valid range. +- resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue()); +- resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue()); +- return resultOffset; ++ try { ++ return this.consumeQueueStore.getOffsetInQueueByTime(topic, queueId, timestamp, boundaryType); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getOffsetInQueueByTime Failed. topic: {}, queueId: {}, timestamp: {} boundaryType: {}, {}", ++ topic, queueId, timestamp, boundaryType, e.getMessage()); + } +- + return 0; + } + +@@ -1088,6 +1101,10 @@ public class DefaultMessageStore implements MessageStore { + return StorePathConfigHelper.getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()); + } + ++ public MessageArrivingListener getMessageArrivingListener() { ++ return messageArrivingListener; ++ } ++ + @Override + public HashMap getRuntimeInfo() { + HashMap result = this.storeStatsService.getRuntimeInfo(); +@@ -1121,7 +1138,6 @@ public class DefaultMessageStore implements MessageStore { + return this.commitLog.getMaxOffset(); + } + +- + @Override + public long getMinPhyOffset() { + return this.commitLog.getMinOffset(); +@@ -1141,7 +1157,10 @@ public class DefaultMessageStore implements MessageStore { + public long getEarliestMessageTime(String topic, int queueId) { + ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { +- return getStoreTime(logicQueue.getEarliestUnit()); ++ Pair pair = logicQueue.getEarliestUnitAndStoreTime(); ++ if (pair != null && pair.getObject2() != null) { ++ return pair.getObject2(); ++ } + } + + return -1; +@@ -1152,19 +1171,6 @@ public class DefaultMessageStore implements MessageStore { + return CompletableFuture.completedFuture(getEarliestMessageTime(topic, queueId)); + } + +- protected long getStoreTime(CqUnit result) { +- if (result != null) { +- try { +- final long phyOffset = result.getPos(); +- final int size = result.getSize(); +- long storeTime = this.getCommitLog().pickupStoreTimestamp(phyOffset, size); +- return storeTime; +- } catch (Exception e) { +- } +- } +- return -1; +- } +- + @Override + public long getEarliestMessageTime() { + long minPhyOffset = this.getMinPhyOffset(); +@@ -1179,13 +1185,16 @@ public class DefaultMessageStore implements MessageStore { + public long getMessageStoreTimeStamp(String topic, int queueId, long consumeQueueOffset) { + ConsumeQueueInterface logicQueue = this.findConsumeQueue(topic, queueId); + if (logicQueue != null) { +- return getStoreTime(logicQueue.get(consumeQueueOffset)); ++ Pair pair = logicQueue.getCqUnitAndStoreTime(consumeQueueOffset); ++ if (pair != null && pair.getObject2() != null) { ++ return pair.getObject2(); ++ } + } +- + return -1; + } + +- @Override public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, ++ @Override ++ public CompletableFuture getMessageStoreTimeStampAsync(String topic, int queueId, + long consumeQueueOffset) { + return CompletableFuture.completedFuture(getMessageStoreTimeStamp(topic, queueId, consumeQueueOffset)); + } +@@ -1354,6 +1363,7 @@ public class DefaultMessageStore implements MessageStore { + * If offset table is cleaned, and old messages are dispatching after the old consume queue is cleaned, + * consume queue will be created with old offset, then later message with new offset table can not be + * dispatched to consume queue. ++ * @throws RocksDBException only in rocksdb mode + */ + @Override + public int deleteTopics(final Set deleteTopics) { +@@ -1363,17 +1373,19 @@ public class DefaultMessageStore implements MessageStore { + + int deleteCount = 0; + for (String topic : deleteTopics) { +- ConcurrentMap queueTable = +- this.consumeQueueStore.getConsumeQueueTable().get(topic); ++ ConcurrentMap queueTable = this.consumeQueueStore.findConsumeQueueMap(topic); + + if (queueTable == null || queueTable.isEmpty()) { + continue; + } + + for (ConsumeQueueInterface cq : queueTable.values()) { +- this.consumeQueueStore.destroy(cq); +- LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", +- cq.getTopic(), cq.getQueueId()); ++ try { ++ this.consumeQueueStore.destroy(cq); ++ } catch (RocksDBException e) { ++ LOGGER.error("DeleteTopic: ConsumeQueue cleans error!, topic={}, queueId={}", cq.getTopic(), cq.getQueueId(), e); ++ } ++ LOGGER.info("DeleteTopic: ConsumeQueue has been cleaned, topic={}, queueId={}", cq.getTopic(), cq.getQueueId()); + this.consumeQueueStore.removeTopicQueueTable(cq.getTopic(), cq.getQueueId()); + } + +@@ -1852,14 +1864,18 @@ public class DefaultMessageStore implements MessageStore { + return file.exists(); + } + +- private void recover(final boolean lastExitOK) { +- boolean recoverConcurrently = this.brokerConfig.isRecoverConcurrently(); ++ private boolean isRecoverConcurrently() { ++ return this.brokerConfig.isRecoverConcurrently() && !this.messageStoreConfig.isEnableRocksDBStore(); ++ } ++ ++ private void recover(final boolean lastExitOK) throws RocksDBException { ++ boolean recoverConcurrently = this.isRecoverConcurrently(); + LOGGER.info("message store recover mode: {}", recoverConcurrently ? "concurrent" : "normal"); + + // recover consume queue + long recoverConsumeQueueStart = System.currentTimeMillis(); + this.recoverConsumeQueue(); +- long maxPhyOffsetOfConsumeQueue = this.getMaxOffsetInConsumeQueue(); ++ long maxPhyOffsetOfConsumeQueue = this.consumeQueueStore.getMaxPhyOffsetInConsumeQueue(); + long recoverConsumeQueueEnd = System.currentTimeMillis(); + + // recover commitlog +@@ -1894,23 +1910,25 @@ public class DefaultMessageStore implements MessageStore { + return messageStoreConfig; + } + ++ @Override ++ public void finishCommitLogDispatch() { ++ // ignore ++ } ++ + @Override + public TransientStorePool getTransientStorePool() { + return transientStorePool; + } + + private void recoverConsumeQueue() { +- if (!this.brokerConfig.isRecoverConcurrently()) { ++ if (!this.isRecoverConcurrently()) { + this.consumeQueueStore.recover(); + } else { + this.consumeQueueStore.recoverConcurrently(); + } + } + +- private long getMaxOffsetInConsumeQueue() { +- return this.consumeQueueStore.getMaxOffsetInConsumeQueue(); +- } +- ++ @Override + public void recoverTopicQueueTable() { + long minPhyOffset = this.commitLog.getMinOffset(); + this.consumeQueueStore.recoverOffsetTable(minPhyOffset); +@@ -1949,13 +1967,17 @@ public class DefaultMessageStore implements MessageStore { + return runningFlags; + } + +- public void doDispatch(DispatchRequest req) { ++ public void doDispatch(DispatchRequest req) throws RocksDBException { + for (CommitLogDispatcher dispatcher : this.dispatcherList) { + dispatcher.dispatch(req); + } + } + +- public void putMessagePositionInfo(DispatchRequest dispatchRequest) { ++ /** ++ * @param dispatchRequest ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ protected void putMessagePositionInfo(DispatchRequest dispatchRequest) throws RocksDBException { + this.consumeQueueStore.putMessagePositionInfoWrapper(dispatchRequest); + } + +@@ -2054,7 +2076,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public ConsumeQueueStore getQueueStore() { ++ public ConsumeQueueStoreInterface getQueueStore() { + return consumeQueueStore; + } + +@@ -2065,7 +2087,7 @@ public class DefaultMessageStore implements MessageStore { + + @Override + public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, +- boolean isRecover, boolean isFileEnd) { ++ boolean isRecover, boolean isFileEnd) throws RocksDBException { + if (doDispatch && !isFileEnd) { + this.doDispatch(dispatchRequest); + } +@@ -2082,7 +2104,7 @@ public class DefaultMessageStore implements MessageStore { + } + + @Override +- public void assignOffset(MessageExtBrokerInner msg) { ++ public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException { + final int tranType = MessageSysFlag.getTransactionValue(msg.getSysFlag()); + + if (tranType == MessageSysFlag.TRANSACTION_NOT_TYPE || tranType == MessageSysFlag.TRANSACTION_COMMIT_TYPE) { +@@ -2127,12 +2149,12 @@ public class DefaultMessageStore implements MessageStore { + class CommitLogDispatcherBuildConsumeQueue implements CommitLogDispatcher { + + @Override +- public void dispatch(DispatchRequest request) { ++ public void dispatch(DispatchRequest request) throws RocksDBException { + final int tranType = MessageSysFlag.getTransactionValue(request.getSysFlag()); + switch (tranType) { + case MessageSysFlag.TRANSACTION_NOT_TYPE: + case MessageSysFlag.TRANSACTION_COMMIT_TYPE: +- DefaultMessageStore.this.putMessagePositionInfo(request); ++ putMessagePositionInfo(request); + break; + case MessageSysFlag.TRANSACTION_PREPARED_TYPE: + case MessageSysFlag.TRANSACTION_ROLLBACK_TYPE: +@@ -2278,7 +2300,7 @@ public class DefaultMessageStore implements MessageStore { + return DefaultMessageStore.this.brokerConfig.getIdentifier() + CleanCommitLogService.class.getSimpleName(); + } + +- private boolean isTimeToDelete() { ++ protected boolean isTimeToDelete() { + String when = DefaultMessageStore.this.getMessageStoreConfig().getDeleteWhen(); + if (UtilAll.isItTimeToDo(when)) { + DefaultMessageStore.LOGGER.info("it's time to reclaim disk space, " + when); +@@ -2436,7 +2458,7 @@ public class DefaultMessageStore implements MessageStore { + } + + class CleanConsumeQueueService { +- private long lastPhysicalMinOffset = 0; ++ protected long lastPhysicalMinOffset = 0; + + public void run() { + try { +@@ -2446,7 +2468,7 @@ public class DefaultMessageStore implements MessageStore { + } + } + +- private void deleteExpiredFiles() { ++ protected void deleteExpiredFiles() { + int deleteLogicsFilesInterval = DefaultMessageStore.this.getMessageStoreConfig().getDeleteConsumeQueueFilesInterval(); + + long minOffset = DefaultMessageStore.this.commitLog.getMinOffset(); +@@ -2551,7 +2573,7 @@ public class DefaultMessageStore implements MessageStore { + + if (cqUnit.getPos() >= minPhyOffset) { + +- // Normal case, do not need correct. ++ // Normal case, do not need to correct. + return false; + } + } +@@ -2741,6 +2763,18 @@ public class DefaultMessageStore implements MessageStore { + + } + ++ @Override ++ public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { ++ if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() ++ && DefaultMessageStore.this.messageArrivingListener != null) { ++ DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), ++ dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, ++ dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), ++ dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); ++ DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest); ++ } ++ } ++ + class ReputMessageService extends ServiceThread { + + protected volatile long reputFromOffset = 0; +@@ -2810,13 +2844,8 @@ public class DefaultMessageStore implements MessageStore { + if (size > 0) { + DefaultMessageStore.this.doDispatch(dispatchRequest); + +- if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() +- && DefaultMessageStore.this.messageArrivingListener != null) { +- DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), +- dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, +- dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), +- dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); +- notifyMessageArrive4MultiQueue(dispatchRequest); ++ if (!notifyMessageArriveInBatch) { ++ notifyMessageArriveIfNecessary(dispatchRequest); + } + + this.reputFromOffset += size; +@@ -2850,9 +2879,14 @@ public class DefaultMessageStore implements MessageStore { + } + } + } ++ } catch (RocksDBException e) { ++ ERROR_LOG.info("dispatch message to cq exception. reputFromOffset: {}", this.reputFromOffset, e); ++ return; + } finally { + result.release(); + } ++ ++ finishCommitLogDispatch(); + } + } + +@@ -2989,7 +3023,7 @@ public class DefaultMessageStore implements MessageStore { + // dispatchRequestsList:[ + // {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}, + // {dispatchRequests:[{dispatchRequest}, {dispatchRequest}]}] +- private void dispatch() { ++ private void dispatch() throws Exception { + dispatchRequestsList.clear(); + dispatchRequestOrderlyQueue.get(dispatchRequestsList); + if (!dispatchRequestsList.isEmpty()) { +@@ -2997,21 +3031,15 @@ public class DefaultMessageStore implements MessageStore { + for (DispatchRequest dispatchRequest : dispatchRequests) { + DefaultMessageStore.this.doDispatch(dispatchRequest); + // wake up long-polling +- if (DefaultMessageStore.this.brokerConfig.isLongPollingEnable() +- && DefaultMessageStore.this.messageArrivingListener != null) { +- DefaultMessageStore.this.messageArrivingListener.arriving(dispatchRequest.getTopic(), +- dispatchRequest.getQueueId(), dispatchRequest.getConsumeQueueOffset() + 1, +- dispatchRequest.getTagsCode(), dispatchRequest.getStoreTimestamp(), +- dispatchRequest.getBitMap(), dispatchRequest.getPropertiesMap()); +- DefaultMessageStore.this.reputMessageService.notifyMessageArrive4MultiQueue(dispatchRequest); +- } ++ DefaultMessageStore.this.notifyMessageArriveIfNecessary(dispatchRequest); ++ + if (!DefaultMessageStore.this.getMessageStoreConfig().isDuplicationEnable() && +- DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { ++ DefaultMessageStore.this.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE) { + DefaultMessageStore.this.storeStatsService +- .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1); ++ .getSinglePutMessageTopicTimesTotal(dispatchRequest.getTopic()).add(1); + DefaultMessageStore.this.storeStatsService +- .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) +- .add(dispatchRequest.getMsgSize()); ++ .getSinglePutMessageTopicSizeTotal(dispatchRequest.getTopic()) ++ .add(dispatchRequest.getMsgSize()); + } + } + } +@@ -3079,7 +3107,7 @@ public class DefaultMessageStore implements MessageStore { + public void doReput() { + if (this.reputFromOffset < DefaultMessageStore.this.commitLog.getMinOffset()) { + LOGGER.warn("The reputFromOffset={} is smaller than minPyOffset={}, this usually indicate that the dispatch behind too much and the commitlog has expired.", +- this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); ++ this.reputFromOffset, DefaultMessageStore.this.commitLog.getMinOffset()); + this.reputFromOffset = DefaultMessageStore.this.commitLog.getMinOffset(); + } + for (boolean doNext = true; this.isCommitLogAvailable() && doNext; ) { +@@ -3138,6 +3166,9 @@ public class DefaultMessageStore implements MessageStore { + result.release(); + } + } ++ ++ // only for rocksdb mode ++ finishCommitLogDispatch(); + } + + /** +@@ -3180,8 +3211,8 @@ public class DefaultMessageStore implements MessageStore { + + if (this.isCommitLogAvailable()) { + LOGGER.warn("shutdown concurrentReputMessageService, but CommitLog have not finish to be dispatched, CommitLog max" + +- " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(), +- this.reputFromOffset); ++ " offset={}, reputFromOffset={}", DefaultMessageStore.this.commitLog.getMaxOffset(), ++ this.reputFromOffset); + } + + this.mainBatchDispatchRequestService.shutdown(); +diff --git a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +index 989cbbe31..814c6d1bf 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/MessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/MessageStore.java +@@ -16,10 +16,6 @@ + */ + package org.apache.rocketmq.store; + +-import io.opentelemetry.api.common.AttributesBuilder; +-import io.opentelemetry.api.metrics.Meter; +-import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.nio.ByteBuffer; + import java.util.HashMap; + import java.util.LinkedList; +@@ -40,10 +36,15 @@ import org.apache.rocketmq.store.hook.PutMessageHook; + import org.apache.rocketmq.store.hook.SendMessageBackHook; + import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +-import org.apache.rocketmq.store.queue.ConsumeQueueStore; ++import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; + import org.apache.rocketmq.store.stats.BrokerStatsManager; + import org.apache.rocketmq.store.timer.TimerMessageStore; + import org.apache.rocketmq.store.util.PerfCounter; ++import org.rocksdb.RocksDBException; ++import io.opentelemetry.api.common.AttributesBuilder; ++import io.opentelemetry.api.metrics.Meter; ++import io.opentelemetry.sdk.metrics.InstrumentSelector; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + + /** + * This class defines contracting interfaces to implement, allowing third-party vendor to use customized message store. +@@ -545,7 +546,7 @@ public interface MessageStore { + void setConfirmOffset(long phyOffset); + + /** +- * Check if the operation system page cache is busy or not. ++ * Check if the operating system page cache is busy or not. + * + * @return true if the OS page cache is busy; false otherwise. + */ +@@ -620,9 +621,18 @@ public interface MessageStore { + * @param commitLogFile commit log file + * @param isRecover is from recover process + * @param isFileEnd if the dispatch request represents 'file end' ++ * @throws RocksDBException only in rocksdb mode + */ + void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, +- boolean isRecover, boolean isFileEnd); ++ boolean isRecover, boolean isFileEnd) throws RocksDBException; ++ ++ /** ++ * Only used in rocksdb mode, because we build consumeQueue in batch(default 16 dispatchRequests) ++ * It will be triggered in two cases: ++ * @see org.apache.rocketmq.store.DefaultMessageStore.ReputMessageService#doReput ++ * @see CommitLog#recoverAbnormally ++ */ ++ void finishCommitLogDispatch(); + + /** + * Get the message store config +@@ -691,13 +701,9 @@ public interface MessageStore { + * Truncate dirty logic files + * + * @param phyOffset physical offset ++ * @throws RocksDBException only in rocksdb mode + */ +- void truncateDirtyLogicFiles(long phyOffset); +- +- /** +- * Destroy logics files +- */ +- void destroyLogics(); ++ void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException; + + /** + * Unlock mappedFile +@@ -718,7 +724,7 @@ public interface MessageStore { + * + * @return the queue store + */ +- ConsumeQueueStore getQueueStore(); ++ ConsumeQueueStoreInterface getQueueStore(); + + /** + * If 'sync disk flush' is configured in this message store +@@ -739,8 +745,9 @@ public interface MessageStore { + * yourself. + * + * @param msg message ++ * @throws RocksDBException + */ +- void assignOffset(MessageExtBrokerInner msg); ++ void assignOffset(MessageExtBrokerInner msg) throws RocksDBException; + + /** + * Increase queue offset in memory table. If there is a race condition, you need to lock/unlock this method +@@ -835,14 +842,15 @@ public interface MessageStore { + * + * @param offsetToTruncate offset to truncate + * @return true if truncate succeed, false otherwise ++ * @throws RocksDBException only in rocksdb mode + */ +- boolean truncateFiles(long offsetToTruncate); ++ boolean truncateFiles(long offsetToTruncate) throws RocksDBException; + + /** +- * Check if the offset is align with one message. ++ * Check if the offset is aligned with one message. + * + * @param offset offset to check +- * @return true if align, false otherwise ++ * @return true if aligned, false otherwise + */ + boolean isOffsetAligned(long offset); + +@@ -971,4 +979,14 @@ public interface MessageStore { + * @param attributesBuilderSupplier metrics attributes builder + */ + void initMetrics(Meter meter, Supplier attributesBuilderSupplier); ++ ++ /** ++ * Recover topic queue table ++ */ ++ void recoverTopicQueueTable(); ++ ++ /** ++ * notify message arrive if necessary ++ */ ++ void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest); + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +new file mode 100644 +index 000000000..87ccb5474 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/RocksDBMessageStore.java +@@ -0,0 +1,169 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store; ++ ++import java.io.IOException; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++ ++import org.apache.rocketmq.common.BrokerConfig; ++import org.apache.rocketmq.common.TopicConfig; ++import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.apache.rocketmq.store.config.StorePathConfigHelper; ++import org.apache.rocketmq.store.queue.ConsumeQueueInterface; ++import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; ++import org.apache.rocketmq.store.queue.RocksDBConsumeQueue; ++import org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore; ++import org.apache.rocketmq.store.stats.BrokerStatsManager; ++import org.rocksdb.RocksDBException; ++ ++public class RocksDBMessageStore extends DefaultMessageStore { ++ ++ public RocksDBMessageStore(final MessageStoreConfig messageStoreConfig, final BrokerStatsManager brokerStatsManager, ++ final MessageArrivingListener messageArrivingListener, final BrokerConfig brokerConfig, final ConcurrentMap topicConfigTable) throws ++ IOException { ++ super(messageStoreConfig, brokerStatsManager, messageArrivingListener, brokerConfig, topicConfigTable); ++ notifyMessageArriveInBatch = true; ++ } ++ ++ @Override ++ public ConsumeQueueStoreInterface createConsumeQueueStore() { ++ return new RocksDBConsumeQueueStore(this); ++ } ++ ++ @Override ++ public CleanConsumeQueueService createCleanConsumeQueueService() { ++ return new RocksDBCleanConsumeQueueService(); ++ } ++ ++ @Override ++ public FlushConsumeQueueService createFlushConsumeQueueService() { ++ return new RocksDBFlushConsumeQueueService(); ++ } ++ ++ @Override ++ public CorrectLogicOffsetService createCorrectLogicOffsetService() { ++ return new RocksDBCorrectLogicOffsetService(); ++ } ++ ++ /** ++ * Try to set topicQueueTable = new HashMap<>(), otherwise it will cause bug when broker role changes. ++ * And unlike method in DefaultMessageStore, we don't need to really recover topic queue table advance, ++ * because we can recover topic queue table from rocksdb when we need to use it. ++ * @see RocksDBConsumeQueue#assignQueueOffset ++ */ ++ @Override ++ public void recoverTopicQueueTable() { ++ this.consumeQueueStore.setTopicQueueTable(new ConcurrentHashMap<>()); ++ } ++ ++ @Override ++ public void finishCommitLogDispatch() { ++ try { ++ putMessagePositionInfo(null); ++ } catch (RocksDBException e) { ++ ERROR_LOG.info("try to finish commitlog dispatch error.", e); ++ } ++ } ++ ++ @Override ++ public ConsumeQueueInterface getConsumeQueue(String topic, int queueId) { ++ return findConsumeQueue(topic, queueId); ++ } ++ ++ class RocksDBCleanConsumeQueueService extends CleanConsumeQueueService { ++ private final double diskSpaceWarningLevelRatio = ++ Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceWarningLevelRatio", "0.90")); ++ ++ private final double diskSpaceCleanForciblyRatio = ++ Double.parseDouble(System.getProperty("rocketmq.broker.diskSpaceCleanForciblyRatio", "0.85")); ++ ++ @Override ++ protected void deleteExpiredFiles() { ++ ++ long minOffset = RocksDBMessageStore.this.commitLog.getMinOffset(); ++ if (minOffset > this.lastPhysicalMinOffset) { ++ this.lastPhysicalMinOffset = minOffset; ++ ++ boolean spaceFull = isSpaceToDelete(); ++ boolean timeUp = cleanCommitLogService.isTimeToDelete(); ++ if (spaceFull || timeUp) { ++ RocksDBMessageStore.this.consumeQueueStore.cleanExpired(minOffset); ++ } ++ ++ RocksDBMessageStore.this.indexService.deleteExpiredFile(minOffset); ++ } ++ } ++ ++ private boolean isSpaceToDelete() { ++ double ratio = RocksDBMessageStore.this.getMessageStoreConfig().getDiskMaxUsedSpaceRatio() / 100.0; ++ ++ String storePathLogics = StorePathConfigHelper ++ .getStorePathConsumeQueue(RocksDBMessageStore.this.getMessageStoreConfig().getStorePathRootDir()); ++ double logicsRatio = UtilAll.getDiskPartitionSpaceUsedPercent(storePathLogics); ++ if (logicsRatio > diskSpaceWarningLevelRatio) { ++ boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskFull(); ++ if (diskOk) { ++ RocksDBMessageStore.LOGGER.error("logics disk maybe full soon " + logicsRatio + ", so mark disk full"); ++ } ++ } else if (logicsRatio > diskSpaceCleanForciblyRatio) { ++ } else { ++ boolean diskOk = RocksDBMessageStore.this.runningFlags.getAndMakeLogicDiskOK(); ++ if (!diskOk) { ++ RocksDBMessageStore.LOGGER.info("logics disk space OK " + logicsRatio + ", so mark disk ok"); ++ } ++ } ++ ++ if (logicsRatio < 0 || logicsRatio > ratio) { ++ RocksDBMessageStore.LOGGER.info("logics disk maybe full soon, so reclaim space, " + logicsRatio); ++ return true; ++ } ++ ++ return false; ++ } ++ } ++ ++ class RocksDBFlushConsumeQueueService extends FlushConsumeQueueService { ++ /** ++ * There is no need to flush consume queue, ++ * we put all consume queues in RocksDBConsumeQueueStore, ++ * it depends on rocksdb to flush consume queue to disk(sorted string table), ++ * we even don't flush WAL of consume store, since we think it can recover consume queue from commitlog. ++ */ ++ @Override ++ public void run() { ++ ++ } ++ } ++ ++ class RocksDBCorrectLogicOffsetService extends CorrectLogicOffsetService { ++ /** ++ * There is no need to correct min offset of consume queue, we already fix this problem. ++ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset ++ */ ++ public void run() { ++ ++ } ++ } ++ ++ @Override ++ public long estimateMessageCount(String topic, int queueId, long from, long to, MessageFilter filter) { ++ // todo ++ return 0; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java +index 2ae6879aa..91fcb155a 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java ++++ b/store/src/main/java/org/apache/rocketmq/store/RunningFlags.java +@@ -30,6 +30,8 @@ public class RunningFlags { + + private static final int FENCED_BIT = 1 << 5; + ++ private static final int LOGIC_DISK_FULL_BIT = 1 << 5; ++ + private volatile int flagBits = 0; + + public RunningFlags() { +@@ -63,6 +65,10 @@ public class RunningFlags { + return result; + } + ++ public void clearLogicsQueueError() { ++ this.flagBits &= ~WRITE_LOGICS_QUEUE_ERROR_BIT; ++ } ++ + public boolean getAndMakeWriteable() { + boolean result = this.isWriteable(); + if (!result) { +@@ -72,7 +78,7 @@ public class RunningFlags { + } + + public boolean isWriteable() { +- if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT)) == 0) { ++ if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | DISK_FULL_BIT | WRITE_INDEX_FILE_ERROR_BIT | FENCED_BIT | LOGIC_DISK_FULL_BIT)) == 0) { + return true; + } + +@@ -81,7 +87,7 @@ public class RunningFlags { + + //for consume queue, just ignore the DISK_FULL_BIT + public boolean isCQWriteable() { +- if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT)) == 0) { ++ if ((this.flagBits & (NOT_WRITEABLE_BIT | WRITE_LOGICS_QUEUE_ERROR_BIT | WRITE_INDEX_FILE_ERROR_BIT | LOGIC_DISK_FULL_BIT)) == 0) { + return true; + } + +@@ -139,4 +145,16 @@ public class RunningFlags { + this.flagBits &= ~DISK_FULL_BIT; + return result; + } ++ ++ public boolean getAndMakeLogicDiskFull() { ++ boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT); ++ this.flagBits |= LOGIC_DISK_FULL_BIT; ++ return result; ++ } ++ ++ public boolean getAndMakeLogicDiskOK() { ++ boolean result = !((this.flagBits & LOGIC_DISK_FULL_BIT) == LOGIC_DISK_FULL_BIT); ++ this.flagBits &= ~LOGIC_DISK_FULL_BIT; ++ return result; ++ } + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +index 9fa448043..028facbdc 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java ++++ b/store/src/main/java/org/apache/rocketmq/store/config/MessageStoreConfig.java +@@ -397,8 +397,10 @@ public class MessageStoreConfig { + private int batchDispatchRequestThreadPoolNums = 16; + + // rocksdb mode ++ private long cleanRocksDBDirtyCQIntervalMin = 60; ++ private long statRocksDBCQIntervalSec = 10; ++ private long memTableFlushIntervalMs = 60 * 60 * 1000L; + private boolean realTimePersistRocksDBConfig = true; +- private long memTableFlushInterval = 60 * 60 * 1000L; + private boolean enableRocksDBLog = false; + + private int topicQueueLockNum = 32; +@@ -499,6 +501,10 @@ public class MessageStoreConfig { + this.mappedFileSizeCommitLog = mappedFileSizeCommitLog; + } + ++ public boolean isEnableRocksDBStore() { ++ return StoreType.DEFAULT_ROCKSDB.getStoreType().equalsIgnoreCase(this.storeType); ++ } ++ + public String getStoreType() { + return storeType; + } +@@ -508,7 +514,6 @@ public class MessageStoreConfig { + } + + public int getMappedFileSizeConsumeQueue() { +- + int factor = (int) Math.ceil(this.mappedFileSizeConsumeQueue / (ConsumeQueue.CQ_STORE_UNIT_SIZE * 1.0)); + return (int) (factor * ConsumeQueue.CQ_STORE_UNIT_SIZE); + } +@@ -1738,12 +1743,28 @@ public class MessageStoreConfig { + this.realTimePersistRocksDBConfig = realTimePersistRocksDBConfig; + } + +- public long getMemTableFlushInterval() { +- return memTableFlushInterval; ++ public long getStatRocksDBCQIntervalSec() { ++ return statRocksDBCQIntervalSec; ++ } ++ ++ public void setStatRocksDBCQIntervalSec(long statRocksDBCQIntervalSec) { ++ this.statRocksDBCQIntervalSec = statRocksDBCQIntervalSec; ++ } ++ ++ public long getCleanRocksDBDirtyCQIntervalMin() { ++ return cleanRocksDBDirtyCQIntervalMin; ++ } ++ ++ public void setCleanRocksDBDirtyCQIntervalMin(long cleanRocksDBDirtyCQIntervalMin) { ++ this.cleanRocksDBDirtyCQIntervalMin = cleanRocksDBDirtyCQIntervalMin; ++ } ++ ++ public long getMemTableFlushIntervalMs() { ++ return memTableFlushIntervalMs; + } + +- public void setMemTableFlushInterval(long memTableFlushInterval) { +- this.memTableFlushInterval = memTableFlushInterval; ++ public void setMemTableFlushIntervalMs(long memTableFlushIntervalMs) { ++ this.memTableFlushIntervalMs = memTableFlushIntervalMs; + } + + public boolean isEnableRocksDBLog() { +diff --git a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +index d5f6acdc0..70371d83b 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java ++++ b/store/src/main/java/org/apache/rocketmq/store/dledger/DLedgerCommitLog.java +@@ -16,26 +16,13 @@ + */ + package org.apache.rocketmq.store.dledger; + +-import io.openmessaging.storage.dledger.AppendFuture; +-import io.openmessaging.storage.dledger.BatchAppendFuture; +-import io.openmessaging.storage.dledger.DLedgerConfig; +-import io.openmessaging.storage.dledger.DLedgerServer; +-import io.openmessaging.storage.dledger.entry.DLedgerEntry; +-import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; +-import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; +-import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; +-import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; +-import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; +-import io.openmessaging.storage.dledger.store.file.MmapFile; +-import io.openmessaging.storage.dledger.store.file.MmapFileList; +-import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; +-import io.openmessaging.storage.dledger.utils.DLedgerUtils; + import java.net.Inet6Address; + import java.net.InetSocketAddress; + import java.nio.ByteBuffer; + import java.util.LinkedList; + import java.util.List; + import java.util.concurrent.CompletableFuture; ++ + import org.apache.rocketmq.common.UtilAll; + import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExtBatch; +@@ -54,6 +41,22 @@ import org.apache.rocketmq.store.SelectMappedBufferResult; + import org.apache.rocketmq.store.StoreStatsService; + import org.apache.rocketmq.store.config.MessageStoreConfig; + import org.apache.rocketmq.store.logfile.MappedFile; ++import org.rocksdb.RocksDBException; ++ ++import io.openmessaging.storage.dledger.AppendFuture; ++import io.openmessaging.storage.dledger.BatchAppendFuture; ++import io.openmessaging.storage.dledger.DLedgerConfig; ++import io.openmessaging.storage.dledger.DLedgerServer; ++import io.openmessaging.storage.dledger.entry.DLedgerEntry; ++import io.openmessaging.storage.dledger.protocol.AppendEntryRequest; ++import io.openmessaging.storage.dledger.protocol.AppendEntryResponse; ++import io.openmessaging.storage.dledger.protocol.BatchAppendEntryRequest; ++import io.openmessaging.storage.dledger.protocol.DLedgerResponseCode; ++import io.openmessaging.storage.dledger.store.file.DLedgerMmapFileStore; ++import io.openmessaging.storage.dledger.store.file.MmapFile; ++import io.openmessaging.storage.dledger.store.file.MmapFileList; ++import io.openmessaging.storage.dledger.store.file.SelectMmapBufferResult; ++import io.openmessaging.storage.dledger.utils.DLedgerUtils; + + /** + * Store all metadata downtime for recovery, data protection reliability +@@ -269,7 +272,7 @@ public class DLedgerCommitLog extends CommitLog { + + return null; + } +- ++ + @Override + public boolean getData(final long offset, final int size, final ByteBuffer byteBuffer) { + if (offset < dividedCommitlogOffset) { +@@ -287,7 +290,7 @@ public class DLedgerCommitLog extends CommitLog { + return false; + } + +- private void recover(long maxPhyOffsetOfConsumeQueue) { ++ private void recover(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + dLedgerFileStore.load(); + if (dLedgerFileList.getMappedFiles().size() > 0) { + dLedgerFileStore.recover(); +@@ -341,12 +344,12 @@ public class DLedgerCommitLog extends CommitLog { + } + + @Override +- public void recoverNormally(long maxPhyOffsetOfConsumeQueue) { ++ public void recoverNormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + recover(maxPhyOffsetOfConsumeQueue); + } + + @Override +- public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) { ++ public void recoverAbnormally(long maxPhyOffsetOfConsumeQueue) throws RocksDBException { + recover(maxPhyOffsetOfConsumeQueue); + } + +@@ -469,9 +472,6 @@ public class DLedgerCommitLog extends CommitLog { + String msgId = MessageDecoder.createMessageId(buffer, msg.getStoreHostBytes(), wroteOffset); + elapsedTimeInLock = this.defaultMessageStore.getSystemClock().now() - beginTimeInDledgerLock; + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, wroteOffset, encodeResult.getData().length, msgId, System.currentTimeMillis(), queueOffset, elapsedTimeInLock); +- } catch (Exception e) { +- log.error("Put message error", e); +- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { + beginTimeInDledgerLock = 0; + putMessageLock.unlock(); +@@ -482,6 +482,9 @@ public class DLedgerCommitLog extends CommitLog { + } + + defaultMessageStore.increaseOffset(msg, getMessageNum(msg)); ++ } catch (Exception e) { ++ log.error("Put message error", e); ++ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { + topicQueueLock.unlock(topicQueueKey); + } +@@ -611,9 +614,6 @@ public class DLedgerCommitLog extends CommitLog { + appendResult = new AppendMessageResult(AppendMessageStatus.PUT_OK, firstWroteOffset, encodeResult.totalMsgLen, + msgIdBuilder.toString(), System.currentTimeMillis(), queueOffset, elapsedTimeInLock); + appendResult.setMsgNum(msgNum); +- } catch (Exception e) { +- log.error("Put message error", e); +- return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); + } finally { + beginTimeInDledgerLock = 0; + putMessageLock.unlock(); +@@ -626,7 +626,10 @@ public class DLedgerCommitLog extends CommitLog { + + defaultMessageStore.increaseOffset(messageExtBatch, (short) batchNum); + +- } finally { ++ } catch (Exception e) { ++ log.error("Put message error", e); ++ return CompletableFuture.completedFuture(new PutMessageResult(PutMessageStatus.UNKNOWN_ERROR, new AppendMessageResult(AppendMessageStatus.UNKNOWN_ERROR))); ++ } finally { + topicQueueLock.unlock(encodeResult.queueOffsetKey); + } + +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +index 467da603d..aaea7d690 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/HAService.java +@@ -25,6 +25,7 @@ import org.apache.rocketmq.remoting.protocol.body.HARuntimeInfo; + import org.apache.rocketmq.store.CommitLog; + import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.rocksdb.RocksDBException; + + public interface HAService { + +@@ -53,7 +54,7 @@ public interface HAService { + * + * @param masterEpoch the new masterEpoch + */ +- default boolean changeToMaster(int masterEpoch) { ++ default boolean changeToMaster(int masterEpoch) throws RocksDBException { + return false; + } + +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java +index 936db0c4c..176c25a96 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAClient.java +@@ -432,7 +432,7 @@ public class AutoSwitchHAClient extends ServiceThread implements HAClient { + /** + * Compare the master and slave's epoch file, find consistent point, do truncate. + */ +- private boolean doTruncate(List masterEpochEntries, long masterEndOffset) throws IOException { ++ private boolean doTruncate(List masterEpochEntries, long masterEndOffset) throws Exception { + if (this.epochCache.getEntrySize() == 0) { + // If epochMap is empty, means the broker is a new replicas + LOGGER.info("Slave local epochCache is empty, skip truncate log"); +diff --git a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +index f20bc3e28..64dad9aef 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java ++++ b/store/src/main/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHAService.java +@@ -51,6 +51,7 @@ import org.apache.rocketmq.store.ha.GroupTransferService; + import org.apache.rocketmq.store.ha.HAClient; + import org.apache.rocketmq.store.ha.HAConnection; + import org.apache.rocketmq.store.ha.HAConnectionStateNotificationService; ++import org.rocksdb.RocksDBException; + + /** + * SwitchAble ha service, support switch role to master or slave. +@@ -111,7 +112,7 @@ public class AutoSwitchHAService extends DefaultHAService { + } + + @Override +- public boolean changeToMaster(int masterEpoch) { ++ public boolean changeToMaster(int masterEpoch) throws RocksDBException { + final int lastEpoch = this.epochCache.lastEpoch(); + if (masterEpoch < lastEpoch) { + LOGGER.warn("newMasterEpoch {} < lastEpoch {}, fail to change to master", masterEpoch, lastEpoch); +@@ -315,7 +316,7 @@ public class AutoSwitchHAService extends DefaultHAService { + final EpochEntry currentLeaderEpoch = this.epochCache.lastEntry(); + if (slaveMaxOffset >= currentLeaderEpoch.getStartOffset()) { + LOGGER.info("The slave {} has caught up, slaveMaxOffset: {}, confirmOffset: {}, epoch: {}, leader epoch startOffset: {}.", +- slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset()); ++ slaveBrokerId, slaveMaxOffset, confirmOffset, currentLeaderEpoch.getEpoch(), currentLeaderEpoch.getStartOffset()); + currentSyncStateSet.add(slaveBrokerId); + markSynchronizingSyncStateSet(currentSyncStateSet); + // Notify the upper layer that syncStateSet changed. +@@ -491,7 +492,7 @@ public class AutoSwitchHAService extends DefaultHAService { + /** + * Try to truncate incomplete msg transferred from master. + */ +- public long truncateInvalidMsg() { ++ public long truncateInvalidMsg() throws RocksDBException { + long dispatchBehind = this.defaultMessageStore.dispatchBehindBytes(); + if (dispatchBehind <= 0) { + LOGGER.info("Dispatch complete, skip truncate"); +diff --git a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +index ab9fc6da7..2f2ce9812 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/plugin/AbstractPluginMessageStore.java +@@ -17,10 +17,6 @@ + + package org.apache.rocketmq.store.plugin; + +-import io.opentelemetry.api.common.AttributesBuilder; +-import io.opentelemetry.api.metrics.Meter; +-import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.nio.ByteBuffer; + import java.util.HashMap; + import java.util.LinkedList; +@@ -55,10 +51,16 @@ import org.apache.rocketmq.store.hook.PutMessageHook; + import org.apache.rocketmq.store.hook.SendMessageBackHook; + import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.queue.ConsumeQueueInterface; +-import org.apache.rocketmq.store.queue.ConsumeQueueStore; ++import org.apache.rocketmq.store.queue.ConsumeQueueStoreInterface; + import org.apache.rocketmq.store.stats.BrokerStatsManager; + import org.apache.rocketmq.store.timer.TimerMessageStore; + import org.apache.rocketmq.store.util.PerfCounter; ++import org.rocksdb.RocksDBException; ++ ++import io.opentelemetry.api.common.AttributesBuilder; ++import io.opentelemetry.api.metrics.Meter; ++import io.opentelemetry.sdk.metrics.InstrumentSelector; ++import io.opentelemetry.sdk.metrics.ViewBuilder; + + public abstract class AbstractPluginMessageStore implements MessageStore { + protected MessageStore next = null; +@@ -457,7 +459,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public boolean truncateFiles(long offsetToTruncate) { ++ public boolean truncateFiles(long offsetToTruncate) throws RocksDBException { + return next.truncateFiles(offsetToTruncate); + } + +@@ -511,7 +513,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + + @Override + public void onCommitLogDispatch(DispatchRequest dispatchRequest, boolean doDispatch, MappedFile commitLogFile, +- boolean isRecover, boolean isFileEnd) { ++ boolean isRecover, boolean isFileEnd) throws RocksDBException { + next.onCommitLogDispatch(dispatchRequest, doDispatch, commitLogFile, isRecover, isFileEnd); + } + +@@ -551,15 +553,10 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public void truncateDirtyLogicFiles(long phyOffset) { ++ public void truncateDirtyLogicFiles(long phyOffset) throws RocksDBException { + next.truncateDirtyLogicFiles(phyOffset); + } + +- @Override +- public void destroyLogics() { +- next.destroyLogics(); +- } +- + @Override + public void unlockMappedFile(MappedFile unlockMappedFile) { + next.unlockMappedFile(unlockMappedFile); +@@ -571,7 +568,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public ConsumeQueueStore getQueueStore() { ++ public ConsumeQueueStoreInterface getQueueStore() { + return next.getQueueStore(); + } + +@@ -586,7 +583,7 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + } + + @Override +- public void assignOffset(MessageExtBrokerInner msg) { ++ public void assignOffset(MessageExtBrokerInner msg) throws RocksDBException { + next.assignOffset(msg); + } + +@@ -649,4 +646,19 @@ public abstract class AbstractPluginMessageStore implements MessageStore { + public void initMetrics(Meter meter, Supplier attributesBuilderSupplier) { + next.initMetrics(meter, attributesBuilderSupplier); + } ++ ++ @Override ++ public void finishCommitLogDispatch() { ++ next.finishCommitLogDispatch(); ++ } ++ ++ @Override ++ public void recoverTopicQueueTable() { ++ next.recoverTopicQueueTable(); ++ } ++ ++ @Override ++ public void notifyMessageArriveIfNecessary(DispatchRequest dispatchRequest) { ++ next.notifyMessageArriveIfNecessary(dispatchRequest); ++ } + } +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java +new file mode 100644 +index 000000000..30054fa50 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/AbstractConsumeQueueStore.java +@@ -0,0 +1,105 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.DefaultMessageStore; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.rocksdb.RocksDBException; ++ ++public abstract class AbstractConsumeQueueStore implements ConsumeQueueStoreInterface { ++ protected static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ ++ protected final DefaultMessageStore messageStore; ++ protected final MessageStoreConfig messageStoreConfig; ++ protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator(); ++ protected final ConcurrentMap> consumeQueueTable; ++ ++ public AbstractConsumeQueueStore(DefaultMessageStore messageStore) { ++ this.messageStore = messageStore; ++ this.messageStoreConfig = messageStore.getMessageStoreConfig(); ++ this.consumeQueueTable = new ConcurrentHashMap<>(32); ++ } ++ ++ @Override ++ public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) { ++ consumeQueue.putMessagePositionInfoWrapper(request); ++ } ++ ++ @Override ++ public Long getMaxOffset(String topic, int queueId) { ++ return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); ++ } ++ ++ @Override ++ public void setTopicQueueTable(ConcurrentMap topicQueueTable) { ++ this.queueOffsetOperator.setTopicQueueTable(topicQueueTable); ++ this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable); ++ } ++ ++ @Override ++ public ConcurrentMap getTopicQueueTable() { ++ return this.queueOffsetOperator.getTopicQueueTable(); ++ } ++ ++ @Override ++ public void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException { ++ ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); ++ consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg); ++ } ++ ++ @Override ++ public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { ++ ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); ++ consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum); ++ } ++ ++ @Override ++ public void removeTopicQueueTable(String topic, Integer queueId) { ++ this.queueOffsetOperator.remove(topic, queueId); ++ } ++ ++ @Override ++ public ConcurrentMap> getConsumeQueueTable() { ++ return this.consumeQueueTable; ++ } ++ ++ @Override ++ public ConcurrentMap findConsumeQueueMap(String topic) { ++ return this.consumeQueueTable.get(topic); ++ } ++ ++ @Override ++ public long getStoreTime(CqUnit cqUnit) { ++ if (cqUnit != null) { ++ try { ++ final long phyOffset = cqUnit.getPos(); ++ final int size = cqUnit.getSize(); ++ long storeTime = this.messageStore.getCommitLog().pickupStoreTimestamp(phyOffset, size); ++ return storeTime; ++ } catch (Exception e) { ++ } ++ } ++ return -1; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java +index 387c233bf..7108c835c 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/BatchConsumeQueue.java +@@ -24,6 +24,7 @@ import java.util.Map; + import java.util.concurrent.ConcurrentSkipListMap; + import java.util.function.Function; + import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.attribute.CQType; + import org.apache.rocketmq.common.constant.LoggerName; +@@ -311,6 +312,11 @@ public class BatchConsumeQueue implements ConsumeQueueInterface { + return new BatchConsumeQueueIterator(sbr); + } + ++ @Override ++ public ReferredIterator iterateFrom(long startIndex, int count) { ++ return iterateFrom(startIndex); ++ } ++ + @Override + public CqUnit get(long offset) { + ReferredIterator it = iterateFrom(offset); +@@ -320,6 +326,20 @@ public class BatchConsumeQueue implements ConsumeQueueInterface { + return it.nextAndRelease(); + } + ++ @Override ++ public Pair getCqUnitAndStoreTime(long index) { ++ CqUnit cqUnit = get(index); ++ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); ++ return new Pair<>(cqUnit, messageStoreTime); ++ } ++ ++ @Override ++ public Pair getEarliestUnitAndStoreTime() { ++ CqUnit cqUnit = getEarliestUnit(); ++ Long messageStoreTime = this.messageStore.getQueueStore().getStoreTime(cqUnit); ++ return new Pair<>(cqUnit, messageStoreTime); ++ } ++ + @Override + public CqUnit getEarliestUnit() { + return get(minOffsetInQueue); +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java +index 55d080829..c65f2a68b 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueInterface.java +@@ -18,10 +18,12 @@ + package org.apache.rocketmq.store.queue; + + import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.Pair; + import org.apache.rocketmq.common.attribute.CQType; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.store.DispatchRequest; + import org.apache.rocketmq.store.MessageFilter; ++import org.rocksdb.RocksDBException; + + public interface ConsumeQueueInterface extends FileQueueLifeCycle { + /** +@@ -44,6 +46,16 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { + */ + ReferredIterator iterateFrom(long startIndex); + ++ /** ++ * Get the units from the start offset. ++ * ++ * @param startIndex start index ++ * @param count the unit counts will be iterated ++ * @return the unit iterateFrom ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ ReferredIterator iterateFrom(long startIndex, int count) throws RocksDBException; ++ + /** + * Get cq unit at specified index + * @param index index +@@ -51,6 +63,18 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { + */ + CqUnit get(long index); + ++ /** ++ * Get earliest cq unit ++ * @return the cq unit and message storeTime at index ++ */ ++ Pair getCqUnitAndStoreTime(long index); ++ ++ /** ++ * Get earliest cq unit ++ * @return earliest cq unit and message storeTime ++ */ ++ Pair getEarliestUnitAndStoreTime(); ++ + /** + * Get earliest cq unit + * @return earliest cq unit +@@ -153,8 +177,9 @@ public interface ConsumeQueueInterface extends FileQueueLifeCycle { + * Assign queue offset. + * @param queueOffsetAssigner the delegated queue offset assigner + * @param msg message itself ++ * @throws RocksDBException only in rocksdb mode + */ +- void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg); ++ void assignQueueOffset(QueueOffsetOperator queueOffsetAssigner, MessageExtBrokerInner msg) throws RocksDBException; + + + /** +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +index d03d15d65..616511b67 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStore.java +@@ -16,64 +16,128 @@ + */ + package org.apache.rocketmq.store.queue; + ++import java.io.File; + import java.nio.ByteBuffer; + import java.util.ArrayList; ++import java.util.Iterator; + import java.util.List; ++import java.util.Map; ++import java.util.Map.Entry; ++import java.util.Objects; ++import java.util.Optional; + import java.util.concurrent.BlockingQueue; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; + import java.util.concurrent.CountDownLatch; + import java.util.concurrent.ExecutorService; + import java.util.concurrent.FutureTask; + import java.util.concurrent.LinkedBlockingQueue; + import java.util.concurrent.TimeUnit; ++import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.ThreadFactoryImpl; + import org.apache.rocketmq.common.TopicConfig; + import org.apache.rocketmq.common.attribute.CQType; +-import org.apache.rocketmq.common.constant.LoggerName; + import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExt; + import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.common.utils.QueueTypeUtils; + import org.apache.rocketmq.common.utils.ThreadUtils; +-import org.apache.rocketmq.logging.org.slf4j.Logger; +-import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; + import org.apache.rocketmq.store.CommitLog; + import org.apache.rocketmq.store.ConsumeQueue; + import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.DispatchRequest; +-import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.store.SelectMappedBufferResult; +-import org.apache.rocketmq.store.config.MessageStoreConfig; +- +-import java.io.File; +-import java.util.Iterator; +-import java.util.Map; +-import java.util.Objects; +-import java.util.Optional; +-import java.util.concurrent.ConcurrentHashMap; +-import java.util.concurrent.ConcurrentMap; + + import static java.lang.String.format; + import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathBatchConsumeQueue; + import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue; + +-public class ConsumeQueueStore { +- private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++public class ConsumeQueueStore extends AbstractConsumeQueueStore { + +- protected final DefaultMessageStore messageStore; +- protected final MessageStoreConfig messageStoreConfig; +- protected final QueueOffsetOperator queueOffsetOperator = new QueueOffsetOperator(); +- protected final ConcurrentMap> consumeQueueTable; ++ public ConsumeQueueStore(DefaultMessageStore messageStore) { ++ super(messageStore); ++ } + +- public ConsumeQueueStore(DefaultMessageStore messageStore, MessageStoreConfig messageStoreConfig) { +- this.messageStore = messageStore; +- this.messageStoreConfig = messageStoreConfig; +- this.consumeQueueTable = new ConcurrentHashMap<>(32); ++ @Override ++ public void start() { ++ log.info("Default ConsumeQueueStore start!"); + } + +- private FileQueueLifeCycle getLifeCycle(String topic, int queueId) { +- return findOrCreateConsumeQueue(topic, queueId); ++ @Override ++ public boolean load() { ++ boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ); ++ boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ); ++ return cqLoadResult && bcqLoadResult; ++ } ++ ++ @Override ++ public boolean loadAfterDestroy() { ++ return true; ++ } ++ ++ @Override ++ public void recover() { ++ for (ConcurrentMap maps : this.consumeQueueTable.values()) { ++ for (ConsumeQueueInterface logic : maps.values()) { ++ this.recover(logic); ++ } ++ } + } + ++ @Override ++ public boolean recoverConcurrently() { ++ int count = 0; ++ for (ConcurrentMap maps : this.consumeQueueTable.values()) { ++ count += maps.values().size(); ++ } ++ final CountDownLatch countDownLatch = new CountDownLatch(count); ++ BlockingQueue recoverQueue = new LinkedBlockingQueue<>(); ++ final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_"); ++ List> result = new ArrayList<>(count); ++ try { ++ for (ConcurrentMap maps : this.consumeQueueTable.values()) { ++ for (final ConsumeQueueInterface logic : maps.values()) { ++ FutureTask futureTask = new FutureTask<>(() -> { ++ boolean ret = true; ++ try { ++ logic.recover(); ++ } catch (Throwable e) { ++ ret = false; ++ log.error("Exception occurs while recover consume queue concurrently, " + ++ "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e); ++ } finally { ++ countDownLatch.countDown(); ++ } ++ return ret; ++ }); ++ ++ result.add(futureTask); ++ executor.submit(futureTask); ++ } ++ } ++ countDownLatch.await(); ++ for (FutureTask task : result) { ++ if (task != null && task.isDone()) { ++ if (!task.get()) { ++ return false; ++ } ++ } ++ } ++ } catch (Exception e) { ++ log.error("Exception occurs while recover consume queue concurrently", e); ++ return false; ++ } finally { ++ executor.shutdown(); ++ } ++ return true; ++ } ++ ++ @Override ++ public boolean shutdown() { ++ return true; ++ } ++ ++ @Override + public long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.rollNextFile(offset); +@@ -83,32 +147,53 @@ public class ConsumeQueueStore { + consumeQueue.correctMinOffset(minCommitLogOffset); + } + +- /** +- * Apply the dispatched request and build the consume queue. This function should be idempotent. +- * +- * @param consumeQueue consume queue +- * @param request dispatch request +- */ +- public void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request) { +- consumeQueue.putMessagePositionInfoWrapper(request); +- } +- ++ @Override + public void putMessagePositionInfoWrapper(DispatchRequest dispatchRequest) { + ConsumeQueueInterface cq = this.findOrCreateConsumeQueue(dispatchRequest.getTopic(), dispatchRequest.getQueueId()); + this.putMessagePositionInfoWrapper(cq, dispatchRequest); + } + ++ @Override ++ public List rangeQuery(String topic, int queueId, long startIndex, int num) { ++ return null; ++ } ++ ++ @Override ++ public ByteBuffer get(String topic, int queueId, long startIndex) { ++ return null; ++ } ++ ++ @Override ++ public long getMaxOffsetInQueue(String topic, int queueId) { ++ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); ++ if (logic != null) { ++ return logic.getMaxOffsetInQueue(); ++ } ++ return 0; ++ } ++ ++ @Override ++ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) { ++ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); ++ if (logic != null) { ++ long resultOffset = logic.getOffsetInQueueByTime(timestamp, boundaryType); ++ // Make sure the result offset is in valid range. ++ resultOffset = Math.max(resultOffset, logic.getMinOffsetInQueue()); ++ resultOffset = Math.min(resultOffset, logic.getMaxOffsetInQueue()); ++ return resultOffset; ++ } ++ return 0; ++ } ++ ++ private FileQueueLifeCycle getLifeCycle(String topic, int queueId) { ++ return findOrCreateConsumeQueue(topic, queueId); ++ } ++ + public boolean load(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.load(); + } + +- public boolean load() { +- boolean cqLoadResult = loadConsumeQueues(getStorePathConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.SimpleCQ); +- boolean bcqLoadResult = loadConsumeQueues(getStorePathBatchConsumeQueue(this.messageStoreConfig.getStorePathRootDir()), CQType.BatchCQ); +- return cqLoadResult && bcqLoadResult; +- } +- + private boolean loadConsumeQueues(String storePath, CQType cqType) { + File dirLogic = new File(storePath); + File[] fileTopicList = dirLogic.listFiles(); +@@ -189,62 +274,17 @@ public class ConsumeQueueStore { + fileQueueLifeCycle.recover(); + } + +- public void recover() { +- for (ConcurrentMap maps : this.consumeQueueTable.values()) { +- for (ConsumeQueueInterface logic : maps.values()) { +- this.recover(logic); +- } +- } +- } +- +- public boolean recoverConcurrently() { +- int count = 0; +- for (ConcurrentMap maps : this.consumeQueueTable.values()) { +- count += maps.values().size(); +- } +- final CountDownLatch countDownLatch = new CountDownLatch(count); +- BlockingQueue recoverQueue = new LinkedBlockingQueue<>(); +- final ExecutorService executor = buildExecutorService(recoverQueue, "RecoverConsumeQueueThread_"); +- List> result = new ArrayList<>(count); +- try { +- for (ConcurrentMap maps : this.consumeQueueTable.values()) { +- for (final ConsumeQueueInterface logic : maps.values()) { +- FutureTask futureTask = new FutureTask<>(() -> { +- boolean ret = true; +- try { +- logic.recover(); +- } catch (Throwable e) { +- ret = false; +- log.error("Exception occurs while recover consume queue concurrently, " + +- "topic={}, queueId={}", logic.getTopic(), logic.getQueueId(), e); +- } finally { +- countDownLatch.countDown(); +- } +- return ret; +- }); +- +- result.add(futureTask); +- executor.submit(futureTask); +- } +- } +- countDownLatch.await(); +- for (FutureTask task : result) { +- if (task != null && task.isDone()) { +- if (!task.get()) { +- return false; +- } +- } +- } +- } catch (Exception e) { +- log.error("Exception occurs while recover consume queue concurrently", e); +- return false; +- } finally { +- executor.shutdown(); ++ @Override ++ public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) { ++ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); ++ if (logic != null) { ++ return logic.getMaxPhysicOffset(); + } +- return true; ++ return null; + } + +- public long getMaxOffsetInConsumeQueue() { ++ @Override ++ public long getMaxPhyOffsetInConsumeQueue() { + long maxPhysicOffset = -1L; + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueueInterface logic : maps.values()) { +@@ -256,11 +296,22 @@ public class ConsumeQueueStore { + return maxPhysicOffset; + } + ++ @Override ++ public long getMinOffsetInQueue(String topic, int queueId) { ++ ConsumeQueueInterface logic = findOrCreateConsumeQueue(topic, queueId); ++ if (logic != null) { ++ return logic.getMinOffsetInQueue(); ++ } ++ ++ return -1; ++ } ++ + public void checkSelf(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + fileQueueLifeCycle.checkSelf(); + } + ++ @Override + public void checkSelf() { + for (Map.Entry> topicEntry : this.consumeQueueTable.entrySet()) { + for (Map.Entry cqEntry : topicEntry.getValue().entrySet()) { +@@ -269,16 +320,19 @@ public class ConsumeQueueStore { + } + } + ++ @Override + public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.flush(flushLeastPages); + } + ++ @Override + public void destroy(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + fileQueueLifeCycle.destroy(); + } + ++ @Override + public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.deleteExpiredFile(minCommitLogPos); +@@ -300,21 +354,20 @@ public class ConsumeQueueStore { + fileQueueLifeCycle.cleanSwappedMap(forceCleanSwapIntervalMs); + } + ++ @Override + public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.isFirstFileAvailable(); + } + ++ @Override + public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) { + FileQueueLifeCycle fileQueueLifeCycle = getLifeCycle(consumeQueue.getTopic(), consumeQueue.getQueueId()); + return fileQueueLifeCycle.isFirstFileExist(); + } + ++ @Override + public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { +- return doFindOrCreateConsumeQueue(topic, queueId); +- } +- +- private ConsumeQueueInterface doFindOrCreateConsumeQueue(String topic, int queueId) { + ConcurrentMap map = consumeQueueTable.get(topic); + if (null == map) { + ConcurrentMap newMap = new ConcurrentHashMap<>(128); +@@ -361,46 +414,15 @@ public class ConsumeQueueStore { + return logic; + } + +- public Long getMaxOffset(String topic, int queueId) { +- return this.queueOffsetOperator.currentQueueOffset(topic + "-" + queueId); +- } +- +- public void setTopicQueueTable(ConcurrentMap topicQueueTable) { +- this.queueOffsetOperator.setTopicQueueTable(topicQueueTable); +- this.queueOffsetOperator.setLmqTopicQueueTable(topicQueueTable); +- } +- +- public ConcurrentMap getTopicQueueTable() { +- return this.queueOffsetOperator.getTopicQueueTable(); +- } +- + public void setBatchTopicQueueTable(ConcurrentMap batchTopicQueueTable) { + this.queueOffsetOperator.setBatchTopicQueueTable(batchTopicQueueTable); + } + +- public void assignQueueOffset(MessageExtBrokerInner msg) { +- ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); +- consumeQueue.assignQueueOffset(this.queueOffsetOperator, msg); +- } +- +- public void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum) { +- ConsumeQueueInterface consumeQueue = findOrCreateConsumeQueue(msg.getTopic(), msg.getQueueId()); +- consumeQueue.increaseQueueOffset(this.queueOffsetOperator, msg, messageNum); +- } +- + public void updateQueueOffset(String topic, int queueId, long offset) { + String topicQueueKey = topic + "-" + queueId; + this.queueOffsetOperator.updateQueueOffset(topicQueueKey, offset); + } + +- public void removeTopicQueueTable(String topic, Integer queueId) { +- this.queueOffsetOperator.remove(topic, queueId); +- } +- +- public ConcurrentMap> getConsumeQueueTable() { +- return consumeQueueTable; +- } +- + private void putConsumeQueue(final String topic, final int queueId, final ConsumeQueueInterface consumeQueue) { + ConcurrentMap map = this.consumeQueueTable.get(topic); + if (null == map) { +@@ -412,6 +434,7 @@ public class ConsumeQueueStore { + } + } + ++ @Override + public void recoverOffsetTable(long minPhyOffset) { + ConcurrentMap cqOffsetTable = new ConcurrentHashMap<>(1024); + ConcurrentMap bcqOffsetTable = new ConcurrentHashMap<>(1024); +@@ -431,7 +454,7 @@ public class ConsumeQueueStore { + } + } + +- //Correct unSubmit consumeOffset ++ // Correct unSubmit consumeOffset + if (messageStoreConfig.isDuplicationEnable()) { + SelectMappedBufferResult lastBuffer = null; + long startReadOffset = messageStore.getCommitLog().getConfirmOffset() == -1 ? 0 : messageStore.getCommitLog().getConfirmOffset(); +@@ -476,6 +499,7 @@ public class ConsumeQueueStore { + this.setBatchTopicQueueTable(bcqOffsetTable); + } + ++ @Override + public void destroy() { + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueueInterface logic : maps.values()) { +@@ -484,8 +508,9 @@ public class ConsumeQueueStore { + } + } + ++ @Override + public void cleanExpired(long minCommitLogOffset) { +- Iterator>> it = this.consumeQueueTable.entrySet().iterator(); ++ Iterator>> it = this.consumeQueueTable.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry> next = it.next(); + String topic = next.getKey(); +@@ -526,14 +551,16 @@ public class ConsumeQueueStore { + } + } + +- public void truncateDirty(long phyOffset) { ++ @Override ++ public void truncateDirty(long offsetToTruncate) { + for (ConcurrentMap maps : this.consumeQueueTable.values()) { + for (ConsumeQueueInterface logic : maps.values()) { +- this.truncateDirtyLogicFiles(logic, phyOffset); ++ this.truncateDirtyLogicFiles(logic, offsetToTruncate); + } + } + } + ++ @Override + public long getTotalSize() { + long totalSize = 0; + for (ConcurrentMap maps : this.consumeQueueTable.values()) { +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java +new file mode 100644 +index 000000000..268803dcc +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/ConsumeQueueStoreInterface.java +@@ -0,0 +1,289 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.nio.ByteBuffer; ++import java.util.List; ++import java.util.concurrent.ConcurrentMap; ++import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.rocksdb.RocksDBException; ++ ++public interface ConsumeQueueStoreInterface { ++ ++ /** ++ * Start the consumeQueueStore ++ */ ++ void start(); ++ ++ /** ++ * Load from file. ++ * @return true if loaded successfully. ++ */ ++ boolean load(); ++ ++ /** ++ * load after destroy ++ */ ++ boolean loadAfterDestroy(); ++ ++ /** ++ * Recover from file. ++ */ ++ void recover(); ++ ++ /** ++ * Recover concurrently from file. ++ * @return true if recovered successfully. ++ */ ++ boolean recoverConcurrently(); ++ ++ /** ++ * Shutdown the consumeQueueStore ++ * @return true if shutdown successfully. ++ */ ++ boolean shutdown(); ++ ++ /** ++ * destroy all consumeQueues ++ */ ++ void destroy(); ++ ++ /** ++ * destroy the specific consumeQueue ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException; ++ ++ /** ++ * Flush cache to file. ++ * @param consumeQueue the consumeQueue will be flushed ++ * @param flushLeastPages the minimum number of pages to be flushed ++ * @return true if any data has been flushed. ++ */ ++ boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages); ++ ++ /** ++ * clean expired data from minPhyOffset ++ * @param minPhyOffset ++ */ ++ void cleanExpired(long minPhyOffset); ++ ++ /** ++ * Check files. ++ */ ++ void checkSelf(); ++ ++ /** ++ * Delete expired files ending at min commit log position. ++ * @param consumeQueue ++ * @param minCommitLogPos min commit log position ++ * @return deleted file numbers. ++ */ ++ int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos); ++ ++ /** ++ * Is the first file available? ++ * @param consumeQueue ++ * @return true if it's available ++ */ ++ boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue); ++ ++ /** ++ * Does the first file exist? ++ * @param consumeQueue ++ * @return true if it exists ++ */ ++ boolean isFirstFileExist(ConsumeQueueInterface consumeQueue); ++ ++ /** ++ * Roll to next file. ++ * @param consumeQueue ++ * @param offset next beginning offset ++ * @return the beginning offset of the next file ++ */ ++ long rollNextFile(ConsumeQueueInterface consumeQueue, final long offset); ++ ++ /** ++ * truncate dirty data ++ * @param offsetToTruncate ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ void truncateDirty(long offsetToTruncate) throws RocksDBException; ++ ++ /** ++ * Apply the dispatched request and build the consume queue. This function should be idempotent. ++ * ++ * @param consumeQueue consume queue ++ * @param request dispatch request ++ */ ++ void putMessagePositionInfoWrapper(ConsumeQueueInterface consumeQueue, DispatchRequest request); ++ ++ /** ++ * Apply the dispatched request. This function should be idempotent. ++ * ++ * @param request dispatch request ++ * @throws RocksDBException only in rocksdb mode will throw exception ++ */ ++ void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException; ++ ++ /** ++ * range query cqUnit(ByteBuffer) in rocksdb ++ * @param topic ++ * @param queueId ++ * @param startIndex ++ * @param num ++ * @return the byteBuffer list of the topic-queueId in rocksdb ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException; ++ ++ /** ++ * query cqUnit(ByteBuffer) in rocksdb ++ * @param topic ++ * @param queueId ++ * @param startIndex ++ * @return the byteBuffer of the topic-queueId in rocksdb ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ ByteBuffer get(final String topic, final int queueId, final long startIndex) throws RocksDBException; ++ ++ /** ++ * get consumeQueue table ++ * @return the consumeQueue table ++ */ ++ ConcurrentMap> getConsumeQueueTable(); ++ ++ /** ++ * Assign queue offset. ++ * @param msg message itself ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ void assignQueueOffset(MessageExtBrokerInner msg) throws RocksDBException; ++ ++ /** ++ * Increase queue offset. ++ * @param msg message itself ++ * @param messageNum message number ++ */ ++ void increaseQueueOffset(MessageExtBrokerInner msg, short messageNum); ++ ++ /** ++ * recover topicQueue table by minPhyOffset ++ * @param minPhyOffset ++ */ ++ void recoverOffsetTable(long minPhyOffset); ++ ++ /** ++ * set topicQueue table ++ * @param topicQueueTable ++ */ ++ void setTopicQueueTable(ConcurrentMap topicQueueTable); ++ ++ /** ++ * remove topic-queueId from topicQueue table ++ * @param topic ++ * @param queueId ++ */ ++ void removeTopicQueueTable(String topic, Integer queueId); ++ ++ /** ++ * get topicQueue table ++ * @return the topicQueue table ++ */ ++ ConcurrentMap getTopicQueueTable(); ++ ++ /** ++ * get the max physical offset in consumeQueue ++ * @param topic ++ * @param queueId ++ * @return ++ */ ++ Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId); ++ ++ /** ++ * get maxOffset of specific topic-queueId in topicQueue table ++ * @param topic ++ * @param queueId ++ * @return the max offset in QueueOffsetOperator ++ */ ++ Long getMaxOffset(String topic, int queueId); ++ ++ /** ++ * get max physic offset in consumeQueue ++ * @return the max physic offset in consumeQueue ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ long getMaxPhyOffsetInConsumeQueue() throws RocksDBException; ++ ++ /** ++ * get min logic offset of specific topic-queueId in consumeQueue ++ * @param topic ++ * @param queueId ++ * @return the min logic offset of specific topic-queueId in consumeQueue ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ long getMinOffsetInQueue(final String topic, final int queueId) throws RocksDBException; ++ ++ /** ++ * get max logic offset of specific topic-queueId in consumeQueue ++ * @param topic ++ * @param queueId ++ * @return the max logic offset of specific topic-queueId in consumeQueue ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ long getMaxOffsetInQueue(final String topic, final int queueId) throws RocksDBException; ++ ++ /** ++ * Get the message whose timestamp is the smallest, greater than or equal to the given time and when there are more ++ * than one message satisfy the condition, decide which one to return based on boundaryType. ++ * @param timestamp timestamp ++ * @param boundaryType Lower or Upper ++ * @return the offset(index) ++ * @throws RocksDBException only in rocksdb mode ++ */ ++ long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException; ++ ++ /** ++ * find or create the consumeQueue ++ * @param topic ++ * @param queueId ++ * @return the consumeQueue ++ */ ++ ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId); ++ ++ /** ++ * find the consumeQueueMap of topic ++ * @param topic ++ * @return the consumeQueueMap of topic ++ */ ++ ConcurrentMap findConsumeQueueMap(String topic); ++ ++ /** ++ * get the total size of all consumeQueue ++ * @return the total size of all consumeQueue ++ */ ++ long getTotalSize(); ++ ++ /** ++ * Get store time from commitlog by cqUnit ++ * @param cqUnit ++ * @return ++ */ ++ long getStoreTime(CqUnit cqUnit); ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java +new file mode 100644 +index 000000000..d6291d908 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/MultiDispatch.java +@@ -0,0 +1,76 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.util.ArrayList; ++import java.util.List; ++import java.util.Map; ++ ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.MixAll; ++import org.apache.rocketmq.common.message.MessageConst; ++import org.apache.rocketmq.common.topic.TopicValidator; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.config.MessageStoreConfig; ++ ++public class MultiDispatch { ++ ++ public static String lmqQueueKey(String queueName) { ++ StringBuilder keyBuilder = new StringBuilder(); ++ keyBuilder.append(queueName); ++ keyBuilder.append('-'); ++ int queueId = 0; ++ keyBuilder.append(queueId); ++ return keyBuilder.toString(); ++ } ++ ++ public static boolean isNeedHandleMultiDispatch(MessageStoreConfig messageStoreConfig, String topic) { ++ return messageStoreConfig.isEnableMultiDispatch() ++ && !topic.startsWith(MixAll.RETRY_GROUP_TOPIC_PREFIX) ++ && !topic.startsWith(TopicValidator.SYSTEM_TOPIC_PREFIX) ++ && !topic.equals(TopicValidator.RMQ_SYS_SCHEDULE_TOPIC); ++ } ++ ++ public static boolean checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, DispatchRequest dispatchRequest) { ++ if (!isNeedHandleMultiDispatch(messageStoreConfig, dispatchRequest.getTopic())) { ++ return false; ++ } ++ Map prop = dispatchRequest.getPropertiesMap(); ++ if (prop == null || prop.isEmpty()) { ++ return false; ++ } ++ String multiDispatchQueue = prop.get(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); ++ String multiQueueOffset = prop.get(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET); ++ if (StringUtils.isBlank(multiDispatchQueue) || StringUtils.isBlank(multiQueueOffset)) { ++ return false; ++ } ++ return true; ++ } ++ ++ public static List checkMultiDispatchQueue(MessageStoreConfig messageStoreConfig, List dispatchRequests) { ++ if (!messageStoreConfig.isEnableMultiDispatch() || dispatchRequests == null || dispatchRequests.size() == 0) { ++ return null; ++ } ++ List result = new ArrayList<>(); ++ for (DispatchRequest dispatchRequest : dispatchRequests) { ++ if (checkMultiDispatchQueue(messageStoreConfig, dispatchRequest)) { ++ result.add(dispatchRequest); ++ } ++ } ++ return dispatchRequests; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java +index 2545bbf52..8da374828 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/QueueOffsetOperator.java +@@ -41,6 +41,10 @@ public class QueueOffsetOperator { + return ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); + } + ++ public Long getTopicQueueNextOffset(String topicQueueKey) { ++ return this.topicQueueTable.get(topicQueueKey); ++ } ++ + public void increaseQueueOffset(String topicQueueKey, short messageNum) { + Long queueOffset = ConcurrentHashMapUtils.computeIfAbsent(this.topicQueueTable, topicQueueKey, k -> 0L); + topicQueueTable.put(topicQueueKey, queueOffset + messageNum); +@@ -63,6 +67,10 @@ public class QueueOffsetOperator { + return ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); + } + ++ public Long getLmqTopicQueueNextOffset(String topicQueueKey) { ++ return this.lmqTopicQueueTable.get(topicQueueKey); ++ } ++ + public void increaseLmqOffset(String topicQueueKey, short messageNum) { + Long lmqOffset = ConcurrentHashMapUtils.computeIfAbsent(this.lmqTopicQueueTable, topicQueueKey, k -> 0L); + this.lmqTopicQueueTable.put(topicQueueKey, lmqOffset + messageNum); +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java +new file mode 100644 +index 000000000..759be395d +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueue.java +@@ -0,0 +1,437 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.nio.ByteBuffer; ++import java.util.List; ++ ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.MixAll; ++import org.apache.rocketmq.common.Pair; ++import org.apache.rocketmq.common.attribute.CQType; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.message.MessageAccessor; ++import org.apache.rocketmq.common.message.MessageConst; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.ConsumeQueue; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.MessageFilter; ++import org.apache.rocketmq.store.MessageStore; ++import org.rocksdb.RocksDBException; ++ ++public class RocksDBConsumeQueue implements ConsumeQueueInterface { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); ++ ++ private final MessageStore messageStore; ++ private final String topic; ++ private final int queueId; ++ ++ public RocksDBConsumeQueue(final MessageStore messageStore, final String topic, final int queueId) { ++ this.messageStore = messageStore; ++ this.topic = topic; ++ this.queueId = queueId; ++ } ++ ++ public RocksDBConsumeQueue(final String topic, final int queueId) { ++ this.messageStore = null; ++ this.topic = topic; ++ this.queueId = queueId; ++ } ++ ++ @Override ++ public boolean load() { ++ return true; ++ } ++ ++ @Override ++ public void recover() { ++ // ignore ++ } ++ ++ @Override ++ public void checkSelf() { ++ // ignore ++ } ++ ++ @Override ++ public boolean flush(final int flushLeastPages) { ++ return true; ++ } ++ ++ @Override ++ public void destroy() { ++ // ignore ++ } ++ ++ @Override ++ public void truncateDirtyLogicFiles(long maxCommitLogPos) { ++ // ignored ++ } ++ ++ @Override ++ public int deleteExpiredFile(long minCommitLogPos) { ++ return 0; ++ } ++ ++ @Override ++ public long rollNextFile(long nextBeginOffset) { ++ return 0; ++ } ++ ++ @Override ++ public boolean isFirstFileAvailable() { ++ return true; ++ } ++ ++ @Override ++ public boolean isFirstFileExist() { ++ return true; ++ } ++ ++ @Override ++ public void swapMap(int reserveNum, long forceSwapIntervalMs, long normalSwapIntervalMs) { ++ // ignore ++ } ++ ++ @Override ++ public void cleanSwappedMap(long forceCleanSwapIntervalMs) { ++ // ignore ++ } ++ ++ @Override ++ public long getMaxOffsetInQueue() { ++ try { ++ return this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getMaxOffsetInQueue Failed. topic: {}, queueId: {}", topic, queueId, e); ++ return 0; ++ } ++ } ++ ++ @Override ++ public long getMessageTotalInQueue() { ++ try { ++ long maxOffsetInQueue = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); ++ long minOffsetInQueue = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId); ++ return maxOffsetInQueue - minOffsetInQueue; ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getMessageTotalInQueue Failed. topic: {}, queueId: {}, {}", topic, queueId, e); ++ } ++ return -1; ++ } ++ ++ /** ++ * We already implement it in RocksDBConsumeQueueStore. ++ * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime ++ * @param timestamp timestamp ++ * @return ++ */ ++ @Override ++ public long getOffsetInQueueByTime(long timestamp) { ++ return 0; ++ } ++ ++ /** ++ * We already implement it in RocksDBConsumeQueueStore. ++ * @see RocksDBConsumeQueueStore#getOffsetInQueueByTime ++ * @param timestamp timestamp ++ * @param boundaryType Lower or Upper ++ * @return ++ */ ++ @Override ++ public long getOffsetInQueueByTime(long timestamp, BoundaryType boundaryType) { ++ return 0; ++ } ++ ++ @Override ++ public long getMaxPhysicOffset() { ++ Long maxPhyOffset = this.messageStore.getQueueStore().getMaxPhyOffsetInConsumeQueue(topic, queueId); ++ return maxPhyOffset == null ? -1 : maxPhyOffset; ++ } ++ ++ @Override ++ public long getMinLogicOffset() { ++ return 0; ++ } ++ ++ @Override ++ public CQType getCQType() { ++ return CQType.RocksDBCQ; ++ } ++ ++ @Override ++ public long getTotalSize() { ++ // ignored ++ return 0; ++ } ++ ++ @Override ++ public int getUnitSize() { ++ // attention: unitSize should equal to 'ConsumeQueue.CQ_STORE_UNIT_SIZE' ++ return ConsumeQueue.CQ_STORE_UNIT_SIZE; ++ } ++ ++ /** ++ * Ignored, we already implement this method ++ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) ++ */ ++ @Override ++ public void correctMinOffset(long minCommitLogOffset) { ++ ++ } ++ ++ /** ++ * Ignored, in rocksdb mode, we build cq in RocksDBConsumeQueueStore ++ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore#putMessagePosition() ++ */ ++ @Override ++ public void putMessagePositionInfoWrapper(DispatchRequest request) { ++ ++ } ++ ++ @Override ++ public void assignQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg) throws RocksDBException { ++ String topicQueueKey = getTopic() + "-" + getQueueId(); ++ Long queueOffset = queueOffsetOperator.getTopicQueueNextOffset(topicQueueKey); ++ if (queueOffset == null) { ++ // we will recover topic queue table from rocksdb when we use it. ++ queueOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); ++ queueOffsetOperator.updateQueueOffset(topicQueueKey, queueOffset); ++ } ++ msg.setQueueOffset(queueOffset); ++ ++ // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), ++ // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. ++ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { ++ return; ++ } ++ String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); ++ if (StringUtils.isBlank(multiDispatchQueue)) { ++ return; ++ } ++ String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); ++ Long[] queueOffsets = new Long[queues.length]; ++ for (int i = 0; i < queues.length; i++) { ++ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { ++ String key = MultiDispatch.lmqQueueKey(queues[i]); ++ queueOffsets[i] = queueOffsetOperator.getLmqTopicQueueNextOffset(key); ++ } ++ } ++ MessageAccessor.putProperty(msg, MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET, ++ StringUtils.join(queueOffsets, MixAll.MULTI_DISPATCH_QUEUE_SPLITTER)); ++ msg.removeWaitStorePropertyString(); ++ } ++ ++ @Override ++ public void increaseQueueOffset(QueueOffsetOperator queueOffsetOperator, MessageExtBrokerInner msg, short messageNum) { ++ String topicQueueKey = getTopic() + "-" + getQueueId(); ++ queueOffsetOperator.increaseQueueOffset(topicQueueKey, messageNum); ++ ++ // Handling the multi dispatch message. In the context of a light message queue (as defined in RIP-28), ++ // light message queues are constructed based on message properties, which requires special handling of queue offset of the light message queue. ++ if (!MultiDispatch.isNeedHandleMultiDispatch(this.messageStore.getMessageStoreConfig(), msg.getTopic())) { ++ return; ++ } ++ String multiDispatchQueue = msg.getProperty(MessageConst.PROPERTY_INNER_MULTI_DISPATCH); ++ if (StringUtils.isBlank(multiDispatchQueue)) { ++ return; ++ } ++ String[] queues = multiDispatchQueue.split(MixAll.MULTI_DISPATCH_QUEUE_SPLITTER); ++ for (int i = 0; i < queues.length; i++) { ++ if (this.messageStore.getMessageStoreConfig().isEnableLmq() && MixAll.isLmq(queues[i])) { ++ String key = MultiDispatch.lmqQueueKey(queues[i]); ++ queueOffsetOperator.increaseLmqOffset(key, (short) 1); ++ } ++ } ++ } ++ ++ @Override ++ public long estimateMessageCount(long from, long to, MessageFilter filter) { ++ // todo ++ return 0; ++ } ++ ++ @Override ++ public long getMinOffsetInQueue() { ++ return this.messageStore.getMinOffsetInQueue(this.topic, this.queueId); ++ } ++ ++ private int pullNum(long cqOffset, long maxCqOffset) { ++ long diffLong = maxCqOffset - cqOffset; ++ if (diffLong < Integer.MAX_VALUE) { ++ int diffInt = (int) diffLong; ++ return diffInt > 16 ? 16 : diffInt; ++ } ++ return 16; ++ } ++ ++ @Override ++ public ReferredIterator iterateFrom(final long startIndex) { ++ try { ++ long maxCqOffset = getMaxOffsetInQueue(); ++ if (startIndex < maxCqOffset) { ++ int num = pullNum(startIndex, maxCqOffset); ++ return iterateFrom0(startIndex, num); ++ } ++ } catch (RocksDBException e) { ++ log.error("[RocksDBConsumeQueue] iterateFrom error!", e); ++ } ++ return null; ++ } ++ ++ @Override ++ public ReferredIterator iterateFrom(long startIndex, int count) throws RocksDBException { ++ long maxCqOffset = getMaxOffsetInQueue(); ++ if (startIndex < maxCqOffset) { ++ int num = Math.min((int)(maxCqOffset - startIndex), count); ++ return iterateFrom0(startIndex, num); ++ } ++ return null; ++ } ++ ++ @Override ++ public CqUnit get(long index) { ++ Pair pair = getCqUnitAndStoreTime(index); ++ return pair == null ? null : pair.getObject1(); ++ } ++ ++ @Override ++ public Pair getCqUnitAndStoreTime(long index) { ++ ByteBuffer byteBuffer; ++ try { ++ byteBuffer = this.messageStore.getQueueStore().get(topic, queueId, index); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e); ++ return null; ++ } ++ if (byteBuffer == null || byteBuffer.remaining() < RocksDBConsumeQueueTable.CQ_UNIT_SIZE) { ++ return null; ++ } ++ long phyOffset = byteBuffer.getLong(); ++ int size = byteBuffer.getInt(); ++ long tagCode = byteBuffer.getLong(); ++ long messageStoreTime = byteBuffer.getLong(); ++ return new Pair<>(new CqUnit(index, phyOffset, size, tagCode), messageStoreTime); ++ } ++ ++ @Override ++ public Pair getEarliestUnitAndStoreTime() { ++ try { ++ long minOffset = this.messageStore.getQueueStore().getMinOffsetInQueue(topic, queueId); ++ return getCqUnitAndStoreTime(minOffset); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getEarliestUnitAndStoreTime Failed. topic: {}, queueId: {}", topic, queueId, e); ++ } ++ return null; ++ } ++ ++ @Override ++ public CqUnit getEarliestUnit() { ++ Pair pair = getEarliestUnitAndStoreTime(); ++ return pair == null ? null : pair.getObject1(); ++ } ++ ++ @Override ++ public CqUnit getLatestUnit() { ++ try { ++ long maxOffset = this.messageStore.getQueueStore().getMaxOffsetInQueue(topic, queueId); ++ return get(maxOffset); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("getLatestUnit Failed. topic: {}, queueId: {}, {}", topic, queueId, e.getMessage()); ++ } ++ return null; ++ } ++ ++ @Override ++ public long getLastOffset() { ++ return getMaxPhysicOffset(); ++ } ++ ++ private ReferredIterator iterateFrom0(final long startIndex, final int count) throws RocksDBException { ++ List byteBufferList = this.messageStore.getQueueStore().rangeQuery(topic, queueId, startIndex, count); ++ if (byteBufferList == null || byteBufferList.isEmpty()) { ++ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ log.warn("iterateFrom0 - find nothing, startIndex:{}, count:{}", startIndex, count); ++ } ++ return null; ++ } ++ return new RocksDBConsumeQueueIterator(byteBufferList, startIndex); ++ } ++ ++ @Override ++ public String getTopic() { ++ return topic; ++ } ++ ++ @Override ++ public int getQueueId() { ++ return queueId; ++ } ++ ++ private class RocksDBConsumeQueueIterator implements ReferredIterator { ++ private final List byteBufferList; ++ private final long startIndex; ++ private final int totalCount; ++ private int currentIndex; ++ ++ public RocksDBConsumeQueueIterator(final List byteBufferList, final long startIndex) { ++ this.byteBufferList = byteBufferList; ++ this.startIndex = startIndex; ++ this.totalCount = byteBufferList.size(); ++ this.currentIndex = 0; ++ } ++ ++ @Override ++ public boolean hasNext() { ++ return this.currentIndex < this.totalCount; ++ } ++ ++ @Override ++ public CqUnit next() { ++ if (!hasNext()) { ++ return null; ++ } ++ final int currentIndex = this.currentIndex; ++ final ByteBuffer byteBuffer = this.byteBufferList.get(currentIndex); ++ CqUnit cqUnit = new CqUnit(this.startIndex + currentIndex, byteBuffer.getLong(), byteBuffer.getInt(), byteBuffer.getLong()); ++ this.currentIndex++; ++ return cqUnit; ++ } ++ ++ @Override ++ public void remove() { ++ throw new UnsupportedOperationException("remove"); ++ } ++ ++ @Override ++ public void release() { ++ } ++ ++ @Override ++ public CqUnit nextAndRelease() { ++ try { ++ return next(); ++ } finally { ++ release(); ++ } ++ } ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java +new file mode 100644 +index 000000000..6fa66282e +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueOffsetTable.java +@@ -0,0 +1,641 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.nio.ByteBuffer; ++import java.util.HashMap; ++import java.util.HashSet; ++import java.util.Map; ++import java.util.Set; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++ ++import org.apache.rocketmq.common.Pair; ++import org.apache.rocketmq.common.TopicConfig; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.topic.TopicValidator; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.ConsumeQueue; ++import org.apache.rocketmq.store.DefaultMessageStore; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; ++import org.rocksdb.ColumnFamilyHandle; ++import org.rocksdb.RocksDBException; ++import org.rocksdb.RocksIterator; ++import org.rocksdb.WriteBatch; ++ ++import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; ++import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; ++ ++public class RocksDBConsumeQueueOffsetTable { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); ++ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); ++ ++ private static final byte[] MAX_BYTES = "max".getBytes(CHARSET_UTF8); ++ private static final byte[] MIN_BYTES = "min".getBytes(CHARSET_UTF8); ++ ++ /** ++ * Rocksdb ConsumeQueue's Offset unit. Format: ++ * ++ * 

++     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬─────────────┐
++     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  Max(Min) │  CTRL_1   │   QueueId   │
++     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │  (4 Bytes)  │
++     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴─────────────┤
++     * │                                                    Key Unit                                                   │
++     * │                                                                                                               │
++     *
++ * ++ * 
++     * ┌─────────────────────────────┬────────────────────────┐
++     * │  CommitLog Physical Offset  │   ConsumeQueue Offset  │
++     * │        (8 Bytes)            │    (8 Bytes)           │
++     * ├─────────────────────────────┴────────────────────────┤
++     * │                     Value Unit                       │
++     * │                                                      │
++     *
++ * ConsumeQueue's Offset unit. Size: CommitLog Physical Offset(8) + ConsumeQueue Offset(8) = 16 Bytes ++ */ ++ private static final int OFFSET_PHY_OFFSET = 0; ++ private static final int OFFSET_CQ_OFFSET = 8; ++ /** ++ * ++ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬─────────────┐ ++ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ Max(Min) │ CTRL_1 │ QueueId │ ++ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (3 Bytes) │ (1 Bytes) │ (4 Bytes) │ ++ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴─────────────┤ ++ */ ++ private static final int OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 3 + 1 + 4; ++ private static final int OFFSET_VALUE_LENGTH = 8 + 8; ++ ++ /** ++ * We use a new system topic='CHECKPOINT_TOPIC' to record the maxPhyOffset built by CQ dispatch thread. ++ * @see ConsumeQueueStore#getMaxPhyOffsetInConsumeQueue(), we use it to find the maxPhyOffset built by CQ dispatch thread. ++ * If we do not record the maxPhyOffset, it may take us a long time to start traversing from the head of ++ * RocksDBConsumeQueueOffsetTable to find it. ++ */ ++ private static final String MAX_PHYSICAL_OFFSET_CHECKPOINT = TopicValidator.RMQ_SYS_ROCKSDB_OFFSET_TOPIC; ++ private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES = MAX_PHYSICAL_OFFSET_CHECKPOINT.getBytes(CHARSET_UTF8); ++ private static final int INNER_CHECKPOINT_TOPIC_LEN = OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES.length; ++ private static final ByteBuffer INNER_CHECKPOINT_TOPIC = ByteBuffer.allocateDirect(INNER_CHECKPOINT_TOPIC_LEN); ++ private static final byte[] MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY = new byte[INNER_CHECKPOINT_TOPIC_LEN]; ++ private final ByteBuffer maxPhyOffsetBB; ++ static { ++ buildOffsetKeyByteBuffer0(INNER_CHECKPOINT_TOPIC, MAX_PHYSICAL_OFFSET_CHECKPOINT_BYTES, 0, true); ++ INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); ++ INNER_CHECKPOINT_TOPIC.get(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY); ++ } ++ ++ private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; ++ private final ConsumeQueueRocksDBStorage rocksDBStorage; ++ private final DefaultMessageStore messageStore; ++ ++ private ColumnFamilyHandle offsetCFH; ++ ++ /** ++ * Although we have already put max(min) consumeQueueOffset and physicalOffset in rocksdb, we still hope to get them ++ * from heap to avoid accessing rocksdb. ++ * @see ConsumeQueue#getMaxPhysicOffset(), maxPhysicOffset --> topicQueueMaxCqOffset ++ * @see ConsumeQueue#getMinLogicOffset(), minLogicOffset --> topicQueueMinOffset ++ */ ++ private final Map topicQueueMinOffset; ++ private final Map topicQueueMaxCqOffset; ++ ++ public RocksDBConsumeQueueOffsetTable(RocksDBConsumeQueueTable rocksDBConsumeQueueTable, ++ ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { ++ this.rocksDBConsumeQueueTable = rocksDBConsumeQueueTable; ++ this.rocksDBStorage = rocksDBStorage; ++ this.messageStore = messageStore; ++ this.topicQueueMinOffset = new ConcurrentHashMap(1024); ++ this.topicQueueMaxCqOffset = new ConcurrentHashMap(1024); ++ ++ this.maxPhyOffsetBB = ByteBuffer.allocateDirect(8); ++ } ++ ++ public void load() { ++ this.offsetCFH = this.rocksDBStorage.getOffsetCFHandle(); ++ } ++ ++ public void updateTempTopicQueueMaxOffset(final Pair offsetBBPair, ++ final byte[] topicBytes, final DispatchRequest request, ++ final Map> tempTopicQueueMaxOffsetMap) { ++ buildOffsetKeyAndValueByteBuffer(offsetBBPair, topicBytes, request); ++ ByteBuffer topicQueueId = offsetBBPair.getObject1(); ++ ByteBuffer maxOffsetBB = offsetBBPair.getObject2(); ++ Pair old = tempTopicQueueMaxOffsetMap.get(topicQueueId); ++ if (old == null) { ++ tempTopicQueueMaxOffsetMap.put(topicQueueId, new Pair(maxOffsetBB, request)); ++ } else { ++ long oldMaxOffset = old.getObject1().getLong(OFFSET_CQ_OFFSET); ++ long maxOffset = maxOffsetBB.getLong(OFFSET_CQ_OFFSET); ++ if (maxOffset >= oldMaxOffset) { ++ ERROR_LOG.error("cqOffset invalid1. old: {}, now: {}", oldMaxOffset, maxOffset); ++ } ++ } ++ } ++ ++ public void putMaxPhyAndCqOffset(final Map> tempTopicQueueMaxOffsetMap, ++ final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { ++ for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { ++ writeBatch.put(this.offsetCFH, entry.getKey(), entry.getValue().getObject1()); ++ } ++ ++ appendMaxPhyOffset(writeBatch, maxPhyOffset); ++ } ++ ++ public void putHeapMaxCqOffset(final Map> tempTopicQueueMaxOffsetMap) { ++ for (Map.Entry> entry : tempTopicQueueMaxOffsetMap.entrySet()) { ++ DispatchRequest request = entry.getValue().getObject2(); ++ putHeapMaxCqOffset(request.getTopic(), request.getQueueId(), request.getConsumeQueueOffset()); ++ } ++ } ++ ++ /** ++ * When topic is deleted, we clean up its offset info in rocksdb. ++ * @param topic ++ * @param queueId ++ * @throws RocksDBException ++ */ ++ public void destroyOffset(String topic, int queueId, WriteBatch writeBatch) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer minOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, false); ++ byte[] minOffsetBytes = this.rocksDBStorage.getOffset(minOffsetKey.array()); ++ Long startCQOffset = (minOffsetBytes != null) ? ByteBuffer.wrap(minOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; ++ ++ final ByteBuffer maxOffsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, true); ++ byte[] maxOffsetBytes = this.rocksDBStorage.getOffset(maxOffsetKey.array()); ++ Long endCQOffset = (maxOffsetBytes != null) ? ByteBuffer.wrap(maxOffsetBytes).getLong(OFFSET_CQ_OFFSET) : null; ++ ++ writeBatch.delete(this.offsetCFH, minOffsetKey.array()); ++ writeBatch.delete(this.offsetCFH, maxOffsetKey.array()); ++ ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ removeHeapMinCqOffset(topicQueueId); ++ removeHeapMaxCqOffset(topicQueueId); ++ ++ log.info("RocksDB offset table delete topic: {}, queueId: {}, minOffset: {}, maxOffset: {}", topic, queueId, ++ startCQOffset, endCQOffset); ++ } ++ ++ private void appendMaxPhyOffset(final WriteBatch writeBatch, final long maxPhyOffset) throws RocksDBException { ++ final ByteBuffer maxPhyOffsetBB = this.maxPhyOffsetBB; ++ maxPhyOffsetBB.position(0).limit(8); ++ maxPhyOffsetBB.putLong(maxPhyOffset); ++ maxPhyOffsetBB.flip(); ++ ++ INNER_CHECKPOINT_TOPIC.position(0).limit(INNER_CHECKPOINT_TOPIC_LEN); ++ writeBatch.put(this.offsetCFH, INNER_CHECKPOINT_TOPIC, maxPhyOffsetBB); ++ } ++ ++ public long getMaxPhyOffset() throws RocksDBException { ++ byte[] valueBytes = this.rocksDBStorage.getOffset(MAX_PHYSICAL_OFFSET_CHECKPOINT_KEY); ++ if (valueBytes == null) { ++ return 0; ++ } ++ ByteBuffer valueBB = ByteBuffer.wrap(valueBytes); ++ return valueBB.getLong(0); ++ } ++ ++ /** ++ * Traverse the offset table to find dirty topic ++ * @param existTopicSet ++ * @return ++ */ ++ public Map> iterateOffsetTable2FindDirty(final Set existTopicSet) { ++ Map> topicQueueIdToBeDeletedMap = new HashMap<>(); ++ ++ RocksIterator iterator = null; ++ try { ++ iterator = rocksDBStorage.seekOffsetCF(); ++ if (iterator == null) { ++ return topicQueueIdToBeDeletedMap; ++ } ++ for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { ++ byte[] key = iterator.key(); ++ byte[] value = iterator.value(); ++ if (key == null || key.length <= OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES ++ || value == null || value.length != OFFSET_VALUE_LENGTH) { ++ continue; ++ } ++ ByteBuffer keyBB = ByteBuffer.wrap(key); ++ int topicLen = keyBB.getInt(0); ++ byte[] topicBytes = new byte[topicLen]; ++ /** ++ * "Topic Bytes Array Size" + "CTRL_1" = 4 + 1 ++ */ ++ keyBB.position(4 + 1); ++ keyBB.get(topicBytes); ++ String topic = new String(topicBytes, CHARSET_UTF8); ++ if (TopicValidator.isSystemTopic(topic)) { ++ continue; ++ } ++ ++ /** ++ * "Topic Bytes Array Size" + "CTRL_1" + "Topic Bytes Array" + "CTRL_1" + "Max(min)" + "CTRL_1" ++ * = 4 + 1 + topicLen + 1 + 3 + 1 ++ */ ++ int queueId = keyBB.getInt(4 + 1 + topicLen + 1 + 3 + 1); ++ ++ if (!existTopicSet.contains(topic)) { ++ ByteBuffer valueBB = ByteBuffer.wrap(value); ++ long cqOffset = valueBB.getLong(OFFSET_CQ_OFFSET); ++ ++ Set topicQueueIdSet = topicQueueIdToBeDeletedMap.get(topic); ++ if (topicQueueIdSet == null) { ++ Set newSet = new HashSet<>(); ++ newSet.add(queueId); ++ topicQueueIdToBeDeletedMap.put(topic, newSet); ++ } else { ++ topicQueueIdSet.add(queueId); ++ } ++ ERROR_LOG.info("RocksDBConsumeQueueOffsetTable has dirty cqOffset. topic: {}, queueId: {}, cqOffset: {}", ++ topic, queueId, cqOffset); ++ } ++ } ++ } catch (Exception e) { ++ ERROR_LOG.error("iterateOffsetTable2MarkDirtyCQ Failed.", e); ++ } finally { ++ if (iterator != null) { ++ iterator.close(); ++ } ++ } ++ return topicQueueIdToBeDeletedMap; ++ } ++ ++ public Long getMaxCqOffset(String topic, int queueId) throws RocksDBException { ++ Long maxCqOffset = getHeapMaxCqOffset(topic, queueId); ++ ++ if (maxCqOffset == null) { ++ final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); ++ maxCqOffset = (byteBuffer != null) ? byteBuffer.getLong(OFFSET_CQ_OFFSET) : null; ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ this.topicQueueMaxCqOffset.putIfAbsent(topicQueueId, maxCqOffset != null ? maxCqOffset : -1L); ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("updateMaxOffsetInQueue. {}, {}", topicQueueId, maxCqOffset); ++ } ++ } ++ ++ return maxCqOffset; ++ } ++ ++ /** ++ * truncate dirty offset in rocksdb ++ * @param offsetToTruncate ++ * @throws RocksDBException ++ */ ++ public void truncateDirty(long offsetToTruncate) throws RocksDBException { ++ correctMaxPyhOffset(offsetToTruncate); ++ ++ ConcurrentMap allTopicConfigMap = this.messageStore.getTopicConfigs(); ++ if (allTopicConfigMap == null) { ++ return; ++ } ++ for (TopicConfig topicConfig : allTopicConfigMap.values()) { ++ for (int i = 0; i < topicConfig.getWriteQueueNums(); i++) { ++ truncateDirtyOffset(topicConfig.getTopicName(), i); ++ } ++ } ++ } ++ ++ private Pair isMinOffsetOk(final String topic, final int queueId, final long minPhyOffset) throws RocksDBException { ++ PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); ++ if (phyAndCQOffset != null) { ++ final long phyOffset = phyAndCQOffset.getPhyOffset(); ++ final long cqOffset = phyAndCQOffset.getCqOffset(); ++ ++ return (phyOffset >= minPhyOffset) ? new Pair(true, cqOffset) : new Pair(false, cqOffset); ++ } ++ ByteBuffer byteBuffer = getMinPhyAndCqOffsetInKV(topic, queueId); ++ if (byteBuffer == null) { ++ return new Pair(false, 0L); ++ } ++ final long phyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); ++ final long cqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); ++ if (phyOffset >= minPhyOffset) { ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ PhyAndCQOffset newPhyAndCQOffset = new PhyAndCQOffset(phyOffset, cqOffset); ++ this.topicQueueMinOffset.putIfAbsent(topicQueueId, newPhyAndCQOffset); ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("updateMinOffsetInQueue. {}, {}", topicQueueId, newPhyAndCQOffset); ++ } ++ return new Pair(true, cqOffset); ++ } ++ return new Pair(false, cqOffset); ++ } ++ ++ private void truncateDirtyOffset(String topic, int queueId) throws RocksDBException { ++ final ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); ++ if (byteBuffer == null) { ++ return; ++ } ++ ++ long maxPhyOffset = byteBuffer.getLong(OFFSET_PHY_OFFSET); ++ long maxCqOffset = byteBuffer.getLong(OFFSET_CQ_OFFSET); ++ long maxPhyOffsetInCQ = getMaxPhyOffset(); ++ ++ if (maxPhyOffset >= maxPhyOffsetInCQ) { ++ correctMaxCqOffset(topic, queueId, maxCqOffset, maxPhyOffsetInCQ); ++ Long newMaxCqOffset = getHeapMaxCqOffset(topic, queueId); ++ ROCKSDB_LOG.warn("truncateDirtyLogicFile topic: {}, queueId: {} from {} to {}", topic, queueId, ++ maxPhyOffset, newMaxCqOffset); ++ } ++ } ++ ++ private void correctMaxPyhOffset(long maxPhyOffset) throws RocksDBException { ++ if (!this.rocksDBStorage.hold()) { ++ return; ++ } ++ try { ++ WriteBatch writeBatch = new WriteBatch(); ++ long oldMaxPhyOffset = getMaxPhyOffset(); ++ if (oldMaxPhyOffset <= maxPhyOffset) { ++ return; ++ } ++ log.info("correctMaxPyhOffset, oldMaxPhyOffset={}, newMaxPhyOffset={}", oldMaxPhyOffset, maxPhyOffset); ++ appendMaxPhyOffset(writeBatch, maxPhyOffset); ++ this.rocksDBStorage.batchPut(writeBatch); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("correctMaxPyhOffset Failed.", e); ++ throw e; ++ } finally { ++ this.rocksDBStorage.release(); ++ } ++ } ++ ++ public long getMinCqOffset(String topic, int queueId) throws RocksDBException { ++ final long minPhyOffset = this.messageStore.getMinPhyOffset(); ++ Pair pair = isMinOffsetOk(topic, queueId, minPhyOffset); ++ final long cqOffset = pair.getObject2(); ++ if (!pair.getObject1() && correctMinCqOffset(topic, queueId, cqOffset, minPhyOffset)) { ++ PhyAndCQOffset phyAndCQOffset = getHeapMinOffset(topic, queueId); ++ if (phyAndCQOffset != null) { ++ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("getMinOffsetInQueue miss heap. topic: {}, queueId: {}, old: {}, new: {}", ++ topic, queueId, cqOffset, phyAndCQOffset); ++ } ++ return phyAndCQOffset.getCqOffset(); ++ } ++ } ++ return cqOffset; ++ } ++ ++ public Long getMaxPhyOffset(String topic, int queueId) { ++ try { ++ ByteBuffer byteBuffer = getMaxPhyAndCqOffsetInKV(topic, queueId); ++ if (byteBuffer != null) { ++ return byteBuffer.getLong(OFFSET_PHY_OFFSET); ++ } ++ } catch (Exception e) { ++ ERROR_LOG.info("getMaxPhyOffset error. topic: {}, queueId: {}", topic, queueId); ++ } ++ return null; ++ } ++ ++ private ByteBuffer getMinPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException { ++ return getPhyAndCqOffsetInKV(topic, queueId, false); ++ } ++ ++ private ByteBuffer getMaxPhyAndCqOffsetInKV(String topic, int queueId) throws RocksDBException { ++ return getPhyAndCqOffsetInKV(topic, queueId, true); ++ } ++ ++ private ByteBuffer getPhyAndCqOffsetInKV(String topic, int queueId, boolean max) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer keyBB = buildOffsetKeyByteBuffer(topicBytes, queueId, max); ++ ++ byte[] value = this.rocksDBStorage.getOffset(keyBB.array()); ++ return (value != null) ? ByteBuffer.wrap(value) : null; ++ } ++ ++ private String buildTopicQueueId(final String topic, final int queueId) { ++ return topic + "-" + queueId; ++ } ++ ++ private void putHeapMinCqOffset(final String topic, final int queueId, final long minPhyOffset, final long minCQOffset) { ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ PhyAndCQOffset phyAndCQOffset = new PhyAndCQOffset(minPhyOffset, minCQOffset); ++ this.topicQueueMinOffset.put(topicQueueId, phyAndCQOffset); ++ } ++ ++ private void putHeapMaxCqOffset(final String topic, final int queueId, final long maxCQOffset) { ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ Long oldMaxCqOffset = this.topicQueueMaxCqOffset.put(topicQueueId, maxCQOffset); ++ if (oldMaxCqOffset != null && oldMaxCqOffset > maxCQOffset) { ++ ERROR_LOG.error("cqOffset invalid0. old: {}, now: {}", oldMaxCqOffset, maxCQOffset); ++ } ++ } ++ ++ private PhyAndCQOffset getHeapMinOffset(final String topic, final int queueId) { ++ return this.topicQueueMinOffset.get(buildTopicQueueId(topic, queueId)); ++ } ++ ++ private Long getHeapMaxCqOffset(final String topic, final int queueId) { ++ String topicQueueId = buildTopicQueueId(topic, queueId); ++ return this.topicQueueMaxCqOffset.get(topicQueueId); ++ } ++ ++ private PhyAndCQOffset removeHeapMinCqOffset(String topicQueueId) { ++ return this.topicQueueMinOffset.remove(topicQueueId); ++ } ++ ++ private Long removeHeapMaxCqOffset(String topicQueueId) { ++ return this.topicQueueMaxCqOffset.remove(topicQueueId); ++ } ++ ++ private void updateCqOffset(final String topic, final int queueId, final long phyOffset, ++ final long cqOffset, boolean max) throws RocksDBException { ++ if (!this.rocksDBStorage.hold()) { ++ return; ++ } ++ WriteBatch writeBatch = new WriteBatch(); ++ try { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer offsetKey = buildOffsetKeyByteBuffer(topicBytes, queueId, max); ++ ++ final ByteBuffer offsetValue = buildOffsetValueByteBuffer(phyOffset, cqOffset); ++ writeBatch.put(this.offsetCFH, offsetKey.array(), offsetValue.array()); ++ this.rocksDBStorage.batchPut(writeBatch); ++ ++ if (max) { ++ putHeapMaxCqOffset(topic, queueId, cqOffset); ++ } else { ++ putHeapMinCqOffset(topic, queueId, phyOffset, cqOffset); ++ } ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("updateCqOffset({}) failed.", max ? "max" : "min", e); ++ throw e; ++ } finally { ++ writeBatch.close(); ++ this.rocksDBStorage.release(); ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("updateCqOffset({}). topic: {}, queueId: {}, phyOffset: {}, cqOffset: {}", ++ max ? "max" : "min", topic, queueId, phyOffset, cqOffset); ++ } ++ } ++ } ++ ++ private boolean correctMaxCqOffset(final String topic, final int queueId, final long maxCQOffset, ++ final long maxPhyOffsetInCQ) throws RocksDBException { ++ // 'getMinOffsetInQueue' may correct minCqOffset and put it into heap ++ long minCQOffset = getMinCqOffset(topic, queueId); ++ PhyAndCQOffset minPhyAndCQOffset = getHeapMinOffset(topic, queueId); ++ if (minPhyAndCQOffset == null ++ || minPhyAndCQOffset.getCqOffset() != minCQOffset ++ || minPhyAndCQOffset.getPhyOffset() > maxPhyOffsetInCQ) { ++ ROCKSDB_LOG.info("[BUG] correctMaxCqOffset error! topic: {}, queueId: {}, maxPhyOffsetInCQ: {}, " ++ + "minCqOffset: {}, phyAndCQOffset: {}", ++ topic, queueId, maxPhyOffsetInCQ, minCQOffset, minPhyAndCQOffset); ++ throw new RocksDBException("correctMaxCqOffset error"); ++ } ++ ++ long high = maxCQOffset; ++ long low = minCQOffset; ++ PhyAndCQOffset targetPhyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, ++ low, maxPhyOffsetInCQ, false); ++ ++ long targetCQOffset = targetPhyAndCQOffset.getCqOffset(); ++ long targetPhyOffset = targetPhyAndCQOffset.getPhyOffset(); ++ ++ if (targetCQOffset == -1) { ++ if (maxCQOffset != minCQOffset) { ++ updateCqOffset(topic, queueId, minPhyAndCQOffset.getPhyOffset(), minCQOffset, true); ++ } ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyAndCQOffset.getPhyOffset()); ++ } ++ return false; ++ } else { ++ updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, true); ++ return true; ++ } ++ } ++ ++ private boolean correctMinCqOffset(final String topic, final int queueId, ++ final long minCQOffset, final long minPhyOffset) throws RocksDBException { ++ final ByteBuffer maxBB = getMaxPhyAndCqOffsetInKV(topic, queueId); ++ if (maxBB == null) { ++ updateCqOffset(topic, queueId, minPhyOffset, 0L, false); ++ return true; ++ } ++ final long maxPhyOffset = maxBB.getLong(OFFSET_PHY_OFFSET); ++ final long maxCQOffset = maxBB.getLong(OFFSET_CQ_OFFSET); ++ ++ if (maxPhyOffset < minPhyOffset) { ++ updateCqOffset(topic, queueId, minPhyOffset, maxCQOffset + 1, false); ++ return true; ++ } ++ ++ long high = maxCQOffset; ++ long low = minCQOffset; ++ PhyAndCQOffset phyAndCQOffset = this.rocksDBConsumeQueueTable.binarySearchInCQ(topic, queueId, high, low, ++ minPhyOffset, true); ++ long targetCQOffset = phyAndCQOffset.getCqOffset(); ++ long targetPhyOffset = phyAndCQOffset.getPhyOffset(); ++ ++ if (targetCQOffset == -1) { ++ if (maxCQOffset != minCQOffset) { ++ updateCqOffset(topic, queueId, maxPhyOffset, maxCQOffset, false); ++ } ++ if (messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("correct error. {}, {}, {}, {}, {}", topic, queueId, minCQOffset, maxCQOffset, minPhyOffset); ++ } ++ return false; ++ } else { ++ updateCqOffset(topic, queueId, targetPhyOffset, targetCQOffset, false); ++ return true; ++ } ++ } ++ ++ public static Pair getOffsetByteBufferPair() { ++ ByteBuffer offsetKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN); ++ ByteBuffer offsetValue = ByteBuffer.allocateDirect(OFFSET_VALUE_LENGTH); ++ return new Pair<>(offsetKey, offsetValue); ++ } ++ ++ private void buildOffsetKeyAndValueByteBuffer(final Pair offsetBBPair, ++ final byte[] topicBytes, final DispatchRequest request) { ++ final ByteBuffer offsetKey = offsetBBPair.getObject1(); ++ buildOffsetKeyByteBuffer(offsetKey, topicBytes, request.getQueueId(), true); ++ ++ final ByteBuffer offsetValue = offsetBBPair.getObject2(); ++ buildOffsetValueByteBuffer(offsetValue, request.getCommitLogOffset(), request.getConsumeQueueOffset()); ++ } ++ ++ private ByteBuffer buildOffsetKeyByteBuffer(final byte[] topicBytes, final int queueId, final boolean max) { ++ ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); ++ return byteBuffer; ++ } ++ ++ private void buildOffsetKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final boolean max) { ++ byteBuffer.position(0).limit(OFFSET_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ buildOffsetKeyByteBuffer0(byteBuffer, topicBytes, queueId, max); ++ } ++ ++ private static void buildOffsetKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, ++ final boolean max) { ++ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1); ++ if (max) { ++ byteBuffer.put(MAX_BYTES); ++ } else { ++ byteBuffer.put(MIN_BYTES); ++ } ++ byteBuffer.put(CTRL_1).putInt(queueId); ++ byteBuffer.flip(); ++ } ++ ++ private void buildOffsetValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { ++ byteBuffer.position(0).limit(OFFSET_VALUE_LENGTH); ++ buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); ++ } ++ ++ private ByteBuffer buildOffsetValueByteBuffer(final long phyOffset, final long cqOffset) { ++ final ByteBuffer byteBuffer = ByteBuffer.allocate(OFFSET_VALUE_LENGTH); ++ buildOffsetValueByteBuffer0(byteBuffer, phyOffset, cqOffset); ++ return byteBuffer; ++ } ++ ++ private void buildOffsetValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final long cqOffset) { ++ byteBuffer.putLong(phyOffset).putLong(cqOffset); ++ byteBuffer.flip(); ++ } ++ ++ static class PhyAndCQOffset { ++ private final long phyOffset; ++ private final long cqOffset; ++ ++ public PhyAndCQOffset(final long phyOffset, final long cqOffset) { ++ this.phyOffset = phyOffset; ++ this.cqOffset = cqOffset; ++ } ++ ++ public long getPhyOffset() { ++ return this.phyOffset; ++ } ++ ++ public long getCqOffset() { ++ return this.cqOffset; ++ } ++ ++ @Override ++ public String toString() { ++ return "[cqOffset=" + cqOffset + ", phyOffset=" + phyOffset + "]"; ++ } ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +new file mode 100644 +index 000000000..78456cfcd +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueStore.java +@@ -0,0 +1,441 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.io.File; ++import java.nio.ByteBuffer; ++import java.util.ArrayList; ++import java.util.HashMap; ++import java.util.List; ++import java.util.Map; ++import java.util.Set; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.Executors; ++import java.util.concurrent.ScheduledExecutorService; ++import java.util.concurrent.TimeUnit; ++ ++import org.apache.commons.io.FileUtils; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.BoundaryType; ++import org.apache.rocketmq.common.Pair; ++import org.apache.rocketmq.common.ThreadFactoryImpl; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.DefaultMessageStore; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.config.BrokerRole; ++import org.apache.rocketmq.store.config.StorePathConfigHelper; ++import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; ++import org.rocksdb.RocksDBException; ++import org.rocksdb.WriteBatch; ++ ++public class RocksDBConsumeQueueStore extends AbstractConsumeQueueStore { ++ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); ++ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); ++ ++ public static final byte CTRL_0 = '\u0000'; ++ public static final byte CTRL_1 = '\u0001'; ++ public static final byte CTRL_2 = '\u0002'; ++ ++ private static final int BATCH_SIZE = 16; ++ public static final int MAX_KEY_LEN = 300; ++ ++ private final ScheduledExecutorService scheduledExecutorService; ++ private final String storePath; ++ ++ /** ++ * we use two tables with different ColumnFamilyHandle, called RocksDBConsumeQueueTable and RocksDBConsumeQueueOffsetTable. ++ * 1.RocksDBConsumeQueueTable uses to store CqUnit[physicalOffset, msgSize, tagHashCode, msgStoreTime] ++ * 2.RocksDBConsumeQueueOffsetTable uses to store physicalOffset and consumeQueueOffset(@see PhyAndCQOffset) of topic-queueId ++ */ ++ private final ConsumeQueueRocksDBStorage rocksDBStorage; ++ private final RocksDBConsumeQueueTable rocksDBConsumeQueueTable; ++ private final RocksDBConsumeQueueOffsetTable rocksDBConsumeQueueOffsetTable; ++ ++ private final WriteBatch writeBatch; ++ private final List bufferDRList; ++ private final List> cqBBPairList; ++ private final List> offsetBBPairList; ++ private final Map> tempTopicQueueMaxOffsetMap; ++ private volatile boolean isCQError = false; ++ ++ public RocksDBConsumeQueueStore(DefaultMessageStore messageStore) { ++ super(messageStore); ++ ++ this.storePath = StorePathConfigHelper.getStorePathConsumeQueue(messageStoreConfig.getStorePathRootDir()); ++ this.rocksDBStorage = new ConsumeQueueRocksDBStorage(messageStore, storePath, 4); ++ this.rocksDBConsumeQueueTable = new RocksDBConsumeQueueTable(rocksDBStorage, messageStore); ++ this.rocksDBConsumeQueueOffsetTable = new RocksDBConsumeQueueOffsetTable(rocksDBConsumeQueueTable, rocksDBStorage, messageStore); ++ ++ this.writeBatch = new WriteBatch(); ++ this.bufferDRList = new ArrayList(BATCH_SIZE); ++ this.cqBBPairList = new ArrayList(BATCH_SIZE); ++ this.offsetBBPairList = new ArrayList(BATCH_SIZE); ++ for (int i = 0; i < BATCH_SIZE; i++) { ++ this.cqBBPairList.add(RocksDBConsumeQueueTable.getCQByteBufferPair()); ++ this.offsetBBPairList.add(RocksDBConsumeQueueOffsetTable.getOffsetByteBufferPair()); ++ } ++ ++ this.tempTopicQueueMaxOffsetMap = new HashMap<>(); ++ this.scheduledExecutorService = Executors.newSingleThreadScheduledExecutor( ++ new ThreadFactoryImpl("RocksDBConsumeQueueStoreScheduledThread", messageStore.getBrokerIdentity())); ++ } ++ ++ @Override ++ public void start() { ++ log.info("RocksDB ConsumeQueueStore start!"); ++ this.scheduledExecutorService.scheduleAtFixedRate(() -> { ++ this.rocksDBStorage.statRocksdb(ROCKSDB_LOG); ++ }, 10, this.messageStoreConfig.getStatRocksDBCQIntervalSec(), TimeUnit.SECONDS); ++ ++ this.scheduledExecutorService.scheduleWithFixedDelay(() -> { ++ cleanDirty(messageStore.getTopicConfigs().keySet()); ++ }, 10, this.messageStoreConfig.getCleanRocksDBDirtyCQIntervalMin(), TimeUnit.MINUTES); ++ } ++ ++ private void cleanDirty(final Set existTopicSet) { ++ try { ++ Map> topicQueueIdToBeDeletedMap = ++ this.rocksDBConsumeQueueOffsetTable.iterateOffsetTable2FindDirty(existTopicSet); ++ ++ for (Map.Entry> entry : topicQueueIdToBeDeletedMap.entrySet()) { ++ String topic = entry.getKey(); ++ for (int queueId : entry.getValue()) { ++ destroy(new RocksDBConsumeQueue(topic, queueId)); ++ } ++ } ++ } catch (Exception e) { ++ log.error("cleanUnusedTopic Failed.", e); ++ } ++ } ++ ++ @Override ++ public boolean load() { ++ boolean result = this.rocksDBStorage.start(); ++ this.rocksDBConsumeQueueTable.load(); ++ this.rocksDBConsumeQueueOffsetTable.load(); ++ log.info("load rocksdb consume queue {}.", result ? "OK" : "Failed"); ++ return result; ++ } ++ ++ @Override ++ public boolean loadAfterDestroy() { ++ return this.load(); ++ } ++ ++ @Override ++ public void recover() { ++ // ignored ++ } ++ ++ @Override ++ public boolean recoverConcurrently() { ++ return true; ++ } ++ ++ @Override ++ public boolean shutdown() { ++ this.scheduledExecutorService.shutdown(); ++ return shutdownInner(); ++ } ++ ++ private boolean shutdownInner() { ++ return this.rocksDBStorage.shutdown(); ++ } ++ ++ @Override ++ public void putMessagePositionInfoWrapper(DispatchRequest request) throws RocksDBException { ++ if (request == null || this.bufferDRList.size() >= BATCH_SIZE) { ++ putMessagePosition(); ++ } ++ if (request != null) { ++ this.bufferDRList.add(request); ++ } ++ } ++ ++ public void putMessagePosition() throws RocksDBException { ++ final int maxRetries = 30; ++ for (int i = 0; i < maxRetries; i++) { ++ if (putMessagePosition0()) { ++ if (this.isCQError) { ++ this.messageStore.getRunningFlags().clearLogicsQueueError(); ++ this.isCQError = false; ++ } ++ return; ++ } else { ++ ERROR_LOG.warn("{} put cq Failed. retryTime: {}", i); ++ try { ++ Thread.sleep(100); ++ } catch (InterruptedException ignored) { ++ } ++ } ++ } ++ if (!this.isCQError) { ++ ERROR_LOG.error("[BUG] put CQ Failed."); ++ this.messageStore.getRunningFlags().makeLogicsQueueError(); ++ this.isCQError = true; ++ } ++ throw new RocksDBException("put CQ Failed"); ++ } ++ ++ private boolean putMessagePosition0() { ++ if (!this.rocksDBStorage.hold()) { ++ return false; ++ } ++ ++ final Map> tempTopicQueueMaxOffsetMap = this.tempTopicQueueMaxOffsetMap; ++ try { ++ final List bufferDRList = this.bufferDRList; ++ final int size = bufferDRList.size(); ++ if (size == 0) { ++ return true; ++ } ++ final List> cqBBPairList = this.cqBBPairList; ++ final List> offsetBBPairList = this.offsetBBPairList; ++ final WriteBatch writeBatch = this.writeBatch; ++ ++ long maxPhyOffset = 0; ++ for (int i = size - 1; i >= 0; i--) { ++ final DispatchRequest request = bufferDRList.get(i); ++ final byte[] topicBytes = request.getTopic().getBytes(DataConverter.CHARSET_UTF8); ++ ++ this.rocksDBConsumeQueueTable.buildAndPutCQByteBuffer(cqBBPairList.get(i), topicBytes, request, writeBatch); ++ this.rocksDBConsumeQueueOffsetTable.updateTempTopicQueueMaxOffset(offsetBBPairList.get(i), ++ topicBytes, request, tempTopicQueueMaxOffsetMap); ++ ++ final int msgSize = request.getMsgSize(); ++ final long phyOffset = request.getCommitLogOffset(); ++ if (phyOffset + msgSize >= maxPhyOffset) { ++ maxPhyOffset = phyOffset + msgSize; ++ } ++ } ++ ++ this.rocksDBConsumeQueueOffsetTable.putMaxPhyAndCqOffset(tempTopicQueueMaxOffsetMap, writeBatch, maxPhyOffset); ++ ++ // clear writeBatch in batchPut ++ this.rocksDBStorage.batchPut(writeBatch); ++ ++ this.rocksDBConsumeQueueOffsetTable.putHeapMaxCqOffset(tempTopicQueueMaxOffsetMap); ++ ++ long storeTimeStamp = bufferDRList.get(size - 1).getStoreTimestamp(); ++ if (this.messageStore.getMessageStoreConfig().getBrokerRole() == BrokerRole.SLAVE ++ || this.messageStore.getMessageStoreConfig().isEnableDLegerCommitLog()) { ++ this.messageStore.getStoreCheckpoint().setPhysicMsgTimestamp(storeTimeStamp); ++ } ++ this.messageStore.getStoreCheckpoint().setLogicsMsgTimestamp(storeTimeStamp); ++ ++ notifyMessageArriveAndClear(); ++ return true; ++ } catch (Exception e) { ++ ERROR_LOG.error("putMessagePosition0 Failed.", e); ++ return false; ++ } finally { ++ tempTopicQueueMaxOffsetMap.clear(); ++ this.rocksDBStorage.release(); ++ } ++ } ++ ++ private void notifyMessageArriveAndClear() { ++ final List bufferDRList = this.bufferDRList; ++ try { ++ for (DispatchRequest dp : bufferDRList) { ++ this.messageStore.notifyMessageArriveIfNecessary(dp); ++ } ++ } catch (Exception e) { ++ ERROR_LOG.error("notifyMessageArriveAndClear Failed.", e); ++ } finally { ++ bufferDRList.clear(); ++ } ++ } ++ ++ @Override ++ public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { ++ return this.rocksDBConsumeQueueTable.rangeQuery(topic, queueId, startIndex, num); ++ } ++ ++ @Override ++ public ByteBuffer get(final String topic, final int queueId, final long cqOffset) throws RocksDBException { ++ return this.rocksDBConsumeQueueTable.getCQInKV(topic, queueId, cqOffset); ++ } ++ ++ /** ++ * Ignored, we do not need to recover topicQueueTable and correct minLogicOffset. Because we will correct them ++ * when we use them, we call it lazy correction. ++ * @see RocksDBConsumeQueue#increaseQueueOffset(QueueOffsetOperator, MessageExtBrokerInner, short) ++ * @see org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable#getMinCqOffset(String, int) ++ */ ++ @Override ++ public void recoverOffsetTable(long minPhyOffset) { ++ ++ } ++ ++ @Override ++ public void destroy() { ++ try { ++ shutdownInner(); ++ FileUtils.deleteDirectory(new File(this.storePath)); ++ } catch (Exception e) { ++ ERROR_LOG.error("destroy cq Failed. {}", this.storePath, e); ++ } ++ } ++ ++ @Override ++ public void destroy(ConsumeQueueInterface consumeQueue) throws RocksDBException { ++ String topic = consumeQueue.getTopic(); ++ int queueId = consumeQueue.getQueueId(); ++ if (StringUtils.isEmpty(topic) || queueId < 0 || !this.rocksDBStorage.hold()) { ++ return; ++ } ++ ++ WriteBatch writeBatch = new WriteBatch(); ++ try { ++ this.rocksDBConsumeQueueTable.destroyCQ(topic, queueId, writeBatch); ++ this.rocksDBConsumeQueueOffsetTable.destroyOffset(topic, queueId, writeBatch); ++ ++ this.rocksDBStorage.batchPut(writeBatch); ++ } catch (RocksDBException e) { ++ ERROR_LOG.error("kv deleteTopic {} Failed.", topic, e); ++ throw e; ++ } finally { ++ writeBatch.close(); ++ this.rocksDBStorage.release(); ++ } ++ } ++ ++ @Override ++ public boolean flush(ConsumeQueueInterface consumeQueue, int flushLeastPages) { ++ try { ++ this.rocksDBStorage.flushWAL(); ++ } catch (Exception e) { ++ } ++ return true; ++ } ++ ++ @Override ++ public void checkSelf() { ++ // ignored ++ } ++ ++ @Override ++ public int deleteExpiredFile(ConsumeQueueInterface consumeQueue, long minCommitLogPos) { ++ // ignored ++ return 0; ++ } ++ ++ /** ++ * We do not need to truncate dirty CQ in RocksDBConsumeQueueTable, Because dirty CQ in RocksDBConsumeQueueTable ++ * will be rewritten by new KV when new messages are appended or will be cleaned up when topics are deleted. ++ * But dirty offset info in RocksDBConsumeQueueOffsetTable must be truncated, because we use offset info in ++ * RocksDBConsumeQueueOffsetTable to rebuild topicQueueTable(@see RocksDBConsumeQueue#increaseQueueOffset). ++ * @param offsetToTruncate ++ * @throws RocksDBException ++ */ ++ @Override ++ public void truncateDirty(long offsetToTruncate) throws RocksDBException { ++ long maxPhyOffsetInRocksdb = getMaxPhyOffsetInConsumeQueue(); ++ if (offsetToTruncate >= maxPhyOffsetInRocksdb) { ++ return; ++ } ++ ++ this.rocksDBConsumeQueueOffsetTable.truncateDirty(offsetToTruncate); ++ } ++ ++ @Override ++ public void cleanExpired(final long minPhyOffset) { ++ this.rocksDBStorage.manualCompaction(minPhyOffset); ++ } ++ ++ @Override ++ public long getOffsetInQueueByTime(String topic, int queueId, long timestamp, BoundaryType boundaryType) throws RocksDBException { ++ final long minPhysicOffset = this.messageStore.getMinPhyOffset(); ++ long low = this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); ++ Long high = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); ++ if (high == null || high == -1) { ++ return 0; ++ } ++ return this.rocksDBConsumeQueueTable.binarySearchInCQByTime(topic, queueId, high, low, timestamp, minPhysicOffset); ++ } ++ ++ @Override ++ public long getMaxOffsetInQueue(String topic, int queueId) throws RocksDBException { ++ Long maxOffset = this.rocksDBConsumeQueueOffsetTable.getMaxCqOffset(topic, queueId); ++ return (maxOffset != null) ? maxOffset + 1 : 0; ++ } ++ ++ @Override ++ public long getMinOffsetInQueue(String topic, int queueId) throws RocksDBException { ++ return this.rocksDBConsumeQueueOffsetTable.getMinCqOffset(topic, queueId); ++ } ++ ++ @Override ++ public Long getMaxPhyOffsetInConsumeQueue(String topic, int queueId) { ++ return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(topic, queueId); ++ } ++ ++ @Override ++ public long getMaxPhyOffsetInConsumeQueue() throws RocksDBException { ++ return this.rocksDBConsumeQueueOffsetTable.getMaxPhyOffset(); ++ } ++ ++ @Override ++ public ConsumeQueueInterface findOrCreateConsumeQueue(String topic, int queueId) { ++ ConcurrentMap map = this.consumeQueueTable.get(topic); ++ if (null == map) { ++ ConcurrentMap newMap = new ConcurrentHashMap<>(128); ++ ConcurrentMap oldMap = this.consumeQueueTable.putIfAbsent(topic, newMap); ++ if (oldMap != null) { ++ map = oldMap; ++ } else { ++ map = newMap; ++ } ++ } ++ ++ ConsumeQueueInterface logic = map.get(queueId); ++ if (logic != null) { ++ return logic; ++ } ++ ++ ConsumeQueueInterface newLogic = new RocksDBConsumeQueue(this.messageStore, topic, queueId); ++ ConsumeQueueInterface oldLogic = map.putIfAbsent(queueId, newLogic); ++ ++ return oldLogic != null ? oldLogic : newLogic; ++ } ++ ++ @Override ++ public long rollNextFile(ConsumeQueueInterface consumeQueue, long offset) { ++ return 0; ++ } ++ ++ @Override ++ public boolean isFirstFileExist(ConsumeQueueInterface consumeQueue) { ++ return true; ++ } ++ ++ @Override ++ public boolean isFirstFileAvailable(ConsumeQueueInterface consumeQueue) { ++ return true; ++ } ++ ++ @Override ++ public long getTotalSize() { ++ return 0; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java +new file mode 100644 +index 000000000..0a735ea27 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/queue/RocksDBConsumeQueueTable.java +@@ -0,0 +1,312 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.queue; ++ ++import java.nio.ByteBuffer; ++import java.util.ArrayList; ++import java.util.List; ++ ++import org.apache.rocketmq.common.Pair; ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.DefaultMessageStore; ++import org.apache.rocketmq.store.DispatchRequest; ++import org.apache.rocketmq.store.queue.RocksDBConsumeQueueOffsetTable.PhyAndCQOffset; ++import org.apache.rocketmq.store.rocksdb.ConsumeQueueRocksDBStorage; ++import org.rocksdb.ColumnFamilyHandle; ++import org.rocksdb.RocksDBException; ++import org.rocksdb.WriteBatch; ++ ++import static org.apache.rocketmq.common.utils.DataConverter.CHARSET_UTF8; ++import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_0; ++import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_1; ++import static org.apache.rocketmq.store.queue.RocksDBConsumeQueueStore.CTRL_2; ++ ++/** ++ * We use RocksDBConsumeQueueTable to store cqUnit. ++ */ ++public class RocksDBConsumeQueueTable { ++ private static final Logger log = LoggerFactory.getLogger(LoggerName.STORE_LOGGER_NAME); ++ private static final Logger ROCKSDB_LOG = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); ++ private static final Logger ERROR_LOG = LoggerFactory.getLogger(LoggerName.STORE_ERROR_LOGGER_NAME); ++ ++ /** ++ * Rocksdb ConsumeQueue's store unit. Format: ++ * ++ * 
++     * ┌─────────────────────────┬───────────┬───────────────────────┬───────────┬───────────┬───────────┬───────────────────────┐
++     * │ Topic Bytes Array Size  │  CTRL_1   │   Topic Bytes Array   │  CTRL_1   │  QueueId  │  CTRL_1   │  ConsumeQueue Offset  │
++     * │        (4 Bytes)        │ (1 Bytes) │       (n Bytes)       │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │     (8 Bytes)         │
++     * ├─────────────────────────┴───────────┴───────────────────────┴───────────┴───────────┴───────────┴───────────────────────┤
++     * │                                                    Key Unit                                                             │
++     * │                                                                                                                         │
++     *
++ * ++ * 
++     * ┌─────────────────────────────┬───────────────────┬──────────────────┬──────────────────┐
++     * │  CommitLog Physical Offset  │      Body Size    │   Tag HashCode   │  Msg Store Time  │
++     * │        (8 Bytes)            │      (4 Bytes)    │    (8 Bytes)     │    (8 Bytes)     │
++     * ├─────────────────────────────┴───────────────────┴──────────────────┴──────────────────┤
++     * │                                                    Value Unit                         │
++     * │                                                                                       │
++     *
++ * ConsumeQueue's store unit. Size: ++ * CommitLog Physical Offset(8) + Body Size(4) + Tag HashCode(8) + Msg Store Time(8) = 28 Bytes ++ */ ++ private static final int PHY_OFFSET_OFFSET = 0; ++ private static final int PHY_MSG_LEN_OFFSET = 8; ++ private static final int MSG_TAG_HASHCODE_OFFSET = 12; ++ private static final int MSG_STORE_TIME_SIZE_OFFSET = 20; ++ public static final int CQ_UNIT_SIZE = 8 + 4 + 8 + 8; ++ ++ /** ++ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────┬───────────────────────┐ ++ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_1 │ ConsumeQueue Offset │ ++ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ (8 Bytes) │ ++ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────┴───────────────────────┤ ++ */ ++ private static final int CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1 + 8; ++ ++ /** ++ * ┌─────────────────────────┬───────────┬───────────┬───────────┬───────────────────┐ ++ * │ Topic Bytes Array Size │ CTRL_1 │ CTRL_1 │ QueueId │ CTRL_0(CTRL_2) │ ++ * │ (4 Bytes) │ (1 Bytes) │ (1 Bytes) │ (4 Bytes) │ (1 Bytes) │ ++ * ├─────────────────────────┴───────────┴───────────┴───────────┴───────────────────┤ ++ */ ++ private static final int DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES = 4 + 1 + 1 + 4 + 1; ++ ++ private final ConsumeQueueRocksDBStorage rocksDBStorage; ++ private final DefaultMessageStore messageStore; ++ ++ private ColumnFamilyHandle defaultCFH; ++ ++ public RocksDBConsumeQueueTable(ConsumeQueueRocksDBStorage rocksDBStorage, DefaultMessageStore messageStore) { ++ this.rocksDBStorage = rocksDBStorage; ++ this.messageStore = messageStore; ++ } ++ ++ public void load() { ++ this.defaultCFH = this.rocksDBStorage.getDefaultCFHandle(); ++ } ++ ++ public void buildAndPutCQByteBuffer(final Pair cqBBPair, ++ final byte[] topicBytes, final DispatchRequest request, final WriteBatch writeBatch) throws RocksDBException { ++ final ByteBuffer cqKey = cqBBPair.getObject1(); ++ buildCQKeyByteBuffer(cqKey, topicBytes, request.getQueueId(), request.getConsumeQueueOffset()); ++ ++ final ByteBuffer cqValue = cqBBPair.getObject2(); ++ buildCQValueByteBuffer(cqValue, request.getCommitLogOffset(), request.getMsgSize(), request.getTagsCode(), request.getStoreTimestamp()); ++ ++ writeBatch.put(this.defaultCFH, cqKey, cqValue); ++ } ++ ++ public ByteBuffer getCQInKV(final String topic, final int queueId, final long cqOffset) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, cqOffset); ++ byte[] value = this.rocksDBStorage.getCQ(keyBB.array()); ++ return (value != null) ? ByteBuffer.wrap(value) : null; ++ } ++ ++ public List rangeQuery(final String topic, final int queueId, final long startIndex, final int num) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final List defaultCFHList = new ArrayList(num); ++ final ByteBuffer[] resultList = new ByteBuffer[num]; ++ final List kvIndexList = new ArrayList(num); ++ final List kvKeyList = new ArrayList(num); ++ for (int i = 0; i < num; i++) { ++ final ByteBuffer keyBB = buildCQKeyByteBuffer(topicBytes, queueId, startIndex + i); ++ kvIndexList.add(i); ++ kvKeyList.add(keyBB.array()); ++ defaultCFHList.add(this.defaultCFH); ++ } ++ int keyNum = kvIndexList.size(); ++ if (keyNum > 0) { ++ List kvValueList = this.rocksDBStorage.multiGet(defaultCFHList, kvKeyList); ++ final int valueNum = kvValueList.size(); ++ if (keyNum != valueNum) { ++ throw new RocksDBException("rocksdb bug, multiGet"); ++ } ++ for (int i = 0; i < valueNum; i++) { ++ byte[] value = kvValueList.get(i); ++ if (value == null) { ++ continue; ++ } ++ ByteBuffer byteBuffer = ByteBuffer.wrap(value); ++ resultList[kvIndexList.get(i)] = byteBuffer; ++ } ++ } ++ ++ final int resultSize = resultList.length; ++ List bbValueList = new ArrayList(resultSize); ++ for (int i = 0; i < resultSize; i++) { ++ ByteBuffer byteBuffer = resultList[i]; ++ if (byteBuffer == null) { ++ break; ++ } ++ bbValueList.add(byteBuffer); ++ } ++ return bbValueList; ++ } ++ ++ /** ++ * When topic is deleted, we clean up its CqUnit in rocksdb. ++ * @param topic ++ * @param queueId ++ * @throws RocksDBException ++ */ ++ public void destroyCQ(final String topic, final int queueId, WriteBatch writeBatch) throws RocksDBException { ++ final byte[] topicBytes = topic.getBytes(CHARSET_UTF8); ++ final ByteBuffer cqStartKey = buildDeleteCQKey(true, topicBytes, queueId); ++ final ByteBuffer cqEndKey = buildDeleteCQKey(false, topicBytes, queueId); ++ ++ writeBatch.deleteRange(this.defaultCFH, cqStartKey.array(), cqEndKey.array()); ++ ++ log.info("Rocksdb consumeQueue table delete topic. {}, {}", topic, queueId); ++ } ++ ++ public long binarySearchInCQByTime(String topic, int queueId, long high, long low, long timestamp, ++ long minPhysicOffset) throws RocksDBException { ++ long result = 0; ++ long targetOffset = -1L, leftOffset = -1L, rightOffset = -1L; ++ long leftValue = -1L, rightValue = -1L; ++ while (high >= low) { ++ long midOffset = low + ((high - low) >>> 1); ++ ByteBuffer byteBuffer = getCQInKV(topic, queueId, midOffset); ++ if (byteBuffer == null) { ++ ERROR_LOG.warn("binarySearchInCQByTimeStamp Failed. topic: {}, queueId: {}, timestamp: {}, result: null", ++ topic, queueId, timestamp); ++ low = midOffset + 1; ++ continue; ++ } ++ ++ long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET); ++ if (phyOffset < minPhysicOffset) { ++ low = midOffset + 1; ++ leftOffset = midOffset; ++ continue; ++ } ++ long storeTime = byteBuffer.getLong(MSG_STORE_TIME_SIZE_OFFSET); ++ if (storeTime < 0) { ++ return 0; ++ } else if (storeTime == timestamp) { ++ targetOffset = midOffset; ++ break; ++ } else if (storeTime > timestamp) { ++ high = midOffset - 1; ++ rightOffset = midOffset; ++ rightValue = storeTime; ++ } else { ++ low = midOffset + 1; ++ leftOffset = midOffset; ++ leftValue = storeTime; ++ } ++ } ++ if (targetOffset != -1) { ++ result = targetOffset; ++ } else { ++ if (leftValue == -1) { ++ result = rightOffset; ++ } else if (rightValue == -1) { ++ result = leftOffset; ++ } else { ++ result = Math.abs(timestamp - leftValue) > Math.abs(timestamp - rightValue) ? rightOffset : leftOffset; ++ } ++ } ++ return result; ++ } ++ ++ public PhyAndCQOffset binarySearchInCQ(String topic, int queueId, long high, long low, long targetPhyOffset, ++ boolean min) throws RocksDBException { ++ long resultCQOffset = -1L; ++ long resultPhyOffset = -1L; ++ while (high >= low) { ++ long midCQOffset = low + ((high - low) >>> 1); ++ ByteBuffer byteBuffer = getCQInKV(topic, queueId, midCQOffset); ++ if (this.messageStore.getMessageStoreConfig().isEnableRocksDBLog()) { ++ ROCKSDB_LOG.warn("binarySearchInCQ. {}, {}, {}, {}, {}", topic, queueId, midCQOffset, low, high); ++ } ++ if (byteBuffer == null) { ++ low = midCQOffset + 1; ++ continue; ++ } ++ ++ final long phyOffset = byteBuffer.getLong(PHY_OFFSET_OFFSET); ++ if (phyOffset == targetPhyOffset) { ++ if (min) { ++ resultCQOffset = midCQOffset; ++ resultPhyOffset = phyOffset; ++ } ++ break; ++ } else if (phyOffset > targetPhyOffset) { ++ high = midCQOffset - 1; ++ if (min) { ++ resultCQOffset = midCQOffset; ++ resultPhyOffset = phyOffset; ++ } ++ } else { ++ low = midCQOffset + 1; ++ if (!min) { ++ resultCQOffset = midCQOffset; ++ resultPhyOffset = phyOffset; ++ } ++ } ++ } ++ return new PhyAndCQOffset(resultPhyOffset, resultCQOffset); ++ } ++ ++ public static Pair getCQByteBufferPair() { ++ ByteBuffer cqKey = ByteBuffer.allocateDirect(RocksDBConsumeQueueStore.MAX_KEY_LEN); ++ ByteBuffer cqValue = ByteBuffer.allocateDirect(CQ_UNIT_SIZE); ++ return new Pair<>(cqKey, cqValue); ++ } ++ ++ private ByteBuffer buildCQKeyByteBuffer(final byte[] topicBytes, final int queueId, final long cqOffset) { ++ final ByteBuffer byteBuffer = ByteBuffer.allocate(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset); ++ return byteBuffer; ++ } ++ ++ private void buildCQKeyByteBuffer(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) { ++ byteBuffer.position(0).limit(CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ buildCQKeyByteBuffer0(byteBuffer, topicBytes, queueId, cqOffset); ++ } ++ ++ private void buildCQKeyByteBuffer0(final ByteBuffer byteBuffer, final byte[] topicBytes, final int queueId, final long cqOffset) { ++ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(CTRL_1).putLong(cqOffset); ++ byteBuffer.flip(); ++ } ++ ++ private void buildCQValueByteBuffer(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, final long tagsCode, final long storeTimestamp) { ++ byteBuffer.position(0).limit(CQ_UNIT_SIZE); ++ buildCQValueByteBuffer0(byteBuffer, phyOffset, msgSize, tagsCode, storeTimestamp); ++ } ++ ++ private void buildCQValueByteBuffer0(final ByteBuffer byteBuffer, final long phyOffset, final int msgSize, ++ final long tagsCode, final long storeTimestamp) { ++ byteBuffer.putLong(phyOffset).putInt(msgSize).putLong(tagsCode).putLong(storeTimestamp); ++ byteBuffer.flip(); ++ } ++ ++ private ByteBuffer buildDeleteCQKey(final boolean start, final byte[] topicBytes, final int queueId) { ++ final ByteBuffer byteBuffer = ByteBuffer.allocate(DELETE_CQ_KEY_LENGTH_WITHOUT_TOPIC_BYTES + topicBytes.length); ++ ++ byteBuffer.putInt(topicBytes.length).put(CTRL_1).put(topicBytes).put(CTRL_1).putInt(queueId).put(start ? CTRL_0 : CTRL_2); ++ byteBuffer.flip(); ++ return byteBuffer; ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java +new file mode 100644 +index 000000000..aa796c4d3 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueCompactionFilterFactory.java +@@ -0,0 +1,47 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.rocksdb; ++ ++import org.apache.rocketmq.common.constant.LoggerName; ++import org.apache.rocketmq.logging.org.slf4j.Logger; ++import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; ++import org.apache.rocketmq.store.MessageStore; ++import org.rocksdb.AbstractCompactionFilter; ++import org.rocksdb.AbstractCompactionFilterFactory; ++import org.rocksdb.RemoveConsumeQueueCompactionFilter; ++ ++public class ConsumeQueueCompactionFilterFactory extends AbstractCompactionFilterFactory { ++ private static final Logger LOGGER = LoggerFactory.getLogger(LoggerName.ROCKSDB_LOGGER_NAME); ++ private final MessageStore messageStore; ++ ++ public ConsumeQueueCompactionFilterFactory(final MessageStore messageStore) { ++ this.messageStore = messageStore; ++ } ++ ++ @Override ++ public String name() { ++ return "ConsumeQueueCompactionFilterFactory"; ++ } ++ ++ @Override ++ public RemoveConsumeQueueCompactionFilter createCompactionFilter(final AbstractCompactionFilter.Context context) { ++ long minPhyOffset = this.messageStore.getMinPhyOffset(); ++ LOGGER.info("manualCompaction minPhyOffset: {}, isFull: {}, isManual: {}", ++ minPhyOffset, context.isFullCompaction(), context.isManualCompaction()); ++ return new RemoveConsumeQueueCompactionFilter(minPhyOffset); ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java +new file mode 100644 +index 000000000..362684560 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/ConsumeQueueRocksDBStorage.java +@@ -0,0 +1,133 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.rocksdb; ++ ++import java.util.ArrayList; ++import java.util.List; ++ ++import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.common.config.AbstractRocksDBStorage; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.store.MessageStore; ++import org.rocksdb.ColumnFamilyDescriptor; ++import org.rocksdb.ColumnFamilyHandle; ++import org.rocksdb.ColumnFamilyOptions; ++import org.rocksdb.CompactRangeOptions; ++import org.rocksdb.ReadOptions; ++import org.rocksdb.RocksDB; ++import org.rocksdb.RocksDBException; ++import org.rocksdb.RocksIterator; ++import org.rocksdb.WriteBatch; ++import org.rocksdb.WriteOptions; ++ ++public class ConsumeQueueRocksDBStorage extends AbstractRocksDBStorage { ++ private final MessageStore messageStore; ++ private volatile ColumnFamilyHandle offsetCFHandle; ++ ++ public ConsumeQueueRocksDBStorage(final MessageStore messageStore, final String dbPath, final int prefixLen) { ++ this.messageStore = messageStore; ++ this.dbPath = dbPath; ++ this.readOnly = false; ++ } ++ ++ private void initOptions() { ++ this.options = RocksDBOptionsFactory.createDBOptions(); ++ ++ this.writeOptions = new WriteOptions(); ++ this.writeOptions.setSync(false); ++ this.writeOptions.setDisableWAL(true); ++ this.writeOptions.setNoSlowdown(true); ++ ++ this.totalOrderReadOptions = new ReadOptions(); ++ this.totalOrderReadOptions.setPrefixSameAsStart(false); ++ this.totalOrderReadOptions.setTotalOrderSeek(false); ++ ++ this.compactRangeOptions = new CompactRangeOptions(); ++ this.compactRangeOptions.setBottommostLevelCompaction(CompactRangeOptions.BottommostLevelCompaction.kForce); ++ this.compactRangeOptions.setAllowWriteStall(true); ++ this.compactRangeOptions.setExclusiveManualCompaction(false); ++ this.compactRangeOptions.setChangeLevel(true); ++ this.compactRangeOptions.setTargetLevel(-1); ++ this.compactRangeOptions.setMaxSubcompactions(4); ++ } ++ ++ @Override ++ protected boolean postLoad() { ++ try { ++ UtilAll.ensureDirOK(this.dbPath); ++ ++ initOptions(); ++ ++ final List cfDescriptors = new ArrayList(); ++ ++ ColumnFamilyOptions cqCfOptions = RocksDBOptionsFactory.createCQCFOptions(this.messageStore); ++ this.cfOptions.add(cqCfOptions); ++ cfDescriptors.add(new ColumnFamilyDescriptor(RocksDB.DEFAULT_COLUMN_FAMILY, cqCfOptions)); ++ ++ ColumnFamilyOptions offsetCfOptions = RocksDBOptionsFactory.createOffsetCFOptions(); ++ this.cfOptions.add(offsetCfOptions); ++ cfDescriptors.add(new ColumnFamilyDescriptor("offset".getBytes(DataConverter.CHARSET_UTF8), offsetCfOptions)); ++ ++ final List cfHandles = new ArrayList(); ++ open(cfDescriptors, cfHandles); ++ ++ this.defaultCFHandle = cfHandles.get(0); ++ this.offsetCFHandle = cfHandles.get(1); ++ } catch (final Exception e) { ++ LOGGER.error("postLoad Failed. {}", this.dbPath, e); ++ return false; ++ } ++ return true; ++ } ++ ++ @Override ++ protected void preShutdown() { ++ this.offsetCFHandle.close(); ++ } ++ ++ public byte[] getCQ(final byte[] keyBytes) throws RocksDBException { ++ return get(this.defaultCFHandle, this.totalOrderReadOptions, keyBytes); ++ } ++ ++ public byte[] getOffset(final byte[] keyBytes) throws RocksDBException { ++ return get(this.offsetCFHandle, this.totalOrderReadOptions, keyBytes); ++ } ++ ++ public List multiGet(final List cfhList, final List keys) throws RocksDBException { ++ return multiGet(this.totalOrderReadOptions, cfhList, keys); ++ } ++ ++ public void batchPut(final WriteBatch batch) throws RocksDBException { ++ batchPut(this.writeOptions, batch); ++ } ++ ++ public void manualCompaction(final long minPhyOffset) { ++ try { ++ manualCompaction(minPhyOffset, this.compactRangeOptions); ++ } catch (Exception e) { ++ LOGGER.error("manualCompaction Failed. minPhyOffset: {}", minPhyOffset, e); ++ } ++ } ++ ++ public RocksIterator seekOffsetCF() { ++ return this.db.newIterator(this.offsetCFHandle, this.totalOrderReadOptions); ++ } ++ ++ public ColumnFamilyHandle getOffsetCFHandle() { ++ return this.offsetCFHandle; ++ } ++} +\ No newline at end of file +diff --git a/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +new file mode 100644 +index 000000000..a3a99d334 +--- /dev/null ++++ b/store/src/main/java/org/apache/rocketmq/store/rocksdb/RocksDBOptionsFactory.java +@@ -0,0 +1,161 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.store.rocksdb; ++ ++import org.apache.rocketmq.common.config.ConfigRocksDBStorage; ++import org.apache.rocketmq.store.MessageStore; ++import org.rocksdb.BlockBasedTableConfig; ++import org.rocksdb.BloomFilter; ++import org.rocksdb.ColumnFamilyOptions; ++import org.rocksdb.CompactionOptionsUniversal; ++import org.rocksdb.CompactionStopStyle; ++import org.rocksdb.CompactionStyle; ++import org.rocksdb.CompressionType; ++import org.rocksdb.DBOptions; ++import org.rocksdb.DataBlockIndexType; ++import org.rocksdb.IndexType; ++import org.rocksdb.InfoLogLevel; ++import org.rocksdb.LRUCache; ++import org.rocksdb.RateLimiter; ++import org.rocksdb.SkipListMemTableConfig; ++import org.rocksdb.Statistics; ++import org.rocksdb.StatsLevel; ++import org.rocksdb.StringAppendOperator; ++import org.rocksdb.WALRecoveryMode; ++import org.rocksdb.util.SizeUnit; ++ ++public class RocksDBOptionsFactory { ++ ++ public static ColumnFamilyOptions createCQCFOptions(final MessageStore messageStore) { ++ BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). ++ setFormatVersion(5). ++ setIndexType(IndexType.kBinarySearch). ++ setDataBlockIndexType(DataBlockIndexType.kDataBlockBinaryAndHash). ++ setDataBlockHashTableUtilRatio(0.75). ++ setBlockSize(32 * SizeUnit.KB). ++ setMetadataBlockSize(4 * SizeUnit.KB). ++ setFilterPolicy(new BloomFilter(16, false)). ++ setCacheIndexAndFilterBlocks(false). ++ setCacheIndexAndFilterBlocksWithHighPriority(true). ++ setPinL0FilterAndIndexBlocksInCache(false). ++ setPinTopLevelIndexAndFilter(true). ++ setBlockCache(new LRUCache(1024 * SizeUnit.MB, 8, false)). ++ setWholeKeyFiltering(true); ++ ++ ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); ++ CompactionOptionsUniversal compactionOption = new CompactionOptionsUniversal(); ++ compactionOption.setSizeRatio(100). ++ setMaxSizeAmplificationPercent(25). ++ setAllowTrivialMove(true). ++ setMinMergeWidth(2). ++ setMaxMergeWidth(Integer.MAX_VALUE). ++ setStopStyle(CompactionStopStyle.CompactionStopStyleTotalSize). ++ setCompressionSizePercent(-1); ++ return columnFamilyOptions.setMaxWriteBufferNumber(4). ++ setWriteBufferSize(128 * SizeUnit.MB). ++ setMinWriteBufferNumberToMerge(1). ++ setTableFormatConfig(blockBasedTableConfig). ++ setMemTableConfig(new SkipListMemTableConfig()). ++ setCompressionType(CompressionType.LZ4_COMPRESSION). ++ setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION). ++ setNumLevels(7). ++ setCompactionStyle(CompactionStyle.UNIVERSAL). ++ setCompactionOptionsUniversal(compactionOption). ++ setMaxCompactionBytes(100 * SizeUnit.GB). ++ setSoftPendingCompactionBytesLimit(100 * SizeUnit.GB). ++ setHardPendingCompactionBytesLimit(256 * SizeUnit.GB). ++ setLevel0FileNumCompactionTrigger(2). ++ setLevel0SlowdownWritesTrigger(8). ++ setLevel0StopWritesTrigger(10). ++ setTargetFileSizeBase(256 * SizeUnit.MB). ++ setTargetFileSizeMultiplier(2). ++ setMergeOperator(new StringAppendOperator()). ++ setCompactionFilterFactory(new ConsumeQueueCompactionFilterFactory(messageStore)). ++ setReportBgIoStats(true). ++ setOptimizeFiltersForHits(true); ++ } ++ ++ public static ColumnFamilyOptions createOffsetCFOptions() { ++ BlockBasedTableConfig blockBasedTableConfig = new BlockBasedTableConfig(). ++ setFormatVersion(5). ++ setIndexType(IndexType.kBinarySearch). ++ setDataBlockIndexType(DataBlockIndexType.kDataBlockBinarySearch). ++ setBlockSize(32 * SizeUnit.KB). ++ setFilterPolicy(new BloomFilter(16, false)). ++ setCacheIndexAndFilterBlocks(false). ++ setCacheIndexAndFilterBlocksWithHighPriority(true). ++ setPinL0FilterAndIndexBlocksInCache(false). ++ setPinTopLevelIndexAndFilter(true). ++ setBlockCache(new LRUCache(128 * SizeUnit.MB, 8, false)). ++ setWholeKeyFiltering(true); ++ ++ ColumnFamilyOptions columnFamilyOptions = new ColumnFamilyOptions(); ++ return columnFamilyOptions.setMaxWriteBufferNumber(4). ++ setWriteBufferSize(64 * SizeUnit.MB). ++ setMinWriteBufferNumberToMerge(1). ++ setTableFormatConfig(blockBasedTableConfig). ++ setMemTableConfig(new SkipListMemTableConfig()). ++ setCompressionType(CompressionType.NO_COMPRESSION). ++ setNumLevels(7). ++ setCompactionStyle(CompactionStyle.LEVEL). ++ setLevel0FileNumCompactionTrigger(2). ++ setLevel0SlowdownWritesTrigger(8). ++ setLevel0StopWritesTrigger(10). ++ setTargetFileSizeBase(64 * SizeUnit.MB). ++ setTargetFileSizeMultiplier(2). ++ setMaxBytesForLevelBase(256 * SizeUnit.MB). ++ setMaxBytesForLevelMultiplier(2). ++ setMergeOperator(new StringAppendOperator()). ++ setInplaceUpdateSupport(true); ++ } ++ ++ /** ++ * Create a rocksdb db options, the user must take care to close it after closing db. ++ * @return ++ */ ++ public static DBOptions createDBOptions() { ++ //Turn based on https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide ++ // and http://gitlab.alibaba-inc.com/aloha/aloha/blob/branch_2_5_0/jstorm-core/src/main/java/com/alibaba/jstorm/cache/rocksdb/RocksDbOptionsFactory.java ++ DBOptions options = new DBOptions(); ++ Statistics statistics = new Statistics(); ++ statistics.setStatsLevel(StatsLevel.EXCEPT_DETAILED_TIMERS); ++ return options. ++ setDbLogDir(ConfigRocksDBStorage.getDBLogDir()). ++ setInfoLogLevel(InfoLogLevel.INFO_LEVEL). ++ setWalRecoveryMode(WALRecoveryMode.PointInTimeRecovery). ++ setManualWalFlush(true). ++ setMaxTotalWalSize(0). ++ setWalSizeLimitMB(0). ++ setWalTtlSeconds(0). ++ setCreateIfMissing(true). ++ setCreateMissingColumnFamilies(true). ++ setMaxOpenFiles(-1). ++ setMaxLogFileSize(1 * SizeUnit.GB). ++ setKeepLogFileNum(5). ++ setMaxManifestFileSize(1 * SizeUnit.GB). ++ setAllowConcurrentMemtableWrite(false). ++ setStatistics(statistics). ++ setAtomicFlush(true). ++ setMaxBackgroundJobs(32). ++ setMaxSubcompactions(8). ++ setParanoidChecks(true). ++ setDelayedWriteRate(16 * SizeUnit.MB). ++ setRateLimiter(new RateLimiter(100 * SizeUnit.MB)). ++ setUseDirectIoForFlushAndCompaction(false). ++ setUseDirectReads(false); ++ } ++} +diff --git a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +index ac4c61cd6..3ab51a26d 100644 +--- a/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java ++++ b/store/src/main/java/org/apache/rocketmq/store/timer/TimerMessageStore.java +@@ -57,7 +57,6 @@ import org.apache.rocketmq.common.topic.TopicValidator; + import org.apache.rocketmq.common.utils.ThreadUtils; + import org.apache.rocketmq.logging.org.slf4j.Logger; + import org.apache.rocketmq.logging.org.slf4j.LoggerFactory; +-import org.apache.rocketmq.store.ConsumeQueue; + import org.apache.rocketmq.store.DefaultMessageStore; + import org.apache.rocketmq.store.MessageStore; + import org.apache.rocketmq.store.PutMessageResult; +@@ -66,6 +65,9 @@ import org.apache.rocketmq.store.config.BrokerRole; + import org.apache.rocketmq.store.config.MessageStoreConfig; + import org.apache.rocketmq.store.logfile.MappedFile; + import org.apache.rocketmq.store.metrics.DefaultStoreMetricsManager; ++import org.apache.rocketmq.store.queue.ConsumeQueueInterface; ++import org.apache.rocketmq.store.queue.CqUnit; ++import org.apache.rocketmq.store.queue.ReferredIterator; + import org.apache.rocketmq.store.stats.BrokerStatsManager; + import org.apache.rocketmq.store.util.PerfCounter; + +@@ -333,7 +335,7 @@ public class TimerMessageStore { + // if not, use cq offset. + long msgQueueOffset = messageExt.getQueueOffset(); + int queueId = messageExt.getQueueId(); +- ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); ++ ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + if (null == cq) { + return msgQueueOffset; + } +@@ -346,15 +348,18 @@ public class TimerMessageStore { + offsetPy, sizePy); + break; + } +- SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(tmpOffset); +- if (null == bufferCQ) { +- // offset in msg may be greater than offset of cq. +- tmpOffset -= 1; +- continue; +- } ++ ReferredIterator iterator = null; + try { +- long offsetPyTemp = bufferCQ.getByteBuffer().getLong(); +- int sizePyTemp = bufferCQ.getByteBuffer().getInt(); ++ iterator = cq.iterateFrom(tmpOffset); ++ CqUnit cqUnit = null; ++ if (null == iterator || (cqUnit = iterator.next()) == null) { ++ // offset in msg may be greater than offset of cq. ++ tmpOffset -= 1; ++ continue; ++ } ++ ++ long offsetPyTemp = cqUnit.getPos(); ++ int sizePyTemp = cqUnit.getSize(); + if (offsetPyTemp == offsetPy && sizePyTemp == sizePy) { + LOGGER.info("reviseQueueOffset check cq offset ok. {}, {}, {}", + tmpOffset, offsetPyTemp, sizePyTemp); +@@ -365,7 +370,9 @@ public class TimerMessageStore { + } catch (Throwable e) { + LOGGER.error("reviseQueueOffset check cq offset error.", e); + } finally { +- bufferCQ.release(); ++ if (iterator != null) { ++ iterator.release(); ++ } + } + } + +@@ -633,7 +640,7 @@ public class TimerMessageStore { + if (!isRunningEnqueue()) { + return false; + } +- ConsumeQueue cq = (ConsumeQueue) this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); ++ ConsumeQueueInterface cq = this.messageStore.getConsumeQueue(TIMER_TOPIC, queueId); + if (null == cq) { + return false; + } +@@ -643,18 +650,22 @@ public class TimerMessageStore { + currQueueOffset = cq.getMinOffsetInQueue(); + } + long offset = currQueueOffset; +- SelectMappedBufferResult bufferCQ = cq.getIndexBuffer(offset); +- if (null == bufferCQ) { +- return false; +- } ++ ReferredIterator iterator = null; + try { ++ iterator = cq.iterateFrom(offset); ++ if (null == iterator) { ++ return false; ++ } ++ + int i = 0; +- for (; i < bufferCQ.getSize(); i += ConsumeQueue.CQ_STORE_UNIT_SIZE) { ++ while (iterator.hasNext()) { ++ i++; + perfCounterTicks.startTick("enqueue_get"); + try { +- long offsetPy = bufferCQ.getByteBuffer().getLong(); +- int sizePy = bufferCQ.getByteBuffer().getInt(); +- bufferCQ.getByteBuffer().getLong(); //tags code ++ CqUnit cqUnit = iterator.next(); ++ long offsetPy = cqUnit.getPos(); ++ int sizePy = cqUnit.getSize(); ++ cqUnit.getTagsCode(); //tags code + MessageExt msgExt = getMessageByCommitOffset(offsetPy, sizePy); + if (null == msgExt) { + perfCounterTicks.getCounter("enqueue_get_miss"); +@@ -663,7 +674,7 @@ public class TimerMessageStore { + lastEnqueueButExpiredStoreTime = msgExt.getStoreTimestamp(); + long delayedTime = Long.parseLong(msgExt.getProperty(TIMER_OUT_MS)); + // use CQ offset, not offset in Message +- msgExt.setQueueOffset(offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE)); ++ msgExt.setQueueOffset(offset + i); + TimerRequest timerRequest = new TimerRequest(offsetPy, sizePy, delayedTime, System.currentTimeMillis(), MAGIC_DEFAULT, msgExt); + // System.out.printf("build enqueue request, %s%n", timerRequest); + while (!enqueuePutQueue.offer(timerRequest, 3, TimeUnit.SECONDS)) { +@@ -687,14 +698,16 @@ public class TimerMessageStore { + if (!isRunningEnqueue()) { + return false; + } +- currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); ++ currQueueOffset = offset + i; + } +- currQueueOffset = offset + (i / ConsumeQueue.CQ_STORE_UNIT_SIZE); ++ currQueueOffset = offset + i; + return i > 0; + } catch (Exception e) { + LOGGER.error("Unknown exception in enqueuing", e); + } finally { +- bufferCQ.release(); ++ if (iterator != null) { ++ iterator.release(); ++ } + } + return false; + } +@@ -1642,7 +1655,7 @@ public class TimerMessageStore { + if (System.currentTimeMillis() - start > storeConfig.getTimerProgressLogIntervalMs()) { + start = System.currentTimeMillis(); + long tmpQueueOffset = currQueueOffset; +- ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0); ++ ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); + long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); + TimerMessageStore.LOGGER.info("[{}]Timer progress-check commitRead:[{}] currRead:[{}] currWrite:[{}] readBehind:{} currReadOffset:{} offsetBehind:{} behindMaster:{} " + + "enqPutQueue:{} deqGetQueue:{} deqPutQueue:{} allCongestNum:{} enqExpiredStoreTime:{}", +@@ -1685,7 +1698,7 @@ public class TimerMessageStore { + + public long getEnqueueBehindMessages() { + long tmpQueueOffset = currQueueOffset; +- ConsumeQueue cq = (ConsumeQueue) messageStore.getConsumeQueue(TIMER_TOPIC, 0); ++ ConsumeQueueInterface cq = messageStore.getConsumeQueue(TIMER_TOPIC, 0); + long maxOffsetInQueue = cq == null ? 0 : cq.getMaxOffsetInQueue(); + return maxOffsetInQueue - tmpQueueOffset; + } +diff --git a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +index 12d1e5723..1d09ca86e 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/DefaultMessageStoreTest.java +@@ -428,9 +428,10 @@ public class DefaultMessageStoreTest { + + private long getStoreTime(CqUnit cqUnit) { + try { +- Method getStoreTime = getDefaultMessageStore().getClass().getDeclaredMethod("getStoreTime", CqUnit.class); ++ Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass(); ++ Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class); + getStoreTime.setAccessible(true); +- return (long) getStoreTime.invoke(getDefaultMessageStore(), cqUnit); ++ return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit); + } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException(e); + } +diff --git a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java +index 85626a332..2447bbf68 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/MultiDispatchTest.java +@@ -28,9 +28,11 @@ import org.apache.rocketmq.common.message.MessageConst; + import org.apache.rocketmq.common.message.MessageDecoder; + import org.apache.rocketmq.common.message.MessageExtBrokerInner; + import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.apache.rocketmq.store.queue.MultiDispatch; + import org.junit.After; + import org.junit.Before; + import org.junit.Test; ++import org.rocksdb.RocksDBException; + + import static org.apache.rocketmq.store.config.StorePathConfigHelper.getStorePathConsumeQueue; + import static org.junit.Assert.assertEquals; +@@ -69,15 +71,15 @@ public class MultiDispatchTest { + } + + @Test +- public void queueKey() { ++ public void lmqQueueKey() { + MessageExtBrokerInner messageExtBrokerInner = mock(MessageExtBrokerInner.class); + when(messageExtBrokerInner.getQueueId()).thenReturn(2); +- String ret = consumeQueue.queueKey("%LMQ%lmq123", messageExtBrokerInner); ++ String ret = MultiDispatch.lmqQueueKey("%LMQ%lmq123"); + assertEquals(ret, "%LMQ%lmq123-0"); + } + + @Test +- public void wrapMultiDispatch() { ++ public void wrapMultiDispatch() throws RocksDBException { + MessageExtBrokerInner messageExtBrokerInner = buildMessageMultiQueue(); + messageStore.assignOffset(messageExtBrokerInner); + assertEquals(messageExtBrokerInner.getProperty(MessageConst.PROPERTY_INNER_MULTI_QUEUE_OFFSET), "0,0"); +diff --git a/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java +new file mode 100644 +index 000000000..acf5edf51 +--- /dev/null ++++ b/store/src/test/java/org/apache/rocketmq/store/RocksDBMessageStoreTest.java +@@ -0,0 +1,1060 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++ ++package org.apache.rocketmq.store; ++ ++import java.io.File; ++import java.io.RandomAccessFile; ++import java.lang.reflect.InvocationTargetException; ++import java.lang.reflect.Method; ++import java.net.InetAddress; ++import java.net.InetSocketAddress; ++import java.net.SocketAddress; ++import java.net.UnknownHostException; ++import java.nio.MappedByteBuffer; ++import java.nio.channels.FileChannel; ++import java.nio.channels.OverlappingFileLockException; ++import java.nio.charset.StandardCharsets; ++import java.util.ArrayList; ++import java.util.Arrays; ++import java.util.HashSet; ++import java.util.List; ++import java.util.Map; ++import java.util.Random; ++import java.util.UUID; ++import java.util.concurrent.ConcurrentHashMap; ++import java.util.concurrent.ConcurrentMap; ++import java.util.concurrent.TimeUnit; ++import java.util.concurrent.atomic.AtomicInteger; ++ ++import com.google.common.collect.Sets; ++ ++import org.apache.rocketmq.common.BrokerConfig; ++import org.apache.rocketmq.common.MixAll; ++import org.apache.rocketmq.common.TopicConfig; ++import org.apache.rocketmq.common.UtilAll; ++import org.apache.rocketmq.common.message.MessageBatch; ++import org.apache.rocketmq.common.message.MessageDecoder; ++import org.apache.rocketmq.common.message.MessageExt; ++import org.apache.rocketmq.common.message.MessageExtBatch; ++import org.apache.rocketmq.common.message.MessageExtBrokerInner; ++import org.apache.rocketmq.store.config.BrokerRole; ++import org.apache.rocketmq.store.config.FlushDiskType; ++import org.apache.rocketmq.store.config.MessageStoreConfig; ++import org.apache.rocketmq.store.config.StorePathConfigHelper; ++import org.apache.rocketmq.store.queue.ConsumeQueueInterface; ++import org.apache.rocketmq.store.queue.CqUnit; ++import org.apache.rocketmq.store.stats.BrokerStatsManager; ++import org.assertj.core.util.Strings; ++import org.junit.After; ++import org.junit.Assert; ++import org.junit.Before; ++import org.junit.Test; ++import org.junit.runner.RunWith; ++import org.mockito.junit.MockitoJUnitRunner; ++ ++import static org.assertj.core.api.Assertions.assertThat; ++import static org.junit.Assert.assertEquals; ++import static org.junit.Assert.assertTrue; ++ ++@RunWith(MockitoJUnitRunner.class) ++public class RocksDBMessageStoreTest { ++ private final String storeMessage = "Once, there was a chance for me!"; ++ private final String messageTopic = "FooBar"; ++ private final String storeType = StoreType.DEFAULT_ROCKSDB.getStoreType(); ++ private int queueTotal = 100; ++ private AtomicInteger queueId = new AtomicInteger(0); ++ private SocketAddress bornHost; ++ private SocketAddress storeHost; ++ private byte[] messageBody; ++ private MessageStore messageStore; ++ ++ @Before ++ public void init() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ storeHost = new InetSocketAddress(InetAddress.getLocalHost(), 8123); ++ bornHost = new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 0); ++ ++ messageStore = buildMessageStore(); ++ boolean load = messageStore.load(); ++ assertTrue(load); ++ messageStore.start(); ++ } ++ ++ @Test(expected = OverlappingFileLockException.class) ++ public void test_repeat_restart() throws Exception { ++ if (notExecuted()) { ++ throw new OverlappingFileLockException(); ++ } ++ queueTotal = 1; ++ messageBody = storeMessage.getBytes(); ++ ++ MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); ++ messageStoreConfig.setMappedFileSizeCommitLog(1024 * 8); ++ messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 4); ++ messageStoreConfig.setMaxHashSlotNum(100); ++ messageStoreConfig.setMaxIndexNum(100 * 10); ++ messageStoreConfig.setStorePathRootDir(System.getProperty("java.io.tmpdir") + File.separator + "store"); ++ messageStoreConfig.setHaListenPort(0); ++ MessageStore master = new RocksDBMessageStore(messageStoreConfig, null, new MyMessageArrivingListener(), new BrokerConfig(), new ConcurrentHashMap<>()); ++ ++ boolean load = master.load(); ++ assertTrue(load); ++ ++ try { ++ master.start(); ++ master.start(); ++ } finally { ++ master.shutdown(); ++ master.destroy(); ++ } ++ } ++ ++ @After ++ public void destroy() { ++ if (notExecuted()) { ++ return; ++ } ++ messageStore.shutdown(); ++ messageStore.destroy(); ++ ++ MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); ++ File file = new File(messageStoreConfig.getStorePathRootDir()); ++ UtilAll.deleteFile(file); ++ } ++ ++ private MessageStore buildMessageStore() throws Exception { ++ return buildMessageStore(null, ""); ++ } ++ ++ private MessageStore buildMessageStore(String storePathRootDir, String topic) throws Exception { ++ MessageStoreConfig messageStoreConfig = new MessageStoreConfig(); ++ messageStoreConfig.setMappedFileSizeCommitLog(1024 * 1024 * 10); ++ messageStoreConfig.setMappedFileSizeConsumeQueue(1024 * 1024 * 10); ++ messageStoreConfig.setMaxHashSlotNum(10000); ++ messageStoreConfig.setMaxIndexNum(100 * 100); ++ messageStoreConfig.setFlushDiskType(FlushDiskType.SYNC_FLUSH); ++ messageStoreConfig.setFlushIntervalConsumeQueue(1); ++ messageStoreConfig.setStoreType(storeType); ++ messageStoreConfig.setHaListenPort(0); ++ if (Strings.isNullOrEmpty(storePathRootDir)) { ++ UUID uuid = UUID.randomUUID(); ++ storePathRootDir = System.getProperty("java.io.tmpdir") + File.separator + "store-" + uuid.toString(); ++ } ++ messageStoreConfig.setStorePathRootDir(storePathRootDir); ++ ConcurrentMap topicConfigTable = new ConcurrentHashMap<>(); ++ topicConfigTable.put(topic, new TopicConfig(topic, 1, 1)); ++ return new RocksDBMessageStore(messageStoreConfig, ++ new BrokerStatsManager("simpleTest", true), ++ new MyMessageArrivingListener(), ++ new BrokerConfig(), topicConfigTable); ++ } ++ ++ @Test ++ public void testWriteAndRead() { ++ if (notExecuted()) { ++ return; ++ } ++ long ipv4HostMsgs = 10; ++ long ipv6HostMsgs = 10; ++ long totalMsgs = ipv4HostMsgs + ipv6HostMsgs; ++ queueTotal = 1; ++ messageBody = storeMessage.getBytes(); ++ for (long i = 0; i < ipv4HostMsgs; i++) { ++ messageStore.putMessage(buildMessage()); ++ } ++ ++ for (long i = 0; i < ipv6HostMsgs; i++) { ++ messageStore.putMessage(buildIPv6HostMessage()); ++ } ++ ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ for (long i = 0; i < totalMsgs; i++) { ++ GetMessageResult result = messageStore.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); ++ assertThat(result).isNotNull(); ++ result.release(); ++ } ++ verifyThatMasterIsFunctional(totalMsgs, messageStore); ++ } ++ ++ @Test ++ public void testLookMessageByOffset_OffsetIsFirst() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = new Random().nextInt(10); ++ String topic = "FooBar"; ++ int firstOffset = 0; ++ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); ++ AppendMessageResult firstResult = appendMessageResultArray[0]; ++ ++ MessageExt messageExt = messageStore.lookMessageByOffset(firstResult.getWroteOffset()); ++ MessageExt messageExt1 = getDefaultMessageStore().lookMessageByOffset(firstResult.getWroteOffset(), firstResult.getWroteBytes()); ++ ++ assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset)); ++ assertThat(new String(messageExt1.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, firstOffset)); ++ } ++ ++ @Test ++ public void testLookMessageByOffset_OffsetIsLast() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = new Random().nextInt(10); ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); ++ int lastIndex = totalCount - 1; ++ AppendMessageResult lastResult = appendMessageResultArray[lastIndex]; ++ ++ MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastResult.getWroteOffset(), lastResult.getWroteBytes()); ++ ++ assertThat(new String(messageExt.getBody())).isEqualTo(buildMessageBodyByOffset(storeMessage, lastIndex)); ++ } ++ ++ @Test ++ public void testLookMessageByOffset_OffsetIsOutOfBound() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = new Random().nextInt(10); ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResultArray = putMessages(totalCount, topic, queueId); ++ long lastOffset = getMaxOffset(appendMessageResultArray); ++ ++ MessageExt messageExt = getDefaultMessageStore().lookMessageByOffset(lastOffset); ++ ++ assertThat(messageExt).isNull(); ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); ++ for (AppendMessageResult appendMessageResult : appendMessageResults) { ++ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp()); ++ CqUnit cqUnit = consumeQueue.get(offset); ++ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset()); ++ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes()); ++ } ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_TimestampIsSkewing() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ int skewing = 2; ++ ++ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); ++ for (AppendMessageResult appendMessageResult : appendMessageResults) { ++ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing); ++ CqUnit cqUnit = consumeQueue.get(offset); ++ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResult.getWroteOffset()); ++ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResult.getWroteBytes()); ++ } ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_TimestampSkewingIsLarge() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, true); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ int skewing = 20000; ++ ++ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); ++ for (AppendMessageResult appendMessageResult : appendMessageResults) { ++ long offset = messageStore.getOffsetInQueueByTime(topic, queueId, appendMessageResult.getStoreTimestamp() - skewing); ++ CqUnit cqUnit = consumeQueue.get(offset); ++ assertThat(cqUnit.getPos()).isEqualTo(appendMessageResults[0].getWroteOffset()); ++ assertThat(cqUnit.getSize()).isEqualTo(appendMessageResults[0].getWroteBytes()); ++ } ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_ConsumeQueueNotFound1() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ int wrongQueueId = 1; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); ++ //Thread.sleep(10); ++ ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ long offset = messageStore.getOffsetInQueueByTime(topic, wrongQueueId, appendMessageResults[0].getStoreTimestamp()); ++ ++ assertThat(offset).isEqualTo(0); ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_ConsumeQueueNotFound2() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ int wrongQueueId = 1; ++ String topic = "FooBar"; ++ putMessages(totalCount, topic, queueId, false); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, 0); ++ ++ assertThat(messageStoreTimeStamp).isEqualTo(-1); ++ } ++ ++ @Test ++ public void testGetOffsetInQueueByTime_ConsumeQueueOffsetNotExist() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ int wrongQueueId = 1; ++ String topic = "FooBar"; ++ putMessages(totalCount, topic, queueId, true); ++ //Thread.sleep(10); ++ ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, wrongQueueId, -1); ++ ++ assertThat(messageStoreTimeStamp).isEqualTo(-1); ++ } ++ ++ @Test ++ public void testGetMessageStoreTimeStamp() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ ConsumeQueueInterface consumeQueue = getDefaultMessageStore().findConsumeQueue(topic, queueId); ++ int minOffsetInQueue = (int) consumeQueue.getMinOffsetInQueue(); ++ for (int i = minOffsetInQueue; i < consumeQueue.getMaxOffsetInQueue(); i++) { ++ long messageStoreTimeStamp = messageStore.getMessageStoreTimeStamp(topic, queueId, i); ++ assertThat(messageStoreTimeStamp).isEqualTo(appendMessageResults[i].getStoreTimestamp()); ++ } ++ } ++ ++ @Test ++ public void testGetStoreTime_ParamIsNull() { ++ if (notExecuted()) { ++ return; ++ } ++ long storeTime = getStoreTime(null); ++ ++ assertThat(storeTime).isEqualTo(-1); ++ } ++ ++ @Test ++ public void testGetStoreTime_EverythingIsOk() { ++ if (notExecuted()) { ++ return; ++ } ++ final int totalCount = 10; ++ int queueId = 0; ++ String topic = "FooBar"; ++ AppendMessageResult[] appendMessageResults = putMessages(totalCount, topic, queueId, false); ++ //Thread.sleep(10); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ConsumeQueueInterface consumeQueue = messageStore.getConsumeQueue(topic, queueId); ++ ++ for (int i = 0; i < totalCount; i++) { ++ CqUnit cqUnit = consumeQueue.get(i); ++ long storeTime = getStoreTime(cqUnit); ++ assertThat(storeTime).isEqualTo(appendMessageResults[i].getStoreTimestamp()); ++ } ++ } ++ ++ @Test ++ public void testGetStoreTime_PhyOffsetIsLessThanCommitLogMinOffset() { ++ if (notExecuted()) { ++ return; ++ } ++ long phyOffset = -10; ++ int size = 138; ++ CqUnit cqUnit = new CqUnit(0, phyOffset, size, 0); ++ long storeTime = getStoreTime(cqUnit); ++ ++ assertThat(storeTime).isEqualTo(-1); ++ } ++ ++ @Test ++ public void testPutMessage_whenMessagePropertyIsTooLong() { ++ if (notExecuted()) { ++ return; ++ } ++ String topicName = "messagePropertyIsTooLongTest"; ++ MessageExtBrokerInner illegalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, Short.MAX_VALUE + 1); ++ assertEquals(messageStore.putMessage(illegalMessage).getPutMessageStatus(), PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); ++ assertEquals(0L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue()); ++ MessageExtBrokerInner normalMessage = buildSpecifyLengthPropertyMessage("123".getBytes(StandardCharsets.UTF_8), topicName, 100); ++ assertEquals(messageStore.putMessage(normalMessage).getPutMessageStatus(), PutMessageStatus.PUT_OK); ++ assertEquals(1L, messageStore.getQueueStore().getMaxOffset(topicName, 0).longValue()); ++ } ++ ++ private RocksDBMessageStore getDefaultMessageStore() { ++ return (RocksDBMessageStore) this.messageStore; ++ } ++ ++ private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId) { ++ return putMessages(totalCount, topic, queueId, false); ++ } ++ ++ private AppendMessageResult[] putMessages(int totalCount, String topic, int queueId, boolean interval) { ++ AppendMessageResult[] appendMessageResultArray = new AppendMessageResult[totalCount]; ++ for (int i = 0; i < totalCount; i++) { ++ String messageBody = buildMessageBodyByOffset(storeMessage, i); ++ ++ MessageExtBrokerInner msgInner = ++ i < totalCount / 2 ? buildMessage(messageBody.getBytes(), topic) : buildIPv6HostMessage(messageBody.getBytes(), topic); ++ msgInner.setQueueId(queueId); ++ PutMessageResult result = messageStore.putMessage(msgInner); ++ appendMessageResultArray[i] = result.getAppendMessageResult(); ++ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); ++ if (interval) { ++ try { ++ Thread.sleep(10); ++ } catch (InterruptedException e) { ++ throw new RuntimeException("Thread sleep ERROR"); ++ } ++ } ++ } ++ return appendMessageResultArray; ++ } ++ ++ private long getMaxOffset(AppendMessageResult[] appendMessageResultArray) { ++ if (appendMessageResultArray == null) { ++ return 0; ++ } ++ AppendMessageResult last = appendMessageResultArray[appendMessageResultArray.length - 1]; ++ return last.getWroteOffset() + last.getWroteBytes(); ++ } ++ ++ private String buildMessageBodyByOffset(String message, long i) { ++ return String.format("%s offset %d", message, i); ++ } ++ ++ private long getStoreTime(CqUnit cqUnit) { ++ try { ++ Class abstractConsumeQueueStore = getDefaultMessageStore().getQueueStore().getClass().getSuperclass(); ++ Method getStoreTime = abstractConsumeQueueStore.getDeclaredMethod("getStoreTime", CqUnit.class); ++ getStoreTime.setAccessible(true); ++ return (long) getStoreTime.invoke(getDefaultMessageStore().getQueueStore(), cqUnit); ++ } catch (NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { ++ throw new RuntimeException(e); ++ } ++ } ++ ++ private MessageExtBrokerInner buildMessage(byte[] messageBody, String topic) { ++ MessageExtBrokerInner msg = new MessageExtBrokerInner(); ++ msg.setTopic(topic); ++ msg.setTags("TAG1"); ++ msg.setKeys("Hello"); ++ msg.setBody(messageBody); ++ msg.setKeys(String.valueOf(System.currentTimeMillis())); ++ msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); ++ msg.setSysFlag(0); ++ msg.setBornTimestamp(System.currentTimeMillis()); ++ msg.setStoreHost(storeHost); ++ msg.setBornHost(bornHost); ++ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); ++ return msg; ++ } ++ ++ private MessageExtBrokerInner buildSpecifyLengthPropertyMessage(byte[] messageBody, String topic, int length) { ++ StringBuilder stringBuilder = new StringBuilder(); ++ Random random = new Random(); ++ for (int i = 0; i < length; i++) { ++ stringBuilder.append(random.nextInt(10)); ++ } ++ MessageExtBrokerInner msg = new MessageExtBrokerInner(); ++ msg.putUserProperty("test", stringBuilder.toString()); ++ msg.setTopic(topic); ++ msg.setTags("TAG1"); ++ msg.setKeys("Hello"); ++ msg.setBody(messageBody); ++ msg.setQueueId(0); ++ msg.setBornTimestamp(System.currentTimeMillis()); ++ msg.setStoreHost(storeHost); ++ msg.setBornHost(bornHost); ++ msg.setPropertiesString(MessageDecoder.messageProperties2String(msg.getProperties())); ++ return msg; ++ } ++ ++ private MessageExtBrokerInner buildIPv6HostMessage(byte[] messageBody, String topic) { ++ MessageExtBrokerInner msg = new MessageExtBrokerInner(); ++ msg.setTopic(topic); ++ msg.setTags("TAG1"); ++ msg.setKeys("Hello"); ++ msg.setBody(messageBody); ++ msg.setMsgId("24084004018081003FAA1DDE2B3F898A00002A9F0000000000000CA0"); ++ msg.setKeys(String.valueOf(System.currentTimeMillis())); ++ msg.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); ++ msg.setSysFlag(0); ++ msg.setBornHostV6Flag(); ++ msg.setStoreHostAddressV6Flag(); ++ msg.setBornTimestamp(System.currentTimeMillis()); ++ try { ++ msg.setBornHost(new InetSocketAddress(InetAddress.getByName("1050:0000:0000:0000:0005:0600:300c:326b"), 0)); ++ } catch (UnknownHostException e) { ++ e.printStackTrace(); ++ assertThat(Boolean.FALSE).isTrue(); ++ } ++ ++ try { ++ msg.setStoreHost(new InetSocketAddress(InetAddress.getByName("::1"), 0)); ++ } catch (UnknownHostException e) { ++ e.printStackTrace(); ++ assertThat(Boolean.FALSE).isTrue(); ++ } ++ return msg; ++ } ++ ++ private MessageExtBrokerInner buildMessage() { ++ return buildMessage(messageBody, messageTopic); ++ } ++ ++ public MessageExtBatch buildMessageBatch(MessageBatch msgBatch) { ++ MessageExtBatch msgExtBatch = new MessageExtBatch(); ++ msgExtBatch.setTopic(messageTopic); ++ msgExtBatch.setTags("TAG1"); ++ msgExtBatch.setKeys("Hello"); ++ msgExtBatch.setBody(msgBatch.getBody()); ++ msgExtBatch.setKeys(String.valueOf(System.currentTimeMillis())); ++ msgExtBatch.setQueueId(Math.abs(queueId.getAndIncrement()) % queueTotal); ++ msgExtBatch.setSysFlag(0); ++ msgExtBatch.setBornTimestamp(System.currentTimeMillis()); ++ msgExtBatch.setStoreHost(storeHost); ++ msgExtBatch.setBornHost(bornHost); ++ return msgExtBatch; ++ } ++ ++ @Test ++ public void testGroupCommit() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ long totalMsgs = 10; ++ queueTotal = 1; ++ messageBody = storeMessage.getBytes(); ++ for (long i = 0; i < totalMsgs; i++) { ++ messageStore.putMessage(buildMessage()); ++ } ++ ++ for (long i = 0; i < totalMsgs; i++) { ++ GetMessageResult result = messageStore.getMessage("GROUP_A", "TOPIC_A", 0, i, 1024 * 1024, null); ++ assertThat(result).isNotNull(); ++ result.release(); ++ } ++ verifyThatMasterIsFunctional(totalMsgs, messageStore); ++ } ++ ++ @Test ++ public void testMaxOffset() throws InterruptedException { ++ if (notExecuted()) { ++ return; ++ } ++ int firstBatchMessages = 3; ++ int queueId = 0; ++ messageBody = storeMessage.getBytes(); ++ ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(0); ++ ++ for (int i = 0; i < firstBatchMessages; i++) { ++ final MessageExtBrokerInner msg = buildMessage(); ++ msg.setQueueId(queueId); ++ messageStore.putMessage(msg); ++ } ++ ++ while (messageStore.dispatchBehindBytes() != 0) { ++ TimeUnit.MILLISECONDS.sleep(1); ++ } ++ ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages); ++ ++ // Disable the dispatcher ++ messageStore.getDispatcherList().clear(); ++ ++ int secondBatchMessages = 2; ++ ++ for (int i = 0; i < secondBatchMessages; i++) { ++ final MessageExtBrokerInner msg = buildMessage(); ++ msg.setQueueId(queueId); ++ messageStore.putMessage(msg); ++ } ++ ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId)).isEqualTo(firstBatchMessages); ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, true)).isEqualTo(firstBatchMessages); ++ assertThat(messageStore.getMaxOffsetInQueue(messageTopic, queueId, false)).isEqualTo(firstBatchMessages + secondBatchMessages); ++ } ++ ++ private MessageExtBrokerInner buildIPv6HostMessage() { ++ return buildIPv6HostMessage(messageBody, "FooBar"); ++ } ++ ++ private void verifyThatMasterIsFunctional(long totalMsgs, MessageStore master) { ++ for (long i = 0; i < totalMsgs; i++) { ++ master.putMessage(buildMessage()); ++ } ++ ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ for (long i = 0; i < totalMsgs; i++) { ++ GetMessageResult result = master.getMessage("GROUP_A", "FooBar", 0, i, 1024 * 1024, null); ++ assertThat(result).isNotNull(); ++ result.release(); ++ ++ } ++ } ++ ++ @Test ++ public void testPullSize() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ String topic = "pullSizeTopic"; ++ ++ for (int i = 0; i < 32; i++) { ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ // wait for consume queue build ++ // the sleep time should be great than consume queue flush interval ++ //Thread.sleep(100); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ String group = "simple"; ++ GetMessageResult getMessageResult32 = messageStore.getMessage(group, topic, 0, 0, 32, null); ++ assertThat(getMessageResult32.getMessageBufferList().size()).isEqualTo(32); ++ getMessageResult32.release(); ++ ++ GetMessageResult getMessageResult20 = messageStore.getMessage(group, topic, 0, 0, 20, null); ++ assertThat(getMessageResult20.getMessageBufferList().size()).isEqualTo(20); ++ ++ getMessageResult20.release(); ++ GetMessageResult getMessageResult45 = messageStore.getMessage(group, topic, 0, 0, 10, null); ++ assertThat(getMessageResult45.getMessageBufferList().size()).isEqualTo(10); ++ getMessageResult45.release(); ++ ++ } ++ ++ @Test ++ public void testRecover() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ String topic = "recoverTopic"; ++ messageBody = storeMessage.getBytes(); ++ for (int i = 0; i < 100; i++) { ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ ++ // Thread.sleep(100);//wait for build consumer queue ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ ++ long maxPhyOffset = messageStore.getMaxPhyOffset(); ++ long maxCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); ++ ++ //1.just reboot ++ messageStore.shutdown(); ++ String storeRootDir = ((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir(); ++ messageStore = buildMessageStore(storeRootDir, topic); ++ boolean load = messageStore.load(); ++ assertTrue(load); ++ messageStore.start(); ++ assertTrue(maxPhyOffset == messageStore.getMaxPhyOffset()); ++ assertTrue(maxCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); ++ ++ //2.damage commit-log and reboot normal ++ for (int i = 0; i < 100; i++) { ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ //Thread.sleep(100); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ long secondLastPhyOffset = messageStore.getMaxPhyOffset(); ++ long secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); ++ ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ ++ ++ messageStore.shutdown(); ++ ++ //damage last message ++ damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset); ++ ++ //reboot ++ messageStore = buildMessageStore(storeRootDir, topic); ++ load = messageStore.load(); ++ assertTrue(load); ++ messageStore.start(); ++ assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset()); ++ assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); ++ ++ //3.damage commitlog and reboot abnormal ++ for (int i = 0; i < 100; i++) { ++ messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ //Thread.sleep(100); ++ StoreTestUtil.waitCommitLogReput((RocksDBMessageStore) messageStore); ++ secondLastPhyOffset = messageStore.getMaxPhyOffset(); ++ secondLastCqOffset = messageStore.getMaxOffsetInQueue(topic, 0); ++ ++ messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ messageStore.shutdown(); ++ ++ //damage last message ++ damageCommitLog((RocksDBMessageStore) messageStore, secondLastPhyOffset); ++ //add abort file ++ String fileName = StorePathConfigHelper.getAbortFile(((RocksDBMessageStore) messageStore).getMessageStoreConfig().getStorePathRootDir()); ++ File file = new File(fileName); ++ UtilAll.ensureDirOK(file.getParent()); ++ file.createNewFile(); ++ ++ messageStore = buildMessageStore(storeRootDir, topic); ++ load = messageStore.load(); ++ assertTrue(load); ++ messageStore.start(); ++ assertTrue(secondLastPhyOffset == messageStore.getMaxPhyOffset()); ++ assertTrue(secondLastCqOffset == messageStore.getMaxOffsetInQueue(topic, 0)); ++ ++ //message write again ++ for (int i = 0; i < 100; i++) { ++ messageExtBrokerInner = buildMessage(); ++ messageExtBrokerInner.setTopic(topic); ++ messageExtBrokerInner.setQueueId(0); ++ messageStore.putMessage(messageExtBrokerInner); ++ } ++ } ++ ++ @Test ++ public void testStorePathOK() { ++ if (notExecuted()) { ++ return; ++ } ++ if (messageStore instanceof RocksDBMessageStore) { ++ assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathPhysic())); ++ assertTrue(fileExists(((RocksDBMessageStore) messageStore).getStorePathLogic())); ++ } ++ } ++ ++ private boolean fileExists(String path) { ++ if (path != null) { ++ File f = new File(path); ++ return f.exists(); ++ } ++ return false; ++ } ++ ++ private void damageCommitLog(RocksDBMessageStore store, long offset) throws Exception { ++ assertThat(store).isNotNull(); ++ MessageStoreConfig messageStoreConfig = store.getMessageStoreConfig(); ++ File file = new File(messageStoreConfig.getStorePathCommitLog() + File.separator + "00000000000000000000"); ++ try (RandomAccessFile raf = new RandomAccessFile(file, "rw"); ++ FileChannel fileChannel = raf.getChannel()) { ++ MappedByteBuffer mappedByteBuffer = fileChannel.map(FileChannel.MapMode.READ_WRITE, 0, 1024 * 1024 * 10); ++ int bodyLen = mappedByteBuffer.getInt((int) offset + 84); ++ int topicLenIndex = (int) offset + 84 + bodyLen + 4; ++ mappedByteBuffer.position(topicLenIndex); ++ mappedByteBuffer.putInt(0); ++ mappedByteBuffer.putInt(0); ++ mappedByteBuffer.putInt(0); ++ mappedByteBuffer.putInt(0); ++ mappedByteBuffer.force(); ++ fileChannel.force(true); ++ } ++ } ++ ++ @Test ++ public void testPutMsgExceedsMaxLength() { ++ if (notExecuted()) { ++ return; ++ } ++ messageBody = new byte[4 * 1024 * 1024 + 1]; ++ MessageExtBrokerInner msg = buildMessage(); ++ ++ PutMessageResult result = messageStore.putMessage(msg); ++ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.MESSAGE_ILLEGAL); ++ } ++ ++ @Test ++ public void testPutMsgBatchExceedsMaxLength() { ++ if (notExecuted()) { ++ return; ++ } ++ messageBody = new byte[4 * 1024 * 1024 + 1]; ++ MessageExtBrokerInner msg1 = buildMessage(); ++ MessageExtBrokerInner msg2 = buildMessage(); ++ MessageExtBrokerInner msg3 = buildMessage(); ++ ++ MessageBatch msgBatch = MessageBatch.generateFromList(Arrays.asList(msg1, msg2, msg3)); ++ msgBatch.setBody(msgBatch.encode()); ++ ++ MessageExtBatch msgExtBatch = buildMessageBatch(msgBatch); ++ ++ try { ++ PutMessageResult result = this.messageStore.putMessages(msgExtBatch); ++ } catch (Exception e) { ++ assertThat(e.getMessage()).contains("message body size exceeded"); ++ } ++ } ++ ++ @Test ++ public void testPutMsgWhenReplicasNotEnough() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig(); ++ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER); ++ messageStoreConfig.setTotalReplicas(2); ++ messageStoreConfig.setInSyncReplicas(2); ++ messageStoreConfig.setEnableAutoInSyncReplicas(false); ++ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true); ++ this.messageStore.setAliveReplicaNumInGroup(1); ++ ++ MessageExtBrokerInner msg = buildMessage(); ++ PutMessageResult result = this.messageStore.putMessage(msg); ++ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.IN_SYNC_REPLICAS_NOT_ENOUGH); ++ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false); ++ } ++ ++ @Test ++ public void testPutMsgWhenAdaptiveDegradation() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) this.messageStore).getMessageStoreConfig(); ++ messageStoreConfig.setBrokerRole(BrokerRole.SYNC_MASTER); ++ messageStoreConfig.setTotalReplicas(2); ++ messageStoreConfig.setInSyncReplicas(2); ++ messageStoreConfig.setEnableAutoInSyncReplicas(true); ++ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(true); ++ this.messageStore.setAliveReplicaNumInGroup(1); ++ ++ MessageExtBrokerInner msg = buildMessage(); ++ PutMessageResult result = this.messageStore.putMessage(msg); ++ assertThat(result.getPutMessageStatus()).isEqualTo(PutMessageStatus.PUT_OK); ++ ((RocksDBMessageStore) this.messageStore).getBrokerConfig().setEnableSlaveActingMaster(false); ++ messageStoreConfig.setEnableAutoInSyncReplicas(false); ++ } ++ ++ @Test ++ public void testGetBulkCommitLogData() { ++ if (notExecuted()) { ++ return; ++ } ++ RocksDBMessageStore defaultMessageStore = (RocksDBMessageStore) messageStore; ++ ++ messageBody = new byte[2 * 1024 * 1024]; ++ ++ for (int i = 0; i < 10; i++) { ++ MessageExtBrokerInner msg1 = buildMessage(); ++ messageStore.putMessage(msg1); ++ } ++ ++ List bufferResultList = defaultMessageStore.getBulkCommitLogData(0, (int) defaultMessageStore.getMaxPhyOffset()); ++ List msgList = new ArrayList<>(); ++ for (SelectMappedBufferResult bufferResult : bufferResultList) { ++ msgList.addAll(MessageDecoder.decodesBatch(bufferResult.getByteBuffer(), true, false, false)); ++ bufferResult.release(); ++ } ++ ++ assertThat(msgList.size()).isEqualTo(10); ++ } ++ ++ @Test ++ public void testPutLongMessage() throws Exception { ++ if (notExecuted()) { ++ return; ++ } ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ CommitLog commitLog = ((RocksDBMessageStore) messageStore).getCommitLog(); ++ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig(); ++ MessageExtEncoder.PutMessageThreadLocal putMessageThreadLocal = commitLog.getPutMessageThreadLocal().get(); ++ ++ //body size, topic size, properties size exactly equal to max size ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); ++ messageExtBrokerInner.setTopic(new String(new byte[127])); ++ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE])); ++ PutMessageResult encodeResult1 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult1 == null); ++ ++ //body size exactly more than max message body size ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 1]); ++ PutMessageResult encodeResult2 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult2.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ ++ //body size exactly equal to max message size ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize() + 64 * 1024]); ++ PutMessageResult encodeResult3 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult3.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ ++ //message properties length more than properties maxSize ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); ++ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE + 1])); ++ PutMessageResult encodeResult4 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult4.getPutMessageStatus() == PutMessageStatus.PROPERTIES_SIZE_EXCEEDED); ++ ++ //message length more than buffer length capacity ++ messageExtBrokerInner.setBody(new byte[messageStoreConfig.getMaxMessageSize()]); ++ messageExtBrokerInner.setTopic(new String(new byte[Short.MAX_VALUE])); ++ messageExtBrokerInner.setPropertiesString(new String(new byte[Short.MAX_VALUE])); ++ PutMessageResult encodeResult5 = putMessageThreadLocal.getEncoder().encode(messageExtBrokerInner); ++ assertTrue(encodeResult5.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ } ++ ++ @Test ++ public void testDynamicMaxMessageSize() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageExtBrokerInner messageExtBrokerInner = buildMessage(); ++ MessageStoreConfig messageStoreConfig = ((RocksDBMessageStore) messageStore).getMessageStoreConfig(); ++ int originMaxMessageSize = messageStoreConfig.getMaxMessageSize(); ++ ++ messageExtBrokerInner.setBody(new byte[originMaxMessageSize + 10]); ++ PutMessageResult putMessageResult = messageStore.putMessage(messageExtBrokerInner); ++ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ ++ int newMaxMessageSize = originMaxMessageSize + 10; ++ messageStoreConfig.setMaxMessageSize(newMaxMessageSize); ++ putMessageResult = messageStore.putMessage(messageExtBrokerInner); ++ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.PUT_OK); ++ ++ messageStoreConfig.setMaxMessageSize(10); ++ putMessageResult = messageStore.putMessage(messageExtBrokerInner); ++ assertTrue(putMessageResult.getPutMessageStatus() == PutMessageStatus.MESSAGE_ILLEGAL); ++ ++ messageStoreConfig.setMaxMessageSize(originMaxMessageSize); ++ } ++ ++ @Test ++ public void testDeleteTopics() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig(); ++ ConcurrentMap> consumeQueueTable = ++ ((RocksDBMessageStore) messageStore).getConsumeQueueTable(); ++ for (int i = 0; i < 10; i++) { ++ ConcurrentMap cqTable = new ConcurrentHashMap<>(); ++ String topicName = "topic-" + i; ++ for (int j = 0; j < 4; j++) { ++ ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(), ++ messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore); ++ cqTable.put(j, consumeQueue); ++ } ++ consumeQueueTable.put(topicName, cqTable); ++ } ++ Assert.assertEquals(consumeQueueTable.size(), 10); ++ HashSet resultSet = Sets.newHashSet("topic-3", "topic-5"); ++ messageStore.deleteTopics(Sets.difference(consumeQueueTable.keySet(), resultSet)); ++ Assert.assertEquals(consumeQueueTable.size(), 2); ++ Assert.assertEquals(resultSet, consumeQueueTable.keySet()); ++ } ++ ++ @Test ++ public void testCleanUnusedTopic() { ++ if (notExecuted()) { ++ return; ++ } ++ MessageStoreConfig messageStoreConfig = messageStore.getMessageStoreConfig(); ++ ConcurrentMap> consumeQueueTable = ++ ((RocksDBMessageStore) messageStore).getConsumeQueueTable(); ++ for (int i = 0; i < 10; i++) { ++ ConcurrentMap cqTable = new ConcurrentHashMap<>(); ++ String topicName = "topic-" + i; ++ for (int j = 0; j < 4; j++) { ++ ConsumeQueue consumeQueue = new ConsumeQueue(topicName, j, messageStoreConfig.getStorePathRootDir(), ++ messageStoreConfig.getMappedFileSizeConsumeQueue(), messageStore); ++ cqTable.put(j, consumeQueue); ++ } ++ consumeQueueTable.put(topicName, cqTable); ++ } ++ Assert.assertEquals(consumeQueueTable.size(), 10); ++ HashSet resultSet = Sets.newHashSet("topic-3", "topic-5"); ++ messageStore.cleanUnusedTopic(resultSet); ++ Assert.assertEquals(consumeQueueTable.size(), 2); ++ Assert.assertEquals(resultSet, consumeQueueTable.keySet()); ++ } ++ ++ private class MyMessageArrivingListener implements MessageArrivingListener { ++ @Override ++ public void arriving(String topic, int queueId, long logicOffset, long tagsCode, long msgStoreTime, ++ byte[] filterBitMap, Map properties) { ++ } ++ } ++ ++ private boolean notExecuted() { ++ return MixAll.isMac(); ++ } ++} ++ ++ +diff --git a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java +index b2d99c3ed..17a2b5e19 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java ++++ b/store/src/test/java/org/apache/rocketmq/store/StoreTestUtil.java +@@ -38,11 +38,16 @@ public class StoreTestUtil { + + public static boolean isCommitLogAvailable(DefaultMessageStore store) { + try { ++ Field serviceField = null; ++ if (store instanceof RocksDBMessageStore) { ++ serviceField = store.getClass().getSuperclass().getDeclaredField("reputMessageService"); ++ } else { ++ serviceField = store.getClass().getDeclaredField("reputMessageService"); ++ } + +- Field serviceField = store.getClass().getDeclaredField("reputMessageService"); + serviceField.setAccessible(true); + DefaultMessageStore.ReputMessageService reputService = +- (DefaultMessageStore.ReputMessageService) serviceField.get(store); ++ (DefaultMessageStore.ReputMessageService) serviceField.get(store); + + Method method = DefaultMessageStore.ReputMessageService.class.getDeclaredMethod("isCommitLogAvailable"); + method.setAccessible(true); +diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java +index 54174ac16..fa8f41dbf 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/ha/HAServerTest.java +@@ -36,7 +36,7 @@ import org.junit.Before; + import org.junit.Test; + import org.junit.runner.RunWith; + import org.mockito.junit.MockitoJUnitRunner; +- ++import org.rocksdb.RocksDBException; + import static org.assertj.core.api.Assertions.assertThat; + import static org.awaitility.Awaitility.await; + import static org.mockito.ArgumentMatchers.anyLong; +@@ -114,7 +114,7 @@ public class HAServerTest { + } + + @Test +- public void inSyncReplicasNums() throws IOException { ++ public void inSyncReplicasNums() throws IOException, RocksDBException { + DefaultMessageStore messageStore = mockMessageStore(); + doReturn(123L).when(messageStore).getMaxPhyOffset(); + doReturn(123L).when(messageStore).getMasterFlushedOffset(); +@@ -150,7 +150,7 @@ public class HAServerTest { + } + + @Test +- public void isSlaveOK() throws IOException { ++ public void isSlaveOK() throws IOException, RocksDBException { + DefaultMessageStore messageStore = mockMessageStore(); + doReturn(123L).when(messageStore).getMaxPhyOffset(); + doReturn(123L).when(messageStore).getMasterFlushedOffset(); +@@ -175,7 +175,8 @@ public class HAServerTest { + } + + @Test +- public void putRequest_SingleAck() throws IOException, ExecutionException, InterruptedException, TimeoutException { ++ public void putRequest_SingleAck() ++ throws IOException, ExecutionException, InterruptedException, TimeoutException, RocksDBException { + CommitLog.GroupCommitRequest request = new CommitLog.GroupCommitRequest(124, 4000, 1); + this.haService.putRequest(request); + +@@ -192,7 +193,8 @@ public class HAServerTest { + } + + @Test +- public void putRequest_MultipleAckAndRequests() throws IOException, ExecutionException, InterruptedException { ++ public void putRequest_MultipleAckAndRequests() ++ throws IOException, ExecutionException, InterruptedException, RocksDBException { + CommitLog.GroupCommitRequest oneAck = new CommitLog.GroupCommitRequest(124, 4000, 2); + this.haService.putRequest(oneAck); + +@@ -218,7 +220,7 @@ public class HAServerTest { + } + + @Test +- public void getPush2SlaveMaxOffset() throws IOException { ++ public void getPush2SlaveMaxOffset() throws IOException, RocksDBException { + DefaultMessageStore messageStore = mockMessageStore(); + doReturn(123L).when(messageStore).getMaxPhyOffset(); + doReturn(123L).when(messageStore).getMasterFlushedOffset(); +@@ -256,7 +258,7 @@ public class HAServerTest { + this.haClientList.add(haClient); + } + +- private DefaultMessageStore mockMessageStore() throws IOException { ++ private DefaultMessageStore mockMessageStore() throws IOException, RocksDBException { + DefaultMessageStore messageStore = mock(DefaultMessageStore.class); + BrokerConfig brokerConfig = mock(BrokerConfig.class); + +diff --git a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java +index 27dcff141..db5c5af4c 100644 +--- a/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java ++++ b/store/src/test/java/org/apache/rocketmq/store/ha/autoswitch/AutoSwitchHATest.java +@@ -41,6 +41,7 @@ import org.junit.Assert; + import org.junit.Assume; + import org.junit.Ignore; + import org.junit.Test; ++import org.rocksdb.RocksDBException; + + import java.io.File; + import java.net.InetAddress; +@@ -180,7 +181,7 @@ public class AutoSwitchHATest { + + private boolean changeMasterAndPutMessage(DefaultMessageStore master, MessageStoreConfig masterConfig, + DefaultMessageStore slave, long slaveId, MessageStoreConfig slaveConfig, int epoch, String masterHaAddress, +- int totalPutMessageNums) { ++ int totalPutMessageNums) throws RocksDBException { + + boolean flag = true; + // Change role +diff --git a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +index d7d13d61e..edaa5d19f 100644 +--- a/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java ++++ b/tieredstore/src/main/java/org/apache/rocketmq/tieredstore/TieredMessageStore.java +@@ -16,17 +16,14 @@ + */ + package org.apache.rocketmq.tieredstore; + +-import com.google.common.base.Stopwatch; +-import io.opentelemetry.api.common.Attributes; +-import io.opentelemetry.api.common.AttributesBuilder; +-import io.opentelemetry.api.metrics.Meter; +-import io.opentelemetry.sdk.metrics.InstrumentSelector; +-import io.opentelemetry.sdk.metrics.ViewBuilder; + import java.util.List; + import java.util.Set; + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.TimeUnit; + import java.util.function.Supplier; ++ ++import com.google.common.base.Stopwatch; ++ + import org.apache.commons.lang3.StringUtils; + import org.apache.rocketmq.common.BoundaryType; + import org.apache.rocketmq.common.MixAll; +@@ -55,6 +52,12 @@ import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsConstant; + import org.apache.rocketmq.tieredstore.metrics.TieredStoreMetricsManager; + import org.apache.rocketmq.tieredstore.util.TieredStoreUtil; + ++import io.opentelemetry.api.common.Attributes; ++import io.opentelemetry.api.common.AttributesBuilder; ++import io.opentelemetry.api.metrics.Meter; ++import io.opentelemetry.sdk.metrics.InstrumentSelector; ++import io.opentelemetry.sdk.metrics.ViewBuilder; ++ + public class TieredMessageStore extends AbstractPluginMessageStore { + + protected static final Logger logger = LoggerFactory.getLogger(TieredStoreUtil.TIERED_STORE_LOGGER_NAME); +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +index 2a7d3fba4..1ecb1fa2c 100644 +--- a/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/export/ExportMetadataInRocksDBCommand.java +@@ -106,8 +106,8 @@ public class ExportMetadataInRocksDBCommand implements SubCommand { + final Map jsonConfig = new HashMap<>(); + final Map configTable = new HashMap<>(); + iterateKvStore(kvStore, (key, value) -> { +- final String configKey = new String(key, DataConverter.charset); +- final String configValue = new String(value, DataConverter.charset); ++ final String configKey = new String(key, DataConverter.CHARSET_UTF8); ++ final String configValue = new String(value, DataConverter.CHARSET_UTF8); + final JSONObject jsonObject = JSONObject.parseObject(configValue); + configTable.put(configKey, jsonObject); + } +@@ -120,8 +120,8 @@ public class ExportMetadataInRocksDBCommand implements SubCommand { + } else { + AtomicLong count = new AtomicLong(0); + iterateKvStore(kvStore, (key, value) -> { +- final String configKey = new String(key, DataConverter.charset); +- final String configValue = new String(value, DataConverter.charset); ++ final String configKey = new String(key, DataConverter.CHARSET_UTF8); ++ final String configValue = new String(value, DataConverter.CHARSET_UTF8); + System.out.printf("%d, Key: %s, Value: %s%n", count.incrementAndGet(), configKey, configValue); + }); + } +diff --git a/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +new file mode 100644 +index 000000000..b987ad873 +--- /dev/null ++++ b/tools/src/main/java/org/apache/rocketmq/tools/command/metadata/RocksDBConfigToJsonCommand.java +@@ -0,0 +1,118 @@ ++/* ++ * Licensed to the Apache Software Foundation (ASF) under one or more ++ * contributor license agreements. See the NOTICE file distributed with ++ * this work for additional information regarding copyright ownership. ++ * The ASF licenses this file to You under the Apache License, Version 2.0 ++ * (the "License"); you may not use this file except in compliance with ++ * the License. You may obtain a copy of the License at ++ * ++ * http://www.apache.org/licenses/LICENSE-2.0 ++ * ++ * Unless required by applicable law or agreed to in writing, software ++ * distributed under the License is distributed on an "AS IS" BASIS, ++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++ * See the License for the specific language governing permissions and ++ * limitations under the License. ++ */ ++package org.apache.rocketmq.tools.command.metadata; ++ ++import com.alibaba.fastjson.JSONObject; ++import org.apache.commons.cli.CommandLine; ++import org.apache.commons.cli.Option; ++import org.apache.commons.cli.Options; ++import org.apache.commons.lang3.StringUtils; ++import org.apache.rocketmq.common.config.RocksDBConfigManager; ++import org.apache.rocketmq.common.utils.DataConverter; ++import org.apache.rocketmq.remoting.RPCHook; ++import org.apache.rocketmq.tools.command.SubCommand; ++import org.apache.rocketmq.tools.command.SubCommandException; ++ ++import java.io.File; ++import java.util.HashMap; ++import java.util.Map; ++ ++public class RocksDBConfigToJsonCommand implements SubCommand { ++ private static final String TOPICS_JSON_CONFIG = "topics"; ++ private static final String SUBSCRIPTION_GROUP_JSON_CONFIG = "subscriptionGroups"; ++ ++ @Override ++ public String commandName() { ++ return "rocksDBConfigToJson"; ++ } ++ ++ @Override ++ public String commandDesc() { ++ return "Convert RocksDB kv config (topics/subscriptionGroups) to json"; ++ } ++ ++ @Override ++ public Options buildCommandlineOptions(Options options) { ++ Option pathOption = new Option("p", "path", true, ++ "Absolute path to the metadata directory"); ++ pathOption.setRequired(true); ++ options.addOption(pathOption); ++ ++ Option configTypeOption = new Option("t", "configType", true, "Name of kv config, e.g. " + ++ "topics/subscriptionGroups"); ++ configTypeOption.setRequired(true); ++ options.addOption(configTypeOption); ++ ++ return options; ++ } ++ ++ @Override ++ public void execute(CommandLine commandLine, Options options, RPCHook rpcHook) throws SubCommandException { ++ String path = commandLine.getOptionValue("path").trim(); ++ if (StringUtils.isEmpty(path) || !new File(path).exists()) { ++ System.out.print("Rocksdb path is invalid.\n"); ++ return; ++ } ++ ++ String configType = commandLine.getOptionValue("configType").trim().toLowerCase(); ++ ++ final long memTableFlushInterval = 60 * 60 * 1000L; ++ RocksDBConfigManager kvConfigManager = new RocksDBConfigManager(memTableFlushInterval); ++ try { ++ if (TOPICS_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for topics.json ++ final Map topicsJsonConfig = new HashMap<>(); ++ final Map topicConfigTable = new HashMap<>(); ++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++ final String topic = new String(key, DataConverter.CHARSET_UTF8); ++ final String topicConfig = new String(value, DataConverter.CHARSET_UTF8); ++ final JSONObject jsonObject = JSONObject.parseObject(topicConfig); ++ topicConfigTable.put(topic, jsonObject); ++ }); ++ ++ if (isLoad) { ++ topicsJsonConfig.put("topicConfigTable", (JSONObject) JSONObject.toJSON(topicConfigTable)); ++ final String topicsJsonStr = JSONObject.toJSONString(topicsJsonConfig, true); ++ System.out.print(topicsJsonStr + "\n"); ++ return; ++ } ++ } ++ if (SUBSCRIPTION_GROUP_JSON_CONFIG.toLowerCase().equals(configType)) { ++ // for subscriptionGroup.json ++ final Map subscriptionGroupJsonConfig = new HashMap<>(); ++ final Map subscriptionGroupTable = new HashMap<>(); ++ boolean isLoad = kvConfigManager.load(path, (key, value) -> { ++ final String subscriptionGroup = new String(key, DataConverter.CHARSET_UTF8); ++ final String subscriptionGroupConfig = new String(value, DataConverter.CHARSET_UTF8); ++ final JSONObject jsonObject = JSONObject.parseObject(subscriptionGroupConfig); ++ subscriptionGroupTable.put(subscriptionGroup, jsonObject); ++ }); ++ ++ if (isLoad) { ++ subscriptionGroupJsonConfig.put("subscriptionGroupTable", ++ (JSONObject) JSONObject.toJSON(subscriptionGroupTable)); ++ final String subscriptionGroupJsonStr = JSONObject.toJSONString(subscriptionGroupJsonConfig, true); ++ System.out.print(subscriptionGroupJsonStr + "\n"); ++ return; ++ } ++ } ++ System.out.print("Config type was not recognized, configType=" + configType + "\n"); ++ } finally { ++ kvConfigManager.stop(); ++ } ++ } ++} +-- +2.32.0.windows.2 + diff --git a/rocketmq.spec b/rocketmq.spec index f67608c..d377d58 100644 --- a/rocketmq.spec +++ b/rocketmq.spec @@ -5,7 +5,7 @@ Summary: Cloud-Native, Distributed Messaging and Streaming Name: rocketmq Version: 5.1.3 -Release: 22 +Release: 23 License: Apache-2.0 Group: Applications/Message URL: https://rocketmq.apache.org/ @@ -31,6 +31,7 @@ Patch0018: patch018-backport-enhancement-of-tiered-storage.patch Patch0019: patch019-backport-some-bugfix.patch Patch0020: patch020-backport-add-goaway-mechanism.patch Patch0021: patch021-backport-some-enhancements.patch +Patch0022: patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch BuildRequires: java-1.8.0-openjdk-devel, maven, maven-local, git Requires: java-1.8.0-openjdk-devel @@ -65,6 +66,9 @@ exit 0 %changelog +* Tue Dec 8 2023 ShiZhili - 5.1.3-23 +- backport support kv storage + * Tue Dec 5 2023 ShiZhili - 5.1.3-22 - backport some enhancements -- Gitee From dbcc8b7037d6e114a1ecb13ed125af4aee934191 Mon Sep 17 00:00:00 2001 From: shizhili Date: Fri, 8 Dec 2023 09:50:26 +0800 Subject: [PATCH 2/2] backport some bugfixes --- patch023-backport-some-bugfixes.patch | 8644 +++++++++++++++++++++++++ rocketmq.spec | 6 +- 2 files changed, 8649 insertions(+), 1 deletion(-) create mode 100644 patch023-backport-some-bugfixes.patch diff --git a/patch023-backport-some-bugfixes.patch b/patch023-backport-some-bugfixes.patch new file mode 100644 index 0000000..9226e24 --- /dev/null +++ b/patch023-backport-some-bugfixes.patch @@ -0,0 +1,8644 @@ +From 0f01df460f78c383a35338aa77eb0fda4c8f2dd3 Mon Sep 17 00:00:00 2001 +From: Ao Qiao +Date: Mon, 16 Oct 2023 11:37:25 +0800 +Subject: [PATCH 1/8] [ISSUE 7265] Doc: Adding how to debug in Idea document + (#7266) + +--- + docs/cn/Debug_In_Idea.md | 55 +++++++++++++++++++++++ + docs/cn/image/Idea_config_broker.png | Bin 0 -> 215173 bytes + docs/cn/image/Idea_config_nameserver.png | Bin 0 -> 204631 bytes + docs/en/Debug_In_Idea.md | 55 +++++++++++++++++++++++ + 4 files changed, 110 insertions(+) + create mode 100644 docs/cn/Debug_In_Idea.md + create mode 100644 docs/cn/image/Idea_config_broker.png + create mode 100644 docs/cn/image/Idea_config_nameserver.png + create mode 100644 docs/en/Debug_In_Idea.md + +diff --git a/docs/cn/Debug_In_Idea.md b/docs/cn/Debug_In_Idea.md +new file mode 100644 +index 000000000..fd01751ee +--- /dev/null ++++ b/docs/cn/Debug_In_Idea.md +@@ -0,0 +1,55 @@ ++## 本地调试RocketMQ ++ ++### Step0: 解决依赖问题 ++1. 运行前下载RocketMQ需要的maven依赖，可以使用`mvn clean install -Dmaven.test.skip=true` ++2. 确保本地能够编译通过 ++ ++### Step1: 启动NameServer ++1. NamerServer的启动类在`org.apache.rocketmq.namesrv.NamesrvStartup` ++2. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=` ++![Idea_config_nameserver.png](image/Idea_config_nameserver.png) ++3. 运行NameServer，观察到如下日志输出则启动成功 ++```shell ++The Name Server boot success. serializeType=JSON, address 0.0.0.0:9876 ++``` ++ ++### Step2: 启动Broker ++1. Broker的启动类在`org.apache.rocketmq.broker.BrokerStartup` ++2. 创建`/rocketmq/conf/broker.conf`文件或直接在官方release发布包中拷贝即可 ++```shell ++# broker.conf ++ ++brokerClusterName = DefaultCluster ++brokerName = broker-a ++brokerId = 0 ++deleteWhen = 04 ++fileReservedTime = 48 ++brokerRole = ASYNC_MASTER ++flushDiskType = ASYNC_FLUSH ++namesrvAddr = 127.0.0.1:9876 # name server地址 ++``` ++3. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=` 以及环境变量`-c /Users/xxx/rocketmq/conf/broker.conf` ++![Idea_config_broker.png](image/Idea_config_broker.png) ++4. 运行Broker，观察到如下日志则启动成功 ++```shell ++The broker[broker-a,192.169.1.2:10911] boot success... ++``` ++ ++### Step3: 发送或消费消息 ++至此已经完成了RocketMQ的启动，可以使用`/example`里的示例进行收发消息 ++ ++### 补充：本地启动Proxy ++1. RocketMQ5.x支持了Proxy模式，使用`LOCAL`模式可以免去`Step2`，启动类在`org.apache.rocketmq.proxy.ProxyStartup` ++2. `Idea-Edit Configurations`中添加运行参数 `ROCKETMQ_HOME=` ++3. 在`/conf/`下新建配置文件`rmq-proxy.json` ++```json ++{ ++ "rocketMQClusterName": "DefaultCluster", ++ "nameSrvAddr": "127.0.0.1:9876", ++ "proxyMode": "local" ++} ++``` ++4. 运行Proxy，观察到如下日志则启动成功 ++```shell ++Sat Aug 26 15:29:33 CST 2023 rocketmq-proxy startup successfully ++``` +\ No newline at end of file +diff --git a/docs/cn/image/Idea_config_broker.png b/docs/cn/image/Idea_config_broker.png +new file mode 100644 +index 0000000000000000000000000000000000000000..6fbedcfb627057fe9ae68d184dc6e44fc9918909 +GIT binary patch +literal 215173 +zcmagFbyQSe*fz|NLl4q5bT=qSH%O;+BPHEK51mquq%;VMG*Z$aU7`ZgHPSG2!#nzW +z-t|4}{pVZj{4q0U&OUqJaoyK-?{ng{HI?yksBlnFQ1De=DCnY~po36QP)mWB$X{9y +z$3CKri1m>dx=KYCS*)cWKqXNx46@bv)&eyR$0Lv6|sg5K6q&w +zOo+}2O$N+}g-u|*|8itq(-C$k>O5;GiHU-ml+K>$w-aCipeRyK0n@kb?S0zSdbpl^ +zlz+S$d}O%0-I9hYKHOi1J-XjaK3+bSl(0=qC;X!R=wJnSHo05eM%hr&RQF6KvY0*L +z{~mPD&fmz2gS(qz-w>M82|v;=JWQb!V$3}zDIv7fgs!kcA|4-(Q^F7gCE&Zu$D0+O +z`vKS8gIu-j-x!5HcvjJ=B~u9za&xA`JV$`!ph)T!9!@Z(Ki!2PPjlv +zh0=)Ph&qLt6o&#>qicd9CZQ07<>R@u<63dqrV2vOtUcC-)*``|700uhd7zz;Mf3lT +zrKkf-$p;24W28qE&{}*)EyR|gzIuGz3qw~xN5s=ML21W%8ra2VhS^y0IELb|R7mJ9 +zjQ`igAF*75=B|ng#)IOpu}O$GN8)yfB%I&Nj?NBy#?< +zO%k0m8MDo8>^Ja+f?MKZ^A9u$WE_C{O7W2850fhbB4u5(XD +zPRaeZ_+>2nTAQR(L(; +zFZ=h_A8?wW{O{vHM=uJcA9mc&hV$J+3oDG8_v7%fEozabp6n~p_orxJv7dgG4hK)`s{NK +zP9lzvE-Jqij1FHMgHj9ev1D!Ge>&v0;}X#=t%T2&gMjcR +zn%6F|7SyP+fqqiIo<7yv81qw|9M0VHIUg^X{uEPW7kf5&af(-?2GzMaUX{e$V<>11 +zk!$Y3WsF%su?3qEu4G>v{(fp}Umi!}iBPJY^w0Jd@W5HDY4omj9hWrnZLVRO)!1}~ +zpiQzASO1N{lQhB7g8G}8?dKx0;#aQ>9O%9c?L&W4E`R(@%ZRD0DACPP^|`$MQ(yp3 +z)#nl~))Xv#5P2^aai4Xjd#2$48aB+qvG55mc_bd*SQ11aF#94D4`u&EwdzEj_TVHv +z7O(SfNOC0GIksJ2(uNrOKT6n3o#Ty_BrgMAI~^5|-N|j_yHN%Up1M*lUg&-+im^jH +z7Z#*d3I_6e!xVgl>~)UzF&ODL$qf--$cCN)k~W&E0ivt|EJ7aO5gAD|p@;2Q?Mn#X +zWr5w#LAWeTrel-lY}hYstY*6;wi-(EIR%z@c@YqP-=c|{X}k3HC83Mf#L$O7#(4&& +zJRq^4Ep2Nj-_^UR3pw$*1-OOT9GOdH))l(9WThrVkxr_zGP4cqkFK*r3JX0^W>fXwnG$Q6Zg^|KTZ2@3IOGAo%z1_ +zoqcCB4qyqNn6=+liQm*3#irO`A@wEuqCZeDnym`PBOx(NXX>Hz{O;jVwR!d)>H)2% +zSD1sZOI}!8{Yg!hs82%N$Vvc7pY0p!EmpkrW3&Diq +zAU4z7o?2!o;ebffTH70*gL=Y+9Fwo57ELH9MAQ~f5cO}xx>Gj~TCZ7x?}CYFr?%GG +z5H__n@w4ciWVINWQ7DDgyrEz4jbQQh#8+TiaKZlNm2MJNj3nCRR$#jB$Fh#xK>Nu| +zVXhjt()oexX+IFjlp&QsH`lo=b>VV;X(QAqJg9zwU{+OqBGq|A@?zteP;-{pt-ac9 +zxjkIBolRvPEk6-Lz<=h+%}S>oDwl3<@Bu=+PF2Bm*f(s8>%H=aa^4ra5iK*^d&@^9ia{b50_=~mj#+);&~bi7WUark=vkD_F0XleCK2u +z=K?O*LA$`WKZXkOSuU*;@s+ZsJmdbF*SGc=>X~I0-WB?B+{IXM3zS|+e364Za~88d +zvM}5&fF99=1H^VIEw>{@hdr!Ov`XmNJ6?m(rw4WNV@De#nUbzRXGqVn(=#T$l>Q4- +zy9^CW8Bi2kpTPHeNe)4nG;1zRw-Ps~6cGM9tIa-d04D!UZ}x%YPl-y?r0Zm`aOl@9 +zZQqavis>wt*e#msaN8*dvJ3-cE=^u%ktHO@7UeU~3NS_p43un0KKV2&QJGjHZMX~F +zW4z-_zJ$XwN9yJ<0jThe;4|y(6lMTb7`UY7R=p?VJ(qpJ$asxa5~f>^I|)Ud3U+M0*CIiJ;|OjnR^&VGOe0OZrR7$~`gn^6~`Zs+P~zP1N{X*?flMJz&7 +z8P1#gVN{)jG=$#)2Z>TNx5vRAQoE7&=Nhg*7h`S*Pbbs1T${0))~JjFzHMr%lDPvH +zrFWx4C{nC+ZS8|Bt@xkRX0B+Kl|;~RK0HDL-{w})Nxp(bA~#6 +zhbh{peWR4RU_um^@7))mxc>dYRhQ2v(?UA=JS1~PxIJD1-Swk1m7z%79r@V{{GlWR +zt;V77t^0oT-RlrgH!X#* +z-T^MB=B61iIg*Db_=rk~mFms4%*aES4$JDBR3y0QrRMTMN>R}7(w6P~^z`Vgz!o>V +zU-)Lt0s6B~g=&}F2pef72Ka?O}Mp(}t4RlmeC*$$WAz*5|~*(`4+#`RV+N49r6z#;62I +zVdBB)lX!Bd3PeE63a)u7rOy4MeWDcTNSo5NMWICn)5w6RV;C&_qI(}l3T;t*1nIB> +zNM{SLl&y3p+r?@0D%7w4VOg}`N79TRCwFDz#UybG7ItD?yGd`?=-(nINP5K_EYD8i +zLeIe9g@bo40VD=lRhUMgEP|AEXUf{|ra??$SAvnK@&{S)q7uUMrxcysrwO*Jb^N*k +z|GXuKjvn7Ef{L0XN#Q1VJBsP_<1@jraQA0VI%?l{UxfOB{lXMIAbb}9AVx&WmW)~{ +z-!)4RATF;m@!hG8Ab<$Mb=iTev1yR%A02N4m!KRT9L>4*QO8UbZd{ +zUc@mh^PS6x@&~_>;g{dVW};<8yV*OK1=gU(~*wQ|{UO2&ffwuaNi3ZOB)z;YzfPIT&!6xP@wsRt)Im +zs$e_BfER`^pmz6RSfnUvZ>R3{2z?911iFIU2|BeFvAm?anV49Zl;_7UFm(Wb8DRFO +zBo2D3XOTq>H}Y?MHf2yiAOvLt4-@6Kt_8ygZXkvN)g1`!H5wSh_Fb%TSs@f;YyfJo +zTqjXx|3^Yi;-H{(G=LJl9J!0geJKo2Ai>&a0h2dLha%xQL5!X5>llhmen)GBSV!nE +zElLXDy)7Ggzl%^cAd)Di>;T&;h`x|Fi*iu?`2<&Gb$P@zPWH>A{oFp^94?{53RCj1 +ziBY}}`-^f?ew|}&gI|eT-P}faIN#D>_zKM#X{i!L8tx8X3QA6VyVRBK@FuJ6S(0)3 +z9iVIB%$8(Hr}*vbt8+9nhvxvm0FMM{H`mlImxnRy4mW#Rxpy_19DTmG$8O-c1WeC} +z({94?AoUkdEuoUoJSwWZj`0_YUyM`F+n*4Xz4drw<2DEozN-90-G7Se +z%K!GVuOh{-g@dc>#XJsbDiJ7&99=>NF~%S19t$m;P=qf&c$3E&hO1Cu3oxgtaym0l +zMnb$^EFi&lk`kVC@iYQgj$k#wI#CNjUn@!Hg`>8)tRwxCc#`kULR0N$)p@#B;o4%1 +zPQC-lfd$mr7pmip?OcBnjL@tBMPSE;DTTbX6}!IzBO3?l#aAaXy~5qGxYkez3SpG4 +z@M_V>=xaPCiftrDq7Q6@pd^eA^718)QZJPB0P$?!l_T|!I*%L%Iz@O~hY)O$tkc%`0{<-ZfQzw4AaDl+!c@?QrcrJQqoT@D(bE?mL3>=jevHKdzVziW +zxQZ||Th~~IUb}$E_1ttmQnU)0EGU$Ae-S*v`DUbP)hCpR??iD{mA91M%rL)`kquc; +z>wdRQ2+^abWI&Io-xU+Z!sH%l@Wy}T)q0E_Z=>x$nNQ6?-)*FnjQKt>D3%=M5bgH6 +zGF9gN3{*|Jn#I@|NMIP29|E0Pqd3qgnr(CzfM&I2J&yZrVy%AbmOAPB}YBcM?Bja+qiX2-NN9* +zg#D7V>St<{_Tq~Qq+b;)LPgyrAdG@SQ&VX`P;IGsaR$+MlLEygou(jOL@?sLheo4v +z1x0k*?e%TcR=8iEKPKGTOF@nz<{HfP7tZvp)d2wGq*$tpi(61+fw&^G&KM7O=2+hv +zTp`1uKW?Zt(&~DS&^=mURBar5aW^Hla82zsP*Lz7xgowmPiL=}#RUMWuYR3t*d#F$ +zq8Ok+dy1ZdAz6&-Rlt!YtD@}L-V6-zGHtS>aCidK>_hu$#aSbH0kf~Jo1qT+>X +z$s%=*MFc%YE%bG%-4L6$|dg0+M2dIpG$nJjfYR>Y4efvB(o(!Vs2lm8Qx-h(ZL +z5;uxADuxW*;bJ@ihySPD&c{0Y%QA^g9tsnSSLp*{&L7L`MsZ^l(K93PcxFStd1={Z +z{L3zEWQEFpj8q}<{3HH^i3Vcd)THz&+d2N2GE?{82X%jg=nmY8`@T7igS1_A0Gdwxh;{czzm +z-Ne)Yb_9h7nb-tnTlR?wDXNZ8G3@|beT_>ac6+LQH6Jhn6{`Wcr +znT^Els+(;DZ?*cE+um~paB72}n8+WJoXuwj~I+4yY_@1>rT-(q}gg5-~Yq;`!aX=rFM}&&|pNJlXmP +z{yR<=jtdiB4G52a&oyIBNZ^l0jq=JXwmEsra8gP^FP#4Efs@$hSW3bZeJ*V&lWBr^ +z_8*I3XA;?82J3@?4l;7m6hm_bt7`8~0o9R)X1KFWdsfRqEWF@`4I>-S@a=v`1@T1R +z*fRz1pM;fkJ=@9N*^q#<%Fae^(b{Sy&6gSj{-*pt+D=cZM`VT@`+5>Lz&54hhjQk#+)l{ix$on=j?B`(l-fF8Bn9e}`0 +z?Hu?N*<)sU(TDP-npA=nV*W9u-2_$PMTf>U>tw{WOCqk+r%&!h%fE5IMHY7~jLk8* +zpmH&zA1d3=$j@Wem|3W=g7-Jn^Ts-|LoS>P&G+vOLQpRqnyY->|Co7O6n1&_!q6 +zy!Wz4J520iEkw?~61QJk@jl$u#YTxb;;8`@ztEVht=Cf*>t0^2;79EUJEflj +z3U|VTQ8*wanb<7C%)lrtl6Qu0r6otrq&Mffos9x|LN;8e8dvirFwsLlgTKwDlvHeNpzuYZ +zV=wJ4FnAx%@i(VrT{eh><^|P`NNZxXOKvj<$VA8XyPoa2jpxMD-PVP-4fdN_pfZ5b +z{3;Z$j*(X+6yKe+zk?PcKd^TNa@V08^;t@j#Z+bzJ(mgvnD0`6Ua_uM75zOu!WaDy +z01^c8F7`VQJVC2WCvLbD3q|=v484QO8ZYu(5+!HR#`T;KXNB<=R!5Xeyiob_e2N#KUj(mc5| +zZUwKr30MB&|Gx<=l%Pf+*Tgy9fIF0(dbc;E^lxIwvvfq9^vISVSFiHN?CVYS7XsFJ +zME~EArlG{B5t795hVdzWL1+pz)b;Ky_i0x=>g``T$eJT9WjBwlTdHp6JQvn(2d!*; +zL&UXfrhc0_DOXn4%>*u8;Wc|YVTm+tbTW?W!iF)y!0IUd2ThZeLF;7XjQ<^W>v#6( +z=%9@kh1(DjBw4=tr6_PG#hToV4Q6oc-|&^d3@L@u4?OJ>Gw0t`=CrLyCEAOM1BH%3 +zYWGB^eV5;8A1m&%+n_%AX}rMVWS$;}rMV~BBx?lnlvPBBquJsDi^4MhGebggyLcB7 +zfI+7}@(XqIk0M((r^hr~9|S@`tej#xH9!H;WzLt@R^QCad4k{qBZ+ReCjxj7hj;d9RY9>}Bfgk)Q +zZT>EiH5MBe$e(Vbf^xeQ9jewOoM$fu0}`=j{zup?b7`*yz +zboTE0U6kOQ1%Q9Z)s|-9p&vhpkhT^*Dt($Whm4f32t)XLom+y9RAM}N(b{n0tUC`q +z4Ol`Wr9h$_D+T-`GB#ymZut*6OO~2nToCR3xHaiD|~Q +z9T)3q2WXzh_nRRPpDof_Fv5Tm&mYTq{8UwpUI`T +zb^3nXc-d_ocyZ&vS^23<%XXdFkM6e5_J|GV34YEVq_=kcw(8#9oAz@YG%5xJq6fj= +z8UA=c!(xdlEcb^dBoIw#^N?ifPY>&DDHv5$d?qWAZg2nIt^OD;3k24DY_IprIvwZ4 +z=iej*ODd+U=*U1$_C=S%uApG4Km|guR4KCgH1)Q8a{HM0&yIIsx5Vaczi$HPCpPXI +z3P2C}S~!RcH6lfK+N+|fBaw5AP@^nD-KJ}$+#?vSf;&OGUp}bRml6?V%zsVS3n8SauPdzQU(gq8qi2bP9Y)| +zA%mjDoo^z6+`uh372w;~hz9AqFC?_oPIH8I&jD*KKoFZ|KA2H+4mFxcDS3L_)PH-ef&7S> +zUV@5|9*>j=@i{QiL9KUaJ%Q7UjYaRXvke2@h3=ce$MV@)(Lp2H +z1GgOyKH2Z`D|#IH8nOmwIci=Po{R+Y)=Y5vv%`P+BJn5l>gKpA6@(tcV_*%H5eOxW +zT9X6yViSYb-#`&rlggo}Fx&21daZ(v)b|*KS6P=F^fHS*dnJRD3iKS@oF&hkMdBgp +z=GrbSx-?-FzS9fw~@-s@iy=v(`Z2C +zopubt)f*y7sYw5){i8)K??;?>hsQAwANnHKDN#~z;;?TCX^1)>*QUe+=&eXGKUX8W +zO~}wUQK1^GXvB-B!ncFJ^VvN!CVJumknNT3J@kGK=((7KDl5JsKu4=K)i#oIo^dNI +ziDcWJIk@$CJQkvo@HIFQ+T6^15t?`ml;UmHQh3+-03ad^3=FL33KM?TJXXvs-7edn +zZbf9Hw)a`{~H@%6%kO83X-4HlvtLGr&SW*NtZNm_~edvL&r!XU?RN;$NR8 +zp5rN<54amz@43aa05*xB^-_W*<+FMcQ~4t^&(3~x1zv4aO0FG{zVz)2#Kf~h0u_$6 +zOa*FxDx*Q4Opq4!#z#zm5Elz5eY4iK@SlNj1Lf_2R5?;}<1F +z6&g<)OtvKH4$Vb`9MMt0FI2I$EIW#nZG2);Z;ZPgLuG|ayH}mjRe-3RZxp^H$q|A_GyMtlCSAC@e{a7k)17R +zywS2f^*#TDPC3GBnJ`{o+l+zBjG4LYjoMS-Hz3)JWU-{&d_NF9K0_n|+f>F#e)Ue} +zE1O~+HYpeW&hKxW^y(+^W>o*7VW}t&L4yLwvKFV +zu*;oQMMhhSmEJ1J5~KtB7=;5oLPb}MUC^QwW!Y34FE53eCDJemu(<1|576ITdGw|f +zu&=?Ri9_3wVR_+8_|tuBV|xs!M5?b55+#f%|3PIZ<8PbiTiOwPiNpFxGs8Fl3Q=dD +z83V(Ve#f!~)7TF16ruJibi~R}60TGS1C#t^HvMvkDRN_iM5B(vy8bG+5faw+jh6uH +z^{dkH^mOmxy;1x6EcL(eaNTn3;JNZn!znxZj3Md6Wum~<`g?j&^k5|jgIrWNU&L{e +ztJ0;_Rh3Yua4Vljln;dGUbtZY|k$LT?`dRLc*1)sBOG?4LUW$eC +z!zDo60?@tpS?|>_SbX*MXRLH4SA{GNCh_&nX4p5$^xujz*k%b*ii|W4aQW5y1dTDy +z;)OO5=jC2fXyNNE*TfYKFYz1?P_fC?B13?C;I+t)iync}ymqO^1GGVW?}zQIqvR06 +zp_govWC7b0%0th3Yc4yln0YFp$Z33FRg{;}W$8Z^m*fafX1G&!m!qeA!z_AE+0%XF +zA{O3!{r!;wn-mliNFSAeabkhz6ZE(cz$v59-TLR5*JBi7!M9P4FD8!0-SFRT`D^|% +z4D^8=m9pssC2%uLM>UY4>=lUs240sIA3+B3O(RV+`Tq8Dv453NJyZ3cKs%?X>u}94( +z6Kk!jCJ@XP10OS+)c`BHCr4WC_dJ9NP%GGr_|B&zLt`b4XD(g=>c71{-;XhkKt!ZG +z**?5QHVtY3W9fYqeypYGL?FhnGgj1$sxKI5!A;`x6tS5qanD0v3#LP@`ca!j@~}jm +z?s8`e)wu5Vrby{R$=YZGlXBcCN-@TeNv +zKsEzlmbiKbGFFj7F~?UzA4}`QB^EPVTP=OPyS6XvvJ9_X5QTghvo0s)K1G%7k_ZnOnxZ33fiDX*)lFREGaEM_k=iz=Cy1FeJC(;h +zO74l$qv~2bTFjzEdv;;m9*Cv+s`g3+=N4} +zt!E~`lol&4;!_Y3x>@HwIdu|QFur%&vHU8thP~?!=QMAy(D6{{QJHgQZ +zjGwDHE9?rnf-6zIlr(g7yovgcpPJrL<$nE`Zpro9eq;=*DMbsVHX2h+=EVrBr7Ae2 +z?zPa%+2;7?L;(|5vH_r{^{1Sr$yK!86^->O=*j3rq8@mJ{q0Uio5!o+GT-c?+zpW- +z94h(&$&xqtV(`VArSnujQ4Md1tV7#xS|^Vk@FI!G@3JxU^>yfsSv_*Flfwf>x96+2 +zeBc9mZGorQr>TK|xQp498AaQN8xe_oAIqu+Ax0)!BbD#`8QS+5X0aFO9s-^!Nz9B~ +zgX%&k_U}Mz3z4_6%kuP@QE*ox(k*+mP+s%a&;lIQ*ZtJ=`ENpqNBB?fV~_gqx<+$V +z)iP>|KFshMAM<)}XOixze%SGhZw%d=_1SyCHEHl9w}Tr_hxeQ|hBt>$!Pf;|3M~a7 +zB2I8B`)!%6h~<7#uzZ%w|H2kegB_aS890=;U6PUApdhimw^wo(!5Glg4Og^yvv#y& +zUYpJyLxngZ?i6LCV#>Bz482LBTXx{()9`1}ZX6vK!G9kdshQvp86rBeyJFqA?s=XC +z1#}a>oKeTfC}tca&@i2IlWo!Do_%R!v>DhFm}NUoet56|vMUZ= +zOktr>Z@DW3T2t&@tx%;|2Eu1GObph7UKGc{@!#$sjz^zDZm6!OPdH3M->#BnD17Y{ +zeIf35w6#)$z60|}Zs{3bsXO0rbI{D_a-wmlyx*qTp8|S+4lzGnz;+rs +zCPSc*^y%TfW-nT+f&LuX@NKA}r3v63vWGoxQ7b2>p$>`i&F6F6lrKze$7Kk>-@mjE +z!}QNF{`u4yI$_?aHhYAy+PSf9^Ss2US~z>gQbrcnlA3F&v3KolY}Ww!xN6cOvdASt +z4&|U*IJob*VPm$}Tj>@wY4pk4H0l;ecb09Z+4sQK6M=DAkj>oBH>man$j^zg{6+g_ +zTTWbH9q%YwQE_|(IK;2ZDk^H~_USA9y>P1qrSG0u=I(`GQ!IXrd@*e;;4N)@Z)5T` +zKzu3SEWIB0)76~5nC~HGm2p?1RHCP;Epf|SP+5w?3K^PvH1x7e%SCV_;)tA3I+pF_ +zfP%o)`}d=REP60B6|}0hIbrJEfi1Nb6lpq$uEOB$rWwrnDAB+CML~-5>b!lyJ9ORM +z!L9yVj)X~H=29Kh%VwSDWKG#Ed$!VtD{K~V+k@Xbf#luKyZ1(|(U$lw!-Kq7nco%+ +z`ho~vVXipshKm)_r6dc@z6;tqCOaG-59vxA%^Nr%!QWW4o8Q|n6K@PYH^?D7z@gYB +zAm5F-RYmIc&{DqHgTk=fKdY`%ZhW61?>OX?0yKooyKdim;X1cyOz{STn~4p+$>q)$ +z5fGuGgz<7z6=qje>09V%RyEk`xVuaLP!5q?`uSr?@r$pozMiQvkI=$U=KF!I_LZpQ +zx1BhVxzuyfvce*vkL_h=@PWKWuO&^BFMc8ou^DbwGe<>sV<)~rj^A$b8f`WROolBl +zHX|6_)Nf;Wj%6^{|BSLolVy?i%b3`_9e0M%vPMuMOqHN_3|V_Uj@_Aojt1j(VV?&^ +zx}6@2)<&tBEcd6pE^~X4D>KnhP)1sSYem@WUs{CiqiV%jL&oGmj!p> +zVUwBO;^kQgB6qS@iR>rMJ8IA5BCuJ8iu8f_X4OA%o7I`<&2EkX$h>%u#lVq{c*5C9 +zaVVumv1;p?&-cu*UB!g~kdN}`>M4@5U*0j@kH0TM`7H~e+c$DsfZkXD=Y3tTiiozh>D7x +zFBFBfP}#HSo)d@pq`1d!;W9VpB_m_PZ#q$wzncil$sf7t`w?7tyY&fV5#i2LTcdyhx3~ngLQ0{amXwG1+dZY5*~#^t9~7A|3_^5f=eyUJfVTTpeDnKb +zfrEvxh}&riN_TGax-`v#uPR%7D@C+Hp{t&7jlB6dUHV2OEy6^6gy +zf%r)(=Dn7oKCww=kRw2}FV@s!Ljy(%vk&f_(y|47=E{-fHB+rz&5ro9!v|{RlMt7v +zlg{7%Nu6d5krU=^il^i?ub8CcGpgqMZ7KHhuR7~qojWXo%iiIi(axfchS>!FSv>|z +zFil#tk8iDf__f!SUWYQ^Kmd;9HXnBNUc=0@5e_})DSJhlE+U#Y7@@p9b1&kvGr)8c +zxMF=$4o_7-^T_7NZaq7=jP%L8wCub{t?^fvW1Y(ImVS)*@a=g;`-kZ2l>k}M=x1k6 +zjj^iNQWtchTpKx&ky{@>g=RGFq&kcr>}m~{R##i>9_^KMT7(`PmFMS6z0Uoml1H2O +zbJjtENG9yj0%C~cSh+#Sg6On#eh1}O@b@uxTJP8^gt3~Q}qq^WSQjz74Ovp-%Xooy=qf2;R4@^UETk@8Dlxh+tyjsD +zAg{}-z@U1Gw=1n-D@^ulm#FGO(_?qund|HPU!!&(v<;6K#X@g5%{P0#^^|Ne6Z}X^ +z)l~U-?Q?x24}KcVAg8J!FHS4vmw)qzn~aHixO8>NGp#0kB5=8wZ}{6l+uO;!(`%gh +z^8@SoHgCG-Esclv^w9QuSchA@`b}TUq)ErO(NkZ`&mUbcyx3oze7QVOw4pIq$4OON}N4BrC+Cjlm9Xj +zVpxlmLrOBZ>(w(Lu#9y4><~2~+X$+YpJS|$*|H)U@F3=k&s`c(Z<1hZy1!ClHmxi? +zzMCm^XDU2IZ5mkWZr2QXn)Q~rX=ULs}E{g +z5|wo1-2M#U!1UQN74LYtkOfgpWY#x#jyo$mg~4j0?cw +z2Ho`GQL=W3cdRI^N!u5Sp{I3tb08Liit2 +zy$F%$;P#s8PTBs*{Cx24ddN2xyUoTfE(mGL{m>Cyf#RhMFy#`sLQb9W)aJR3q{Q~7 +z;aF*z!=|2FD7M|HhCLaHu9B=2e3SGGzh>jBiR4l(P$c#mj&~basjyqC>nb-`VOW!YUo~6C4Lbv{&ZxJ +zZrT_;X8(#@r|}2bv%7FFB~Huvk-#sq@A3HE##97<_^u*E1*Ej4el}TM0~;t>@3ecE +z+w(}^%|o}y$7{k@%ZVM_rs`jsI!RLJXI8He8+Py1#Wy+}(Lc5jPCE#Ich?ARF#RQP +zThR9NI$+hy5N5{SMn@6qW$M-+w_MciD|*UDTRsWmi{gY~A7~e|$98j+mFMUsLocZO +z4>?>A6Qz6p1=BIEmTGK4S63KfUMCTE;57@)yssl&M6)e=y*R`ftTqZ8w +z^TUhShs|G>I}3E0sz3#)OCo86yO7vJ8JCET-4b{9af&Cao&@2O>4pTS@9F10w};Lq +zU#&%c36G_hDmw5jmxf5cFRsp4Y`o27^4U97cH0l6`-5jxPa38Yg#EdCraI$1o*}?8ehpF8^2UYsc{6d#gqN +z!`+akmz6t<@SwW@+;`2s-}QXyuJVi8e|exSM_k8zz&6mTw;KzJe#kuSp@BOX1N +zHtTpyzQ?i!U`--Co_WdUm#%~}}v+*c9R%Yn=rIp|@$aI#-hUrbZJnKIzGQr*z5 +z?_H(qR&;c{4ZOvyX!{Vp&i}g~J~+UZE-jiTCcYLwrA$itlH|xE@t)#AGMGGu6tvG* +zBrfodE;xgou1KjC&!jb6ciMZ9rn`3zoYj>&<2tX&HS1EuH{(|HZLeu60^ihed0@QJ +zaY@*u{eo+`=}pGxB_*zzqtr5WgVVys_Lh7bB5I*hbvlg{wbY)Cdp7AG8<}Al+1z-W +zZ|dGi7o&H&MKH71nMdhpQ7rk1rrY-5P=d%`3D2kHk$CdNbBH3z7;WI$otD0~t}{uA +zZRR#=SWo>`qU&PZ_%%Cq&49LmB~8iUm7OI?KPvfR>Uavqx}QPSic)SrVX&*)?m=xQOxGWxS~x+l$fpoBbYTcvp*ngg^E@a| +zJVxWrKbL0&kslJv^2dF)S=s5*e=+-H#a=MObfkZTkTTc|T(w;LRNrPgF9N2s1=w_mNn!zm-h2`#-H!j$d7>2pTPaSGUk +zjc&`KvOEnC=(;+YR(-^}n5rfZYnTbq%US=a@t~TD2LH4unZ?x!nn|94G;V7OBm2Ed +z+S{F}G~7it`xn`k2RHc_bP`O*6$s9XiWus3p-)cxsoCgp;7XDGCk$w-^At%he@);p +zC#xemOi@*k5_(09lYS1~a}lm#+goJ4J>a#Hn`Szp6yB&2VdQLpCTe4R^uHY!(bk@6l# +z`p2=*tep*djX$`vNMoUsXtKXsFm1gjPh<4=gSh8~veX6Ug5l-%{DA|^(1^n#SiDn; +z{eCN;DO}k7G1&`;^BWhjCW@6o1}j +z$cu5>hO)&^F$PVN-v@wf8ih^*n#1AI0w@ClYKE>EV%BE%T35Yp@-0m5dPLv|zeHan +z)ZQ3cC7BAOWD3+2O^Nn9$vR?bhCi$B!~Y|e5^Lf4)t#I#iD8Ao4XtKY+0~Wyr!;@8 +z>gcB=X!m*Ki?q}%^@9V~r~LdfWo2;Wb2smU0X+m5GG4I{tsb%+LrShh)b(Hii?tlU +zWaPKe&QhkqNo`%P51S%DzQI3wZrx>jUBw-dTRRI(V{4~pc^=7~8+>QW@)FM_f-^3T +z_d5F=uGB&^1aUuYCw^IAuldZ-iv~~`TZhQ_uWn-qBOcXLEFPjKrI|eI3E1r6#Mgw} +zZ*;E{*Vp-2(_O8U@I2WB_l_cn^5uqij2ynaz-;G7S>>cDNM$_iCWbe4iz(gERgnZ8b +zF!A2vznqy&pEG9Hk7h9o{f5UOJ-|uV@RFU5a$r0q0{dh4{8kHVXN#sg6`bT{I-L#r +zPhK@gZP*Iuk%o-N?RD&<7_yCtd%xER`Q&_v?CDtQ9lbAGuf_sci}}N{1K{h{9yh%_ +zhcxcf@Agu*`&;RT*EGr$U53EY-qB%bx{7&jM|sNR)k_TSsxffCSw5uzReO#DX)jH& +zbYh3^o{O-0kB-g&HRkU`p|;Gks0s>u&8+py?tR;+y~d+Mv7UK@U6H@=^K_+KczdDB +z?0t1Vq>-{PW$`&5izy}!GP@XDig;Rt};Gn>*mCze6xmQ;PRz{(l +zXVK!*@p2S0!LLwf-G8+mtUqLBMHol8U>kPRV;2z(xh+mj%P;ud%Mcda%bGEM&tCVZ +zUc+jb-%1;O%Nudfj$t`{{8oDUccT9`o_8jeTo$RyprA}9pv6w86fzm#D-ir!HJHyA$);RKNR!eo%35j`DSq9x7yB42YEeeuY&+~It9@%tw58Q{|*7IxUaDb +zlaQ$Mi5PPG{;;Irsx#M-%JR|wyBnB~yVSXvPS49;Rz_y%&+*)ukw}?6X>HXxY`%>= +zD=6Vy+d#H8>Zz6D$4@tY*t9PudETB1oOW3ai-EWFhwnGZX0`1@MGN#8Zs%|RB(kCH +zbE{W7z)4Rj0Ii?vU$vvU1I^GTNt^18Uoz^;%+a6sO48cg`X0oXpE<#p=;@L3N&J<& +zvk%W*4^NF1BQfyKovH0G1PLsA2i)nKnJFzTN&futLyf1(brT-=OC`&Q=+0V+!&80-fOR3Q +zg%bM<)~58X(l1pVu(22qoG_K?xx&FhajqJebmrTx+V_H4XN$TfU3TPlJ0}sgv>qJQXnD+|IQ?#Zx{a#4$k_jpTYCHX$ut6{P=WfQ##_* +z@KF15Kl%ikrgOi#iV<2N;hd7B_~AC?afq;9sU4HQ +zC6)@7ZOH1oo!6is7{)|24Bty}`f+hIl$ozw@Qp2wOK_yC$5OK*5ZSyd9%&O6c*2k9 +zJVmfk)-sx2t&=({);{+*YsYKTky`&O4xxWzL>Ft`<(tH$sYYzA|dz_ZKwEJy6=WG&O+_3o=vXBiwkbQ&ppGu +zO`HOr&miO|5t(h~c-RXOR{B$Vv@CY2SMSZ6+5Dpy1R9-}#XmqspPBme-t0<1T^$!C +zCcvTj{9v2LD|EKf`S_OFH_;1ALJTsUzn1w;M~8wR9d%@lL6Ej?E=@qFblzy-3JqNy +z(~$_^@py~w6IX}{I5rAyo6u>}^iGKd+dm#zQ;?B;gsuog7{FmoQu@0r1m&m`!f5B!F=H6#zkCS1{%%leR2VGIV +zigfpX=K1v^eQGI0aje&yvr!$3SrBmB597}_r#6wc3=BLJR(7D3wKYhBL`?E^DiL@9(6@H7t0U!Yl$i7(*4hT~K +zf%@_2hBT|9B^KNtU0*(Ca(l!%T!)))Zm}G6_ZH7B2*wy12FR*LiV^utckDpdcUr+c +zYV9k{ZG&McuI5ecd@#?jbmRA2Jc9e0%dp&wN&GUHW7=tTy%aIe!io=HXiHJj{a;tKm8e@aE{tyR>EF35BFG4@y@1$if?p^^%*Aw%aBho9hvQPgtqNc0)T~ +zlP8n?ROq(#DZ$3obsy9H{ObaH?10>Urlg?#qu{jjXXR+e$1iIyNew<8zJ?EIUg$pP +zq}QY~nUbCREHC0|B_3B-EI-2G4<33O-wzG0LlTli8-@7BeV%=Lt#i_QBHi-9S;W=8_z%BE~-_un`YF+THXw#q8 +z3NB1zENEdq@Y&Q8)4DQ&OIfI6S(#2(%Vm0p^`JMm07IipAliSG*=w140lxZtV38F2j3!^O +zb`-NTUyi{^uW7ef0Y5$hH0d^m+z)2mYQHo7r`pi>;k2Nw0W?bd+AENBpP01P0?~HE +z`OWO6(Y|?a@+!MD!Z|=2-s^+*37cP-@xah>VjN726~oEuX$rnaD9VMMuM5qQ!$lu; +zi#IL<+P4uZ1PmFy9q)JGGLM=KU!LFND0N(WwRpQ97|s=*gxg*P12f+nw?d$xr$!=8 +z-P)M)d2iJ$**BIfcXbIiiQV2$+HZ6r`{EdBxZZa=uI-7(Hbkw*%-&{%?sv}D{qbs0 +zVVAnq-HVCUjR#%&Al(IWzd+qM_N7(ohpeX|1V3r4n|DctEVRHoLg;Q5lE~ub6%z() +zv_LV0Fug{L7+11yb9GWfMhXV?RQ?ig{1#s;VE*28P1h!lOn&&dPxcpw1QZoDPJuBN +zPfg@eQrVa-h^FX6s%mzS8y$^09N;S!!QK3r&>3meLJO3dGn^5lzhHaIs6 +zAqb(Md!N|#T?J@&7{Dj(Oj1m4$?vQ?hATH@=gplK)42`OU7d1CzY!188SS$z>Q}66ADAI=Q*6X56LrO&~y~)Ko_9EO#5%B!@GKbnm6m +z@?iM5vx?j&(-Z%ZFGoKA95tiB%=S&Uuufaco-{K4UzPCFgnGzV>{0J@3J~fMaNQgV*GJo#D9YBY8j`tO^1- +za_VYN3ThVUkPk0-ndxm&5eKeR3eI0pw+TS53r~-=<(A$4Mp*v0D&ftkCE;rHdvoN6 +zcyp@K9GRLx0gr1JCA8veQVk822xRqK!|o_G_MGpA;=7z95HvGhOo*#j@P(mcWo$s; +zeT_T~&84U?DQbnSGf!uoXe)Y9w%rl8w)#b6>j`~;)%A%!v%^(1`Ay@h-yM=#mQ>0W +z7dp@)!}k;y+E6I7P-S^v70BI*1Plj8stZwV^x6U)sI8WpW1?<;C|$*5cZ`|n7{PoT +zA6fVJ@OXYkVz2Rq`m;vm{SwAIA-Tovt3VVG{c}K}LNpxj7Ib@2xHlTf7f)s&p0>J9 +zk{k`rixd0nw&df9_pz$?>*Z +z@8e#K#)krl;H_-d*E?2N4Mrjx(ksDLz*cV}qlHFiKPnyhu6uy`e~rdrsy(d5ac;4# +zcD(;|hf9_k3V|P0y6R$eu=@pJHOK|hF_1OI_Ktnf^`K7T=$jCFKad_Vt-^AMzFh+N +z`=h;DCYRj=*d1K3qYN|+TB0wbdO6}i^ge=Wlw(g4xE`rA{PEqAz7v>*$$EbzJXH6$ +z8yI8$`G-5*0g0h6V1=0zEo+Ng+|Sd>6}ErZX>OV$<@d4KRVq0@tDnW$w68T7$F$*I +zSw#owet7Ady5{J8JdNAdMRP?-)ct++>Nk3W6&mE8Icx(=FnH>nf<*^gue3gYJrsLB +z98G#ykL!w)15l)wC?Ob+>#CqWbn{E^pNMu3Iv`2K&a|W6iUg#Hs4S8 +z*rc*uhtfURWbK~|Gi*p|Z`CQJ?LLz-utA6yj_sOFI&m*-4}4WW)r9S_3(1kYYtpWp +ztm9ZocRjogwaFj0u4S;;C7>i@-03z}vY_r47OIN3 +z@3JmxgR|iflm&HVwM(WMVc-S`yGW~G67G=vY&HJGZvf-79qo0~ikH$vN{RlrcEP9E +zDZP`%mL2Hl;kicIx0pSkS2S0q-16f!E)XWM`Z5Oks(sLh9U$^Oi^vuUw0usJV#FFDT&k4uy)PfYjW?m^Mg{={~vZ%VP%44YBWT?dAeXb&=;sNik*~ssTEIQ3;xF@O#j~`qs!>6!R&SFB6RB2q-~5et^3)@3N)50&lp?CjpK4 +zwI*c8>65WBz~6nhXKQ4CHG2v5AtDdNsv}l9uU?QiB|sCIdOf(S3j>CKRiTAZD5a>9 +zBno*3U-({w&BE86LK&@BvqxpHbzUP1Jrm84;NiyoAa$Bo%yK%=-L5L4VnP_bTO$&O +z@DWWu48P4W!Wk}>{O@zmdl{beccf~g9yh@-JS4KKt;Ty=J%}PSUY8^lZv$DiZMchy +z{RM2M^Y%98ZVyBqS0nnq!|Nu0U*2M +zHAjsZJI%V;x#bG9_=Rono+ICD1J7-BS2igUy3(>zPLio4t|8uP< +zoXGi~l@C`+u=mnBg-rHn-5!YbF}=&oH;(4*yIC+MX_l~8Izet31Zc@(8bGr7{>kpm +zqLyG9NYx(GCh{RZD5@kEBCPc>za$C5F#A>AoH)w$;_wdj +zM$s!&#zq~1gYvNfR&w} +z`TDsE{iiS9+#%n_AiFWfU6n%uHQ7ob{Nx*>J;(c1$yYvn)_@~*6r6N9cl2toXR0F9H}wf?|MfP +z+hfcHe5ybG557yBJ4Qm8wMgHo(4`zC;vmEx>n@?2W3PallUqF#)|blc{%ieqsbv0u +z1#w+_Q3%eec?AY5yV_X+15*r*7XcyPh_;cgvZ$Yx$TTZF~;qL~E#d<5;mT-KDsU1@Z$#-4y#y(|~jgB$ +zI2*!}dAz|lKDh4dLC##O<%C>3qxq3#Qv)QxUb~*@Gn*SE|56_>IHDUyUw2~X@v)AP +zx2`u77&bSxB3(Lflf+I=pGr-6$^+y_qByoaMPuu*h9gB`qB~e6UqT-$fl+ASqv2_9!C>L=IJF +z5KMCDO53XkJ+o~r-X;>5_N+ZwqxsFP$TJi$BomH&{zaD}rbk8k?DDbZ7Guk|X?b0o +zS6p8R)HmMBLaPQ0%pFO~dr5Z92KXQLAv?pHyx+mzb;maP^ckGUN}5Dh^R|!^@5fq$ +zjWGKY$fq6`cY)FMjUmZKYat8wr+6AdvCuzF4$wmIyENcB${2n9qS@MDDtj|YCA4wpvcY{utYsfH1P9 +z5k$wNK5ElGl0^jN9R%Y-yGIfd9kj*Y?bU4`-F!FIl3s;O6^cykvt$wy4&Fp&n!Nic +zmB<>Lo|MB-SUT7_WIH`m=d%%zYe&eJciK%PDfeJ%UNWMUw)~92GXnk3 +zy}8IudSszQ01lXB+qB7a!ebxfjmesaBeFNdtG*g$GwXOsxsZ?nyUTo9Ot+bt14eyb +zIT6gb95;(XQHGRTj343LU!UJM9MO9=5`l!Fr2~)6q5Jkt<##?VFfvL-1Q1t=$KpIR +zB_ +zquZ(O4*-dX2VQvngY(6te&zvshKyh%wSoy-kuzUO(6&Drhj1$UkOwyp3cUs|aoR_2 +z62EjHFU;Exb1i%DMZTRVnmv<%CwP9eA)$dj3?yj7U9pR0y`Rao_(C-A|XI;z9f14*gmtQ +zXSB|#`E4pN&{u8fmhb>1-A(=$ma5-6gOfO?q@=;`qmYyVA03D#ofyic(|1p?ha(#% +zlp_>hb~Nz$x9wrxTqK3P#i#wE2xVZ(!b%VQ?_buNejeo9D|Fw?lVX5mc|w<;MoUeV +z)z%@`&Ocx1@Pm?N9Vy-wain+)%u?)HsbUi*=@`q=vSN(!XWmFBe3cVpf=cMg0?(iA +z9*j>-n8f8ej)uHuo98V%P;}M8tzO+M}4o+V~Tg$+}SBNAyt%r)w&>2`AjdRT8zdg_o$YUo^)n`x8}^ +zWzt@_i*tfk!aOT5?Yn81#RCDH+AJrF`rCV(7eWw?J@4 +z@#Y;O#(FBbnlGAtn5(Mwpor!<;4uuuwu-TL-KlM^1WE_{UF~LVRT10phQ&J( +zKUd8&^kGqv#VuMr=2lOJF#I?`n+)N|!P+(2>_r51TOyy{oRVG*cf7Uwt#cS71j7U3 +z$$|%7BeVqY;vZ~j_a5T#k6+m0*d-{J0fWI9Ei5k?_tJ1E%{Dn_cW+atoBI+CDgUT> +zy6n_e$fX4G;%JbPV2JXG5$(0x!==gtH)3oX!P&5FHYCj90~uXRooCP-s`Eu&aN(`9 +z&Un2Dq#17O#Q$;liMg!&D7!@yGAPQn<)u$J;ha&m9Q_9H;EF!=lA_c%VM=i +z=VPr`wC88$Zi&!PdpBBg?o1qJtuK)BsN$VED+b*@mSblM8!D1r>DRa%3iGLSF3h?9 +z#ITkPqHTIW3Odz<6jmcTV+#2rA>$`0!&F7dP!if%3{0yODy^ni%UsVsabA0yaBjgV +z)J3y9`155?1FAFAA=0Jik(mnGuOasgWQ*U=)C1p7VH+irxk=V;U&1!>BQr1%U<1@J +z`+$UHs;qpb+1%d76qAH3`8vFb%GW=?@+D4rB@9C(lHxw#Q#I!yg0P%~Kj$7z0tW +zLe{8xRmMTY?Gv-Y9X%-51m#{A!p;l(zrW;$hL-S_P(a_sRfFU(VRx}rQx}vjs0uY? +zX;HD!gDGF!|4F?4U(W9l0%$^Pgn_>J>IjY=A~hWUdq9b<^m|TV!AEp7F|sb@5Cz6K +z^JLG+=r1{V%U=yrkLkkM>;Eq&zvKfMYlT?t4jv!UiuB>mok?s&wI0kxhkrc%)f0yp +zD<~8mm4&)^S5Zj=1v^=F_?fGCVPucMO&&r#L;b%LYHA=b6b=lYbj_oIGg)WfUr;|Q&7;!V={E%em>ZK!h!q+r&AM;M +zhCOVE+MmHus$N|x>ZI_!zNJugf920oO@)b8jYPL_GSl$pT6KA|-k#gu?2E6XuWhNISH1k0Fn$1p6CbPyAuLb(qnp=*gQK9?0ic(x_?Eu(^>tLM4edI-f5~FQ+;V+m2@$_+wgwGM_@h_y0}UFqH=goqGI*}oI+`}2kz{O#cQ@+tLNGugmTV}$=M^Hz7mVr#1cs;$lf|W$>pOw>7Hq4 +zlatp5?WELT_KCZ%)6=e4C<6Wcd&U8sg3tJer%HWZ*&t7u@G +z4j%u_@us72sx~xM;si +zZ^W7W>e{-!TQfZF?(my-kFoNPOa(*tw=o-bTDZXy(FO)iFabuG7kk0tOOp-UO6#6@ +zvq%i7I+~&`?_eTb!7)_Mr;tH(rg98JUnNzIm<~VmYR^!yQC7L#Y9geF($tKfqDuD2 +zOxBsHdK!asFBpcG^l>o=sOhUw6BWAg_BwXI>c5V#)p~xTN#PJ+3)sB0P$%vv_!Q{h +zLyRg;_TP&AZ`lY|?tp++^ovSUKSk(TPPBO;;N1(?32ai;bmb>)=BgmsO^9q!_a<&l +zcQ+2Gf7j5jtMr}O1*+6itH8>IY}7nx_;)k?L6!PaP&ae<#o +zqkH0|I5;~QnZZ%=!HPqQu7hYaYOwNs%zuTIO-og+7*{sO_zc=@=^I-@%d)RQ +zQ>ITe^Fx=F+akHeny^w(au8O}AX>$XkPMCEGQiP*TRff)TVU}uOy@XBCtNt?oGkW;X$FL +zqwBxS{5mv@vaMf)u<0K<1V#?p{NW}wAV0_XF^t-O{0x1@$k|5vZIY!d=U9q`UHY?q +zd6QfxWRsUD0()*nrTWDplW!~$5nT|Kw-GER+a!J$7k~ZVj&J&WhvFBTX=k911k}={ +zBbw$>&yT7SKZc!;=Xoj*;MI&`R0BWu+c&LvV~Ksj4VFvL!E$R5+VBm7YY?ik@WWoZ +z5llCNGuDd3WyTTbXVIp8Py2QO-y_0@#X+a<9GW*#`bzZGuRE}B=8A{zj$3(uYAQ9c +zy8@&m`Bb3GOLr&74v`L)YDPxg>DAA +zC6u6LAzdDHVePC6Oj{Ce*}Y1^X~@Glt0*?r +z#sO8Ad6xeXNv45i0-#Jq4Jvr$3)*d?M23lw5-49|>eZT<*> +zPBQzV$=&p~l@+$#$Fm55zrdj8ZvVi@!ICvi9#jhOSjy94PtV?kzPG&!Bl+S8Id63@ +zWG64ch6^=$pG-h3#wS48Xzu8$m1qUJ_)9-A5=r=+R-V +z{^bQEck!BwyG_y2#~bw07$6%LDAf;ix6VHqhnSI_mFC=V;ZS+Z?VAkhTfq!b+kF!j +z+=DpqmrsM{**f$}Pi@_Q +zEnn2G$1bfC2kifBaK+BXeZ=boQLMXNcwjO-{q#AQ(GDlyHYct!_&sCn+5TZ!*8 +zgOnEzZXMxR*$A>ns$wPeRo7I*xG3OhNbritky&MmFw;r4nHaHZ2R`+2O-_v75>lFX|t?cJVbyQX+I>Ab9Tyke?^zkmX*xO`4;Pu2Jr=r +z*bmJU`}?g}9V=Ub;s7lgu2>5r27bY@BBG~;W&Ju$eSJ=NhGP4k?{_^`cT?QB>{c0B +z`nOfK`UojvhS4VJSMdXaw`%cL*v#`la3tS +zP7XRNET*++rV(m5f=$`G_v~)}P5mAGzo|ESt=GXD0f(vNu_|Zzhs!Do(b820td0YF +z)~=c|7?o=ruMokNC6aA5INfa`VF6vslrnt4T4EdJ9d1-%h)R|e(ptap@^F`QLM(v$ +zgK+9EMrWNNl)YXVeNgh6AU;NpFevp)ognjR0-51bDfIgFe9Iu}$Nf@;~q0qCs)ld&=vp|ffTG** +zej2LH-rgPg5e~?*s2QnLa;Bdef?n;5Npg0E9-`V6OyZkLbQ!ELff-a~{rTA0unet5 +zTV(>+Ptr>;t?#n9ao{*4(BiPu$%q-nFbpt>b5A|6D|p>MCTbK;&>NB2*{rF%`t;a@ +z+TJl*uP3~zFfdL9c7>Ife5!v%MWxuEjL`XQtb;p(DJ58$V@F6CBm@R@eP7o>Fx&mx +zl?=b9t&zEJ34dzkq@RTC*uYiw(^CYZxo*m1yUro7VzUh#?Ww7~4eXAmOvaImJP;F^ +zXKIYysOFTE!VORtz%9+G~43r8Z<(EusQc-f#?e2l|{Tk4*u8}D|`Ll1?>cWrF9?jMM(iU5B(O0E!rm)<{GTiJ@k9Xy +zuIq(CmSs_}$qndFn6{)q^l>rH;s?g!t5mw{xyu!#0t)C~aRPT`bP+;jF0C=?@ly +zarS)?(Bgu2F{sz#LmzLRqhr+I6M&{1n5=xd?%jTLK|)o34XPLGUB9Lb7w15R-Sf7k +z2-(VCv_i-NoCc&zlVM3%DymNV*SK=XY|Dex7@T+`tDD;c+Ik*Th69AupUbU}M9T6Ge_Pn%G)3FlQ{yZsrlUz{!aHv +zBjTE4^G)lw4pe@U`hX?_q4NmbvTFW)c9Dh)(u(Lcusm1wtDIh6O +z*5A$we{7$v!_jTV8EjRH3vb12yp!0C&5f8)3Wbl5?VZnbIFZ9}et5p3hG3nPZ89QU +z#2pG#q)k-!P*P5np5!9jR9}H3hhwKWBIgJNFaOA8K9S#&X0@q5>1+=#J(>S+8ma^@ +zK2d5lh_5qjXXO%YEo3Pq#2G`oqTx +zWPX~F`W#r(b_dPVr^0<(tOW}2O(u@#CPpCT?joNX4ypGQ_uBJ;k*D1yrRU3`S<`#& +z2PJJycs*8kX>YoK#ND<$M9=GfPS(~RL~AQU)E3IKy7I%Fah8v9^lfhw#dw+89eGyn +z9dm7m@3XT-*$LMv!-J<8x{Ey#>mF)RB#OT*wQDgLPJLJRgkQ-$&qB^lvYH +z#0}bz;3H24a?sH+LTJ%bl_xJ8XsyKu8L+txq#NUMmFYwhJkL|j)LZ%0jaoS*l1m+KRzq662YA+^n)>|o9?m5)o +zl&}@`wUF8CJfC+#hhW27J^r%L5^r-Ofd7?UYc(HewYF-1Hp4``bW2gsH2l_DaQ>hD +zbvQsGaA6KiV-Bw9ol&XRA*x)CbAsoJ;#q;czmI+J988PTy8=>&-ElMa((Z5aD6zkC +zaxRj#Agjr|ByziitJ`}tukj!vUhvXC_4iY#i@?Lz$S*A00s+`?wJEV6fBJ6K)gg9^ +zZIovcv$VFMG3SWG+Fs0fOSt9=TZBPOCpdW9U1Hr8JJ;SSkE>XNy!V=NqL>f5bPT;a +zf#aBgOPi9xO<&E8^oyn?ZdX#jT-Jkt5(=#T=4}LM{8xAaPd6N%u9rrh7v$TNo)^>N +zo>zzau1`z2iWHPvO!RP%}{hfN^#cd4R^SG(IO4rYKEmgLChHau)SBR7p?^KyzA +z)-<_IzS#uL4VyUVRuoqiI5%of6t@zvD&_r2JE_g^q*1SXx(=r=L}rdS&iJ|B*Vu85 +zT3_wIQ-UK4WR?uY|E{K_kw1qEs#O8BYTNmIlhV)*wrS7(Gb|>DJv>Eb(VTG{R&QsP +zoAH+j*KYlxcjcaKC-BpK>bLm4nKlAt1faFRhC-i^ow*kG$g`MCV=;0s+6&y*$rse* +z5oq^?n$3^+2gJ8zymf4j$-*liFNVEOd!g$!$i%h8{cFr-7k^VkXE +zjU<<^sKg;9E39-Ex{Hw~>;{9v%3J3qqwrclRW0(f`iluP8sn)&$1a6!$p$ohXI#b# +z!_-LekIr27F-)|G0+>kfi|su){^Lr*+oGsZf~u+I1vy%2TpOpfj{2;w=+UV9hhaWK +z&KiVgTc6WkSn>FW>VR>FbUAuxyZ7DSC;TXvh25+lFh@M5l%P`D)eQxVO*{-7#Pti_ +zHK+!KTe1vldyP$xQCQykY$O`^ahbKaQGdE2S;MoR;=HJ7LGyeTkQiODY)X+PSeySj +zP-Cn3g#1My%D(Vh!w2E`i}uY+R$O=@zQL_$qlD)q0~fMh1Ha6js-JvR>FCm1qQa1| +zS=!e@0mkFgm|zOcVUcyWQb-sht0fm?j;qYVIu69xuR +z##?H2SAPmv&P3eWn<@H9#Xq(lI#L^jN+i^p{ +z6B9ixw$9{6sL376xf_HfY~1C%3c`_g%?^Ah`} +z*C$Ep6P%*!4Kp6%v4M2?-MvR!PzeVUw35}l`w`#=!>;@~eFb+kTWlB#=SiFy55}|o!eKB_cuCn>(Wejz(w~p_UunO9)TW(iPi_Oz!EiB +zdAa{q$8}4*%!c2$INL|%2x21x)43WDzuj0^Hl7)BxU9tBROb@25oXpYyT4w{qubkc +zkTEa#gOn0`oXcaAJ1)vZdyPx-@%zRdu7QIMCX4sEmsMt)Q0I9V_CVJ`TP$>tNtEkB +zSc=^*M?7>MQ#Y=2%t`MrBRsN`@^x+YDP4GJ>H|pYUbrlsZ|EfMCt9Hkb>UWr_ooW* +z2_IgF?DYh%dyZXYE(o$7lY{s5+!LO>DC6#wEu&;1wih{6x9l8)g2ND*h4(b^MoE+X +z@MTV+Lt>-`{&NB>y!14axqzQh%sX!p$t!UWcK#4wYD~so?7Hj+Xt>FDj8{TIkzdXv +zruo=+D_kwA>!M}v8aE{G(7}`MSV;*i@f5nyq77&?dc@x)51YB3zanS7;vYq~BL_4R +zslp5!Ty!b%`9_Zm_D=N*_U0B&qM!L-SVx@qm?CW3$U;35qvfB!b@ogJ2#OD`5VG#i +zO*I*Zq-_kLcIg+m@T~g1D_Li+hK=H0NXS$0Kj! +zPk1Iu#K_jA1G1G6?0JP}1+trIDks~!5Qk~bCaQ+QJ9`tp$m|euxxG^w`c<`g!+qhq +z#u#&k0AtgWEYaKJE`p+7j<^|FU|o-6Ol8pe;Pj@n@`L2X{ByRE(>5}Dq%@`h4V2tx +z7KjxW(AG*mMd$kbAgJtoX2V%?7ecMxnVz< +zcTGd`i=%h1dA=XEp3%vNF|3|vdP<7!$e(SQOebG})|iuI-PNBR$_V7X-EzrW+kNMCj`j7GI|46^yWO)AZOjL2*Fn(DsONXz>KMxA=t +zuv5$3Ux}IW+JbRSFH)Lzfj0W`hVzQ{N|by;n|Hgc-gHs<>wrH-XqMi6@+F(A?uPAc +zu|FC9Cv2Zr!H;!pMxlad01fFYF+I;D8JG{KRYlN?_6ij;s4GM(5R;V_!*<(MX~zoo +z`_u{J81pWOXVTwsN;O7p4Z6$d-`egJS2^#cwNe*3-y2NzczYKwGlBO>+!g+{3pACw +z8=4N;geXvMq><)YZ|WS)QUr!SvD#ftPAj}7>rKN4rJWCV=atYp*@#m*le;iq{PP5JW-n9jTvC6;E4L%g2kr +zrM;WpEkhv2P*6X9;1af&L79x4_Y%NQ2iM`yNK;wUPf<9KrU|{tQ&@iKOX7IK(G8C+ +z5RRTfRX562rPm2gQ#NY)!sOz4xHQMEc+(Veg3Sruu9Vi$du@$T7UT$oLyix5)zHz> +zbo(zwu(GAa7Z0=RH~?~n<*=76#J0Qn!C#g~;FKV>Lsw6CSQoQYd)s7e2EyW!v2$hj +z_p^sCUS6!h-_mTafLo^3#CiI|XNe*ygtyp%npiNX4BYB~XtyAYc@;h?9rq@u`s$Vv +zDrJF^XYwcIGN^;SLX7NSL*)!EzPO|wv$FloH2fLTk~jN;M2kxM1$5r_O#Zj!NpX2a +zINa%(1k)jkN*}Pq9rN&oHN(GBcdaVw$}0hQ|D}1%;u`z>`4YIoa}D +z97|f-TYBypo#khd*q2^77PcgBA(+xTPZPucsuTD40$Yy8MV#_yRWJ-yy2p@7_Pr^;bcN=B`H#=^%;A3hqau$2!%6sFG<%;n81RDCW +zK#XmbGU+j;$-V7nX{yLO%zJ3Ykn&Ch$um<1?06O4N|lCO +zRl2!0yXIyzs^LquUXRw44EvOJ$e!#{Gn?_E(a?`UgUYOdzU|K7$~yR^$MICpmvyCY +z`~hPys>W$Z(_U8}%!xhm;cMG~ANZP?QeBU-(JfYc-ff0^b2BHWK0cG&YJI9%QNsG1 +ziMw>V`P~?=?LgO?lqsQ-oHxLwVMuuSNgUbg9Z<2Mv|^1!PN@h8V3G4W9~3Ty$SlMx +zqVyJNP+mC4n)>(mXenw2sAJ*2yntFmqM9>S7yRT|$i|+r!#Ar)27}p_8PKj*# +zIc&Q;+SM@oMt*)6c3fE)?I0iieKSWuQ_N^BIkcSc=_!QG*-BGjrKB`+CM(b*V2znc +z*MCjo%_LiJypr2%(v2=aFeA+J<#KLRjSUwI>*I3F`1|SR@D^2NMnO!$*XiL;@!L+Rnb0MJxby=Iw%&ognJ|Ap +zCoCzDv)UB2LjVMI^ic!}bo$UOkID{aU{46HAh~Dsz2FnpdDv~Yr0a7g8knx!pGBQ< +znWyMEcOSdo1p!C{4KRoj(#H+U7v*rx#In#;*-HhM@in@{go%tN15c8>7|t_dustS@ +z5yZsoO#yz=zK~mga@ksKm|*1!S|oD;7~~N^TVrLS!c?&6BQx?G+4Su~=FLU#Irhm_ +z<;b8k)VAXR2S@cw1U8W2&WgQqp<5MB4p4RVj+pdSNQMCuMb4q>r9&Sf=_}Pd)@0`m +zYSJJZ&WNJAGNmxrQn)Wdpjx0%Dmqf&!rGf0BTw2ao24v +zZ{IR0s}g%_2+?DU@3yI3YbgQGWzW4pUq8#6=2IFIFPC$_&0l}na*hfavebIpQb_j4 +z@aAQ`M~~Yto{|@j@0D`Jghl+dTFvgqDZlZEZz-R&g`D;P3$wyLy +zi@8a*=Y#WDEdkKiUnG<0gRH6jPbSkIa1MB^qLq+<(l@J&yC~*%*G0 +zF4-2ujqux*e1mq|edUsJ%tR)n`#V=*_4RXGHToEYAGCu*JOCYm4xbT3%kxClGVy{3 +zv+wO+aqSbdS?m+lb^x+!UZone1&20Fs~ZK6>&6(Nz43>qbK0jYF}5nET2te3-y?}> +zs>K?V^gRsg8D03T=N+%yPdnYQuzKF@r?y^H4$&R=lBPLMS6{&%nXL`*j={Nl5@Ahj=Ar$T{pEg2IF)|Ah3FvXFJKY4R4YhUFnBZ0 +z-qox5;;}cef*^hQ!}Q;IaZ*`zCPs&pHY5MqDyTDP7EOBO$aA?1VS>>;~uo)*54cL`BE6GCjMjgS5m%T<7 +zr|N2aL8IF==qI*iG)mVc)Wc64vTE%W_@=SwEG7F-sTs>6ZAJ2o;Kf%zBN}ap1CaIo +z2JlPzK?7cNB1;m|j|iNfFenL-4h;Y63}MQJww)+U&#(n@3OoqVOU0QyHtATAD=970 +zh)wN+fHYpp2!U&mP-8h&hOFuqZcwl3eRIDLBd?}HbfmJ(GQn*3c&{fGvBM?GI>7bn +zQB7%Kg|biH{pSzu2%(K@IMsRnLOgPD{%@ZUjB;==(cUqFLP!Q44)yYXIm4w@-~}IQ +z(|!)m`)NHUct;vq$Jmvmg$C&XUqtd~ro0m(g>T#aK>3cyLO0{Wjk*yd^(gT18m?u3 +zZNO^+ch)p+7ToCNy2-mI^}T2EUJ$uPD0E(p$PS>9o#b8!=hlX#$V~}6|bh0kz?*lGxML`*gj&f +zPLSue!lnrimhpF-2NeS)6iQsUy&P%MS9iE=fY0*Ht5niss1$)cIP!{-mx +zsDcCkSoQML;GWrw=|_CZ{c)irC-~T@IPVy~$7_m!>=v^G&l?qy_QUq9W@qM+Xi4Il +zIRe*3>xFgJtv1NTvFV5Qe}xTAxBs$2VO9z<_Q^pAfrwl%K|{{420ubTZmjMfjXgwI +zTLV~|9ZZrceo>*e@(N*&BXV(-2c@|p(cNty9FvMAcgtbIh0wl8wmh@)*Fquv`6I|G +zg1JLNl%CrC@PJiz)6K#dZQ$TeGRi=hnx-S>)+WJ-lFFyAoL5(d;dnjz!;6@|9ma5% +zmCpe=O@l_7_ah%_=_=DLcZNV@qx|o-O(oDV)ZT#0aa>okKnc +zEZHAX-Bck#OgO*J;?;?I^D)f@XJQ_Aafu95@AFdw3)=eCQz?-H8H3GtuHGa|``1-%V4-zs8^`V4)Askx7gKYV}Wm +z9@wf+UL=}3XL&iloXkg)CRm*g$S9v-P>>0>?j(>`0T7TS1u*D~07Yk}v|u$w_05WB +zhOLl2y>dioDT{l&uKqJg&V-eGVs>X}1E;`^EL3RM(61LM@aE8E2bDJAH5K2AH5rpm +zbfypOaRzRm0Yz#pasNEP&@V~{)Pec8L6jrRGoucH?pZ@Z95Cs0&&n9Pe=UX^+#&5X +z$9Xn=on?AtDJA$O#M*kLo@R_PbXPOJ*)t$1>p}|6v2eVo{|;f_jP%x1Mw9n}8=U## +z-MV&K{lLfazR)PlurT*_BH|5ss^FaEeimTaJi;nrnFu!+@^?Av$GaT83GNlyIFoGK +z*eh%D9`wXKN$ao`}X%67RolXobU@{{DqSVldxY;;TL7r@*M9s +zhHW5@>FuhB3qw>E6g@4Zc_^0KA6uco={8e7yRE8o+p)b6Y$hx> +zol}VLkFD~qTY7NhxR&xwCiJRtH1HvkDlUbDQ8(l<_CSQLZvO8wM3xRO6-``Zs4xLF +z?T5diQs=g}tCB8R2HV&eTR~5Ocmzo2Ang&aG>E3&hKQM3w}d|4X~|uy(odcEKW980 +zEz)iX3sCb>e{1Zks%jQqwHKG!4NY8d<)>J8dxQE +zCkAL%n1TXPY479L^oq^v{$o?)pbekP76U>cd(jBp^R>D4W^UAAxBFQT`MkzaIR@>a +z^>&UjPZl1$wXF^V5|w_vlL7bkJN> +zo-9M;*c-1%2?L&%J`PGjpX;O0S*8o61L5c^r{dTZG{|q_`8pkcq|XG?_iCNhu{%ZG +z_b`wKp!C+o+WzuP7xIjqSOgwSnJTo(b(`NrF9hrC*93dQ$Vchh$17GBI=MU^KExhU +zYv4R{rApTFu_#4zxniIRE6DGW?)}6N4qVe5I#Cco7-CKrfq6yw_SrqH7+jN*Ot&B3 +zDFLA$TdnP~%y&0Cg+u}|RZ8qUGat~S3?GmSW#5Q;HhH_NCf{M3o{YBb2w1-Rgo!%8 +zLbiXxA(}$XalF&4r|XaMV-U)C&lo+;862#B`bQU0X41VN|4QE(6b=W&mLQx89JI{x +zLLQ{-6&ieKjCdA|zispnr?X(Nt%3ffE>POx7hQz(gb*$7AM0&-aloBCJaCJb +z-L_&c*;0x1S!rPScIq#DG%H#cW;5&iMv3KhsFCtqJJ<3AA(8t*`7&|3hL9-o{jxKt +zGY^&A50Nz&?M?58)@oni{|<+@2AVs#r%rQ!xw%!`UNF%{cVF4bw=hsFh%+u@#P{oTJO>;G4oe^750zgZa(Vm +zcB7`xao#SYeRyX+n9a;5t2NKE<7{qxzl(ohU@VfQA6ovcW~Tw{f5U(uNc;&DysBV; +zH+O*M_${_7duABN-Y3!fWQmUlPT7xJ9oU~5-L=mGBpovMHs61pUmzN`7r@A?J_4GuZ{f5LVZo*!3 +z|4k&Wq`U=fpa8J3!AB+er8K0>?rj(3`QZBdIf|6+0mA?iHe&jtu+qfU~+PN&^{MlZa&q&P`0l& +zUoRXjp_?O&ggZT&L$P0aUD3gUwOFy?9ZD-=$tT%2_5m9Fc^HJl)Nc12%?y9CiUz}B +zWfC+1veeh>mpu%ifhi3{)K+SXJ?qErDsdaqPG*7+D%ST#+!_n3VM@qlv3eNUEe&5x +zXWAX^%KrbLj~t2`xR*uH%$Df6<^|*aZYzsTobMtvs%XkYD6bt^+6zWq1v;rJbGN(3 +z%|Hz>qmBG=dKn{H_m)sn)s}Z|DQK%wKdiC0O*c`8RP1;V5;Dq``XT;z1Flt +zKD(3LeaPDh`nEe=8#atD!O;n;?IM*~9Xy`HcwH}Dv(mQ@KdJs_Ef*Sge`_q>t^XfY +z5GRmDMN3w%+?6tOF^|FQd9JQuPo3S*0u=i}?$TO{75^cO+vL&waPkaFW%--YJxq1R +z1E=ALr8NTjK^VqF6J{XsT``krhBT0K8nC7up=5;#Tlv2Gw^B5WTXGY_a|+;0HmATa +zb4DX!FHOt$s>ZT1Mo}Ut#}YolW8^5&{M2r_*#NamT^_oG9L6~r9IRSe>xcFldp=e- +z>=Zv9?Jbg%(SCyzu?)W})B=^s*Xx{2ceF+I=PI04SN6n*DYJ=-5u1Zh*Q*(NWZkg% +zbqypLH_nh9_RrhDO=PSsWK*dfZ^K(OQFXKxwAiXnh>w3eB1t+;1@=A4&vwYT}&joP`F@Ya<<^wM4 +zAT-8A#HkvC!#sGo#ivV(X>^g@=c#Y#5dU5&j+yww!`;H|8^x!jB-IaOoUPJJMF%GF +zjeh^!3iY&|D1H_Q9s#4kdHMaNP)cCH;J^cq_c3L;MSpJnD#k%U@o-*Bo9@l6iy4?8 +zq(CrAH0#*xq5-;6RJIkmI4z@J0d8Pg2@8n|2Bc#io%Z?&gy)Y}(uU!o1_mQi+&g9d +z(hXQV=-yTP>Ry>&aOS5&CA{})G2Z<(f1aS`Xr*)4!9Yk@_UvHYt-+J_hU&!*2S$Oc +zXb@q2lJ%J9752KcW%0k8|B9zf5IZ~L20E%4yI;SA8#7~@2klkKal&q +z@KL`MsI&mDPWX=Sw(J-4mdQEyy&FH*{8?7#C0n7-_(;BkqBGlaLEKSuX-IE~ifzEmNqx(g6b# +z$hgnYb!>O3Yv`Z`2ZeIp+}xCCD%KS!sW6Nx+>-w@FQ9H;&y!IwT&g4U9x+ +zG1Syu3)ph|eD9&9^1*jxg$X$iqyhlBeVxjimRg;ssA3{-kCl58&>^*;c#OmqVg?&| +z{OIW$4cF$vjPK5FH{5BNX1$w_S%H(uM6jIT#OTJx_yZWwoNqO{2;SRT>Sw6Er>6{1 +zms?A8;165r56rTEy-29+^dt{n4$5)c2=`V2)c^cUSLwMdJt*(TY6X^WYvY=D++I;m +zoA`dK{rsgJycFq;kAY=J@!RE)sMa&a)Dz_q6%$~S7qs>>NRDR)0}>K$+PCV>eU$)5 +z$CWd2QZ!w-*1&*DMmwY!uktnaKk}emxUVFUW|uIR70Pa`dOv!FLVlPM94Gz>pg~vo +zu>Kshq?t=9aTf;67oJsK9X?iGYATqnC=~7MumNHP(<3$usF%UA(U@%rV^Lin;$)( +zXgn-(+)7Six-TagE}Sx;r@#q5PJY1JW5SDqWYfNPNQrB4J+OvUGPK;#3n=`#n)j(# +zc$X92JCCYO5|UCQVBm!5RlZ1Tv?%mQs$TSA(v?T-K~z#r^{5rQ +z)BLVvyc{5E)V?(Wb;K2X?E04JU_b+RQXf3_CWBg07vmRpdLOo4Rd={#k5q&%7(txmb`Je*v* +zo=M05Kfabd!GHYc?^z~Tk|Sfa1n~*sG}Zt)4W%F$W_TDZJrL8mQF1{QSjubjgcmGU +zUIqsEgXDksSljIM$4JyGgWy5q$0LiI6AXd*OXouOdx*Apf|0|`@>LGk`IW7N6ugQA +z7b&BWPo&+ed`{XmiZhJJY0OM44979*WtgI>lqXpR`h)eS+*NitLRxCoy3bJeOs#b> +zsnZ9Vx;p~{XOPXTVRS{<{arrw`kHz>7?ltbRaGdNkc|>V5fDKn05mj}TQwBuckC)5 +z%hDTv&m&dpF;37W4bCv +z@@kKf@uLnJxsNOm7cy5=`Xd4_5~ly2PJuEF3K*5d4yxBKkb$7x|46>8JFA@FL7GHp +z_W~QKD8RT_jN%m%WXXqc6v8qzQMv0%IsM)opAD8P1YMJ@G_W5K41a01e~2zF +zU8Azf&wT?iSl8}**FRkt8IThPXJWS4qr%nQF5+r>Ke}!j8f;2ULoG?%Jb85VnZUju +ztcUs052^ynaLEc3qObzx3Ehs-ioWGaoP*zk)&GGuG?p`NF}e3FFQUk=$CFW?mtU8M +z88RHvQ_KzYwoVDMGVK~@F073Dn$>LtmBEZ~c2`G5Z$5R<#Ji>3(dox*+)SC>WWfqV +zb?>04(Aw~j6nJQX|Kn0vnCt+c;VTy;qgwOOt^&X%7X6-sYBzLoJVWIvAzgu>r_1za +zyp~(6&?-4LowgwhJ(hBks$oMwaB;n?CfHUk}_y#0~~fDd`)S3 +zfu!n*jrMCcqml7gau&F9|1uzE%LXG>Y3^L=`u{!6&pIBk1~`#~V04*25r|M;C^;ZN +zDxBcaIuR{&lZQy1DP4%c9dku(Y6&A}K1q@~YjJnSY_^y4~QfBhfCe)m6YZ`nqAglZPeqD%~$ +zBN8Yxf{7Vom!2VHM`}<>fyxn@1^v8%ikH9TdWkPRl+C1mJVXBNh9bnPcqILU;!Efk +z>L?60_jiq%$b7gK9Lgawc4!k|Kj_{$p?1LMKkN +z3GJ!7M^8+TNcj17=)vd=?FSL+3zNKsR7py75IBCk@tJ!zrj +zfiuKv;$nu`zemyituJiUON4!t_kG=4FQiL_I@)ED;$8(ob!ZrRdg{QD!DZMWa@J@G +zg5GQcLpv_kaajMm^?#@fzGFB5`oBaC=l|}m|L!m#71|~r8XU+8;DwF)zcu;4h1tgb +zpNtZyDg<7g|4sP+W{q>911fyK(cL2?)e&Ed2iMf4Ei_OZbek$>wvABJ_T06dD?9%m +zc?aCbEURy#ml8&C9_H= +zS(2Iu7M2t2|2v^oe{fLh7l=fb6t!hyj9PK&E<|$lM-4G6ar}$O?mR8!85Zt@OgmI8 +zQ`|AKiW@b}AH~ZhIO~@Ts#!j6@wyr +zT-7uS6RCzIHCP{_rxDc>!73_1L;iH`HC~`^0SDX>32VQIh5{8W!cA4M_DR7opP@m&#}B6 +znM|h=3r4;v9{^hprO>a}+_`w|NU#Qw*Ay +z_D!uGh2>llm(FBQP6|nwoo2{W>X#m^dagOuti=jE9F#Ryu$gf>vQ0o-Pm2n4?>SuH +zxFm0f7jhHvK&XSS73cI)zSpn0$pCVNhO+zbdFA)S^RqEth +z?|2!J`drd*s)anog_8PJfh${|7|<}q@!H((TU^yCHjcSgWv|j@m1MXa=N8m&TI7Pk +z>)Ut1q^UKosM|OzIc+N5RE)<U`e>V%(+L_&H}H_#@1}?<%OUg +zo_@BH;srMDP|$GU8V3WOoOy0|v;)Gbckrz)i8#eUh^KnBzRq)Y_PboqCl)5LxHBvP +zFIxi|$R}b04LaYo;n0X{TC!}oL_w8H%Ai}M)OhuzrTQoUYP5A?Z4~#fi2izWEiUWDc{-M#$jiNQZd2(#Jy}T9{~Nn&{&-6WyBX#hs&BKh +z%?A8r=BbooFX#2q`}O+Keb>J`um3J9nmyzbW+0$CfK>@PnKr`KBO(FnfFK0gb@~(& +z4jdOhob;9-d|>P!IFMV>zdQ`^u2bDlL?@9P6~A8iwnrY8G-p0!^5 +zRdR$tED}@zYKmnSk*03wWBYsE2@OtGr`*Q2%w56Q;)HB@xurt>Km_Q5^(lHF;#enl +z0m{N75SpaP{TPZ6ENWsYuo)Ow4?|aDXB6~CrVBtu5xxptlKp&f`-y}0gbR%Oq;1P$ +zAJ(*98h^#9Y&Z|43N&bFHL8->aP +z*0P{>~}pKB%`J_9l4nAfqYRsDKnDK=al-J#>nW6~4F)w{aS%iXNzs +zq$Wl?K28^J7hGH?KK_oCk^&HKljf1;kfm%*^s8wRN>a7mkV!gym-FBBDjS +z-;t*ns}e{KB+OgTZ_j>J*%M&efcX<4;bu{W_(W)M5@9iftl5gE^@}<4kP!?d=K29) +zZc=r9h)&*sLQA~L^U=9k&j_?H*zrklNg$Qy2%VkF`w7HK$IA??D^Ntt#Hp&*A)?D` +zFQSc^jRSvVbDw9*y>`ggQK`MsCab)>3Ih~~$YGCqC=`@bqb4@M#>oFw+F0Ioe)piq +zOi?Ly>=Z?Y<_uayACVpJUxfKc&m1Z{`EE#Pm}0^z69Rtc_F0_2AA9VieI$jPF8P|r2Gav^9u$r9pb{kcWG?` +zLb=VAUd8wSt%91cja(taZ*NF{v=lTrO-)G6a$RN|S +zbc|97!4O;x4>S=lWYPv|zNIq_@?BUp^|eUNx+CVbLOl(Wh;-4!C0ZbYO-5%I1i?jH +z)FPsS2Ff)>tL*ZID2&X_<4kE#F+sOe3tD=xql;%*L5#*)e)4k{jy0x+Z0#YR15;>2 +zr#B^it?24${1<1DF{ZlVU|jby%*jdG@;^10+z6`Xq9(NF#~N&){TBf_6FT}@l6?Bo +zELs&gv?8Tg4HcIXtUiGWO$AL|GFfet{A;`wQoTT=*WVJj%j48p_x?7+q~*3vJd{X_ +z^a$I#{~bhqtLIL+;U2k{N=lsGiS-{1aZVu4{+ecP^dY)1ou-nmyhm4inFlmy^NM!yEiI +z-|Xpw-F6sW9@l0vI~QrS&5{m+f@N~Y_noG0FMGF`WA(WdYl7Nui%%s2qS^*lqZPgx +z+72v=L3k*#!tu*-Dc%WPnBzVb2^{wsM9$U+Wy@D+!yAReS`#efM>1df>xdPy>BL7~ +z8}gNc&(x=n0*hyb9!$Kq7xQkxsd}8xC_l9I?Q5g2900;Nl&h(%-CGs2t&3?Vvf;b~@EZpYML{4i +zD1`^=9=OW|`<&+w0GegEZZ+U48ai7Ogl}~0x}F%i`NAbya#QbfIN6MumgL^?`U=-$ +zWMpTLEI+{<9v)wh2pNL*?-j>$9qnZxbW1{so3{vPYU`();HKi&Rt?|HcYp__#0v(( +zKv4j!ZL&i6cAwbT2u56K0{g7gmjSiqD4>D#fVNB9%zit=24f>9^y%>R<~AhiWZp1f +zYHKeEymcp5b5<8Ttvw)^m{?+OE6if#877_R^z$+zY>IOCZrF~dau)vrOZ;R9!@n! +z-C+5drh4<-;_Z;b<65XCxJ{auD=;_0U;My;;&zcl_I7`g=8eLzzdKg2^&&J0@H2*6>;}UAnG+60mYBeh{ZoNzvCMUY@s}iIPxt%emXE>H2?;)q!r@MwA5CKLs#@_P7ZLxT5agzV= +z+GH&vdcMg347`toVA^9z8RST{$ai6iR4=B<5FsKmW+a)3KmU5sA7jITJL(=#T+L@V +zpI=ulHWu<0hEY{xH*PJ{RR}rh8Y>N!$&{gMTwT}D5Ctqt)eG(ZNPI*HXKHPOWlhbr +zYLMxZU?}4I3p8&ah6DN(llOpqpAaS47WA2!D*JsFO-dPn07c?@`OqPiqIn_r))(tf +zpXJv7gqpfQE+xbMeRHD?|FQBh98RIU$qrA|jPox{g#E4Zn+QwHXf}n<7nI>ZEJ$v* +zKH}ailMN1vK3n_)#!pYeLFfLb0sc +zuZD&-V>}nTF#)s_EpR|+Q&Kjh9LaJ2E_kWvX7cLc?NT+(ud`hK|JKq4wsq?EZe%3Oq +zCot$KCRUGVbcp_iA0Zd>oZ9cN8Lb%G%@~Q^S*kb1-zuhu2Hmxf-|IBZ6gy_`qy=Qw +z-_0DX%0)+Wxx|<4^Y26QEGNeHRzR1yx2>j%yL;{mN@2%w_N!}KLWy|#qSAt#c`~{) +zl$ouudap%}mV^92dR#W{c{7D=IQiv6cE2!o?gx&W<{&BZ<8(#nzyyP-(GBC*0_rsw +zrt4jP-Y`i6FK)COairqvBj5(X%~+o$70 +z6DKeEefTDVE41fOIuQiwyA9qGOHftui7rg{bfoI)OVt_bkULVg$oeR?q1yvqD!QmB +zmKkRkABfIDdhmgvL-;g}SJ6{cxRBH%gtkpS?m2-BIdwY5+QQaWgF|zaf?~F~>YN3t +zh_nGQ)QOH`Q7*4Hzm(KQV0HgHu(jDVDQG-(c~Eba1RJ!!?n0~}s|n~ElJD3_lAuuq +zTo*!lOr2fCbPfqBxj!SoASn?|xtW-F!dplOxrCB$)8dM#sx1LWW2W~79)_UY{m^Gw +zymM=;K?U{PQ}p%;`);1Ca&}8Pyc`?8T#MV`r?2w$1QElyVjyfApON>t#;^I*K@2p! +zQ9T7#O>PKAK;zlLVz!%!UP0BM*J0GvO+V+<9ThB>oe1jGJbbJ?u;rtP62y>inTxm^lahis=ZyNhuJc8CO{iGTu&3DKv(>bu2TgXR_@obXlT||Q=fcwlHU|9oj(^z +zPTg8?1YKf*>Jp#8VyCCO!^4jhO+{0Y)bJv3mo|;P*43F{@HTY7SnNko>g_Ci7_Y%D +z&7U}rlW1mbFQ2REr~fLzW!2O79DObPto#=C*S?^kMeR+ZF=jSJx+lEK$GI2S>g3tk +z+(`7Vi+^;X?2-7SwYWsj2+Mur@N?XC>(_fqh-Ggexn+=<{~vfYIq9ke(KI(MtNg_p=>0=v8jBp;d?n_4}x +zO$QSsS;R^w8v4n-MZITMb7y29`dO8|@p!XonH;f&JBOR%W$J9EaXdW4F1RSuKPTuR +zz2r7-ev1BduDy@`Q`aTomg6Z@CE3Q%^1I@XO@VjTYVO9=3~6%AlBV`X|J&0i7FKTP +z5bd)bNnJF8r1T!4+Sv>>DY|_%Xk=u?G%YD`&}|XJctFg +z5wpStaNK{})}NLv}ea#~E +zF>^vgm#pt-AvV+iLwp;%g5rn^R8or%JwjO3yV$6x|J=+S#OD{0iHXAxUOEog+5^HB +zTpTTNKi4X`IJjdUW9`;H1NE?uk1NVicl-ycPAE#$KQ +zAX+Hakoit`RL=uioX-{Nx|=GOX6NXta2Tn!*mpWfe$*+1rljsnTGFh!!6OT6SVG0@ +z60JkRN}E;ydRCN*TKK?{S=Z){5?jqr=75J(FHqq=D4QmB+-u6XR;osfq2?9u9! +zDQ|2NU!`oCQz&RsaAcmGbTt(!=i<9J?wvUJ-I7g{f;z|Mc~Q{*IoB5FwqO}QhEKep +z4EcfF8WjQomds82pwCoMl8ChN0RQy!+uJGTEp##}Mg4ljWB;9GNU1F(w|2qUtl#Y~ +z1Adv0uW>UgbECquGpl0FU4xedD76g+#qT4b|D>xXbr)u-UASny8_M$SEmkf+ +z2;g`+GkrD!1a-w`vT^ad2Q5qAuZm8LW6<3FkY2iTes86CBUr}gbOffFoBR6$h9@oZsaRsUa2M3dGDXzJ{2zSL{ngY2yTeKY{$=Y8SmQ9hv&nLwq=A0*WcMFhd=L7?Wy< +zWfib>%~q2&1~Q_`>TB2V2nM|E^lpeCC*ilC_w^c?ma`trK4BFO%0-O~wzj0HX_!=X +z^Kzkzs=B&$wVQcG&6NZycprH=1eI{m4`QhxA|o`}`j*Xl?6|eN!K>A(>gwrrzqN12 +z=iKudQM0rfV-56=!TJc#ndbNClarUnbK%?!OdXLQ;2|aGFR}rlq5;TFRRnx!^s%g7 +z4n-Yr7_gH2`!2#KPWF!n>wjhKo3bjkm;$(t< +z`pCplnu;Ia1Te%Pw{oh@(9%wxvR21ekf?vG-V)9O(G?>QiP*|X!NIGhM06eB +z>kve%(++^m2CPzWjbjJ#OMJ3OS}(hc9)|fSQ&(E%*8jq+wF%l2*UghQc0L)MQ%_j8H%FBWeBoB|C_rgOz^l#GVuI9if(yNAdxw56hgGTB%iU;Z$Xw?A7rzpYcC=K5|Lb-UT}7O6d44Q +z{XXmZ=tYo6w0vdr(ESt#gxb~(2MqK#L^wUO(H*ga%~Fkn5Rr>|+G(44VwcPSE=`nl +zszNe?Qyx(X0Z-PuO*S}n`(2D*e1`aS&HnL8?A6&VKZnqV2LE(4K@EY~qVMtn+a~d# +z^U7MfTnE8eG^UizV&|t;i~g}XJ9Nh{Xg{K_DT=@I%x(uQA&KZ8_)*+iQxLA*-i@dz +zfU+kE9Z8fp*S&DA +zXcdzC>uZ=G^aJ&s_6~P?$DW|Y^9#XayuzeA&BO5cQMy=P@Jj^G=_RzMdiUlNhs{;# +zIg&E8kr6tG4UMLOVa%Q&{euF<(I=)@II;HyEyIIb<*ipdh|70r}Vxv94R+2UXDuC&XN3O +zeFx7v2TE~9O!2(nf${Gjd?j2g^}Sfu1-S%{qr_oK7t^Wu+b;;Ur&I2ST*YQgr78N? +zR=IgIbSXUzJCnqZ82-TDWdD5s{2^GT(QtbEh3qdf>*(0-?isYO#w`_^p-W6ios8lT +z6JU5#BlIeyL|X%?k%$2>Zh+O&-jKo5{C|qGnwsf#9ChB*25T;FP0V`BT*ki4@p&p!56=RnW=*>yu$;O*2?K9 +zH)V8%GB+{$!CtN0I?ZsmtT-%wn)g1g@^ZZ~l2jIB0|>{|BWi +znmPl5Q{j(tyMLuQpYQv$KPa}PaB2JYEK5)Tc&lL7$U)w&Pt37f+Qs$b5Eb#;ae+&w +z39XWZ-|_P^0A-?5p{gM6PDG@Fk)wE+*U_k^Z9+yJ^NbY6;KODrStUS$2DCS|A +zU+D4eNcVb+yj%&3RDKFObb; +zyp`SUW5qX~?;-AiQp!WW;*61eLDJIZN2>>vi5g``2lUOr3?WU0a(FPjtequWKn~!H +z@XQMtAjWSUaXFEH-1;k9E10(WC;IL +zRsMzz>P7`CWs&Q~>9|@k1Y83f>&qe*Pu`YF8|bn#EB!jiYs8p<8`po|kjQ{ILg>Ar +zC?a*d=eVnJ0rv!$3vSGTq2z1L!bF7`!4gj|)GgzE*n1Ob=sLC^P7w9zP77~HLu7=N +zA4G4Ve1S+}G#M$j5mwA&yz@V9pTZ~2%&^~Egw2H->ejO{0Pdl6OKeTi&4p)jt0xf?^*Q{AfKe}2)=$N$OJeOE?eO +z4~XE~ki)Ad)T`dqzJ__=Z($-kGtB~jIu~6P?jYLhF;uz3#mh|n(3QJb_tec-Y*Hf_ +z%c7WuM%shCRpaj>X%Je(CI~fvwIz<6_{2wyzr?YWgjOILw%iF#^(FSkvat5)3m0uq +zP`Xf))FJLoj#fMrSjXHzGX!xa-66+7NNN%6@7?3g|4H}b=1W^!5W~ecf&f6BAmM_0 +zR<)hTHK7Otokap&K$yP#)W%;{tT@~H&*Xu3M`IBuKk02 +zX+CbuK#bZRikygw5V|f(DM%>#A)ewEKwHeGKz+jOVx6B>u;n8`FGicL$82jnwq;|Ox1 +zf9;5OfdK(d5w4#FA(|EWvG#{vrWe95!EJ4Au(V0`k-@K!AEmpQBTr>td(<7zH?*od +ztFjGSM)whUJIct}U0-C)9&s`oC3ns=dDyGDkUONS%Sl?_KGf+bxU(Yw2BNSR&nctq +zy4M+&(v@=dqzkh5_Vao3Yoa|cT2oT?AwVbG|oh!-pMrmk3+O_YhnUNkKrro+Q;>kA3h^*jB+ +zu3aIaCrTOjv#L@u89;DGEaYY5Aua7rZJut(!#%{SOkW*Ea*kN~$)){M2iFiIu6qbS +zs{HH#^fD^I@Cg4_IE=|_<#?Ous15O_Jy&}$Rlp_4jcWXodGXffHs#-4p_)3Dj)p)| +zOHt#~9~O~BL`2^=l1waq;p)qAnLp?!{ajO!tb*$BviVGdS{?4RkjEXIOxH-!H +z(xitZmT0dFisB+FCt3+EYCXN!P7g60@c1!Ct1c&e6)s_6_RW75nY!&MUIY+01d|Lj +zF6p-`h6r(N5{V$Iq9%#}Vt&vSivtcaLUm&l(8(mNgAJje-bc;j?=Op>m&^qa*r3Xz +zlpzRJF-3v|d?R0X1hUz6Qc0S%{ZK*cwDDZ!tWqpvIsz-0Js3)uhrwUMI)mXsC`2jb +zc;LtNlz-U^YW2|R`(83X9TlRj}Oo&Y!rfS +zLwF9bwRE@0(>ZA{;Oct%Jw>_==Zri;%{n3OR1rz{+SnOps79cD(yQ#UBe0xF2@$1r +z$QbUnx!V}O6k%(W-*L0 +zyf2mkq%w=ZeesI+JO00j0y#5+>5QMy?a)Usq7j%25QK?}Fqq~HQ)EZKlVmG+P*@eL2+g4NF{PEI$kmu74g( +zF5Ql#0(TCcKwM=hV);Wj#KK>(Pdm|lpjJ*rJ#!chC+@UP#R>}h38wwWTHh1+1Li5~#Sh1bwkGxN(<9fuw-99g0XQlI73A|ZDDqI<@;WzJCBWDJh +zQDJ5z`53Q)YyeXr>v=cGOK+e7rluGfc~1?c6xEdxAtxP}lH`%7Br-BUe37siI`0ij +z93Tr|OYpO65A>FTb5jkX`w%26_5(mW;_uayAQs5pVS)%h!VM53*^!s7cgo!bvj_hr +z6xtH%b0YK&y#WO8VhVG6t0L~tDm19gSnE{nli+8(4t5dD#I%Ho;=5uOIoV>@DKIB7 +z2Rjwd3-t=agKCAd!_^4W3Fif|o%=0=W7w&ws8cVJd31#Tk)} +z@=LnY>+DpXyMHOTy}z~7+YC{L;d*JQ!~Wys!W8RSn`!;Up``x+lFNuasLt1fo=d$H +zZ@B@)tF<+6bY=>yT@t)B+w=8V!sy^I%r47YkaySUjkKQLBTW(aZrBfH;h~a8G?=9%4IPNggR&w`73`b*U`#&u^Ekf}Sy* +z&%BBSlfum~Ple*%suR|}&yX(;lE%Qm!2ZcSIDNm0o`v1W-{}UfXZJABf-vKir+U$= +zKND~{ppw4XR~oO5N?!|4iwN4?7TPG{0{gvVlDHJmp_vnX3E*<__$&(x$8^@C+ev;} +z5i$d{q^4#_JC8Cg*~CbNT$*VJ +zhTp!**`nD&!h3vmZ0mFSC|skFK!@njt5;Pga(NjVm3NNq;! +z_`ZDb)LrT^^6?^q9qdizO}v@&PSy9cpmZ5zo!66ShNeo#aJaBVUHdZYOmwI*ge;tg +zSR~eaNmAQ&96j;qdlKhC5BWFpJ~uD@aGSG~P@8#wV%?ultpwO+J@W*R*Nmz= +zlE%&ZB?Q~MF5frsXr?Oh*BI1t$mwYPef)hSsUi<(r$3$O2x-z%(G*N(Lm&?o@LwT~ +zjgF|D*sK;!$q>Y!037!nB+{pM<<6E3!~o;Mt_PDP7QAW{kt7mb3}yMuN}Wg5*B3HB +z^KSIhAMN0`GK2<|llfx_;!#ze9sK0E=n4tK%g!6>1{$OK=3mQRn!WV%RF^Rv-6Jye +z=p!;vm8V1%B6n5i6?Z(Uy=Ob?utEceFZa^P@@i>|4wVROYYM3g5d9MAlsV~F8@8Hm +zOI{P~lNyR6ZsLe)*d?1Juq7L!&C>20lVyt_E6frP4yo}eP>(JhRh~4Ts-G$|A?uK- +zID*)pnSUf{2x$#`2@hzo&@zaCd-s?rzfM`e_U +zFwJsG2L3(GNC`gb#X2Fl&HU+vM#KKJN&P80ymDDQU`0}UR{OpLYvLPD^@EVKTyVtl +z4}J;VGM1v=&snR|TrJwFU+#_ca}TRi7o_~#to{~n;LJ?G;ps#LvWFiSKOV@aExnz@ +z)-V8Vq)HKjBhYgkC{42yPFgF}mK`MWR}GKxO8WPjSHCIZt;?h%{lJQ@w?g^=NjfKV?65QQ`yL%u(gL`lfZeKsIobR4HZvE(A +zd+gDBjjCF;X3d(*Uvu#<4@Jq|F6V>0K)+#^ERPAIvr2*m85>^fL5W<$2Wm@PhnFl8 +zF~VNn +zdJa2q#ay@dz;nI&w#qmTRtHoAf)b`9p#zW4eDw1T(`ZFL}_JG7lQN(YkRt>4Pk(^l$tU>4J5iO=T&xypBl9cW? +zTgHwFj}uq(9zLBCs;|MI2@2sga8tlH`B!`9JvpVFRoB#ZHBp>e``yhgt9$h4M +z&FGT;zNk-BX1$CCihn=Lv~=L@%OoziJ{$17XCIeFCh<)hvGvv%P(%DG*il_Cf_BjL +zdD?sQ{b`#?@?xMq$lFJ_8v;kE60_oWkn*~6#1{4Uck>t;Q?g>h)C0Dv1+0iv?or=1 +zh`?GTU~^@&LUQF1Lxfq#(S=KtE%c{ietjGkBS+sh=HN!q}iL^9VAolPl +zmV6T`tkjKoG?~X9IpB?AiOzv|X~jWA=)GVMHDz*(@AHSvkhxI)AHn@>yMN^(0FCoU>urQFvw?LAp3aBGM%4`{sw?2_>=#DXL-3-kDp*mk^WmM>7RSbOm1NKEP +zk;a*aqjmZ%!*u`%&m@d2Q=$$Mid{8BJ~$9d)7N(XA8vrE=Z$nr6Uuy)3a_aRQnD@SIk0U$sQ21?4-g86x8YnV4@{UM7H1C= +z6d!pA4FhxQu?>*2>5%g(b@rbg=fS+oVm(5deBJMG`_<2`J +zTDKGRo^!;<@y?n?YbgDr-4StaQ>JHhk;jDpDO}lx^+t~cYayT&_tMkH}QP{yQn_q=?5RT({5Fuu-_a=ni9_ROmO(vJ1b#qCUl +zExVb8^U7EK?&$E6S(u@!r0`&%E-YpzMT5`%>xwtJt?O~Yve$y;adA#g#b(Wx-3yV# +z@3sgHj9Kron5!p9k}~OFYn@wu?0Y3WSz)I~OE8W<>0Vny*YNhaVXTLwK}7Dy{!V3q +zQ#>p!7k``l@`%gGEY~H(Shs;u8u>TopV$Ys%*AE>pi9c~_Txg`$;5!9yua*enz^7Z7Ut)?8XpNnEa|5eZB-_!a`` +zF+1lZ%jlqy$>~LqND&`w^<-X! +z%SLNgezb-goAJozA36F#3)zn@1NuRj6|YOPCJAw~=&hIsr3F}c{G9x8oBV^w2FoRICsWE8KI5+Exb04?e`NGwnx4>WjQQlQfEgTk9?4t@RA?bq#a*SC3N5_2fFBV%HRWurP@K7bgRO)}Drih~kt +z$CyInkfS86W+}S4x{q4)YkzuZA0W3@Y430BwiLSc7!UTrD)I6z~=(-*y;$K8Cm;Y#D<8&j4J)q?>UC~iK +zsV%ms%C<_EdQO9&+P2cd=3uXqwB+uWtKy$G(@`^2q9v-A*YvO~Z_?;8bd}pL +zYxIjw+sCSd=|Xz$2k}fIL7)ET;_;HJy0LU(l(M-e_eTXIjn8H=B%h8UHcqG6W8iSn +zP0*mu{m1OfTqHm;T4-zQXGjjkb>(Oov9?gDmLERBbA|Sj+Q(1JApyMaykFWLhP)LW +z^gMp-(^X(Sq}y2>y-V_JVD>mh4Hy3Gs!5$qw(5BBqIx}9nO5&DYoc<^bGiHYwS7bH +z_?QfzE?UCkdqdz;yf8dJ=3r;s-5URTMagEJkh}AxDsv#^mtzXCL>jBX5|lIynYL5% +zwGv6IPDZbxum99C?B2X<4DfkPMQQrJwIXI;(|v2L;Fs}h9pIOVzLXVV5Eag!QVEQq +zThmK`=;b!v*x2jQ7u@e+;^5@hF-B_5YADEHlVuvCmkrG$yJXUv;a&EjOMNR0;l65g +z)(gu_tE$59=mj)_K>S^~lJ(k?w8M;8092_Vmu&t?m6~|BpW-{6ElyeZRacSDqWsIP +zK}RSq1>e1NlP~JMKPav0<*pVe2vD?#UabiU7G($FFjVNGhR&s;WY%B?i8qO`FNPO= +zS^CrC(f7;KipSN{mD_mGHq(W;Kt2?;1S!GLMM95Z0O>_F`p9IobgN5scsJq|`srb; +zXTvQ72bhhn)Wux~P2*$N`~4BYNAkH}159h0#lzLiXx-l4QN-O|%V7OIq|n11d;0JX +z6`uAB_CZc^gy;)h$ywz{f<2{JEp5*j&9vn2%fpXUg{#lo!gV>juj@+s>!JzkBiOf& +zKbWUC_TQ{pCSu}-)mJ)@VF<)(aTm%Gjm4gg;4z&kV@1t+O$_~R007)e;HhHb2Q5GVL1DLs1CPr@rc{5!YGqkR8^jJaPdeh +zKUxlZ3tvjNrCtTK(XQ$2n6vMjPWlatwOB>e4N)68B_~ET^I6ZlG-`{>~Ckh +zR9#>V9nZUQp}~HN%Yk)4N5?)oGSwu%>w|y5C`1R;<`;*}Xzdb_cW2=es3JGaJk{IL +z>23w>1VOV*0*21Q9#=d=z=rLd9!-A-BytF2!4$89}tAF`Dgsul?hB$Etpb@;Sy6+@8z!YY(-#p^5IK +z`0>r>pug-NaZ{Akd3Rg_UN!X0YMf3}_r$eV7jA5=wsleKsaL(f%>?XM4@X)wr)X!IqB+r96XqaCZ?OH^hR{nKSY_)9jM$`QMrY2=xm +zu8`#O#!Z<^xJzu^G9UK`3s0ZVIfumU@6Y(B>du)rx&amlJ#HbwYGSkR@|83UC>G8@ +zTU(11Q-W(a!%}Tlzbszwx*DEi{Q1!_FB{2OWxn}?6>D3m|5BY^cvpNnq{Ut+V_u|` +zkue{~&c<#iOB$ZPSu0AOLPh-tLt~3=HUZT>;cYa8rq2gNYB$ES8LO=6cKZ8nVeZIr +zk9djE0Eg$p^zCfIoKf5i%)_lY|Q&I8dvw +zi(mT(T0F*YXU#1A5uM|07&TrmZX&Pq|Jv0FdPLcq{l(LR*C;M1=-0#6sLw#sXZA@_ +z;}N2$tTXTFoqr3#QdT~g+)9U%Xf?7lOeB>&hj3*RQOsaRi3`4w5saQz5tZ0VCuUG6T+uJQMeY +z?dEZyY5bB@%8wvUh8wGPHy4& +zskjg+O6YHb{`Ix%w-s=k=Uw;8DZ0V+jPc`KB&Xz7XMx}B93QJC!3jTaqwKw6=|{pQ +z{wamif{%wgkKxvFyJErvCqXB`)n`1#p?q^IIF90ua;1j@Xn!8tmZGcLN74j+wHawb +z?Cy!{&cS=`Oa8VEuXTquwf($bqdw^t>ZF^V2NWGjM+PN|zvaV5LNr5u23H==*#lb= +zbw7}?3xw+8?0L~`nF{wXnwnXu48esWbS(Tu7k+q44TM_w-9qjAA*k%!o-(LATZf`a +z2VArrKUQ9n>V0GG(%HAZbF0z3Cr%}8#1yHIjPYtm)!w2&*ruc3p;(q;{IuM9%#r@p +zaLZe0<#vvn-8Vy#n12la?u@;eZPz85 +zcSWFsE(%u`d)=V@ixf!w*%Rwj&0-vGibMbF-!%oj$XtIUpc;`I5X(E7meHWl=~jgz +zK57qpgz#i$THyV2bC;WZr^Jl9cC=bD*}d|r-gc{e&fPN+X~wRL+sJlq`mpeeOr0I} +z5wsQx+!q77OyIM>n2qW#|NdY>V9_2s?v*jjo}EVA)iq>6V)4=HO7_L#-2ZRfg!2l@ +z&c>dg-l`;uHFjB9K~5si55pn6rQbrkuT)3;+DETrB+=viAPTSBwR~@%Bxv0I(?qVx +zF8SWv93PcMef%&SGv~R9i|YfL$F;I5Q!^r9=(Ni~cujPR`w&KffY~{&Hwpk?B^p;i +zRAKNqnq!lzswE^v;#(Gq<9J-+EFl(nw*5)u?WcWOCp{Let8w+)x!uOuKaoyK<_W>l{ETfgO@U%H@Gl1oOO2YJHD7nbk&WvEE8eL>_QZ{v +zpAPoP2?f`^aS>d1gT6paK>=4JUAc|3cQBU9=@mv-_(~9eH<7`?{!U`C(|Xa+AJ5qD +z6Oyl?V_W!{o)jWa`x-dhOG3U3*_C;bj9-_GE-f!1Ccy>5dbT)6NX^~RV8hox^Q?lx +zqPv$9mz}hWZ7|JmIR&PS+26C`G!`1}o)^&c7DeO0|^~=QBes&9AT57j7wTYf& +zqqx`u+~0)-6m`FOBdP++Q)NFLg0&p%!+md0=0c;NECQJ4U#gGCA<)>3B1w!SD(o%{WuvZGFk +zF23mpLZ(~3Zxd;5k0C1s&s3C@tJoOAiu>|Ag4$g;o(|iR+AeiJ5JCm|B%5N_OMv_! +z+=@e?vS_(QLA*#q^U8MC9O44bZPQo`x2?Ns-atD6z+(}7iFG*8M8(GkRzT%MAZ +zV6mduaQFNeBXbXw3bem0+G)T3kSy<6dPzxvY%IXB-tyc*iy}*ie{Qx(opew+Dy8jO +zS*V5}{t_UYGvbuLq=qJ_Y*wR!Qt~JA-V9qDmH0C3xFNnnC?MxPVMQ)+WtYY|7RryUyOQ^x`9>^s-g7#Rv2|zyy=dnh1Re{x?>R^zZ0= +zuYg#8WN?4uf#D4yXjkbC_hD +zO|=`fZg-&j;PRl{JKb#C+40_OiYcri%)^g+PpSMh#W&&%1xc?=P1vUq_!jFU+^>9< +z9E-Pd??{$WY&;%ABaO_i13xUwAzTXg-+P?(yk)5&`}ON!n3}vV9WfYLGIT9SZZR0} +z>(iH4C)0Zf3uXzSNxslicW9GDrVz%myZ-~);D0iXy7waz0ONO`>dcZ%kR?qRNgG2S +zn_u8CQP`3+y?tJx$5-kWhQL;+L~4 +zN!K*k1m!3N^fI&{Cw7WlJwIPu+rm78k?K0@GmUUiG&-@q-yM=i&KQ$jX=_#OWfFw4^rVZl~<#qJl%h8W5erR7Yng4cfH@ac_2eWJ|F +z7TXzx)dR%sR(^mc!Q*2IGm1a1$zuauu%_EWlBT-$q*e8{-1IxlSViY_v%~$Tu+yI&&-<7pRBK>h +z!YF%(A_9`ItOCOdm|$3eAq4VsWo>xpNrSkxabO@IVyHN^GzEKd~v=#a`8EC9cKGDt;q}iVpQ#^pc+F(qDXk6N8GO*;y +z8gP}~JYR^v_r0?gbbIZW1Z}R+N+1Te=cj~-n@rvD-g0q)2aRRv9e&BQI7b4ZZ<1^I +z-|l(zlw|Hzk)wG^niWG)@KvspBF8OCM;Jo!`;Z=}a~>rJ+uy>YY++TNv$KLY&7O17 +zXqfrV(V~KlX!pOnEprEa(v!8~NiyCI1UX=tzJ$ynYHw8> +zF0~lNT2Ay@2DqF+>@s}ueZnENZEUF#lvA~bBh6(T^|fs>Ke8mKg?r?SI;7E1fkRlz +zqWQlrd?u}fhJ4->D38U3?tTgyCRrjd`0z8(o3=KFYV6=PX5#10F?yWL9sY%znCH)- +zwbq@GuHYVsOB4qJ75wh;J{ciA{5LhYQ|p~B&eyk^nM25{KWnp3x|?l +zO4B4_IzD+ZI81G`wdn3$K)oh@2U@-G&trp~D$Vmk8%zh?zW1=a%GByY<{||>zr8}f%xyR?vdvari=L?l2$5mhG1^#LB$1Febv{C!n%^*7~7iu0&u +zDP@Us>8<2a`&=5@yKBs<+26mVZ_hqUVxNi?S53q0(~ +zA|cK1u_pTc!ixtApZA|E7V9B%{_(Q*B@kJ7YK(fs@1KrE!VCC6P!{UQsCg-cZfcSH +z^@gP2o(F~84)l&9x)_E(V8^W|;QOP}13f7fZQ#w^2ob>FEF6kDVN +z#LWRNq)o&DE~}u8NGKG+&AXjXmhbCGLDvin^hWw@W@W-m=J>WJf_#mzz+9>X39$PJ +z*9XM~UfQ#kS32dbZu`B3_Ss3G$y4Sz&zF$`ZWR0qmkI(|%N8mNrv&?)?q +z@&S230O+WzyHN-am1VI00t6BPW0D~oscrdUo5Hn`!$Xt5iY5XnZC2BbwgU7&knGJY +zi7=ZZOCmPBtAsmeHpc)ovFBw9Qc%s5-ad$8zEK$`3MPVF9Pj#IG}-qNBTm^T=P1Q{ +z*$?-$7_SPe6QCM;>4GS$hbfX~aKTn7TiZM5Pj&W>rR8UdRYZ1~3>W}lpuHBz#GDE3 +zkL+DcRXS0I3wghdMFYc2TD<7X%(*bQXW$jD>6*Wje+|chAYL>C0^tSgdmGmpy*B`< +zDl20CL&&S%%EASj1_5^>^g~TJYk48pLAf5u9MWde$=&VSN}7w)DrVWGbLSWpq1)ax +zzt<`7l3tg;M?E1hB*>L5Q~XKOC&@Ate!B8x(u8bR +z#;TlYSZR^*@HEd14k8j05y7sU8U5yi17{k>&LN_5l;M{wtF2bwA^r58vNLh|y@%)Z +zxF_;gl`U>W;6>XXM<2QJDgeDf=val_LFV}U(mdOvffQ(;f;LTTjBSrsP^4Xjrl468 +zRW4O0YOsG}|LrlS25W9DUDIm0@U+|Oh>zD2CM%jk_TQ++e +z>Jv$3FL7vmlJZYtCtOrvMQ^jcp7_keC>~4d&@t4*=A8I4E^ +zth)L*Eg2xX;+l?kUz{*|Q208$u4oXqSsykI+y&5B$ZcklLJA~1uh)emKWLjJN{jOz +z%f*bN2mm;dTVtC?@>6UO=LV_P(k*XJ5&$M;@grc%U$IH**%2H3yUXN`zD +z04a54Z1%#PRl`%1R)URvZ6S&{F$_~EpZL;bW1PCYjpKu +zq#lw!lsBfZi5avz3XmWa(KY%;O2`5O73v+QOIazae{KeMgbW4BTH6tr%jn8h698xV +zHM__L#>H;&NI#ehku7;y>}d{o@JmPv+y78y8W5=HQ=G%^@<@B51fpT +zfKc5lM+?_yv_4I<%(kKXwp-j(>q&!w?J)X=EmR5~ajNfNcbI`L#F?P&dt6nF-9)dQ +zWE3UXwbdVJ{nJZbr@I~$PlHwUm;I0XV^1PZem@}Xodv4L@5~v1#N{J1@L$B*P#AFlf&`{t1QlIpz5PU1 +zXfqC!G6*tbba+?H`9kLO?(J}-7R&Qnxp@sZtZL^%Guz4OK$#wwa4)O^i178v0>RbGC9CPHX__Qns!6aLV~p1Mcw;!78HJkH +zE0o_dv|wO63gyZC%<3{HcWKBI))XsFf6%^Yto0j7dQU6FPmVhe_cb)jSoF`RI)&8O +z+eM0uuLSUUU+AtA5?z->`1c}~5W@1E4QJ)jZYk7ctg+X(?>Z6PUCuZ#V-&}30E&#w +zRO&Xc5uox)aA?%QA`-gWw~z!P2m6zm)iA9{*X)!z%&~Mcx11yLg4k1X-}7^h9}~=~ +z!o~`X9#vb0xNL?3E?encoa}ci|HL#n!S>n+Cj?TtlcK{H>45`96Wx53Ji))ccn*c8 +zQySj^f`YC@D3ajP_^9#RLhDA2t#bHZLhy9`QCt9e$hN^na8Oa@0y#}?^fJ}q=3de^ +z%>jko?H;C3Y~!suMpWuleV0avd)-!SipLXn5O9CaJXtA67-B(ut>#QgT{LRPSW!xd +z;#ONaMvE3e#cRh?FD=^G6V$+MdJTu58@zqc0HTV0Mo_11z}Aj7BZK(as5FGTj)aTc +zrtJKszdCkqQ12_iqO_4kTyG?DYkZv@MWiPjVH|;tTgw?LfZ$X;lX1?dKD$$n1>FWy +z9|ydgsf=`a?u6U`z9GJoY*qVWiSLh6hUX1>Pt}y0lOo_J%RF1C=&Acj@&ogwwBApO +z5sDL3-0=FmOdFw9iUXB+jmqN+SBc8$YZ+g9ye->6VC(ifk6u4a%b&X%6@Oa7FAJ9v +zhNYn6j$YfprK0>XIIhR~5J>OMGAOrt^m1|e|Bh0KR#&iwFyb9T8>~p3+ih{swpmBE +z<%`aec!#)xIrA{DRYw@WWC;6!+Y`H$eeGc+eh(p^=0>>~37tm|`x!Oo`Ran;d +zka~CSf6!LpS(SEq;DVfcTx%20!a?0q=DYEZQDL^+ddW^@j^HJ^2xBbR^OsDCxeI(0 +zdyz{+xYd3Ca}YZ;ic_$NMRR#P@Cy_DRdsfd!PYulEMXlP0377IHhn$d^+cFU&YvJ2 +zQ}q`o`P;4h_90Yfr|5-gOe;`G^OcHu36((hArLc)w{?5G2WA_-{|srVE=()n=;9@7 +zC(#Whh3Sd}&ZBSe|3cZ|OM)(ayaSceIPrIaV+k*_1~Jucsc?_8R&Rk-Hxj`0&^ +zw953a`_D@P!h~9{H!JCxra$NBrgmPw%5P~=Q*P)E09BajoY}%v7|wiY;AdesM7(fcdqh$QAF* +zSb*|B2deZOLu6pXZwe_z-%Sp~o+t}98lZDAB6Xns(}RM7Z7zMi#iEdYKX=y`s5Y;c +zC3P%X5Du)_zz75J!h#|MXn*-yX_HPIoH=Bo}|>^GOHO>M-G~5cQ-MuedJT +z(`&{IP`~Zue5p5I7i#$I&vdR{+WN6xai2(by{-F`{Oum +z_NRNNunHO%)F^xRDhKIdfom$YLI +z0O$gHMpYz+Dd(E7R2`P#=GYIH9!k5a(hd#q5A}U_z8ATvI!K_F%mB4S^yrFxcN}c(Kgrcpcku^2@FboVKTaOF56P0JMz7Cw +z_q{6ao<9us6>%MS_e4PH_}*CRqTCltQWk(448$UiQiZOk-@kE6m& +zoqz7VLT|gp2Y?*5eJuZUEL+71`Zg?*`3@@(^LNFO)U!hQ`zrQk38cLs-k!q(Uhz<9 +za0*hTyr~ruQ?!xDvv~qgVKyp)1iI{JBN-n4o?YNc9YmGUxHQtGHw +z)nZ|9Khh`xvdr6)1k30R +zTXmo>{}9@+9Rq1|&$>7PXlSPAOdmZ(OMz4j%D0698c3>7=_S#Yk?6Yx71MyA$Ur8z +zfm>?KPZ$7ZIDodd30frzLx5F10!Lhqi(z5IW`v~gj4_2EJREKlCCGZ3 +zCjLo$KLL5-3!b)dis7jpzRq-%K>(g6X$tRQ-XWrS&7m>X79@1D#aJ293Z3fB5@qO< +zN0+uhLUVG=G96LL>5EyN9nd;Xd?$!ge81-+UFUHW#uEs`0U9%9E2?K>*KgUKc&Sub +z(ima1Zm!dI*sZz*gHIbPynS$Q%6I*EjjQdy<`q%v#QGx&=s9ANN#d28W^+Z28(Gwy>XN=f}F)Q2_#0)C_B +zl?q%c@dW=5x*MWhG%7gMg~!l1X9DI+>}~*%zW$KWWg`d9ECG2-hoel!cqC&SD+(pT +zdeW(+7i^1zzf~d2uo8b^9r-~gES%v2cr}K*(+W9^uc|* +z$;&+(S6@(3d0;4j)p0#^@gEaycCTg`GpSG*U7JG+dG$-YS@4WGJ2fNX`^S5&YA@vG^IuM3YB +z!hCRD`>ko(S*q6prJx}56sZ?FP2;h&VV=#JjYTW@1ug@Sch>{SKg>{Jv@-}Kz7w3| +zLB@1q2J`@0C@oz(g;g9JG*r5OpH?V&{M&&K%1?oFzRs>k7`e+rPy6QN`0$WDDaPk7 +zsFKrU*pgTykKHGG7~m+tIlZu(#?3WO11SPjl9&!UD`nX1{Bbfp7eFA8bp?E*k)7fi +z<%U}u^7PHJ^3Aff+EOa|bZ*~K*0iuSkRktS)YmO#V3Y7iY456dh7ZQIxf|^zOlEj! +z$8d-^e{YH*?wEOtz5py`-uc4E?mZRcHW#%@#mJr{^^XJm%R&|+*?t>Q`KFnpMj^j<^zqmprf?!G!&f0Qe~{N+c2*+=^tHz3izs^Z)F$|F#XE3#980Gm0lh5Z0@! +z(~I%gE64eJ-_p*I#qOHNm!znQkhP7^y>l;NHV_Rk<$GbDxXypbjsLdZ@g7{@8iy`c +z>swbe3tDCw*{xkZC0b4qNXb+{K=Y_{IhL|V7ssL3s=ydB@|lplIsT4w|7-7@-#>(d +z{}@}Gmof;BgH=gW!^!~PrHSWb2Y4aHtmnUTDq9%a@MT +zhu>HdiHM%w_x_u?@b4EnA)jSKdH@1ydMX}DCz>-qhxWZ0VLB|MCsBoxAqW{Vxq%p? +zo)1+=r%d;qc#yduE2|*E9sE{$v4U8gJ`$piVR{@2RB6z^=uZFo@m_^7H~~GIXlR*h +z?5)CY{5j7JsQS8Tc6R)B=ahCXWHjcH5~f|zWs(@801iPL*lD9mzJNtutODnQP<_Z8>MKix)(vaJ>N1>VNxqjZ%jdHiPYhVrvny&!^0!( +z9n(9xRKjISG;w1z@_tASva;=7S+Ae-vs60jP7-!r>cNmv!1EzNI5kRU!~OT_O+Q}@ +z_;tcpjK~3|Zos?zFfLG7H1)W3P+^&v2PxW~)-FV1Wa5PIp?&36Raj5TVENJA*;{^D +z2*ussSJ~^7HGY9GuaBBeYk~LY#yL^z3Vi*dh@~6-#+#3w*W$ZbMg-4f4mw85vJ(Ax +zJI?nhUpHzGGqfCycvl0-nLdwGQ`1o(hUoNb1%dP^ZQ!Z+p1zw&a{iA?-@Z$C3U1zA +z^ZN&5rBa)dUN16O$DkJ9}#<{o&`p0{@#nfoC|fAv +zi*-*(64|yhtp6$8ypYq^U&pOCHuV}USv{d?7MpMp(X^SN6Qtj0hAA})B^?BtrvWW> +z#W(0%ne&1onq&HVuo0Vx3#Stt@Z{e(8ypm#-4FQY0&la7UGF8z-82w_X`lApdnA}X +z88nQce#zh2@%=DfvH$lt*!S@ZLwAp~Nau4yyO%7v6V#q`0qX*C6yvYvhy$p63R^Mg +zgEZaHGkUt}OJoiu6bAABn#XPntj{uVC8z&RwhR_l4{Qio5>3Ri7`M?t3|WBRMcmv4 +z07Q7NZ&x1w8#1t1;^f?GWBcxWk@yzEoq<}WlXKd=yKNo$eDGy|h&M9RRQ +zn`~X=N~dFWAE~Mx6xaxOWge39r<`pE$1@EBKR!G^pcC8bH-r3(hL5NZ-jE>f6=5+HNj5VaDU_e;n@=YF5nfWL~ +zX#Zx_3I|ydbP;A7;1hDjlNfU(#@DR~lY-`++FchCEWmfH;H>6);?7lZF#qh**p2zd +z7^y}A1ZlxX9i}(bSI)DU{%^+Y*~miq)6}J^mx|VK!_eS`$)1XnQw(?`go6PWas +zvMup^#tcRVbE+*1{M(W+cdXy~$|wJ(BtQiM<-^ +zUW|2~2<@!!l5KJX(6}s#b-kU=HrWR6f`A5u +zq;CiV<6N^!S`3FZ`v)kNiOW9`x5f_7uC1|LOX*Q2gZ{!~VoSo~MDv8a(j4F?tJ=@ze{$4jZRu>;5|G#geq+r-?gwMjbgg +zZdF7M2APN{zF^biMkKCN6zVdI%M9<$u1r(eT6}oDITkL_Vr04z4g8wP8YmAUdY4t0 +zfrLf1cR%Qup81V%9H0x<)SjEU7)1&j$Ms*AV~xv!X+wWlTfG0=iqeNog}B+?=6AnF +zUNVJP{F?>=aJcRISXfPY4-{R=h9qNOgygA@vAN4HQ^lNS!(Y^F?9^}j`rT4ixjrrm +zIN9Gjqx6~o`1!Mxnbb6fgGa<#Px3bdX~eJ110*fJhTRl!t{ +z6vAcb-lEHV_@UbY9`TanrUDNx-hXhgx0e%y8$i(|fy9T{L>0k@ha-U$KBWPj?$~>- +zC#$=XTLZ;8GKqNQ)_}y)GETpqw(2jZ;JPuxYI&WZaBxw`P&Amo4p8Ui{$xeyn>>YI +z$95$XQ6E0|rdJW`NBt3U?$=3$18nF9oDjlO!vO%v7z)Yk`<&nXsE20uC1z~ +zER-Po{ln9=mOD%{z(U(&IDPl;YjtHI*qA+RV&b2o@!!_*?jMAL^z}mpaWVv0gBOM9 +z=Rhbx^?uepzMv_@@adq!vBclxn=a+uGmiWS!<$-=ssWK{5S$BKA;F)Aa~(K!Yz4sa +z<1mjyIfaTQn%>#|aRLjbv5D>o#|ZBLnmu;>2^I`zNr0?dUC85n$S;y3CT{)E2V&Qx +z4ka@{Gj)K=$3%fTr|-L?NV-E*knF`lzO~YDYJ`_Yl>k6P*fPmmgxD&4k_@5Q*%fJ@ +zFsw`D{Mn%egkD@69@X%tU~?Sy`30}hk8TloQo58NP|xRv-pNzZLEIE|awz_zHd#jc +zvc@ECqlHX^(HKJSZpjn1RZwl6A0+}9elO|?J8rTiCcSmaFj2JW3upzxj`#~n8_ULx +zKbxD5gQx?I2)_6K&y+tl$nb!NqwknMsC0bk&^j@9*{F7;;ps>U$2dm^M7lrK$piy*wrIlPssjB#w_NY&@fbb~A +zYP?Z9L7_m9$2H5Atji~oid#f;AGkXBE=u&)iP~iTttZnT8`xjIZM5&c4eJciNN;a{ +z4@Ry8;*QEQT>U-qf!&mMwx6H$-A_|ZkIv&uNPpusi>psdYyuHWjpGc~0!BB9L7R>B +zS?Ed^^is=o7jtv#?~^F3G8nWq*D}I@b#`*c4t-sOBOHLY6vA7A+>wyl#!eWj^a!5Jmm&ZBOao3&Aa7z+oSH!EMTr=mkD%%L(qOg;XNGud +zdKl+`lm&APcq>BqpmCG2gB)Yg{5peXDPj5{m&hkQJuNx|U_8oYFa>h*GPW496e}Y` +zMedI*8bQHMmAPvcYV%{#Oj)w#!q`u)iHOW^O{pNhe*W_~spmmGK84f#62w(oknPE* +z#LV;B=0I76@rwE^M1e@abx<~|pSamC=lB-?t-W#9V>qQJ9*byKkY&Rt;Gn&Dq0>bDzyml(IRNf}44IxQ5l-HYZ3rn|h&R|Um#hzbZwptSUO$W)8u{U*b +zy5Q`wtnN*>&(8ZH_z#KGfIUnFjOI`@^NJ7^2agduJS;ruz~})|Yq(Xg4l{wN8hj9< +zHWp9-#v62<4!Kj^aDMnYXj7ReMnf%~OXi#?se1_ZO_6)9^AHb|d8rQrABJ;NQ=7{Y +zd+yf28?c*RY+N9eulJ#)rED(?2nZUuB_p&vj@Suz-LJl*XVf1np97O&%+IJUa+M75sICdw5 +zpe=L(m5i;5IbVG3YP|si|96GcKYc1vKI<}C4*l8%CeV~TZK>k8a86Gj$Yoak_%ts|Ndwgxk^AHNBaCdOsPW3@eYQm!A|_^7dPmrL)63nIk9{6T;9^s6>m1@*y~+I +zuLz)=ZtLE;duJ(oc1+v}Du2vA@Ovn1`rBli5{q6lmey6R9NW$s=T1D9X6Si8>#-su +zFmsE~xqawu=hryK)!d;@;U{R46)Uf^SC%=lm%3@!8sKX7Pk|6}i(aDw>KXq~N}2(V +zPa=%GJfmV`E)mq)u>}cdI;*QzZXc$Ry?X9;$)*G*B*Q`3Q)?2^P+(&4(1<0Oh&JsJ +zyg2ZvLdv*j3|msg$d7Ho!mC!_ssDV275pzancihG$F|KnoV7+Q1E}Gfc>8uzGYaYw +zpw+|&u%`XqIu=T+1M8UACcjUn_OLC#!ww@Wj%(bJ?7A8D`3js{ZoL2PTa*vo2x$_~ +zrhf8W;Xcx#9}E7ZQ`9R>^qZ?^tX~j4>DJv`?ShR)NXvJz6N8$Aw1Mvb2vQpFos?fJ1dfKZ8e;n=#S~0YzJDcBTo{)Hif~ +z<1YSS>`~aob+s3_V93#4z0au5+r)dLE`@tn38O5}xa7g!d;AvI1W(dgSW%ZJ|KT2I +zL9`7YU2I`s-_t~PkYSy~b=KKZiyir&#?0RZCmahCcMX}xpGs~SV|n5xfC=hu9np_Y +zeQeqP@{Kj74pI1C86PRpHnol7j4>Ml#X)6M?d{ +z@V^lc?^U?GhGps<%N0v=mcq5R+Zo}7k?%;MP{D72WqZY*EW1sboINtWsBipS9IU3C +zSyKR$Kgf#b7>4A^<0-gk=!TxOr=GMwbnVa4d|nuwprX#f{@b$mM0*30E7Hk~z&!y06&bWZl)CV{O*JrFlK)P^-enX}ttaQOU4OQAIYhmf +zgdC+L$d8;gO<@2VWy&NT3xJl!QaOO!^)h4VILkwUfH;5~Wm=$ED$}(r2fM(sf$Z|3 +zC*;$hd(ssIs=5ya0*Le^`@k5llV0F`)qM0z|7lVs0Ll^9?V1Ms1tNwrFJFH~_9s`9 +zZQs1CBctf1>yZP`X&LQ1MW(kRQ~e#v;eqOUCP@DRhP-Re9{>QcIiBn3o_*%V+h|pV +zJ<2)0Th*2Zd1WG>c?wQu?EF#vhnV$eg6FN8XY$r34!I-iz!x2xoq4i0J%ERhkXQzj +zYpG1N5i*DC{YZ7G)y158`mZ2w|&|FUh~Yks`q<`P^qakTM?CWAo$4n +z-}Bk9sN_Sd{7T5h;6;!&(oPI$9XZJLHwiV2X&%*{A1z#9V0xk4xOZgRy|~S{A#5et +z8#yT3d>k9Y3FTkM!FOfGKmkijj3FehU-22i(@rT$3uH|3KL) +zr(BYx_-(lJ=2~O<#3X-mwI+#gX=GrocQ^Ph_YVedNt7uL+_+3em=A!as8`CP?XsOkWj +z$Jj}d4sx>>x3Szsq+k%At{uzic?Ykt^2HdjxNUu6repHDLc;X>*XsH|mfxrm7Q%;R +zJ33N>6Zl0A2D)_qX{5o_u9!N=;d>Q4X|KBf9tFw<-lV-su0L+BPv<-xn9O(uoG<*1 +zS$||zQe=sg3MxGxw95#N9A-16A0BjUC1lcR${`Uu;;XELB}?5^8s?syCUn(JiyDT;ldM +zDw@%FObMoHHKQ!iPJMCBRXhw!H6Z*C6ikSycwzPb(e~Z{aCcq1(K{o0CqxO++bBaK +zNP-}eAbKZy??&%6dM{B@M2%iX9c3_r=zY}aMmNlv`+lDHyyqV{zkYpuX79DvUh7)d +zDqAhMy(t}#Y~%fz|M#mOCbznbi#PK6KfS5Lu0*@N7Itj1(kLA6MScbzn0)1$Q^s(G +z94B=W-Mbr?zYGXv2RrIq2w}8g*xVESc}YLVRgIb;&*Qey>QdJo#>)Xw8Ep5_hUo`q +zVgc3Q`zG6S7@FYEvhPdf6k<2pkRqko8rTOCDL8?lq#kuJDzfhhmN_1w>zYN0832^Et;6PH>e+qpDQY$3DJ4LvIX*9OlKclj +zR5W`_C$r_4Mao^}-gINw;yny+z4h*~IHdhmTa?W&u-mge+1m}W^kf)Y3Ff&xOzyXM +z=?kWbMZV~Bo!b*$mx;3xBN-rs2U{t`2u^n=%JZZ0dj-rt3DXjiM_iq)*7@XLsBmn& +zc1}ddakIH{WpQ&@c*layRw&tR@$@^}oylHA%ioQ$p@LB-O{)+O#_SiRoJgq&({j56SO*i?!cK@ah%CWP@QkjJIw6Zp}!yM +z{7cT%Aom)SPek!>=aAM!1cOcG-%^jflliq?UrT;n!y=#_heI96op6_EUc0BJ$`z +z%ysvV;=v&#fp-ePJxYKJ_u2@K%$Nl{4LC$q*Jh?nPQLa{6uwKVj&v$Sk&(Rt#|6ni(qer0mOS1Aw%)$Ux^>TCYc-Vs +zaLP8Si2?__u%e!-AFh9mW>eUtn2b8%TH2^Heb%c(O${MOj~!K@r?=Mv9&M2R(J7YV +zBjErr$09A>&D}<09wSyr{#hB}-)c!B@pL(sgQ=qBo4Pd$=vr`o=1Fd^ +zxxKCZk!;vVSYhx+;b05+8S?oQN8&65XMuv*WAb3F@pznW>Dd!0k-;k{O}q88K`zv$*TCnq~u{5`fw +zo+`vlqxi0-*DZWIO04NBC}@O!KH+b*z0N>21yM1K{(Z^!?u +zsx^vNw0A>=_bh+Jy(YoPBjPM+sfi=-p3siL${jy(#*?>qZ7V3&z=J)=P@nvmQ2B!c +zHiaT*@R5)mrVPG!=67`#VFID4lp$azdLRHjkT;ODy9c>xn;M^2m5Oag)wQUC2X`02D4Lsw +zmHUkF>z^3Y>q+sZRDjtJzv62Ca%Vx|O5|?X`3@^z0mXQ?n{*%Z<|y!0IAJvPad&X$ +zDyH=$5Gf0OmIS)Pd$lbEzHFPClX?b0jQo0G5ugd=mjBTi;Ad(yQ+wzpggHWiQR+^L&GD*ifGDnvQQhE>qDj?Ah*Y +za^}#W>-BugZ67H|V`7i7-|HRV@&?U&Z^pSBN{Is$UxE=TdFXpwUW1Z&AWC=RXS?!K +zo|WPulPj<-@&x_Ln;fEQl`ktEcqBBayR2Llpb+?*wtR4wM?C1v@yTgAWusu1*_XI& +zW;WW0p-FMoNjmgP`y@2UQ-zzAuWtLFogT$MeY3;_QTp>^r8H%fZDMNx{klov#-F&s +zc>+9eu5sZo3FSd&mj|suCiZ#I&qq5{4GTCJ70@CrYOO$LAs}=56bS7vjx0mJ;4eh+ +zYdE8UG>;c{;w~ON`IcX{h6ER#maUXu6QDyRK{vXeAh!~AK>n}1LMj)3wf}-{$u&^> +z;8NM7d(MVL6!RQVUi#y^M6$q*Q)QnwO{z_FABNTtLBYZn?%K*mVh4rBZ&gMp{0h2_OmD{L}gLg2g1 +zzc;^B3tyvEZ}xvc1j&Og5*0qLte`G1;}~?UZu<-Gci49LA;-Dz_Q_+iuqamvz$Zy( +z>DAyc(EaJmuE7f0{^p(nz5Ec6rpJ-!eO2b;ai7(Dj}y2}0PeL52%7{C3iY^BvISjB +zu}AkwCl(@afH(D!8v@3{k0pA_Br}&nW#o`!eu$7~==e#80-EVBQ}#oCg8vCl+ljH? +zJy5_y?bey`Tk3si4~!wtxxO;_8hF>H?t_lmzsDbTZ~i(vGYmmzf6RbrAPZ6cr)czk +zhqrs4YU9>5k)*kWwUf7+$Aj-$%f+it&34IlI&eCqN_ci&yggHSrk#OeSjybKWWnx{1n=7Y7IciZd!b6Z +ztDs7yLIO=x(2G*r_Kj**C7aH0yeM>N9i=Aw!*9y3Mu{tBkJ+MWD7vdpAJT&^aWh!& +zDbf;_LQcD3!v5Pl^i2mwXB25)nq4J6bR472Joa)4r39WMHf(Ay<{)3b?-I#vJyk`H +z?YSM|2Ru&wEgd9F%hn#}av;62?HljY`-;-9Ady}SZ^4VLFHf02h3<)>{*q}W6(X?z +zW;8nU-OHd!7-#`ZTkpzQ_%YAh=i040(K;6$w_B)P85 +z8|Ae<@4ey_89y@Zp77k^5jx0;)NXc(z(3c=wb{LpjJ81rCjR7(^Y8>K4VZ}F2f}(9t~^&B{3S?|z`u&%_a2=fIJ7WLxDjqY +zefv6NiqSbbc782O)IR3qUC|C8{ynBVZtK>>v^S&U>OC770_Wnd?L&DAE*qtkq9190LJ`1hp=Zk4 +z)XLhd%uf4sBK{o5si8Q1^l(Z}kqZUy(Ri0_fAZJy_Y +zJM6No+YLu`gIkW0{`zq#2zDUxr*WaTwik)3JY0OCS}#_)r#PYNDtlEpdNt-O1-5|? +zVKdI3Z-)~Ic;!_LmzI)(yUGaf#bCj#+C!SDo +zo9))6Z|wo@O-X-DR}LJT^@?~O2B>F?E(P%Q9xj;n_`x$yb_eLs +zHZLM2NW1WxPCxR@YOrm17zoHwPtXIa*nb%Dy>*Q~8TMvH(1cJ$EuOB6$);*;fAiAy +z@YJZjeSK4hnunKtU!;HnN&BFUR!0el}3 +z#|t;$MLI^A*ElXgF0jXvjfd&SorhNpIIaoffLB%Q+YfRfq9y_$_n +zoqEeJrFBCB13uRY?w`xeho~Y5Bay(Za@_E<8K(UicA1Qh65iMnUZZDpXPxHn+}Ive +zx5KYvlnUFwaQ0Kx=*8cgUaOOuxIEO6>fn!b#<6y24JlU^>edluvA?(rw!L=oAyKus +zPGL)CZU-d!B3F?`nC{K+O405Iu(gDU)>mUG +zTn{FYfFGliJ_caxG%j^l-{@jJaixaULCan+e +zCL&~6zmEADT*N!fsjNv%DAfKO2Te64nb2K*yM;Kx!rBk``&B=(#zuyxZzA_Cn1@zxv&XO7*X#*lhisQ)w!tSSe> +zZ#lsikiNou4UJwCLhzbEc(Ij{!sx5Q4VqsL+sj3c`VrX|+{u1Ybqc8l51f$jn7Fk+BW09fjClP1Gp%3pJmRt~ +zrX-9?m74gJiwsFCAmYLhsvpbldYnfEDAygw)gi80{#R4s5RYj>!rUgdt{y9<8v7M7Xz&&zS{q@_!5b-?u*)w6k~FcM{dEv5 +z^7pzda@YMsDQ-BE&%s&Qui)eEZM{Z6v7n_H9&l64$z4mhE7_jGWsEBry6C?~ETzqC +zWvpoURuRi=eTtS7p!I-W%D3R|QQxTM$U}U7_Wd_Rgg@$WafwOGc%L|np9lxUGM_P< +z%8B1M?aP@;nu@M4k9lKb%QNO}D|ViIT!p4oz>TD?S31xPe3@a~q#3`{veiad3AY88 +zx?$B2i;M7`T)Nj!BWl*hHQwUk{-cf8yT|n-K#-ux>f2xdE5%LUU{a5+3%ugj=k%-mUBOt&^m0K2v}`xv8p{=_muc#XCPE5)L;Ajj*A698QUpVP_4Q89z@|lg +zlTvfx8_9q4{Py9tSrf4XZcl`*VHyIyTMF9J-)SJPB)?cgK?3Llis)%gCCI(;6sd8< +zWBA=lbkb~OXXm4E#=dCK1WMLg4{-ef&14L)9st8m?&k81YF&m{|T$e?Y%f#L}mw$ieK +zCj|6Yg9tqqw)a60#p5&gYNlsqH+h@Z6ycNGRWX$4L`nc{wO$g_5jeK&ht#^%U4XSv +zXke5{vNTAvfRG@m!}da;NLT6o5+UNDP3JDZ&_+1@5kV42Xl7Z$4j8oV$@!-J?8cW9 +zP`U7nWX2gB?;H1l0js#qI$0mo{g@I}$(f09`HcYQe2Cq|a5bOS%VswOr^8&!N;FM2 +zc2DHfHXVPu%02{edCDf{kzSqK7yU-p_v(o9zv6k>+N}J`Dn9Py#&(koo +z4x(+H5u@gZfdrO9X8&t3oBLsHY0CtLJ>ydBN6wnK#h&6%tR2_i9fg +zAVro_pL}k%I1NFRFp7Yfmzyuo&aaYngqblQAZb$-9f_xUAFmt*QuI|Kxq)RNR#2|* +zG!%j*RPPggRXcmrDnQ+R(-gw~2v0;#A+t2ek4tcz-tU(TveaL!_B`WrJo^Y4?vQa9 +z%pBHa$-BjRSNL!MNV-}c7Jx>QJoLD|d@S5W*99gC*OL22qn3oiBcZqt-h>Avr0{E+ +zD%OyqGfZN&7vL5>7Zf7S1DSe_mq&V@`>InTGT|n-pRL1Qg3UO~9dGIP5>Y&n@k`;h +z2c#nkr;ksE3Dj7TfLMy|vuP$hAHyEd6=%lL9nN|J@;EmS@SQQkwEtcR@H+9iv7nT) +zKyS%YT`o4&-#pcX8k3h6*K3fH&2JFXF^*R=H<>9^Y|*GG{F#Hl;V@!SGk(_NIO44< +zO@p3QIYv2*;~O=I)>!Z|zuR)us}CX}j^h_$ABX}zjhmMPEG`qo(F#eylz4?lOq!{w +zc;L3TTt1p5*r6r7N!0S3?0OF$zvmTTv*mqK{mjg+;abHx@!A|q*1MDa5%15+ftnOV +z|6sMExOVL4QBSJ`T6Hz2mZc9Oq!I(q +zgri@w{uTDQlIGgL>nkHu*A&oS<~XXwj?@)eqsfI6oX{V^Qv=f8ll6?JPyvL#t4tZ{ +z+IO1du|*eut%Noo`36uT?>23pW#>7+EQ68zIWQ|gf-ta~debDK5iXs>a(xZ%A|zr3 +z*V0dM_U{OUsxCLNVuHdniQ?+`rva=Y^LJLqG-S^UU9gjWQz=44mrX^!5A%GnTs0(p +zIh{jR(lWI6zyx&KE<#P0sQhY4{OYskaRfgg4RXf-<|T4|^)p_G*fJiVR1nYShWPUt +z#O8)FMty{m0IkGoj%O)&8Je(@{5!l-73vAFrLZ?1#)p@RMF*WOn26&)CYwyIq<^&g +zCQFDCHLM%6YfxpO2Se^K{ZMjUqUL(7+M8;CW0DyE1^e!CLeR#GPn17CWz@bWLxu1@ +zk2pWxrt9Hf(;2_gXeT|%A^OTJP39XzEGKBeczC@*fqCwzDpL+uo%QB=qo7A-gKHRr +z6+CZV8DmF?CVtNiak!Sc7XX)r|;MxjipK?~FSITbI`ai*Ajv(?N` +zt&y`ga-{b#gm(b)Os8{A4>MeUL-Qhdhy0)5#Bj{f+yz-0J3zKQuyf#!G2L_Y;ydPVXB$2;-w|?>?C_veX~(kxuH~$74!c1! +z{aI`x%HpadcbJnv(e%~PZ3X?ogNg(ZJA6Ai6pHs +z4zr4Z5T(l0L4up~>}#)jsm? +z_soDS>9#?G?aK<#$)G+yn+?TIO!7lI(TTJe{!|ZXYCv6*XS?P<>_ysbcjxVr4c2I7 +zQ0Gd3`FQ(LzjOu1zEtj0sBo7+AR&8JX7UQr89XVMjNTn(u1FBN!?cn0g49-IKhQTq +zrF4g>ND6y^jG89YP0*Uu89GqYetlsN5FbPaEFPE~o^WZ2`EjU>%qG6-ipTJ)H*{VsnW&QEk1*YY_0;h_N)=Ei7hB!AAy}-NJQvX94StG +zMR;j{Kt|1jK(3#6*wo})&Os1LZr<>-OZU?+<30?egf2qdkQ2p0BzHn|3eTox=kra( +z=j3Ke3%NYNT7{@@yzv9WP>$>MiMTPSoIJ^%guKcp1+P!w!_4RZ05?`Xzqg7nduw*; +zpb$wzBt!mGt7QEva+3l_mbNeay`Ttqq#fDq$gKeCozB+z8wcT7e=oiK^@+TDM?75O +z2XjJYV|G-k@zJJ8n?CZ+0NIVEz~%b>&_wJW`C-+6#q8xmsz`w&MlNDOA<`H~O?b6j +zCdCf%0`w!glj-L;EC9FtJ7B|j52t4OkpvEtp5o?YPNtBA$&h#p9Jlrlfi}{LHklOK{Fq7Zk9l7LlsI%*u$GEl4JWAw3II^uA7pz%1VO% +zPM{)3u(^_nNiiysb7v?@NP!?^D&99Z7x3CIh$?nf>iXp?1PQbFertK_&7ijUku9{^m +zlw48An6GwZd{c3ti2=^EK!otXgasX4qpWiW{{h$Yi|{D<%$$IsZBp>CSjZzq6`y#x +zpZe^r#(evtZ1?8xrGXkBjZbpvuMPZA7S&?AQrq1EkFDJ{l(8fan(D4Wp4YwOW4dTZ5sIggq=6` +z^yTjb?9eYwGFJax+_hb}Uts+A|KJD;wg0uqXkz-xA6s%J*s`A)fXcmr +zD1rO;N+wKB^k; +z+N?CQjp<-!^^U3H<`S&hp6`{NvGr*-KUj~Co!&C} +zjkoZZ4Qua_{I2f6EfeKk`B`YIz0F~OuBAz}oMGJ_XYT&qBy}3;2mruL>2pWKrFK+Z +z{kC;|c@u4$ytG_5)M>1p23Cn3-lG&M4NkrryW0J4%#Eo_ETaa?ofj2E;BlvNm&2Z4Y?L^k2+vd}_-;KIaj44T?MkIuP +zLtIv}wqF#|U<{_?zyH`m04c?kq59vgir6wrVxeEwBTpV34y&1cnMM@`cG@>j&u{Uk +zUTk9&*QfCDT^m9|fx-Z4RB>eNQE(XbOtWu}BVO`^U76?d*LJs2`lKspHt1NLeIkMyagLW_S2_s~OD4M2u)~qK0bi +z=tZm}4aM+qm{c3evcUvog}ehf8ip4)#oY0Af4i5q_JWFCx=|9O3{h*NIwEfS;!|Yi +z0ok4tED0D(;^=nXt>dG#7siY4%2QW=)@>^l<(;ca=bW?@?kE!b!8lhn-u) +zB$!7M#25H{9`k&8W1r27_`jmJR{56Mj;RSs17`cBz#2I2k*1M-RTJ9#($MR&O^lv) +z5@faYsPbbIj$O6jjPU|GP%8`nLTayRzx%7Jn7U!M=lJ_(!Gmnyni=cQpyl_{7|3^e +zeEkk{SECr7f%&qLpD%W(Vc+=Bs2Ww-aGPQuUkdSfWRxsL{j%$bq7ed>7(K +z&&+e-fv_xDzP8req(6IweV7zcf{)8aw(iMT8=L9e-=7c1I&A*FenMce{wpF&&Z3qf +z;7vj8OTFlc@6Wgs&cg=Gw-(=vt`^v5fo!trPbcRmr0i}UaSC$9Ubj(x8GwslDh!&Y +zno;ViFppOdG@V0>%6LVcL??v_G7A8 +z5kyMVRBCsvP|I2W639Gdnc}G0cR8rNz$ckhDh{`^Vby~htt;~aJ +zW?a#+c`9Kwc9A;QDK-?ZAZCf(pfgjGs_=N=S31_>9piCz^XRnomb&GkQJZb01u^TyWfwx*Q2?m$z4U_Sig*(i) +z!_xJO3_`u`CvAuu@T?~aUN +zcJ-NzoVjYuKJk|mJ1pV>&)d4`ZFYf(BRM?!56JL2^UCV3D?Vgj7o%mOO~qrXNS%)o +zv!szyjF+mK}yk +z6^)>sigH=r=++o!LsKkla&E;;T1Rl?1qkJ-pV&un${WC2%eIBLp#DwV$oEmpJyzai%D1qwJoZ|9P;X;hb +zYLY$rn`Y;B(&lE*EYtm~;!MOqFukG5uyTM1`3L0SuK*S^E)P+Q(Th4*U_gQf3C#r{ +z4hSKpMlpl!1a30Y!d+|q+L6igPm|C+$9!*EZ&f`nbI*Qjr&Cu}ptCbEVS6E5 +z29|XIpN^s-(eoA{X;dn(VF5)4cXhfQN5?=dq60~833Q_luZyAxZ(k87$HRz>E&}rN +z3LWO}f6c+(78rY)Uro~^(C1Cdj(co##&c|9-Qd)!VlQpY)w90(0IIkUgm@vI69zXD@}QF{d%J@?TAyMg`fyFRR3if36Nk_6^=4QE}D4sPjAV9Mgyp0^%j=J=;8pQZhWKH#|Y={<@284hA +zHuO&XcH09yb#MG{jBKlZEF0{Vq3(C}Mmwmmc~ILxq9 +zpl&ji%;bqsyBqm!QYRsKC-7y!O^c3n(0alYd)r<&+vIaZ2#zCTos;nJnJ`?gP%=E5 +zmI*{~F8+o=lAz8@_@hmH0J+wrs|SnJ+8y(iTg~t@els8Ob$U92Lvd1{0#XKGf8YBB +zOQXp)SZ)}O`!o&#Lj8*bz2hS1z#BRX-&&yNm+b2eHv*ekQAfEkly&(i<7W-%;3}{@ +zA?%dWqVnqoE^Yw=wj9GR7*O6DanP=Cm1&0j$XFw2><3oQ9;(yl#zFahzTno;Ri{Yc +zxG#IwyO&J?J&Jz<9dIE+S45kVwT#tXGfsI?m|c-BEZdVQZspopqt5iqX2r)0TSV@v +zK-mQ4u+66iil)Z1_qOf&C#K0FJ3eGb?#qC&E)Q2R$4Xg0F>%u3kb%ZzTCfjU!jwPe +zO;J`T3{{h8q@S4Ukf5~{b%JEfkU-k+*?bd8YqO8CmpuF&w8m)ci}*}=W9%DdD4OSY +zYb2mwTMN7OS5vHkODv3ke%IE7&$k!tn7TPCS4ZlTht?79$c*k1uXXmnX_?^yA7z0O +z1J_&Bju^tS&FY!26>eWLo?s|i$D6m)3hr#>_@8bIq3rw2M^Ae7n;keG;r+qnL2^-@PhFQ&Ixnii!M&(u@2k?Y +zNl(4HIvN|HPM%$m^B9Z&MOq8`_#{&U(*5gq{oeW?ig=#xxa#~&q*P7dWy+g{Y#M0? +zUu~3+Il_sG6{$uUzUE-NWOF*Yy_UVbBJNu@*mc=87w4@GSszZ=>^WTr-rpUNRA?yl +zBB}N1C1GKhrezTNR3FJ|s(*4VAAowVQ!bD>x$R_n|Jjp!DL{3!05TEoyIv-?B_}^m +zE`Rqn>8Y{~b(#NF-ifcj+tK7{JcK`Rr+@19d9rs@AlOj!uJOG5UwE(?;**65Yg&WC +zMGUXK_fn%QkA22GF1*S+SaTZ>dha>mFh!>{kyzt^1MEaOWUMRQro%%5OOJ1f-1dEh +zIgcSLQc%s(qSvHwJJGLlj@C!OmnJnHID#hV+31ItIgUx0(S7c3Ye(AGrL_4_W9ph*5@rouQO6xA) +zLfJxheQYwRipojT;*`PWow9CG7bq`!yCHj-`7(6IwLIs%2fY-$OhhPNFl-;$BxV@L +zU#Zt~ML4Ov)>oDw35Gn&mNSbN4%V&3A00Hg9ns>&Ob&rvD*Q%_0qrluM8p +zlbfG=IVMgjYF4*zM=DJAIK7eFFke`Mn%IYp3|CiE(ve@o2$RkdM2-GxadY47f@Iep +ztCJ)+mnV7lNH0DF@BoCD1bFM(u-7=)YJF=H34~lH==rV$)Ip8-@cHrkeCgzv;gT%y-oTe-Akj8 +zTlj?CvY_76Ub+5Egjk^QDe}642M@XSHyRszfn5Ikw6hGaK +z=$Dbc*taiNu#}n2@ICw%#-q+se=sq-((DOw&4tUs8N4p|#pAs}o{RL*H;Stb)CeCT +zc&=9(7`6)LsFrS5$R2n1}IA@81e;vifoh~=~9B>-8ob!Efo+a7_rpYrD{ez;0<*y#&;*uxw(Ygrt +zkAANVUX^63ln`7Jud^L-@j4ry6v~}rCjSa-M#NKGK3O$Q3~Q1aGfaP0>`7@wFP>(r +zK6zCV+?6fW`la=}!RT2^k?RImUcep8+9TtX?sNC*JmviM@IzI_ny~wB>!MdUv5`1r +zB(kZ?+KM%-glY*SQYNNU7-qT8!C5xhi-x=7mCbgGNMcn7wG7)4s`O5F|WjOkrNMDNi`>-Sf>8!y3~G1KU|j +zC*fceR%j9)&?;&g@!tq_a7b +z;aqweSJd6@(k#$bmR>?5<%uZwK<)iycP9$Vy)HF-r8LiQttD=-yg_Qa^0NeB9I +z#&a2XfQQSNBp_r!_d&o+Uc{+&z`cEBUJsNuBis`29^HH6{E(2Gx3i6JWX7Y*XI6i +zk0`)8mBq6|Wb#KeLVIgXLhWXxMvR*tON9{2pV?$Kw>I*1&K68XYr}3!;EmF`w*Cf3 +z?}E^fAMP2ly@-eS7iDtb{kfThV&_b0mkcnwq_cvp^C-4ON(*A +zObnejTj}ra_gP6v{jYNzp}BF6S<4R`yd5s%;erDUkL}tlUJTgQ&CBt_JACkE3;k3k +z^qrTnqDqZT0#n7J=n`gYOm|zS3h)}Bv)iPP5LU~4LBID#h++av_GghE1}#sGYkw2M +z9A*}EB4$qKRy#(H))3XEN5$KtXaO8zY{TJ3pMF||S*_`=iVnBsUaD%m^je0a7IX17 +zex^fB9v^*PhtI;4d~ZcEGPGS4lX5egz$CZ%Vkcyefffq=36Z;-9w)6dl!8krYNkdu +z;iJVW=(~TIvy#Oyljn*qS{%#cH)#yb^Ag+BSF5CqHa^l?<@S7#i7tl6s?5b#O&LnaE3^9cQV?_4KPie!hnZcupP)8PtN4Hlxz3ZMQa41Op0b@B@~ww +zHJT#I=OA^9LA!0%S +zv9Yr@KgV*!%l9MPuD#fV^2uk&g#rt3dyohAVXEQSyyFi`=G +z|D8!?@FZud3Z0MC!Nv8?gM;=mo|9){YumbR%)wAc*CD5GE(eF4CvhYO7v^dXf$ +ze2iu*<7^5P(oczN1{OyY7UHM~uNEh3tOCv?rEeBy)bFedZBXJaeC;w=YZDDg@h`g= +z+==nq6YVxV4ZN5|jj{Gs=P2sgO6e85M;zIk{W@p>v(rX*^@U~Hqd{4lz5cfE>O1p; +z6;Hxgao(V6jB%WqutGKmZGsCJsmYq +z{2k1po-ZlL@3$(ss;!tB{F$c^UBBrxOq~1Szpd?XCWkKpW=lxs>XpX5uaxYD*);)) +zh0<0R&P94=nZlUpGxQq`10TK^+1z>MKQ18X)311q<+Saq;RAS$1zL>vyT`4$XJ_%Z +ze^^T1%hgOez(oTQA@h`*bxzoaF;iDEAVJ207fUZvs89@F$^@So;C}o{_ibx|+t$qC +zFC&`^g(4r*OgwRhu)M4taM!Q#EH^P7X3MDcbZh)8)aGSZAhC==?%wKZflA2T +z_YjE-41{+z9XiH{q&L~FtaxbdC1Vk3SR^f(Kn9M~!FrBS!JeQo0omBdoRcg}wts}T +z!X!BlufTC<%*R)YKI2@`cGqQ*p~%a6`B1;B&J~{v7L6}Cvx=0FTHRtVgwj@zQXAW* +zSs#j}IF?QpndT0OzYxR2<-wKSxm&}_E@On9F-~d +z+*mK?6NE_Q?IrA(XJ*g-vL?6QX&e>T;dR+mNeO%O0PVd@DL}Wi(n!}Pg#({7DzI{Z +z^~7?IKqRagE**57FDE-$!+#OlqN2?t&Rj-vJ!q!Tt$nsx2WRJ +zq$7Rf=yc?EI_1&SSl!*oN(KAAiU_>tUy=_nWo;kM%oT*^>4z$DrUR{SU0DeOs@mc* +zw{2hOD4G^-%}3drOdD9m&xHk_AHKlCh+xhG+7F7mTe8^KQF(}vgTi(%5LUW*Ah3)sfR@z@emU(!ZB4f29h&I +zM*Nt_*AnC|vYT7}TA^k87|C0`$@*L_Gib&-j_`jh?OVmwF5#!_q;^yQ+r9-&S#pWJ +zuO=YRq2{JjM)E=F)w0_s1YSagemoKgdxC4`9ak-Nr92#2LC@xZJ1T}QFGzUErMnjR +z?*YHHWb7_Gb37#AZ%^P){qo)sYI +zxmpym*$${f?2(RHi;D%QFRpbGi?ixo +zRBCQF!rk0to9a7&6u6C;DO#R`t(I5f!Pg5;;j@7ZZh#&S(G`y&>-Mw0RnjUk2PnqE +z3)lB{90tXbiMMB!|GoLmZREs;6^oD3C{FNHG=a^9UAPXQ#^2iGF}r1Cp=8ZZZT*=a +z*w1qk3#8O>_M%G27X%lwmvE;JyPx5t9Yk1lU%<9=v3oisrw~9tb{8T{i*hoia&Aqn +z+8vxV6VKjjMQbCp+(X4wC?3eNAU%U-y*uECCS<~yGudmBO4rn}Ld^MZ85@cG+~>#~ +zqZqNw#zJ}2O-62cvs_!~jZT(Sa>X+;D;F(Ez1s}!BX@Cp1Lpod{Z?kDG^!q09^AE^ +zJ;3)s>)1ujwfUXt?O}!NMnm8n#<3m|96E|QpU6yRH0v{7+MBY$B>OgrvzbI?a6)#s +zZbO-LSmiECC`soIcT#HH4!lym1glrMA&W3}`X9pn&kQ^7Q@%C1l=S#0YoSb;Y|xmY +zV9!ZzGt?1_>N*kwFvXW(Pb +zxiK3OXS6O(t>N~bz6W;>rUOXv>1N9i!HITfR9cH;PgEH6lQSV9rrcb$wwG)X!+IoG +zJ-6s~{JPVQ-^^G3lE1!pePB?%uRvp%Xi>q5vX%X}~4T_KK|78`c +ztxlD2HP*8(vK99HAuN8cf8fn05hxujB2u^2(1D>IrROCW#JDdACk-w# +zaBp%X3)I80_U8TAYn&?SNRfYWahH41`rLwnhkKNB_0Nn>*mn=futJWF8S|*EC*Vr~ +zI;#5>yz@1Qv&;JT7xmyq^474mI_7H{{Ho2Pk+0TZsGGIIfW##=1vwjd=$BoF2wdk~ +z5%24;4D$z>R4z&?PSR>ZLKn_w9%d#Z;&m&Vr#ON%2xN-|Tx6HTIZJ5kn4e_P=XY7s +zxNUh97TI59=8@?_RWCHUJop#|qvpg{V>On-#(_oL{?Sr2y?RPgD7*_ojyJnQ +z+6Ouf|Fp<2g%zwrbI97%A!FV+lLL3uS!{w9?mZ+@I|n1dHZ8|M;Q57>j$nIciO6Ny +zDlmf$7Vl51Raw;ayf(x<-XIoJ`Ra;HG-bRh&^zgqq@svmBDTPg^7cr@=9Pezp_1xXO@LUQ(m;KERj%W1qYk8`eE?MuzsNNG{hk8{^)ZL#cLx5Hqhv=*%YkzM*ZZ?U$9Y5+*WA +z{}tl6bht$Nt!=;vR_6+_bEvJN2ch93-dpH)csisFUbYa#Kkvkzl}&eV-8|+W8fm(v +zOW)f2hp%5~OIFtL@;ikw3^~HI;Iajb+#F^X1Fa9|&0a|N5OU7V$Q)%BCK#9}`;ljx +zv>BFITw5;>oFRQ1&bx08X0820S?K_8St-0;g)JSGvWdvbL-^G5%~0x{oP;vT~H5ASu%n=4OrIgzCAc+mX?Kw`}09n=o5Ir9ys2?z;>3&u4f8rWaGU+a9mi3V|Hv<+LhY4{ +zoD28|n>Fjm{Om>_N=VPRZHA^xm?53!mS^_vTEp%=q}xZ+gVyPycHd=U(|UyplSX58 +zw}Z<;uN)UqOP%P$Cr--RL0I#+hx4}&AI_@&lO~UyAd}}~-?(}{cpjC)>$=jqr8pWW +zyF^}N>+{rP@)HQ%;4X==PK8SJN_>kiEF-ssDri4N*`6>ImM>7kqdABX*U4^+I|={9QQj;h +zbF4=j)>PLnKQJO9bmfBi%`6!%jD&p&{Xew5Wn5d!7dA?9Eef3KXG|$+l7*(n}czm`zFWNVi|CjwoOJUTX^hNgmyNPS@8}N>XkGjGi*-G9{y}0Ho +zlxpT9W1k|0fr&qgV?@QB&p&>iz?Ne&k^S!a1~a=p-P!Y_t&Mbbukd`HLhlVVJzRN6 +z9{(SwqA|RuyUTzIPko7?M5sxWT?$m0)9YJQikda*SCCxtY$$OYa?J({m^$JOJch*) +zXDkf8-F|-M=e3ta0}YXhEui(I8k~weylE<0VtHU+VgG@5w68y?)VolYBB7M8p<^PD +z`R_N*{3lTc-%3q(kx;%XACu`~Mc5zBE04}!Xw=07q+8bM=|IJ2>TMsuuEMNaS1KR6 +zM{OV#p;`YhI7CDPt<=ccLWBeUAn-Kf>W*>#ANGcq#*|;PN3F*Q98N-}FiV8tgDjCZ +zcPHeh)bhIgMh^?wi7*pB!Y{x6_~!=0oyZ0oj=#!(IKOk=bMqf@2*W;OeWFwg>DD6u +zHNnFyj(=2?@h}(j-l+&z)7L2h`_~+cQ*DYoLllEpK^(q5-3Zul7|W}yP)ypEw*Dsf +zw{g=s#@Hc!w)Nd^mjCEBn46)J7Aq^W^u{psnJ~&6J5zrfI-~x5=KsHzv&Qgr{s5Hs)>SF2kt6fm>>sM!Kci)Cd+-Vp>}9)_ +z|7$bt9VY2b`tsvnwgjSEd=R0iqcl@O&d17tBg`pGjRFj6G +z%E$3sWB=~2WX%53)jUjkmnuaitj_zgP}y+K^M7oqecTwO%1t-{hk}NEKwYE{4UNfL +zZ+HFpzlIBjTy=`A(oabEK{btr?U#nl;N5?dH6w@A`hlKMoc9iS$6SUFjUgP;$>1#w +zr!e}5+`v_Ud8NQ}CBh-?dRe!>h>j5}+mhST_Pt}|6$IR%RYa~9sDC5s!f6<*X2|8! +zS@LACjXd@LGvla+O>+8oofLyliep5x9BfJNzTGW~_L++ZYy}{pTCdU!|CJD*mV%l2 +z$YLiuZlYV|ih$i?-@rTW^__so?YQQJ_dlZU-zO#KVvhD`{v> +zVJ;>pK17O*M0>bM=#DGw08ZE>p9GH&Osz^<2)R{P93G>UBwjzXkX1M?wiZcLk}mei +z+w|^goY}Vv5%MhYa(J1@b$l}nxmtX71rOm%E6M?qP5F6*6qN&+)J3wLt9xQplH=%y0YGzr%DZgO_wmE&!s5Ei675chNab_zU~_& +zJ~p%?%zniWiaCFZ$aCTO?4?SX>&X|wOwPl#Y!l5@oW4fVzm-omRIXDE*SQ(V3uyvB +z(H=iZo%(nfWpo8<%4t5;m4ci!pMd}~UGB*J5b@g&?wU^nU%#>6XxKiSnXlH%_ieBL +zhcXycpnK2eSNb6xF~Shh(x%(-mDLtLHI2c*~Eph2G##0B=_ +zadq?@yS)5g3cK_c!zs%{ytLNLn{E19C(uFNEq%mqHZVLHl>?>Di75DC#X2Yf +z3MMxmQ5ZL&s#9XqrWXBUDoaccv5J`Gp|cAdbLuY**~z~%B5J{UL-s6R(U}6}JviZ* +zI~ZIcUm&_QDhdfytgsT<9@uT@|MA-NK!(p*OrtY;B%rJ$@kd?!h$}D6<8}QaX|><^ +z?w*DtX1vCa-cV;tKFIJhaKFZ_$xZ@VvV!fXTI6e#742w5e=+mxuJjqriFT8kPKbZE +z6fpj0amO|~71B}g-X~=g9|@u#MvjEqU@E$KnfKY>cfQAVdOPY}o~%H%{nsz#tA0X6 +zX#Y9&m*1#FrgL;$=)mwL!8#ExCB`UL6_KLMQ*Hssh+(A#z3MsI)P1E!D$0;Y6hOQM +z7G-xD-%3#4f+v(52&;$HQpm6Yz8@{8+(&@u**}nLPphL{V2^*)+bRPegP7gMQg7KHil2>-=Wl1iTa}oC(8G} +zH&JRKUW5fUqe5P6?~?00C_o6;JCj6nqKUVX*hUb$e>~Nah4nSAJB4m-`-g~cksCAk +z@~2y3NO>F4Sgy(U*(svjeHy+A1pVJHwF1xppGcb#v*B#LaQtT?*!p)^&nKA@fy*+H +z8zC@p1@5nS@8=ed0aeo|lq)93_X|Nls +zFsuc7@pfTn@(|MSOML_v9bhMQ(bC*AKa^%r3~07(yym>4X;#S$gJi$1w7Akz(0mb= +z34DZ;YZbrEvtPe$;yfy>Ig=+qxW0I}SR1^|^av9F2%@Ak%G1;M6qA31(EC{fIX8Q? +zkreBGYos(n&mqBm1pK4Zv)cfl4Y=f#=9rT38)t;;ptXsqkvq#r$PK7VA@Cc1HzxPy +z4h|O`FPM!s53&>iXRcO59U0#o_3>YgkwjNi>7Yv`!CWzHA>h|Me$~Zz-8}C`txC3p +z=Y+t4nUlU~qM^{w{BHIY_U<536N(#`TQmDTF6m{*^M8zCVHtHL(5ch>LzoIc2q-_p5Bc@UAQ +zqQq&82WE91WtC#e5X)%Z{|I8=&vO))GRD^j0-LKn22G^=*eP?rsxIw3&x2sJ-tqUJP^J2n!K<~z+SS4ri%n+8Sgxab0mqlxSoDB|N% +zTCFY0<)qjrlGKkqzmV7sjN6z_i{bV$45Sms^oP@InkNry*&T{BchO|U;P~YPC7XV +z6OE{lPEYsY+a9E2E=v~JZtjfSMzFjv3%74ztWz58?)pF)XoR+W +zbO*3F!MR83%&S`g=44<@T%3VqpMe7%aEce2v92sd|EPNZ$l($JAApRFb4t6*(>6vv +z9cT*_$WWa+ir-RxUIidjngFJTQ3IQC8v}C=sNLl^rVB=K>RGY22d&&{{qa?w1^RR! +z-Hl`s*iJ^XIekjB8|g3v{50^u$L%6?4BX5Y>tOi0lhV^#k^_*cB%GMQN#U1KFf8N` +zl9Z@TBDO_t4x9%#*onW?cy7JC1P3e+Bhom4l(aY^v7#IYBKBTo(H6YhQUL0~oECmy +z^Qz%Fo~hzT%2UV3CC$HcA(|V)X>qV4xvF0cK@1%$@3ZwuQ=Y}-1Kl~b$8^41k-C9v +zu{apCx6$kQKs&&fvGQ?jE|?l01r^TgrD1$sx5~cOb#+rtc-ljbz%*^t8LRcp38$wJ +zxFSY0Bp7p{s=$ND!VdFnI$W>aKVpSpVO*uQzXr>XQuSkiU4p3kW|$s&>jhxely~11 +zDnDZNMm^!j!&8&^(MVSG?Tq9fbJ2kv{ez~@UkFdj(Me-TiwG$rL`&(wPR*CfA3MUM +zC@zt%HuK^I;$#*!xG*51uFQ4(k3qMSC~;CIvhTb~y74uxkHRg*d7SqSlCMyq>&Z1f +zi_4))t=jLY!yE3+yKkqsxl@(BrH`USDqj57kH!zJ%MJu^yxTB+930&US9Ssbnx;PX +zAEgb6_?SQ7mUw0jg^7*adFcY4@l3hnw#{_m&(GO?js_p}aH5VT83c+%DM)z82RMm< +zOhU>M8U2OulutFUAWEVVu|ymUqBc%tTO|N^0Of}?#)ym=ZinRRTy&Z8HGQB%)R_n9 +zo#UtWE-j3HedX-S&DwfPw&zVDT&-66or?d98s=I)y;uJvCeUp2z-n=EyZc4k1EssV +z+VECa7E*y7-47tkzmlCNqA1S>4A+v@&W_uVr{~M9!C?xDU0sxS@b*sWMsL0iGjZ>IBh3KZ3BTqblG2N@bBJM741(Day*_#ug0hi}`yrjFf(`VeM +zv`hbuT1_Ks3etPOUt$*Jz=tp1X_er`3t-Ce3Wv# +zS?qL)%(j}zUM(G&F#1@9+-c5UUDiYHEvEpa8kK*0Vn54}?&_>1ScK$1+NM^K-yhhX +zf4(u>LS9zgIv8+Ipl~kwq~-*gx7)H#Zql;Z_qqx&$Epz|9UuJz-h??qTk=|TZryuM +zIMLmq_4HLq1r=P5eq+D3XgAGRuP4I +z%I6W^VfU@Rg09woS_m)~0rNXP>FHiwP6V#}js=F*;`s}E^b8M1aD#ESAKaiz&AVN` +zjaVs{(9f@!rlhUN$4A6G8zxb&d$q$F$o;mv3VCqtx1=ze$o57OjF;J6FjX@smFKUG +zZXRyv*4qGv>yc=32!!fuu_x+RR}L*W?v$wf{x+4RW>ZR0VdQre2Ar$Oj|X>mvfSOy +zSvmvTVG|Py$cs)3=^H46GZ2sP#_rd5v3uYkuy9v~1Zw*@EF?4#$W7*=B%~0-*_mf) +z5zit#{Q7*(BLse|?5v|%>#618WEDKk_u;>x_(En@wc#nk4IFJl-Mm>>V!0 +z!XlSoZ$F-DMgGgUNN%vdm08kaO;c$}BM2J5zIpo9>+}aVXMZi(&?fTFAF+G9dK)lHf1Roi3z65?}&DIICTx|d)m)d>e6;S1*8qln%>Jt=oIZPT8ssF}grG+wq +ztc0tvMmja(vJ_>ay6PXCbHxB&QV*R7RrfZTaNYUdOQT&&5;v>!yrDTeyaY5?GbG+2 +z|FF{O0@(1aRgzmNR)XKqt|&B9Rg_ThC3YZT6HNi9pSFbe)RjvCcwC}xFJx)J!Ja%C +z64?oc&l92bF0STNOsOo#e+GtQRm?-ZiJlz4O~@5hN6s$W0mM4)p`A=gXDV6|2R_EM +zu_E96Si9Q4(VrBDV8aRXq{=Vjt*>6)7pe@W7%-uKzB$LZX!y)Bk?onf5!lQd-S5(H +z#b)tcnE7qW$?=~I35im6=7Ig&v3`I;V(13|yz%tdu)BWaUS@dOS(>?r1_H2_J( +z(Z#vBjoc~XsdJv-Jv+GrTzI$)t|}_Tzb(Umi+Gf(0h`|Ef&Y#O1jXKzedte`5^{r% +z@a&FW0jPSVtATPXR;x=XHcjWBbk$&!bw7~@2W;8e3vU%TeJ6%nWxIIS@?`8E~KlI +zQTW6?WA`)AWvm0aVaxx-4Z^Xz35um^ePhej)zM{Ayew->AMJhB@^v76l^;&u!;)-$ +zQ?G?wg~+E&tuCcnd>t&S`#>n8e4Lf3mXNJnAA;rQ=`lz?e+jzHDXY{aJ)N=L@+QiN +zzPzq`91t`W@^rq>g#@ +z!n9ruKU?vS^TPz7#bRo24pmqno9#9pRTvu|XEn>QTi-jp3D}M%MM9AGi{DVWDIsN6 +z8l}%?M61hA1MR|Y8hYMD8gn@t1!KYRgWihaJajMhq4|>0U%w~D3ZNlqT~+SjJ&I6L +zR#u?+r{huQqd4jZObX-X^0-^rP_K>YZ7*E?P6t><$LuC>9? +zd^s0Z#V#Z#!`H)CJ)@4IJ4D)w!sD(? +zk_%tq&T7)zT%0kC7a1lT3JXA&XWI>F^ho+}MjU=v_6OG^`yPxy^spoxz-ziqO>D8+o3!bQka +z)c~7CiLBj7Y!(%DQPlSG!(ZrrwKMVkG;5fM)?GOfNc)<@=Z?p5Zf%1^(5c8Ise4H= +zxv%Q=+IAS3ZBGtQfB3g%ypNX?8e>0##@`yIB=PL7Rrf=5>4r(Z0otf`6J_*}@JVH3 +zx?GM-s|1QmF_>m;$hAla_zn4JW=z1%UPuGm+Fp<_`V4(Pnc*4#_AdRVznq^ZAPKz^ +z5*>A~@a9Mtqrb`6&NN?x9@dAL2fmsI+bClY?Zxq;{KL==L)=rG06R@_#Xz&sRkvZ2 +zaGjy&Ya9GyoNl>R6l2_y>rt=e44xUqY?ShbbzIT-2|zfv=~a{vh|%h|NZ3)64_Fb~ +z2c49}=ym-|!p-Vecz2gSP5a!F7#`lK+{Vs>SJ=R%U$NBfJfIur{*KRm-nT!DmgGAv +zf6GZ}CvRRS@ObU?vqwj&59cB0Cz(>)cl}C@iVayVYe8f_^@pp(gj9FiJmo3w(1p#f +zF!*8teEFw){^QRKZ5blAe~@Qb}90(DAaUO%m_g* +zx&8VAhP@PK@%yO&3#H86&+QcHHvz&o?AYTu}(|xExKQ#;W +zSPLDyK36d2b(GQ+Kf}TMu4f|a(EDKF3WjF~3plOqvL{@TYrvr$i5Qk*l%lh%+j?us +z1xEXlcXc%}W)--ui;p>JZ!-%wS2FWl$tGul9;k$Qtc*gZTCa0Sk<_AFgH-W}HkRh2MYsGEIm*bq +zJim?o^QT;?6dg7|%SAbolg=*M1U162u&Quj%3}mGA4M6va~CcJ-Db*=Z5=P>SWCX^ +z@K#E`c-(q^Ad7lNT3O&(t;+czuhogw6PhqbYr7YePJgzs_uCtZ*bow(Y`td(;DKnb +zg5}=arAW=O2@EQivPOE=ctS6lX1PL%0tR^hoTRY{YeMN%Iq8IZPK4Zgr`=yq&-36r +zi6lxD7p9Ou@l-WVK8aKV+zH{YUFJz`PVyJPR=x&v{aKdwc9J3HIi5~@spfjdnI`A0x7OXwN0arT#T(diyZ-tK3(LUxjy5JJMP+g?F;i>g +zPBz5fnbD)W9#2Z&ofgMv^!1$>jn>0{Y+KG8omcRRC9c~pPKh7^Ff#SL0bd!II<5Z`JdhJm& +z=rTiU;Ng@e@SzO{9nH=ZAt5CSw&5et)(t%NxuHnH_qII@ZS@O^iMG>Xx>rbx9lEw4 +zGh?ZXtaC$-TWO>ML+uslo-p0>7&yplZV#UAYz_ExFHj0QUI)1<(?A2vK;3R3f>0Yx +zA4M92wNFjBufEpkZseKdt;c0QZ39SADiNZ14VP!wDWEBWZ)NZS2 +zlhY7voz;HN9-EUyut{Jt$3Z&*x^a_ePr&_RU_zVqZ8;oo7QgrKL)JL!9hykcb)QMvYi(`0Bs<(X3Fj<= +z;)yW_lT>Ysx32)j-(M`&E(i&o*+^8pCKaLu$27}IH<$y=73CXTXu4i|(7Kf7z=ElN +z-E28^gjkcg2pkhzH=mSU`lO|gN2jOLr;UY&9{To&GMASrb_Br`*}jk;kk%jXP8Q)t +z8vdAnWng0QXe-j}a|TxQbc??U3T38meNJnK@K5W?8!IF4NG+Clp+eUciCNS3?WIZAQ*VqB@TS_{00m1~(3l;|ysz +zAC=i@dKB?&^Kd5VFoYqXeF3)9B9HEbR3bwH!j`2wynVR%UH~&l(Wn7}I*)0ufOcH2 +zTV`Db^OV4+v8U-o$283C4cx(rKCEE4YX0dVk@clm5s_DvLTp5wC0^WD_LREYMq5e-GLW%v+F +zVQj=Q4)QALP_$HEzrRlzwKuA*bq8o23vtN}Lo6(`w)!W$8zN}UdG9emuNvx3_T}rI +zueh0_nDl$+r_lRYC7Nf`)Ha2re^ +zLR*51!)PvTt#@C+@R%=db#nk-B_87|tg!^iysRPyzX=nZ6L=7v%O@4)1y?rzB}$?- +zwpCLtmH_E!z4zKtLL0pV(hjZ25;b;3qYn1YHqdXS(LbolJG{NjbRf@gTC&s3R%W{< +zoS96eXmJV{yvWr*dj{0GFR-HaW?U>vje8$HmshNO3uFRwol~See=k6x^mc1`Q^$H{ +z&jEGoFSUUdRahgtsZb!mSoQ>7enZA@Xta28OfDyeiX9!;XhasX5S{CD>A8^5oa~xK +z=z%&8EYIXQDEL$h$(HS1V3<%=A{Nck-H5Mqa!6qH0&6{-K$s~2Yp8ibgOx)08FwO! +zl|_oCLu&TLdeppCY}TT~#?s36w-{H>H6+k$ttTSt68j+$VKIgOzSg@6NMSgUx9eIo +z2P+06dV1S?-6<(lI9qKB1N`W^E}=%cjM>n$FfwEk$)T6;7<+%mqc`lW;91;`04K69 +zuV7C~w>FdIE{|!knJ)Tk9acn8eFX*H?(-oQN$4^9y!DN)?b>K2{K!XaDu!p22;}W+ +z9oW&V&i7}sU@O)#IYFBV*jYFAW{lQq!8IJz%@-q2+$IRqAZBW62a +ztd^)O$PqSJF(BRaT4tjV>VOfd8_>>L_=viscOLYl{6P6p`RrEV{ +z+7MEQ^OMAGIu-8(I;u>3e!8K{Cq9frbza^DKr5PH$>iQc%{(F^smPJf?yuo9pJM<^ +zj}r_hnp9SHY*zbs_d6J(;quV(rq0lRvLdVH*i}fTir7 +zz3QgcAZunn&-|49A9^p0<4dQYXVXz||JB9l$CsDT)`*n7`E~|2Hjf4#$@L)K76+8sP+L^Lmu;x^`kMd^*km$GS$m6yOfbbic&sO-WU}v@q_oA5hdF;Bl +zJT3Y|Z5HHg56||R6WS1Ctb%vKB=0atsrZe|J=qbB$gGx$>{ +zZ=*IBqt^&J7m!ci;ZN3UMlQvQMy@X(lovi<|GnP!Dbewip16#gd4OCous7bidoP6F +z)T$Lj5^I6?tj3dcjKG@B0@=;OsF60W3b;P|);F_fkHg>x8ceH-B&ud#MA|%~e*E+) +zT$Ew)_J|v{r#R-NTb82&-mKsY6iml^-xf^Q1e@T7*d#M9hu__#=mj7nRz4-Si#nTf +zpf7&gNuQ4UMwiPg%%H3PScOLw?6L{L38d4<6A4RL{gI#4zHjzI7gS+VtDy`!4iftw +z#G&m{hkg~_pCMo4=r<1F3jnSopX2yqZfa(_07As_yE3j^4C)Mjzm#71oxSiTnyxE1zwkf(koqNxJW~JSz&VY87FZmpD5F_ +ze(I%$V-N<1;Dn|{Rf)BAzzpKC?6v(%4E5){3AulG=zX!p?t8*DfBPsJ$HQ$^gyiFe +zr;5=q-NaxFWjG~-sinDq7?vN(>5haklL6JUA>WLTDvik +zTo2BJD9s-P1vnuDx1{Na*V||}Iah|@ovvTu2m9)+eqPK%M=#@C*z`qZAIJ;fo^FM7 +z1p3_zLk#kv+9-Z#g|@S8C9w12ZY<>3!K5xuKhhD|yo;mvQ{bB#TDu$&dGF}o;e%j9 +zE%Up=NZL-mr2mY-5hDz+{X}XOu)W7&+Ty~**Aa$$OV=zfd5`|SkR~5AbgxY1`M31} +z4)S$JllireDup!`rU`7S6#}7|ifo3=%H6r#DA<4_tVoCPyQu}9iAdEVD7GTRUr~m9 +z$j?*Bd|2D&>graDbnhqi{1c&}DNC~?2t9qs^ +zv#Aiy?w_+-`dXPQwqP4C|ehtHbvtaX#L-CXe(c)uEAS5siqeIP~Z=|iL{8yTuU +zF>p{pUD2rq@8mz623vEOuZSf-c-x;=EuYAu^V?FPh18Qd@1ovc6l&BxZH;|wO&Ar_ +zxK?BNJ638EnEFlILvX}h#O@&aB&AFTKCwvx`~B8F^G*;sHP3KF$j2{FRcbVfggpdQJO*=IU-YC| +zEpP{>`{jL!9uvFzim?j@GVGDtW3F4*e5muWf^v7o*pi79qf5hOKiKdoxy}4e+#4Ou +zYes9NuYuF9=hpkKfZL@}H?-s$41Hrb512*${2sY8&S5I=NzvNRMnh`5J4+2a8e{xk +z_4)a!R0r6zBkaHiCAD*sbKV+ot$cQ1QE{acBQM?MDpL%0;bK=F>(}4?cwlaTAtr1V +zI^V*g_`kyZg=+>^JFcZCb*(=P%#8cGJysZ`;LZBe%%rqJ +zHkl;YVJ)wwazNNJ5MORy=O(k)UY#`}BkE?OfS$`85r#TTA^X+oN;mTuQ+ezxgQ7oe +zEA)1!_XbYCo44V)YRVtl%SY;zd1*2_N1wn>;fIThK?mv9pCV%>6P``dfT;4rBqKqV4r4|QeDNZ|;SJ>uXdITq_ +z4u-2f@Re!Tua53vt?H2`7vs9LN#fIPTAJeIPBb=(vPpRIQjwRbq +zI2d(dlN9CJvtTDema(1fITU-2C_>VAwIVwV*@jhUGV5^h&z+&^VVT|0{rvrX)YUU@ +z=QN0F#MI*=mwZIZaBCGUk|^}r`W_gAem|WC#GZZXGYC}Vn?I@JTPONSoKWsEdBkV8 +zw_+?9m|eep;Y-yLho-FLGp;E6G?I@v9l1jx%ICqipP!I$P3weolNK%48 +zKA?%%Ms>kDlDC*LJo;sieunENHZ&78A>-rN_g>CMtCijtVymmRX4(hLTBWgJAT4wJ +z1JL}d4yHaX>KUV?J)SdzPgHT3fTj)Q+!cY{l?E% +zM)s*n|0}Oy(I;9Tew!gEZ_fFTf}K&u2n^@RZjNm`(umg8ds7HN=!5o +zDM+M==s9AT)w=#7vYnNERvTgqfB96=x0yc8#Iuyg|Zq#Z{OW&$-y&Y36r3sBJzzc&dEL +zGew?}(y%}&b}F#_2G=NwChb%T4;#mudD5nuKsNRsoxrbS+*qM(pr_J%aFlrTK3AAGq%(SC%5A +zp~Vu=uH@<{e(K1bOjpB%J2OX?{svcvK5&{zg1BRi4E1>9O-yw5kT&py`zlk~7LjZf +zZ%5D1ewD?krS65>L1&?1TmBgPv{}NIz +zpkmVmTF)&kwgzqJf(q9Ec&YSV)(c1{m27J+1zhE5Qhzg#-7bUsIN~LZIr5@lOk+H` +z{4CeIFp~lVQ2_!zb3!nd3pZQ19-}fRb}H<3K${~{157O!{Ku-nHT{Nc?gLyfANTGp +z!ScX#l*WEQ-7V}0B^t7mmMb`J6|IwScH6Ol8mPZ2<25b>b_vrYbx=v-;WoR=%5UH7 +zd>|)eTM@M?lD8uA;Bebg!)ApaSLnB`q5JT0JqU9f6|#4(C*d5iH>F!m-;&=FDCCDTb&w +zB;3|NJKz9dd77-20JZIUB3?>3#YQE!BfXZ37M*mNZoMm^y+N2}WGo;C5JypJ@czlL +zGLC~DOHzjF%}5UflibyLYVxRUmsB3&Xj)NBepL{i{ez)b1+m3t!@j@i_nHn5pG{%9 +zb|*%BX`J5wzSocdn<$%tBc6-I1Sa9wP$a!p1tB0v41#L?ex?+k&NR}Q*=%57o$*~g +zyF48W-et6{8Chh7_GQ}l6a;7T4SIfrln<3d0wwp!on*xHXr6WG0;>R90OIG2k^0zH +zVz&b1JIe82y6LkAF9OoA^86T=z?jAK8;uwZ>>9b51-^4Z>qI +zZ>R%JAXKiQ|1KLlN1x*rxA0{4n|TcMdH)nj{Df32rugp`mNW4K{P3Pz>hCpuQybn +zoy;!R%Zzg>$LosY6YSf1?%7M`2l->KR}88so@bIMvgAoiGrD64UPviLZQS +z{}gP+?^F9|o;vso#QQuV$b;a1F(RTMN~4r1eJAu%qVZB5FFBD{i#~zcQB<5aX<&T3 +zD${E-JuUa%c(-WQuB=-w8Ij=K>=NYF2qCM0@ZiJ^$P|?RA{SMq79Z%!HS9a%;E<_< +zq231CD9%)E>l=BDTE!PL=!ZljqYIEiJ>E)A<}&-l7`a12c=)H49ZFWrp=Yh3@H}| +zVCmgFGyZ-p{7f-(#S3(p8R)Ac^F3tQ@n_VBQ_dE@gbAu{Rg_j9 +z&x9{zx;)wfV*DGzpn51PdogvRpx%k +zbU(d6_i74LuX0RGyjPp5As2g)^8QGmhePI8qVGUUZ|&)a8{0-0)AOj&zA=`8cU>DFE(3d@d`AgmZt{{8^yT5>?T4hgn%f9PXv +zzD4lLIP%?s^@qIpuQ-xa_wDd(5Hy_k3^VCW5H5(}w=)byb=#F8()mVN2+((V{Y
z}m! +zh}p_J+|;v(i)Ix|bNGPV8A~h8d6Wc2l>B1dA5E0t4jp%ViFh>{w`DeQoikj3JZiR8 +z6WL@p28~_p=@lWER(Ue8JPPIjV$f*wdRsBSs>o9F( +zNE(MPgL8Da5xivw*e|KJ9R%SpgaG{mD`NypH8> +zS7xTnROo&%y_{$yEBTT3JX-wIS>)8?m7jb71Hx(dxNM4y_r}g>g}P5WIo@$vOVGxd +z_^)?(c3n?=tYUaag5qtsCBVMrjx*};vRj1s;p8I!9cZotvL&7uR(&VA@3_?T&g7*} +zzQ~o$hYBJHdBIs6C|Rchvi+)(#Hpzy`YD3)zV}>#4vmwG{>MTtli)%Xl;bp-|=J}mKKiB6Hum(S8W$ZPZywd6vgE81~rvLTs+8wd^? +zhj&g7DLH$~rHxD#JD4Z=UCI1bV4T<&Qm$g-4T^{LF~^4A|?(Z?k$W;CtA&{;O0QGu9vfnW@>( +zDjJH>2eDGg0pBb;*SPr-V%a%Q4FbB5Z&7zCK4~dI{o?pUSMJ5zp0Rz0JK*DQGI{OK +z2?w+MJ!R=NO6#BW?W3+xs~6xv%h~Ue_p>HZ!LKhD;Ejq9dj0A@<5vNLik_ntn+SeV +zj=K;ci-#Y(rS)Pi-(-g|##{{-P5rE}P=t59K8ksY?DoC-wNAe?)P?pK6~0^9@NG6x +z95)kMn%vw~c%;yr*NyUp7el4xh*wcU_lNEuc1}XHJi`iVPxDCnwmMtyldMiPJ9pGw +zPuxy!7w!6Nn}#Sb=3S`2xdslj&TtFutQHDCbNF{EFuorD>%sIdE-Df)zHa{Z(f{Ps +z5!~SQ*T4_?sX_tUNII>MtO;f`JWS*tVi-nLTILbuM+Vi0)!&hPk$7ZS7H +z!GRHT(9l4!s90DH9eU+WGx#*%ZXy~27U;+YcN%vb4a}}wsbqsZ+vkZ-_-Qqj0Y%Lt +zs9IvEfcDeiyG0$%_1k~N2qn3-RHNM%`>@?_3J?%&5k-mg`p8*HHTpnne#Sjj)@<1f7Y +z*GqcTUv*?n74Kj3>)*cD);c5pYmZsvXO#}4{NI0AB)m{FD}WE3n706=|N2H0jaxgh +zu{Zzk$O6Dnw`IqkJE0iwFL~fb(+aCIS|kj!W$H|5v{qmqm?HW(iQ`jHlp- +zeZ?g{e}$8ZLxnZgdWP2YO21zHCbS}Qd=cZFHif$U28Fdn?rofo$dCy6<9c|Uge@kPl$#_-7q#G#Cr41rp8YnXsF1;5K}MLgSr1nRYgmYJj<3R +zoJj16D@4E|68$?ORN*&%U~w3KlGT2~=n3FkEtu99 +zj%Tn#0yQgEMhOWAeH%+B{kHwis#<;R4Nnr(7uFjawpelO{_*X==5oRR)=-R@AZzVx +zO2q-^5Cybg%cHJ)pflM?|=Al+rO8v%MocX8sP +zJVDk+UHON`GRb?zy!C`dWcla2u&6JRu$UFItqs~&vomyU61ixrcs|^+XM_VT)ZG>%tb&)%&fmlbqF_&*d@UIpILgP#ew&FW$|w2P|T08$aVe9-cOz3NW3n +zF#RhA3*VCLy`YDT| +zNZJX0%=)r6z~>2N1RZUKvTG8P&(t!r!4_;psS}!)-i4`rqj%#qqOjA}d^M)Icu_ou +z=KJ5MKdoIBf%p-t@b{sOnTW+?98`xU=+zGvji|AtI2=oq-FuR1f@3J~=Sq~l4ZT43 +znP^6S`UETL`3KfwBVOI7W+D%zqM*BX-$k6owiz4U@?9&gG1rzcyCJ!jpG%a@HU6jY +zaQ%bbQR&2lbdFW$c1BAwEoq|y9%muDA}m6ae6T<{g)0`{;@uOq3$UiMB+HQPR-8pV +z>PFk5;gmaiS-Zz^VShN(_Z)B>f@Q>TR8ix;!~mvT=i_`U6{k}2UV%n&ca0oRb>@wh +zT1LQkx^=Z%7PYgIZS@%81Y*xpyI}?O=M0IqHl-G@tLL>q*PAYyOB{R}{RAvX-4{{` +zLBF7Y@slZSYEmv4S*(^#{*ac%V2IASRFMUbN8z6(dBoT!*vnuiv+6LdyeZq?x|Se> +z^wWHfC@DsCaFo%j{Z7rrdY2@l+q)DfT9Vs_h^?{G=>IVFl|gZI-PXY!26y-165QS0 +zU4kXJ1RdN%@Zj#jgS$&`cXtTxGQfQEJnwt&{i^0i*VO5*(_P)`?6ddUYYj{}a)q6r +z%Ir>c*IXI+FMoSF-&i8t4KABUTT}idbo1Uyg+qv)L7($Da%sjucj<`JYlbRMh^>6K +zJG7vN|4MgHYlPvC%FvD-kb0Ldz5TO^pw0qI#;7kZEV&30!0e>*x%~kyt%Z1C*FrGn +z_B`kj54k9eK}nSo`O*o*Jl2<7sQV(2BFmj7!@zK9uh(^psTqNT%Um(rPz)Fo%s+zG +z8Ga+)Gvj-AzHTHA+xUiZtB_FWkjZn-fm0#Fto_~9qwM*Dlu_V+mWF$1=zfU(hWPD- +z|Jf}9@iWZIBs^tt0;p%CKFh-%m_zQ?hn{+Lv-c6b!g6`ao%_j=ZQW>-Pwepa8OExc +zOtWe@GeWqGNrqjievBk~WU*Z`Uu~KUIoItf0!BKB7#r|cBoDezP+EHir12zz+pW96 +z$v#m)!QA+8T(o677tr{idER>=dj>)VWlmOO58AcdGvL$PL*0rO*ucX2%CtV^yxurrgiRe37M^~4vTrBd +zSP}`AQ0o`>rVktU4ZBKW53hvnNGN;7cO|SGT{OkQG$r!5KqR|Q-s_g_3UBVqh50Cg +zv3{%Ga}U>%6G`O=hl$QDN1)9RSI45T^^4uYoRUEdFY&??M$q*6C@th(Uf)7%nUg@?EFHKLA7$7cdH`V2$*!pMc7)pxDk#LGzpxyBRT-|X2uDK0zNRBv`N +z?yKXpbD@>lQ3cCthBJO*MIZQ*EKEY&m%(qcmmEDdaGDUg5@%Hp7w1Kr!H-826&aNj +zlL|}yq`N3<1%LmgdQ$Cg*Xw&l{b@7NH_=U#r-w!foK2fNAf*jku`0&!m%0=y2YGFm +zC(^-yQ?ti88qpnskd{ +z6=IENXpDBacQ+YiT1H(c{dnJLZ|R?m?=)rjT8=Ee)^5FD4a-q-&^^VM_oUk-x@O)Z +zLywYp?0S)l_QzeFpkJ??GvavftOrO9-^dACr{ +zqsEy`YoZklD>_^(RAaD(yvO`c{X0W&q2b^HXbImx_y9+m%`E8_e;MaCwxQlZ5=)p@CaQ%t+ +zur0bmM`JK#IA=NhXGLy&%lKQd4Gl_vfTy#3_EmcyAIhF(E1`KIa-7q8Y=lmg(|Pj* +zZ#V<`?O}W_uO@F-Ub-~Pq%R{*xNanbt`8gLP(;1#pR`G+0 +z5HcO4$|W9S>eC~*vdGbneNIvI_gys;7kM>dzJh}bN=#!*J8H({md)+Pz9I{04F(+; +zL9WLk$bgtU#M}!+HbrXi+%%($OWXwcQI6k5>=O)-#9_G6C?`Ri-(A7S-@s#*lu_JY +zpzHQwtMoISQA;aO8jUPVE +zy)8!I`O&zOiRym!SRlLc0`9P?NpC@MT+iMf0v8ivS*Z;rLz_u_o_q-u7=J2|hzi@5 +zmM@HE2*4?Dd>iHp7Nj!al_b)pxN_MxW+qy8MmXl^etU|XoV^q81qqWsu%cZmi7`0h +zkDvZ%NF{EiC+DT2sNKL7c~prvH6>jlt?_X0 +z$%9qa3Ng2ZQ2F0>LW?z@xNNeEc=MaIQ?4i@@G_o|6JqIxsBh7w;|V{(k%<@X9^Hh7 +ztG*FgYnPxd`pW6brDCR6-4tFi +zC8La$N%@~$A$J)*G?%ioILyv5#(Z!xd_0KQB2sA*&H%Ui*!$+%-9CHtcWB7fR9{Ac +z@GE=%4)a)Ugjdaq$l4Jzs3;Dr%}{84FKZaHon|&Xrn;Bad}xVZQ^#dBde{m9|7=6K +z@*{bwSB3T*nE_)OJ-VI*zditQx-kM^yQ2c2!5OdEuZDM1S5EnCd(Bp+gWzugXkg4& +z==c7YS3%jQ+b2(|f#_@^$D1 +z`JLrTT7E9|8ccW5H^nqoyAr^wA!0yBzyurIu2;l)Pr!d|az`W}1Hxc3S%}W-VLM7} +zH0DTz?%P)X>se6i=!CNwe^TgkM;^X|W%4zRlG^|9;{Sk;4XU??iQGGtmyU*_gbVOM +zQFZlhu^b{L(DQ3>QkdR-y7$VKwc~MGpV`Y9ZM4I@CB&st?;iHsh;Y^cFU&{67KF2w +zLV-*jzUdvYiF2+cj}m1BAW3|&R>OAQFiVI*c(0eTPhH3-d0lB9(-*(fz!5yc@q47C +zqwRro{F!wJKnGNKEz8?Wb@&VZ%MBkIrsM^RK%4w$8&AOuB9_E%}>u7ej$ng;(#>5R~;| +z4#flz0iu^q-{_qDn%w=7$MVpY-;qZv3te|UZE7y3tTr$jiZRsMpD?VGoITPbXqrBE +zqvSIBZN9ROe%-8p2v`#hcB%XGxORts#{03NeeZZ=OhH6GQJFkxUL*=g73X*&`Lu?Z +z6FzY5C*5>8`#wnL_pUAQD1YM6UmxvB7p>YIcuj@u2-!_{nDIbPcFUp-x{F# +zJ>GGJ+D{qC^t*8}s{PiPIZ&*YbRuE`YrR&e`MmCeqk9f6BHv4Kk#4niK3S!*QLSCy +zWW7z~KUr+`Uq=6`PYbVk{&|TwR$JaAsFE9iWI41|v6OBq*s*wN_uwsfiMoX6a+hh< +zOphjNVkSI`8RIOLKQ@@H+F-;8AfW8Q(>`;`&K~y5@7-TWCO%aj%Qlag(5YO^%WGb5Ro2;Gqg^wZ^l$7Sk4&eR7h*J`6`gv)ra=Z8l>g^TCPb+=)(K=V%7 +zyg@17E4lS*^lg9MKjD|1{h`_^!Z!iiDm{;OuSEjUjtMvgoEJBt$ElQPihtWVjUg>J +z09)f84z-3^`%-zS)_KY>uM((2$DqJG2bJBL^HpL<4oe#*mTdl*V%fCu(q6 +zkIp>wS-s(2JgNYN$ciB!ye_2SiP>R}OMCN%TuY0XPe2QLULc@O|F{O$9BVWpVOA1)9EXMSH$8W}%^`Rbrq$zm{u=bIBk7Y9Aa5i!jFK?`?rRId%j2d#T(e5H*fTF0u?P**T9@RcXz8 +z0wF)DA?2JOj-3&Ig<(na&W) +ze=|Vk_kj;_M52RaNS4!HF}Zxi6D%+@hA$=tM9WeMR8S)@($Sp}lV;E5Xmn0Azh~Ko +zLnBd1h2i$-rxN&v0^iCuayeu5+SFe*84Y-ZUr^`1o?S}LKGrLp?q_m2|KU^C?R;$l +z`9a78P}rrLY~JnRf5SzNcVq2H8x1t%+MXP?+dIL3;Di{qa8q;7Iby>wM550H;|f|q +zW=NLk@EDnaY7GV>g5AV8Q1EgcF}e5r_IFy|%|wk#X}l%%#Y=#ggiEBksTb?6^il=y +zhvgHdn|0J*T#4Xh)-__*$HC(2r(EyP{G6u!GAc3>qrE2K>MZw8yF6Rm28vg04#FBw +zz=>w)hSd4r%)`oErmK6MRl%|TOtdD!=bz7tKn>PqJ>u!q* +zn_?$|2s_T-EsY-E67<^uEig-OQQ)@&KEZPfAF +znK|d(vri;K-_)ycCr#h!4q1Kl5WeEjw$MFq|3(oglsXgj(?L)>;3PO%{91<2?s>+~ +z$X5+>B8;GVN|ALftKqTS;?1j)pi8nX+USvs-s6_WupQx% +zXjWISwSCPQc=k)GTDWfXHoeWHC7|~s;k*+-NC%5K0y4R=?cao5ACyzWIt+|H_F8ht +zMfT0KSxEggK*Ao`8KU>xeKsD}E?S}7<{%gRM%G_17;AQ3Xn1v%QCjQjz@by=tr0g? +zBr>e(Ef!WB%kDSs&N$a)z*8JweUd+1{<3*2Y}Syx$0xldKClcOBRS|s)`l*C;xowX<$NNI9UGH(A{-Zm?2+QvxYH}I5?K3}F_WFnHprTJE@*H&J +zCFjc&Q00~9`846M_1xB9_>_zh&QWg=zXu8$JYHy7ZEk~voGblyD5a3f2BW8CmK>k6~Qq2CDW~&$J +zcohi9XhRdz9Q)vA9}*j!+`zSHG)FzUB+;i#m^Nk$j)8oPcSMpS|JN0bwjPpm+0yreZ(hv!<4EdSSLAkmXKT3 +zGZ>1dW}rYNgi~oylcKN?LiWZOEH4VM^IL+39t;`%YmG|4Dbs(HHkMdR-1gvjZtD6d +z_#a}id-NJ3#k60-Dey#JNWsmPuq5k|^mk7H!CwjNzKO~z4t52<3-;*)X0JxB_kGE>nYRNh0`WPm-GUf +z2{Qqim`u81!m6Y2*o+dI<(z9rj4Tf7XMgwpZ^rG0$m9Kz6}(j1jIgXdE7X=cCdk}^ +zPP$@$f|n}3_*g9EB=NC?TqKp&G=8_FuVGO4**!5n5AGYYB+k|B<8)aSQc4te@y+v) +zHFgVN$&3|KOn{OcKEfOv@$RNSJ{yitz}B9onxUb*dSOxwhL+&mD4^Mi+>hXDEf$+? +zIDe-8_~l?KB^4SHtDyMO)o`~|EleOHDtqg|7AO2VjryHr{guDI3NPg7h;fF5+Z_uY +zHHRWCGKhBCsfRBnlbzURvNM>8%^xAP#=o7W+5}V`--I8JWii)!09S+m^1|@Ltf5Wv +zij-d6QI(xgILIQ3-&Lktwr1ye(bBg4c+)M7jFBoi7#Wr2P);3UgAtAJkwWX&(=RHE +zM{$WmlisJ1dS&n+RI_L~)o>V-)5l+*-!$i2ybdx6gdN)^i`<$Xc4TvK`b#%*%N)S+$G0Z05J-xt6T*^h%V`j;1CGTYn$`+_^gzlv)yFGw8w_o;1=yL5(J- +z`Sq1-t#T`9$PtD%m(Aim@sq@8f8wN_#IYyih%n>%)2}Muh1Thw);7KcSBcF~TM(^cK_-|E`tN^qpV +z0mdr6GevV?+t{r)Fyx2qs9RnP7FRe~L=HHBQrO47?0YVM+m*kj2tytGo@8$t_a(jQ +zjwl8@F{Kzo7&_I@l&PZFNY!LWgYumwNVv$6!EquT3>fXed}h7Z;s6e2DBt*fYLnko +z$?`_3KV+4p(GFRN6Rz5Vi{@!Z&q}%MLAT#*LV$^g1Q+aR^{(d`$<;}U!nVYMp?3n9%hI_-Pf_k5PEFMx}C +zcvF%js~gzDgR4Gu;y)_=8OHz7KB$N78csBL+;EH>mYNb`MGjzy3;3C}!}QphS9)05hs!27jIwWgV>_ +z@NEwxY!MCz;4=d){=(qzi-1mEnkV`b&)cRjo7uc}Em%5%L6Fi4w94)=bt~^}TA`h_ +z%b6X!7WW@orFdyeHZ=6aTl*tfbSHx9&i0I(K**GSM72ay>%-3wpULQnoMs~gsVs53 +z=Vxq>><=apT;2R)Wga;DGKYW!Ui}l +zLhs1HhLX*S{D9b>Ibv`dl5u~A;9kZwD+5IgsVH%2D1lMi4NHgf^F(4^r1SPwm}pv{ +z*qk%gszm(yiflA$HWDHaa##otw8Vp+g4nhzW8$dyESLnl&P0E%fUEOtt;Hk@50S>) +zsH8_E{OeV-{nq#2*fvxNj?-To7CE8qB(~CjWO_2TPRQxe96+`oGU*YDEgGx%kA&OEFxjDIn +zCL8b$pZ7aa+dbLhM~d2<_CL+v&#iACLT0C&bWY(}quk^PmC{}UqeF0(&$7wyGW#zx +z?lu;Fbk1v-4V}@xuxNfqGQ~kjsUEz&*|^p-^1+t9ger7?=^Hzq@oX_bQ_Ct~5EG9G +zmNbIsbPq#)dbU}~5&$Qr{4}<~;bM4l#mfywWs$&R3(#WusAtmvaDB!yIBJ{#HcVcY +z-fquSHlHjvf*br%Mul8(k$*7E%K`z;&wUR^80IYvl=m*ZLg8qHTq=Akcyn#A1;4+F +zHtOYRN;dKOQ~N#=|LUS*5aIrI{+;W{R0jq9-jmWwUfk?_FY_Lxhyt5}-(8RMg#ppQ +z_q}Y9HuX#`xj=h`osY!WTs?O$s!%M%SfJ9{_m +zRo$)XvDt5B69axs4j^1&+HdWlhb8Bg1(S=XDn5y1hjBdXPIFRFn9H}}{QbIPY5g1u +zx^205wtpa`eDUX31*3k1pH8G;f}Z{xI7wlgJDp9l{X)&Jar~6HeycsQv~-obQ?n;Z +z^|;MD!}_#zymURa!?qV7)fH_jb2)@z>DWPrvEwHc+n>Of^MmF$ok>*K3}1;%0Rd4| +z9xaK)4+@8{c+_wJN+{+-xVgM9qQ=v{;3%a3k#y|*+fQ9Kjp5&wfz!Uza`!+0W}nLR +zOJJ}>yu@d1%Tj1)b0@A*-&b{3Q%;v@@yZv*-}Yeg>_+nc?Sv78gZ;L7Qcf2`#EVuq +ztU8>A#K}j_YYeY^FVKT1!~gx8i47&+GV&j@*fE$E_=Rs@wAWXTB2?qve+o!8NW+&8 +z2{t@C6F}`WWF)7I1%`70X%?rDMh>vxk@!r&dI@6N9 +zR3R~X@gT8NI0<<0xDk&&OY6ri84?q +zrE$wHRn)7NzzvSCU|1YoXx8p}EZ?k)GtT*XtN>VOTCyVFa6l@HFmO1F6Dc+r`$&jc +z6LDsL$HNpNi9!)eD)(OpTAKs?3ZR=nkb6gcSC%BDrV0Efji%LN5MT!x?4G^lf +z#b)6pqH&WXR8WCTSUmZ|fnn3daE2_e&C+eAkWz+Ew*lb#?a!9E#2f{Xcj8{Z%c!oT +zz#|Rag<9`vo7&6q(Zx%00a7M~1VzYI?{;)SXcuOXGSDAJlt-hL&J&JiXP_++!L6g&p`a5;?6U5 +zh@-oRDu;cfdI6X=%Sp^toa3%#^JjNazRMjp45o`FwPp +z;$Z{vD|Cf2fFh^}Id$A4-Z}pu?m{7W%eG_c)F{?6*P~njCd|w7$2*n`eDXTIfhoKH +zkc#hzx&)Kkv$k03B78^Kf&0Q-9YKp048{gQ`xAq(-i6ju9B07%k{8OiaSa}6OXC8+YQvYVX4ZHW#aE-0YA4=lC!t) +z6L~D;1rht%mY`Eysz$_C+j+`*~(J?CGM_&a4ZcJ +zTiz@or0j25@gDH +z7Q)2^v!3nWA3xSzPIoE4$iG-n1eI(- +zC-SYGiilGt(rup+UtnS#C7#k`66dbdKDz4b>SkXbcnClh^{^Y=aJ!Ck)1Hhd=~CwD +z7HuA({JJcj{O~;KfIrQWS(eq)Zuf&+gO5Od@zs^|$?Dsn~vq=tsNSfcnz1VYd}bgi7KiB@3jN +z+o{{Xd0AA=kKZSHOvU=w7QTU8Wxgd9irsm&|a%;fEq^FtFK{vx$Aw=XMSAr)no$D~`k&pq;a4-9~Z- +zPua%FCLv!r7C{D+S`6#eaP-`d2RKWYf^eY+6-&gW`IIk~QpmJgd5UG_eX +zu=!?0>P|$)#mXW!(pPc$tWUt>?-HZkU##!sce{HJ5OX=kKG+&>7w26eLD(A5PTrRT +zhFKd+R>+IDX#PzRn%<4j71pD7qMd+k8pzgqC8esQ#Kf-R@SKqs_r=@|^YqsB_L!$bo!=kWb*SGNqTXAjI +zuH!$FT-ch5`@N{lck|efT2s +zXbQmipP92KsnX({NWX~CCta!H?)XBsK3YU1y*qLlYDL%6yPGY}1KACH +z&=>Tp7UD#TVdflh0Fx$jP|2M?QdV|nP{j~X2`fTwem$}#Lm7%1OR7W|7#)p^I~NNK +zv;`yJRcW}&7br5pqBl8E2GNzth_QWAhe|k +z2b%%kY(P@>c$CEk3-SYt?}Z*O;6=fQK}%3c%es;pQcSO>m8`5|hLEkINW;jA{lhTt +z<;;f&g25GEsldS+vNOO=Eq%KC`Hl7&X~nRCXtvpBV{W+<96I^u@X?dTN>WyN@!WV~ +zwsJj&3`DT0icp+rizP +z>wKgz=IqTzz6mprp7Fc%{-Fci0BZ>8BIvlQa<|ssiM6~*Ze3l{fHhlj5$u7P#h;TB +z(cVCVWK8=1{N|b;4ca0snbD&ls4K@qoNEc);pDs*zVF!RzFnKC-$FEg8RBeiR+zs` +z;Eje;)Y*GISmN*=5}HREF>UtGJQUQNgK_`NCv&5)ldglinTH8zP@#?WYE!RL=W*El +zQGL@%$n===(>0yL?YDsrdrZ>J8AG(pfw@wmpGmR+F{8(rzz@39-@aS&LN8A|^N8k0 +zj@Z5!IYU6Vlw;lqK`!TCNJRsVnl-lgK7e>swPpqF+3^ET11fgB~6h{ +z2QDQ&cRXj>X(~ng!LbkyS!@6tnCc0lYi6X39`HOFz=sJplAU +zL=lQ|oYQ8R5J_?frS}W}st${{M7OUdPU~=w(nbx9JXZ>OEuIHGlk6#^8dgM3qX&q344%gXPXomF|1GA^7eyt +zLZFyBHxrq2NpoFO!~Z}zxjoa1{|EDarcb1L11%loJaUVcfnIvL#`WYoh-rdVn*V-^`Ecgz6OZLNFHgj +zRIToVdu?Zdc13R8zBX*5tQ>O929oi%Zn{6PX9LKDd~`bhs-}xzta*~i(3n(=jkOGQ +zf%7;N#9DD#K~w4yKJwH2OvHB&&|n(-u2ZG +z4>WWbP71~w`f~lYkgGhbXWb4flFP}SA!&nED*L$cQ~b+)e!p?z{az`X&piMdIe3N4bA-=~p)Cm9FH +zx3D|S<9yrRBmMVvouu8N<7+OO4I8zkdBrhqQ!$D(QG+7~zZdU&MsC}*=O)4(nvyL6 +z5vSF;F|Su7){(F9Z_ii0{GXpbpB=gO%*B9^$^7^2q%`jTa)sPa8dJj02uRFI)HPY1 +ziu^i~rD5SPH>d$(NP1MHU0$1{@?!#6JtoM2oh8HnJ874FfLb$n%a=VLZXz?xSHE7DORqRIW5T0t$m&}9o +z$t*g&|8$jYL)wy?p(PwQR(=GMb5sNOBMN`}iZ|(SBs%dV?+U%Bx(YOIQl+*P~)>or8I}uPxBi>#2#fo*oJCHnG*<=n} +zSj+6J*rWCojB>yC{tfoz!v;Of#>pd#RxrEOkdUzqm-h>df^`e8U>N(woa0h^ghY_8 +zTx9B)k49|Z6#@Woe3HX8%zbodau>EeE*O{f`*OuM^rB&!FvAeO852cI&6d?XD4w1o +zUdqiT1X78UOuAEcfD1hf)S1m1uZZUItz)UJwF8iR-Y;it$`W;9#`{Qfd9|@~kXzI|6D_^nUy9G^>GC@m7ox}y-De!Y#%>|H@D(*k9z{t&d$igJv)dKF=!$@P3LL!fVe_>MqBM^$dZV#`jO%fh~22q +z!pnVB;PXDLw`^t|P}-%P0quV>nb(_0h$OZvAHSI+7!$oBs)v=_lE(Qh3kzKBiLuEJ +ztc9q;#~3~ItVAO=FU>+BiV{6UgpjLYZ5K^CJ9qwC5lB4YBDdIxFh=OLzOn?dA^v?4 +zX~o+OjGdZWYgWGS#;774=STN(r%(vn}52IXtUxo(c2<9s-n}m5LhRq_NrZs!-|u2P +z8e^#0$M_s+nIV#3Z5(cdveXo2uQj;nXHpJUxDy(zwgf$mL}*UQj46n4O-t0{KQ5g~ +zI?E4RF{jfMgr{`eU7IG|LQO30Qu=U->Zb~iW?zT0$kTq2O`+$=&c*i+WU$9?_XeiV +zp%^dA&(%r8@NQa>nB&-dM-avZunU5Mi(6A%xE1+IQe4g?#2~qqYDB~nD-!T0dh&ob +z-nPf9Y+12mIk_h-sC +zNGeDv-Jocm^_i?f$_jk=78c{M9`EV=S>bq^%+5wfIVZV7ZJ;FJbIdg(wpSFFjIrSF +z!!(bIPDSS2nD66+lFGWL4WoPpL`qA@q-toSG`Pk(e*_z{#-UBQkzKA)};u$rS`q&s^)Ki8z<`L|o^jWlctH?+P;Xc)gnqzEQr>PAqD +z?x54UpOr^rxI`L7zB>v2S}G79A@nnL_rS?~jfSV}whzV(o3WE&!eX4ITmViu{zGpd +zCwymBh?T1hruwW4OeEG<-iw40Y#4Jv8@L#DY|*~FN~Z+C{z_Qd7PL3@iuwLIMzBcr0RC|M0WiujR`%|l1^j| +z+wmda=t?N#2-~jVq;4vVAK|{tuQ~f@Md-HVtkdq4k)54#n?6`CA>Y8~@H^IWTx@ue +z5X=v8QPMd0lYl`5w0AJW7>|Tdd@IsTtrk@{&+F8kX(A@k4qINLbLi-+csX-Sq^>lo +z8^7}gX-bo%#%8I-z&XLPSChR@OiwbaqCPseN#6;F7-GCREaLZA6#ikMTL5TI^hrSZ`!P`j*hJHTf +zM$|;pa6`R@enofNqs}NR^0LFF9sN?)QJxUq6jMC2SE`ov*4n}=>DXLnBFc%Jq%ui4q4LD +zh-ly9c-c3Bm_Wd`sP~-61nrn|fCvBhe8@SyINx +zFDVn3Ev{M0mcQm{I=fd&FncWm(~<$x@~s~RK@o{4jv|RokT(#&zAYMUZ_-AZ&LtU* +z9bA7YOGN#lWUI=kWve+S@X6&(PUr`sRt`4~0ydS9_oJtZ?38Ua&}!(<#H +zTWm5xpl9&dm(@z?U4OPAZGwBtEQ(hHr|9aMgZE8$+=sa>&-vl9_G>?{I3T}0nNLdU +zX5rC68llM2^CmbqoFno%`&Snh}J>~4QXo_R$mTU}7z{bMjSog4}vs*!D_9iFS+u?T2e``)c +z$8#=e#Uv~u786jgUJbYT;5hYoAdROGbW^%;8dmd`7QpLek8^i}3$}MKoP@c>mEKJojr^M?);| +zE!QQ29>8H~T^}~L&{NfAbIx2xhV+g$V+BnwSj1Kx`j#<)J(1!I<0+s0s^*yS@{94! +z3mKy}Y3DwUyYdyA(M(6l@^O-N6Oz%TxOK4BhUG<7NB1PCk^vYtpZW^?KA>2}fUep@ +z6pirr*Eo4G?X|>wG<&^X}srZ;3insdm!-PSKI(Xm9VQ9sz-85+6x{yy3vaq|1+Wo+n_)!J8F +zeTlp`xs=-5HfNLgcf+%F^%2ML;$|(3*GPAn^Yr3*6JB?6ue$27CA4=DlX!yvE +zLM;PzMJw$eR5T3HyCuJ{JIq1E#>Kx2U%kLg3bM;bMTIu<>M!AmB`FpMu#&Pb+o0c4rtwfHzdD*RZiRFq5s9Nu6lsmKf%wxtJP&PT9L_+Q+fE! +zVcBlYpX?vsSkAm-Whkh9kc1Y_cFU_e7fdH-U?fLNO=}N_)6-VQdrnsuy!RQZ(^F31 +z0xi2dW0UY;&e4y#@4THU7n`D&-onDB>d9NwVe8h6h|VBd#J*&DDMoUKbg|^<@7ViH +zDzj#rM@-SPEx{J6#-#cAlp;eOib0EPFtHNdcEh6|QQyy+997l-{eGpLkP+3s%;8p% +z@*W=E+SF-7oNgf2^elc|3~(rgS_9qPXz(YQYJu>#4{`Ybv>hJhJa+NHu;S!PjKppR +zsjfZ(T>R%Uc0j(6XtOVLn7+$LQ|#8&idfyMExR@A>^ZLgKInGH#sbIic8e95q)*8- +zdifJNpiHx42SmJi_kiQ%7ri?K>hQV_*G=9m$m5fG4*GELyr;FzR7_6f7{I~fFr_ckF%SgPvZC4?NEguOUqh$jur7vSUhgm(Le9%XkY;^ +zTwYh9W8dlPk%Ti;7c{w{g9DQ4y1@&${`|_dbb->kG|oNfJs(*xv_^ +z+2il??Qfae`>}O4)k)^lg|RdNRC$~!=E42uN!%qT83Ob)O^|fJgZT6aYMf?G=| +z8r#nDT?|hk(GoBu`z*+6P +z(dw;+(C=~TqXwcD6zxO@g3C9j`47uxGm|N)wRvAlwexV^hfQriDus8yqm(d03)$$k +z+c{AVcq_oqE&iLa|JgC**w}cST4vuon`>Y1`Ixo7;UssZ&k`OuX2vy3uotRQ`fW|a +z`GU;W?qZ{qZgJZ%izS#iiivvnZgO +zFUxCht%8L`Boc7__IKU6WTJcCvbhKkQ~?jjC{Qq6l}7t;fg?Qf5tCr0ci!2!_NYu` +z7{1Q0@gF&)PMXDHTZRPK8f0K)>`58_OIZdZphr^leK{Jj9;xaD^NMnP|E?{7p_4tQ +z>rB1IAOl*_Ob@N@G*SPRY1*8D&r_`&u17EJl;N82Cs!SFU5GOA(T1PHl4K!ngVlel +zCVfop@7gQkWwR!=6HTpp&-^XfaHz`X<45;$&$L$))xrH$AKhS}ibEVllhEv%_2h

0z@jF+(kjruWaLZN{-Ks?YyU4-Z;-_c|Ix0$Ipy^ +zs-BLNsy^m^RO4N37Zno|XAgF>vEX}<8RdYVN3C=x|Z$u6+nhf;*;-o|5v +zH=77V&H{a>5l99bF9@?EJ+kZ3W=Qhu&fYb?vHdH$#)Wwd(*78xdidlYdKGCK7; +zQO4;9>`_pm)U=;$HvBhit(_pjA*wmy?-oekFa38n#*NVcKTohGWlWtFXgso<>?H71 +zHEDFg{c7-{6X8nEZ_yGDzgSds$N!`Qq}QyStC@>n&7JxWQrCV0am +zl1Q(h4symMtM$T%rGj{`0X=V|KlVmz`p>D%2k*DvVu-%SagFx8GOK-A5JT1F*up7Z +zYh(3uc=>#?5QBt2ew|a5bQ>u+j_05Q(%zxkhRS0~9x~^+Gpdp*NJIeSJ!6(uoAY5V#)uTP8Gcuc8f5H5o|!(vR)VYk +zPvk@M{S8>C795-=4z^xi$GOqcr>QQ~J)%rC)TIHv+27N-oy&p=4eC;4ib`T%-!3ZV +zHL<+Uq}*ewHiOD4I)coB6>2)uQ}a=u${(_nU_%j*lj1%TN;B-X(?7gy{5<5pz*wpHwzRDGESN1UVn!B8ZnquYqj>5R-@gR5T#IR +z#z^Dcl9Lcj2v1QKIBqA1>i^(<@{&@+Hnm!EG6}a?jq>%xAJQKSeD-9zoT@9W8;zjZin83?`U64a+|Sp(zyr7|{b?$jCsC;$ +zS#(^_chSei{~iQKff|3eaN8p`mqqWV0c~`GR)TVC*@V~vCp6D`oQ|7RTsn=%-^1C& +znstFaLAG<|3?eqrlieE24wcC0?^AT+8+(L(AgBkHibUb(tYC}fkAstNXaA}B@le(G^nU2Y<;bBq$ROfKkihhv)TmsSci65EQ(QSJvTewa#Gq&Y9k*!e|4zw3 +z9OV`m0y7jfl^6XD%kl|9T`>}6>(!`*0ForzWc>5ncl;b>c##unkCMsP +zycx>(k@dR9P%54?trkWaX^7R5}ikgk1($_Nut9SN}m +zUL=1Kq}B}TW39mD(d&Unw%8d&1S))Nv4|80}1&HgDe +zZ*R}_{3NpglT-YYR!IGqM9N_ohc_L_^QlnxT4$BxjPMZB{OoaDrM0`r^{2=!d)W4| +zS}kBDIx$=GOX*zMd0^0-ZVnM86{+p8yetDa5)0Za_>Owndb7|@Scqdo=G|MEF&Vgr +zCIJFzA#U31_J-C8S!&5V^D~Ikbe`-9X`@;7a +zP0pYG86it}^>QEA6V-o>mGnGl?H{uo4Gc={18{`Db>vrCcC*sQ~+cBt{(dl6ep5Ykn3Rt)?zzip@T?qw-${!Sr8x +zR1JBDc@>rd(2u4Z<2fUXIqR&v?C+yxs70s%wu@V+Cr(pM!9k<)+vsXnrblfn*~ivH +zNf9nv`|00r{^a!!%ml;!ls;sp0*zY0Qm4#c8*6CNe|?ps$$4MW4MU#{Np^!4n~pKZ +zz*k73ogc@NNq~P}-;j`~Z8&IN^X}+p9)|b~cKLijS30ko9wY0L_LJR6hm9l~CZw6z +zqy9Y~Zwsa2r%Oi6$*anw +z|E#@H)M-w$)AHG2h!by73UK!!%eQ +zek+OYDIXmq-Yy43ObEwi{mdv?_JAuX|*P!{fF3V1!Y>Yhz62`^d-8u;%9D&dw>`7fp7 +zWW(|`_*&4mxGR!I_8aVWA1*>{(<-q@gtMVQX6h^0fsJ{4ZMG)98%FLc%K5(_{diH^ +zuwgMHRQ-hc%ZF-s@@WFrldL{T4uzXZ0957iLD^yp*HIihQ7VB$$9IKD;C0iqh$Yo4 +z7DD&Jd_Qx0mr1ua{>g2jD>fFk5cdGr%iYMdeOdOR?ON06=YNqzK;&QaA^%(v%MDlT +zbkPo`ovdJviXWNB{-vo;Oc4Tt=Xlx|2s$+ulDG8Y$`~u`L5t$%ZV^qmr3|1kyyjY) +z8du0W>y1U6k0~%uCqrBGk +zD_D#Eudn|lR4m1R6DlMS*s6ng8C_jWz|1^`hj4SSB<$p8dkJtKpjhlU+?Wl#_Cg^J +z`WP+$?lv^WZ}ShdMm@m8I|z|kV+#y5_|DG)nV;UXsuHUCcm9QI|A#z!#P-rIDk6#e +zt`OCt94Xhq314pW>p7=(HF<9o3I0CxFXQoV%nioq2mhY{Le#wqoKWum+rq!o{ePlKwF(GuM7sLFbvs!3zn@FL +z)lE^@06v$evSLd~dz)Pa3Fy4N4Yn(ii&pHvz}Y@J&HRQL`;pl4kG)%@Gu5YmAu$Gu +zmUo0F`_X18m#{1!e(l41bp`VPCHu?o5!e5yWY@^@QdCiI5p1aFCx +zf^(F?XYk(u)MR8sU$>6imQ{+DYef<+Qju4(rVy?p?B+h~RUf@+^?LGC1- +z1#a^c3Mz%_B$ovFx2_6hnrZ|t7L(8y8XHyrGj?r#%;x2;ePN}_GmE3}YnYs2&F_cH +z4f|$jZ^EM;)S4&};JNT-C}fo=a`;#5bVZMO^c1_@^t>lqGKP(%L&;m!QBuxdJ=ilL +zG^wV~gt-Q|X@=1&APEGG{-^}`rp(YC?+o>-cWmt$TPHqWo*VmFW%Ng;KmaOlF +zn&KsOSFu$dK$?XnEQA^PvB}6iLVOqAccIE~Lwqs(h#$F?AW}9k`P(sPe}QbvdAX8_ +ziUV3J1JG0*m@YF~z1=>urk~Z7hq=7MRnC9JJCSYi+wk?qU;}Sq_PNO`*jw}OZn6p% +ztNMWgyLAc6{QD=b-nDmZU+t2o1qYvhh4^NJMn{I=+P<@T+{ +zH97Fyp+YYvyLN^^hWKvIX6J*!mm2Ezq2-Y2h0}2OH;a&(2Cbjuhb6op2CRl#Eh +zh!AA|-0tN7++ZS=fxer~b?)<-*4SSHt8bHt$hOSEB~hrris|Jl;3|PKW)F&)qAk~`x(CE$0wl?bEfFwLKOKrAIqy1h}HKl$ZRXF9l +zrXT9|nR|I8&|u|4`7I{F=t)Kp^y0=4wJ>msGaa6BJ4}5eM?H-|*!lVDR#5(snUQUY +zN}-6!(V`}wU3bD{3_?KtR0X`%U-Qw5Gl@b0Bbhx>I_ti)u+Y(8h=O+GB+*yvf+98B +z$CFOVm%3)cbzTbDjrNrWlj|-|j#b|tkjUp3PIxn>8Q6cJ*$WSk +z4JyL8=dZ@jkj|>hcn&0w`dM=tu~LKiXey#Tef0YM=>YLeR~_67)WGpP +z;L@UeNRrLd_t+56Ye@<_+_C1OMDYW(TiG1!PAogP=fTa-Q?e!j9%rdg@1AvEwUO~3 +z$FCB9@Zo~SII<|aS8{XgSIUY<)AiI&!H*pB_G9J=Uu<+wTXrKQI%$P8%e}QSU#BiWO5-|^E(VlV;w85>u|A6|vr-bwVn-4R`!SLj+qgNuE +z0vP~;0F;QupN1M6dL!U_lxu_IV(~#sw}G_IPPU!D8l_pI3x9#6cOj#5xq9yarG{HV +z)jL5;N~GNxY&Lh9Qs_54SYgBhW|ZlEwcJg93mz%$%5UHG@>)GB95A5YR|jfYi430O +zI}hf)`7WEuyt!#TwX$&6x2>$$=_L~||5Wt3h^1j_QVtXY&nM6z8OEUAtg`-CVCPlwaFmc}9?b9+*-$+XRYi+EPQkaJ +zEVp@o^=Ws7^*ygvPnnU^Ie1aB#kbQ){X4$`{ld22*v~I6#D0fvi9Se8G#B8OiILZh +z86xhmxKMFXh1zI#JDx1<_MbPAmioh5O~L25V2dg*$xM^WKyg++53XHgE4i +zxBqy9+bZx79J%^E;A1tbk{CHBy$j|xApCIr1ZU9XZ1zpxH9W)XvI&Fdznv9$Nbdh; +zsVulkn%KWtoX92r1`GKlGR9n}Ahh)92_8L$K1jT^VOmuB-EuCuHxtO0CkMq?ktbT> +zeYaQEuZD43J+?z=v#MpwdNjriMJ+W#LMU*8SQn_H2dgB>iouVRhhv}6t+%_()Mtge +z%+nDLmQ&1xUNpvt6upiyVbdl=S4fC!M@?f%IXa4mI&=SpR+SSh(Y7mz7 +z?6Tu3ceW&|B(NF2RK;Op?cVHuukdzP)YMZ+6;J$vQlJK?0{C2Iy&8Q;St3gbu!XBx3Lm>8 +z+HFkKhqWpxfuk0!dU?&P=`64VeXBLVqKiNTq&J85*>EW3L$*TdDAldn{wEwbvSRtN +zYv^G(B{wPPb!7-lnZJ~a!BZI;nf+F?RjNJVDD?t%fzAP0()2wmWtkyl1xrv3bX`7RS +zU*|A{H@Rq|x9vxVsl+|kkwXfPb4`&gPbefq^Ow}stRJ+ve_5FcsdL|OD6z@r>8QH1 +z8#ES0I`}ENoO?(qwTFyZDAsKuRBg=iPTRmLOo84~TJnyy<3clQ)}!O?4dpTTVHWx} +zUotoc(+-Wtwr8RrVe4?ge($ZaCoJ{j9}ScjDxXFEN=m|@?nW`WFtlv?;4TLlbRlzW +zAmx9KI^ck+&b}WN#5lXLBfjJXS%nWen{6W&vQF041@q6D4o@G;)78kW)VmaeBg0cK +zDU@=aN+eX|s)IkO@Stap8tMSJ-#CL@Agi}~QhuovMUooPNq5<4oQnmO{MWmP)YE$o +zY$O2I#|7AfEhRp9{l2+lG8%-*H}BFgZFMH=(z3SB7N*~x-@a%)U;5&*FdulB#3Fkc +zt6$WnRi_#egMDiqMHs_o&+$v~v6Sz268ws;8d-W-Z1-XOxps8YHCs3hQoOa1}DQOgRInX;$eU^#~67nQym)zX_u`a)IyPz&7@89$pkIzWS5# +z^v3Q_`dbEFX1tVsIlLZ=yNkzGYD20sVD-%Buz_k$n(~gY%{KT)EjV6hqf|V;AD(D3 +zs7Nd{j0h`<$hxD2*#;l47atq@AU+4T3-yNk3oW~T_ppRbZx1&o%Rf-cuyw5gA5X|g +zh1%fP_{~FwiR<#6Ek^MQQXEg|z-8Lt>*J0oge#%rJ5U>?w8t8Cft0Uaf(>ZIQSqmE +z3Eda9fTP}lv%Z8BTD6FePm@Y*!~$=%TfGmWKu$!@dkO9*aGz|@5~dx6g@x@UXRBu> +zpFj76Zz*M1kOxHCLhk6HAy +z1hNJ^omiFm?0xSi>IN(@wP70PX@?_MUzchobgeU#rz)yw;Gl90N^)*2BlHBVcI1v) +zwd>?v%mV5R6<*Jan$-t!ep!IMAY;PV>UzYs3)DbFMn0rXd(z5_b%u$S0g<>{ANoV{ +zlP&i@NwDh3ZUxgeA5iq-0{4v+!TYOj*j>C#FG+}8-yS1BC6U_`g0_JG0n5Mv*#TnJ +z$kj@L(^iPb=`WFh!xP+TodcP*uBr2rqXnyNOaQg39UU~ld`Xc#o +zhQs1PK62w1@l&H;?>=fQVcW{eLE|0iHtlTuDh|cH2I5GFwD0g$NcN68SHdtCm^|6! +zt9V4#WUN_RNQ^|2VA#G5_@`Oai+P;xY^|wBhK2^NQ=1cIkGvcwf3*A%x0C;xZP}G1P@Mxki*(c3!M?EuwWW7$jj9%b$7P-4lPfNR=wMVWq0g%Lk`xb3C*<8mO +z+pL6i6R8<__eT^gTlt+>Ff|X+T^lwzf3zW4biq~o>KpEIc6mPqRmZ&wi+h4(Nppar +zse~4t57qvB3$1r%3Aq|BkUg0#Nx6rx+T1T&#H`#3sqw|i;-{ABXN*XctZM|WuT2g&L;DOtf2)P@+yDxV=kpR7jrD(SIkbU#>3N5`*}!#7rCrl{~E+7vWF4hJ7+pt +z4*}jMpsoYeA_N85u6yBWf&c8$desO6D&WeSMHHD_Kp{WViLPSjtuzX!3UL_4RHAf1 +zN_5D9In506eeog@`gnNhUii;`G-(I@nqs3X;$mAnQuM%!ToL}b?0e9!bAC|}#51;h +zan@p_U&W}Al&7x8PX(h0)>Vg9f_Ksh9Yk?*)2p +zl!9$!GwOHL)wbQ&sDU#BMZ&(sdbwW}$*z0|Wj=S2t40c~R&IdLL_7gD5D3J_7XCT& +zp1pVdeY^ZHx^R>txQD5J_31xcAtqH+Q#`b>?Zx6>QXd})6fB#-EhQ%3u1nj{v}-Hz +zft~EW?-F%+$w*U2Nk09A2v3iH^@BHGLbJn)yi66_AmZXxz}-$qeOQ}D66dJt@pG+w +zAiaKmi|S?#lJWtXmGG+1PotC~Q57_$aC0Qc@ihG>S$W2e?|6K3Xl9}RiKN~9*||f_ +z8x;>G#-Ac+INizaY0)D?%O3TLf+03c7PN~$w7qT+y1e&#n|QwQ1{i!X)iKpNa&=!1 +zP}olO;D(ieW-{)p%th;+wzK$t3QUJ5x9_oJ$e;Rr8MjGuAvVmKA6yO&LK+32_Qynh +zC#0p?ZRRWkE-yKM0+{;~m+0rSuQN9IE&Ivm80f$GZX@i({oEg~9ufxywTiw2Fv;jF +zZbVDf$iU#YB43cTFRIB6?a#Xows-sTR>*~`z`AqiOlh7w5u}`=Hb>?GH_J7z{IujZ +zY`4&D2VTDmJ+u)Vbd38-D +znj=nEjzZSH#hbTNq0tsRC77`C0tUAYS_Dka72T=r?;RSZmHX0q7B|x?1 +z;W)5Fy${5}Ir~XsrLG7YkAgsgGwwO4x$3YI5uFy05UJe?`-if#nYR#QFF#0h&Qrle +z9^fjYJ=FJPZ!EO(V9dM{(~&E7Jj|E!{N>`Z$hDqRsZ&GN&^gf}Xx?CrzI*J4sZL)H +zfs=X=j1^Sjxw~fn(V698iHNMgihUvnw+qtKr1!|v)eAlMOW#3oe%bf3iO3gt@9`az +zbcp37SuquADKKIu0VJb`EOn^4Q=QuU{BFmxHHlsB`ThNhOsymM(xs+II}K6b`UVv$ +z6N8n|X0;qR2|+Rw%e~vj+-8Ov#q|tv^~H;nLY;eTenB|y@H1b1K%|pbA|cL{xNBj+ +zKZn-|0Y=4Q0^aEG@5S4m{on@Tbo+9ZlwYBoboFQNq3yLnVO_eLg5LwEh24-zz}W_! +zn}=SR!UlSaHJZA1qA8N)2o3hjte)3yo>fE;+B{Z*w9~Ipao+Bi`ntSaNVm*$tT%?L +zL;83n#xq_3S|;rABIf{Bhu+!`_FsB8h>|_jFI4M62aH+WarDrQivY&Lw!Jd7BB~=5 +zjL;t2S+Y4TfTSW>0}TV^A4F5l_IT^Y +z1eW#-n#5k`UCtQw$QP#m+W4bC>f$`Z!g;SkVnU!AinZ00sZ_L=u6!>rNxNm#6lRtnupOL`K6P<68&q@KeKq~2 +zG^7=abLmI2L|;{uC-xW>NFSJFozSI%?KzU$p*{%m?D7yi%9@0ZTy3vp5I*fzp9$~V +zaoUb^ZG72BYgXq|d7@n?@1Po?>qZRYicF+)zteVK4D7?@WP|(PNd%l9D-7X5gu_@8 +zV)?L(9aKP16$cVYh;GyrtEhu+g)zT4MUSf+ew?mukgtR}EGp9!ELYw2!_E42Q_lT| +zVf0V8dOk`K3$?S#hmM^)R>geuH}7y?2sUYdfU4(e?K<6T9SKCU^gl?tR~^WuSMZQy +zt@>thYMn6~bn;+xn~+{0Qi^y9n_$`VE`_zq{^V)VB0s!Ouy +zi8|)A;u@3r1CB`yq$pf=@0R=Ud}$BD9_sgyjg8N|NvqJt?}j?+Ug`!WS)5Fq8`G2n +zrq@t5XJP%ed6xL)hkPLW*Aa8YG^Y8E1>Kysn8i;*kw0l&f>JciR1ZmrwlpErVSa4s +zIbTi{YpM7@V(Kz+N5m(Y5Qw&TXNDky*O0OMCc(H}b#GQ3`(EWErSfP6FNhI)(xAT$8JN=s>6V>X@67H&XQ1MnFFfrGVi0Az|6~b-SeIUBd<$7 +zQC@8geTmDmUju?#bHzmzpJ+0C*a{0bjK0~y2&R=0_=S01ft8Y%CsEhmr0JZ +z!NMg;^p^!3f)x!|p_84AYcz}-MDY%E)~Skq_d6YbBQ{ko!j`Sc#hhikxT6eXVzDNM +zVoeXmEMO{gGeo4&PmWzBr;S1D#%4S?-4yh{()d}hVn_~kArOyb%Qmz#1&VFNfb|xM +zi(Op7h=Y>^*ePcxQ`ZE0v7Ja@v-982mI4O6Q0xRkk=eM*%`DxJU_A57iv +zXo0Vpv9Uh|<#z;RnIRgFy-p%`Ze3UO#NO;a4d_09Z{Pbe#|rDXuhW_ATnOS1n5l)p +z$PFZjx=&U5huXTpQu8lh@^)49f5<>+U~lN<`r`INe`v-f?n;=TIgdA1Vfh>YseO}V +z^AgB_BI_i@v5ltz1{L2!zgoVDro-DOEX-TDGMGHicb`{?A=zo(O*_ZwSVlpMpGZR4 +zO@RZ`ZZi*fi!f||P@)MZs~1%#Oyh+I160IA=Jgvwj5r9ikChztlDy-2ju*QhKAub@ +zHVe_74+UOD@Ne1I^mFPDGRXg;7mBtAY!G%eo8K)5Fc2-c9URojFzL(EUQC!Anwmj< +z3m(uI;a}aF{bcpQ`v>*qE01^QG!u=cI%U%oZiN_mUIy}kZUA8fNbb*}t=}QF`l&2& +zv{03QJ1960)5nQdV)Ex)a#WgUc(o1BcM~?enA0HA>aSF(`-$FS3}s%;&f4LLyQ}GQ6-)e +z8!|KSeUgZuBr}ekuoMeQ#Kjvi5zCGbf?}!oVZ=518o1A<6U!=H@)L_h<8M)U$^sAl +z*sgi(0`Da6Yu-r`t=EO`%Z2bFu;Hv=Hx?G6jR-oL8e%dhh+(Vqov#a7Q7N25mn)W- +zeuRg@Urp&@P1|wG?z1;UM2E_qWws#1oAU|AmgGap7_N^k%eVOytO^EdU=R!iz{MsH +znvclX@fngO(|&(>hDzU3_l4u^eQ~hqBKpzbp8t?Y!w2algkEUw`gMd$3-+##Feo{;7FSlfe|?(sT^H8_pimWDvLgs4v7g{&qik=3hvYD7fYiqt(5OT<6GE__*g +z8X?IJMj>qaoCw`w2f>hXjH1x!p>2r;N^$8ZdLZJ8ke#-tb=1nAZ9S-9Cz*eaf)Y+s +z8%KhS!ieRJA~7MHP=Bgr2;lfD{HJi4E9zsSQAaPVTD*|-N|mYhRAxUq_yFueW}o!k +z6Sw${wEraov+NsVF2~-;n!xj-lC&fUM4H4BXsp;7B!8Hyl)GH~&dOYw5OAj`;jU1r +zi(P#DeVw|z1B4!04Ah@Ko1W6ZSXr=o)<|P{R>e~&^JI0X_i!u$t&tpFE2XsXzzpnIj +zZRmp_y_U*i0HFTBWfEE74D6A(jyQz?iC1oayv*rpmO}icJhvblubgOeLiy^$(6R~} +z&X$CPnDK*we@e0x?3g+vy)YmCd2jNAS!FHpN#B9gpsg-v7NHb`ZvLY&_Jo#=U*b;h +zSzksFP<^r~j6f{EdZZk~>v&SS2^_umRuSU0Pg_3H!UFZV2}fjuIKX%6H9G-$jznI<@2NY+}5Ulx)OF~<`5&})!sXbY}x +znGFiz!>Qy4{q}~o3P!?EquY7*Oy7jpMud|TjzuEY8S2vQ&h8~)o*&*{w7_28SQw3> +zN!L5i^o%4_#=vqpD+DEna?```0@b>2Lz2iAXVvf|zr(HItRYDpGRp91TD&I^X=}v% +zI>&CNr-dm*G|m$dJY_=fDiU*_G6@loV7v#={O5s-2Vn5rD}r(V#89Ydw6J3| +zk&90bv{K9d(N+b0-S_Pzoe_;8v6j>~{4qMV2?>?!Mn5zG=GvAgL_{&pm-*#yZwD>9 +z@xp{XmwxZLD7M+G2J(%V)~1HQ{hlx!#b-)wLMMi(SCSQREgV*X0xl63vLN<`mXsK% +zXe73-C4NLVuK{7#A1o{gmX@m0Q#Q5*u&sp4!owbxT8{1sD!Q$HQr})+uWAU8e~E*z +zh9se*|6X8UFyLhI2j)dDr-KI(Z4YZQ=Gf$=IGlF@w!jAI1f9L5RmdHNBgI4g-uZADg02N)&cl%{$qb9Fz+ +znPvUD)DUZ_1&5OGoTO8-^CNTiye^r6_he}ekF1R%_#oQ#}U-TW|g +zmi0qAIKyS22#uetsSFiaA$E +z`$b*lEA*B?hsFo*kI`rI@O5S1T{$kDnAX{rb8h$Kqch1b7&&+*L$7dn_UAF(nP`V1 +zCI91Je1eh*t8;>zDld=2*EQ=Fyp{qlm#71AO$_?6w2eNjy`Me7R +zutUP012V09Wkh{ebW!&qbU2X`aCZ>mr*x~{#yGuviJmO5*TXs@C}_^bzXpuvnW?*~ +z;yaD=t6ONe3k)DBj^4qVSBV_kyzt$Z(vMEM<@U6Ggiz@p8~B#|rr$q1t22lbIdq8Q +zC(zl0wHWhAgm$Rm?GDoGqchWx|8Kk{o?yXOmees>8HW8u=Aw +z<5*G`yR&Q-1K5UIF98_r;ho7RIAk<2+6T{{EqJ)WB^Y|ziTd9O_OjB1;i@aXI~Ipw +zA+cWJXu#lHU{X)w%q&&^C^a^W17DSh%)c|lyq{HD)6Qyg*k1sM0cLG6*;%%$XTcoQ +zii(P${P_B`a-JJ5w}YGO6-**Nr)h1sG4nlsk~i8?`f9mZ?#oXXm&6>&2=#=%$n(P39JfxG2DM9Ho+S1G>!4s=zp5u;}VP@RjdbK`<=VNp|yR3Ot +z6-%a?Jvp2pxAy^6buN-1gRZ7(oSr)i%xQmsQ%E<{P1Bg4XrD*x$LlvmI7V+BNysS= +zOSz%gkk9iz(92m4S@z|s@N}?fZd(fS5=f8*9gCB5eNhpC2w4~|@EsUEz|?JjNQoT| +zI|`pK#6255z+5jW!25u&I_Ee*tipVdCr?jG$C3#}_(Hq2vXGLr|`ig&Lt-X9a}_9}=JhRgt0PI`41RP$4a +zZA!WJRqMOf5|x6BG*`?jcG-WtA7+pmS@t9UV&1^ne?QKWQ`7KL@K7;Z(!YssY-VCI +z(GBr+Otc4ZG5;PeG3#ktG>40ANgfl*BVv!h$(8mhh6H1M0w*}WWWR)<|b;yA$dazix^ZUI!iJV%-Bl+g8LjP +z@Cs;}bkI*H`>n%RQS^3ZllP|+2jN&_i`N;|$N=5vZgThN?l4s4pxDB?b#;6!6{m0) +zyDVR=!lD*Hp3jF~2lNdF)wSkE*K!a~-plY3T7>8|2d?ABaAnUtPVnnUa{yh+dksU= +zu8YZfBYOON*xV=|1Py$9vSHp7K^pJml4tsdvhC0wL%ATZDc=r@4c`EE!y<(0SSOUElfV*YVu5sdS +z&QtraPEtmoR$YnKyPe;D8vv3`+W$cy?53iZImHdQ3TAQ_#S>;TUv2hlDGdJ3>PZ&v)lk?IFc +zXTQAHc1QJlV*n`dxzo0L#({A&?|f0#{jA{Y3U=p}+i2RRVIku&a#dEAv&rQ%w%w?h +zQ~ek*GWthIAYX<8=i^Qt1kua0RxW-H?wKQx!C&Dgww83CS+K`shHu_&hrQ>v0#r@E +zltQSMlj+pXQG_|h`A*!p`FkS8h=_nhszDGF$WO+KF*-_Y6GI#u^~9oFbUR4}@~_^8 +zD&UEtVzNRLev%CjLDLB$FBnKS32hTfuvbWUOGGlW87X0t=*b(7t%}8z23CiFmBZto +zj&&U8)ubH&Bs&s2trIRmVqbNsnndLV{_o-=Rdp`U6hyzF~JIz-tM<<^AxnRbPufm>{L$_yZV$)4&vU689t3waA)`s2n4=9g=W-nT^Do4K$#uI)ULk1pq +zlwMaH=Z-c==$@S}OdqifQKXq>iYL&}4~v+ve;T*eXY!NkBqb;4(0bAujZC-y5KG7b +zZ7b~UyRtP_^eDugJ%Hn;7;T!oVi1`s9PPO7Z~MBQ{jozg|1b{hc_?P5pKI<}v2xZP +z!SOO%>KQ^WSBooj`z^P2D55&ux=X}W?aF#odfco1*mdl*VBo@M^~xTf*hMG-JB?d9 +zlK;)TA2+=V+!ABa7f1h(MgJ3yGC|AA;MW@WI)YX@KRmM0b5j2#2c=x5ZR+Cpyu1F! +zCKvUcPi!7r*133XXl*&(WpVMd!N?g%3r*(+#vEU+Ihy-13Ir2Rrt9Y|xHH^G%3{s2 +z4SMSurjHWGLmYM@c2|*?Hf&f11Z%!*8+wKT5rv@!BND&c47dXLKIwRe*f$#nK>zCh +zFZC54r;n8mtcQ&s6DZwrmP$-??O807|LbcQbSG`^&t6phS1n0_S>4+;CfG^{?UkN= +z7=GxNGCbT(Uy_|K?^c=qwQYyI0Ox!lv+;oX2h2T``_JS1k8(bP<8w$D4RJpG6(Np^ +zCsspqsSSsYPKLEN)U4rBIqC2p4_L`{Q~EO-v0Mg`7H`4E#J_@fj5lDis4I>McyB53 +z@YL>j1?X|Z0``CUk4V0TCMX{DMci-hei25D@c-M((5I74a&1B~_pQU>uj2Rr^tAu$ +zO!#+fC;w7?!tejx;Q#*C|ND*#?YjrsYPTRXjt~z?wyRaR^T`9`POz)r@oFCgT!{8Xk=XTCrS7@rj0_7`)XSu=7B{{r0_N|5*8E&YgDM8dr +zTgEt8E_u~%B^8Pi(?7eF7`gD~U8PU%KmPec2wBSlpT7EixY3XKj;!JOK>JV9*ZFIuq`J2d^!_WKy+nLgc9X8RKyGE&#xmshA8?io?0H@R6R +zhWkySpeRX^X@-Cq-XDll?X)W7G$3#=yeV{bbN-92P4AaX%4x{UqT|4d&GUZm4)iR> +z2>w{hzpYV?Bu>Ff^`AwMj*V%GpdH$WLFfQg#TXuph$RtJy5>dnR;`R9u!z)|?anxI +z#qy*5ju=vgV*u^Ri9Tk#@>&z4L!S71M(btC>@yxB4eXazh=^ieqZGLzgy9O`f+wH+ +z^&0)N?%gq$E}Ybo-N$Cnw_7}j5a!x;Y3pw%cz0GKb<^SdS?HpCfEHgE4=34kS8jhZ +zxWJG}AnVIC_?90c34HWnkG5{IRUr0_q`Ut`N{9Ixcf7gZ)I3ejdh%8AL1-w~J-yHu +z5tP$%Z|;03hwH|yk!{(2Y?2rSyz3G1-~CV`h!=@Nh~6WWx(l=$SWL=>B%Rl>S9-x&Zp)D<#qkFuA!IqwtWM +zDxNC($-Y#L^W7%wyhE&lPw#gGC5UpnUOQUkd||DNA#v986eGwPS<8Cr{EDfaiu8&n-Mx|{M&KS?^osJ +zfxT(o7=!&g0rzxrt1*!E_S?Cun+IVK+oMq@vWK%+AGONmfQYuwB}DU$UZ8@aD7%D} +zmaS~2%~PRYUu9FPGJ|^7B~q(O{G7hPZdWAE!aSG+R$%?x$8glCH2rGc!s;%$Wfrc%pY>LEPkUr?mXOdY(~qiQ*_6rs{JI1~AWfxX`d9qfRTH(Rb;rK_ +zmW}AgRGK$l15Ey%`0Z3AH4sW2>4;&J2-fi(!R{UF{I_%-^XPVim^lEac<*-n|Dx$F +z2&nyhW+;beE{k +zu7pN%nWI^e#;x~VbjGKUf6UfzlK#4VwWi5DdPlz;|47DBRR7t}BZfq}jpxMrt`WRt +zlk0*ESZ&gAs`^NFBY9rl3B6*|p~!2&w3pnZk?Q@ZcM<6l>D;dO&Eki+4lBlodE+uZ +z$81dU?qlzA(hV=}NSdcp*vFPBiXMy?vdE>ph2~*dVr;*i|6u;Rb@4>`Rha{JqDp0p +z%X=r=s90%dZxh|%4r@Dr@@^*turgM1(y4}(Ud%1B$^>V12JMt{J__vQ8^b)a92|OH +z=hs{OD|A9Wm^F&h+sfgYNuF2miui%Y{_blflaR&UK$oX}B#YQ{9o%%usC?xK4{23W +z6#pl6ihW^A8yLQkoG0dc7o|=_bQLIOzv_ggr>Gy3GW2irj(!shVBiM^#o-`4Z?pRf +z;cUCJa2)5zH8owsFIn`JUQ2^(pXS9B8d@`igl6TR|=_+6R8Ovm6=}7r`js +zN3w#=(#7&n^8|g<(cf=2=So6awEpXt*3!C6VL5_WRyoHVS?O+!_MKlu;t2og;s<_; +zgN?ssQxoOo&IRjlB`dLr?OLDT@_!6zMey>VcS>~q*Aq_JDWAhFvR{vaWJ`t{$9zG@ +z)ay^K&9qQ9;Q{240Ss(`yO{U#3nj}FuIxL%um>>-8+6J1|+uZ-TCG}OCOmF!c* +zEOQ?9rk~YeTchd>#C?;Lb`wW>nOhrVGaTADn+iT_#9yDbBv#d-a8x25$qTL*XAt~H +z_c<^xHvmJGTFB2?q?9S=P=`0Ve9Zq@GZtX`t`Cmbf8S&zO~1x#s?3Qu>2F2ZGB+rx +zlP|{57ZZikZ=t1&?lRvr*DMI8Czk?EKY*?-z7<|(XI +zMFJLyR!cSyD$;mc6~-^YGKqcr&x20%YeYI`?#SbKtd#&B&p55cXI2}x^Yw;UGG-zX +zag)z)L4Ru9FW4FMJssQ%lDj)MndSCGlDOHPh>dl=Ah>1LrF&;~32cuQVsXEjh<8a^ +zr#tWLWto>{k>!AD8?A}uQgFFY(njN2&_%nS#1vqycbI6FsH242$c6gfkFl&h_Gi%D +zD6uX3Qyb>5M9e8ghJsbDJ%;`}7IFU+Y&RRT(fvRvWqiBa`y_rZ8&o_}#e!m$i#Sm} +z?vx2ReoDVAZi=NoiN;bNtqedp)WMHCu=Vr +zy}{NO{TBm{Yk}b&N|Gfe(1wM;7+EdUN@4z6_sY>bl9Gekix1lr%?Z{5wW;AETf%#; +zOM#=~3+!`^`!J?|_8Yy;?UPuq6W$&Y&PN&IXi25d=EzAB=U;VQ1o~RvP+i|y| +zyDstTR~E(5@08nTH=%|27q0 +zp4+F@H(T5jEfh-~{CsKDQXY07ccq?CAHEj(Zt=rO!`AFqeqAmo!TFwyI+jHH@W**c +zxtT1a-|@;`{Lt{klN?*1^cu78qnKr{;GEQ>k9^Y9=M!ZrRS{XuZjWOy3|Ns#+=r@8=* +z@rztiUU*5Y3h8r`1oW3_4H#DqO1(VsuqqKHbiEu +z?HU=@_z1aFjKt5JqWqFr1=h69n@uVnHwyPTr=(-2jLKJi1xZvoAM@TI|3xvrEGTN% +zlwEnDaNRALl2s;>U;iqB&@B?DA40!0o>P&B!M~@yQ|FbrD +z)QEF4TrVGwnUG!o +z|6Ij57KO*$$5IMYg7(~VzWp>LjnBQwLIQ*vSGkgIf4kpfgd+Z@8a2g2E5a}e@O<(y1r>D*Zg3d+D{y@wR-Y97J+cDgF+2mXZcCk +z9|ak@&T$vQc}sAa2_?|2h2P)ypr}=OdoDJM{D>Dcn{vC?Z9=tWlv4yLX^o +zKIyNsMUjj6#CJpN11eeMFluDj5!Rdb|o~Z-8Z215xJ8P-Fi_3GYuFs*@&Y$Lm50ot_C{!n5n?gPWvnund!CeR? +z#+NvTEvD0{Xu&v3c`ub_x+5|vIp=AbxDxPGz|kC~qsr&%1P{LNjD4F~kI<6zooQT; +zWG!15f`S;)Z1iL9k%UB3=_A+eQ926r;nvW +zuD@YN?6~L|DDM0z{ODG +zaQV8szSt7fzW(p}(CkAGs@$YOnQlh>^^_ABRK#)ykbUj45Uhw56MXTHpPAlCY*tk< +zsukG}OL5Ms5?Mnm{}Hb1E4XO~TUWBOAr(1+I1E-_a4J@QR1^?tpcz1rky{#8vn +zRL|}fjFyiGR7<-sa%s7`)Q4tLlg5Q_*6b6zO1Equ9iRzcb1mIO2K&j6h}??9Wd~Fv +zyTcuTXP%}zp-!RM7ri4iZXI$U^BNCNhI@a(lFRdUvL`*=q^nV +zAyZ~q4uRy8T?OJMi$ZFJB}+zMX@2f|{^hF}EmoIn5~KN%^~}&|7BFLw&NW_auv8( +zr(#rNg=^Y;mylTX$qPLxBffO6@~NK1Zv7!wYo74A)fa=e0^Kq&EXLcU)O?YXLN;?Y +zuYK9wg1i?lMKIhkY+rx3_gX!c3%EYUP0W$ZNkkaB_lFEckDX+OQe^IMFqkis5=>1| +zzuvkYwfFj1d6Pn7#kkyZmQUy9M%(Ul!W-;+Ot|>-CM~yGk-&HV$6Mlsz|-h;%z(Qo +zEjx{;)GrdI=PzC5>~MGO1G+3%r;^ODo#F5WwIzXV>2AE8V3ai0w9npJ0gE^@e!;2> +zH2+hzHn6l5O|BOsd0=LEZ%veW8bdA!9=@tiY8xf;_z>x^(DUGQWsu=pg==a7?1H&9 +zW9d~D)^m}ZdnL#H?%Z0iY`6w95PQIXm)>8LDoI@`Q6s`HqJV@;foMep4wC~M +zU%87s8kD0J`|jtLgPg_053yOXu^)1`vXF~gM*F8DNppLr^~8$Va?4FgN)hLa;_;`hwtE +zC8@`-|1VEt4L*7s4`h890ztshhiYcIZ_@07;lQ&Ih$_cn0AVEjZS^yf!Z>O-O<@fI +zg)4e;s6{NoosC~#cG8(yKItSRFE@C*2zr4slU5vO_SXYRmV3BtxMQZ?xax64#X3uA +zFYMTkJpUlC4FaKx2qtJCM--f}k=6PRx=zAo3aD8|tdq!80IK9s&Tiy;pK`nXf9{@_ +zQP#(;QQVQ-{lk#5jKu_~r6bKZ6)__3^?KW3Wn$W+kQ7VYb +z!)bH9@wFdpEAI>>JTYz0x&1U1hHXPoYux`?EN)Y3riqC3wfd12jE;NA9hU3=5s#k$ +zyo~i#xQ!~h<5L)B2|DKg*zeeVhDA3^*&_+cWtDC_XAyLhz3F#NM26VbQv9V{hV!?| +zf`^1FnB?kIJ_a@=O&x!0Evih?f~jM!Z)Gg+6(hj&7o8-Zzy5Ubdj)Hxfw{cX7uUeN +zH)v4%WQ(vSZ2T9n20&5|5$NEBGp@lT!`7=_ZD%iVvc-szaKDK8*2!M_GsK4eM+EgZ +zz%E?{Z{-80-v#eujf(fQ-WmQlB?x9dQ#GC7Zo+xPh-^oUEt~c2T2=k*j}@(m;HuJ^ +z9ej8VwEfl5SdFRgHtr|>CAHW-IED9s?6QR>#kZo`R)N&Lw8o_rY7}%H^{rF6_qxny +zK<%pTbKBR%otQcu<#oDmUl~}tKGZ*%Sk6J%YNFTd-Vg +zx&C^c9K)dy1Koxxh4bKF1*Zuxj9?UK9Z!-Xj`I+{p*oIotv-89SM5>53%mWx1-@6o +zVzy6`O(HH(AJ1gAr*86&TGj<}{87sHvQK~9E<=cLPQ}Wxy3BtU3@i$Pt0i7(%eIN% +zt!>(BEO!C^?Yx~uNUX7~7VR=d9|VFBlRRV+O%Ay00S6rPT8rj&Vru3ysovDCmn>6? +zT;2xe#p48oquAa^>_N$cMxz(mN)<52>P>itZ%W33)wJdA(u^uPp8>+Rn^}U#Y$ZF- +zWR)`&ZaK%@>&XSi55xhACUnwVvCT>~b%xsgnW>Q-|cRU +zt#x=|`2rN?n^o%LHI|}K*)VQa>{uJHH7F~T9`*S2h&!}b#j4hb7)B@^w4g}0!!Td! +zzh%S)Mma9)38luz+KzYqI6W!}RVAYNUP6%z6)fqB%2^)i!+ahV=d<_~>`T*TM>eog!a(92i%}#+^Ucn}L!E%Fm;k +zZLmnHlSpc|J|LGUr|gFha=srl;UA;xhxu-$xH5Rq$a-~s!@nH;Fd8q>RAnzOO%8Zf +z3aKHJV924E6XTyJ$*7yG(uc))A?KA2l$5gNV@aPLr3;!|o7`?HmTKi(Wnpc4)U;{* +zn(k<)wl(r5veaw8HvWticL|M4Y7LIH;MFyoW?O@(N!3%}`uBoB|K3@1kGxa=mE}JD +zR^)*T6l<%wvWvNn)u_8z;WwFa`8Zvhy|mV*vVh>hT(MK!4o;(|TfXc{L(WM8 +z&abRb7gE=MjoHwRQy6}=h>MGTY$AZY{*I^1|9D1biAMt|S-L{fRL-)cKi0flu>&Y2 +zP`8lGpHIhE=4{-oUX&Y#sKB#Cv`3!i>;oK#K$>sO*JB^{w8&}8Y2T2x!AtyP_ZBdP +z8>S{^+V*3?Dp*>BQ)uK?74D~#`#LD55kQe#!>XM%Qen~%&)uEb0|cz&>;uDAw^2*^ +z-Lg+4-2KVjS(I>wkn*#>hpKadE2>uu<1YkBpTRqvSgn}y+g^2SJBOt_gn`yM)A}`T +zi96F&ZsfXkL58A!E_U9PDd!+do>S_{(+~06Yt0HJotlOgiGX-Qh+-ot+tdX4`lf&0 +zAc&S9>6gs7QNwOpUOKj6Pl9zm%Y}`>ITRTV<}R`!#M0;RxOv&$jRHu~=OWgJR0H;; +zE5SG(qB%D>G(>)89tCBAoDW~s==9B`W-g_Ppn6W8YE8rowLX`*kQdsV>=B!Z;{&mz +z6g`w7o36MM63#_O<=mL%%PqK;NgR;B!Q-TGeq&9N50X2o(hNfTzw^MU6qtH<`Z>jC%;!*kDX +z9QONKccuZdpDsi`vqoni-O`oK>MpjXU{Mb~m +zL}nS~W5aNDmlgkr_f(*#Al1>RRzDVDnB7|FIw;*(CMhrErM!0TPl3Ce1)_e(z01G9 +z{>bO_d5vs=8Cm4)=ttLldl|ULr;qf|rj%6&U5(2v>)Y~fYGakF?xN1;6y$i1i^q+* +z3^l9F)ZaoMQvT_GvdM~vVj)&CASKjPCox0LcSZ^Ap +z3CZOv{c0zoK;Kz>z`PCTn{p_gl0z +z!t8?NCU{s3{-vMGVi?=2daE*7F;3e=D^qZ_?@3a(ESVHLTEd90FU{?WZw+OgY*eVI +z8OCqkA1UZB!4!I98v9&t*fyeSvexGNuTZ~vm3wmx_`4ApplEC-iAirG*r>xYVafu~ +zIhm3pMo+srAV829>t2CfmvN1zKrF3m?zj2w5{xbR1OcmC?p*b*B2Ba0cDF_a3L)2N +zQ2z1|cHM&y@hs0CnUt@f6;9cl=Yz8bov`v5>k`4Z0OFQj6|yoD?Tsr30Oi@b>=96k +zX<()+`+m)qt9O(Mt4{0bzqNYe_wP1F37bLpeN~||-XoYfPw2Y?G+ZtTgIJw&yUrnV +zxPHqucolM=E>u97Ew1?q{~ta0#5M^>7`8o%o%no6?ZN6INnnkKIr&NKwY?rx2)$|h-iOf6KQxb5u{2O +zCswB4`RLDIvaRmCyJtu)!zLK0Lm$)g;lgn#?QEfvmoQ4h;pAMI4c0O8@-@sNyZ}p? +zPz+B4yP}2_G7U1*Ws7a8Wuse$r+{5$m3g0kzwwQY8pPvbg%u5z{1$CM-FNm-r~7FB +zOpzZ^pQJ_kvBeI`4B&+GTA5g@YJeiYKZ7HS;8R0if~$9u5k=DB_0x#}N*H4VQTMK^ +zDTUt5cNwy&^Tn8=U?0%}r4P{Q)l*|mnBNNS@P#7)u*?d}-mLlIJC;F!VPn3j;tM$0 +zXwKH=e4h8?#hh(-J+75vo!z?2j&aK?lN63IoEaAWSBH6LM@6#?2n9hab{>APEDPOw +zDcMwC&&w-3&pd-)N3%abkuVdwXV}_3L;-yfV)V!)2qVU-0$VYXis^-bfGO~5Y2(Hu +z?)Enu7n{lV*>nBlDWaCTLHDiWjY;dhgg|sm32Cc;Ynq3penaI3 +z&#G6p4dxP}L`3774~O)#(&&H95%fTUn{WUHMTN2dJRZ;rEbq8%F5QMrdqGiM6yE%{99oD +zo=9D=^klOYgv$-jh^b4WzWAWQa;poPHSt5=nNKyH=}bYzDttgm^|)zn=_10BrV%L(`C#R!(&e)XoJK57y^gptW>N~k?sMN_ERP)5~vk# +zU~`+mF-4pRN#BPSD5$JLda=+7CiW`Wimmb5h4O=~328Jg^LB}# +zldR(F5BdT*Xe}0ThdT(;e>w`F$XHov=laKrT)3K0%Qtl|>3v{Ajgk9OIIEjc@&Evy +z5^ybRR>c0^DOMxG_npzJBbm3THk67yRu>mom$vyAAy+P0I&Yls*Stj) +z;49{h&(~TZB6Mi?TW%)+g?H2Gy-;cO*PP{HTZ$Pl+ci;D7ZHD^SW7J{0+vB`k9#=6 +z5_}>{;`-^Cp8dmBOp(S~-^^@yel<@ZH^!A?Q-Pq`->_my@Q=B#H8BVRag*p|0|-EM +z|Jb)QKI?@G(?IaB%bck}1CCIT*Q{C{Ct)L6L&D%NS>eH$%?^IomWLwbgh4v~U)#M8 +zNr|#s^mm2MH_nm41PF`9pG(k&S(u6J*pH|W6GJf +zp_cDWw>OgBXpXR&^1V0ZT7UdJO(>EW%Fy_9g>3a#(}m;h#$TlX7_8$Na9*n`^DdDR +z+o2U6r%p8(Wv|T6R~`N{Oo-v+Vi#%)i8c$kqoQnS*#1F@inQr030bX4Je?|&&MSp3 +zPov%B)9vC6z`%XEizyLSLB7(@q|v2Tk2?e?C@C&^yD-pyC&H#NSvW5V>N?-_kHzM@ +zZy%!`ISvJh>_;Mm>phF>L?Q_^9bhmotq=U^zTRkCUjC5_A(Q(D&$Hz-nHe_Tzc`z_;nLq;4+fN17KX1(0f9Z)dM4s2zDz3an1Yw8a0s`7#YoLJu2*B}hAnU46IywgENn+d +zh~KLW+Z(lNz)a{gxeCqrV|U$GcL$kdZV1G)$fcBhP0k{XPQ&n-ELn>-Hgd?ZcaAAd +z90Hrlm2}QMeK1Fx9?-~V;+H(U^yo=Y)oLx1by^7m9sHVk77Q}t9ksOr7FgV`ok?LI +z0KogJ}th{LgdHY$?<^htJyRKlu#ZeTj8+{yK0_Y`b>8?(oTM|9V8WfQ^NV87<5 +zzX)|GFdnH}$Q6OZ-vGh^Lw{+<%x^!G*DaSKa60Lg!j|rO2(lvva9HnOA&^xrCuZc3 +z?846P5D1+fl)%P5Tpq`ro=rSA6hhF1)0LbhmP3g;ry$7&cA12XE4KI)TjC#w8xf~= +zIJ#ToLnI;MD!8{2PKbnvQ+}cv-LFFBwJ^jQ6JL1snM>3>(5JBz6ZTeP7)=9tf4PjH +z#vKaB_}*%Fvf+de*PF%+j$)*qcPmc4$9}X2XcvN=5zJ>lG`qvYavZY_Oztqt9rz)t +ziRK@VU+5YoT1H@HFYwxy{Prl2zr@r)LZ9YEi{`0>iwrCSuT6Mu@DRgi(tUaPGqKtB +zwxf`c4FtSdlPd>DvVLa*&EVHX#0-Zw5tK7!RQ?qN_AGM29BA=wOFC+5th>CN>^p!3 +z-Wx{G42Z>n;aJPF&hIPr1Q10|n8lCdBgK%|2Z--L#kWUHpzSS&l3vinW}>mmy{NYS +z&n!eVPv^#dVmkUb@7RvjC$YN^X{%I^SfLap)3d&Oq%7dzz}!iBt^L*8uLSB{-qW2x +z-$M`hm4Y7DJm;h<+>kLE0ro%7en_xq +zjb*Q5G?dI0fEh +z`N0-JK*c6gGQ{AQ*7`v(`1HeNE2^1%T8V(5rgeREh5o;enXnmDg`6vgD6=QCPMs-D +zRrCD_=oYM)2a{6@S4Vy#L%O3jq|%Z-r$9DVuwT!JL6_^+2F&RiS-0qagGGdel13&% +zKmF1Y;F(ZyQu!Nc%H)|I=|bB}BavF?H!+SCaXqb(d5+~C)=I=gpk@TD`GJulg-*TH +zLWQ5adC-+}XKLnh+IO~?s_C@T{!xoFMV|J+ +z4e3edSFwPNcBnO#jOpqF^B*p>3ix^Pzdzlh!(KmN^8Kgo=n@Zi~m-A_U_L!Q=vAD2K`@ogL?_+$^|I=sA8$O +zyw0QXcQ9quYgj^!0JKmQ>RuJa4q~FNeQJf>+R>%tP-5(Ip>KE%pKZk;vwk!r=M80X +z-H=AutR^4+u(nt2^fjIr>l(#5zymXv>-KZJlB;j%W_fVGS5-&~z&<;WIqZ~!DR3<( +zbvac^7s=Ufje9_AlC8Hpys;q*A|cTrk?=vM0)DM{EQa%PS(ZI?Wk5NoV?0!IoqH_R +z;ksn`N^J_W_;jfa5~Oz}2y!E$F3+p|R>L +zq#MZUvU$Slt@V+;f(CoaWyDfO65akSwuT8;_qz~-O)}?L*(0+%IVkzBk>7=*cX7#a +z+E7rDs!!c9dAD&~TH6`a2eq{FHK528BXX>Ctx84Z8_REb)t;EIwg2=?ry)lLcPr`U +zCp}f4LPQ6IfL+{q7*{2pk63B_1JBjvd!k&m%6GpFFF$Ml8yuztn|+iF&Mj~888znx +zR6#lEox%KMjj6EGHGy0|&SCnj+w$UgT7|TH)3Z^gQz~sIr%zAa+t-tglEpalOOGhI +z^Y9wPyb+Czc*!{-9}fBbVY)4(!Yfy~_x;G7UF?WjTa*bYx>o;`RxP`io~NdY6Nt*v9+x +z?3h#}@=%CL?s-gJU<8-u2aWV5XPh2*2T1ib47=B^uZ`zA{4aZffXR;c2t-X?;wMF6 +z&ZtlUHZyE8LL{JqYnG2Zz1H$G_h1iA(U+QfRjz8K#-j(rcxo=;!$bgpqQI&=4*9+# +z7j0l(i0Xhe*B2g%dUPbq{%J+8R~Yw?s{BsA;N3&_sqxR2VHO`&1x?$G7osPMA;_3P +zF|I#xHY4eDAOr93qW+Y6_cjK02F&x#MK_SY{$9j0cT)l-V;VCjf8V}e*d+Ep$cwQ^@vc#J!+1`VOB6&Yc7sq2H>VF6AR8g2&$W4k`WhXp9IwAK2 +zQ#YED+&f3H*JZd7&dVsqwJ?iG=@8eq>lZ`D1QR<{q%LM~o@T6p7UI$^bP=dzJtpP`tsCaqC;kAN6d#4`3M3KIN3;77QjmDlaVW5&Lz;dBT@LnNLmB}PJsZtd# +znWhKtK+|yTWU;I)MoZx=DON2I#ap~3$hVEyR3Psp5g~(Wh_L>l!&Gs+fhr}5EA4@L +zraJpTxU@vj2+gbH-QJo{Duq1l=z0d8kz?<2*7*x{KS4yDxK>J6zwsnIwLqNUT@Y +zM73TfkB*_nm*0t2lfEk-UVeF4WxKzb9LhP4JJs8jl0FhjsrDLXG%tD0$*jX`8EAWz +z@KsX9%`yvwz4?V%VAtVqw?I&2R--VZWRSk*798g=hGh8-$TQ-BBv79zYsGF}dC)qtd=Z}oBwvqJ4EZ=zL=V7PR|LFTkgh5NSM&!P>G*GY +z|2_aSH^Mmlx@wb_{srXyJ6-bI1!B}MUa+EKkrVMRU5c#jm(o}4`;+pDb38A}vghCS +zBDJn>H)I~U&q4@EPjv=HL;Knw)F5njWzq?EPdEsi)p9NM9Z{Jz&#qYU0(mRTtatk~ +zS>1>&{NX-~O@H8-WVn8TTwsqX;59+U)R4DK?*@h493(A99bJH0hL#aE;vgg~Rq^aj +zjPCm4*vntLG5;Ziq(vLNx{R};h<3QGg1-nO-sMF6K9?J(uwv+trQ$kp&AWMl)9� +z1gVY>p&Wt`Wp>L}KSij#BkPBIF}$AJrqkwNH5uSjpmcou)q7*hJ&v0FXK)ncks7Vt +zXV_%cg!M>wB97wss#m0MfHH+&`HU3~uvYiujDzuiim@V)N6GU5tsn>Hk8Kp+#m)uaL +zAFr1zy$@hDVp!mWTj@w>&P^VSnmk3a(`nEPA4Lfx>O>4v#20+3j*%jObaN1|ac8E{}Zc0x8~5|gLPt3kuq55h-4{haIoLfgxNVxFfM#gO9SMR&Ub0yww~Q;@ +zE`wH>B7B9*3#y{ndhl*;z)FHkIu6`Oc0UD6M$1noMv#>d&>WMTi!=t^lFqC#D+O_P +zUJ=K+;FE3eV<`~u%j^72jt;nu{=#TAwW2_>y^bMTYwBUG?vXvE?t>gmyE|Brgt?Od +zz>e@0I%y~1^N;f8pggMfP@S1Uv7-YFe8~Iouf(O?O&bK_=W6|BS^ZzTOFl7_pahY%TWO8nnN$=kgd1j145I~2`8*b|(^X5r_)P|}Q6 +zM8Ihj-=(tX2WCwUir@i>p273;rHhlxc6B|LShNP6%H+uqvJt0-`Vz+*X5c;C*#4%| +zGm1v2KQ|{?5q1{Lb!od>C+h!od3N0>ECf-r3)A) +z%3}ET({^sGJCc5em5D%wyXvgEKXmWktQa~!a_wMG#qFK +z(Nj>WP&Ky#Yc@~APdn?~2FTcub51o%Ncp~mgP6A#E$^P0|9Nb++qyaxE&aTFEl2u- +z+<74YUQSjh-r3<2?DiShmDmGsd;sWl34c_rl_7J)#PI +zEdr&3Lf@S!k^;i=QeQzJw=Uoh2H5+1+&tMrCrp}IDnK{VEGTsKJ>}`nC;MK$gX8{C +zK|f3hjXd5e#>DCrrxE|i8{TMnoLq|e&fYh8psf6=Pk64hpegZ*uV%p+HV`-ZVW4Ah +z-Yj?5=AQpBhnuj9YLK8weeV1^bYIRdGLHbH6O>DEqn7~=2QD?|=%FZldcvqF1thfc +zVsPeo3X4NRw|xl()-C*tF`pkD-lt-UkaNBGDs7F0ic1f76bR5}Etq#u#05h=SFcpd +zJkA;O_$~LguiTRHj>7XVQN9L?DXGWI{U*4mcg#$##n=y-*P7 +zK6FtH@d#BSVB~zS>?e?vwrFA|6QT%$j_e;00$7tSGv*t&WBHJ`E%%cz&@S@K#@Z67kU1Rsga+|{;&Ht%;u%|NyFsvGJ%0V +z`C2va1;_9YC_S{oaSLyG8l<@#A=~vkT>bB0Fhmw0O>pKcIynBR>61(efve(i4P)=u +zwiUihLKoKvse^vR;pzTA5?iGz_2o-@YU|H`SO96EV +z+1-q;gsJ*1g6EJfIQ5$OPW~Qxr?FvqjV@rB4uy~vC4UwSkUIGM(2{l(^7HFoJ$NE6 +zuKq{V7eoVMXxKmPy@f}fe&@G6O1##g#NSNRfpAQHaDNdn(!>z$%)d?pJBFuBHiK+( +z(zV}b$SDB0e6su`^SR=kC<>=v&XkRpds!L6_)R{wDR`0$fhGx>aa%l(N**=3N1BKQ +z>`}Srbku*QzG&K&*d9VGp0 +zS?fY@1~pr>ccoFF(aaj1Nt&I{L&L!Or>qVJ#irQZw^G(948sG~2lmp`&YFp5A`oy# +z3S31{q_4-?^V|UzNw0TnUv9H|_y&?CNl8J7PjO-SrgT{I9dE)>xyX}KF +zi6%JPR#Q_Gk6x`MwgjICyQ|hJvvhM?{kDjIq0)%&gT~Jt2PUBxZps^>ohpI2=?X@c +zmhlVUWpaz1n@PWft)I|T6gSE7b9#gb!c~l_1ImnkTE2Nz9%LSX7W;`bIMoQa03-49 +zAY(Z(FAm-FBtwt%0uBa5o@IQSWvKc!3^sB@)JgeTR)zL|6RF>eqNdtO`u-$mKg{}C +z5_yabV|F+`W&SApP@?!ys{GV?c)c@QLG3FOlb8(DXq`?t1+a +zyP-d2$=YI(a4Z4SLXL$hB%a?0e@(^&4>j_dFC(-HkMXa#&Kj!Y5lasf8b*Dn-Ol~p +z-c%v)Pc7;0ldC17KkiL^(sqll^%V5W_f~wW0x`8~Y>)KIW* +z=>B@~(B0TY3}5=$*pR!{d-LGTi}<#FAG1s%w0P@ovb23Gne%q3Z^3stf&7pi1)&q> +zrw@8YJ2GM*lz&>!z4kCsy4oAu?Z0b@b?O`I!jf9O)q4X|{6t%KZ+~=uW1vyOUbopR +zav&LUE~4`GUzf%6mt&F-G#j3BJ^Vjh$%M8+4~}@ymbEU{Qs8jiOhdV+@-hfhdyvS> +zM(+i6c&_^$9ry0_%+O=;nzHF?91kH7OW6(2Q|ZMK7A3{TMOopZSMX=1fmYKeq)vd1Z2?)g86>DB+*sCcR5}&d +z2{FiuzEHJLwV>pCsVm_-<*okHP`})wb_ms!7-#RVuFw@gxfWMdu%#=Y^ZrAw#jF0k +zbl@2iDyxwzn=-oQv}ioUmA>9nI?rTgXZ<=mH|Xz4^>PgQd{E=vFzx2Cr35&bNljkh +z%}V7fxkdx7Xg{;wfJknStPuhwxAiLUK@B3dp_TbrRaHZ*fkKmDv+3UE$WJqt+Aq$P +z=H_tuDhEX-(f%VQ_|jisT+t_&H{RMmg*K)?ro8T->US#8iQ3&tQXj3DHy+C@h6HC6bz=V;#i70OYRUd!7L&?#ilO +zGPoTnyLf+{go0$mc*sUNn^pI^Dz&ZC!)}-ga-ZHb>$WpYwY!gnEv*aPNKAF#P@|uN +z{=RWw=9Sq`ILP7)S5rQ5h%5Ovg{E4l1k!x7sN%~RxglylyXpIoDW7p>Hoe)f)SC9> +zy16xJC;P7TT`kk$c6q~h*|D42AuEi_7gUQOLDOZ$!U@!s@F-co{nQ2RN&(HWg60Zz +z{Ge>lnE?8q-|fiRgFv{rxDnVuI6{cDq?!0$f6mOZu>*kpq0>g=Kf4vpaRSs?@cOPR +zQHQY^clL}qnel}v(5({M$)N&H<)QA{2zdc$-3-pI=waqKUeewOL=0f$3Rf<+ol+EV +ziI;sZJ#m7`d%4X~?aB8q);_aSKh`(HJJHOo>RIVpx&7R3Y?1bvJJ9!UWDI%jN;$Vr +zPRGVvlT#&^w2#>B()QBbM6e~A>_+6~=uPU(o1ErW1po7dd70eOLWlCSwIjYKKZ5!$ +z8^Z6J!{u3G4RzJGp&!MASw}zCCR9I4{BVx0%|Iq!ghfo&EL(3D*&oD5M+;V!}a_PD8Dbayfj?2{A<;p!G!OKn?8YcFV{hB2#WO5&}M)Z +z^h6akbN1Z>Q`BeFL6syXp<-2r1?-$w!JqR5+`V3(MO6i5#2Imio6;Hb#c3+O{aVVz +z*9od75w1Uu_#MHtE3yB+-n`kpp-0uY(W@0nPQl5-wLXKRCLKv;eAmy7E8-7J>j=ON +zj`G-QBUF9JpDVCs5KtzM%`ue8c7ET)b(a(i*d)x@`YfMXTr;=5*sx3*SxAT9tBWtE +zC5pQF_OjyQ*Y~wKjNTX~rjqQhZD)(f^pn=%D{uChP}Iu^`e|fsWR@`&5hpTi0RFGa +zp7Jl4VSOI&FA`rr$NY2W2mM{Su4<@c$Y2iR@V_H?wt?@U))sRf5PUk*sl+yp>-GDF +z{pSuw>{?V5QPD$H@dV9DxTw6*9@9n~8gmbAPQEv~h2E9VDcdU2zJEWXM +zY2V%UEseqHOVP#AJg&XJ3trPf`_bul^_KE~=WXa+CA5cWv;2$-kgLr{%>37=N68lW +zN8IAgWe)BR?gho3*J=Y4N@YCJ9@vf6)`b~y*s#cWLcZb}*Q)VBW0fE;tl(@qoH&XJ +z@q-G6(|nsUNOnVYRtM0XYB++$-)03DXmr)P^EHlxY>KxhoqLe*H-!ap^HOBmCdFr^ +zfzNeKapbFSak;ba?!3E=nX!NKi|AMWdklYPcIqcN7!(y-Ej3Kkh$!Oflsm{97!uXK +zLZFB}{Z!`eR#yMjk99aH^E)De8a2hiht=N`zaP+bbc-)t05$ +zT~pNm_Fw~ToWq5D#frdi4nj=(zmFKTce66K627Z0ta%g;{Z0FCy5JCgEtL9n%eR+o +zGV=&c-qfxfny{>4u7W@D`w)i>)2*_^A`+rT<%d*gS3b3trwMv@U(BYsPfixqLe5uK +zT3}O%TqDSeF{HRHx&emES@L&$QeH0ZZbiHaT+CDNH_m+>C8pqP_K`tqD6o05Y)o8@ +zF>c9w;9D|f(z#y0caHyYwm>b@jN5tGeoL)pu;~nKz+yVX&r!!l*s}DVwOOJDM0oqWE}d4Y*y_g?+OhrK +zE6`rEBIm`0&2h$!`HQMQ6|W&66n+15(d}O*$s*0z`UlS$W{kvPNE53G +z`p%bRqx587$lNw>TYC9o`BCQl8vx@QD{llt9)* +zi2w(aM+B>^eAF8JiTlDkI0fV|ZMm52!kHlDfAF)6MHDtNXoDJgRjfyGRaUbw9_UF* +zb0yT9caP}_<KgGNbq_Dz +zbVhiqrgtPzdupYKooLuMvGcTI2k> +zbL{0H=$u;d`gPJ=!GY9xglz2$vQs1aX2^=qrUHh%S&473EBjig8c^a5%1VPzkdwvhPh}D0EQ1HK5UUcePwXj>Xd3|*B?4?nVTUL|83$~cVy}-J_(|_RV(23Y2^&(K6|hjE&yW^w +zq@FuiJ#38c%pH1v?d)cS_tpBYu7L(SpcUFBPzm)HZuNH=(#8Zpx)M`0Mv$xkO3*KK +zi)>E{X7`===z07s8O<5kXqDnxCaP6TE%%_r&Yj#Bf4s;)<~4IUZu%aU;v5PCee_*R +zNVr*eXr;*Sc2;2rF1X&~0ObcL{~pb4iiBtk5@CN$%mT1*!1aZ)0k=$O9~}JaC?7e6 +zs4-}@^2a}$KtOQk`>$<;a(luI?V=mWO0u+xL0|Tvd7EN0+$gJykTc!49yPRvztLTq +zQv4|rher_#qFjeKyfw-*WC!I|%(igw#W_Qw_xp{K%0Q_IJs)&9ctOcWpC8Y#6dRN- +z^M9aTedl)?g)%APX+<41lMp1fZ<4j-nnO??$*8D6LPTAXP@f>l(ccx^f%RWM6O>_3 +zzSjE`CFA@Egis1>H}7(CmiPtu+8lSd=r!`_Sv6``sTO?jdGQdR2!Q +zi42_GBxz525PVf_%YJiKPhh6jpG&<^il)tblfP1a@^ykrJP>?}pMT^e0KWqRB=}|M +z0pXz09@sY!ju!Y`{cLbgtL7u>-xfM#ow@z%KK4QCHK8aiLZTAzk8?WQ322_zM?zA0 +zW=Vg13oi!5gYm)8E;xZ0`uCWRB>602ONEXBU%DQs2jR}$&C^4LXh@3v>3n1SRCR>& +zaN#X|IfZ||0Yg`fogN2Lt~`!dFkv6p&!>cFf*U#a$!lok;0pkrFc??xL71J>c+xVP +zlJ27;Jz_6iOHmFR9WVd3GKPO!S%^Kl*}IP?eB02LqxL7AwD{@y!DnP{OcA9uBWyK9 +zBTZvWy~H6l!XYN`Bm~M@(kyU@Gyb1&X3mo%G%tveVjd)XpkA3>(e-c9$>=>Yz#->l +zLNsS3l}q{s7vAEY@emh+K7_ONhnXS&PYI(qq6s+xtSo|kf?hYE?bZ0o(GzG|u*8+{ +zw)zV9TJS@*hB}IFTSZs2@Cy(FS=5-m_k+uu3R3eDFJRdl|?>t%V%&_ewrLL5%zn*hE*CGFIpMUvkrOzU8(A2is +zd0-dRQ1$n(+qymEy5|}HeF+HmgQq0(;e#S4I~;He`$QI?-GhsS{XpO!6;B%y(GhHn +zeC7BXLaHV{Iy7>jjsLljWY?wBF$(YL+W<$zI$(d=UNn9-^+C*q>kX3e`v!EQ2nl10 +zehv1fVlDjg^U(YOzu-9%=6*IkQR?Viw)4imV&KLpF?yJ4TW$MRc?pZGg@uuiPPq8u +zS)EKALM{e_u8)i}!*YzJcCy9ZF8htFe=&Sh6j}?61I?61=3yef`6YS17rz^ad$t1- +z#t$G7cm>tMPl3CpC%Lz^GxNn+-+rXJy91owjWzKCcJH}Vo~^b<361SNx4j8@o!3lNdUFx@Ew#9#g +z67545=T9(-eDglEc7f856!_E=k2AHQfjvcA0 +zdD-So6iAz++K}}d6ak3o0f56_!^gAnP@@2a5YSApJyr_CbaleONhFqU&g}iIZx8cI +zY|6PLxQ%bKs-^49m$*lL)!M7YsRnYK7UF;V02R3^qzKDgiorTV3SST9q$KAwgYTRBX=swiC +zR|<)9hzMEyj6cKm3`-@&joOUZ&I<;;+vE2W)T9-3HcAC}a%vs8OTse*4NqS`0FuTp +zCYT-c5UPc~WWD`-7HZc>&jG#tZ(lY%sxJ=$;e+8_f5>2iegDMv+L?TPH*xa!oxr@; +zmI#nMjyL8wfnev{8pk~gzbfdu?)g&LYuff>H;7*BtXXv9*7eLSU(=g=bG!Mm!q(xZ +zm-Z1~{06{GZoKuU!ItI@KL=RA#JG6UBJS5TgRRtS&Oy%MNqaUi-$$qFR{Fix3C|yo +zE86yD*#R&;Sk?DC)!kU^N=NO!%9ZwZxS98=cYHrQb0FcvKjgnG1Bzyq>{zQrbN>yA +z9);Rc38e5y4!nqf;9{C%MEn!BL+vUi8S-7vV4hrm$oI&Pqla%6-7^Zw}OK0dGn#fLGu0>^EPl00Vh1%g?O|V*+-_?V>3x{Sspoe~>0V_M1a( +z^)At2PQ%!}!svt^$orCOJ;@;+)D)c!Vl{y7(4;SE(SL$x=5!ib686c|{wEDz9%& +z^dWByr)SyR@&3ec()LP^9XWj4s<=$oA;;Tu!*)U-fN=VFe0AI`+98f6PwC@FI9oc^`turn&3e +zciwB6*%)c3`HD7HY`$l8n@AFm=}cWIbg3`2fn*mgIgfkPJvxVXZ??>>dLU0S2c==nCIgewYYNak`F +z38Gp41T_k-q=45l|MFfiO#{64sdCZ2>@Q`*w>z0*Jstkh2tXFqsQ5Ysc~-A=QjlVT +zzAHb+^x~qWQ(4W=)jS}KlAd5jR(O-_uiuZ+>Ii_u)G`Xrb`*Mh2{jHXwPZB!b+VV^ +z&QFMgwcS?SOom;FG9Ff?KX`?@yQc_0JFSaCs2MPVLYN`DP)0jk2eqQ`)E&DqDoFTaPHHp1 +zn-1AM0BEs*qQ4i*c6FslGa@KY7wR&0;oG02zO|y0Nn+6_W3k6>|CmA)rZk +zyW*H;glOi?G}`Gje(Ay_xL|bRGes$^QKm8a9)EmZus@rRwQhK#_oI#D#^#lDVcw$Wbb +zc0_bL)ap92gw4MT+Q})4Cn~yjWO_({xreK{SOe2Pj51AHuT{!4Wgs=?mMbpqTv4g8IIF02$ +z^%VY55-P44FIeG;sTV$96vJVCg<<(jDakUFN#L#zCG3xH19Ua#f`Ks~0$3vdO6Ge#N8Dzsye>u!%b^U2b +zPxZc-hlr(&3qsS9c8d6|uv9w^8iz|>6kJDPprWi;#8*0MpYVCN1yCtY(u)`DrPBb4TyqS)1l}VH85=Yg=FWiz^&Y8td)dRD}Ld +zZnYNHI*40~D5o|e+=#(KMxo0=(Jf+3CSLv`O2APY;3idc^RaH^C_a)D%d=Zb?SZ(R +zz$2ksIDZgo=R4^|9qpmvC&cNEg}rIRRL>&^DsfX7<)}+D?%#0<@wAQIWK~E>Up4h$ +zC~+Nfmtg9ZM!S~RAc%Z&#Ztsj(-gd6+(heeKo=$4G4v%FB^hYC7w68=iw)UR75nqU +ziv;%7R(Gu{DR}b1$TYR)Yvj!SsiLx@C+wax&-I!yNdu}}0)v@~pvx{djhpTuyRqaV +z1v$-GfKJKRm4>glEQX4e3-(2H9Q$;bKTLg}s&X48V8H{$J)P~5HEW~0X0cWK5ZDrr +zaeuUokOnG)jc#Z@MEE7R`HZ`F(3BN@fFD8%Mv{32Pn|g0~Y-wy$bc_{tkT +zteBNb!#ZohlD}n}^S8NPHCDtpT3KHjDy&pCX)tK6&0y8PTu@b!_mlaLhc!0KWaT7(p3QJn&0K1B=&0GeV<;%+ +z@V0zD6Kizav9tbSTQ+vA@zxzaS(8-HqFtw~k%`l$Muql>Vns5vw@d|x6;l1JN$?W7vfp8!WjL@Wl~ +zf!AL!1L1<;a_iKP#7koG!<=_$CW<{*Er8z053d|n%ltG8<7O|29xsgAb&-%k*nFJI +z7RSem&~K*dq>m`10*MGXVMpIPflX-VDHe!l93G^?0a|s|gzEDZek_%C`4zm4QVoq_ +zIR2tB!4E-iFyEzgduO0ukuNLfG2K`NZ}7!`83#^{nvxXucZMUTcD`3EgxR +z#*GL|X-tnIMR$m%qMRGetEV_K?LhvTH|_I8@nqq`oNGK)bXudVRRMJ=N_kChK?WuviC`~6I?3%?a+2U`%9VR3ae +zCK=ZcQ=xCC6x$L4H}V{a5hK4@K`8ox<;V*X;P|tRr~$LZnx<+%GkZ+H=8~u +zogwk62qaZU^R5IY3FZn(Z6lM47F~CKAYRSR11A;03H@jewKOAisGC;%v17RU5Hy +zM`NpI$ZXRyD{9MAqLOk15^SOMg_tcvScEIR;qRpqBM^lv)G*SsytMGX?0BWke5*^g +z8l7VM>K0Wxp(EY^VJL+m9ZbUo50DcKz`@&c+RYH@bbls%q%4_nqvWpPr?G1|U@H60 +z;q~xFVNJ|qkjkJZCoi@V`dbW0M`sV*#)=b5KzavLhp97MOjw9E7-idmu6tB$37t#} +z{BhhTCZQE@3m5TTniqfvV1BY1pC~JjL~^In{{Gq&##=VARWJ4>2!y-!oFuxsGNMSU +zr2v^(prLssyVn=6v7B+4h@T}{C>=Z9J8u$`bZ!qXZ#T}kxg9=zwj8T^+>OVbb)o5m +z9D*9JW$R_=gxbDwww=lQ;i&+d0bJ-AtvPk*nEWjP$@;!9m9xu+JTxQb0~+DL;%O;& +z@&LV^KY-m*R*%DW0;a7rGID=e-Wz0~H{#15)$H^Nac`=O>P*`cOGq3cn>)^9F^{;B +z8o0c|v_1GP<$22vR +z0_^ym#8XwvfrVEe{RXYv{2n|GXv0RIBB=d0U@e(&V8%z(Qy>glA|f2A2FecYq%n*h +z^J#7HZj=)0ARpj^Pp9`50ZB))SxmzkB9xZDbdue3dFFg)Ohaq9vdo)%KI3-ra4y3Cg1;cM;17Xgv>qqkMD2M4OG!SX! +zG8wR(tH8nhjd8z=V^?yMQpXhpTSAalIwdNHf=nh}aM+becjJ_D`Nqt=nln%btfAB3 +zv1nX5taPQbn+1p9n&dTH(reb4#R;f}h$ED!)cXd_k;~TPA>_qCBd(sz$T?9OOcI(7 +zAe=_^hRk)wq5Rh|s|h_as`?;Wc3sdXRk13-_|W +zS$s#pU?kH(A)cobN3DZD&LCnjg^9kBh^Chihx*kgaE(`k0b^jWLQ#xK1*GB_6h|&D +z&5-$wmka_>keP9RXtCfr;K_sB_tD*FvQ#xRV^K{OhN)|wb7egm02Dnd=`ltsXW69o +z4_nCOGjar?fL?sz6#`Am%3ug#>07OJ01r=<2pNKrx!?v)>G6~Cj5}FfaE4q{Y@T(Z +zx4A22F>gRC)V1Ypdk&}h6AT6wD}vGIp3ggiXN#$GJg&_0Ics9s?t(*}i>msjg!iu+ +z7|sC@@22WI4)UsJ?cMd}$C^kLg(D|Jv3f6in!{nGaj!yCEXU2({Xk;R{ryy362d)` +zKXVLeyR;eCcl&fjaHYCde;vV%Dy@)cqz9|vvH=x^=LmI{%ex&Lt3oLAK~;z}az}*_ +z%BPg?ous9wnfZPOk#v}e^EO)00T&2`)N*N>Dl4t!qJ=v{P3+y%>$eaBpD({y@*Z>X +zQ`JAt?h_+Qs0sVqZCuAhPy&hhp<98y<|APNPFsPlFdDseGx6$vT1;x_cBvl_E{B{s +zZSN>BD9Ijr34|Ge9zXhLV0MU(_O3a&f=6IhQTqAU$p(LBw&>+TYfXOnxu*97@maaWqxpi5yBHIVFn5*38g@rzB3Qlgby6UQF?`Ubl7S3Rk$e^#@L#sTkLJsZ5rCgD~r5r^Zya^|^NB`gC`2p)cMKuR^sA8Bp +zs7)Fm0B)2J<(sJ-J8Ujo>YjJ`_$_QZ5nb^g5+<^%KM{xDpCdbA59!@B01!4UA?Y4j +z0ntC2k7Q4cgJRi52G*B|pk35Nh=S7vD!K_`D1{?K??SW#Ip8r0bjVQ9QT<1x6KhDl +zo)@`aWmBXxK!v=G)i})XgT@bC$(Zz1e;`H#ENe&ZN<-53pI>O4_^l4Sjgw+=R5Sk6 +ztkq@@$K^?gg}X=jx8VXcs+O6Liq`6qq4luaGc8+C2w4Od92^~d7U>x0;Fyo^ZLuyAxL(F^4(2K{>jz9s52@{ZKZDMVs +z48qt3LVW~kkp+;62`BKf9tG7x8N)H7Z@*MvTvwQQdd+b&EcM*GO8l +zU6RU4?w}??SVqW;Xx^Awl9WxiPkc7giR%vJ8qDv{hm%*U+y@{4pbOS0rp8OU9oDA6 +zOc$?)U4D3}>L13m2DnB4L_ABN25s2lj0erHZa83!c9#YtpueXBV>>waR&c`X$>;DG +zN^?SSc#_XJDArf;Dp~Bt;#@Vyl{W^-u<22;uGb}@J6Z_F!&em+F&V`MyBM(rccTuE +zJ-jc+S&XHra`N6>F=PZ2Fb${?LLcr_w12ibF_WnV5Fg5>;ModR_y)!7G}p~qK72!B +zkDKZtTfN4sPHohMNa=Q0ipsv +zE@d$#M-8~L-o{m?jNrZY!kBL#2S4l$7sDa7vgX^e4$~?el@5)6z+fa*BO8ecP*Ueh +zAtN$kG+P#lCFVbuepmRO5mRX&@`=&gQPLgUTd~!kGl*!z#41kG2t_x-4zw)_H0(KH +zgoH%N?Ueyih^*Dj+~Z4?UE%>2zMvC@y4uIG6_^E({!8cI_Ti=?;(xt~LRY=vg_&SG +zb7bD9&FpMpKMIh4YP+(`Jz#s{)DixJ?HvQA@)L(t2q{F3#6MP{krevV-A?&4AkF^K +zA<8c+^0P)42Mvq&pTf%of}k}6;oxCJer;o{(Bg&#ae^=lMV5xh1NdbJk^8m5_L$bL +z&FVokCWRzsCNa`)WDKry3r5{tZUfBSkSg})guDVuzRV1&kQ6Z@;ol)D-Dn$IlaL>v +zUhYLK(W$~Xsoi`;gkWZsrVde~YMt3rxD_xIWwn#$D_1|lp$#he<(wXAD5#N?9qtD3 +zgq^;AP~CPG>hLZHnm(vguEV_4o$9uU06>-EJZ{Ls;v#;DBMG=wmbkJIek9q&=uy}t +zxb$~bT0y;wTqq@baz)C58Vjm2G@e9%T?)b%sb2z9^FO&CKF_xJx +z0YD*Rsvo_XJ`;P*nS#!BtMM7|$!(ovhd|BE^}yyq$AsoWc5^SVhC7zZ*=_cHgr{J{ +zF20>JTsz_P2ki*#C-^~h4TNvAV@)fTKU%Gvm^9kAu;Scc68q0!o%hY*;1YxqtTN_{ +z6h$U57NIUF%xEKX=S@-rp48@>BV?hv(X9w5Lr}M&1D0STilCewfog-^!e7vV%0S&Y +zTi6yp6WZPq_yclqK!Qmq@gN&fO@fU`F?mHkKiPddQ{7XXjd1@w*Sx4&I(U`L1YV~=`%%r4=h +z&3$&-gn^=PTZ0e%Y*mN$%k6pBc9`2;xhJvnE;p#+j9P#K-zkk+p#Xc2i9y9po*w7`RA +z#pr;70n7wAfKI0xz6zx~b}*Jr6Jt^PH^Q&u=r$7r@kjys5#x29&(P)m+!`{nX||={ +zvsAco%+ojMOQe1bJwmelUSCtnmRNv}ye$R$)>^Q$C0-+RuIUYqiO`Rn9!3TP2v`6G +z<>}*p8sHu9j!y8=fR!4FE^>M|PfPiN@Hh6|!VP#5@+eX+2l&7!L#^fiRmaf;Bek40e2O#5G4st6j2mv +z1{Q*zWvndFzU`wuh<7ns07nuomv2P5)A(|mlc`U3BEx0+-vkL{y7*pdj=W4ftALSOc8D@1u9YwcS{r9-zx?$hVS9`4nZU7 +zAgDOfZ3s7Ql+axGmA5sowgL?UVAB8lY^b@tf~~-Bq&SYOzcx%nA0rs%j1*s&Ir@-7 +zztPb2#4?h!)@;bjO7@1h`ChNxB>0ByY5+{Y8&MXvZU8h7CKVb61`&#WW2mf3w}8JL +zt`Hjj801`tj&Xq=gfhy%?6g*@-{^ppn{&&)z#)RYIO+^QAq-^_qr^0XdlVle=&K)< +zi3V7^!)VoR8hkNCEToW?FVhZS&aKeMN@T3~wMUX?ju$k$K*i3qmyUFDmO`tf1 +zdDoc}N{1)#Ou(9p7>EMut}+>SOqbdfU)>gDlZ96q7@ii!1Of5uFbcM|5vdQw2}E>= +zPodYL9{Cz(lqOpN^?_5A@>U=D2SoPHiMO1WN)_yt!)V0SwUv9AD+Z8!rLPm}H59ogKVg5$3oS8H-*=5ya;h}>pY6W?+tZajH0pvy?j +zD<+%d;5K8F`~m#%v~D>ilzAXW%m +z>bFGFyw(1OI?{l~RyJ7ikKyW8*T0neW`W?U^jchSwO|V3cysYL>g?rWm@V~ac2sDp +z=ED7ATTo4=_FQ>cUa!Tp|$kq^-J_SjTiu(#lY9T5Yky8-GcyJEU_- +zd*i5YXz>RE&$==E4+P5nod-K1N-~3zD7pLB&+y1c{wWaYu3gg`Az8#3`!Syzh)Sew +zTx&Vq;f{^yuKUPfy|Bhf@CzT10#C8=Mw89DiH4pGNs*cjQuOy0TV0F{TiYp}VFl6i$$!H! +zl$Ub}5KchnVfGAv&}n)qr+Ka5-)>>18lhMqOY`^~0@(W6oTXF@cJIPgPTqJTplhl3 +zNT`o~$Un}Hmm%Ksrh&>^A#IMRj3dr}IV$n3KWjN=bXDt*5EYfqNs)@CQITq5uOi>o +zZQdF5Od%Xe%&iE1vqhXJKi616aCxm#chJ>$eP8nDx3A5s#&(cEmt5D(sm1t+0opUq +zJc1>q`TRCxgYWM{ey_dQ54{lKjVHt9_`Gtt4my?WZmC5a;pgSCT!TbKv8FnK$=}z! +z)pC5_t}{6~OyuG&)8D|MUja53{$85VxJo0vDV&>Y`*M;f{aNCC*B|s&%>Vo^bpU` +zjNXl~CX3#PkZXkdGB0D@O@DV1I%T?2mg<#KR?O%>1LD91-DsiMAtPPsCaz(y85Dju +zl~|6SXNmx@AwQ9mh)ZZGti331eE7IWJMRoBQQGe-9w +zKF=tHz|l+}cepZ%!vLjMnyjwAB!0vpCVhm-*gctFuZ=(+b6?BVEbYv~woIDuFpysP +zL@7T8ebSN8^!ujaq-3n{ds0P@gmm$bCj-JG*|I=q4FfBu(iN3-u`-*Q#mqgNwpTL{DKB@M4QArPM|}~6-lKT%U_p$5`fINx9i~G83T0HHSl)$IW~?;x*6Jwxuq*o|)8a-zfe`bedbF@A6B{9~2F+==pn9 +z{r~H#qBp_^wiCJWaMD+7l8W@ +zBJVHA^Pjc){y!W1Ul1b%EQ|2}+t*I@1m0(Pw!Hs)Y9Ee&-Jt#)Y#@Ii$TQf{>A7gv +zc=Xcf{Nyjj)D9=tXfWtDdxOX0ev^2&OE1b#HW-5OV0Ce8jH%b?x*| +z-3TzVvXUtukpfeQ%<$9-T20tL(G%XzvZCymZm;SRB&aF{hNOI6v|-%*7iXHH60YSy-iXJ-hWo^xq>u +zHr4;HP1%zhSXw*tKtG{Z?yXdL@pj +zGPls2n76dvZ;s7A*j3qJmfd7!1&pWXZ<_PWY~>@j+Syr$x-s)s@VX&y@3E2{(~eMm +zkd&}7lD#~C+EB1E(l2jX%S_#1MOT)H-j-KvaC*aiPvCDe+o`031SPE;jmn +zB1Qq`W>4qob{B1ohAT0l++J766(?n}OCwkE*=_hE3i=ZB#S3ikCKq)%(Tm%?)6Vq5 +z!SJ6CZ#P1`6%Oy}Hn~9UVl^0EZx`Lhm+yU0?m39l{51|LwmuM(6Kv#`_17bOS@IH| +zY2{bDbEkWEf9(Znl~12&&M6Sru{>O>_I5HFHlTK(e}$78ZabAfJ&3S5IG_3e*sFft +zZIKQ_g=-$kMoD$2=8oMu>ER(|R`AcDczx)+vsn1Paz*;1V-93bV +zP*;cbw#n*iqg9`MoJEuMLaybq3Yw-s+xI@}HH%xLjbc4L&z{NDEUrtPJAetj->^k& +z$e%LNqqY)lLQoFuU7q37`EIfbtS&ej6|bI~*Q8RXHWsv8d3+$c%!%K}G}e2u=M@!nW{p1v#1b(;co)Rju8Aydz35T9ZkgZG&XKMX^| +z#|2hol&%6h@nbVLdZ>x>bFOKt>mO=}id*$|P+>)zEeUwdCj{IeAVyi|8cR%YEL3iV +zux!DUc0MaMs2B_wwBjw?V7Kl5I+AD|7xl5o%_%(U{%QX`0VT_bRp5oo~BqA5|KB6Je`E1?VrkX3s*)%bY;k#gd!GoPNe|xIZ6U}8Pw}I2f351+4 +zMq>OUd5_tWF#bdXM-s6f!UCZ@iJ!Ys-G2x))yBGdG*2M!<87@h9K-pHjacr~GNY^=-D`+{w3*CvM06 +ziY&J;V*5Ekc_a1Ax}~nx-ffaFC*L0hE?Szz!QanMlO*E_{BD0XAUq@1tQ7aG-D%6- +zeC!k4+TAFpEEOv84J?I7#SwD>En`CWeHpdFWaNo;nDO=#UHdjxvyb~i?~7Nr>2vyl +zUK{L?n-_EWzaG|>%)>{9LehHJ07C#y@IqDi{sz_PITwpE1Wt;Y?t>MhT+SIFZd~Br +z@nO(huk8)0t0nK)6FUqW16z_mt|*Jhs@wCR4Y2u+cSfdez?nxr;JM~rDDnJWEk8W2 +zSw*^ftx-zqaZ>+m(Z+#yDD~u?tpIgitJln<#=-#Bl6tA(Gpy^yLFT9gGCwa(|Kn|ySC +zY*5A`?~JSVy{h!>jX=sxI@+G|;v4fXe=8+9g}yh+1BfFDm4tQ4=>jWbU^jBo2Cak^ +z&GLiK&zygnK3A3Kq!D{7j&kK4JywnE*|>d^E~XQ8{O;56mU#|tSX8^f{bXPXuC>Eu +zUMco;t{He`oP41sOi!=f^)9%379fpi;2;8svPqR-+rhkuUEZp%|LU$$k3IGp@jirF~6b&ilJ(9B*M`A +zx>QDsWzK$?>4Z~kjCQR{(QcW-q-mGxk9Obiz5Loe&rarR0SGTB)5vP5O#;KwkBknz +zakg?Rngx5FeAlsh2f`EU**h@@3hv`3f0qppR<(G}+$e^uc}t`BOYu2xmuf3Dj`Y{; +zD*PP8QMV20Pq#!ZU37Q%MEphOCL6{*CC0)7r>lFqVy?qI*(<#Bzm2o=dYp8RpTYa- +z2pF@Q_Q_VPEc|(|zUKuE?XkYW=DT%O=VR>}r{PsL`=PJtw?l6)%ou=)u0nnPb4~#* +z!9{+E^W963xwFAenI99NH=pzUCy#jdPag4>fdCUKVKYL3ql29QuBF08#QuOyiJi;| +z1lfUjLC2udJ7U4I70?qXpUw%?d?H@l6nW9P{EUAhVVSnryq!hl{$!GeOdNM?(C((W +zh`H&yIxXkk&5s)B9p9tc_o;6LLp3Vb2-f(Z8;hP-JIpov{^XKj;Y_U|=-_t4yZk`; +zMipB%g-b3o7#Tw)-Ro>?^!s=QzbkI(Po1FXMG!(>WoyY)`;@Ql_^=IMkgKqSg_{U! +z8hAMjQPFS8l>$N#RB2>RtgOVL5)kAuylO7?xQiWPS_VGcu=#K{ZF<10S7~)kRJun1 +zIQ#0Cbhkd*gQtBvcqQ_mWeo5(XKp)| +z+;@pSRp>Z~hI|mac8)r()fju&Jc?g(-E7wKk?|hn%wQI +z^UQ)8vx075wXx&+-nO@!;H#vvXL4yD?`QvSshwwhP9%8Xb#*NJ*}DPjjGxDP*7Nv+|!k^b5pYm8#-KRL6%9&CYuR0f?j9iu2} +zNLZRSeJ-dG^Un_0{w6V|f3coLk`6wiWCXBHI$B!iODKk5eVsz@QienE0|y6Ov4pC>sKV +zGth@v_)<-^`Iv=YvW`1v&Ybe+;tE#8^-(r;THt?wFURHO)^)4R)*ne-FickW>P~rNg59SH) +z4yAcQX=?gFSZU$6GRm^h<8FWtzz?=Ug~Kr863`XXcrAG*%d2}CA1pE#9K}vn3pyEf +zTU77U=M*HKJ8{~3{4D)ZajTbO7?KvZk8!J3TM=nX?6GD2;=Zuf-!>-}Z8 +zWCtb}hp-?Ct>n6j&c-}LQkbIKK4)zqJr423gsB!C6vcyIO@E3@JA8gBl#?nL=O6W1 +zp`hxZ5ikze5OAW&3mKCvweU#vor=YzZL&>dp-iYH4$x){!>?&j_&)E33KFGsx}dzh +zY1l1AXWttX+U~2A_hb{0_w!>JjDIIw6vqE`zmD)sU4+_N3SWf5J-@I26V?PF=S!Y}kw*v!W +z@=_)Rpk~9X)V!E`*OH}1aZ8$eTYgIcdc@1=Cvud?qFO##NrZoLpz!4=r_~1~pDVq7 +zWk+P)sMbxMUF-<#Gwej{b~BH5*m!eauJXR#L+EDZc)gB#KLX@dgW$P=n2A)1xx9in +z8_-HY+t$k0<}C=*e?#1V99smeTvB1X*osvG2IY*%CJy)zm)~|!^Zbo?q=1m +zaKCr8gm7x@9>2u3dNkdpHg3&}1($83?F%LN4xRgX??vzXz>m;Mht~-XHTm2)jqj7G +z>*3bodaZ7Fs-GEW6!*hjdjTj59jf62udYw4d##Lnmg$q%PCnx|@LuaK-)Dz%y?Ltx +z_woaK-CP$?9ZFkUPlv6TzxcPDYD*e8A9Sk{yzkx7QnVBL+*25sf2*V{bo2RLGj$Q+ +z(t3>LCPmH{S&Meqp+Dwa7%pSsJ{BRcE4?{U1Ja~fJe|X5uoGOXE^Ctmw9MONlybZ0 +z?B$x0GkV=`K-x*!?!9}d*}a<*-3%W8*t4A#TB3%0nP-+~##nX>`IxEa0n61xe92{2 +zO5#D5>`LCL&Ha5Ri^B6E_qzRAEB~*P?ha +z+Q9eew0P(U=tBu-*!8QTFxmL^E5Y`DwySun@cR5ZBB&;(@ZAkhG{}wwa|6ZOLPswA +z_U9ATs+6%@aF`^F*nMWtG(tZrm!HKm)5(w5Lv``!-);I% +zddXi~fxu+qv8Vgcy4JIo?n8_?MCH6G3(&W}m<_7VidpR+>yyA1dhxUPTL3x_`d0yH +zML6P)hK!EM-a*{|M(jnpbze9b*>=L>z$C!u1bTzN)_2lz)%!(fk#aM_RsEHd3mUxP +z7gjQ$Ly5q9i@#-Fmb;R +z +z?5Fa#R4J=yZm1ct6JGD-B!h+%-^ZyTecZpOOog&8ht2j@vZ{oSc$gWD$Tzy((^VjT +zFrj#jNlc`Dgw@6b?tY2dJ7dW)h~tckQG8(ZbDg!)lp!#A#MIloqAHu?Y>eq^LS0V$N8DI5z&*|c$d#BgX@_?>d$1uY +zc$LS=E(e2wJKV7Wbl6I(`H`ufg-T=fgr;}}A{l~(Hk-8v6B2q8pz^Rlzrh&WAKOqm +z0AK&d{EYF(g4S@Tm3O=t34K@i);HoYdDGb%B7=01aF6GAXCA&K2*b&{;g%Y$MqTP@ +zC{#3T+%k7u-*Md3z{>u68;Ww$jV;W*ZW1~0p+mpsx`u8OHrE+y#GekzZ7VUi-1;qL +zVei*siqi(#0!NfSTT^*yau%f$sy&6Jd%sL-jqE`2f*0@pvY$uVgdq(WvGNhoTIu1} +z?zrloRBt(KGp<7S-*QV0pprl(@`INw-G`2Nn0U-_<~pUuZEF?eS~Y=R_o-4O(Soas +zn-}i&5zgQ4Hw`F|smaC5Lq*0EOd8RD##<|X3yvi^f8+BCs2AgYu3hSQh4vgH$~11s +zIV7z!3N3m$!hD#bhMq>&y-`b9OmS?U3tRa;bj}seQw7xUb>4*#4euErR^Ltyz0WY? +zUO8 +zG*}}Vt1~mYRmy!HSCYx^mVNyF?6={|4+MT2Sy<^+N231Z894_qvDvcIA*qtjA1+#E +z<#NP2n^tk;2FJ!A^1oY5F!^GEY3C^?ba7!wA3m#r9#R^LrMMxNm(8U)ojytH$(KuC +zNj-fb>$QCP8R6MO$m?Xc)iaAKp7t}sVf>wGJCn3TPjecY)OMVQnsiGC5sBDC +zJj2|YRH&!QFdwk2nLX38QmTR;TC7zQ4`9*s+^{tevzHc_;5T4D=jD3LgZ$;YJ8q~L +za_chcTw<(%WQww7{>EsCY#+wB==6QahB1~ +zR4WWFt-8FEZGb!f#QbS?XeE8#=tk~kZzVV9nxO~bo)p!jPwwXe?S$LyOoSt|wCZzb +ztLLv=cYG^OW)?JNBY#ovqj=cagSnMPaGS+)7M^`(@b$nf^-jZxpBU)|<4c{%fQl9& +zj;nUdzhHLjpC0jton=>uN!P})Zd#UPnnRo{f=E>hwe=GgS1wT1Z> +zqgmBS!WAvzJ_*Nt&}oF$FpZOg5Q@4+g}j&lLY^QYhO9}6z~Pg5nl<%b{qEf^Ytv>^ +z2f@;CIoInAJmmKJrOSXgvcn2veaB9lKJ17iRui0bXC#O_kck;_CV4L2Dt<|NDkpk% +z_n(}@#Psj{7Y8oV$RG#->^_dfg{wrvo{XoT@4q%|NPrzQR`lR+E-c1MwW-1CjEXxW +znJb_xiLQ(Bpf-MN_uoM(jQz5lQgIH(-^%d+(^dNZ&su~m%T>5c#M=T3H&XJy#Z(`< +zP-eR*oz6PNd&5^isv-yE5*EAH69khhmPt%q7o0C8s;$|S-@7@R@QjKQ&smjj3hw{^ +zs8;4Ogcw365BR~_Nf158?c_0J!mAf?L1VP2G&^45lb>YLIPz5VhtI6SVJpW8=CM8GFa&BWMgOVu%kJp^_gs1x*T6JY=EjVyNZCIn3+y#y=3efqKU^n*ymk2wOok +z|9zH!`+6ioV)$|YJ4*k;KQL+uZ=BA;{`2jPz`tM!+$oRRC2A9ogCE!Fe=@X7vp;GM#UJ3sD+x}eta|dczd%uf$+4J~&pZ~(MSx=uR +zNk*^!*5S4OEQs$0dxhbKU|u!HpYsM=#0#N+U;Fnj#)y-xowEups^)MghRA^HmPrl| +zJ-I0WexB>cnD4xW@|6x%2i)Uf;SCaPSp*^v^KTzsW+N_d(tsO)jm%hZ6)p))1Uh)! +z=eq&aEqo|QNeRP)u&*NG>{3+gX8*L98JaD{4Fl8upSQKZ<4)D+(<7=)2Vvs=>|yl( +zJ~zRNuTzLOa9L(6;zU{j1m#hX<0DLzQ^6P54nPAvt(Syl*<&cg_*l9IaqtOHb1f1m +z5vl^ZcRV%VkqT<SMq&;<7!;S|BmZST +zD1PQcZQ$XT8?Ob0VRUg~CJ_JMgNg-iU2b7}T0p?LZ(&8WMgVx(z#==??zc$3G5Av19N(?VfH +z-m!pG0eSO}^f7NCz#vtGd&cY?gwr)1-d)no%2VH~;rzD%G5$mns0pDhPKGI`H +zBRCH@Yy!r~r~}(09>!uFj^Qj$V1i_1RO+=iksccc{2nB~1kqda-d$K+R&c{loTHS4 +zmM4@#ZPv&C<1E@LesL`f&b +zLw5d~!!wc~8g0f6Vb}ZS#q}pth@+>p5b^;g}$#fE0BIE_hPAyA1 +zmqhKs_qzv>Gq$DZX;-C7EGn&&;&KC$wRl~Njz~%re>1m*@2p|S4P0>nq-KOm%uoT; +zR5hG&Ta1g-qY(XqWN4TX&&t}n<%^A3dtss^XpWTHt}(HTH^(%u!1$8G3ip1<8t(IO +z`)kGj8)$rQneiqi0WielYWw*xZ1sg~21K21p2w-E>3lgjDgnw#dXLC^fNzd&l$3he +ztQb>D9{?>FgqLr}h5w%SutTgxbza5E3UoE@++Z~a?qw_He#;oGyoYO&JeAG>Zp+

W4BhMPD~fzjl!@=WQ`ALVcJcvoIXc*?zIydruk +zse4GL#odycyE%7>_tl+%IbkLy(wJ|XdMAS{iHJ&@?aSrsanljC#q&g!&d-_c55IR< +zD8l?rhET5`){2V^EQLajhfirdQB)r98NEy>Y5=*#)$XGR_5K8ZjDmQ7>0=nH*u%BS +zSPAbVDIFm^l08p%vY>TzqobyQz3JLp5)>a1O<`MKQ2LP+Xpt+>ymv`> +zT0#62bNu_+ulrD%SrgMB5W_-4fDE#TRRQBdFS013?b?0X;|$cUH^Me+v4lLP;l(6d +z@rgL>-7<|w(yCUyPaq&>MK{0Apw14d&L1EBaX~8Oej5`}`e01G#O!yJeRs+06_f75 +ze7ixi$8*>GiMmou-(!>x11GSyt4%|Nm|5keN-RPydqB$3OYxi +z_2h)3M(79?0OvpRt&R8|#mTaUho5OfzJgyjs7^UM;YRTeg4U^;Ih37?5O}GLI +zDJk`^F$=NF^^-eNH95tg?_YCwhChuRKf0JXR~zJWr8jRH{C@NUbkPg*k!Ep#5`vzN +zJ@3qaPj|RGdEofvS_~x`MdxM5ADS+TW)(w~l!m^UwYRAL0>yAX9Uq7n?g@lD-FNLnqU98;kQ5TjKu)Ev4PCH`WkZfmnkwzx4jSA#mhVHqhtY^yIjS1ieckaMeY +z;*@$5ETY0DdG|S6q@zD_Q(kH+Jo}J`9Qw6UEBNac@(;4IUL(%iyZ=wB``$pKA +z`vpRX!2pw@)HWyb8hADCjKAswwNqC%yJQm)acAbKo-<&2;&AL*-KZZI;RTpgrii8X +zxD)=g4+@-?$7AokAOWX@v-a^Xo!wfhJsYGEAzDr>zs!tldTOVpF_7%96GK6B2@VZ}2X$2Pk65CN)fH +z-D1bJ-%PT>f2FugV>#H@#2*pE{=B#ex)>6Ped{A4Pgx$e*I6!;E|TqCKf#qwy&84+df1x$bJ^Z)>sR?e&TiBJbeL37)Be$@DfhnCy}@Z)b8#<z{nnV%Faf-GM$5R+;{bO#M)Y@|@FCtm$BiixlSaoc8(x^I-NZcs21`-Jnr_kqpdo;?p*kdA+p +zJM2YIm{?p^gL<&vuB!PBmS;QZLYsLuWPPq=iQJ8pe4Me^m9D;OU-t?AK;nuIz%JR^NG*BZeEPkN^Zd%k+Boom%V%K5eaG5a#}ulHP>`bUq*we +zLkTM1e!~_I3#)S5yuch_VwIXEdNM8$BnMLI|BsrdzHf2LOEy%C;N>!aP_UYTu&0ZKcATEQ__SJMkV7ozy9vz67EuZI^p@BNAF&peak(Td;UPUlZE+uc~Wih3MaDcvSIWdCp;dkSif5Gtyc5cg{}|gx`T5cX6Nlx +zwvi?U2MFR$dmk3Ap3;(z<2mlFEAIYAgchDC#NutXofFuK?Y5_rt$g!Y6KnChP0wia +z5EXJ!Pe|2^V&&j7ow`}WVcRhKJ@m-qc$mMm92`{`<1^QEkLFFBJB1U98hUBC-}rS+ +zLj$JhV?$JR4(f&a9>YWma2};PS6;)u+jM=Z+*mvtDI*zUbP6Q*Kp#K@q3oKdsBz~g +z;}_+y(Fil53mB1CgMwox`|c?7vM4g#c1`kI_DI-$a>I@#(?=W6sS2s8`@#zD9|rDX +zX=r4}Omf^2H9nUH03N~KYcqC}*PHe<`YSyfC4C8%4W0@2=YZKZM#P@2eBC4$aqtVX +zZuVDssr0~FRDHzr48bf{LrNx7r~mD~dQ}RW1V9&EZE?ey=TC|xx(|}j!ur9jXFikZ +zucY((t1$0NumPFdHYb(c>Y|RBCt>}bWB_mTnHfBgb<^+%=|ad$4uuIyZ4p!!olPuA +zGDl|{5|<>tbc6&8G`=r}Tm}GeeB6j)L#P3JWRky4gel_l{Nj_Myfh+?m%c{mi~NLk +z-u5AEru~(l3uEQog)AQrTu4b#nkcD!@trc4@jAoT5sG$N+>`{s&qwB_%7)(7Qlj(w +zDZCB0c;n$}^f0WXXi*LRC`Se!JpI6gs@v1frzl=a?pA7#1;`GVtgFld=l_rzhNT1K +zYBdig)B-|k4O3ogQL@}u^0j0nB!$0qH~DNar2{w`)|mo-_voQSdBhT1h9+%|cK6|B +zcszOs?R9j-04i|t4y6EI7oD1_-_u-r5S0sMMXIa6{OnL@slC-?Y0z$vjYdS8j8gB` +zCwu(DcLn8sQHDFJj}7Y$s)V(+ZTU2F+v1S(;*W+WI!;M~4j-{RVkNdVxcrOJnA9F7 +zjr|BOu8^wU{*)&PnuqxF_TW)wH#xNjwTnq1^@bL3XD +zU$36_!@`Nj-JQbsl#tzLJ^;?1mRtYG?V0e!*f_4H+YWLCdF36R7dQi%=`8xv;#yK=~eV +z?B{6Z4b3Tmtx%xEXcBvg&|zJ|Xx_WpT4?f3i^?;vKuzq$iSXY8Us`l1HXndCPZ{g94o|Z}lZ##}8e9eE$y&kTWWar8`7{4fLspIq6cWWofm~Wk%y( +zs^LA&I{gI?c^#@skmsP(hj}| +zmXWzXbGOR={(==>0(0pr4u7#mc5L*#uNsW+iIb-+77Cl9fd5G>v5A@=%J2~>O`ONd +z#Y<(PdiBj}#*52@INMrzEC +zTZ%f}HN!Szo+nkQX}T3ys#e*5X+R%5>Oc|`5$r)t6!w4ou@Bty;j?EKYembY$!P5j +z`<=Jv>u8?raMUIX>p1@g0CFRGFI)9pjzKIAzx@iQ96E#0{UbA_^H`qcWiBJ7zAc~J +zM@_lGNQ6Bxxzo-}=Zo&CTZwY)IBTuWKr8ff!nEab(vO|9Min2tqovjIdb(il=6{e-ncn +zBFdju(#p2OCFMMigl*`4zgz`NxY4x!#1g2q5c~i&j`NO(^qkFyzc*>Y*(cADgh%iA +zYK=-+K%y?2iQD>CHrklESc#Lfqz^YQTQJy4!Z1_W)%3j8CHAaQ|kQdv?vdO97K=5}lze1nP0Bl8Xx(WGN +zh0Sp@1?YNU7`dz~@15IGfxxw+*X!YfT=YX0)USNK7D +zYMwN$X9p&=b%u=pB^7G?y6`rwbfF*L$|SLWY@B4Y=(6eBm7T}&F;z~pGg-R#e!=;f +z@SNYF&M!k|__MAueAx!)Te&yI!@7#ASCxI!wHBa!*@D^b|KR_ME(ybqs*a{G%98;7a+6Mb-K +zh&-MePO_KDzxDHwVe}X)F&o{wM^Dx>c@nY;TN$QKJusaU1x(0Lhj0EhAer@`iZGra +z2dJOHLP<+%VPs~$dI)YuRi(p5bzk6T%Wb)Z{_!7pP8ZXdxkU1xcICn@Rv#P935=~M +z+CQPt%xmRCyTasiJ-3*=nx}07=VX>RJkF4Gph01tP88B{+WKt=m_P#7)=ok&JY2tY +zdx&yh*mS210e64_Z(Pv$2@N|FB>W=lz9%-D@=z-2VzHX`zLoD|+eTTDR;TwMCIF)&MyEq>kF(foF +znul?#sT985D$5L)Vn5Xuuwrg8$^MtQi})N@tY|5^%ID!qzMn{r;xrLzaq_!p@!KFY +zIQ~ev@d#H+*$sI9$g&Hp1BAWG1RkSD96ov~_0Y5Tn3Y{l{VW~z%yzI}-CPLCy&BU@ +z-zhRT_d&bK3m~F?ep1rkn2BoK2Q2|4gwACR>$|UmfMp+wWtdd@{Up!rV9$cF(BKKq +za8GKtA?*K*V`hI)_ZJC#?kMIoI$vY$&_bhCoPCsaCCWz;4O8@|)0eiwiQO*;`?lED +zY=`<$PDYCICZn4bW{WCP7i7Y&W=s(4fQ|6=uFz|>@`$~b_1WaTnUdm77ca*wI-t~(LtubZku3KHI +z4wlh{$L01duVs^fbH45_AVO_h&s}5DWudLFy{d9xw%-XH_9lAwI_E_|4zycKmxMidrw8x5&(1EiWPuX~ac9 +zV9I~UqCBuYyEVRA+ciY4r=fWy|9Xp2U3)hRvnj4~@pAwB+=kLa32UO~>5Glr5Qc}9 +z8yGZbtjA<2l$b5ocjo6=9eq+No&7f<#Z-9J()sjl3bk<0*urdVlqp5Sd1=h;iaiR_ +z3>$IZz_nlStH6Jz((}VmGh3ohI;_8W{coDj+OjnJGf9AjFCZp~{hi)WPl)_O_6y$# +z+(S3G|IKOsznr-gt5csK@;VVq;n45=fW@o=9OopEbLVI^Gjj9PIfH +zfDY8G-Z*p$3hVRvSwdF9dFk5Zk^Nky^1kD)6JoVM(V7=j7_@*M0gtu%DklN9IRx_v13-@6 +z?vP!(N*m)q5Usi|REdrt(#a3<2=v7|lr#r6g6spug81WMb*c&4#xOZ#b063HhD{D{#GRU;?pcZdr(}Yg +z0~fQf1tA#S1d#zSesC{HZ|<%ip|vTuf@G11>fL8TE#m*F4urww)lllpn24hpqlKZ* +z0T@p0fM)I+8F7q})7B@!o@07zZLmVRP}cGWVO#<&XpJ*l$r96GJ-GmhWpN-(A$`+K +zcstA&IaXRN6Iw0E`D+M(Pjg<$n;~qta%5#8W?b$l*zMN-bSFjh2{aOZBfUX%wkH@T +zZaA$npo7?#edK!Y$mG23{>?bJp6<_o#`MqZ2CfcFh>6sGs1b28Xek3pgGFoxM6k<^ +zNV9ity7&=tf%$Q9xeN|_YKA%hgkas7`xh!{eROJ1e4MAGqi4;=nGLo%RiRpRlJe2;#)U{pL{LO3LUSlUCzOZ(#L%aJN<`5(P6caH}GjC-Fj_#+9lM-e~KUY6M +zS~NU<{Py|n%baFc#qGyhJG(}|R6ADH``fpW@R2bwY=EFp2pM1@%kJ8agJgi +z)Vw#RCu?fcMI{d=cc3Ekan=oo1LjJcDPw_WZh=!D3lbz#ZRYFuEp+(?QLS$r4^OIf +z@Vt4drU?s#!=&Qxcdyz|;pC{~e}^qSaM9SLI~7C$_c~bF57v|L`$1<3HR-$Ut9Ja} +z%tX^z#Em9-Ov{CquH<P)23ufc8m9n4r4r|1J4FqPfP7nM5Jz=HG +z+2Pnaf;Q3z5sl^#Bs(bMtl7?}Mtz*Zt`^p3BrcI` +zD45M`INyBq0csQ#DMEr(y*FhkqH0BRR;~Lm$GP!TR7WYkVi4fTP$6AOtjUT4rGkxn +z(KlD&j2kwaVV^PX(w38zXe7b(r;J+awGTb3Mm{hUy{9nSfd-eBXXl?%(Cnn-T4TbS +z-)g4KJRN_I65voN&AT2f!&WD~(3X0w3gP$Nt;KQXwr}E`{p_A +zU3)i&m^pOln1_kY2skgGj +ziPuHGO7V(?r`q)}gYDhm5h1Ve3;ox`S~8(5$TYSs4%X&xq{$^vr6<-5H%}Y;xhh6& +zof7-hBtfFh)P%ZHFJ2)COmvz)l076>B~8#XWWD|XtS)>LTEY65t5!+lDyNNW0gY?s +zP1Wy#JvAt&j+q)uN{(^F$s8%$`_(V?-wD@V4Lk3N@H)5}J^(mtynga1i_L%L62%VA +zG$1eDK4oPRWCT2c1Ssa{YQ|3}Tfa(9abp)k>Z?LGKQVLR|2zOG_RaNV$S1O|nK#yH +z{@PiU0aTL53^juc2D_vNc+4A-o)v-j$?D9y00Wx?lC +z#_xGAIr!gvQXL_U=U3h?r-yWZ1T1Zh#Z9Vd3@6Kp;SvrrjDhirT8oU=vOX$m`3P~B +zXTL`fZ1GU>6tmu6fs^Vz|{<1dok|$ycR}rzh+i>TdJOHmf0N$@-+!C*idR;NwaP7L#MK`y +z5s_L0qqb@4*L$(V8Pw>T0wD%1e+;y&%z^Ap +zBB#Gr%&9uVJ2iw!SRc&4w>C36F-v=Mj;@dpslI)3o}_2UW7h!@oi5qs{R4W;2Cp^J +z>HmiWOwd_g5r(|l`Lw)a)fSZ)sf{b)d;=t6X#2dpq64#=^F6_HXM}WDUh@kWFz#}w +z^zG1ZFnGtAEzZO@AQ$Q8r~s{Aw?q_6Ii2$8X0JE6eP{sjp@U+J9ql|CnPTe7{lDto +zx=lb17KBk^tn3-1T+xO`Ok}!+)O&>OobL^(uRAz@XV>jduH60MA-j_ItHKk6H_N4G +zXt?wK_i;TEHK;bA{zI4M%pKt$dR(?~p||o{DL`U5X07PKmxnqpUx)86G&{%Yy%94m +zfs`4kHvj1&yyg>zuXpW;+$WQv2&}O^z@=Crv+%LX5~DgVtcYRUOcu(MaYw(^xkJ(M +z-8R%>J*h~ogmLmudk38bmLN46<3&h{XHOO{_x{Zfl)r#Is==_?Iz8oDnrl}4P1QVh +z6A2GICh6==vvvH`{plTt*HF`NGhoFXNIvB-6WsjR1;JXj;_yB1x{yyF?9CRd3Tx}D +z#D+bB@sEzlsj8+(_L$}x9#j=y>cdvtIeftPEetHJ@B91+WNvZe!FLgPcd9_+Y^9K5 +zYi~leCU>@ZRV_&0*jjd-2DIfP&D#*?ACMVs`dN*K-BXK?#}q0L2Kr@ROiueHWN3mCq#%;Ub8J89=S}r +zZx%+(u(1`c7wpWvlJPr|y*dE(%?$dE&}LN9j^9hjkTSXb1ljoN?>7Ny7)NPuWq&6h=mRQFdd+s +zr%Lb**EAgYWCk`h0snMNJ(I0}qID2(MwdkrxawN=wND>nlYP=gKriQ{d5yJ`X!iy# +z@!ap{!^_;)wjhYdY5f#oiSf7FpT9POORDP_8D(yX=zi|O5pqaD_YQ8rK~kG4Y}EVu +zdkRv@e$ETc-#lz;C*`a$i@mz1$sBZsHFkQwcb7KZD?$J@M%BEXs{4Pq{$Vnl*+y@o +zdGfj5uRJYq_M+Jg^ozezdHCI39C&3wye)M{;x?Xr=}cDD$$znK-<#9dw=1nx6G0oZ +z_y-(!62oN{5eH)LokjjpoyJ%}WS@rGteH>dJR;un{9J}Xjw9NSS%rVn-vSe+KF-CK +zG327EB4gq-*o&wlGY&eoTv{WdG<)ZulfWwfj+A=M!a|BKE~B&*{3x*uh@qpLBBPCN +zc6Rd$NN@b5(Jd|NM@kHurRLS#`@=^c%(hQ1fj~bMJ4K$P_XsAo +z&Jm{(uO$FlR)X_E=|Sio7eEWerA<;BK)j@lTkaU7)_KD~T-dqz@a3&&lNUPL{B8-+ +z*}z_uD}vSTP1N!+maF>hJQ?6Qs;r1?#A^*C2fZ{w%|XWb$}~D5g&moMiV^TgM?bEO +z=Q@F^=wE!#6xa&amKMJwt-y5{PPmOyz(FalYP-=UCXb_nLgHRic?>8~WpGo$lTwZNg5j2JxGA(__8?OeU>$jNKL|LdX6}}@c_iVqh +zTzhRkI!4&wyvHa5aZPzWCht+ix|c(VBAQpbJ$KX3ViJB-%(}{BZ2knkC}EZqCPu00 +z-9JHhqNqCE9EsoIjXbK6vN{uY`&4dtQ5$H|>x<{|ktF-CkkSYByAq|otYgVR +zywdTDTZ1MeV{%$o%3`QdMySvaNp=>{30e;p_BAtiMVqppj>V2|>naQ*D*Scu;q)qZ +zYzkHft9aA)Hq_It>t(F7!S_#J3z4$V3i&BgC)F8l*sgyczAvP5dv3>lgN)iKj6ZnM +zRauyH$=8i)7vWUkFJB<-9Boqwz6JVj231KvK@SEzQPnb5aG5of{g8Jf$90U1{>lb> +z<-z9G`U%Ncs75#1-RNX1uPjZKJv78SDN~i?(^s-uV55_LG7HZT2J0rx$?a>1&iq(O +zs2%6oveXr+jGZM~Jk@GTmuocqMG}tcn*Q~&^9=T@y0dOU^(xmla+HvG^`|a;`pfzg +zd+p5iMyKtMmkjUBASX)?2VBgoV`)t+WM4Q3YW_)yw3y<-i3yR=m+c+aJ_rAIz92w= +zBJA}J@;7=dJ|cYSy@~Q(BNtWfEeM(Nn9U>5nm--8)@qt?qOAd|B29_d)|U0*6!bFA +z#-V9YyYwFv>2MYUB{5a*>T}uNOP-&~B)6Y~0+xoLE#7h`B|8(LY1X;iA35n~E%_!$ +zHo~?1$yu~30RR(~3xQWv*PAo(YH9;2uZ#-9WcUs0jOzJGDI?mO16|Uu6I1mf-@1cW +zc8jus?NlsAN;ZXYhElBE+#lUAFWwFF5B9uprKJwnG8_Te$5mAuHsizX&;8Zm{n>sv8wyZWg4@|yz6Su}Grq~}n +zxGhqcy*w8wub+?ho2-9CDsxQEq@>#i@7d7t4SVJ&v$JE^?MbNIM})UVNsE$)&y^No; +z;vBnBg8}JQo9l2p-Lw<2-a*!nDsV~K&s%BId{~kp@GPh}mY;q2YoTlgaaKP3`DcP3v+g#3BJH+?$w_OFI +zR%<7?7^|e6JazJy11m9A^+FeF`oUzyiqr$2aYaNf2N0^qA|z!{<%GW@c904==u0z# +zo~|s{oc?sZIT77s+O6T4dImAMs~C3_Lh_urzvgJuovPF{S$s(bSjv&>T1l@ce1Z&K +zb&tB6+Lyhd#I?Spy**M^$~lRDKE`BXGgC5b`1u2@McKBeJhvM^c`}5Bp_=fkpO0v%Mg0yufJw;t%UjL4&AnO#*i*1KjfFHfJx`P>q`I +zjQ+N8wHq8I;d_ +z>mt)u>X{acblGPj(e5#P=%j#DJLeI&|%i_|G9A8z5d0@13MRvT?4e2=6) +zMSNas!bjDRNjRS5gk=a5cp;(aQe)N8B2Cf5k>9f9P!qxWHurr$ujNaaTX|LRGoW6D +zouT`)hfxwy2{Mj%m#wOZsKdO#^{JT#o-JpQye-Gt#xgg?{U}rC5`pgW2SwR=ar-;x +zuB%`y1pTeIbOUg}o$XJFGHz3XYwL^42_o=kAD&L#BVhy@)`&4VznYKFZJ*?u9Um9pK +z&m#AstKU~Vd`l#vWOP>?==co7y_Wyg&Z^D9gS5(-ax~d2UC|pZp#AmBkHSG+nHC2X +z_n)yk3-lDRI$QgyM>$XXkA>fujn|`dqu!d5V||PkmsxwjrdFsFTAqiGs34_@W<*aC +z#g0?aMt#)mkL`g5{?1%m9Hri<7iYzbhmq0KE}Ml>FuI?4We>Z&qNCJD8n1WTkm(ih +zItkPP94lk4pqd&OaDf*Y+gtexnCvUW#08=XH*WAqGNkL~KTXHc3fM`Q>%!x@rZTw}0S{R? +zt9;kY%n)?Qqg!s|>mH=O4n=dmg}j=3THtiCpsb2{iOn7PrfZEHhyLE@`(bQ0cW +zOmsi#N6OTposqa|YEYf=IDo)o;#gi(t#&*$t`fT0(($RyYokO28esyY8L>EBW-m%o +zzFlI2C@^b>BX!X5lT)Is#6k9iD2 +zkfg=%Lhu#F8v3rZGbNeKzy&hmf!KjWjC~n)pZs(zz8*FqMj7#Xq|0>UBBN?IliWS7 +zX!7jQs}L=vFU0D4c3r_6AYi0}*@!s9h^#y1vp>o#tGx2+Si;hcJ0D+qu1Qr`UoJMk +zt82si-!i#BUKk3}_}|s(QFVKt+(>I=PnM}Xq&hD6LYm&rg-&;LGmF^v!Wd#LAY$)j +z`|7yzhOA;X>NnRk|C+SxO}?SHnIp@fYoom=noh~lfTYcsQn#}ihwo;2px^2F6;cske7vU_)QPnGsePuONy@m +zLEkB$=qDlPK_?_sxMH@2N!*@TB3eRd7+?IG?>Vk4s-sWhla==Dgs8dYXGXR`zPSq7 +z+Bnh54Z=|Y?^hgt*4jkTdgX9;a}sF|E{Oo6{P;eqB?n@m`8OLTBzNXTr~3&Ce)i>j +zR6xCD@F@6nCF7A{$Z2P)pJ*m^W!KfZqzIv;Y?!v_$NNdTPfsB$br6*DpTEvTRa22S +z6j5-X{dzJ6bpMOBt!TL6dbMtj?AMz6qN3;U%iK>LJsQJ*s;2}N!u}Og)daR5xY-G_t6E +zhgHCkf-S>)=CR}`bXC~$eis$cR9(4UV|;WN)pW^pEM-Q*^%&3i_Rt$3JSe*AjheE? +zUCeK|2iG3lxkU%onr!{X&EXh6v_v%=Fb^y^&uMP;8#GRNDHbtuuX4A5n&~n?iv|Wduo!bVi5ignReU$m*Zoz7vWL>hT8dTaRf@HXo5*Et +z#H6i`<@krkBceq6a$d35um%;yKnrGqC^Z#+$KPEm8=%M+auq24d5xfL!_dP-A&r*o +zF?LCWjQ44Dif^=v;fqqp&0V!CVbpRImXVMB{JO7Bl?oSv;=C-bH!IC>t;xI)7nRJ!67JAyJ{pobMqi?m*)eC05lg3po?8?J!-Y`;OZ8Xiw`NeVyMZ9H9}@aWWHK +zXDrbMKJ7exoWb~!RWfl)v1@dP1v{ab73!Tb^0CsM`nSE +zybtZkR3P*DVp*%SPZGX&O|&++T#do-`=?cH_`G&Vr>kAUlV?nppmDwT&!mRNQK?t_ +z^!I~?vyRB`az?tSDgm&lLNUZMj8+NmAgl0W!pYvLGV7!0>9}hw@w(ai+@mXL3nlb| +zcc>P7ed35)3ie29%hlc~236OGT`qdl;Da4h4KoeoLS(=)?v7CcGt8~O-je*`>=Oig +z$2CStL%*98RaNh}gWqV6YDq0p%GZPUR8}3jL`Olb+_4${^trI;0nGR=^u$4hS&boW +zvV5t|_MU5%V#@r}#xsKc15@a7-I%46n^YeX-ZtI?{QrB3(gR6aBFu$|7^K6Y%7&lg +zStbeUP&vGnM?UoV2WqD_9Idcvq}>DT_`LD{2Vn;;QRrRwTpBdX8c$RJh%< +zuDQr-lmp)Bc$ebi0z_-$u3Eq5RcMbJfcLA|zD_}YW2^&?DUNGZ#HN9MRZyWWCV{du +zs#gb}Mh5$G20o^97MdOs0lB758pCh@DKJEtsC)z?*>*{HDBrY{JUI<0Dt+l{xNozK_x5QB2=jZDu%LIOii?0vur^DquI2LBGo#_kbM?~I +z$plxqN@|knm2(jc-tn7PB!fk;<4P&zdbBm*)@m7f)IMwx2@%jns8hV_#z3u(fSe#h?-B8+BNaO>d+-v!x?*Hj~% +zA}%C2xlD3e;^p@9FDGOD5Z(uk@JS-9m0oz6jK_5^TXT5c{eZ_^5B@WW8qfA#Mu)G{ +z!j_p)yq$aN8$a_=qfLIwQ4pN8*YJNSO+g;DKRZQYjZIeJ^S3%1^qio&-4UK?gVSC) +z?&2`{bkI|11%eUfSFc^q9_tL6dU$J4?#PT6#OJ}%VUTCA5zIOyHh;4=mi9+FRS1lJ +z$D$+2tz5I^W8`x(F?ZUxC10u8C*3MN);9;Y4a_V;fSzhi4&Ib=>-eO;f^bzA}cObpDE~gu&SAWB`8vazoo99#q-( +zyjKm9hz(ChQqS@E>fy?zmk!@7tVK72avXN)(>D&gk5_%n10M{ZZk!#rE?pO4e~Fm~ +z+2{)JzD*qt(yim@I9=ap_u;#v27zvUi?&WhO$Of|x|E))hTZL!O7)I8$+_n8_p6KM +zf4?Tv3)h!q^jA8?=ePUb{RyU~bXl{+sQuK*=~PQ}SD{vWWxD|ZDhbN(%l +z6&)-ePnZ1CJD_s$FY|KA+ctPM2Rib1X`CZAq&&~u)-8A8WM%Tjj>|7-*4rEDIHEk! +z$L>4zoM*;{NQX8yfz3SJx@C@8E5U64WK5?03eC +zEfdiHi>bGcYJ+M1M~g#oXmM+S;#S<7F?ZC&R1*e>6v98UHTXH2ZMwd4$RFC +z?G#s2?)GR@$cJuIoks6`GiF`~QXuz^{SoP4?163QxeNE6;(}M4goe+d#85tc9n28- +z9_)H?b?ijHNbHdLvlbH#X4OB&CT}$V{vk=ZwRn>)*tAP#YKCaH|3vvZe8*VG!aQ$1 +z;>e~b!Xa)B%eMlp!(NIkqDo`PVH#jyTY8hrZ5~k8AP}?aNq)=6@xl|$H_#-QqV#eE +z%!9nY$tV+o3ly#r3aVNd*c+=NceSe&#w~7hFe!<*CqhfIxXN|%n`C!IdRo|E-iEA9 +zI>Zv}^b7IJYXZubL3s@YfB}n|Mcy$HQp8)0txNlb4LB>D+_SUP=i4S`CFr)r!hz}jFJcQ-2o8p~a}cYG?g|4O;Wn#|jP$^F4*mJ$Q(a?N1|>@BqiXRsEl6wrsS> +zy&Xorp`m`U_RlT2f7*bc)!U#&fN1+gnqBOfO`6X>ql8ne@;@%I2}$p|G(nD0{)kN9 +z?_I>(?Du~udHtu;zn^#5O|3(b`7O;5*54sGMm-w$gPy)y?%u%>2aXLvRZhQM^ +z1atl3@ZP@}3mWT@BMf~8bl~>?Yu%0(9B^%(5beve3bVL>_sZ`dk)oda+N9-<57S8r +z6o2R!iw2odpjEO*eSXugK-kvRXz`EYKu7LH5}{@w_Fuxc2%(Bzq-dod>V@(5LjBjI +zH{00V8MaLYg^iD+Efz~4FlVIKOQ!1#&i5P`1W5;{nv=E +z0}64(%x&TXh6fQvUSBDu(^% +zM0*Dl2fer%9(x4dx3&$0d&PhD=sCT)CPogB`S6vIIU?e}*1dnvKWOXJ0M_%J@7T{q +zZvx_Dg$2$5G#%t&Bh}NGOnWz$B;dR}C>s8Df|hz@!L)O`^UZr^gSLRkWxn5UQ}Xhf +z(e6WvYtxV;o_qddWE}3lrP#b+$GT#1L_&E;k+nS$r3zwxMg`Bz|;XW*0j;v +zNX)%YE>BjOp`xS)Ao;>3O8DOzo{Rrm8mO@LzRKW#JI8sQwm2CKiHMPaW5dRM`^Y^L +zjB>4TeiwMl0T+M{A4UT;yKc76QCYUm6Yw2>?>p%OuMw-DYc8ED9NRY`s5F!_ly8wE +z&YFer&HY90!#_z5hC&qn;W^HEGZ8XcCM-rNCI%#MAuF0c^umJ?BT$VSZggP-W1BB( +zH)IYJFm@U`ax%nFIfNm9u9Os0_%Gkt(BR=>F5P{Cc-$ZY%!WYp1WOECw(T!RAmwVl-S+?J8bbPy +zE>B`)UqxxPYN_bp9>Ysb{KWmJjnVHjHM~75Je|(ZI92POQP26m-~T!ks~c1BG|Imf +zpzN{s_@7vu6a3XmP=XxbdhWKkOchHrk#|kS+2oPIPNW#Y`{$5amopd=9^W_5G9R3y +zhLr(=*u`%-2@C--!xg$oDH=WH!%=Zzj$q&awg4#P-xly}M-T2fXN1r8S^F`?N`do~ +zIEfBFxaG*8%7MRJ>amR^{J<;p^38BN)&C;(?h$NgeDU?Iw|=l6bnE<|boIggn +z8aze@K~%&h2}xKt*0Xpu8?JDn@xJhDLA=16)E@0`q;{m)cULsReJpmS*pc?X%|iKavlw|2A>ZP_fu>9g{YeD2MYyriqD`-q2H38M{wF7(9C_%5| +zM=V6>%6VA7VRoRB;yLa3Hya88fxFvUmiT}2B+xe}A~V7$p{tqjTtOS|!}8yNn5nc+ +zc<7SgzkRV1R3VB{6=HQ7;R!rjMsvJ>;v%q$x#i~fXCFN{!*lV&By#aGA3YdLo7rI6 +zeH=}j$z&_-BHcZ52)p-Y-@S@pHFV)l1AjDCbl5dN7#qTMDn0lsj?HS1Altu;+9K>k +z6oXL>(m98)Z^?9_UmV~R$4xL`?_POC@pd7_vBE+bBj_YPz97TH?a`H$6`%I>dutO;x%zCzG+p`$VT94-KNFOEe$jBx9 +zOGBZAE}EZ%qJNVk8RBl;s1>eL{nf=aX%=@TQu$RoyZFdolN03km%}~%=Nwp`#6!MC +zh;A?gh_IP(&&n9RQ?S9)$lmCsJGhS(7&lBB?8tUlI4f<@S-By>C~PmdS7}?rsO^VqslY +z$eVvFRbzjlx!vmZcw$Ur{=y$CCGb4&@6p!N@5qJadNJ`izpOI0v!55E*9hA@4Q@4b +zdiVpQZl~y&<|Z}f!1h6g@(smBrG$l%h%)ii(KdMN)*gHr_Qg*=9nR^VxIF +z(I>m8MubX(IvJ|dFr`fIV>%^^@$_$I9)WyOySRx`@@<|7N4%?pr2%ifWVSa)x9`kq +zTRvB4>1X)Wjz1-fINDTE@cHbTE_4>2hd6I)nh2*TF?O^7rC$0sC_AEE=U(dfzIWdk +zOia4SO=Nv+MjU0JH1xYhPFm{2#=aW^h-bw3V-ztkwstIghFFTD{*|<$Kpe3H5WhG%#u`sdMW+KN3{H#{j@eK> +zLa>y(z?_|xQA($*#nG%lW$b2cQBTjA4pqI_RlGaqM@6W|=Jb85VK$x_%&a5ODPK{qFa2E%Y05#Ev$tL3-7>$5rw!LnAteSFYA?24K7D_o{yP +zy9gBOOrSHrB>b;d!u)3($fG&Hy+ehd1&^i$bgNxoCzw1%336G>2ily>@L$Y7kegnI +z3UNMskxoK~>rt>0gaJl~D;iU0EPGF~mg7zxlK`gPRwxa&~q)gggj +zQPO6GaLqIntXM7# +z>t{E7W2>E}8rua5tMW+|z8gSc9F!3QYJG!3EScxp%!bis<+B21c6o5dRT|~Dcj^}= +zc}?FX>q%dqs-A2=^4JHfxVt@lrg!-e`N<|}^MqRjG8V=0d57?%TKTE($w=Q&Vx`pyNzo7u;UQ*+!m#?}EC$UWM +zT0w1Y*5BR%;Dqje;sYZcn=9?5fEqYUvu^a`@Y_|Es`{*AZ-0tnu*p!10eWJ|w+8~l +z(t|d%I2nEf52|HX%++__ +zawO(pm(nn4-jd1N=7hSomfEV;ysNdmmm={4G0rh*U5jfzdnRdcQ*eGor@3ZBx~nYq +z8J)o08?kC0&tgu6d?>BaUE99Wx5gwk_3L0+a9EFH;O>kVS?G%W&Y!Dcb~q6hU|fOz +z1fu+N%Jz1EWzgUi%(kd~H{}}_*<$?k&%H9ECJqp+pY|7)_5A{8z_k<|w^&u$Wa1;< +z1dCbvNTG*HZgV5^N59;Dl)J*YEgb!{YROsM&G!RY?Y17DM+ike*?S48hcfWX2 +zJy9FFU;p6kNk%fSoy$)GGq>=_y&!q!U+DX+r^J2N1H5};i53X04p1gw$I|{yga{ke +zP3sGTcWz7eS&TA(FQos(#sL@>yhjMUhmF%my3#=%ymBQA$erZTKv|i=2lOq;Ih0LV +z;H7Fp{BoceM&kzF#4Me@2&EHUp{;{fRJ-P&tvL^;-7R9zgDVXZ-&5+W7Du>rUTVnC +z`h(oj%ehP^(BTUG$$-{I&yP-v1fp@u0KJM~0mnNk#T+kV{fzsfS1whCgquF!dn;M%;uvgk0Ab*9I}Nqg +zGEfV{DYa#_U3ZELO1_QIswxx8m|V=i+^7Sy +z&Rq*u+;Zx--e5%hzW#LO{LHvJ;n>0K?B5w%NHb11H4YqSr;NV2sKZfQ5P{7u-Q*@% +zP~lxQY`Ya!x@|*-hQ*YATra#_FZOXkdE59Mn%G?0zFx$YQ^Ez%bX#&cm$Y4VSDu3g +z88lryMnq9{+muy?f8=AHx?UafCC4D~lWhzG?L3Hb^f6Wb1EHcEnei6s%IAnzgUA$IN9+_EQSu{Qp740TcQL=XZ_BB7OQ8hpHykv;A2lqI?C+>u4!}#zJp`$ge3=0S +z$!$ZBH10Jk3Z?93kH`YYP6|xb{hLabFC7tn#2xOyZqRo43SXDQ(b?lIL**OsmeaJb +zPP<)i71E)8+kfFGiARhmOLpoa-2NlnTJ7tC=tQFQ8XWP{r25m@l`84l;nJ2xwEvew +z#4uxh^f5gw`@uiPG1|P*Dh?tUn+3GWL(F5?G`Pm}wtnBBAykOi>L)ojOtrqJILbr@ +z3X@~A_^>OhRFwu4$!(ZWXMjgG(0S$t&YFY2wB|KZE`K@vhd97Q;iG-~vw^ESlR9NP +z!T=`^@YKzAu^FFmiTWo?Rc`=gspG}+!>8E;wFRV-7mMG~YF=HM<8w)X4Y1hbu4bE-%cF;E^QHHJ5Yv8#2CpbDSbGe9-SD +ztz@I|pG=Z|vgjD3f+6U7H*_vr$>3@=et@W=DCb;0+Qejw!9%PFxF|sckRJ@WJH)%A +zM9{8r?f58O=Pb9WX>)wNgCS4>MY^8xyuNlQ=&&Ix;2jvIN~<^GM0vbbUYQKQT8o1g +zn%}&P@kkc!vGcw^Mi=OC%OzrCq5U=p!dEn1D_MJDrz~;5Ho19BhSmm2@E~3zKJ^j0 +zon0oSsz5ClCYs$qP5oYaV_xoF7rvMNN}~t-C0x#GX!W42fH$U1qfKqL!a8Au_ScszGrRFDM +z^bly$OeI=`*|#6PTa)O)->QKI={N#DR83`?jb>vrZR$%1)Q!Wwn>x5$TIypWJQ5Q! +zR%Z1dNqMrl(~8ZF84X%w)8T9%;iq4hJ*_w%u5xl_np}q(K=ZzjpcY<)>#!D~nIAw3 +zM>Nf^VU|;2AaX9V)_P5oqd{JhzkI7;}$vT)9@7TH_+k$#kl==*b3&!(9fJRIwSf4~f=pxco*Ut4E2thKiHy5`b0`~hcj)tn2>_frzX +zEO6Q)^n5frY)OaMmW@{piD2Hp<|ml&xJYE-gu(KIEq31)Q_76wwLVc)5_u5PhCCWBD=%wR)7u +z;SXQ)qNexw1517oep;-zgyO4W3@Pe|3a81vP>H3wD|>oO(dd7^Ml +zKp|(t!omvUl5OaIIZ +zRcremcW=)1zVnHtjPCkmX78JCON^G`LuiKIS@?CbF$J`Ulx*_ByPj_sXbAp@H-cas +zF54wcvZXyjl6B>j;mk2SycK;Gz>Sd>8=hxFyF8OPa)P%1!Ic~RLp06{Cocc&S5z)e +zPrRQyM`zz!i{VH@;>c=#$zLTZcsgUrGw#jb3lw5ceiE64WL{n}>Q2u1Bi^)X!MW7WT{EoZnTMAtx<7hu<3}tKgDu$3ZDXLo8nkL53fW`) +z80flEm!k=($#ne_h)rHom!-+PDh9#`&^WpNo;}5Y)XH<0Vqrb0tThMv +zzsl3V5i*ug0x!ry=a~kZ;>73qIrFG!BHUxG6cU- +zvgH*I^d(1Im9`~w+_+Do>ys02-e`WgTYO?NJ3qLM1cOw7((oCS@?MvM+b+JVzGFeb +zx#>05sF^&gLY+vqw1oagA22d$Y#rgt6zvZ^xGEnjsxTk!v82f2;sWxNVl3EJ#KfM5 +zk2VH*aqbp2Ag}muJ6>V3XoO2v?XmcFhNjXKO*;X0StI)Id2ucdc(-L|&FFChq~|9R +zn}0X%FVm{ZrW*AFi~@!NS=gJB;J2ba +ziJbc16cHuwavm`f^bu0?xLh#dTz1fxQA~JlV%z1AcEh@U+SBI@$u8|XTl7hifKcO9 +z?Ipe1;k>XFlBgfqEV^ldQ^&{5Y@#ZeadtX<{Ig^ex$;A{|*@DHIO-@+XkuyYh8kh}K45vCUbb&&|iV4cnx1v>2$`#BD1l +z^UjW?GamEuhUK;XAk`kwZIZpRvAA3qXed>H;CVmlCV!+;1ewlF38EaDFws2w)F6M_ +zla5+_umV(VY! +zefzh-?D31QXq^@UHeX_3xFkGFe%&)TFyLCR7eEUmonX$fyKSD2xI*h7UO_W6e)XZ! +zFmK-dVmJgG3pwhiOFTv=X*5D(?uF*a0gW7CaI}X3L@@?@R~GV^4VGElLz{nH?x3DO +zMmBJmKaL>GgdIuy&POH*X>+=lYEp1=k-{jk=U3r~zNPK&oa(4zi!4R_p29G@=G)0}gf*)D0|XIsU|> +zw?@YK@yz)yDWaP)@|It`uOKCzKu+FMHi+B&wO2xcIln-bANS<8nIq}qn@Ni=j>Oky +z%(f=y?0s&VLVMJt7zkZ^dq>8C_`N^Pnu(Diw3R||0jZRprUf2PgKM4-PfCu5$8Nt} +zMELOf>6uW!p;5a-L&1{*x)EO?U6`_s6#is%PEU8y+Dvw-euqWZ{Wx&s6fEKM#ey+j +zd4!>=ml`KfF2ydGB_5h89Dd>xx4G4Ld$k$0j%<)x_*{GLdiSH8J(-oc+*0f-Q=UAE +zDIzQ}@}u9tpUL;XZLJaC_tdAtX45EuQzQ->jS9@D?5g1#HHhN%);m%0a%iOZSyH`;6-?9g2or +zmqT;!#bDDUUB3i>&xH(u%;5qS&T{7;q9Bu!u*ubXI>_xwy;64aBlJCVht)YtBg*Q! +z)wdRMsl%BgmX-&DzoEq-(Sm383`<={*c?DajY)-7HV +zKg@ADmoiORejKowbw-Hw>6Rl{e59+k(%2o|Jqc6K!Zx%>3AlGIe2bpo65XAz?dH?xsw8r-%GCmn%|x5}Y`_Mes8IOtOswtS9d3QXxf9S@oaSgL4Lhw=Y;0?FB$ +zaGN2py}}F9b~6EwLyU^WI>oTeRtlmMF6jkxTeMb;_2W0WH{w68+DX=< +z{NYgmAv?rOk?x26N?b__#yMB)psT~%#4m>1fwY%0Z7gXa*==!~v*|YbhdRN$9Sn15 +zK|2gh`>@_iGu9ox6koqdkWA30u{%N7TT-J9A*^?d#<*@&34aN&}{#D8Ntas#AC +z6f@|1(+*kYuJIg@YLHKt4Ls)o`0oWl9s99WK5+QVT|onK+y(VE*os;0kro(Tfks8h +z4;We97pcsJjX*>^MnKqz*2WO(5DRU4dHr{ta6=bczZmwjI31J)xk`DY>8SXP)F=t0#_IMe3%e1f_tYokOIXf&i=;Ho-0&=XEJZg%wXD!e#dwStJfoVy8n#i)SH81#9{ZE4-Gf2 +zINj~jxsf6-TUBcX_Qgn-ZiR<{naZOcv%Ti_4~!nhnkOMJcxs>3O>myg~Del*QRC@0eiLP +zeeO?QDNBEzcLUEu&Mnb`wc|y}-Y&@a#<>SyXfOpi4xQ1kn}7A1oyC(gJO +zQs!$}T|~xCdQWjqg6(%8qKdpO)G`cvvq&CN%=d^rC$i>+A&0%d(GOxDQ4hO47X@0> +zEpj^8t}@yJEFyV$#$k6HKol`lmW$0(4`Lg|Jy82bqVT!F)5QoSS!D^kS!1Jas%WTZ +zlkXe{my3H*s$#&c_yn$fUPHbRW7svNlM#*`@6Sq2+B%b>b*J(dQp{*<@J<=;mb8B$~-|_f2 +zLgP0$>9yR&kJ%87*_GXTQD +zWw%znJhF?7G{4edCajp_{T6y>=>EDxT)vKQ9_E}(VQ+t;I)c?q%6P@?uu*MpQ{!ky +z!#1k+Xvp_OA2GXaeFGb1ybjtrVDgbEI#os#yFDIQ6cN+hxPSlSW?gPUDHDnpJYLpH +z>bVPVefR~z*>3_F8!NI1ybt1;4lo6R4Lub3rmrhs~Fz;;9C3&VMxSjHkYtiIIv +zfULXRn*0ldYGJ&61Ki&SjwtBMGbn$QHU3x=m;*D*3|$kR1W+JtaD|lGQw;uZ>>@o;{$!kgI2&&MuffP +zpLlKTVsT$FCT{-sVj6h94>jvyp?3>xTe23V0&;d<+?0IZUAClgzQ77XX_Yw_*hY2? +z5jhrVRe9vY08}`FZ^Z?^%K+E$l{lJ8zt7xI;MnXl%6I@mCZ3q8F7+wiECY~B +zF@Y44Odwv+{k9@7aT|^YR;GYFvEyi6vadlz^}PJ{Pykph#;pH +zFFK^5l4jfhKN>5X#G?CnO+M861;7_d_1A6UYKXC~<4fFwCM|%DSuvkmUo;J|=jy*g +zgw&XFYlVLkQuO{by(2XSD)#qzof^KXAq&#EjpJ*J$PV~;>hjXL`~Y? +z+kYlG%KH+W83tAkz?hkphXHtF$?i3H9d=-&spPulp<2WiW682~=KxuZmLi#cjxtk% +zv_loB4tbbGD43LX%qo?*e@M*A9D~Dosz*E=bzGwjy*v=sca<>#1ezo7?)ZQ-VbOk~ +zUjU*$xi3IzeHIYyb#bvfr~0I!54TC +zonTnec`U|V*}RUeQDTf%Q{Mg5Kk>tw2B}2dF4j2B2&Q9=nOwMu1WDq|0UY=5xtfd_ +z>Puuzd3CP-tfC8_n4;g?4(t*E*J-hN9lYfbbc6ZMpxB+L6IZJU7|Mms97B&<3t^k5 +zAhPHTO;<7)z~vHNmHb|oQA>Jbgn`m8;9BkXmRiAub^h>uzD?2q<%Kczs@ILk>xgHn +zHcYwL_mnV0m~J~-rI6co5|q^EoNZFwFn6pd|s_t6(>PX=^vMGWX5s>&)m>jwl*LkMsa +z40TI?GGvQX<0?y>rHcsx^uHGzft?Y7UT~;=Fd*A-AveYwK!o$#azp(2>@`%!-PI%O +zNWnNuu=;&Qs5`ag^%dMgh3_Joa;P*1o-3>P0FKNAX&BU}-CIOwK=oeS6&-aTK;{8r +zj|^dB*4m&G!H)@b)yy#7K22&ko8T>3@+(;+h9CVpe2tN@S2oRe9Xey$dG*STml)_x; +zvp3llxmO&Fr^GaF6-&4ReLb`zKa~s*3;J$l)45=nSyDVI;}9Xfn*hg5!a0kIt`J*6xnhRKss7 +zDV@MilQjHx1mf?ChzEy3a;AGf4aH+B%8eQzZ +zSN@3a;sK1!Xrb90u@3EftE#qSSDOvXGYPW9sg$+akzw(FO}OK-iJ{eNEk~I^FsO}p +zjxE##zoZdmG{N}Jc!G_%UFQ1j`z$t;lymst!3oJ`U&O&s)O;Or$Cf$}Q+CJ!P+0uF +zGO@;jTMP`p8TF(0E$yIo6KbLCBVQqIf=UO$^H*le^6kxuoWR;1R+JUi4u?Q?eYCeD +z0n4^gN!_V41tq^%WZqbMxt+rwVSUdC6S?7uuC?eUx0BKx6N8s)&A*S7L00PuwNd6e +zOF9fQT5{H=9PsSj4yCktA6PyhcPIC~ZA< +ziZW*}w3b})59@UtoSE9Z_;&pO1xydTx=*{NH#sknUwNfLzpBC(a!^oF+>d@&?IxxVn>j7y3f&yF&T~$s=y1fV?9`9m +zStQ8+pg!Ify~OVye)SdLDdshcCoDB{F9LFvIKrbyV@IHi)xWN+`IU%MCQ{?e_kOM( +z6fOT{PH_VBEKd}(YhWxCR(PnrEtHI6rZ~k3o^}2Rk>%Ia1S(|&$ch3U>M@u(KmsPz +z$E$78e-KEfh~|ddLf~b;BJWx2ru?bnRa{nSC*|cn6#OHQA4ZIQ1|zF(#vmcDu3dLfnRb}g;pS)N7+vw +zq?Y<*>Vo5K!3@Sc%eazbm0C5wH>;;+Zh@m~S1=`Z_gL39<6B*eVn68&gU$GbL@6Bb +zN(9}|>oK3%y&s-m3yv`N;0+HEG^veIIJR)$z^=>@ +zH-l`BWCs(jbcWow`3HEl`AJ#%1vgAD-Ld2B>fp|sjz3SZn)UHR(U0L6mu|T&_59&v +zLB7pm$!VPsJ9+;b!^xJj>d7CFk0=YTT^;UTan86KH_U}hHZwCWUW+rXJ~6nSJz_7< +z`+m89cqYnUEwAy8VzqftQeJGw%NSDtZi)0H3GOdHGeuwSQkeGYx{^QgyV^p6tL|Tg +zT+7AJ)e%SCJfo2UbMuZitIL=jOl-p?fGeLy1PO1}Oa&Wvr*^NU{c5t2OXB+s|# +zlRlRqX3c;I6Lgdod`6t%@`H;V#i7z$4cBDJe}2Rkn4Kuh0(@3lDS?>wl;oQBoOTWV +zo^#kz>O-KlUJ2Gg@{8aNd&YKYdPbgF){&{(F@CJXXS+?$HPzi1N^`&4`)RlFI*JqC +zL5hj5Jj)75Uf{BFTN|F!k`;MSV~rswQ?jhncSj+ScXrh+$9E-j(Au1+gG?DSG%Vsd +zTSf>EsY5y~3@C;PlCE=*dOU_V6F8TDHFNw51Twy2zwNls`fC{Xp8T_+{X&Uw+&Eog3r%|1)D>w&_r`*|e&9#tJfYp=YQ5u4i>e$3jQZ~`q%|K3 +zJkR1t>duMb6mMuyo)uE=Sgu^}(H<>EP?kLQZ!p)9Zn4^XAqW{$lx}b +zC<_qO4=tf)DYO~0DuRz>HJfW<1zzhpmS~yS>Y{vLZja>E6PApuhF@=;+ZVjKr9IJ( +z`mDtF@zbseZvK(QBAvhf=afqF1)huYR2}zES_)7fJa- +zY^HY8heL#ElLvq8k}IhL){VaS-K)a^2fle>y39y1Nn#4G`z1Zq0tFp>|JIRL%yET% +z>i#OXunvei$3&Wa)ESb}=_i@O_*4hp(J8$JTOVHM)pf@;2Saq`CQ?m%0%Cm{TpXlFAo8-4p{6)h~yH@BlAd0Bi#YGj# +z3LJ@3Uaoyz$eR+#Ow`mI+k&2 +zIWOYJ>^Zjoyuc(q_*-Lt`2km5Nf3Fj3~Nw}Vx<^*+#fZFlnxsBm2kLC-U2hc~4AMn1m7yrz? +zP33_2oI$l!?qei6M@#!yv}KAEHo%e{V2W56?Ku`6}Mmwbf3aI3!#V|r_NvD>Y)7+D!1m$1@Gx|ZZ{Kg%gYMhq1>?9tUn +z(q=&f0`Vu6B_HfH=$Q%XpZl5JOeUG=Zv1X-FDr%fXPMV)aP9-i9C(7F!vv{*J=h9O07La~}5(F?Xm`{FnA&RwD4F*r;{Uq0~N +z-oMy!`yQzExk?nbT1O$L!T(0y_wc=SDp?=-iZ8)P8_^=KcPE0O+x_IK&)b=ljc#p* +zI7p~$Wt&RNCM;i0Ybe_6WUxl$)1l$Nn^Sf@X3!c$VjWspvT^OF#~>SsYmg +z)!c1yk3QW%eEW8re?Q#^&-MU4!sCz85I+wAlQ# +zLa?!nq-Bp}E&yhLcd;pRIIdCkZ>{5JJ=)ypHdu{`Mmd^Xy2Z{4aB^JHCpelA9lB9c +znQsLkST?Bn-`l<$UI>~>cs;s&&-PkZb33ja5YS7;xcK5>sdr1Jo7xpIUs{!CHODER +z?g7g@;rW?MD5lDQTm*He+xmin&kx1??4He*(5y!9c=nckvi^mAv2G+m$O(P-g)xh-@xgV(e7K-~Tb$Freq(Jr +zGlkRw5Hrn7eO)G74)zEZI^QZ;`|)%j?62PdqaC1~l>_Oh=vTObs2XZbGnmh+qf0kx +z>ddX-R1PSNr?Bc1&B44AFCOrhnZ_Qa#)so)g?WXTrfT{Gi8xX_m0FV>_u|?D2ef56 +zVL1l5Cbr(pU+t)7qe@LLKwXl$yeKQRic~%fY&J)RdEU!pz5|D%_eCA*WY_6k$iLik +z0^*}`6vzv_r-dy2aGrdG#8qZK2pGG)v$92AN+zE4fodR$BP=UzXnC295C-MfVQT?N +zvy86xr9w7Kp|!5no=vcC7pD#aDYaw0R{TQ)f4JS@;vB{{=w6*Zzm2TWG(FzO`l_mW +zAyI~=>&_`Lko;ThlTQ<(<_@ddp&RMzTj1s +zSoFQNslkh35bm8FJL}m(pMpk78(L#CR?(_LU#BkKnpAQV6E)g3e0ybBj2X9!j2$5) +z(7PS|Hv#33?6D?6Z!$cd48l#)Pgp;e11WSW6u4Fj{~3MKj{0 +zkxUu7tjzuhB4K+$vEZZkxiOu3CQmruhzT4V)2=(ANo&1=?-biRY>hSPSl-8o_Vj`u>}V7r{Vv>dwLaTO +zKN)l5rcUM6$AH-EH!lPSiQJU+esJctPkbeef`Qdb?D2wDf_NtoiFDjTJ3x`vsQB4) +zZV%!2egX);GEWp_Kh?3Ao8p_j;4aCNkNnD&!e?$*_9@xpRU!g;F-n&+=_9zkew-FAQABQHSCm?49K6^uHJZF-#uEJ4+ +zIY7^k)X@!2+55WgcsWLFsxtbeTBD*5kXzOmI6TC`o~?6LQ=Ux&WOr|c8!45k`c#QW +zOc@ftFT?lJ@gz*a1BwNUiD8k16UMu{m~1`|9xAnWUOa&zjvq?jw>97@d;^H8Hirk- +z;bLgLQQOPyxmz|Dnsz +zaJNUJ8BNaPx$hg*F#fcO4Zzdv +z1cfce7YUo$_ax?a4`i+GzTp)JVfhG=vi%PU25fLABY@Cd7O}7VA=4%KG0PhIGf`|9 +zT&5(B`~rt$vF`WSRVM?LbI_0(5aHsZmCVgfp7RYjt#f-Nt>c7JKEs71Kp8=m(OBY} +zdA`dpIY)>K+x)3yQ5pJ-W>ygU*PvSU>e(!F*Y5(T4rTBy*Ltkq&wOaJ{9es)cZDtK +zb;S)^@f#QaD2-3UOm?4aWa)W3+C!j)HMJYFzj~$MdUl8RsTIO%Hl^`qfu*DYra6e( +zGR*}I%mqa9nww#p-x4qK)Zh|NxT!w6oXxjNhL3Dz@iNPZ@mqgY@v;5$Q7DDzqC{>h=FCd=UXg +zgBABigu#YAg{%5`l7+5SSo%~YYO}oLmGnlW#ZLPR;GfMosj1_|V&LFe26FCkHm+9Y +z5ctNvOvEr6g)zL%9dvyR?qw4tjUj{b1DRcyT0XQS_DMD=or@D?vo?KeCV>~hc{`fg +z^bk6Br2?;0t4CQyre`^KLi#o(@v{U1(D!np1p+R0W|VJa5PsSj#}&o$-Gleu&ZJ4ZJb${Q~JF1j60hxh0E +z6Jg$n>iTd(_{tMO$;-VHb*pFWQWtKP1-NpS{VC)1>+2!cb|S(}GoU)O)oa@+R)BX|_Gjqh$DRD99=Y3SEL&v@^f) +zHIEDy5~!%b{jK)bRCly7u)Q+nQxk4|Re&F<^C>giZPU6%t89MMb^gLC2|33YoL&ul +zY!=a?Y{NwrsX%M)@%$#mMgNE1$)mJ>%(50?bps8OuuK6Or`?g099~Kmu#@rUzv+zT=QtkCd}z-cyA77j$`ykB&XO-Pi`_K1y^1pZuf1y{Z3y&b={h +z-@J%t16odrUlR{0L^D}23?HM>T)3#eVc@MuBm+Db9tV;`1VLcmFE(6H6u1ajK6A4f +zkkBw0*{b2o1r8PxDiA|c_GwEUQH%f@QGb1#Q%$kT@c-=fei6@Ci=N&BPd}SlRv{Tc +zuGeo`Sv5-nX5iYQri?sjj?S{*HM0MIO?`D(R9*MBc?Sj6vQ}-#1WbFkVlu%pdscKaDDm3P^O%wA0-E%3MnU6LM5Z4It$H +z&?L2Dc)t~BE=?Cpip|(KVX?5s%T7T_%M~o53&P3Y(x`m?qwp+eWR;jy+PC=s+Z7OF +zEn;*%0v(s&bNSA(gwkUpN}2jbdNQzBTMLoTV=C+~0#(g^6eQQ@99`FU9o&BgIpD&c +z3~_=sKXX%_G`m@YCtdhUH-{v!ieMGnP=IK$q!sT3DE?u2LHGEaiP{ND_4)2aM@Q4m +zH(K;wM0~VXnJ@F>mmUjiC}@K7l~(A4`C1C^%YWtbB?Ae86XxHq&DVVT2YAn5^$))8 +zkMt2@(tpOGa|PssWw9_US_e;I{C9Vg909O|q6r87JRt8px(hd~mo(`nH+vyl5fCy+ +zbJE%Z??Pf*hzJ}!AFw<>xPOK~vd&2?b+ZSE+Bgdkgqsa0W`DbT4gKw;kGFiHdx=JN;C+iS +z^=n}xyP+?9dPb*>Yrky|M51gLl87A})*?n$i`KtmjSz=U+K9gk_OChfYL`S%hgeIx +z@lq{G{SvAD#*YtKr`3QwZlk!{qP2!nO$`AQ{vCoSDT+(Ay<>$1{1;4unK-T-j}&lk +zMvw{uv;89FBa;3nxZjk=QuvrLhHF|oTd;^#Wz@5Xj<@!)(R>_;k0O@)0VDJUk`_7+ +zwLjX)2}ydhnPaq0!A|y#ZWN<1D1x?dmfDR{UC|*pxkp(0!3O+OLCylgmZN>*b|P%k +zZC$M|EzkJdCPOnFF-sffxbdtj28*A#IdiukpmYR{%s=n1pxufSO?}op43QRjL}Ykj +zgT~0nbT>P!Q0PIK+o;|zs`^92hOEIyf3WO-8Uv-l@6uNWqcCMZi6O^q{?1nAvQjO@ +zn~@qb%nA^VrgTtuYjh&H5b?1g%_mB0t`tMAvZ!xXFXS#W7CPXnniF<$7OIYah(g=GTNv^!q442#o1A3t#k_#|;}s +ziE3YhUw@6R_*jpzJj7n)8z`ajkj4^8tNFN-BJ~4it57-Jm#eI;f53o$Y_vBTfG_An +zg(lBRwR2Ih*u*AFpXe`-CB6$T)``D%zjV|xq<`pOgsx94WDaH7=w)oye(hKAx&C4{ +zBXxS=qXOm>WHbT^q0U}oGoa@&V4mEV-uLj_yQO@*bIYCWa|3wpb3;pWdQF;r6&(Mj +zGlYK8jW#4&x$QRejR^u-JcrOnvhQ3F_!C}}4we0ueYGYX$4;&e7<;l`!3xS|mJ1qC +z=ot@AT(2g;Ziw&44eMLf`wkI!8UNZ|Q~PIuB&v`2998J$Fxj;!@=aJg +zyAml!H52{9v+!+UL<3qz@?Kx}3mFH6u#4-Fg-k$F7|+ME8`CU+*VP?$VnA^GX)Xdu +zX1SGg2XSVpF!H%^`TX_A-7S%MZ*gK~`}X|Tre&hc_O$5}BhTNrK99Nm09J2Zic^w& +z?4)g=)98}2%%Dn?Sh2>)YzN$&ODrY`ie6waRw&-QfACb^<`|=hD0x@mFhe=_l8^to +z`&;t&3^ipzJe5lH0i0Juu}X<6pH;Evlu}9v(5n2?_}a~}y3=Un4Rv)?#kUb>2xHtZ +ze%O=rrTWWExSQ=TB~|qE&J@O0b__avI6;KiWTemtMu-c0>F%Y*$KRJkk9TjmX}s?U +z(=LO<*X9u2EF=?Y-Ul1k0}-|mH?gcYb9XBHoQ*klb2q%1?dwwf7K5Mo`WJuL(wE!d +z`(I7Z+}Qlio!KQ~i|8{R_42BEW6@UAv)Y>(FQrmOi$ +zj_Y)KgskJd>-euT#Q2Z1u}nG4Hv2I|lCd}P$mnF|!#Ot_v2(y3kwZox{UD8_Ga}os +zK!zlxbbTT10g)}5hK1{pnL3inY-BqhH^kSsMaD&oJZ{)=e0(NJS*J}?SNOi4Cl|s% +zmJ2y +zZkI!#?Z^BMI2QU(ZQ0(26N-4jc +zlU+KqO9;PHPwx->y=K3c2){lSH!7bS2?{Bgk}B|;M_BcIo3dyq*@}WA!^wotoHcQD +zRQ*IAfkkwJKH+5WhFM;Qx`oQw06*hs4RTk&bd1>ZL|=mRYY_0MU%r5{G1KltU0Tfn +zh$K0Y1*XX(zE%4`T~ekj`G@t6ccX;=+7J|kj|+;xo#UgjhHgAf-%D?I*BxLSNiq6T +zbD6LjOe$m1o-F0f`DGLL&Y7!lRKrbO%BZKR7c)9egv9E&si>R6f<*mO&O +zxS@O0Q-S7E`97?Krde1vUOG~s7ZrjJ)5h8NHcP)Ctf!_NUJSOHnX7`Ee`;GC&aL(~ +zTF^RskGdnhtj94O>RQ7y^Y3g>cd(;Tsc}CQ#L*|u5rpO8kGt5@9fWQjUg`M315|MIX;lcCqLQ6{S%g4# +zxLqt*WA)~ZDS+=bujls-78_SvAqYD<+kHX$CAzfqct^vmG+k1g0C?gU_w@tU`6We` +zcO0h}%S50twsM{QH;JS)Y>TT6>-+sidy6aY8+;cdNqtXd2-;77poZiImRSh`P2O&H +zDPV_eh}3V>OR@YNV{sC6-ml%O&``XR*q>rXHi?KW|Jy3L{yV=z&Z58V8YFEaF<4@ +zhx-8tjX+bAy~~iD+%q4t^%mp>@2+U9*3VMiGUf>XJsTfgJsd`VUXl@9P^sM{JuRq| +z$_ds)igb_1Z>d&x307_y85E86@OV2-XhJ|P_w}rE?2*{RHl|l8_u!D+Bhj`;dtb!R +z-!{cDpF9$5v&ovOa%{s3LiN&`j`PV&Ax*sybV+Eez;R4WKPa{$PgL4`zYie`6_>Se +zu)t}6p=z^!+%^Xz^U2TQMQ~ETP_?J#b06~RPdrPXLL7P_ydRfCI_;4fAyY{wNS?wK +zbFsJ8LeDyUarW*Go>XWUhgEeFoM!9&7WX1IyM9luD7gzWbq27@9E-16YfsEZf2l}r +z#w+?tRuQY_&x7r?$H5!6p6M^(f&pJ<*=TzL^doWGz|f%LhU47_rt-g^Zdr#)UaojN +zhCGQPdx)aSPndN4vAp9Goa9l{5t}e8Bgp@~gN@$Cli+@`IWKmwQh!0%Uk~SYmj%Lr +z_q?U#Hc9@z{;U5;am*5>PI)@T(4RH+O9wabCe#h4AaS3-diuL$cDDw@m!IvX(KBAe +z@R{Qqdx*n3XV1D&9Bi8v#quVl#=NFNZ}d-REqUu6h1mPOIH26r9|QvA*w~}xH6R~> +z75xv!@wsiMOiTiAB#1p3yOLGT`1L5T13`LffqX{fKCWl_JDs-_>H0j;y@t4uj>Jo^ +zcpNqongqCuR`(Shvey>}JG}b~15n+2K0gqnyYzC<#YHw)haFFprHEF2O8)wslG2^v +z(?JNm&;HL0pZ$5Wlfpg1{hd{#X}=XbCxCz5-EaBhd(&sjt_#z58qf02#csWKD0sZ@ +zjkQli$z*IbR3o=D(_-16Z=-7%Q)HO*1_E9B@ +z``_9K1)k-`;sKBEf@LZQcJ5cP+!mzjQyur>vB$)G}y&Yvx}MEgW@+ +z&+yE*JZp50^!WO4vW6l_4LQ5+ukIcr0T%aYw^wz>fSd1O>D*i6jp^xEn>1*1VaAJB +zSu6vERB?kJttmh`0x4u{M|7vM5?WHV_^?xaVZKde&8P?6d9SYwpVfg*i%2rVd-!an +zK^aBX7}6#lZUd@cav&1TE)gDdoH}Z-0?CG)@@!dN6$e*nStqG3@(!L?jXK{tL^B*- +z(jOY@YkBTts&|Hv2i^F%-444H)3=Ast@9L>As}s|LE*1?uL;ad?2vsf=3~QE91?E# +zllnP1K7_y+fJE9bb5>;uD(OOx)btm!UsCSBTYWS>(OTa8m@7f8TSoh8;-$4J-nWJw +z5!4~>FW+sf711HZ4?`&h!5nr*?|c3#K_?!qgw#4}4j!|}h8jfjx4!LKXjD7ai%~k{)d9%A;$?0QqhlUI>NW;uVzj0 +zvkc4{@yo_KN}Bx0%SJn6qae932R%lL&!tv7Z^{m|hfyLgmjrix&N|sBh}RSe$ieMn +zCd*&J;J`!*5xn@i=9dqaNMF@iSP9ae-X04~Tg#)%8lX!7Vm0{RC_HmT!LP$)sJQef +zAP?VEoC@}$j_7dso?|Z9))o|5Qb#L9jO8U)>}W7DJxS^y4kW;EBTF~$t2jw635D5^ +z8LT5 +zca_@j$uV1OxN}6Vnwp0U3tG3$m^*I3f_233wg&LKubR{PWQoBd{qky2Df<_1kHgbMm@-fF +zszS7>Ph1&MO#``(p)=_w;F?BK7N*~#kLRILUm@9R +z&I&$Zfk+R2@gdBxwLW^Fqmqb_({w^{F_}SCHy4zJrL@P4`dlH6{Pk*;!DKv9v^;<~HCDoKY#6pEB2txmU*Hz6*vCO=C892Yli&iA~|_u`>oe@Ujx +z$1W1GI6+5YCW%KGvJj5n-A8EuH9-n~BI0hDRcSdn5_)kkS?qwVr`r= +zMZQ&KNDA$8;g#M}7;PISLb?qkaxj0QLTYrxok?DkwVnQe3Mp%NE=gc0!=QUOD!&)} +z0{Zu0NCCr&Uyw6D=(y)7w9}(JIY!es +zvPGa_k+^ieDJ+u3d>=Tm{*DU@mK;h+PSTLkJ<*eslVeh%K4AfJRU41uO5F!OeN!!G +zZ^+6{KQUEh&_dJV=S`?JK0|rr;~<>Xc-k%w2)y|S#w7tOsVb=tKC^1Muiyw?lUK0# +zzE%ay7tQEp1X*$5pVwqMV1{)RVm0*>lKnV?e%gHER6Q(m7rkw_vyFVSOljeB(lx3J +zd16u15|bbJHPZXo3OPJ)K0i1NyQ4SW!>aZ^S?*~>O;?-_R_rK9 %iwAMEFUuC8Z +z%94XICx{d+9e$a-iU-Lo0^3eo-EPy&x+nLUPx#7YD!Wn=aU3ue{8VLz;Cvw#Oo1J2 +z|2u4lj<|@@&@N}iM6=TxYqay$gX~SUxilO_@~9=j@?|Lj|NE8!51zh4Qf}W{U;I;H +zhHiK%rTgKmQLShO0T9@BCfHC2q0Wh2I-LBS3d59{JiYPV9iZM&{xuj5my7bl*U@TR +zzNg;U9v|dYq0x6cL#g@Qjm%MaXUUnt3j^LusfxkDV-!KJvHLk&a65BrtE#->SJFQ9 +zmIyBA!;BAd7FyH+EN*?+%vDC+MfVzc4O`=m*z6eadr=ZN5w*33)6H4pA{0zhyIo5| +zHhh^Jn@p?=Q@<6*(B;_9n4osLRJ+Nc3*~!JRdqR)tYlMOy^F13`iPA_ZcUM%PXml& +z5ej%78}bKm#PZY~Dk;g-e0kCPx8Ev>%QB(q#fsQ(hYK2spqulgfKaRfbY;nIXV9>T`7u>cMHL>_7|D_?EbZQ;s1I6us^DZRQ+|&`uCp1Ib97iF(VTj +zc}vnuI?l<(tHEyWf-+O;aP?8&bb>e%`=##p43M+Shvc?MCU?hysa9yj# +z#!JnF$dl95*S(x0Jk>U+cfeOKw<@z1<+qi_V=8*&n}7V4gaZ5#9JL-Dq363~zXw_` +z#0PF=7Cw5>Knc_HW?1_Y+tu!u8so;7 +z!{Q>&%Xkpp4#o`c(Y>N5F14|0t+~HSKCHPxXt6K+TvNLhsNQ!t=}4TPi3?Gg-k<uzI3Mhp`UDlnLhhG{&$==o5{cvSuKNkk=mn^|YX +zXz)UO&6B$F+nV2g=2H8Dr%i#8Njbs?`5_%8Z`$f^CM!2&r1I=rLO)<-xUVe}j^^&4 +zx{6KsnJ9EVF*8bAKVZu!TB9AX#DEWpIE2A>;RW0Q$?w^hNx+^WP5or$Y=%q$jK1n` +z3*}hLA>o+pctdZXjNBO>5RiH&-6^aS{|P8369JEwknKp7`Thy59_hO*qw&ntg77DV$+NCjYO?1j9!Vj_lPqtVg7 +zEWU;wLN)!Awj3|AoqZiNu=buJm6gyC`*?0UNzd#`J}19JXgtAFJ-vqLp4%B&5luw~ +z;5##zCeUQyHpsV0V>J-dE>%Lx%A#xE&9-m=;l~lIC^*}LD +zhJ@W{w)s7t#bSdwA8z@n&2($x3@4Z}zh{w%1sXA$8NPY@Lh#M7R>A4XBkbA?8BbNr +zCg=HB!EGV*0fjehWGt+GXLR8Q{H7T2SBlSzC<<%ehvS47&6s}jgssmEcBA +z=c<_XO)8vF98MuisWnha?7-{^@4oIg&=y}YmgeELL{`J2bv>r*NIktAm^ +zN4N87(Z>>ZBpT8Mp%~p==3g- +zh`TCWiTT-tS^Q%Cr95Cpl8+;SR8ZR>g4d6A{K7zi=6@=~P7E#Yxt-bRb +zDHp4$_pYhB^Kl=dD$9cxjjoaTS8lgs9R{A~H1*r;6qfun*?=Cub1E4_zJ2`K!{*e? +zf=IquN(}fc9c~6?e+e_%Nho|sQz3mq&~`SpJTTZWz+0qRxoL_5^DF;U8XC|i8VEeY +zS`TvizepgB5xSwdff*@VW<$xCr_zEfC8F9rRvvaAZ24WHIzPw>bDcsE(Ok})9-GYP +z&6JyU@OGHdKO|S)K6bX3+B!4sAc~R3VD!XZzPQB)M$`yw-OMV8qg*b_SYw8{w0l06 +z4C^t^>wR7Xk(HXW)Jb<=fWN@YjhpcNnq>28Jnono40=rfg}^81&3w3P+Bm +zr8KP$8^GE~vNm!&NzD{U+EE%n0;bUG`Kg5KF$NpOA_3E#n7FAMz?!Qs3xZuMR$V4t +zIq_&}3Tr#*BLy?G{5D90@;W29k=2&5+Yj*^o}{9t;cwR*2OOPblPc3V1=D|c6<#k8 +zq|^P@x_Tcp+0=KE-T=Qmk=(V`nwW_GP+?}?%BY%X%MpmR@hfUSNH`4{(|2`oyuyGT +z(Ve51#su|!lO5O+qfz1(0m}33eaWavF^S%Y#Hi7qhQsn_nr0$2Mr*Bh!+a;%C-F-L +z$+?*RW?obo*>qszJ7J45p*~l3+u}#Ho2N0hh&VjAH^b&sk%<56re( +zpqJzuM|>R;1=YW#-xq%62IJq~5}!UTVoA_T*W;3Zk(Y}|Q-V>=9vp`EM7*ixN9u*D +zO`;6lpv0E@bp0=m?>JR|)WYkipN6nN4bZV*GM3Qsd!{fWSOFc-i(_LgfIe@W$)^<5 +zMC5g;Q2K;dt%!x=zWA>d?O9K(pE-kAjG*8(zVnx_6Erg?ZUid>)_tU~u{D%}@Ur;L +zZemsrtQm>B1A6>4uHmuSneR_W)*T3Ny?U~J7byNZe(6kslu+2@G>2=}+1>dvC73~Y +z>DAny)Xcaq0WBhg8@fNNGkjAU9)*A38i5lz5+!K87K3|Xw|l_b=N* +z^h^%gP&=Opiw1@zDX(i8qsm50a;`XPS_(UTpp`9H7m4&79$xR{8c{TC-oft{oi^lm +z*N7Ia#+_VDV0;w;wgOr`Pxscpr`^W~wqlYOxj1P*=OOw2{^PqdYns^e731O;GsMKN +z@4d?wIj)WK{Vz5lr2vP$P(x4hW!y;X/PqpUMY3LAbK%fhAmIrPE-`cX(`SA3j +zF8HI+O?zM>v%Vr#)g0+iqGBE}_(hc``cP1Z%(?|(aa~rg3{LV02PVF*1@O56`UIMu +z4WEx&v$V7 +zt#iM`+*-k=h7@U5T4igF8fRHk9om=8hcZMelP%va{24q1+=WiNnl22+B_H5dxP})1PQJ5$Z$s`e8ADKNXLv)5c%UF2xia1N@&p*`_sXg8`L;UyI +zBrX0f8O>%Xq2~AB4q{@ivQsgrLoqR57bj~i7vKYXk*?MlaMR*fp)i>+Thuvqc&Pce +zcF@2wI!m<4;7Crk`(w9-+qw6GP(-R{1qkG_EMEphjau5}>P>bBpe(MaRe{C}&!TR0 +z$sRDz53?AiMz__>@-V&G +zrL}|T$+RQ%jzAqXMh8qeDASWj6JI5C)jl^U>-W)WbrzcO +zo35zLbLGQKP1WtUtQDc*mI`z?9A&>&c$?A-X +z@2MlCPdvI!DuxJzcfIU#y2gKUF=1nmEqsQbsW;O)|0JUt0y*M)%I=t6MIRX5h5X#b +zaPHj?zO37&mn_{HrV)6WxEg3$mi*lgGpyUG8(~NZ4IT+FxO}IVE_~em(-%K-L|CgJ +z!PZvi)ALfVGbJ6Ow<8tFQNcKxv%Wc;v}EsZ1uuPDiWh}c$XVOx)%xhkgtd$)_sl+w +zzR#SRe}Y+T7tp71cuZ#UZsVQ31zTjtAVn~SSai) +zdp91w`foPj_=nTuy<-J6tRtD&7_sV9OKHbek@Q|wu~{)mW17XB;lh^Bgab50eilW( +zeEP*aAhv=K=rNu2*ibQqS<;DXkV6R2CCoznfK{#5^%D3qM^f>0O(VtR#KT%25&w1s +zqDHbl=UWY#k(_xlwi=tJdjemQjch#~fiUTFuj_XV=K;`r<7TfjOPp->!LEAwBc87+ +zFQdw@1c<(`$$hS)9iAr3tt&k14Qe$pkHW)UNfO9H{wYrJr!7jucWTz@P)#q9G`xQHX4P5h4o);l^KnY<5O7eh&eyM96OPsIcs+0am8g +zn5C;?Z+jE3h`s4}{ULco;pB*K*j6V%5}CFfCS?z?f;!=_W;WUGNuQq^ntQhKC;S*Rpsr~3Lz5n6m;YTq#qsv< +zPS4Jx&fRUwQx<-*B@n5}x#2%!{+NvWvysh)FGr0r+Z>k1G~}DdU>>Of3-!)ddgl8Z +zxVC-VV6Zrh-Hjv}D52R*W5R&7Upw5Wyq;Yfwu~klItc!T=W>%*XOT+q6N~y_s6=Ett9`BRnAg)`|_>R +z^O4;A*=sd65-J9b1g-)*R_ehf;jHKwJY(nXfG<&x+O>f@j(W=*+qQDrFKK4b+&2VU +zd;9o`2cJEqBvUM)?J#|AIk~^9s0G&s?!X!K7(X!s-Q!#9UhNH;=6Yvaj-J9*T;Nxb +z!w_IUv*3`-t`JYYqP8IMf;ln&l+MG<0rCw`=~00c1$YJ}mXfjF +zX +zWrmQBS=tK{9&L^FdL{a=4WQ|6L<-pNvF7AEh5Wdu6v!7{D+DIb?RpMB^--h*UjpPJ +zX2k))d~>1f*I%9w;%~vm>bFLt{v7`KxAib5p$tQvgMC;5-!PR(2L?SrF@T$!V(QuM +zkGHodheg86J-a;M4n4i^?mqEn_A<-X?u5--igOw)fc)PH@ke+DKhJ|ur{MYmdE7EYZJh +xA6VoRP=7G^gB>l?U9!|&SYv2mNUJaGTFTkpS$+{fZGI2^QI=PitClej`+uj#O=6*%fD{dG#arAdZo%DKq&O5P#XY!dai>^u+ETn&kpy>&6o;b4-TkD` +z^S2nq_Y7>thmr1^08 +z3kpgQiju6Ls@ +zX>Pce!)}K^F7w?0HV3{!c1QWXqaqh?5&%IcEXf2cA9Z)UO#oyClPKfHrh0mT=VO77 +zrnh~Pk6f#NB%fHgPLDdC?w)=HK8`&_#y^*oVbgUgUPhoziG9o7wH_^;?`=TAi790M +z->Y?IN)Y`3hO$5{AYt!6doN>uCxaS9V)n#Gfso`FS)n<{f4oDkj$n%QIsWr+<==LK +zU#bE*t*l8C`!eJmomiVwCYTW^(wqS_F$!8la{qIsUVV~1ayF!f_eOrl%cg0Ne1-BJ +z@C)=AhYTM8?3A?Y$R0eXl%bH+{dY@w?c@4E~KFsFpfA)qq +zpu!+AYW;}BHt9UQfM7J)cX~d$4Qqs+TX?qTF;8W#`IhpMHrQFC!wnwUrr#2~+-H;UlPzR(`LKB`QwojN|?j +zjMumtEi6ek45mQ;aH0`dl;9b7hAP+7vLS2Ve~M<9r9j^>R4|kx{Uz%rHP=+RIearU +zNk{f^%HDO20=rb6xmyu5u|6u2X+m!Gmn11+GR2Q6HIKZXuI*KEhMsgYnw3$z^l8aE +zvxVU}koH|$2 +z<5IwnQ8x-7 +z^$u^7FD94%Odb3<-%0$I)J;MH!6cPrv?#DMhnA}M8sWjZ-Y&?dIf{QqQgIG+5MG?c +z9iaqD`FKkFe)USn(hj3!YePk1>EW!r{;N4`p6;RUL9A!)q`)9MKKnprx2_^a}zf9HpRq(w+OU! +z|M`%&2Klsxfxn%5&d1MGey_fL%eQ#v3D3bkn%c+34=tjnUP;xR)-ia~)|#$6ttFh& +zhsqj-wHg|oR6W@^$+jL0n&5k-`K;`t6gs-Kd@IcV)1@~y%Ko|X_j6V9gY)D_oOYyL +zq}s+?y>Gd`qVhKwgNs>9T;PrtjkZgD*x4=}zL#Flb0qiUJ#!#mywvR}h_E^q5#Xhg +z_X9h+rOJBn*=YUQN290RAk}j=!Fr*Avfk9@t66zc`VRe2hJx9~te55iGDv;2`dAlx +zD=Z#Gz?`udDlvDMa6{~WeoU1ZvYF(5vfR$)OnvjhRYW(p!gU4aDlfo4wp%;@A^6vC +z!AMM%Z>U$14W#o32~H0sc#WB1PdC%2`wHGwV*l%kLy2AT92E!Sw)7xP<=i&6;(Xj` +zw2Q6>#J%)c!=Cjng&#@5SkLH8td(l>{`z12U)7ONO#{J{4;0xwd$chaG6P}(nZNmI +z$KX4c**-1o1qk!%{O}D*7`3kOrkkSZx=H=VNe)mCG+J=QL=n +zHq$^NrrE24-UIpRgxtL +zI}=BU>t3FXh1m(ZE$50a-wg+?uuvd1t1kguqAaM`N)mb5rkyimq~id8>eF!2&|D@H +zLSZg5ENp#XP%Y!}HwQSqT5)c_hkKL1>+*pL4>j2(w%DINb>))uc~8Yo2y=i&B5@21}zg{^@vm1!ucqG#ukssQhu1*0BCc +ze86@?V=UQc)*=+2(f#?qsH{q3ABXZl%7`k_=_BiFEnx&r%%VC>{WxTYx+e7t+9(FO +z)<+8i2~14ZVgi+)6l3%9l=xoxgMmBiien=?b9wkk%lDxU!z6;>_p;{y3y2SibB-#| +zARZBhQI_61an$%t`-VHQTOb*X``&D$%MF0eecL%5Z}K7UH$$(V4HspHxLbKky8*WC +zaB`O$uJ;PXc~*GIn<>FNzV_1a!gcd&mf(FwERQ7D8o){8$ +zuj*DUh|YfKbAs0muzfl8UgsF@0_h3@N2OiX`)s06FVx;g3a`xtyJIPT1p)!5GQPhB +z8eBwil$SFR3N-n7#>@v806-pXKN~|M^t4wSa6Y2Q#Lt)TcHVc>1m+0sPdqiWP{x@- +zosn^e-WB5Yd5h63CY-ML`Rj=CHx@m@9)(X^4=&+68%b$;s+Gm>P9mTWTAVpuFi;A` +zcPjs0RH<&vgyOg(^gPmel&xQkuspPvX|C9tPze|bGie@H9v%o4tb8Eu_w@M-WggQ+ +zL!W^N$-h)uiW(fih5&=fKKg;u7IR|vhkqgHN`o90BPq57Ab70Km4zmYGzEVn%lO#E +zsZnMNgAeE+(ZUIcIsKgy8+vg==IRVi&}0wFkh)vNDa*fvgT|q(D9h(H9hANM +z8?}lFKvZ0Ck)k$#tyIFcjl$*LH+x6n6uKX@`e%BMDdN>cSB04Mq)m5AH%f$A&?~cF +zo&EJ_pwu4Eb*J}^Wy^C|-OQ0?z*wDPoi{V=Gi&}>lzEPa39hr%2Rnj8G@6mU>FPX+ +zE^3|u(PX*Y$p^0Vrsr|xDpn3DFkPr$wK!cY7*G`04l{>B!Kx7FxxufLP4;Kym0$Rs +z7aj5Hsl;$;A#gMKPuz*2)R<@8kVuz=T6(!(i-!{^#ct#(po(!r>D7rc2I+t1?MKSRb)JQRGGF?OT3KKh-IZ=HzO +zA>KR6_Ms8xQ8$2}44N5!$@?c{9wPZuzUf%z#_y@2igA+RO!a@+}Kmz(wxbs*j +zHvkaP@)8Wkad~BBJTb(D!*i;j&d1C>6`w^da;vEknl#B<)r3xsRa8>;&z-qsp@|ol +z^3b3fj4n0nz&TonPrq4~(KGoEZHg*uJxj@gh0qzu3WGUl;m5r;!xmlrbvSNOW87wR +zid`t9T$MypfL>ngo85@?py2rWp>HD|#Qc16?$tOhu-^b4=H`m5dx?y@SreXajN<2t +z7!g(YU~6Sz^I{oUxOX=tw}%1q&O#@~XwjnnsODQOr*tjnB{^vItynAlNW1Xam#@q9 +zp)LoJzU>OTWK>(}?)Dns*2oNLIklqhfO7Ltk=ND1pHYg)@+ivrWN}jQuMmontJFvG +z?7$3A%)Y|U*Mfr+GQQEFTpT(UXsI*K%{DU7_e^CN^zyz?#b9S30N^dH4{fGXgq8z +zgmEwY#<+z2b}4-sS;-0#hgMZ^Q5sZbICi&7OW*&kg$4!)3=T2d95PSNgONpp2Iw(frEY|L8jGzt`GqGaX})tIo`3pv>U(Kn;b2M#v(wT`^T&j$}FH*8W5rMZA|c +zgqkt4ej_P*3!;crNJl$AA1S?5hrPXi*DTM|xK2xX|K>N&ns>0k`^_`%-6AT^!9lL% +zh7go@AkaD*01qO@H@vc-0@eIm=<4{?jMe=O4vGyUTPsN(q=-JF&3is^DPInElW!>% +zk6Q_OiE-Ro-NOWf%;w<n9Q@J)4$g89h632jga-h?@V_4x|M@U@WxaTOTJXH-)*r4JnaIvA=;M_ZDAWz_K}T +zV~%f0Gexm&@HanPccKJewqG +zF-f9z-;HH4H`0>yn0sC6Df61`6}vt=?d>Kt;f$+i40kV>0H2uz!n1)0FdZcI&CGx} +zw^>pil^t!$WSqu%jMNgJuVYvciX}M@g)`SKE*Hua>ejddMaDxzkc#AK8gGpYUeO{6 +zwI(B&q*sQ{aGulwgog$O_cNu%otg+th~9`~h(=qiFTL(O;D<+ed#Ev#6ld9lxypc~ +zgtbgJ@OOhzWR@?l;bDMQ6xxk`?GVgSw^6Z5r%*927!U3DJShh1f(LwGx+zq^O&BJa +zSyT)H1WH@DLQ$k|-yd~vyl(G{(!O?-YJ#63+@$l~Cus6uT3wx8JsU$kh&aNY=-P;4 +zHz!;5{cki3=RpI>@=!a@E*!~V2OJXN%knf<8`0xMmJw#<;WN-@P=O}h)^2y))!|*$yRKJ)YvPn2^!C>c}s!AsUOClGPYE%?#&*5FmQnH0>{(89h +zGN`|6_-9)fU~i?y9;Nbc0|pQo*|4-wW9%G0@2&%DUC@2SK!_9-EP>z#>R&nv36>Dk6P^I}9^*Roq))1J1 +zy0#<6bM%ZD<%O`V>oX35&H4m`A|vgIv+`t?{N28M4_!zZf^>})m)CznuYOhJQ(8|K +zA~tV{@er;R(T+fIx$^t +z2&e4fJv?aLy)lrMh1Z2Q0_V{VqPK8XfgZ9;)?Q5M +z*WD#kuaRJDaPcZy=KlCSIlF_FAqQ#UXqoB&-ETxjn-00n^^Xg1>DIa=``TP9eVVmmIOMi;0hhs}jGksA0` +zdNBwRTs!+y2$}JSA^$0Fhckg8?)`a0E_C&A?h9~1dC9OK5%T&h6!|1uY!xm)SUqljf;ON@J7vy(i|3U7G&U5hm&t$iF@U9 +zuI3peeoj1_M8{ADjdipSjPf;mw=OYUdUs8&G-=zws;o4YV&H2(6uF5F-o`*B-y-z? +z;wp)RwQ?r?3!I_eM@b$hnN)owq2Ohke3>j~8tY2BPep#S4`6vOqYwkGgn-`}=`XKt +zC+kY~!Gn=|tNy`Rj%2M_-g32*L2ZjD5wBubp}!>d+BjNh#lZ~1;a1ua@Oi(CIV~VM +z*I=j>d3nD7b6co0g2&Kain1V|xFB!Lk_BaSojj8^&g5B)eWg-{PsWV?=3h@l&FL<> +zttIUBthcug?}nK5>A_{g!q~`W7oW_E^p~RFf4R3^{NFZsk2X&*aQGEkPH+CItKDxjGgZ3PB7%CWa&aX?3&FCbIhdMRL22(e$E(BCoURcCiap~la>8?Ej2R|UmG<%Mqkbz9h3@nz@87NBH8?-U +zGZh2PD6=!iBWUAt(TB0!er5VmHE^4k5eEbaAzm>IlFn3@KgXmdsy4fO9P1;*;3*IK{I6p9 +z?=_hD|NnN-3B%xk1miF`aajregU|MS%{!#@bVDHxWZ&387Cm1&XABb)OdssCxIc0&sS`7#nz)k@57 +zK7+V8^fxDsDy%hf?|naLYofZxq_4v3vP$bSX`~7Wz-H8)7DU~|%pFPdtRJMXJD7mg` +zK_YuipzkUKCve-gJ{~X@J!g5;=T=L50NweK_Wy1~;}$ADq?Mk!VyP(>KwD>I`64}IJ?IVUa3A`RtHA+l`{dIc8;a>jTz#a;ywi%al6 +zm}Y2dp#r5Vx?n1^Td~|-n_kB}ts-$uVI1_WS0K4y^k?QcnH%6Sw`jV(%2qrSW5=dPY{ +zSeB2BRR4ttl`kgCWvGAZ*BaztEqNb>gKyhGeP;mw2t2Zk@YW<_w6qO9i9gn~@<2tl +zA5v!WIYU(0TbZx@l)O3kf7DM7mkJDya=bXHg!{A7GIs!xFaKk5Q`~d_fc);K3q#xO +zZgN)U@2PWV$9Btlh0XiNys-QImlt@$MadJGtQ?ku4y;)euD;1EjMH*8_OYV!icHC? +z=$P}{$2B_V*wQ_dr03^z-?Y3>&xOT(9+Q>7ePCgAv~YMf3Rv9xhVRXkfGQgRz=J`H4eyUYeB4AG%j?(}<9f#u +zdO(a4x+u_VzOS)Z?KK&o0%d0TUZ_#HW@Q_!=3eo!FjK>FP3}GHe1DS?3mh4tKHjCE +zv9lZ+>3;Nx+VZr5ztrR%iAzPDd`l$+>+XiF_~Bwx73Fji82CW)ruU*e$hksRfzs$b +z!mUK>s)hTD7P~uC#r5^rgJWFWKwy4kUkFm}s^I^_%wt(-Gk;uI$2~BOpVZqt(fOR7 +zlCpmLT9RP7_U=GpG_tBKpBen>{i#c&`S+v7(GDevS$qHhhYra&GP6~c`rqO86ND_c +zOR=){x<-5`Xev%t&kK# +z7N($)H?eegwfz{1fmG1H<>?q-+~+=wTEc=_vcz}Tzmn;c%D5 +zC@T#WT*fgIg2$!Zp9UE##f}56mYzi(_1V-mKEwr#1G-$UgXrGym9^nt84N@RU;GiB4jnugL +zWOKf!C)A^ZkN$bw{}m}r*l1A}HmQfB +zVlr|u^E$5OK;arT-O;O*J+09&+MOZxANe>=3B{<1r8q{14uz|$$LL6QZH)!K67aZt +z_1Z*>?E;pI0cA}olHHeZ1ur+fdWp&>~ll)496R?gsdNoV~KW?l7pPHeJ`NM1j+_hQFRi6{M)oU*qOV@$0glUqGOL}Q%vbMdaNE-`fS%^?L(F3+?D)8bxVR=( +zc?eQIuBmIH#H5jnPlq6Zyt`4*7gr#F*eBq?Ud#4ExoQ7|bP^yB0hNkeL|xEvx#TBa +z{;yvHd<6M)OdYaohOr9AO3pX3uXkw3h)zB?HfA7uoH}kp?8>KIl-&#f61s-^kzM)( +z$v5drnJwifrLDgIWitPPtuN+Azz`fUluIfJ^|8Pk!Z*|W6R|KR%H}G*!#=(s8e_3c +z`;~ArUp_P8u23d@>tptJSqL|u`wg_Ez~eCLlarXQE%@T75qu#*W*p{lZL}ziZ!`9Z +zO&E8VG|AtI`RZKXKLyLwhIaZdpaN8YjI^|H)YbkI9fAsc4{j6Ozr@RgvubPGHIwJ3 +zziD2Obmn3tSTRMERRt6z$Glak23M)P=*I#AWbr_n`$sg1W-)oiL@^#PS$$TP@5S+> +zCdy;|1+d19wZ*7}AY(FVa@FKGMO1I&ff^aL|AF@m%H_S^MwVXm~Q +zT{jP4$Tzp?bgq+DZ3mMr6~G~p_>U6=5F>nBY#SI=l7MFkTp(T}Pt0b$B_<-iGw~?} +zRNxoP!9DaiphDupE +zj$9FclkQ|>8+k4ok(;Vq6J-a{%1|7}#Xg4e(1&nPM?qH<=>wJ1K_B)H$v;mwcp}@) +zj2f60v`}c2s)kl6#s7^OPzu)xLwWGzmt0m4+Az(;65c^OI1~wcy-Zxl(`rs|iYO1S +z3SR4yi`j=IwE0=Us4R0C9is5k?58Y0c7;fT3T1;7Q`-BoSrIrd|4OTgzwA87$2oK^ +z3MjtC@RW1TvciT?NA$#2!#TnFr4ZtS&qc?tVK;SPTpEQw^6|cK_uj9RVVL&o+oTBi +zz^03@pSHsDF5!#?jW4zaB;=_%d2+aThu(FWWHF}=NCf}&|E(5u;9+QCCnt_=n2fPQ +zypQdz`u2M{V~J;b$L +zau(Y2O|qeyv1aDK@elAP80Be?-!6l#=YZ+_pu_xH5tbOWa(!FHF-~28InYdWdMXyE +zJi=bfoP%;HKADor1Mc`&NO+qv7uQ$N?{p+jp4p0AT((N|~w6*@<#h=9TTbf$;)oim2J=BqRzJ&ULlLB9#&h?Vz1nd94I +z%hj+_;k>;xuTT!gFq%L@vzc(Nz$>i^yWRZ8e@R`{@--CZ>zA7TMwL1c_vLMErs}^+ +zB#Q(pt2oA5IKVf!%)h!$#^?Bq7{a(K{@f?<+GW$&tQD0$VL`QL_SNDL-TnPW;lE@u +znRy(qhZYvSo+6;4Jo++$D|^6#_g_V~`Our+H_x<2O3V2*1MBK1*t2He{^f_F^wkZs{2zv2ye~5^#SiI +z74|D4!l0?v2vbRM9$?5!z6?VQM!r}3>p)s2kMyTlEZev1 +z#G?8w;?_xGq4@{Ut-s>EIt;~Elol{&FO}As-e`z@XBExIn)#7>e`jo4j4C{bJGckK +z;*-y?S6?Nn#qWVSx#uFi_{j>efRr8uTLZM>e-r@=1$qdpxp>E9F2OcgXW%&_C}dnQ +zQ70cCyJD=A5SBvoaFwJ9FmJ+;ZgS?vj=~UV8kG#WKF;H_Xy!nG=%ueOkOx0_*Hupf +zAuAJ2Q)Z*^UeuOn*$Gcg*6JqWzL#hdf_^|}E_oebs=o+r4*!iW&*j-y*T@K|aZCJHzrK6oNjml9 +zcDYHJz|i~IL`IRr8bXoqJ3^zV6ZP!hS +zO~ypXOS|$quXeC#R&zB3|3nibv5Ef?9A+63hviF +zxM9zz*OB|OyA6zf`yBC?`qOXW$h|q)@<5AZeEda|C?W77d>~k7` +zsD$kG$;!>D2e>=Ax|O4F&Bz`;R-&SJqzy>0T)}~i^|l)fa{D}VjhRPCB2tu9(yIze +z8yx3OIGs7lh_e$_`PN$N1NLUz*AMyL*13>cZR$-t7P!?1C-^^4gA((PxbH3V4YE}b +zBqRl?fwk88hgdfZex~#%ZI_XiWgD5${HPx#IQiVsk$;L6voRNf3?Uz4jmjDkn1y5? +z;2MlZ!b}tlSr$eCx2ZJCWzVQTB_%43KjW-?bT-bPk5IK9NOx4Kv2Su5(%0w=X{K8GaXO8Ad^?TW<^=IGw +z@9)x>a|sUd1%+-z8v+&&`Ug%0UO`^&95z+1G(5+Ns&7PUkiGjMA_anE)=eDs!6xr~eWy@Ba=CO~#jneSD2_}Io +z7sQoH{o%exp?+V`K|kStis%v}c>t)fuOPzT&`G(qegJjw{`?K-2 +zgd@{_Ot`?03=vr{%6)TjeaPE|`SRAnoO$b!!uEDRY5uLRZ+T-`_J_UQ^3sO*w{I5) +zGIDyaR2`vxEPuigJvcc*QUZdH@Ya&^o?b57%@!`(-}6)f +zRrDeL!IEDWq<)8oW4cUsV-dNG#5u7M6vHtD%08ZmVlY_UisLZ$P`Jz1+p=x+)xh$^(1 +zI`A%FrykM)&?+gBsXy&*E*BCk3msj)A63gUobLw$xq+ShK96gGzi!n`Qzo`glg1tU-f8}c)nS3Y8(X?jn`6mj^U|KaG!hKhNJsJQ>$9ZvFvvKZci(!-? +zg*94$+IR6Kt7k)HgxD!rlMayY@?1#lOYc&Oix1LQP$TV|XGXjkjPcnkMCgoDNgcyT +zIscT1-)U$DC{bB$y!o|{#xsCSsbbHAQNW0A~b6XQ;#zNFDmk5o+K +z?6cDB+YD_4#uwl`e7RX0cRoX~eXymWWqL|qr(jA~jp9yZ0Rnf?0O(e!S0g+C*^KsTD1qd;lwj?T}V-`>-aHI=G!d&a?@v(l2qR@;NZdaKW4 +zmmtI$ihPE?JBLO2Uy5s!9xo{TVyrb3gJLppQ +zn#0qqRY5=F9jcM@A%owtce0NXCT&z^#(#ZLalf3vdD~^>dNZ-3LrqE7=5$G2!Zp2L +zTu18k>%NC=u3w|j>YKoL)|qZ@)ADTXt&U^D#31@@{7~?LB#(sKG3@B=kc(&h@ +z+S|5(pXVcF`NFW_8uyotBR6*L$#YE)hROr1FEm_5643A_<9=3cul>L{zgU*p&={i2 +z+FpI7!|S%Qa1^;MkpFZt=s$W?&YlG|9DUe#yM56swaL%nelg8A$Wl>|P_*=V@M6~M +z8oO~vT!h;_{_C!dQZ}yd^kU^>s_O3HVNvj7!0yrdOfX>g;Gpc%zlaFf@Ey0bkpH8U}H0p08!KQL4Ij*FC%_G}<;5EH}$CDVI1lI($^%27D{*m3^ +z>zLi&Z5!L)|C-3(ZR3SMpL-7P;Gp-fL)_w*fwa@Hx#e;-(ZE>WB3FZ$46BUg$y@mZagb)*_&tYX$@y4Xs$^{WwSv@CML@|ZfHm;zs7p~Q- +z0UOsZMG#c1i(_wpqBj>Q6QPkuoXubd%$l!o7}o@==iEe#!uRU^XH0ndkpp~lWX9dO +z7ZPuO-l&N9ucA{{l$94YgibrotJyTAh`R4T_iCX^W(38gj{-zoPl35VW7R#QYO&Zh +zghb|AvJQ}Q=jI;$r8)xCuam6u$%??>wctbrCJoafzAeY`kp-^u@4mP_Or`yARi`$; +z9pR>=#JOU>*R7b7XBsn3^Kn?X5VzXW({2@2!#>M3rTMJ3(4M01cNCmrw8c5!YKfQI +z;rZ7k&=D5BV!Bsv2=P^b5js|i$kTvNYK +zaO?Po=hk8gSHyFd_h3aJ|7uP-Ks~Y;_p&xj&9I{&?OUU^i5AXM%T>roZqPaFu2oxp+**ge%tpXpsb8aPq+RC8HRo>G1b>;7(~v9TIGH%RnZ~F);*o#eYdiJ) +zO^toWflblna*=5h%~QIy$4HvQ`IK<1t-7}1jNg3C9g1;_PyQWS)0%HAM~WE`l>Ual +zHt9Z5*J)AdK>Vy&L)>RKne5>Ag@gMZ0!!dpPe(ifHU8wp_|JZ$7@D6plWN}QoP@k5 +zHS0T@z=j+W4pSAuqqbsVJHg359`z6Q2L*j&PpS$UMy1|ZdfEXlI2hB-5_lqM1 +z7@1&Pe1Qyo2}SF17@gm@QqxT3zKG#=&EhTR*0pS@d?Kw{T&;PX)$NeD#eVRqd;Q0& +z4~~w638BmF61%&HC5x>V%IC$231h#13r4|^oe`BKze*(Em~-icR<%|`bJLasMiSDU +zGrnQ@A91iKcnE47Y2?n>tS2V;PFVyy!Z3_`QVlc}$_*^v(*k7ZWT8<%uw7nJG}H&FL?irqPc#8ihpRpa +zHLsgpj(U$R6PHN&+iMvSw?_)AccfbNlO+82!S3=Zrt^b7KM*41W{a6A-MLnhcx2{& +zwEpD|i1V6Zi}Q0Cw3K5{?Nq#^0er|x@JiXby%?9_kF;s0IW^L%Wqdt2d5y_@n@v>U +zV&wKDnc#&hhJT!B9@-+{a>PVAqe>itznb!%l;hTq!OJJGWfqt#LIaMlloJG%#c_2vsR69d#Do_V_mAt((l%xchSSR +z!#DQCUhN6CL!&g;My!oesyS7b6cwu}6kirY@Avi}Qlz-0!D|c^dp#(2OFC +zZav3%M7&M}aE6UX(eEiUh +zCPu~xOE*0J9~YCkcak~|{!GF~JFhigD+oGDmLa_JKVEgU%aqBzfYYKDO1a7>!Kk5% +z&<@}Q&DghZfjniVtkUR}rq%DLI!qb%ROn)amTXh?^ko_k+ly5sT^{fKcXkaFKV~?5 +zw(~NHDcAfn%L#;EF5T^5sOzCe-aluvvmWrj;QPtK)bZ(0$7egi{E0!XhRN2k_La+a +zCe+3kO=7r&kYO`zIb^$E;@xX#^<;aE{E1A}G&PR9C^au0P?#tvbFp3i*c2V2ary9hY?Sq~%n2Z=+ +zL~`o63XLmYW_S#{aX<;OW>R(aC?LmG_BLL<;*3%igRDwWjh* +z%AvDJHCEu&7nyy5;WaMf=IFgt_*J)-nhYf}M(1Fv?0qS1R~gJt(qlaEyPV)4cWUzY +z)~1QQ8#QSkYWTpP`*4CAxGdfA(RH#gAaE3zIi;<8+$QAZE=s0$vECij@Pd&p)3{urEOO3Tj^7~k5y#U50 +zTc9S>A(NB9#S$(e(o$X(nE2I>f*ChOEhoUfHRFC@XX-baMp(et^72Ahq{@EZ@ez_b +zD-1FVd}eIY6k{EIivIyL^f`@^`K7qmFn~Sw0C9t5yHrY5?>a(q>vur%w$%}fW41c{ +zufwA1$7zS69=pb~8i)FK2Lf*qp&U)km5&ybX{0kc?9;v2Dw7I)YXnxhdxu4)+pA}i +zXV6nO@_tH7%0SuZ9H&AMbB%GBmO9yzrwFoj{g#)Ov){g+L4;-c2TaFxdpM!-AH3C) +zQu?kH8*h?2SIdrETF}Bxa-#Pe&+B2*zH-rkx^T?y5Kt3cHVszAg29gfN{9Iq96bFq +zs}UUZ9fkOZ{+&PRaQpH5&sy^LU??;?P9ZznRkVVuB$Xqz{C-J_?!Z&;koq3VqjANc}@mBkiDMbB8#Z_4&!ovq4&QB(K!ZJJsnUMhQU2kJ^<(CWo(`6n*_hd%1~j-&M}U3Q1C~?3Iji-`aHFl>3@J +zGj4vAJ!GrOds&$#6yd2j#zLwQ*@mI6A_1XG)U|!XmO&ROx}*&7E+RCkcMly-7lH*L +zYmqWSo62$0J+BitWfCdi7;wQ#m|&8ns!6w1hI`UE8-v>BMvx{Hg=>Z!wSq(RYT}V$ +zOZbodAy(asYb|RNw3j7o-WlW5xpzC)U%sVwdkk;hJ1qyOW06)BHC+=O+D&|Es)!C~ +zc?hv?xJ>HGVjp5^IGES)#m!6Kq^RsBqTU!Ov@<%K&k|4VafoUOM1+z>@$!tmehe7i +ztbJe)mGWi6Ml!=a$j(zO;YIko0AaIcV4zetsK$XBD-o#iQiD(r+N()65mezaU*{l8}kn +z@vCO?7`>IQDwAxs{($mi8P}eWg?e$WKIfog-6nvbs*}$?(X{y{>=cvD%9dV?i03XL +zQ#RYnPeB+hZ7nUD423KoK1pa2Gt(qFVjW6-0=BMpc-{B#_lKwk^S5dvP*{Ew*VrHj +zp)YsYXX(nmcko!4C1I%*>BR5sgmUj`+Bd&FK_HOo+b8$E8Za7JV~|_M(RjGs)t6?s +zhGN6tzk1h+>v_^s7$=x`kloC1tBkVIKgA{A1XQxPIqR7G%p1+K7+Q5|#6Slmd)1&Y +z9MQNSO$QJt)??c7Ho4DlteK-Ks%$;FPS0EQIAp;;5@wElhwDNK6O;JhbW|LS5Ptnu +z*9rvUq>+u0b>j7Kub*65p-oFqKWpk1z5s@_5uqTc(EoB4T9$EEYZ-P&R2atUHCe;Y +z%uZ^#kmDz!WkqQlHp;%71Wgp4AH7r;0lj!XtI86iL +zwjG6ow+6gyXyIS0!zQaTS$6vLckZ;$Uidc+e4;_CD+bz`CzVyaye!sI*LrzFCl5WS +z4MRuP%2}@fh|MjU$8YC1UpdaJ^1F46-I$gFu9I*7vJ-oWgTZ`FMZ?2vq^0IrzD#|P^d?V?jBlB2M0^*@FR1?Hw#Ez);{e$Nf%1(Rns +zhIk$cOZ{vYAMW)OAwG5D7sFJ~?daKTALwvNXkMO?h3YqZMuG~%nozpPID@v!Q}|-6 +zXeY<-vg(5++cK>$*jN7a{uv4ahh7@A`}_nG8r=NugtLv-U3c%K}>iF1a+@`Xq +zz3WZE$nH>}N2}}lGaRZ8Ri#J2LALS|JHb3{-x)DiS6?VQbJd4}Z(>2Vw)twDhjqyR +z3xKz`-qH4VED`fNoDSPj{N%vj_yRM8!WCEc_|7z|%BJ|5GVjVZsu9J=#z$lvCfL)b +z7Z$GPrXg^@6jP(*l>N6#}+5@;!kO8pE1S#eky>tC0+mhix1Uvr7hmWXz% +z)iP$L_Kj0(OGX!itav62V3TzY={lxbL5`vQ9AArC3w;LJDpSZ%8Mpb;X9GGT +zFL0iJdTk!BD&UU9qIySSDO3=NIHgp06!wpFS$E}ZG1c%e@d^{9cREonuIDJ- +z*OyfUj_p`*e+nLF_&OhIyErU0+|l|s9uXODNroVaX05UP@Y5iCtq1ro?|||d+oiuw +zR`Pp;BYo!=S2Fhsx=>DPS<4{U)hf-6)O6Gg3i=hk)wKTQ2~P+Q*@?+-%=4#C~so#GJO9g4eCT#6NUcXue1(&A2WcW-fbch_G& +z-+P~D?wU;g$xJ5aWUsyUlGpoOMQumN-d0~_V8QhE(Xpyq8n)phvZFYzePn&0`zPHs +z#%NdI^q74lxfM@R_*K-OpBj?e`6Y_L4X$)Oi_LyyqDC#g%7wY$n?Tb_%FIb~L&b2GnDD&9g<30WTGOe7gFDK9>ihuQ? +zl79L4btiauJ76a*s?%gND{37{gSrM&5LKq8F5d*a&RUwE3K(03c0G8Xc#wH|IBpqt +z{SFiKqGu%ciKdjIw^pq~NtEh_h(*ktYV?h^U0HOvhzrkg`hcm)xh#?o+V+*`?Gw?| +z)M8>MRU13CGkjDbSatlwt0u`~@I{#)#J_;EM8ZX>O +zkmzj@7CMBo0Y06w!DI4!@2s>7PG}7h1&)giwJb$O{U1l%vNox7Ui;N188{jpo|%8m +zqQR=mK?E^O*^duTNnYG0{GwD35|=0=I~#ZTOEJ_{(6L`R7CW*im3=!rWkX>jQ6CMP +z9@a(J-P7Y;g&_s57LDpM7QxOktTGlf;Q3*-E7|M}Dl~->94aS!%Uf{va%^BAKH6pB +zCd?zK=-95kc7Wa*ZZsmj +z=N%CPJD6Ord`*rwth(`8SsJu$FsEQ&JxDkfW0%tlTZFZbFy#ld!(af>cY>N&m5pib +zYG28-L@dn%=Hi`byeLQWGILe*yF_FcF3i>f2r^tm95y-!j&06S(K}}$k7s-~GswYa+ +zCYbOC75GDmV6jm%c*O8;V(pXWEdD-D@7IJEiwj?rM +zfbzpR_{V#YH#Qd0YMmMP_G8RFEZWaEORkX(o`gDFCz_4o%G3rM`1+Jwg{EZdE80>0 +zmU#aRBXNa=v&|mePn8DsQPHD}12;X&$YW3@25d&1`T3pCWDV~Bw5)R4ja%GV2KuR8 +zvxKfmPvS3x-nmE)#7>K_G7eckI1D>pLXnt=hEvkn?DWrt-;mc@Znz+2>k9hM3C6Es +zJPseq^ZCT!kfkoLye@kn*z*bWQQM9!^{W$>E2+A_Wux;2on$ZQCI@CLtpz31=$7%`w>cDRd^Mbl^U@SK-X2YOy1Q?8i-4<8+jEK{Ihn+BTvxS0+ewP>QTvOqAjp&T*ibQ)tJDLyKxWRi59)C8C;{e2Lp>4c>*MJs# +zLeCO{?qP$Yq`c|#*7L{ov?m>O*d&Q=osy*Zp*y=K0H%>6J(D6&lq|e=F0j +z35>_ptWq-E{AvK^*^l$iF!y&E1O)zDTX%pbil6)5vjj){LERPtRYP=~dCTSkw?`ev +zw@{?9lajgf!z`u~STzymW}g)j>)=>sa;S<9uv?E8dH18r-_%+BG@g?xJa=9CFqs9p5pD*ep5&7neEi%pv`p49RCZ+dkbu67szpVAFOEwtDaF_^AsdFR9%G~ji4&dx7E-)R +z(Lg%j^^j<3O}mkmG&!a@*~M)xk}gs7Qv4-!!+*z=Q717`c2b0*6+UGHQG|2p3~JJ8 +z_IuNM6Uy2A*nU@$+knhNgLX5RHSbI7^Ra;9-EoLQfXqgS?o$j{I5jHy{+xf%VdJPw +zLS||fFS#VEd*&AJ=FyEb-vFOHNx#RkB_vaoB}4j} +zVz5%5AT=0CW5O#ux5j)gWzArZIi4iqb12#VYpLDg +z%ve_qML?P_!BZWit@ENXouVD^{Yhayk&Y%|@{;_0tG??R?`u}eGcQxD&>NZdF_;-jjAeGS)T2NILv}fRcQj6!}=E +z$VBt}QJNOF%F$J5xIw%dKP2JjhBtTDHr2n_&lz5%k#Y1l%&zWkDtT&QZloR@g~S37 +z{{Td7G64XdW=i$Xj~DkCG}hRWW5V-*^_CX{f2z$NU1wt*W(^V}V2@)Nt#|4Lp?@T^1IPdQm;_k9H{1>VxJS%opM;O?ABWCoNdi2ZqdrvwI(h4GaVl_BHpkB8N9ah2V9b4@MFD) +zM&0NeT@*PZEreJy<#K=sqV%)h$U$`{k7k)!M>gxL^Q2H^PT`_s;}c7$8Z$?LROU=# +zzqRAmo=tD~2d7Nc<(Wcu3iiG%Q3W1{(z;hId7YMgOiz;1zm;_K +z=3|KR8$OEbW%fH_EBEHRqg9YOli0u-=Qj`|V`T{+0)iGhsAVE`q1#~}PZv=0Z3<)m +zXGKiUW@E{(C{$O-nyJestqtmOIWT%QR19t*GiNzX$q2|GAXDK41_}CZko7Nqgmx*G +z>z2AF!8_@IR0jvanyrFp9}w*VQpp`3ebGmc4YeyRo3mWD7f7!3+)My^_T;(WSu5xM +z*5j%h9|+B&2_9z`lMHl8$YPDemUj7tk$}{1!smIE4*}H8*V;ZxN;U>z7F8aFfQdWS +zRnvXe0Y!Dcw5Z6P4Iynenj~2lFx*lLxU73wTnZMr2V+?W1ST3Au)tseOKT0U3%kt= +z%n%|jf!)mF{@Oo9?(EjhAi&w(2jq9M~)TTZH&^tUNXfOoWym +zq%ef|Estxb$Hw$7NaS#-nmkbyaroRi(SZ$D{E$h$CKd?hVfBv#zqbQlG)lB*XgWU6 +z580v#s;f0p4C?CN$didJ{4!xN+pKgubjjx)+Y&cDdNLt0sL^msq`gm-b^pG15Q|-3 +zW&8RpiVoqbFU})&P4R(}pchQj;l~(NrB`EI&GP7v!(C0iVUNR1QZnTrbFhfJ31VdJ +zM8`SbAWiCtQ7(z|K0Vb`G}zE$`Kk2r5=W+h95ud)Q!?9-Lp;-duU +zWVwU)AqdVplNw92mnk-749$k5z-k%4-qFApQ{NpuFAiB^2Bylfh4Y!Dodcx`SUbfq +zd<|K1xaO=53-X4lANi3oI6`!mX-$$O!C#RY6Z%$IZ{2KKODBUG6pezsv`tCN215pz +za-jNYFWsD5epVGl!NFONxrX1OFvpb_{n*##t}l;+&?n`!tsz4tPeBJH+3Pv%KLSmd +zM1P{EqL-s+qTp#(&2%yI6bR%sc+KH5GJm1H=xt1W-b1&v7go;FSH3(!99?!zu6uM_ +zZ~DGFUS2%4aF*Hv6T|Cx08W*Uj_MmHV~lAOj%s*~sjL$cGehg9*|zu%q@rEk<)7c{wNqUh9;8@g^>>}O06uz6Q*D8kOF1%T +zdj3;F!d=YCc;m{HWldxc^EOA}I6`)qDEXoY3ILGK(;bFq=UmYK`WpfE +z47NI^eLFlbFtB5ng1ozQ5dmRpU>G9tcWGTjPXH+K<;=@g@kg83AV|l; +z4qSxZO&@x{oU +zJ8934OBxUTR0rTZGW62jJux-NZgnGU)kGrrZcb|o6C +z$V-XVp5aS*)>xS2NK`(LKlCmH*Yh&OTV%hIQiN|lAH8B_iVWEAFt&Q(@UDBjX~YDN +zu17I)pE-Zy=4Sb38-an?4w)B``aSQM#!bPP$wGC;2^2=A9COM;S+TPW; +zQ(fni@1Uaxhx(5U62S@yh(x7VZz#b^ouzUFQ0<)y(XPYUuBaqB4Y^#O5f`WC!P6g% +zz1=bpb;ZnI6t>rWxwhE5=|)^J^&Y^FSL*g4AgIfK+?wlB-N?1veoinU47t&sg>2W! +zpQ@aL@|tZvtp!|LFG-mWX}{Q>4OTBc2GA&wpgRz`0LaNV3EL@sXw#)cGZ)=1sl8{f +z4-c0L4(GocOOAJ+&bC6*_MoRRix*ihn%hYu@f^?dBD}5-G?#Y4w`x$a>2!+!} +zWZz;&{HTGsr+c5kr0Ed(pwB=G#q`tS5wNN;A>xPmdPK&;vE7KBav`We0lAj6<>0tAt)jcn5IO+m(J(h207%R`1DM19 +z?#j@{k4?uQ=~k4@yb7Zb>Tm{_$74()n&}XQ@Of1_pd^_4Vtjd&!Efo~_)08-8|?`{ +z1v|t~d$DE7JI0Al&guSYSw_Mr9?B&oyfW9#G=?9%9b`NyF^tGECfk1QyO~Q97+OGv +z%ngIc@@5_>1FVW0iT2+Rnxiq>1t~d!zd7=zrqpznE +zD8sXCv?N8x{63odYaczrKw+(g*vh3q;ICBuBXk@|L`&HP1g_NYNRgNSb!aY0+)NY( +zzod4Bs5!UDm?Txp!<`mq#Z1$;rbSo%T)QLn7^PBlj?mqiphQJ+R*3z`ojI= +zip=r}5S(v1210;exM(l-iSYLK? +z{)jQN2lteiHZR#NgEsWNY=&d!5Dp-GO6nH1k_NQtZ7pDsg(}W4e9Au|;)<}2im1E` +zlj<*rYA42|60x}#d>0#4;~f#_-dwegAk+aHQPoC@+C^sxR!=_?4oNMdg$dx>L&4fCfNCMx*n04)_a;c+Y)3C79X%! +zzc1oq0sz6zhHY=jND4$2dA1rWh5!Arv{z9gqb%i^96cVTEcE5yi|c&ea~V0%h@%)V +z=`l{+nNF-mmUZoE06_q?k>5ls<(;KD>Ri$E)jtRS@EWF4w)F4gYN<)F%%9>W_oa!4 +zX+0xAQ6ga<=osA!$%ue +zA@-i1=^rG7t%G$Ek)<|*s-zuEwEwvDJ-@)Al|@HBKSCq~N(4z_=AJ`OAEQE2kpg-qbya0r^!hEmHP{3ql^rt +zito7^suHYh8z=pLOzK&sLIHRNY^RMqgA;vQ;ejUe>%&bXMS{=uC`neGVCFd0)iZFoP*L~%!-{Vm=}4BuPsd6JTI#$e84=L%NS*{&aenuOma^PQuZP6hFkbdcE|QNiEov+^<}NIKIzeh)He1 +z1@x^$Tu}@PQ#XeN%zgjn90q#Z^7e#FEpBIP*N3La9J*BAc&8{he%%1Q=2OVC3!w`k +zB-&IqqL~e0;uAy>Y?9?&p7hmHDI|;{*eZW{hM-(v>9xU(fL&n>@Ah6WslSDdoB|oCd +zm4+BS!Tn+D9ZdVs$V?zF4Q<(6ujPsD(=kbEZn{Gy-ql!Vo(?Z&!fnYbf+`00;e-HI +z>Oe_U-6rV{BOn60;SxDJxrylj~A|W(o3*v4CeCydsvW<^u?g*fKq^ByAOb_n) +zjc6v0?n_E?YR}F&a82m%$CvaFm*JV{A3tJlK}!|Dl(2x|GUeQR1Cg+KQVq00UpCGX +z;c5@~W=!ChEEdf4D^Rp(N&_}j_7p#;XafiF)B(f)&4OzWBALS!`oP{262QK%U?TIo +zfhv!vIe*50mGWb;ccEYOr)Ul8_;ds*8Y(E5MY#BputJ+aSno%kwuP>}8sNgEORlfP +zvI~uE;jtU9cT^=f7`S3r(m2y&PSr3LIfT8}JOR0?oz{R7JMn;n^-#zqy^7|#d#hVy +zkT3_S|HscB1+K)bd8xNf4Oy(T9dXemcrHw-s2DBqR1221BdB-kn0FY?{@tgR +zl-UXv?1&fyDvW%HQtKS@|4Wqm1A^xiyruRujn+g<)9y6Y|2^uB{vxF3U{LCZ!9b|_ +zSA?Zcoy|h_eFuwS@Lm(20u1?)c>k{&#~Tnx+hf;69AX{B#E@sui6U=ux&Y-{Rag=& +z^p1jo0@x1flYrH1aCqxw#7$2!-D5?^VNdrn0qb|~!a^SiH~dD0hnYMN@p +z(k0H=>8f(*E3=NdUDm{fYEYK7GUz^6?Yhh+2R)!_rZ)uL&>7HsM8&3Z#_s=pjamdn +z?c6>;W=COD#SAEL`gqKpDHSaK(A7FTz`;JZpa*P8cl(tTI@5`FydeG0ngK7-qVxzu +z=yAM-+*&I7(s?Zmmm*Gmh==FM_DO|U2*v%@D-doxN~7#TMn;=0*9dGt3O**vK3e7f +z`amXn9a(Oed!+~zc3Ir +z18_^_v>*8ksupiV`s{Z`(8`)M{D~Ajn$rtLKWUJNJ0=oKD=XZ&Ju7}kebzfSD4?LI +z58KVC!|Ipv*mcyv&>a!i?)!^S3`W^ +zhP%sFD8(uR>~yVO=@-JFZdMd~-OcX)-HL8|OU&+Qd#z=;Cr3MQFx8r0>L7@T;hiK; +zSjWJqZ+lk2#|M7P`6*H9iKS%t;Vxm*SsyQ4I^NXuD@=$v*5!Vf +zrlFpM=Lb5m5qKQL`y4TZ!SEZ?ED$pPlF%81QR|B!Gs>a3S4)f>^E)#)Tf#*So5dk7 +z(^z-t?<;0;En`vwA}D7qZnDZK*~QTLXQRd_N4>8mZ3dSJN66Ndtqw_N$*0iZUJ{TL +zIsO0NRtZ<{fq*uQORaj*$iCBwh)=vI-^5Vvd*v5x>`*@=*cWlOAdQ+hKZe;m*T9O) +zh3SxAZUkfq;{CSMFR}L|AF~M?`PIsJSfjHP*25b8^gM|B<;#iMXTyZFe@ZAQ@x95i +zT)bV(tmtt{l_h_*&0XTE5=bSinqW&ycAawh!lHlarKS$B*S7>WRgqGl<7)G&>Mqcy +z17pMYd6=0iRM&X=cmDBxI~RI}4_DP9?d#v=_E653{us_o?V}#x9NBU`h)e@w5i@IB +zeil9^(vw}^vH2pxe9gvMX=Z09BO6~VJ5XgAdOlhRS*sEl2>)lT{`cMt*DM@V$n;dfJS9x|6lSBv3}3@iZCM+aDbCW*CY +zF_9Y-G_jcDZ>GbuJ6RDYwIGNc9v+u&V2)Q(A9lMz>W>aW3_Z{H|KlhoolbBU7a?nF +zjCB$!puW%B?5Rr38WP0>rQvWUVHZxB@dFLq%$DQOz +zigtvy3Mb=88#YW3Ct9?{AE7)mawgedNo-45?28b^S=NH=-L3ooy%P(AofPPGSdbMR +zWHQDULZuFGSAx+b+tID3C$l2ZLiKOh6;@RhGi2;c##GYHbeMZdD#;od2+FiEK?dgF +z>Le+B)bm@3HsL%KVXFPTk&tp7F9`C4dly(A7{fcWc2h9AM9uR^cPH1Q?RU1BQRx*tCdTtCPs1`4O4IWNj5?;k4FM +zO2iQrX>(q;toMxR&=)6y(mZL=xm`u6NFdL6sHyuCKT~V&%(teaV~`}!;L0LyQBybN +zI5H) +zB(B3^5~8Q~H7rC3J&8zq9=4f>nY4v$z@FHczd0{Edvd)dVZYe_-@S};(HbbuU$G8F +zT=cVrX;J>fl&vz%@czW63sf~r9kl6j(0fc>x~`l8tNWkt3RL%Yi@~#icq=^ciY+;E +z7x`HZ7Pn2zTtP#$R$E1X_#At?H}UuJwB%Z+lE}mAZ!sCg~sz +zl;YHlmn`5FyAmS;wt#>hN(wM-nKg%y00_RcS@k7bzioQ`tEN_1CXMb1p?=Z;u-Ecg +z6S9`W|NEz{^g%-J;*T&iq3o_UzYCqMg9=XA+gqjB=UT|Aga74_qc6r$d>sg9YWq~e +z8r2@4?JeuDq0R82t(y@HvEIr-2Nm+eO*CG8EIuLc;U;fW*XJab;kB?4G3R}X80?7P +zWWG{V*~`}`6i2ps`8`lV<2*@d~<(cIR72zI~{ +z^D)s7V1+k7J-u4Rl19&paIrqE}+^w;$!)`jCTyZTb=xameLl(dHS +zV?Ui*L_*3^7+1U5#O)=?w4u0#J3K9pq{U5qm~$D!0$_TAeJ#tsO+y4zWH!!LW%@O5 +zcjtM&N!6(5H#eW`{imqv!U6A=2DbkjpnbN&hm2!IO<{|Oy+r{6Y1QLd)kjRW +zCYtWt3Kd*Vw-W0xpc!+~FcYf}5x$$6))-~QJ~En6dXHN*1B-dCe`)Xtj`TaYN3O1h +z-wN?!8PF{ZyqWR5m^69q9l$@-+q0#}h(XLWV6%x1-F@CzKdgv&%Z4Vp7l$FzI9OcV +zR*K}*krw?veQ3`jDLF62I8sMR*etCPVHZ!17-&SQwoQ?F=A1$~<$^$InI^7x$p~7O +zhf~fD#UABH?3J{vJrOr5gu=~HFk@&D_D@Z$NSa0R^7Aepc3Y}geQ~>?p*K%Fw!n+Y +z6)j3NFY^nik@e-yQms+T^0lw?rbaIw*eW&GifzZdjm-#hPew8Q6>_r0hkPp1b62AP +zlQB_+ar9PR=v)Q~CJTs!cCSyRCIAXvOtJ@WEABscu;{)9(-D__!B^qKL)3^v_P--jKO{y?-%Y?8V+0+fPy +zRdcfNP1Qq2P({ix$d0@nVOvOP(K794qG0nE)*RnT3nAv*@!ww7cYK>voQFI_x_bs~ +z+}!3W2mLT7lmAA{ru{<$`%*FGbCa}gxhNI9I~G`Lc?p(0CG|t|T2y15q%9qlSii?+ +z-`t@Qrb*Rg3^>W8pJLV6ZF}p=bwsGo7*Rm|?Z?#)cuI((;ZX`z7KM +zrh+_;rAm65k8*}ebqK}TVx?df>t|fpuxczvoFHdG26$c=!XQ@MgEy^M@SneI +z!8vOn0@rC}!Th+$g@Pf4ZAJhwPnYTVu}!peq-)HBs)f6tF*&ejy-P>ks=R@Upc6cQ +zSNt>rZ~_p72%ISHrm02e+)HSe3FaZ<*9z1jY>wB3=XIb<2QawmbRSo83oV?3 +zPavdx31>Hq@-l)OFVc>o|CI4+Z;i&0xT61nedY>hzTCXI%Vv!8(x0)s5Q{T_{;6|v +zQGwjcgu|BhK8uH^+FB|ng +z2(UHJc(@Rbdk~0-mK3&!Nwc0D`E>gdpP&t&0<_}7Viz*<=?P*86RlY`tzCqWCK)4T +z|8udI9XoOI)n&9&NUS&u$&n++minSf>Llh}L10-+3zUtTX6<_nLn@qrp1a^YrUn_b@&)v +z#RDe=*9}lMsl9cRr$)b)Z3GPwhcvoCx$%kv)8bEr!ciaUnNC;$i(s}UWyRLRO5ZkEb_YR*8VUv^tib#^Y}P7 +z<@_O`Qe092>J;5gepQEK7<(FU@x2+~Wp8sQUT?pou~nZlQX1(>vV+}XY=56D!-u5r +z#r8g(f7_3Sj%??qMxMrYcRZBY?6(FkI+)HDs=0!?L#4k;z<0hZ!{Za@t<2FK?UgJ9 +zl-eOJM05ZFT`+n=fy_*8G`rpT2oWgEVoS%PoeoEN`?1VK9j3H8*>F@qablIyIa@BOms!o)6pC~p0ZDF +zt$NRY@ioj06I-fC1f9E02cY$#zbtEE*yRsGA0II1e4@zl^vrE@my_}6kabX~`(Qm1 +zgoRjGH8CUspB@J;p~O=?ajJtk$c5JJnZ-0X;MdKSf@8?R3@zkd13HQ@iLP(Dl_Md_eTok>#p&O|Qv4CZ?Dw&EhX2 +zU0R2B(~TYG6uhh@M-D#^uVB{+@`XE^EwAFq&CSa8-WVc~6M~0|2&bYu^VsN>*LGSR +zr=s$r5OgMqzeS{A7!RBkp@ZgB+!MBOjHt;jus5>**VQyt!q#^ca9;!-co3e9Q=}K3 +z%&?!Xz(2mMzz;gDaU!(|=p9x;ZRZUGTX8Ba6=WslrhEDcgFs|M0s+X8)#2TxL^tWQMGkC^*_RU_>c!lW%}%U +z)@XvJAwjxqMD*e(JV8DoF=V(e^V@R(Me{E_T{!qoG5ITAd;iQ_%i)si^V+(nPBi$V +zRp|E{$I+rU^fV$$*mdNbBtzO$K3VzK4*Mq)=|}UhC3&PjX;Cmn@tp8*A$Uc@Q0e)P +z@D>@(Tda=tBWNKrA&Lr6S^472KJ;B^T5cfXCD5o{UxDg+NW6H|bI((F^ +zP1wZxTwZTdd@j!bBcDBC-zH)!slp!a&8p#AbX=W$=5y6EN* +zU+H!&xw)tmCN4t2(dP9JqzDy_|Iu3g%Wj~4sw*@B{Dea0|Ckhr82VuQvOa)z^hhc4 +z{K(aQdrf2`vI=9I`0;~o{0_^KBdGffG8sXI&PPu5kF4CLP{dFEeb{Yfp&YyfD;ATZ +z8s1YYCG;`7;{HZS>US*2_m?BJ1y^b%NloZ|eC%=ZH`#2550=D^LW%teu$Qh4DE_6b +zr4w1DJ@Q{A{N?RUwS1n&%?h6YcZ^f#V^eBl>kYe~;Fr?)`*RGTt5eUT*B8wpJAC}S +zx#x&;bH4T1dc1tBC7r>&IJeYlIL81E+5?`e?T7WD`4Qv>Ka~o0IO&5h6naLdehHQK +zP>H-gggKG1KJNmr`)}&=x;~DSIB{*j!Qp;RqsQ|$ucy0&%W3ls@roaxEfl((JOwg^ +zwhkg+p0n_bhBSf~F2|d>A(xi(_kt*@Aw$p30OEV*l0Vj(+MH^1Om8J;JnzXkp85ld +zJ+42grwi9)G4shrJDT1g5`TN$;J7=yK#xwKyfJ}x{h~eXzYUKs^r3L;^or24?422> +zwRmH8sON_^^KL!!?G+whPKxl=aUi#Xh#OtDkoZ~UyA%Y7NX&astAOA_BrC6bh*Reo!ytZiG!>_O7KHV7m*&cD{TdNua%Blejr`=W0mo +zB$HSLQwie1QFZLplA>rVg{Zs*9Qt44{X7wdPoY3C?qmMb^noHn_Q>$u91tja$K2V2 +z6DMCb(s&RTYEo%bqxjt@A^B@8QS9p?%6CaD7rp0 +zAKu(+R?=#T1L@en?@}^l~=+pmvR6pTOhI|^> +z%5D&UJt`o%(?}n;fN`ub9XP84L+K(-Rh))O7-S-sIg&*T893Px_^r5}MYE9hmUG83 +zS}cIStR9zP^q-@CZLtLcl#B|#1>qzst>&2~f!{$Y&-)C&w-_M713^scFbUk8~DtC6+{9|{@*b0m85R`M<0-Hr1iJzqQ%%HDc +zW2zDCmEV!Q{iu7>5P)wg=UY`2>e(B8Lj`Z-wqMEh<3znO5JO{p&#`=OkI*ic1<8s$ +zZN6x`tSFGTL+TYRA`VY7%0mJF$h@zmejUK5y_fAbUkf6*>yWzNllHBUy%H$r +zdMjeTBA2*#U7a7$|F*}C3B>Ds0Zkf{TmEpu6tpHQdw&!8Mf3-qv3%V;C~=u5e&BNd +zTL9ze+vcLsd_uv{3)x4ag|CW5rZIomgUGYnEoidvr@Ipg-ITD{wJ+J{I}!x!0hg_# +zj7x`PzsE&Rs|=5yD(`~n^HaTBEZwzPUwt2sjJH-Lri$AF#ATCtH(rQw{za%6%tZ38 +ze{lUW$4!c+O?oNS7U>@DM_N1i)PA#tR`cg;7dpnPP)$2i&ck;nFJc<}-)D%1p +zkB@2^Tma-TtW((u=W$aroNpN6Q*TU2WAQ!vl>?_)1%(dV1VARd8U7oFL9!?QI#*;< +z)PJV0M<0?KJ&-gKUUBzi$afzVOi!7P#FMLc#f2e%M0;I$@A6a@Ka77wNhpx* +zBpBLz7wU?x%E&($28oBZv6)=YY*4Iw!?gxU)WH7Yc%4Rb_Bw?Hn%^8|J>hGhy&}Q8 +z_0{0Io19;Lz<;k;Y|aa9Z9ommuh;Q?ZT@-ehcAVyaJcr0?>6<(71?yUnNTN$(3IPa +zGD0fy%r3<88(-*#s)40l!A~#N#Dxq_`JI2cd;@9QkDX6`hVKeadg>MRy10+t(RkoC +zjI4KoJ@lIrtIh;4g~t(cMz(-%4%<<@eB1iR7K=;9$?-iu#0Ad816CEg5;MiPNfaUiD)gM0o<4xVj4}u0Maf_*WO#|H^H$F@ +zI*98QwUlD|h4K4K%cQ7XIR{}Uia*#JQlR2qWd;PLJ(uNw`}{6nz;}bd@H1ii=)gDy +z+>i5bqJ?fQxb&Zf*XxlJa!_L0nWUH;adpQX$8okmzvb5FWwHXJo;3oi)f~%#(Lr0r +zo3_;TEP3NIqY}LQxxjBzduM$sOCEA +zd8lX$K!TkSXn&qOI3&oFwyn?|&COg(y?5icceq}h+rY`1GM;`8oO97#drR>;=(|t| +z9IiHpl5X0J9Mr!&ZQkgADR{@fFnC3#K+iAn{Osm7GaNG3%4xtBWOA8mzWd%K<-Y?N +z7WR*{dGPiPNqe>-V`vLNHugn&Lwlui{UyUtjkn(}Z?Ph8;)>g6SQ~pa^nrCh>S(dI +zV18O2Afa~Vl6g_;Ev?16oB-z_?AG24d^ +zY=%#m1+Pi&B+npuTA=SFykV~QY9$Bl3ve5cPR3C7kxKm#g!^{KO-uUm$=6bNnAK)* +zcjqI1`@Hy`@3*O&L4p|+gj +z>@>YnI-oGUGOU09H20c4w~(8``Pi4$GDxP<;fv!2DL?g1(4a4JKcg~N77W592+2f` +z8b{BLIl>aqMo2yfoR&NFN#51QNc?QUtWM(8TUkVmV3UG?yKJD>yXEj=okKVK1E&~g+O1rj$>bbO!uV57}M{~%w +zoDTw<#x0sO5MqgJFB%bmTK%p73?MSq~ +zk8?XRb+T%CVSAjb3A~No-ecR^V3UD-rTsgB#nC`HD6MznB3C3iTg&1gIZ*lr9(}26 +z^g#CZQ^tDHHQ%5(KP2CzMxfKjU=O;CMn~W>3%XohDToS#Sr|gPe!hYq$O{S)t0O`k +z8@s}8@%M=EdV!&pw|YpVGRAXWMRD}}EJ0`VxE|5+IAE**uD66As#;Xi_bUH7_!$vc +zfA8=Q5-el$KlC4&=u~|y;6us+LfcL)e}eCk;;T9$Py+bj$ew9jnTA{vK9Ti&Pbj>5 +zMVoPZm%$gha~l_V7To=e(u-X&H2nP|IQNhpWod~o^Gs%JoaoTN7xh8U*IC-f+3N}U +zHt&hfz2HqDR`}ZPmf1i4W8z%gbP5!o_W^5P(w!sc!eb$z_Rk&8{NMuwf6X}U{ce5e +zvs=LKb=_rt{5UmWX7ZF2AnGp#2F)7K^+`wbG%~HEqB^+)Z_qxFyays&y3P8btudk# +zM53y=z=Peb1%)TC#-@N;8quGG?FtnSQ)U +zJvR3RwzraLxJJ|Lb+?-1vD<0p>-o!}zzZI}$cx*!;iL9$>cdvTxc8yX6X1@^X@@ih +z555wvWm|^5ueGa$;kTEjk)LAO#@h7F}TQ(*8g_TRVmPaX_1(!uYJQ?~@@h +z_${&@%$Mnq(H*EF^g@E~AfooX^pU4@YL~;RLasPjwUc#_C3qRfoyXaXCbK0AF#3&=nv4W_1~>E4Nz$hn$UmSVj%-P0yZ-d +zwW-h-G0D?JO=$8EfgU8XkPVHpsL@U5*aVZpXeEBX2|6ibjX_T!{BjU}LnaqnH$7ox +z02lLWnTH~nRfe4XC`F$9_6G*S>YyedsFIiOz4r|bOqY(H_LW54-_kHgh{>22FI@Ao +z!_Jk4(Lg +z;BqSbcJ8oxfb#ud+_=#03E2y&&Gf$cyZQbLe#y>@O*}@tis^OKMla!c|C%&c9jAm1 +z$s3de+6OA9FU}$ODdQ*EH^lT+AFt~ZQnmJm +zvBYBvGhL%4YQ4%kRdF1(o3h|*t*)ZTyAwkCeIJ3%(5CqZinT6XoZG{e5o7g1e?gJ$ +z^XFHLyM+#EI^wYJncuEG=^YIR{N|yo;y-YXWsIby{XbQNeRc-gZGqg$WA=?6bOC&S +z9vSq?jFa#&wdzqHG7VaWFv3}uL+g5EINo+Y#J#5&vM(}e*ms4k$WFOgt!6t~$C70Z +z+$;zt+L(=29j^Km4<>MpCbni>l>XN)ncso57pd#`83uH5Y0AWZyL)jcQV7M}y+CnycP+HIySux4 +zad&BP=j8i)&dizrx!>%`O!n+O$t_E+&sv4-D;up`v(f#%R?HIvIz05Ytyx4mWB32CV<5XX2ca!I<0Fs%IZM +z2^-R1K0V8z=n}1YKvVs+p=Ki@?+-DA|pN) +zgy=b@)Lti3W2vi5EWR00Wt!ziNFu>+0{?1L$Y)23Q~vcPD|(p5tE#%_UqKw*R|Uzs +z?H*zwb)fX0VJqeCk&;Zf{VMi@x$=8GDyeW7)I+Qv%+7vW?Z>;TndRf`bR;47z9uNK +zIuD*ql94}`(EJPWgaq+r-+guCPQxk(-pJSyYg6;9TjzN9S)5Ixl1eEE+>OR4;^i03S&a*bx*Gwheu|F$4bYyKLbw +z#y_QyT5%>3FiB4L;NDH6?y@9i)lm*ELk9%nHpA%~H~d2Tt#)b9B-FBCMnS4Y4LIjv +zPdK`fBM9x55ufC`9KHT+bZVw3{4UDYcB7tYfZKmwHn82_Eu}0?9w{)ry<+l#Y*T{s +z&_+p@^N1gs`{v%ZwO8{b$acR{EyX@Palb3(2Dngn_~mlxW7@*VCU2&J)*b$FI2Oh~ +z7q^4x7t*vC=h)mY=MpCnZTF7nIrRo}xjBrL>q*)l_fGsg^?`zDD_uhVjz9Xy%a%sc +ztEcQqo3%76_>kLnFetvzx)KUXniJss?vVtZxIItKs1)b!)O7X-~ArG$p3ux?*^G>AF=oWL~ +zpB$v{QB?mAVtKT#sYVKZl|?gm5UJqecbPTq(IyP41hP!D3*!bQfy&7OdoN|^u+xye +z6U87rQWO|SazXqH^J^2`-?s5y2%tyzs0e3UE8VxFoE$tJPtGv(Jmf@y54N&63!JPZ +z#Z8kM>;vW|t~`k>U~{gU~rRlu8^dv5X;fhy0VW8kFX8(--Zp)d>VLo~5tM +z3kys6Wo@xlc4~V4w*d`#_2tBjJtFz*Kp +zldIXIp`fH@Nds#FFM3^AyJAa2Tp@{-8N&U*MLd9lzf6uNG(wh);z{oF;lqHMKPcfGYRAJn +zt?>pPLu_K;JCDfozDqo?yXQm)X5GN80R|DU>H|cZ7e$*pe@VaGs|l@3o`g$Mz3n(nsPB7Y(n_ER +zfM91*(PI5ac9@Y~xX` +zKrr%s$=y$V%dD*y?hM!pzSh(!O?Lz-y(-fi2RWVNJCSgKlc~l5zu;9#yta?%Du|1( +zD&eOhu@r?qL}?5u{2uHsIiiJLo9Uv_sTjdAQ=FoGwW!^_w~8+jL3yxn1US``Fq&mII;s25%lbIL_>d +zh0Y?nTpBe@c7*RF8T>MO)*g7y9e(`NEt$%R$)iOm%t)@T*F89k_<_I~_bO~1mop^H +zb^kAW)OhCyhVr^iGu7JH!1qN~wzu2vU(UxAU`g&{kvE)Zp=-vmmt{+x7Z@s)cN}lL +z+j9iD-+r%GJIohBquYU#$Ag{tqVnIGXS(0uW9_@+443)Qa-N9t!n7RI0-k3H#D||{ +z(!KY$LKf10oLiJ!iNLnAV?Lz6B9Z0v-5}aTM1}9dMDy0wc)d5?b&Ya?&%pq>#6@@dySe~BGI$i}B7=cOKF$+E0W0Jmd{JQqt>07HFZ?0~L +zY-?;n9HAyS>fyY!G}`f{47^1ZzdOw1j!8_RRWR=wPAi$~l>70(NgX +zpP~}$uVVKvApEdz?KL$mpydpJA!@*%R=hF`iR)!vl01&5Hmz%pfSQJmfFEW*&#S8) +z^4f}^5$WwdH~j?38V!^a?~EfG+SvtN^G4ypwaa|}?q*`|8vzpF8Q5Kv(EaiDG{Oe1 +z7QAi??E4uDS>J3Zk>d$T!{Z57UU@_8xWwx!fIe{zAo@;2lZ?XU^)!_zTE +z^zI;s2kJ}h#jT}PC&0rQ=u^;dnMo)YEDx6RJ>DqH{3tVO +zmsR~NuZ>^8yQT*BDP%VJx@zO{j1nMAj~bt-DSc6HXHR?3@)K96Scl-%+aH_Jp|OZm=9XCh*_0Opp +z&1$TLe>P}|T6{XD?_}GNX2(891oU2P?*TS!Sh_77VYrn@GwZxFy1JgWJ?$+D#fPEW_UM+cRBPJR)Yd(Azxi9w#Wx-t{1*mxwOyP0@zMJ0u24>#jdR3o%L* +zLFJd)5K(iWB+4fFC^0hujD@kVe$o|(N4TDnR1LzlMvceJm5qx)Vo_y2buec +zDuOv#KR;|gkgq|K{t&NE=$U6851mcs%rIsF=aXU7LNDoWOah;ZATB5O +z78iQVc8+~HBJF}7m>6-p15@-&EG#BgR(R_3Hzk!eiF^p=-dBkTSHqg?u$7P?1jcJg8a`id#3<^$^Yz3Q<%$aX*?*bIs@h*^UwDdD5)e^P +z=vy)%*gCuw`XR$e)AxhxqW+%V>FA}6x~fzaQxju} +zZu>Up*0g?CEF|RpeHSzsbS*)58b@KzRXy2_iv1M1ztT-*hHUITM||L%u7ewXI3;~O +z_Wg7ulJ9I8apMbnFa?)a@E-?TUd+PG%l#bxL~<<{5vpiAF`iIa>=g8u5BA<}-<2vK +zD;9q*Rxi7>-fu|h=G9zL=@v0d0A+2VCI>{{e#(aQnksm>|6I*B +zoEEfFh|n@EewdZt^OvSVQOr)gIekhrfkJdKw;U$P_usmZI;X;Em*)5*d&Nt?9~S55 +zAA#r>B*}S|Z*nI&A$pHFBH9rwTBp;U=C?eY${ul3a%riD%OAu6FB}!^Ln(t_RK=<~yO@njcD5bM)z*TCSR4$wr2T +zND_{=VoOm1$NA-`54F!&L8u(D4rh!mLGi4qIVwZtRx2|JA_IS@^$Rc06#ak7f_3k6 +zSeO8!a}5{@C_$y1?1XO=Xo;2p1-PGLNWqL|PoJB_9LG3fl_>zB!?8w|KhWI5beN#{ +zV#PI^cvKB@k20RVF7T43y1drz62c{^yA-_EN95~;!+_8s9PuC}1RXc%dkY9+`x9ch +zy@E_4m4F=F_H6EC1WDhI!lwx$GL~ma03CtDlaqE*@{IdbV>nPz8;I`80FMBeR!wbf +zswJfNb?<#h|KHkPe6%P=;+pmG`*N=% +z1^oT$zehuza%}TbU%czAaKFKHY-Uh}Au`NSf1cLv%jxKWEwvL0ERfKskX(W2W42Pt +zjiv?1<$a8^axR@`HuWYY#0-4o$m{pBX`}0wT)ARJW7`5=up%)M0xxAkHv)s)+&SWG +z?`mV41?El{4ipOJuPrr|POQ*1X)j&Dh7B2(Qwc$@iR_6xatkWDVhuhh<|_0YCqg=e +zgiKrywWs4oUY54A8&WKodI+7qZ!)TLl+5@*27daR=P +ztek3`6l0CgZu09t3vB;AlD}o>=hr>r*!=#x5kEP@$6cG@l!TnF&+XBIA#l7db)TZ| +zj7%nS2t@R=48xvjo1UIMw7-D1&hO#)!+DmME4H^*4t$}^KP?r!c9BMi6824S#&y30 +zfX9TX!?B{5$?EEvOAi6w|9!WmP5S3M-?8%F0DSZ +zaN!#=jZqtbr2kDEaOJ(+Z5mA +z_-~yU4_;ZGobO(4%!dCD*vbl9%MqIq<%;fe3<2YrCOF3*OhD8Jz}{fOP`KzSUDMg& +zLZB`JZMAv)BO#|C3k%v_(lIIFCO7jTCjD|hM744_w6H$L67XyBOzdU{!06b`G%HprI~xbfVYFs3fusiQQI>&zf9(l>h22jv +zkFS-w4~W<7%{9>}lY5XfHzbmc(8w9dlus1!3R9~O*4x4_hyUErft`U+{}~t#A4mzn +z#Z~+7UwI*_5VI`$>-Xb8gMNe;W>Jeb6lIdMD1Og8`I7h`3GYTxa8hS~x+^W#hfQ|o +zDmwLn2Juz-l?4r=woFI_%&!~*eCvfnkGmIAM}8p8UdezA1`t{Qz<>+t8T21ZIguo +zA2lZf{>s1_=c +zp8{*WCiAcuJV{S{iXe)c|LHI^EvhO4 +z0o#Tl*fn3occ!3mkZgSb+l$QC{T!?_dNKB)#K+=L^b9xDI$Qk2ad+T7xSZWt(IAL4_Yw1HVpr8NYbG}-(j +zyqF8wTz1?72kh$p@ou8m4hOk;a +z9@VtnIrgmIiK=y{Ql=Q`=ycH`dM|cMCu8h^BbpYjP6(_k&M_uRe$`zorb~zO(qWw3 +zR1HtJZy0iN;=q&6XILk3T5k!B*Jy*xI4ae)-<8??-}&uL>KF$2pH%J>Zoq%P_x}w? +zn_oz95YSW*CG!6k<^R1O_kTi4h^`R+|J4%fTLtaWyY$+pE+pQAQ(d=u*{2>h2K +zWwwc4)$-7>nWHcVdF6G$|NDrVD@f>~4i?@BH*M$(Zp3f43cG3q{Z%J-WGOvWm5S{6 +zvWc_h`LmAJnKQ+u%F%pWQuvT8=>NY*i^2rpzVg`~Hm|!!g@w!k@{TJG}w9iMl>X@1{TOeU4PqF`3Dfkj62K@WHuW5l{ +z3s*jl8O8We$0qnj#}L!MGqu*pGlGtv-_Y^tNxE=^HM%DWC%89Rw|eD!6f<0r2?DHKoCp$$do;{ +z0AE*N#mTvUNbK6GKy7VpOvoa7MTMPFN)ukKys6n5jh|c#Nf>?1;FOuoC4uJ?W!4{Y +zX92UHmN>oHe1}{J^{ATd#1|Jo=~yDv+DRf@vy=7*?X}_Rzph<2MOD?ihAzmx91d_R +zQOrBpL_ic3tfj@uirVgEHF64|sPO{&h$ljZ8v`|<5>5Jua>=g#2ieG{qW +z^;V#|`u%L&ud=R+*UwX5*x>o;yRO5NR-jjNuHQ%3yX3a>g~~iEw1zRILa +z>)EJ7ouxXM_PbUP8glJ&+$ +z*M&uH)%M3vIFxek=oI{X)c`m}5lc9@<=$o6DiW*mS<^KdP&zfcc8N;W^*_xokCG65 +zQuC+foPn*3mf5CF%V)-&@l=;PZ(f<+U4g35+uIXa6+iCt$*g2i;f({wLeKf>Vyez; +z;@-*AJyF~i^b?fMrK(OV$TvCW6LNeVHpd^f8pn@3W)E%{{%7N(owLGFV25!wBZ0J_ +z{#*hn$cKcHu#Yq6S@Ds0Rxv|NAf#5pEb{FgCe^qhe#n|37sPJIEV4KZ;UXj?k+#rc +z1%WKseQiH!W)TnWa41ePg%9?vzoLvWTGMsIL)4@YB`k&7C&|=Vlw8s9(Ng{a#Z-NnMb(w%!HX#%67y +z$ccgsT=o9jiGmLda)Tx!&6Tif2z>4fg%B59E*m*bHc>5x|GO(?Z|Z +zHhOfFm;bh4EL|H)-$qdnttt0<$Ah+N-wX4d(~FP(?X&l=yR*&rYW!u*B?p>?PEE)b +z+9LelVb*5VX&wLw`;&#KqfZzcNJtWrQiC^u%pTDBi^M*#cZZ-Zz=XU22dUym{Wv7j +zMTDTX9`nci787k}Wgr=V;&L;dw@z+-0l4u4AusX9qMr1m!d4VG*yZNgm|a`sj(^kr +z7BYX3|0b?D%@iIb;q`(A37W-I?tM0gtnpLN3VSjF8(2Sk5gr~Dq$fbRy%n2!;ng<5Qj0p;s +z-Rl!rV5{?zK5-s>lXVC~Xp&c8zIjg2*L=P^gWWR0j0S;@xOrP{o&CD6lVn+0Nl#87 +zb6F(g=-K$CV7YcUg!SOt=~Gr%u&eRz<9KrJ=Kc4AVigJX=V~{HYTA#K{T8mbgM^&_ +z?yH8|$OA&I)XZ_1pv$wZ;aF8g9A~3y!f=|NCQTo&{j;jZ&6tZ0} +z&-yZVBv=#bdLX3?$o`weX%5qXG5TRDQGqHi*UQ3KEoGs_8=K1)0Rh`gldwg*YoZ|~ +z1SHtlzF{;*VcRvf4p;|_8!`svY(f)8Yx3#os~r*?ucM`EEy-%L7Jb*GAWNr4!jp+l +zu%gd{wqg(>mtkKrWrDtSNWsD7m_-mP!R5}b%(%&kI5!PL3sKnA; +z+tL`h{}^ZsFrfu!i)d&p8QZuZc4Y%O&}{#0hi;qm7Pf|!>0V1S&7Ve!iy1O +z6ErNohCCfB>T9B+ru9)|JbkFGb^9!uf&G`&FPf@_O6vzX=RY+HwpigqM7bd55<0lC +z()`b$JcbT+Yl7KsiqT_7k)gbI+Qw*;o^cEHM(&DI)col`jRu=%&1!|Jn|*^R+WEha +z`Qv^6j2Z~pBW&8`?O?noob<9=M^IDb0eT>9QvH~fg*D1M&{(_+@|=3LPJ1dSi)A?n +zQs~cFr+%f9QMG9BQd0RB-Ryg(>zP?`llrq_HSa8bXN75()N?R)R+**sQ)1+?J?XQL +zoK4y$qVv=QB*G$7-VbE=X}B@>6z%Y}4`Z>Q%*{#NI8@6?+J9TRcO~FU;I3U<`}NS- +zKyk2nc5zg>zPI5kKk{olXAZyfry~BvlFIgC)IHr1{ft?X*ZB0sB{!xpFU8YkLx1|>f +zKA{a=bNA{mY>k_yIdJ^csexpbr1TF!57xvdH43{!cJcj%A(hGv`p9PKsQP3x`LUFB +z{Z))og*1u-k2R%o(FgjIJCTLcJa;z1f9G?qK`DB?577oaYgA$8!WhoSW0^v61D=H}PtFd7Mm;o&pwi&&4hIPh9PI!}H +ztBoy~^htscZwA+12I9Jnt^8b>|6%QZU~FjmX-|EeFiSV}&9Xv-9cx(@UN2$}x1>O# +z@TUS@F~{sO=-q1+9)x(?psUYOU9^4~@($?@?xAU}=lCO#`UnMNu`CzBN})|VyR=BV +zr>2%8MdUTMZ*-8{xsP9*x977I@k9JvyX-=(LgMjsuFj=gNqP_! +zqWe`^4(-$H$%AJ1)O1-|P8QLC4TM+>>^8VYYo8Sr3Mp&TG$q72a!WbxbS@q%xjQ?^ +z;Y|Zv{@!)=)L%BNI|PEku@OW~vJ-L*Ue(uCqh(tr7d~^RvL{g*1$3Ql5*3AuIe0N8 +z+iujkPaD3!~=scBHVvrA%$_n1c0J{>dAzd8q(-TWf&&BDl%CX1fqBPz^<$dP$k7IrpONgFdREb<~7_4Z=&w-}4JK*a4SE##(Z8Hn|YBa`Uz6G3y=3<6DgUH>3R +zBA<{NW)h;}CtSQVl#DN1Gl7eZ%uiX^QOkxIRqR$k9E2o=3Or3sO;x>oZJ?w7$aARV +zt&?6z#@GaxcB#Qih!H9tyPHuBs`kw0Eb~9o$MhiCx)2FhC0L~eyLNKD&(Gvg5Ggu5 +z#3m8AmE-$}(Oc^x|B6l;uEF)|Z3IA?E9umbIfSUG=m9hB0ggC$r2SO-i@p#!P#hOQ +zX>^`;n?sp;1dc3<&mdK<-f`yoN5f5oGejb(5g398{w53`)t^fzO3{8Bv*f#ZR2br3DZRf5!V@~_#wGG7e}F)6 +zZ-qSXY5J+LYJJ-_~;Q+qv(G@VBD%; +zpU=SL{!{cGEQSzHHB=q?cGd+H+rrhQubJ||{W;ShTx4QD%M7|2njpGQ+H!{}Xf57$ +ze?Eh7(%~$v?Sp1VPV~Z)hn!vqteoHtmjf)db&aWO)P81U#EfMMRM^_j23No(aiShy +z5M!mm=KSz-L8)o!068Ii!*q{)k-+~Qiu-i&(}fGVjM8iqwS7EnY00hZ2saSkD02Wj +z$gunPj}#YAMk>RimA?ySeEZ*F1Tg}lYlDD~aX?L72XtB%E$u2|(kR4l`g(hhAxj40 +zlGncfIZqdz+$)7M)^;e#=ke^DtObE2@IM058ch=eM^ly{3290qXIKjD2nk**GddXY +z9k&oj=@1N`zVmB@+75Af4`nP2!fd|ygPxvalGRwWJt(Ci9f8<9rcK~j!swyoRS%pq +zjL`zGdtSW-oTQ0My3RUq@a(UB#;SpX-Indc&KEE8q9fixPv9pT0Q`*Y1MQSU_(4z+ +zQd`M6(sO9h>=JiyMhG)n(p-<<`&+Nt1Zz&u;k_gwsX@S+juZz8MPU~aQ32Abt_SsW +z&8vry03XE*le@hy+=h&dw4C`t;B=$gZ3N8x!fFiF_m~fqWnTAiiBXjO+t;>VzY7(T +zqnDb*v-zHoG?%7+6?N&ohq^t)jJk(>7!yb{$W8H=w!e)4D_Hb7R$>nOg^!QPK!e}K +zlQ*o#vdoO(*TUM0Se`g1mvCQ~v(DsbzP$Fb-SrpT{u4S7Ugr;wEHj_@;^!pNRJC{S +z*w#BsR0>Hk;CFQxpRkG?6**vT@j}R%pMh)SiY?HVa0YCb5yByhc5t>HdjVTJd*NySmr0(hq5g7MeV}5=Y +zMJ&`)go#mRl50h0mZ1~QO7{v*wmRE=UOMprOF3DkBo96$q}fi594>~u&e4+jVcp%Z +zZ6u*m`c;(4og$|#hBK+V*j#&G-;6!<%H6#0!$sn>six{~CF>#IO$d`;t{`Bm9U#sG00WU7?d_1l&XGAlS44w5!;AD=cxOMXwZd`#ahojR;az1~OO%|F2w5 +z0T}1Ub-dk8-@pKrUk7PxI~%(tN1`;j#Ji#3I`Vjs6p;|}r@T=ONM+vp+C$7xZPbO4 +zmkxp6+jWjlnSV^TD2_5rxA~A1KUF05bu~6pv16|cY~>PQhqQOq!V_tOlBLFKeO_@H +zj)Ml4FgiTNAv`9e>{miMlI^TvUPO2dXvtQ!XY6VBx@~}C8w~4;KB-2IL+a5hQOmco +zx|sk*Cz(m8geSD6?_t3mhvfI1(1IcL<(V9ME{(U8V=j*FNiL7Eu7iRY>s;ZJP;3Kg +z#qGB1GJ?TdDV(eY!8>--DyF6HD>Q9 +z1U$op&>&h~!A53H-1B%JwIxvwuZM^zkgsJ+g?*m`>0}`_ +z5EN$%a#KV!eZeQ@g;d=&y2<|Uzlj=}Ft^20?X*q|rQ@`I@jE?Js8ar!@JYP}pR_y9 +zY~#)ae3;hR2Iad@s&MJU7PH1BUrDHnX^*0(EgIXMG12>{lf2Pc-ju^+xnqIh@Q*EIOpn-SNp;*$WS;}0@3 +zk6u*iec}ypgJ_$ja+sNYbl)emG|2nJ^KtZi@{@hhpJ)y&1=eFi8gU$t@DtD&Z|!U# +zM#briZmYq&p;UNLND66NuAYP&TS8q+8`XeVK(Y@>?J0-?*7KS6q^p7wtQ(qouYaEH +zqcQi7+wF|4rVuR#NAgy8f$KYR!jh8w#XxpyaYXXT;2mkpEBwePris}Zmvxrlz|X|B +zi2>yC0np8BcG@8E!5<@DV&C^e0+l%-O0BAYoYSV#u6pHkM~a|7+2V)I$Qr`Pw?k2O}c+fjR`?sVf(+1ntP5G7^q#`RkTQ_Iw!!oB2bl?peI +z`O=BLJ(bkM!IxnuO`d64pxI=fR6WTED+7TIT54xUqE^tZyI>yuGLIcbkBPViZ;}J? +z->}G&443?Tcf!fh)RI7}tUafA`0|czJ56uT)VBMLjN^%8Ipf +zf-VD`(4r?tz%#z4!8uRdt=)2D*MmtQ`+z +zB`{y;enH`OJc#|C;M|zR|AYHeYF4n+s!dv+x)4K-BqL&0k**+oCyXc*1Ojof8yt$B +z|FJn5J%dWVx-46MG8*Kj^*Q9HT^{=>c=l-@NdjbDMs`F4%;`8t~^qU9G8j-x51 +zH{Jlw;w~|i)YcyA=m!QtFhV0bOrAvieD(g;iYJ@kL%gQu4lDEt9HIF5X|LogSfUA2 +z42a~?YMK>%APW7j7M&uaEJ?*S%Kt#zP7g=g*s2zQB=hU~W#CDjEpaUK#>^n}RS=?; +zA1tTQE7);Hq)VwPKFy4NG)I!b-nD>MrmUTWp+sns-5!XLMZ?iC>$BFcK_Ews_W$d= +zkBTH?k&9dlJIm*Djov2O{n-t)ZlOVHAgyVro&^|%xiXa`+11!)L +z005K_GKEo}G8;mnsQ_Lk|EW0A5tC)VwlYa}HhN5G3yY9kR4{N@6PR;^SiVXZ0v`Pnlfsf_byLzkC4&m|^e5@SJv)k*$I>u|%6RCjWOp~VLwbJgXS +z3`t}fAIW(Qls=}nYQ@&QQjmab2|r +zzJk0aZ&XWwJLSY!`wOTwdb*_4M|Smm_OuF#u#k-)tQvai;dZpSz(ADQCF)DC)6#;7 +zWyn?<8=#6OIp_%JqL>kn?57x7{H#n>RDet57t9qm+|V8fG@b6S1U$BwyZW+ +zZvRchIEA=74Z8%rhNzWnf4trBld=j(rny)s3?wrwhdQE%bRjF9Bl=zWrU1&eQ_FF- +z@YigPEBW0Rw&t3RPv+nPY17lUH-Q%(0YpHBjo%zP^gZ3a7Z7+;3?|0{+~?pTApy7e +z^grc64@u}1$-t4lR$nSCcD*TnCrEWKtiDY33N|%i>d%CNksqAbzU$+E3;wdaCbe~?Jwseav5P$P5$+t+3RTfEeTN0u2 +z$F?=v_6=E-FD11b*#CK^G3h$haPQJ>Kfk*tjP{|{t}ZlFQRhivh}0Z=xFYmaqADjH_jFLDR{ayHe(q{DqdO~ +zr5!(#l%q0h-$-Sh(|iW=Q8|BmY8@Q*7s!+^eI1L&!l=F_(?}_9%AWZsZ>qi5Fh47l +zf{Tay2ee<5S(ICp{-_?iN=W;6zlrR6XIx$Fd>hWz#`IZ??GbDU7o0{#3#2y1t842D +zijc}zyfm_AEw5LEF~W~a@qTS;Pxg;5voC(Z)HFkd3Ws|~tSxmoPfbzp)cO=`Y4IUH +z61M72y8ZX>y8%u4G&eff2|fdT=DeSq;s)$GXibBJe+?Z2~z&BpJGP0@*G0U&`DgmhO^Zx8W#Lck`j#C8dWfzi&) +zEElr_TsdpGod1YG;pjFx!wb=0)j-aFN&OvxZ+bp^i3EK>rFiFxmY~-uW?Q6DhSD#L +zP2fQp!98Bh{bCC(Q#?}BWnx8HBGWBFxZqWMGyWaTU6`@;K{%2EBz){|xB-^_j_RD_ +zOH2OhSUh6Z&G*wZq+q+A9nxUDf?OOx=E?SBEBLEnx6oYBV}jLX{zg#oE+Z6ZQYT?RxZ@n+B)?i^M7d`P(rte-9l% +z(j&u>4V&bwI_Bxgq6nilTB&e=6gH9CRZQc^lv@EZm)z7gOGDGobDy<~2xZ$-Mk$fb +zE@wF-IgFTl+ovKHUnD_K`INvCFY${;1Mp_i)TR>_S3IWji&)lgJF?2li1c8AY32$A +zBKFIg3xonv6A~Z(?i!`}2Tjv+u`jTyo15w)fNOZT1x!t$m9@C0o)s+-+zNiAVpK+w +zaMMR}F+uwqK6htPqY1;8*s>r{CjcN+cencFcMq3HY5hCL +zIx6rQ8HxhqE#Ly?_$8^Hl{zMSUXC>-xqbf~yBiK;eVzhzk+tgub*Fvc+y5i@BR&Pv +z2@lmt0C%Vr3CTNG?6pVN(&i!Ys#3hw0$5>GL4|&Mu(D4%EXS}Dmn*%h0t38&@1=Ut +zRlgg)K{2ApF?=;<+RlK&g5JfxBxryi!f)a=tjf=gF!wbvG&KDC{J0Gdr*zvm2qj{G +zpi~0CboF{0iR|Rt2ESAB_72aQ%EZ5%RIB?+vfwL}J`k%ksmeS6+dhusz<<++5BsAZ +zKW>O0OYEq8?sn|KmcQPtsqbJ>JWWO^)cJ0_I%%O7CCPyGuG +zo!DOr;4cgy?;F)ah;Gkp4r@HV^Ttvs9Na>6+I$cm?1<ru(2fERIS=RgS(di`RRKy6D~I%)l0f*K$zZ9IA6vTOe0Z +ziIE`Ospi&#LT+*kwL>!bep>|xka8LWM+PDoV{C4<=(aVIv{Z&o;I2=#Lp@P4RoBJhn-~w>ZsU%(a2FiM|JBU%6C)=@Ye`U!-5ZyI33; +zs>f<;Y$ziAO!C%WHh^nYK-EH+cLt^}6-nZFIOWdAewye6=a%`I@GX|z*YQ;`<901j +z;#Efw!R)oNf>h0lKTTAP^&|v3odj%}{*Pwx@V}!12z%Vg4)BlxFE6 +zvvEWm`v$>hq&r;p0c=16j`BV%H1rKZ$ +zrCwolh1uV=L~Wk$E(;4BVs#rvXX#g4$XMAO?_92Tin42FH_i;XL_eDP$8guaU7r~+ +z9H<$?t`hda4*>=YFPocQk*vc!*{~kuq +zo4qcM+j9@~qiookbI$(k6Rw3LPzy|?w=sa#K;iMZSDO+EX+-oDD_TXkVDSzIBo=2T +zDlittB@V_=$>r=7tkKlum5%pkn-h%|=)p!U#^yq%_}p3(7|91L0WMSFU>?7oW=XK^ +z2tw`SF%|(I@^NnJKFIC{;$41M&e{%t_=Tg=r@BDl#=&>sS0iT2m;c@2LMh7qw4=s#!d9Am*?tU{Y& +zF^&Pl9~bPRRRHSonhnBZaXeB!01O|*x+rz=r3|4yXDCxmZHdPL=&e5o1nV+EYUCBE +z`F>dVZ_t6-YEyRF%W_`BDrV7DcH|1ecle>_fnquS{ydsGm{WXHwb43!0a6A*J(LFF +zcv(Vj>=-k#$1cG-TYW41O>n_(SO8$b3ZFiT)n#tt22SF&B#DG~yr&OI*?MlOW!(Me +zk38k9c8i9=y52h>a2Tf|c|u?7jD{lbwB890h{8%JNd +zhsur*pE=rNSb!Vf)4&_%QE(w&$S5`@Fk&s0H^}y|$ +z(P20k{xyPpY?h4jw{FCFdAA9Uj=bRk0snp@k1jY6kY%pG5eu!!L;ej7hn4dt3~eL5 +zirmd%rEP|9q1a#yU;(p}GLKjDLAw8kSTTxic4z>5pR|}pEWIb=ZWHQttEU>3 +ziE<9xU>S!Rcey5xI2F1iUx#FZo)RW6{5c4BN1Z$$$rw2hVXm5)*Ad-{o?K{508PaD +zizpJ@Bl%7(rM=7V%cmumGSc|4I!#pY1!>x=;pDFy#4o%6M_m~M3ZVhuqx6?4vXmgc +zh9uj?`c?b~#JPYB(TTJ2-9}BK1$;!kAxmr>gbMX&(+NY8Qmz>XK#K7YvTrttGEHAZUx_tDO9vc+T+hJCSSELjL>M!q;(YcZ^mHj%HHfpYIE=d;5||< +z$Wz8v5lj$tG-eP9by`$Rg397esg`Mj;CqVMAL +z$8<0;!ru~Sqf&15z36NZ)3H87J)}h&F_NvKJS3mWG}wK6R*<6TIIA=vn4JRN)HsWG +znxbjrGmWjP+EMj%5-F6vOLQBK1K^N_h~o+7=KrM>7Ur(D%an@DQ>DUN%3yi5_^~56 +zV)o5aJ_8;YrD*>&8LD4IHK_8Q1X;#3SoM;GtfAdWZ=QM-DaS=P6Yq*7yw_b$Lp|U; +zuQ5duLPiH^wEuthN=o2=W`mLjuE)3syOt@#5!Gibow$ngop$}vRL*_zhT!jX#wcnm +zNj?P^bHQM&Py4^G#eZ<`AeQ#}Jq-|j9KBW@U(-2LmESlg+Zqv#*|L;PHlSz^Szk@{ +zrleo=BgU?RimtJ)Q)Ctry5;lZagh^tJ`{GS5MnKudnMAe4k||*r2WMyX@ooHsz;)# +zw0ue4^p^F)jh)hM`M++s^7$x@8Ksck%`IzWiqvZQyFxe79z#M&`5muc?3wxK$dUD< +zG|C&y7Qx;RxJz2YS{|p~6eOj1)_NFN(G}OFO)M4d1&lAJcOTYXQf|_l884Ux`Z1zk +z5yCYHF`s`JJ*hm?Et8G)0qLG7vKeITg}%0FY$kR$Bz;1hJ5*!c&)SL$F%iGSiFUPO +zZ781FVu+J$48fLMfRy4OX^vR@D--9Q0VkO~vp2-xF8*@Zm=^6LZ14E?e52IdeoC-d +z)q*(CjWE=($m1-kZ6f5}T`Exa&%fm~XQZ3lm2U&%k4Wt$X|1`KfqZw#{pXxAR^f$J +zcEy*l>3#hWSJVoy*m3xV{2HtCL)Nn> +z!B7umY}-LRD@>AA+rK=xVXU8>9*uFsc-bCTAZMhOBGJ3<$^wh_Q%wR0$@0H65Xp))d;v!JbB;9ig%rv@8G6 +z8rDDjSn>dO7j`Yxf3oxJ=M*9yl1D5*0hUT!SA5hTxR&@Xb~obr2iM-W*zAuKF(}+O +zFCt{xT8y6r8s?8DjVnCZQB<^@3LS`Y7u^yj3ZU|9m?b+$(IwkpE`mbrSaa17U*p*G +z{K~xxgi3Pf%?8y^EY8w8X<Q@)?X_GX;iF`KDD6A!+W~*(pPPBE)Tcf*Q_D`( +zo$hLI2pDMW+XQ?p(rL`N3M~JBxO&T=IGb*3m_Y}3x4{ONKyVu*5Q00wH8{cDCAgDd +z1BBo%!QC~%CBfa@opE1IzukiT81E&en!vm3ZN26ywLG&OBmGM6RGXOeLDmc?AVvV1meg`C&(cef67qMnx7HPXyCHpZsH(y_Pc=023H$|AT +zUlFq+Yi44qKSxxs?h5T7aL#aq3;M!jl0#dL&0^0{TI{O!k>N(VRL<%o%vH2g8{l-c +z($*MhgpnWg>}4U4Ls4CVdVJXai}z}4qv0z6A*6}}hQ2yXZ0&6$?mvqdUGYp4$LPh8 +zR)sTTRAv8kd?AX`WJL*h_{#)do!@LjJ%jeYcsOP30(Bp|d51fwUNj*TQ1|t9kBP-T +z&QP$Vz(IU~uvF+VnRjOk-)jK=_juLaqnB%1fJD!C#XC{dqQG?$sD7zGlhWuKr3tQy +zAUCcUb5xa42;MXo-&dERVx5?_QPGlq;Bjm2-9sjvTM)v#nDoXhBSIJ0+=ko +z+Csz;P`?J=GMv&4jd%mFaX8K&D0D{)-EbDP4f{h3@ +zNF^1Y558xG@Hju#Yd>qZOF4Mowj2IM+axmbF=%tp+uC*0yURSdU|IS$CqL!7@u!FA +zVx)vzNWn=Yj6(2cGNj*)w6*zh*Zc0UDO-F1M=7zZ1j$B(aL~)IOO7#aqYt;xncjL% +za>H$rkveK^Q(*pUtRGc4v2!`y!UsfN^2+NWryR%Ps7)z#w+aol{gi1cM=0G2>IaNo +zR8G|v{zgg7nu8-9|L5KCAlT@Zr2!SE=BJweX_=V#Fni4&^QqLqc!nQ!>PhtXYr!F$ +z{%1nvCApWATEhWCK7LVsox-xBoXN2hn&S)ey}K(^cJ?Y&E&?;PV=#HA!Wtc5k)qGE +z+q-6ud&qoNHz?>~dE(E9B1>f^oZ2|vY0e)aKZUjn^pzd!o2^e;^P}7J#mGoAldf)V +z;;moTQn2F2nWq;d3iycsiu#k0#(u+CI7c2H=Zran$Fm|B4jEaTj0CXmPO!RYQ%{-fA}(%5K$GaRypYGv +zy22OWj%4U_GKgzy!B_rbs~+%9td@0)FF!9%DAdFI*I?5GA|iB)Li}} +z%k)8caUP+ +zz1+$bIo*$oMR`AWoe6SBq-Ig9hepH?N`|$w86bRYulzi(XlVlsRX1K#gN7{zl_kdyepcGVC_F +zW5-c@xCBno9)Xqsq&)jFa-^3N6Libmm-5sxv2x^c%KtUquzSu;)ijh$zFDkhMUWP5 +zdnER7YT$;Vbu50zg&f)VS3Vb1{_Ut9h~;&M)HU9W74uCeimpTYweNRwF1{ln +zrl5v4SJ?&(8u)a4vU?w?`I!c_(Zg3J%MalojowqafwiegtU7-JAHrL67HJje-P%OO@*8LV +z7TVJC*xXTVR}D_VW8_aCYY<())MFCB3>RklRBnTpYk1VZ4m;{Bal!W-Pi)EKN1MCx +zB!^Lg69(VD<)bsFot~nTrP;9aNadHgl-G0S6V5hR_M@w7aO(@bhQge>N2RIpAOD7z +zIn$F5VV9#;UqrmcZD{FG1X{075GWt*TAO%9^SLW+^I!DV`?!q0<$fdUk!~I$oy&H( +z$S4_y&-?o$9(Y{gB~GN~;$*8@XE~>-5tNT|?{O3NDH2-All?Iwe!_Xlct`Dd?d03_CeK8JvD(A?pj*U;Rxy| +z6s=&lfrkPQU(R}xBD8JwE`ovU=fDAT+=bL#P@>OA+VE3#p7{LZmPgrU3L9$vi-2h`vQt5F6Ab$#mewB3e*hWPKkgMX4dDZdTT +zz_;7rM>a;i@s;k~bP;hdp~1zGKnunh;??Tb6R4!UIWX)>3uJ!k8ma?w;iwAv44Ny9 +zikjlOKFR1Z2_MWXiGjR8_W98*vUt}^%7);w-&3nrjCJS_cz$t?tcQ*Vd;-IBDvF}oK2^E(r=S|A{Kj3(458DfcrAkO@1~@ +zCQ!2@thlkEG`}_{NGyAMl1FW!+zFD1JQo;liGK+HS}KcXq0Sx4`5B9Wi;zdVepAkitUC +ze)#Y8?U@wNS9*Wy?2!QA;kCA$FAyj@xJ!T`QvwaR#f-fSwa|;3>X{44XaO%Q!8)SF +zwRUvlhaGr-RW>Wr5mQ*oELhG{x3P05nrq+?>hxrp!zr+ER~na~GK%0|>oh9ex++EUK$pu%3vzuq^x|B#(!TGBb%UXh?0z2$nS+ +zU|Dl}BDh>UeasWe-))MDK!9rJi}&!o`RN4 +zzY@{6e;sXV_Mq7^w|W9p@JPtd@u^$Brp?g&%3m2jEf58-`Ssp1t4>SC^T%7+c?X?v +zm$cm)f4$5$*$^Dshsx{GJ>7IO)ZfO)3EOi89UtW6;~VHJ9(D}(>Dd2#(`X8F +zJ*G0`PDj_fp4M`*c_&Cwk^8*KdO!)qxZ(tInmAD +z5sL_mG%#*xdd$225jax?H-n<9={Mn0M)c`hKYsCn9eFXcT<@P2^;dnM%INWV^59eK +zJs4dcd?Nk>3EZ!_#34n8ZDsZtgQ}+xA@ZIV$UanR_!~JWn%-tVVVI^b%3QImuiGVd +zTNvPICKmD@+-~U1CsuWz`bJPK)vY{!?z^bve8kTcy-pE=b?Z| +zF!TBLEJ}AF-0NgwEdQ3pmO1$1tAw-(8$@93O-8AGYKN$R4_J>sk#nJ_-BRP>v)OG7 +zvW#UxIY}Oq@I{talaiu{*!r!RxpVg5Z`}A0Z>mesxlVn!o3xHJ@QOM@0Ff>xz=-Mh +zt!q`x$CzUoj#UJz-X^29c$=(2zJXr{b2TDnjsY04m(Z=PbUS{<%eIrVC&7Am?BQ?I +zm>r#bx8rw$=9GgrqE&6(`7sDrPODv_u+m@DozBQD#)D?1lVN+vG&gwE3!{q`VLWR3 +zCufO=afVP6C5Y<=b7k}VhO}rBCei!{tZwX*WdPA1=nai=qG{K~`97(g73vLT*)S0I +zX4%kuu1$w80hgXhGYxYxXoGY?pg;;vF>i%hoQ{csXLf~Ez8#PDr{U~ZK{TaNXZ?6o!ok@H-C +z#m(TwrC=U)SH}K9n^Z~j#A~h#d;OM1Q@sU)rtG-Jv_^z-AH*{Cd}g}o#j@W=zC%Pr +z!EsL^p2K3o4n-&$9pBc^E!ZaH2oRmO2<6BTDQI~Bm!Pwesu0hRZewZpGhpg~Vw9Ki +zblNgT4H!1ydwOUAkW9PNqiD#)+}!Z?Q}Emze@RUwqwsTxtCJT)&i2*fqYxe|~uDI}EOK2TT1#Cs*J|4NYOqHCBp1 +z-ca~m$(5c?{3H5o+9h5vN6hAwA`h0c*pnS}^q9i~Q2k#{x%S=eP2VKa!k_>NMpFj1aMLK6`+M +z5%*HKE1s`Vjbt++`uj(jI8#@imuaXv?+-Sw28PD_Fv{|a$u~|Z#qrf{A)k!;vKi~za|(99ItFYhJW-2Ry%jrdzj)1KUKvQuvWM_w+i)_!m&2%GYjASqwgKT`0Yw1$yuxQIgUn>A{ly1DZU9Of3AL98Oh!XWEuHKN0OEExW6lx6t{>Ecl{A- +zFvG(I*23S@E9c1j&F{f6bP^R6I58Ybah_rYk1k$^^6R)rU?m&JR43NFiJfL3&{qnw +zcaW8EMWJs5zD5loiK~pw +zT}(FW>Voy7i_EKp9C<6@K3G#vDMm{pwH_ewAIR!+7`7o~=oUYa9plCrsy{5_%yNwoskZ{cou;)}--Rvs^`2M&}}(C4sBvOi@jm)?OJ2{`c1I7qFwd^_=Og)W2(>> +zbaYHD=Z7<}i5aQ+PS5AF76c|5-}I#cdBS8O7sEUs=*!s4fQ<9k9d)D^+CHk$kH5Os +z22tohZDgK1b#9p4g0IN2t9mA;os!8stTWxA=+Zq{NqvYOFv8f7jGRTF={^xyOj|+3 +zU0@8466((p3L5FNwmp>`64BDG369f*Bmi)9(t1K5#*&<6GZtKQH){OJa*7%CfhwP{ +zBdyM@t^?)v=rW*&MEHA0rCh0$g_Am(ODka(k_rvV#@7_|@S%W3x&v;?k48u%1VT)# +z=j1-AZ<)&mQR>DJ2^VXx{JlfN!{DK1CXbRx_>dIBZFt<7Es`y)E#P*xSCd@3)NTI< +zfE)9sFXRBkkHfkX6N+lqh@&dpDSo2CHA$KN39kv< +z2jj*F;!hZybTVYLHU5(J36h)?s9c+)3Ze3Ka@43<#OgaxIS$7JalsNJ%L+cOBj}{-E!8GGJ^mba23Le+y5@2(HN24;+nqct&tX}>tD +zsMQMHAc-D8pe@Rl2_D7;Mp>L%CK;W)?c>ZG4@(b1xkf2@N+6uc(b*MGMw)luJ=-Kf +zyvdB*ePXpQ`4%+{s(-z_RdxD9%?P?tEpjH?r;pF+n^yYmFJ)czRRk3W +zThicWQDEl>HVos+&UP1&7`$;%wTGi5M?i$*j2W#aqp1Y#$2?zTuEg+yE>!#{S2xkw +z8NP|x>D{ZR{@vPl03rm_g>e(hU7~j}aDP`8R)|O8Bj82bLufT1`hlMj{ooP7Z#`I# +zYJVV_e6Q?Cr>b&rCQ}w1Z-OyR9}Sj6^!dsMyh7LLR+rnQhIcnENUyOWkm_j7pjuBL +z^KODW5EBO@uJ~t}^16=t66HVywhn9}4VWpgeASdvKc~vjKtnm6juZ0nlKyT+2b30v^168BTDcx=s*X6 +zMAL`H*CD+0hGtI8f6hwQONTJ7D5LL)N(<=&By9vAN2w1xP@p};S{&N!^jB$2JXQ0& +zYSnGJYYzbze{EhyycYyAR(`!Ys!h;o6lV^7Jvzv|h&)E(lFHdQvCae6l +z^t;ek7?2n2Tv=y^-Lx2;jH$%9gy_J>gTzKd-;m08LZnTFk)+XZJ|fdJ0D?Uzq6I*5*P6U#~rG?)|72%f>!?sJOgZ`*FzjNBO~BlJX`tSd)#FOK8$fxj-owG9R%qJEis4S +zw>RY<-bexep;-BSXG5of8Fu*;HI!uCC{;4ygw1}zOMytPm#Yf`DUxo68j#p*ZsNp2`f-BQ6t1o@cj&* +zdI6(?9t!cyl3>@S(MMAV6k+{K7?KiMU_`#+<5zV^XN1fLh@-s0WR4@1&Y+pu?H%fU +z=@hNia;DL`zb+m5?(`%-*c4eBp^mg1&XMz<=I30wP$6oX>Ehb~0A~E}5t3k%!1E)_ +z0_Grg&=dKsUCOCqf`{E;SCiqQkQNb8-$MrisvOKon8uSl*FAS~c0F&ldn~p+_*y|? +z17$`>#0jv|gczB!qWW^Id2p$*oAC^rk=V!Z^}C?%Z>E2F#DxcEEB~1b!EZHxM32Y6 +zB1J^3!<~Xr){bez;Dfya{^2DoeTj$d7;xK10F8`;|AfzBt2Eb8u68+2T_8@G`XGOg +zG(MF_|CJ_Us6|G)Ul(Gl+K{t#x?%%opl7cdoX?9#8dobq)SvJ>@dzZ$bRrf`540d9 +z-5-)~za=&4k-u3>L!L}XHM +zl94c^rQ +zG<3DF+0O#o>W>T0l7Z{Pl;(vkR4~y1h_1FaUQ-qaO<_gbt0!KBD=2&w0M&q-3Plx{ +zhU=5^JpQF996$h|S+=^DiHMvQ@>8IFPqAS-8UkbfF?x0C5FhbOB2U6ebZ6Uzx2iw_ +zL;5buJ`eE$k3_X!5W!(0k@&rcFrxZnokXB9Wo{Nsk3^{@B+-FG>^4{`2nF{9truBv +zkZmZ}<%&az!UKRPmWg+WPMYw)GPWY!B?7i@sYif`aDHKJTFuldGCEXDm;T#l*gFrP +z#64m^`WzrWg%ZAvzO3#LzI}lIb-@v~ETt+{gyDaBCcdT=J&gzp@bJMm1%O0|Af_}m^ukl5kPe)WW4_k;UXWR% +zkCX_g8v|L8gRzl~XZ&24m4K>RK;Rv=5;p+nr0~Z7uRmb$#~mho*+!0>m#Sr5fFK(h +zNfYr{wr`>vH_$S-)t9>Xab2xkrVdCuF2F(EM|zbtGh4-mvlSu5DjFlWET-v8E#;cF +z!)=C<>yW8W6yd<|)e-S8y8v +z-Vq6ZnVzh(kz}Z~O_6FVD;Y}}D31V4<#KE@uvLG$Q6 +zEb*^+7Isks4`N#tZS`N&_WnxSw0 +z(gFQYm|^&3bmUu+5D!V-u$^zQ&v3^@~w= +zWILxbZrso27Xe<^l0bjCuq;M=cGDoZ8n`WLBJ}s@@QA*6flaMWr6H7HFRD2HK75qy +zuDZJ9MRJ1Cpu|b1bHPA%#4|5ACo>!OY&!>P2AhNZi+vs&8*7GeeDlue%K)Y4+OTdV +z`;X{|4hZCNu2;$EdL&qJdn?a^V|4VUg9`(3cdmiexRHhV?8dWjfZX~DDPNF+;wobm +z^-7~{-~Q&-FN$F~oB{amHH46VAOPCwgH~E(z{la<$)*k!{cNc_jL_r*A?4zb#1tL! +zAkTGtMC7qs-!s|Gilj)vo}q7GCE~c(zEg|<6eq-w$kxFHa1s9BOEc7X*(F(K&h5Fv +zcXovaRmHH{B=6Q3sLFytrhf^w(ccrZp6GlG=i$eJZ`X~f-A_iDQEi-KVM)c71q|9| +zNbBFYG)X3{5u4^|(UEVdqf@+>N6^9s1>#aD7a^gQ+d1p~P_UvdC+QV}o1e(G!ro56 +z_BpW~-dElE3zHLs%2Glm5|BG=-|Tq5mL#L7a3p9ZEW_aoMB1VP;YQ-PdlIX~gmAVU +zA166S_D!o<7;aH7Bz%H@1!4SHVnT?My15j91O&gAw385Lb-76pK>No1*Q5gXYxNMh +zg+u@-ppNc6#mE=uAIGi%TVm%910x+Q{CzA20r}zFG}$*Osv_B{simH=2@2=Pnp#lU9BgI6k88V0^(`^Tb`001*^FvTG~*4*rC@?ux{) +zv4eGA`_)M#UrJ|~nCh*iQTF0q{34}Ohs&05k2<1k#>UVXUYz$-N?OVXO!7Z5?N?Z +zU@|@n4m064(O&c4z-; +zquEX~@Ew9L*tk`qfKnD;(Avs3N-0MqFMPCY3oF#h`vgFlV+8Z}KxG&E!tYEG&-cM& +z#C~RK^0%8{Z68j-Mk>!fb#_`lkCK^*SBO!*z)nf*;g`477a^NJ+^sCk^cvLa8a$%n +z!x>nQH=|i~4OeNEt3Ol{-=+q2$CJZ2J3K>|;W?V#7Fra)_D>zB!e^DDrS}{U(vgBB +zB*N!SYV&(5ebJ4l;xu*3%U79QEaYP1@}`Gh*>`6`W|scMAGCS?-S`$;mVFsfq>Z=D +z)X`=S$3#BFGGdsKx_a%axgt44|4QMSrSTGm!>x{^{1toLLvEEt>Koh&o>-?CzXU3K +zDaZYB;l`mqi{12-|L_Y(M95hw1yeFhDoZPedRS$6o%+f*_~yc?5oOMKnS+#wCA3qw +z;)e66s_>0ELn=;wfAqtm!#ovVQSw!rQGLpa*UQFuf*DSJELUNRQJd|ujgPkx76nC% +zYN;RVbAwO9pI%011yhwP#lp5n!^W?ToYI|-s|LS*AikxITm417`S+>(N;4hMS94-r +z60p+LFcA8{Sv97WwsA9WAGZnWuU$Kf)FIqICgw+znc^G4lbx+O>tJj9)*7IZQ~W4D +zZ!q%XapB-`d!48BFiV!wXpSwZ0J +z7ZT0>Qf3PIQN{)WV@;~QmL-7lvPpfdX}S(Am7%ZByFdgm?i*hnLG@&nBI2^oPBlQL +z@KXdB_uryl35;Pl#;C}9tz-Q@S@A}bRy<44;RFApQ*i9Taa)UlY?uIIZ-`o!Myccec;ouK5)v+$yJ{7B>hqSr+ +z<%P-$0V?rd2Kgeg+NCO$3Hzt&p~_!ipln8(Wjut3>&_9zo4iF)?7AEt$&{>kFwqoZ +z8CRkSy?bTIuf`FiJ@bBjEk3J#;-&FP=Hf(u%=^NjcmDc0%A;huwa+idnn0>u9$l4U +z=NaU2y8kkCJ$!t-m6=l3M0J1eAfG3&v?%qPP9vkbFM|G{FG-S3%>h`YvP9Hx(OKf# +z@AU6mUO}Rc4TS5`m>bR~NmePIyKBShIKhl={tn(n+W$Li6H*Qdst$oJN=(eM1gsQqn*2YMZzaBLNFD_NKHKRmFG>4tm +z8X;Qj(uhM}-vy*T<^L)UVd>0<%2TQF&-8QG_ecGZnx|tl+5vD7uu?y6P0gAp3Q_B0`IS9~MMK1E +zz5@kvPmm}lwB^fbb9)k+poxS+@cOAlC_sNxIR!AkE$)DQsKZ4`HStJ)jm~!~o?PRJ +z$=i6V>mys?p7cP*ySmnkc$2n^7)rgebz27TqkeT$Pk2tB+sXiad^IvahQ_w5l&mhA_WD +zSSRO1HcvxD?low+Owo9k0gI-{H~@4xT8fa_q~FIy&OiHiFwbaK22Xz~obpuoK2=Zg +zdh+_~*neKJ7kpw`kc@sSe&p-_XP+E)Dty{nvxTt>{;3tkBUQhJzxupg(-ueJuw9gQ +z7i0;eng4g3-P~pHL-Vmo1xEJ%6+RfB;eS!lUHb_d^l5f=XT#uFM9w6rl_9K@a5pUw8wzf=^!1<2r$3O=?vqR0R9 +zO8zstTP0C})VT183%0NAXdRAkCYhe7tyhS4qy||R>3n+wU=&KhH8GW27z_U2efaOg +zU*K*J5%GIRbE>XTcuBL_UOkQ+9}y6dKk+2<Vm+DKYWlZKm3^$VI})^vzZ4x*Q;3cT)1K6ngK&=WiGc^^HHjU_zAS!reDS3 +z!ryPgh{Jig?nc%Z_`Z;p24f>)`-1cX)=2;7O~B~-qGCb-K%Y3pXO;JFRk*N&O#2Q? +zO3WgDlz^kHwAJ0DPznx(?>uO_m1gQx6bmoOXkHWHeQu*)@x`$@Q3tm%P0^8|g9T(I +zVTX&$;{VjJU&I-?)_yjPeMU37YcV!4J~MkqUy<-^pr|?Qjf*pb8~^YA`_JPP{h#Qx +zQQFAy)(u02C$FGAM?45{UnRlCWVPiOB>jaY +z3#&cAoKE-82ZCs~zVYyh<2z8}<5daRNDJEh{Duyoxnb^b_^PTUnLZS1vfO1IJR_^& +zIZUSv1WF&!vm23EYszQet+JjwUu$}O$H)ND_)?~#n3NURsVQ*;__?6iQ&LQJ4>KH} +zWKHeg^LWF!1nbgy5$F|^esA4AXP24|EiUW?`ZmV_Qpmk}BoVgb=5&>S=bd4y=^u91v>4?08NZe{x8gJO4Oow40ajb? +zBzmc8R`zSSR#p@ED;-OLLInH(c|QU3Q)|CFLf181{+^Jq{MPIKwIuzW?($Nq<|AB$ +zX!425+uw+7agiM1HCm@(z>t}6JPkriJj(uth?J-B%Uvl|j?ch8QbcM!v?PL^aUB-A +zU`PjHpdAmeh$*R<*g4kqiWip#74FeFQu$6@5w6A@zp0 +zcgP#EoV47KUY96HXy6%K*%uZdn29=M6z%)7ORb8%GiktMFHZIHTF%#!TW6GpMN@QW|BB(_c+zS=oqu6M_nnt2X)9WuGS880ph +zPzVJ;RaX~QM(-&n*UuU3?Won%Z5y}34>5l)C*7>9BiSEksh+N-B;3L!--c~tW_X=N +z%r#wZqH~oa6))y7g}B7=axZk3W4BY!|ET{&8W1WyWGe6e9lD$m=#rnIr9ChGole$EoRoLNv% +z&n~p~G~akz($TCKiav8*_ZOe2bqAE4C|E-atuDLu))Rq*G1&tI40(9>~M(Q +z|Ffri@mH$2CqP2ss>~T)611$4`@j)ZhoRc!l~AgS%xp(9B?7705xW@R(ZC)JF{cY~ +zPIE<1_PRrT{}1seBP+8gcl`~F7{o!pf**l)Z@9cbY#aJ1uGPc!-E(j9%+lPu%te&C +zh7Ngv!_mo#2~13i2|i(+`Lm!X&zk9aeTA;8qhGM|P073IOs6l{s`eYI7dif|{BTus +zK?oH`M5`G+S!Utyb_855-)@_zf&;7g>7xT!1v2_L1VPB_B1Q&$sxEKI&tCO;@??@3 +zM!f0!8r=TR%pja9Le3JG2@_oZFR65JuL+*C>MZ#azz5&qR3nBOw|u2ld?q-iC@PIZ +zMHU-mX3H^4*&`v85B= +zU{^y30N<|bL~TUckdbEPZziMPuPpOG1}(KyipLTbwfnbPGdVbNRAE*wUVbJcZyTnW +z@t-pmz+~i|%07>>PwjDd!*6-bGxoQxlDr?kFySdqi6x(h^%H*3>|GM?-tH#?>OknN +zJx(KFTr?LdRhV>3?aJ6m^NH*Z0oTUQW-?tC5|m>RucP_F&> +z#1IiUF{!-nnkT@9IO&fFSYOBUdcyti=3Y$9>x263j+OI+SjFcP6)xy^g7BPxyR;i@ +zB`eEY6|tHNDtQfc%T>@ZRaT@CJkI7^g99$o)GX-*l9@)TbYowA1D8Omko+{&9Q?h?aQvHWa6`4Z#T!0G@=sa|}wPjv`=@!#Ua +z7Q|bOH;h;}deG+IQGlFD?wHQFN;j@G)E7VGnceQes?RO_tqN2|F<)d(V6cE-*d{6X +zbT*~jx}ZOdw{kH(1#m4TnpZ<`jTtj>(#<@tDyUBo9uFNH8uF_|f2FQUzQBn6y@eR+ +zn6cQJaT6C;Qf8SVxlV9QNj0^$C;FENAR+OxC|pz%>BYfXj_vvYThgeoxc|T7E_Xc- +z7pJ2lUSf$g2SdoACaN+}+X=8YKl(y|6}e=waflseCL1Bn9~s6qwL6;vcN5=d*XW +zrKUCCo{Kok>pnI4*QK=Lb1tmGn3e5gU{Zm|$Mw^J>Hl0v6dLG9%Mi&I%+c +zm}2lSdg7^9cEf{aXq${Gz3{l5=UjUYwb07JrjLW%aY!SfpaF~ +zRGrPxr5*!u4pcB0$=2!+fWT_VpThKei|DR7RZvPk%BB3(#!pUbd%tA(ZYELPh5X|z +zx)1cW#Vuf|GTX(v0ADcpfznT^&c#&%S80(@LCsc=lQKsnSHqsCh_q;7082>3k7H1} +zCun28-{o8nZYs-qA2&SWE~be5{!z^BHa1GV`(cMesN*R{w>2C%DCKaPGX+Za=I;w+ +zu@f!^D>EtOsk^wkdcRw29ZXT+k6?ngjcCu4hY1J@%@xt#nO@KQ>Dv|%D?o`J-MS99 +zHg-%=;>=N%vWXK +z!B4wU65v7@kyPP3Z&vs1sRsFCkbTP!yN{>V +zv?~5{pa1I-UxZ3Vro$58$~8dJyrg8b!;oTKs^kc9R4UI)H{qstWcOUGSW$r +zIfUp9lkq==&%QiwGU`_j(!NDZTl;)MHQhDzwkU{>_L?t_0JZ@p|0Z+pzl03dx&g+} +z61CLzssv1=y<5XAU3;VhyzJEvb};*-C;qI6=Wa&t&dl+?PyfatB(Njo+;Ns!-PS<2 +z9nojINN!gcD%C9A +z_8#wH-OTD*0~BPJzA^5O7sE>N6ikwROedBIA^cz5=&esY>VZal$_XQ@8~S?x=(GMa +zLKe?efj~H|9vV=cd#sAfBZ$`nqjdbosUih(j{S1pw?LJq5y~nDRoc*Nazqi(<1G&@ +z{Lmacc=Qj^#GeKE{T8jin^jejm)!^8dn#jR%7N1sA~ddaW7zV1e{5+ST_6 +zk_f&W5##VL;zK65;+l(n^WMc|PJL>U{jl;!B!aY#(pIWC&U)=kgHs2l*Q*ISfDQLR +znq+T}G=D*XHEb&l>rO+37GbKF2t9)Oj#>*Rea@i<2=hcMh=cN19zJ|R%}A(&nP=7y-D@1Ps5eZ-Pf%=2@Yci38}8B5+$I&R9A +zy}d~i=}RFotU%PA*Y;}Xh`p<6Eg_aV)Cz=yVkquNz3~Ui#0YkHTA4lP;J(z}=9beO97XlsRj&co{`XFw +zTe8FLn%UrU2+16xFKcb@_uJXpIC3;|g3tO*TyW&W%I_RSMPT|`0T>T1UKjVbGkniA +zO7{QUAz_-f#Z6lzpCxff7j;)_7xq$u1JV3@An*cno!;&ZDY665gWMr1r(FwrVp|x% +z-X?i-6=mC0;6LBfuJ^*(o_$^6DE|?r8bF+Dnlhv65VNa<)BIEQ|AQxtel|;9(e%E= +zF)5vH1MwmTM}xS!4!@qyp1Wm-S$A--EK3YryMC^G9X;^ydFglaZokp=D!DLtU3FNu +zxF;iNLnG1i_T4`R$`)bWK^(m+MgPC>hmpF`mEn4h#vY$aEc}KOeiS)&wNu+-x3!=l +zk$CFvF%yTaN%LoAVQ~i&!>UqsvC8=p9R7J4PEPcDuSJR#Hs|#K9VU*D}s +zxa1tmlZ>s?p`6_^R}D|ij-}W8%aFC6(Rh*@9_o6TEfN?Em|}8QmT**)`yT*xIY;A;Q=()_rQ!1L;2c<6mzTRj?-HFy=m17PGja +z&Oe`g$+|13PCCfyW~ziBuhE#&JFq$msLUDNVK72cI@oaJ2)&dy$a0TQ>F)=ZD^}daH%vba_?o+AsYLM1hWOFf)`GbP0 +zQ@NLUc0@|n=C;8&@rM=S|M9WF!@29fqXOE=g~gwwH19h!C!c7Y$u|ZYedp`=LR|K) +zEqZe+FNXSV(mcx!W>1=`)@&d;G`hX*X%IMMm`vP%eYHXnb;ha+&zS@T2CiCQA#H^V +zZ6hMKsYFk!+2219DV)yT{1+6P2S{2sY9=N-rAF2#cpok(U+U-e5#i7tnB&>~d;{<( +z4U1RH_E@Sg-}u$n6f^9~jPILH&ehH7;w8YYtwqn|@;AL{G5xJsO*iA#eNh|DLtjbU +zFvm{@fgGUb0torL0<__`M2=(2_6-a81_zC6JV?2eg~iD^I}RA2t*3B|6^{p>wnZ0@ +zmueIm!TqAKI{V|xGF1xq^0|Njrv9a4vp%iAfagoFE9iZboe2Zq*{k^PZ5ac38ZHQ1 +z4Ulgn)H?;{V=(3>yGT=|EOJ+sbmB~fgF8a0i3pG(O01sf>6FjH5lSfz7Wtf+7rAEa +z2a^CZgZz!Vhj%`Cy0|Ew*ghFK?0Reeu=hH#(Edgs@ +zdaL)EFzsDGXNIJUXgbbTBohC+xBq{qN?gIXzb#3w`6oB+sSVB+T2aUcKg}&0FGzzY +zcCukn`CUk0YD>FD&noMW_AQ5xprWFnDppomoDDTujiDMg3Lq&aGx~e~@l`jZG!Y)s +zZCBGQ+ZnNE!LNTddWNGgREHnOkEOr-b!a-`-geo_(!%)5`inRIEedDc#WzB +z4G^Q@bi@|S3&e-+n;5X`c>zS2CLHpF0W9)pK>Cvr950sVE%;pb?`?oM>a7dz`&+G~ +z<$+659efD1_)JykIRUKYHAZN=;~abxKHMF;M3XPvb$vcbd1t)&*4Pa$8qv-g;*UcM +zZGdZJWn=DnqWngT!=M7zk!IS66C%7F77key&L}T1SIp}okNJPJz4cdHTN^E0oZ=KH +zZlzdpcP~(iLyHs*PH}g4FH#&rp@mYsxCRYQu;Olk;sgkun{&?l-TN1OW8{a7WQ;xb +zUi;BG=X#c5f`nv^x}NUuCOIP2`81NQ)0v5f-8na$$=|3qFK*DI@ngT4*!8 +z3H-5CTC{bJOEKd1!{~`{;O_x>@Ky7Hkqm-xXftVM>00b@Bjmx%E(jhe&F(Sux+s79 +z>%xs}(J035 +zHb1OU^F4?C))W8ZMh!y0E%3Vj{qfxG8E(a_Gy5`fthy70v)}VgP#}OQ0H3F=<63q( +zV?m1`L#1;v{?OvTTK>!?l)cOG`4@Q83)J^V8ty;*?T#fv6quGltQA+u?}^M<7M!Uc +z@zO6gt7M9=P9Ip~zXy~TPcy$ZZDJ8EcqsJ71z@%6za#)GT4;gawxz-9zTW?Y_dupD +z211J3A53eU1KyJ<14%m7lD6C +z2rIc;Opb{pYnkP5ojlydH6$wOXkLYr-n)MhOHsVWeQIu^C54;u5zQZ`OqL89HTZaH +zV3lPx)CI_{X+;`85ph*HO(S8eV_M+!-WsFg)}If0d*2sZ6~#MSUW9!ctA-{7=cNCc +z6Pik|AQ$8pR@@aCV_qlw7Y4X_zxntaoAK_C8G`9j&Rxc +zdD?*N#Og{w(<|*WOPDwcnl<#V2}s41+O{Y&6DqxzJgo%KEFf)Jq<86Suucimywi*n&)cf5JHx`WgIa~4xb +zQZ~#^i1`pHHA_sdd84R){P^Lk9TB7Kt$24P4t_bmkW4RIBLzhLlnIWcQX_mJ7_S>63F$k^vpc +z->ogC__W3d?rQz7*xX8Q|5sItd=T|}+NdV7IQbG4$pAX^TT7!r4tYW#vWE~{elrSc +zcO{;KNmaPs?xlnFSIda6dch|kt)KoGeeDO;W#Pe?k)mI=e0*_9E=7ZS?Xy}oZM#*0 +z?xsG0ly7Q27XkOg)cPJo?DrmO%73RpxvxdVq-Z~EVb5sZQtkH~fQnNWPp37rz}c~b +z`_&cEyjkw?0)FH{f5ZRCh950(5AV<{W6(75wsMrb3Vz6tQ3JBfcxH`;W8iBlYCVWQ +z(1L(ZM!*aU-4fhS*^nT86nrOJU><7d{B4#jRt{tmRGH6}~(FY^XH%Y!C8i +zT>3}&3Z{>r=?;OX>r#HqsLI!BclX6#qN|_p>_s4{4~-wF?20sL7J@g5+MNlTqhh*} +zXLfq+e|eVqX@HN1Jv!UBJbEq!x{vANONTb`4Pji?J#p9gZ_{@!qY^6BU1CItm_yh! +zkf5eFL5iKZX9hIco7*QlaP#N0;&9qbCP|rmB&G`tdQzvN%PXFK@2K1(nG8)3CLAB* +z9%`&Rw|`UPX)=0f0{Ev-iKro61BO1sjXa$OKHZw>DyY&wiiDIU#(CgAKKUThy6!te +zd=wgz75wP)l<{-}l$G88`slO-ATa&84JW_3qd&i*ccqgl +zGP}b9sgBb|c$W2!^5k*cDd$hjKths(kDEapfM6Ih@CGr>7~;k}rBXhf-d_%eT%+CH +zgKn`r&mSZ%p&?hN2S_>gtGBY>LkTfE(3=T)_6ZR?JO=|J!P?WeN$ZbvS9lsTrw^OJ +zspgPPkrrtcV%g*gvq>KiB1=$QNHgy5%e$&|M4{uN@7*m#l@j^vpkmePk{NQL%>Q+# +zr`X4_5b$d%jvLo&{azS`zI*mw@HL|J3Wxyxw7i!1phs6MK_>$tmY)@daWve)5IkHV +z_bxx*2&qiGkU^f@uJqaXF6h(4h0M&IZHTG|79uimnC`bnN4qc&hW|^u+rKiCcUcID +zBo_o(1|NUcN0;ier{v>xDu +z4>36Nd)N5-x4TA}@8?yZqAH%JXu7~_7q;0(MFj1<$SO!7&+Lh7{9!BP$1M+nq!FLS +z!%xZn?-tfh@}=9Fsk`Xs^24dqsTC1~KvnRqh+sPm^7GZIidnP1)@uPxOL9c!Fa6*= +zI;s1VhTFw;t@$mVp}op)@oD`C$XbBUpqG_VA<%s8@1IB0k1C$e_*2(<0thFcT^1H$ +zbfR^#?!P@{AJNWW(~Nt?E@kJ0#&-dU7wvrdx?(dmVt{$u&9gV6ND@ +zN5?AW2OZm(?*m%}WulkU@|2)W_|Ep4amq%2vDO-?SBaHHvI$ve8-htmwoLuKv3#>N>Aw%WU2{${hCdc6Fwi?JlzqvRb4+ +zG^ZR;<-2jP$*Y56e^e~$0-J>l~+jL|lcnWh+qbb7w*F{qpU#o*XqgXLRix2R-Hg7%aI6Y1XwMlCaKHJ7BD +zF!6_AW224FQy(dbUAp_;CgECio29x6Til;mT0DeAE2IGVsUIp=8wMXj_NwySPu%Js +zw*HQ5x*e61r;^1|^F_SsUOo+sw$ubA1}PVUTRBd*N0KztP{YYM#=sI6Ix8(+HOBYs +z+#Qc$k>|f7L8_*a<4S?P*BCgjqTDs#Zi&Y`oCe45Q6^{^2*14u7~R<{Pp`?7)ZT9i +zd_+sk5$y97`x#oq7GKOJ6UBz<21LER3TPqa2iSD<4X9@~o=OOEx4*_bg`O$>pM3}W;oy3z?Xr}=Qyl@r*AI2dRxO4Mz7&CJ=5EW +z^8?Y6x&UvJ>9a;asKLC4Adj&l^Q#v +zU03C2HG^>+3h%H+sP(pqU7jnQwy;2}`}9RAueO^3Q$5Sr`Y8(DnS-$+`J~^V2WNb` +zwO;db4V=cwm0ASx4$bj$o!;R?0mLY3T`9PDYcHLv`lv2hohT +ztr~<M;H!ghfgT8 +z1e6YVXRs`nydQ)sgWXhD0N>po8Ab+99gOcQ@_4QHEVj#Krx+T*TGino69_Dk}cRy5WQ(*9qC3L^mouau$iUv8fasRMAmbAt~nD!edKt +zp`SpEO;1cy}&Dm1Z$Nu%2RF7kTlpKU%DH@n{KHirf*7WWwvQDF5iz{apYO +zaRL*j>jG*wHk!}+Ii_To!q=_ziu}gO)RwP_pT4^N859BC()$!cP;gEL5V7P)y1GhR!ke;^NO2yoNI8}&t*sbA(RwKPd~k~eQs2( +zz%j+^MGUVZY%QKrT~a^Jh6bnlb1pv$y(v2x41rlOSZ*-9eFL>TaLbevxcL03?(Rm0 +z=qC*i+-6H8?33EsJ>y?ny6#X};yp=k65aX0P`e(rO9;0jDE~cP>`qQ6)6H|0cm`79 +zmgiSzeCbn7uxEQbMhSnMQoebuDz(=1`-V927q0o +zfz6vxcDXC*jKZr~p^9*g7 +zo&Q?0lpOixlf?&j`e!vAklQ!~U`GiP21)I^#3%D%Ji&39xVNE-XhNz +zBe_0+r)2mSN;ctpsvQHqqFoRjB5lALkD3oUnT5UgvBXU8S=^FhcF0bE)k{1@h7eRe +zx;vEA=$w6My5xu`2Zc*-c!^nw@BSOo4^_P@ed|mTAi3#+r1cwB-k@c5!3k*=OFDWS +zEXZK56Mf0m8KQeu3i!?Z?xiHv8-8HI(NDtmjIhGuK4s1RCKIe$_i>wkm{@r~kLE&9 +z<@DiMwJ$0g=bOROrsJQL)gs|x^MepQolxu?ge~M-C@-2=Yt%xEeS)!B^siih$AyJx +z*VkcwFmk|1#(~ZA2oj>Uei)NBikx?Qo+CQ0=YiWa07hlu_m$!c<-nQ(B%4RX0Oi}<7}#SS#aZbcAvzAWYHQq95ks=1 +z9ATs;Uyygjxhz}NXTkZIim*JkwgQiPTXf!F_k{j?vBJP>!8d=EEf@9=M12mLC=Rp6 +zF0gU1Mb;kiU@xnBnI1dYs%XLmT(o6d@aUz8z-nJxr1`)yaMOJro;7?Uf?RddX30!a +zqBI^c4u(mzO;x@J{kTma*F2V%K)=6n#|3y9n{}irre_K|HvN1%OiOs-sL%E9R}bOj +zu?bXp5Z;M-{u_qxS?&LttWU4wG?Xxok4`9$O`2*fT7RuMtams&WM3k3g!Pq{Fh&Bw +zBg#NJq4{u64KfVHjBTA81m3`7$<>wg?08?&-r)SP^cnJfo4Arl(Sea(_b>z`t}+ZA +z*^|m{zIU)fSGD4N(I1k2rpzM|gT#A!nfU2qXUBSq`$Cjz?OG_HQQ2*eFEN|*`(=4R +zzOt4eY!Bge*?~%u?2hvnRAsc?CHzWTS!MjFrg%6v){g1jljVcF4yWWd^R9f7?p0 +z3|KBcXBAy-$PlHZ$k}udzzBomdigAFMny+N7~*B0jK?INlwZFoV*)ji38Xv{EMnkW +ze8HMp*H5N2S|6y*w6rmVB8b&C%zAHiOt+^*VSbhVf9ZeH)lqhZnGEve2M(WM^7$^X +z?T}@ve%hD&oU(q%5>KE(qs=yyCOB3ATIjaMwqXzN@A1%(%@(i1pv;jTGpj|PDQz}C +z=}^3oz(vFIF)l4f8h4;;eTau9%^qKbX^d&x6}!%*8IT#Xaye0SJVm)*=fna +zj^EUc9t?v^balvd>Z}zJ=B$!r=V+rk)Z>Dky*VGX$FT6pX%{-fxhro +zTbL|M^PNDl10ZI5Eb&a^xUs4@$e8I^=!A}n^K6?1(r6aha$aZHYmO0VqJjdj&JC3f +z`-XkXe8oZ+t(h88h?W=wM(ojIjbWtL5FD@4ySP<>kGK{2!yd~^=_{G3j+5UV^8>M*d69m~PvHjakU`$hw|NFF +z30v|4bmIlF#G~Bi2Wu&t>ZDAjd>B=OPpp4!XuwRj5yX_OFL6^45;o2cc!em8{SKdM +z_s`6TxhbMh7H{p=&Izb&3o?A=DRxGCisWu7_=6>0(%yyC`L3U)vgL<9nFi~m)n7=% +z7T>zqV7hbISS)UbD1Ysrk!eP0RnWH>u9GQoeosv|%r=lcE>e3F7oxm2B>ceQBlxE5 +zRN*jPhT)S085*X)wwM*hOPR+IM(IF~yZZ~`UnER-3-rXp)bWVF36^7jJ?s{*gno0B +zzLM~Ht;7S0J;0KVVf_>p9vx*Tc1_lSp|Cl;WBOv(Zqc74!j@iMsgg!_&g|{g&g%Xt +zPk31%!Ar>}{fra`1sfZid2to~FTnG?(kK-OT09P%gF%%4b^gU)amS@Lv_JR?sk&N@ +zU4dW6H4RF!u_LS!^>igDjGCmQ$DNk7jdcLY+8>HSi_8GX_tT2o$<848xMFYIafiZf +zx@{9dpYsEQr^}xXH?uNq5a^J}=_-TXcI@E|G-L36ILJyW;;s7e=P1ne?&u@m);d3G +zZ?W~E>tm;8t<^5(gH0&pNU`?xtp3Gd447ndqYgS&h;jLut%lOscxy}~Xg4aKX!~(v +zip2fcDLr!=(&5-22Q`znMGtSJ_Ssq*PvQ01#9?B{_(&i;%G&Ze9O$+4A~H^weB;VP +zMD5e95C+ckteTM6afxf{x>p}%4M2YS?x*3qzJh&u51iWQ30W?@qrz*MEo>SzA$uR| +zRkkuHjeE6H1YB0TVK%xdhr6{Ck+A1`oCe8H5z|?`|1Vs&42-sCtYI1VPRcNxx@e+* +z>zy(PK4W?Dm1NSzoQAn?4scr^BDaaG8ZRMAEmp5S5k9<3{Y|`odd|pH_I#XrUEy#F +z1b9l7vE2Ly*VYlCMzbC}uXz$3K?b{DOo;9krggwbVG3W}anRV3>v+o^RByb0K-WZa9(bk)hF<}FN>)yT%+tcBe*h89 +z<#C>uNHlWz(sJ>M2T2>nRjN718ze^M@u*5C7n|uxQ76 +zS9~d0*=W-(a5W5LbR0kn*6yJ}xT#X)D@U*rfyY{LYuyeBDwi9`D(0M&Oywvst#2mz +z0|4z<;mh~to5gzdl98x=9T=*-ZWou=n&EJkHwLxt=xvk7X7SznZyK6NHvSME5?pu7Q$A<-MnJ`V)hzNuGhPrFxu%}-7xpy +zzD?0NOI66Z|pm|&%EF5XA?5wfAQo;rFhQD;iJEB|Ab5F0{tXY?Uv +zisve}L})u9Ax{*fbLxuD2U;pVy@7EX`d$2L%Ypbw^_oQ?m6#Ypx?k9+Ff?tL11b5F +zaquz(?^21&l_~=p606>9622uKsfd!1AT)gr3-^1|kt)7R^qI7joOGVswU~ +zH(R`&b6Db^_%VtZRYLS~m*jT&DPxa5(EnHw&M8ZYJ8L_2I-Q7+axiRQb#!#^yWPx+^Cdw>CaW+0c$ +zA_D?z*l*W%4Ofg)^Nuk#;O^zmWW?S@@LOc!Tlou&vxJ&z&SE3+Yf6j6QBhV7ED#R* +zX1e9WNT{C~tN@sro-a78l*t+%_F}y|bp6PhmpAX8c1U7XHNxykWcyV#neyuXxWUr` +zd3O8lOS+`FB4EnBP$bb%d=5SX)*0@LLQzR+`a7(^n?{lKerqu9SbB~Xdtcgk7N5=9 +zHqufmI&x5ae`klNuntNFJFo?e-6W>J)V^uWu>2Cq0hWPLehIvWyRD>$ibPP;+2mY& +zdzQ|1aA|{+mLkWZuKhfQTK6~2>&}(H$!*2bL^nx8WB#oy-(75MxD=y?7qbP$uaOdoC$3fxR>^TgALrPFU$4%K`z)L=PkO6YmRywiKjqM{8DdMy(`S-ax +zZr0Pp1Q?N*mL{jV|2m;*?P)OFP*$J5Ihawo3O{eAGb;1!{b+_N5!^b-a(*a +z+6<0kCQ=6T6`{#p61uCs!fc?p?y8hz(zzs@qz>E0jK`4^BNvDk*|s!TbG#SArrI%- +zKDp+B`@Z-|)A+2f%thg~ttcjiTaUl0Z#Dr)v-$^LVZIdH!=pcxm<{vhHts#uIiCtJ +zaz>D_RjB@qZP56MQ}V^-rkR2qOfgyxtSdilA+4Gn5u7%Mf`WsUCIVW=CLxu_yc4@k +zwe7&e#AqQ;U-xCO^7BZFO=aG->XP5xPW#eAY$=&fv{}aQmuULTIGpizYdYZsEZK>B +z*l3wJ>vuGJy8?;A!HzG!WcSyV-46v!Sto&njbW4pJ8i=CDgaX!u@*j{}J%ZZ3|~)+hWl$#L6o +zuzlF+k)cjeTfB$wnH1ibXn&$q|H8F4!$v^XTIj=)9BB%sf`wH~qTCUA2|}B1x^>fn +zS0LK)dCZo4G{rL7n0u@vI)CxrP0=&kMQh_NNPqXUPKtf@4I!n@L8ke;*2>iA*2vAc +z9M4N!S4zbNPSt%Vq^E}YY2C_?Xb3NdGDyVc8yqF-Sv>w@kl#fJjhv=%;36gVYuhoj +zl}N2=Se_O-Xt)$h7Co$zt;3Mda$lf^X7o&)>hAJg(WXpnvD~EPNQ4E-Xz1;ke``4- +zMhpX{HMBdiJy7~Nt25jO$13mA_4HY~GB?VZo7vA6<8+SQkwbRzoAG&`?9)r}0}~y^ +z@MM)=6VTwb^Ys6gnl3T)%aZnb`ApaRp}5&hXKoM=d>ubJl}_Uh0UzyL3u#6QU9<<2XC4SJbBlWrSE;P^1Ex-P#z?UIw$IwfcmQmb9k;&U1Hs-{dqUpQm(A +zd|%JEq@bZ^ky4Ua-QLpRQiw?!q(UNrl5D|?maqQ-hz@imDUhx9KQ4p({P)ZMkZmOV +zM~M0V_b-xmC}HbD-rs@$dM&ck0H$L}>3#gS`yYsSNkl{VOIPo|yUUCZM!T%S}#>Dwuu-9bq>hfRgy!`m8>?e*`)SnC08xZn>eN379 +zIGSIAj6a((M<@`Gb`o1*c@T|q9lHaO7roGw1u(ye`p9onrPNW2n!55r0tM`Yp=ig$ +zIygPp4v+btAM49vQ=di{IP>AW|NKQ?himp#jcC**g6v04cT?4aJMWcFMEJCL|Ha76 +z-J1#S#hlW&!L7$N4FqJnP^LzD^;09V-EZA)6&(U2pNW&nDK|q48&~~mS9$(#GVS@< +z(1q=u@aBt4f5cAFiq^ckZ58tN%!hf(1q4(JuZ5~r&bz6@^zXVJwmvg9*61J?|A{@- +z%^}zxf9rPl0=aot#LxDoE!pRk{6DL?EDbgPA}7AulbDvb#^-~E1uqB1Uu>9Dx9duH +zKm%Sq-mS4nb=@;db#y8D2hXU0x+3y>f03f}5QohJWu2!wuRgxlT +z-Dt!_iCFUU8m$szmlF$~LtSbR)Np@>6@fjmDcgU{Ch4Db +zXY<*Fk`jr!*3j}};wO=uspb}>Kt1$Laj5_7onFGE`Woki&B{HcPv6N5-gD9-d^-2a +z0HZC?%hjZlnMdCJB+>TBDQ>#g2YZDRKPx+_zkAuTCEr +zZmu_tiC?9bL))07#!PpioXY_RmINgX`o{-MSoYp|~GaO<`;F +zNSn3|?`ILo`iSxN$5QX>6Ol(_pZO~_TAcUPybf#ckEAZ~yy<LT6Ud~;gFA+txmexQLQEtTb32u@NTPCrAuh1M1$kpA=y^GQe_7k&=V|u0tIEY +z&zhE+0Jvu%q8<;)n|LmFwE%Q}g{7Omq}`kf=%>Oxm|uBr8Xz-YR)%mszH|Pbsrp%b +zx~~%`=)E!%%Gk%M(oXB8X|I-4-+t33k@?=AZu)>7pHN5kou(#a(~$7Bbc_kaYvAig +z{~MoTXp2NT+t1$b_OLqtKFeuVPhZ6!&2a=bi65An7lI90n-_56uKK`$qwg;lY{TKK +zHA3^tAl+rN&)Gir2vhTQwfvSv6NA;LJkwICAAFC2;!WJ`HM#+sZ%j^`0iHqJGd0MX +zezV6`>z^o-A#qlVf!<5Wp-T3$$+81YKkeAr!9k`%q@Pw+h)xi@HYPl)t@=d|_yX3Q4#LSfM~ +zi%i`v9X*u?pu~@0pkH--Jv5MuN3L~Z126Ws})#hfvvUEVte3uHd*?K +zy4G(upRBbHIuLjUcMLAi>V^sz?BqmKUHPunvrPh+3rvQ2m5ysV*eh=T+|$^dqb)xc +zm3o*J^?xPgbO4>MQGv?gU?-nk^ny%4Hp?c;%Nih%#6pPJ0 +zXWfh&zx>L+i`nI3u`KRval&t}VApX_>}T8K7fqtX2Ro6HAIz^sMqHEjysGX+5B< +zMPgdZL6RYxz@LAz#Zt1#*r2*mhntH+z^&;vZ?moEsI%MMno2pb0C1uibihw|S<7&# +z#a7`gKjkGESDKl%a%x(KM!7C<>RBIar!IMNV1fDrM+P;1BKCa#biHh8~FQmlA6Nw^*#mWkti&rB{@(fN +zbB~D(OzXku+-_}_a)e8VOPxw@fu}go$K8vQ5vS<}j6F)gz0Lg=PiGF>y9>!afRu#7 +zk2XNYWgS;(h;0Z28sStR-#ba1{v**gPt;hf7PJ!(@3gORt4F%Kwpl;ogZk~=(s_05 +zDem>mb*tB-HZP*-+iBbF9cS7zEz4Rt~Kqyv&2kwOBF)L>CpG!X(!Grw)8 +zj^LAMV|eoIE<;ki;5~s%M43jHO(F%s&dvQ1u5GOYf@IH>iYetds5kQdZ%zd`A~eFd +zi*tG=oxto3q|5c?eTp;|fxMV+@WX+=>qCxydc;g2mX8{45*7iQ7*S(=(&hVFC47hv +zMXy={J}|NT)55B0gQfyRYBA6B6={4V|%N+AQ}ad{*9?L +zyG3-t3vK<9cjg<)kbluU`ivdEJT*MFZpKQsb0{C$oqN0Ej%B;g?<6_#P+FPLk%OzEO+8UG@t(%L2eYQT^mL^$$_Y>G^H=CgP?w?HW^8 +zUyU${@oBnf^J)xh+ee9P+p;AxWX2AZ&bGURl#RpiNOGa?ZxYlOI_BfBxPz+4@`;B6 +z|KQQtP9IN$JA9T*7krZsc6p@K#3NIB_UIhYxdb-_IBo6f#!UR`PVDTo-xdAr +z-xboe<*TY>T9&qJ5P}$I80%PWn^RF>CN;J{$DXj;5$N&cRj;nRl^JOI;B)7x=d{57 +z*!!{L?5`rzW#`$ORy*I_Bn~8nrjT-cBljHpy48l--OCQ&uSsT(@i1a{Lb6S~$lPCq +zILzL{Q1aWV{CXU;s5~7j1DjKy$(m-^6EhXC-ORm)b--si +z$kk+I!#oF;XnhYZtzpIk7r!R?L%tsHP_iJP9$se)uu{a2*boU5VlH2xtz-XPX9 +zL|s(O0*rb=&QI$;H{UO3gbX@5@Lf!O^{0a-@+GckIrg6V1~qenBkW6T=cAg}vjmNM +zJ$MSzVe@rP^Qq3usR460sx_gFdEU*?EVw}amD9ai*DYOkvu}ae>BZxUDpcL%t^3GS +z+vkM*04!EBNryGL!9xGV?@4~v0X9npFS*wPQ;<(Ox|g$2-_v#3UoMaaGExc!6|1FayZjX_PcGwvfG#Ou098#gXo +z=a`N$eb>cb0D@^K +zk70`HyfnYk{Ho3Jw&EWODL(>JQp=rllCtyuVUe?NyrW$3I4?5&YQ~@iQ$49M;Y)}t +z+u1VN6M0kgI(w&KVx+*Ho{uN=e<===t}=<`MH>lNxy>JaVHYcAU4V(3^cJkhgP$2j +zLVhDsqkgs5Lfw&!$#OVAIdme5flj5#{8b$+>lN)JS5Br%K@VtS%Wr8o-&JSI6$o`C +z!Ap;RbC`mO2k~G@+1;@r^yL{{9RJO4O!YKjS6T=D!7!&9E`T4Qp3NCR<*lXw$f-ip +zXH_Ffa#~YHTmO@;zU;0`jt4BpMc6JOQ?`5UVO>>ol)7q!3ddoTz+_^CQmHC+&E`5M +zM`<3^(8ifEA+Kl0ZH{a?ibONNERsZErLuMu7X}s-s2XEJ*aqxWa&;2i$c!d#dU${J +zqY$VY_&&cIQH%;Z#1o%LyC@_$@20vp#sdEBoD{h3;K=&Ml#3MCmgcB>i|Wo^N%P_C +z-g3M^JKi`acik<3oX!17iPDTraoyc3^L0eHk$!L4DuY>KM9z8$g9wBP!8kg-7)>|9Hm)(3Z3@EGBF19sBSNNEhy;&{xZ+d^V2$xN;X3`1WNI1 +zS10#ZYpp?iYPt~J42%13=WUh{Hzrh3l(2c7#O+SW!Mh)!vUQ>J5;Z?}9h&kS*fKuV +zcfFACam5#RDN+UM4X^PxTL_EV)cn~f*Ao&xJ4W3&jNsVKDuVtqT691)_5$+rgV?bw0DL;Qw9p15>H4^2W%e=dvsD{X@bcNidn1(C(f(BeOPV +z*va?M4PolaOu0Y2miw;V1B==dCz7xSN&()TXNhQ1Cm*|eR{r9*pZ(2gmKa}~{8Bsx +zMxnz%Cs_EjSJQF$Ea{cpW2}lOJ6c;hk6AY^e{A4gl!!JH&$j?#qNAdaSiUwphAGz~ +z0Kj&-9XW~?4G`xB*u`~}7qAT%%j@W-nB~^Ap+|Ez^~T~sr{WRRu&-S(Z26R)9H$E{ +z`wp29PoY{*6Er^so@DBtg|yMvk5OXbv&Q39N@QZ;D^bM>>gdeuVQxrH1 +zc5_-TC^PSdkq#TI;iafjEM#3s+HTtK8|jBo9eiA9I4;U5_gIq>^tTtCS+HHF9cJ%~ +zZ(QEZ9UXpH>_nyTKKxY$&P(um?2nsQTLvuK1geqW$JCR%P)_mX8UdDWW*`MU$x@o{ +z#2wTFBwk(mDo?3p(@z>`EDz{Rdr8Gq +zush`bsmZ_MoS}&Ae5bx9=K&_}&%x^yc(QNO?SluzDzwa*NeJET_3m~rTIhwQ1flm? +z%P2%dhNGO?uEIp4@@;pez8524Len61Tlx6DPnXq9 +zB*OcIt2>MVvKY2#NI66+0B}1~Dv8;2!;FJ#`HF_*IJZ`uh{|qvH#f8qBU5TjPz>9!Me<`ZIPSfJ@*Yp^^I|R8Z24wOxdKG2a9%HqvghH@951W>~kHZ_NCwU@B@e& +zxX^m(0WY7`dcCzo?!1|@y4Z-MHVUTo`npEvUs@m9{oC44hs_fe6)2EZ$PFJ6=uUe0 +z{38(f80TcArO_-14D@OH2sh_GI#SYi;wMYrrj@T> +z!V-b5dpBA)pT{gVsY)>Ty3I_;z*}c|K~lyA@DQ6F(jk_OfQ6cMcz%FoFa5 +zYS8TGGEGu4(XWgW7LE=K0IM)Yx$%vwNo;RT3v)0y3Y-#ld1Q+;_YXeDu2kg?E7|XE +zXoq8W(j9rMP8P~!@?>$G6%2Uy^5+P-w4O(AtdVJ0eWy5@fSA^ftPDbP;8S07d#4MQ +zB9Izv0%?yDE60o6M0TF_2fSstOxOc|Gmjp0T$}cP@EOodc43~I!BtDOBsM~C!hGu4 +zwz<>cgeCumgae*)7oI4?Ua0GEys&`pHgxEydS1xp%e(V#Zd^NFOtJYbclE;pboA0F +zFez3t&6&p++W4#-$`y~SH4HfWzOz$xM1%|##&ree+_P<-zAdmh-uiQFL5m!{wsUvk +zX(u{mh4^Tp(Qm;Nw(0gi-E|uZx4kBBJnGj4Ut?pf7zeDMhwtFsa4NVE=uu`~$!4$a +zwg~WwDQKefQ0mj{OQ7^nl?nb9mn7Qv-?tgljUsQh&t0|+k4_MnE|?kDZL*m3}G9V)k +zDXD?uS{p&j308Atn1$*utll#|T2y)R>?W%_J6ZRXM1>R?m%?UWlz|9|W+*)qgQ#!e +z?jLOs?YuyzBjKg2Yx~vAS70>T;g>Cg&-ZS0*n!ojq&$$R*wWB$>2U&Y6N$q!icLE7 +zBi4MjZo`W{q=MS54q|~Ax@=Vk6lCE=p>nHibm2J02o5ctwplAqp!N6%sJmR1!%G`g&;9zLT3Bva!gn#)KD$AIl)VZvd79{+Q}py)Bi={pw%h#hpA@10@Wtj{pT=*B&a% +zv=+SNmTkxy^!IB5xby40t1K9W$JR-dLh5S72u$P?5jZjDrlqEkAEDp&&9v>h4$u6w +zBNC|<)XC)=D&&Y{V$Xt!w27IblE6Ivbri!HT{6vUeEmx-6N=8(9F4kLVyO+fz +zdgg`ZPNlO_^d&;qp@Yi~F8?m2M#9dQACKTdYO^QPAPg`?L@QDL +zV^jlFjy3Tg}6-v2H+nVI+m=1dtEO#2Nff^%J +z+fE|Wwq1seP3w8p0id#@n1Mw&WgU+>qke|B7KL{6+nO#0-v;W+)>3O<)9^s*CR!pH +zKgzz7f;@e3xJi*be{n;GcL+uh=(+fgU3xiU86yHNhshJEd4OV2f6bDDXu$!!CY~lQ +zs(Nn)BWKJZbu#_Du57qA@3uQ%+tv(q#JTR+?h^_r-I06MK`9sPX8LPhy6Q|c<+jvE +zS@ZTiHylDz$p!93a0}&jlSzutu!H~N@(-7Qq|ojQ+x5>rm!#`;XmYf9A6N#dbXa;zyg%h=+IjSuEhXGwRG5(5{Jy` +zZIPR%Ek?Ix7Yyj=y#cL-K(N)IhX{mX$eBd3iM+|DDL@)goqU3<7{$&x@RogiTt3vx +zSc?4ymN33Q-#9yms^{4Rcti^+YEw0kqV1CcuG@-P((-c2beMvZ5R(s^v=4Kl~=VN5=^=?(Oby +zWg0BgC=`|#7)=czaz(1PnLAG6NNU3Gd24CdCF^V4?rF}X%t2eOqmez(Y_~cW|Git0 +zMDrdvI;_hJO3ZY_>qLstYWN){`z}P!7k_zVpyHG|>}OHN=&!aWE=6tstm?=2@c%_v +zy3){%+LPTYo2D26vHWu=!dYvihZM_RkW*ClN1YW2uuhDbBq5P*t%yPJ+AlvEHA$ID +zL-56MC4-1OLOqaPd7QXBFlhFuo`F8$d)q!( +zd?fuK-XT?K)Ro8R1#fXM6g~-|GOFZ=hJbWo1$QE+9=oSVvSW|$3F4~*>qSsj!wQQS +zv+u~FC3;+SX0}I$cL$=5=ibd^dt4z)RR`lGUrC|${w9$&1E0WOf{jjmfkxF$Qf-ZY +z%K>Y{p(;)h@HD8in<`Nngnd93N8XX7y3%A3b~<}KMSa;8`bE&@I1TguM`-*~snv25 +zwet&P+6Rj+CmxcVkU2Nm$=okCN*XB5F;zTg8XV`vJ-$ER*DSBoF><^o|R5dBI|m +zV(5b9&*bW-=r4KD;_BbVjpWnT3*V`G{Ov3Nrc0DDJvO72mO1<%+TQxD$@l;N7eq>= +zyE_yZh|(}xLRtkRMhZxSbd2t9X{1!7TViyFf^;(mlA~cTw$Gl==j(lZ|AOyv?3Z1~ +zwe7mjo#%Ny;&y*La{M0guu`^53k5KMK4NSV&$jhn5`o^pHTfxQ1tjYH^LU(tcXhs3 +zdfolHfQ75WUe(u)<2DaJCehcZ)$_}a!OM)j-RN@_RY-Zem;B00rh7Jci~o2ds8Fx) +zDra))o%d#WWPVPwk1V8D$E5?iRXZe(=TcICt?b)TMF~Hjx~_?8(mHJUKCt4Vv|Q|^ +z(>Hk09}lKY5i^5VOg5Q8fw#imU_W0BQ1HNxrdA3tb3h6o@9fO`5h6O$xB!JXkjMFvztcgR#ndtc{9Dpy|lf8|Ivh<`7e4J>N3ADU1~my<9z#dThc!^cP}uLLYl=e +zpIhwlp}Wam@54&Wg{AwKmA)S=n~jELvWs9IjwZ{nRky-_wn6p6FD;L7P`3(g!y~)0 +z_%BBi$lByBn~2ICC}BVW#EMnly^H9@pFz77GHm*iVuo{@8g6JAu8D)V@j +zRV$g9m<5uvwF|U7(wU<7?)`_@FVe4n@X!x^d#k#gl?!v4JeT`7YyJYmp$XlvPcN!$ +zoDPJG*CQ2Jxs^6ghsMio{=tmsG3qvJHBh?!Z3{*QlL@fCtt7fKS`y>KKhYhm@%(RqF#xEDUvq7L4+oK +zIrjY;^DK)&la~^7kwUn?8H#i7GNNAH-m&Ii9AMxgKL)YiHiPwRLefEm1i<_IqW|%Y +z)TI2*4fW2V0vMe6IiH}!gl{J`*nf)-w*o;{c|$r<}t7Tj{Iv){mV2l6aD{v +z1T#6=^iOV}!)MBydrF)~;QwQ(?5)d_gN{)#Ay$@2z>kBCd(ah@u>T=Q#&Vd`Nv8mc +zTJjO$rk@pjm(c%Rf9%=JFkgKs3q(xZOp~SI9ebwnzi5=Y_df`bfWD37S +zDbHq52LDo9n6R$44u1bnW(ZJn@GB7@Q$I8>)Lo?d=n0wp{}j0ZRwYcmD9H;|+0diy +zG9-pZ`ggQQaDg#P>lYrTLlsO`f8@kTHhsv*5tO6~{|wrf(wSWB&;dgHO+$E8asM{fD@O32M-*FlZJQczOpAQw0 +zQir(L)qRtSyoXn>{~E?o6Vi-cUcE=1e&kx4a>0*qrlQ<8E{9Sb$erja +zGK)2|cU>h%S73dNR;bq!s|;7DMDc}8$)$`5u>bNtWwplg>a^{_RODyB!b)H>PgYd= +z?F)oz6(|c=R?&T>@tFZcx_}>^Zrd~KF*WR2T-(^=>W_G&D9o1g<9T{pc5ic6G7_)^?NKd8Kn~82ZfhrPrh^LX +zKcIh^F_TA1vxp+Mn|7EpovQa(9COvKJV(w%_q59aP4Sx2dNt3wK0KRQo2$r>$^A_7Tcfx<1e+1M@mxjJ0G +zmg)y&68A-&`fiaM7*=Os`KANnUE9JZPD(zUuuwrm=Mfyr+>^DX9OV4Z?N0{WQ>dm6 +zPJ;$FV}!L+y@aKaDSB29e)FEHnQf&o9)hHv2nc8n6LJ3#W=a_Ld?9!i??xWepDf@D +z4HO3DQzWPI?^%pDiVpu=V0rEx&A_16cDE(0k0t<1kh9Iv%uHO( +zHJLEJSYo%xr09F5z)PL+BC^ahLwzmowTusRTE()tgF}y*jXu+&`Qb>^AH`(c@1+AQ +z9(!Xh7$C}Ca}K21tAVNO@kgF}-bj7Lamdl3%PwZSSgx0>M1M(FT8Gmq)|dCiB~>{V +z-BS!gL$B1F?)h$z)FXCQ-)q@@PcXh3LV +z>ZE~P9j(mT(#@2z+J^BqG5Yb%uOaEW{EkXlK5|5Xh2gtNY#g+i9TBe=aqLJRabU`6 +znj_*ZZlF~hyu*B9>(A`dpf89y^mc+qJTgH<$`HmRjGcqG-4&cJ1LX~VMWkBTI7`&b +zB0MhMmq&NS$|27oEMdH=*+v?x89&BgzWg*(lrp)V`k~465mny|ha0Wcb5@l+09Fm) +z<9;fLf*^`_@Xc`Gcugtb*ULakN_M@-auo6h@SE`*XkiZYi?+j(_3n%l%q#&jwxM(T>t#)ijL32SqRK5GBRX8+9g?)7!MFR%9=?2nBQMi6x~yCt4P?;uGVe*fxTUbAtoK5Mq~o +zD0atlILRXN?u#he2(wal*=5jee2A%obaZXH<|~w9{#hYY`p-+_9s&BNVuNkk1=?2> +zU8+Q6zJi6$xe<^+cbtku{+`?^{sC}67oOAktC2uND$O@^2(zP^1UaO=i$Y@B>t4@m +zaXJ!tW>SZ?$Zei#X{7M&>Wp7be{zR60?9j9Ls*Y`6!HZs|I#ioTS}B({w!~={}8$G +z(~qdDtC{IE`A^0L+agwM+yK4~WAd4$R8sikvI!0FK@ +zn3oMBU!JR2uKgu^O`DfQV6{&g9hJ~(zWT*ql>Ocvtf)yl_Th!);$gAf5AA3&a&oA4 +ze61ldcS2Rq!V>#yXa4?r-vrv#H%?nG8opSXAZ+A-t&-D0=vqJFJOv96NTJAK4{PL~ +z9n0Bx?jJc4j885tdBZL7Bes<(Q8zv$Otnl+5rDSkpLr8Mf&WBwXb2xEYq*7}3w +zz)j{Ei*9~cS;M|CXcW?1Piv%yGUG#Mudoo|2sH`{*FDmZ=s1PS#!%7t%D8BMP8zkaGkEB)pdTznHKMFOJ#^`kCGO{sRv0kGk@$wgM +z%?DOCX~ljJLFvzh{XS&~2k*s9*GU&Y)BRSm<3^cAQKXZX~$n|1?cQZ0TnlkOl}zt==Q-s5TytP;@7 +z7V9LV;!@TF=Sl3w`?4um>S)vJMa>E%kb}dvixs9B$sdY_xBi(p5st_fD#eE~Xw=!E +z4C7heKJoizt-j3gUOy6{_pVsU?uB1M6QJj=uzvdf_@S8eutwPC^oOhodNvy&^2@Uyw~F%msM!oywrDf`guX*r +z0liS`QyPt(gx=ck3c2X76X@ul?7pIosIfRFlnfuDQ2Nao(D)4aYH3f04_!M3#^)HK +z9uX0+`Sm7J`FzlFW%#KC4}yfMsIHmld~lvUdzj41xZsiSaHV=2%#b7Y@#B=-tgxDD +zr-7a$&EX3oxj!s}xkCn9`?z~BWXa`%}0its4}fNKgma$WoA$< +z_Xlrp{O^ou^dB`eXP{B?+LN{&x(@yIa!iO~O5Gci>sb$G#ybst&UwaPhkPFH5xcDX +zuNp=-2nQoHIniKjP!ZeAWyn1}s&cy7@6^0Gyybq)B!eRlVwC8@*MjGS0YS?V%e!-% +zB@PTH*V@-xzal^M-%H^84%DV$p$lcjqg|@u{WLV9(4_hKt^GI#9nP>**XV|e7xhf} +zc*|HZ;GR})Br4zZM(bvCRpw@Mkd2=fh;0-ig!E-wk?tf1lX7hLc^sTBAGDulT_;&0 +z8jGWS&q*&~=9@Rw*HbKF3h~=QxP#c)YbKE{Yl|^Jv*jd~HHm2W+G@z+#O#UM?fF(v +zz}3>z01unTYbnUUTP=nze>Cg+as!TMhg6CJk)NU8PwrY;px!P{eJ-${Q-aPzhQjGG +zmi^NYdpcQXip43&6nDo%5I39qS{*!_-~gviK1iSgH`|BN8hnqTq}2NL_;n2gSOA6| +znN6_lkJjDSX61$f{Xa@{wKa{EqP5|SsSe)enY0(L!Cj^3F|p>&lPhJ~Y$l}_n**P1 +zX*R~rhkogpB*^#h=I~@e?^?V!D$kcs%8)^B&Uk}Qs^ucZDB?am8Soan-e8$;fr23| +zbn7eW?j3a`24-u~9j@HeS`OW49~V65Lemh4I`f)L8}+)`S{y(dE^lLk`{0axoz()l +zj2G3fvywif!t2b>O|8*A0{gYz>v<>Qn<@}cGt_Bt5LH6 +z)ROZXz2H;jI@gafU0EKW{N>P^YUe&Y+m<3-qL12~ypcsD4w<4GQ6$$}xs`XP{v&Jvo%e!1R;SZpEdh3mmdAW-y|W+>Ya +zd%6fG)&B?%!4!s46!|-{a +z5Q=qb$At?G0xd@_+b#(;mg}6cq#|vt?$6j#(SbKRM}aq2x5R1aml}gPpS=YwB+$L= +z$W4I}G~Eb#cJ-!m8+^INebW$XC>7Ycjiwz;{I1k2?ZONu^`CnQO@d3u#MYDdyNBIu +zkNV|F$kK4>Wy)#`o{{_VK21Kj4~p?;N4a_cQK(5XB;4Cuo!J|TNp)S+)1y7 +zYAq&}kf&E>GY}-~{{CilATV3JbZhe6ct5MWx=OIaYE+xuQ^Rn%zQ7Wuk-<77Uzwil7SZ^4>m6S0eQ+qBO*ktafL5+C{O-MFVya_PLK?4hyzY7*$j;ofUWI`qW{ose;JG0{n@ +zmy)!U$}8*RDN_w;WInZH?$|V8TyG-<_p8=e&2(mr|l9T#s(HCPX_tE$Lb%EL5k1j@m>`eEcK` +zE$MhCE;m*UFb{`1n?Z3HQR$dAtD{(w2SD1-Oi96_!`C&3HrQX-@5DS^HOOqOQzt&h +zig)6f3MctM6rgHA3Lx;xNnTjxP5OpI4Ektjo`Wo!m0qAfszx;kv=F}3D@{}x51n~j +z>3XR=0yJ&4lDIh=r#@U3ee@7lI#l3UeSyO-9|@4yf+Vuo}~)BrJZh +z9!Fwk95$b*jtVu~<|Itkr|sL}>N{!i_BQu16&{U^O*SwIdHoIjNF?W^ud0()H!G^8 +z&^SJ=-PZgnJ3#nvqJ^D46Y_U0ZK;MUV&Gl+ZiLYHpM>k3oiqko)hg1xuqRCZO0@xc +zcr@?iqPJRT47RSumN~)7c0c&L4P2atKJ&ca+~xO>X3>U&qIZb}sH~*IBd2kKyxX4a +zCJD$(&qOH;G+|s1Q(Cj!IERYNt^P^-Y^iFw(K}k)xRfCoa#`_C3II!9+{%gE8*0Ck +zMMQ!)V!fS +z{hWnP{|&Qq<3V_H@<2j{EWF0X{Yj(#r(>?vHKc_WKG1I8O1@ +z&NuW3Aa8mBBPl26I`8u0fU8T98Rsf}xlHn$`vWljDsl8SJB}h2?XI?KySr(0uuiTn>Wm)VSN7JJEs0~DQQw(-v8oS+S`~m2?WALkhaE$ +zQS(zuc+zq-r6c|Yeg*K_Pf`oP{D8mOhNqIC?T+iPC7$Rqb?%*Wfi$cc~(qM +zJ!y+JTp>uB2C*q5E78t8

IqUtAm}BI<=QyxM5c$W7)YvIwS?i15vaLxSAd*8DY( +zlVD$zh%O)|E1~%Z+*U<^U?K$gM5xXAOlkYkW75%>jfRF?*H3pu7h}Ziyz?D(l^Dcr +z?t`db9E|g1B16;Xa`@AD&igQli9WaEe#U@Lbx$6h)sIm={9HYG#IEoW#eRWVaVy0G +zK6e?g=y0PTjA>;YM=}MB+)q{Cjt;rFpUq$Btf!+}V?{2qNw<4`AMwyiBq>JHqTFqy +znyU(;vAFNai0soZv~z+@)WAx|=$rAUn6QuN(#|_5K#TZN7d?D|EPrvwX(wKD^J6!zhyEvjd9K7(_tB`siu*n*UwL0{gSLATZeQq +z47BtcJ8UpD`K6Y9bxeN`b%<2UNA&+lG58nf+A9gO{<2@9=+a@vztfLaPH@+_= +zHj=HDo<0Bq#X{EA@1$BSa#mPKOU9$-lU@%b7#MIQ8mcYnf|YP*>^hph%_b@i!QL +zT+iQ2zuFHwIxFdC&-kAzdwDTV9T~;f2s*2TG`5uYDBr;0EWP`590_=u$+-zRUTaN( +z7_%i3zP1+Ii)Z%u|mos{rGvpA0_ +zM`f*dIL5n4{cu6`_v8d75^apRG@_R3`Z!=8?SiSEva@l#)7{+VlRs8a7{V?YDYlhA +zpSiVPX~M3{xf$hlwcv=nMBBnU!#LZkMYPpLw#GkxpcPgTR>c1Me6Mqg-{t4`pFi7| +z-g)h2b;G|@(d3m&W8Zc&p-Q5wOw}xzyszx-RN7G8GCMj2Q}_mMn|ql +z)4`OK{PU&V;AZ`$&U@&(2}PgF5e8lUCU!Fznpx*?&n|tp!G_%TZ`tIjS&OYJTg5*R$|JuhKNae)YT +z-7~ukbQ5p26hG{umx*^sl8nS7k%l;ox46=zx5i8&A#6D_)k3bqO?_p8U60Q!(TU{}bm?c^;KtN4!R4L)1un(GvK!P3@%Kq-7;RBNZJ6x_(U30(nwZJW5Ih +zLX;%a`oIA@k}Dn`A6FA;bW><-*5!(+Cwmd{5p;Qw_oKGj98S-7wKfu>5}^}O+8RqO +zTXEiS<&cIWkLf1LU5kxvx4bsU?TK~maM=NKM)%d?s{xjLUD;={24etY)vbz})p+A8 +zr#VT&sD{?N?e_t92oCa%p_Gw|6P}v<{t*Ymu1QoSJ0DIzs&t`{>~7r;`2clzDRt67 +z<2(fRyJP+VnoHbqRX-b53of8RlR$0PB_rlpeiGNn`gP(YW4fjs|DN0Z)X3grNpvSh +zvSI788sTnCJz~d@O5UN&DZbJWBp8I+60mf^P6B^mITmkzqOq$Mj2d(@Y<6XPE>Q;` +zm}!MQXbuTguFX>gX +znsv%V>B&jO+!hdW;v0ZYvGO@j3A;HraPAdjk>(5eKz=70y$V=~x0FcaIhG1uCUNOG +zL4I9o@j+ozmIk1aFpis%hN06z0}N^iF+Tac?8VA~<(HN&@9iSR{L^0ybu(~qnhRfH +z!$Le+X#PoJLBKj=IVHxj5;N4*`YHhWl*GP`BsP`Bpzg1nP|jfaZ{?d@c){R)8P$r9 +zIbYD}=;~klqvUf9$+jYY-<*BO`om3Iz*=}eusC0XYZ+q+J +z)>aG(4nld{6zrlwU0#ow(Y?C;r5##_hMZUGwI;EfZFq?ok6t()pt+eWv4aHP5MQG8#XtgGT(jtcE@J7kJsAoVdb6RT)_2zA5||sig?Qx!-JCi~ +zHJ6pz1Q(CLC6y$NF0XZ(#YRrR*JJ7;j#`hFgpr3Z?JwmvoB&zT1tkp_^TG)Vy;NTwyX*%M95qutfaUZfpus0-BX@M|&A^(290 +z6BNorOG_&&Q~KZkTFeSx#mnGd!lSNts*Cnh{d36)M38QyK;kt;RXaH`_(p8l#BJl!#HQ7{gz#BPP1ir@LD +zhK_gONYn&ENekLs7~E4x)j#8R-T&1hc$1{&YW7Esn2(YZO82Sx?XTbOGdF@YiccTO +zMif8tIHg%fb2_hy5DN0prqSFCxh#jn1Fax96(*W&NiM(!7Eqh#GX+iE7kKVBhaqNGbv-*Zp@$Wxel3eOZf&Ow1&;^d2*?&L!En;tD&J0)=`F`^~e>Z&>g +z^2WuiSc3kBuH;Im`k|GHSMM5|)e-KQ0WM!v*vb0lk4?i2SfnE|edtXPAFQr;DQJMRBn@-_rbQz3gChFq76sqn@62T-J<#4TIhJ7k<9 +zzWpA;$P)%0SGd!`@3)Sb=fqo8L&Qa?spi_)je3GSA)5lNunl%=4y^3~R4R^rMks@9 +zFBf~?CM~fuV{c)nFMXNb8{@>>ZIj6x2>9qh5Z@IQAM1(-X+@ILz;p;LYO6`A)_s%J +zoxJvbJL6ygHH$B*(rAp(2Q4JfFiVDZq^t2yvJsi61L7vHL>rr3r!{K=-pHv!@c@iH +ztFJ^^Nw>c<(@Fcy3lb0}aZo(Aa6FYp4v@gRJ&mJE`Sn%H`h3nBDq**h(QjVFGBNC8 +z*0Biy%ZYKl^_tXKrh}j_jG)N{fyAOLLj_%pGPQc`npl@$QEuPzs@*s3oRiB%y-gk| +zZOt>sUo@c~!+m;#62Uv!pHF0+_>AHj4%!>@KWzdq%W+TMao12da1Ru|iSOFh$z(+* +zb@?)aBs(c`AlEmi2p7+F9f`)8i@ac7Fd9I;!u6d +zDwOu11S4+Uq%X_z$^Q=4c{!4T5hYS?I{P7(Z@8KIA78r+wj8g=DT#+Fwo@P6AWd|! +z@9(s?7OFF7L3sm(5I-L3i6dn$o;`>c&lPA@0EMTUeEXEDz5q>j3VB4=#+|kgRynSq +zMkXs-cloJi@x3mGp@L2xFt@oBw)}xwqf53lG}s~vrMuxqc7ZsB@7J``!-Zib01H*_7z5w)F7Xa(3oAlR;S;Pwn0GcLCV~YpvEZ3UEk! +zuCBpMfH7u2bljZqrBhg#o)_2*!7jyL7m+?0LBJQ`1F`nQE8BVCM +z>F>?o+Af0(NsNkY^aEeY&QEgTf%M*t#?e=fuv)Jo3K!WR+QN3X+-HwnFzRxpAa(|Y +z%6W)%xzEqu0Kt9?1@|jv*1f_%OeCTqx2Ou&nU@dk#yION46D~$wA3GTKK^V@L40VO +z2n;~^HEy5#AjWuHEw9yo7`+)Rj6pxJoyiLs}1-bMat +zj$(a!uLcQAf8VuZj~z(r-Bjg9`_y6od6G%kCuxp(5F-E>@E~c`pVar8Lu^PBipNS| +zA5G)37{A`(i316`Q*mTsAw|zfDizf?vvu~jG?~b5Y(~Cf_DX_?}t};OJ|loj+f0U +ztljRqIhDQM=$PdHk&+cKqifm{Xrav#gdAPH`8`(HzcQy4 +zzEEZy3Uz!Q11KpE-K2QD(6OsKP}+!F_h +z<5?G`foU$Peg&g+eS(h5OJmpNCl0?E6WN(HH`bQtp@>)7McIiZYN20?LxKpNSNGMJ +zbrL|*yAM4H7F04dj8yt%2|^WmzsTY?7Y>k^{iIl}62MnI>9^t3*Xz0l?>{K;InmnI +zbF?Tl3Hoh^rLjY@by=acVX)pt2T48y-^PSxOX-jrZ*M;9Sv)}o@nxl1q7!Z|;e~_G +z)}`aQ$T8vh5hxLT1`L-ZF`bfEduMpHWwMx%s~qjuy#wtjGkt*R#0;^lZu7A;VbebU=lnm +zV$7?o^BTB7KNvB;*;0aBD_P|b;mydimaSdOD)|FI<2*hr~~B0ByCl=$If`ytP1nRX$iKlw&~kvvuR +zM`TiYofqb5dU?0CuHW*dGkLb{Wn;Xh+u^EYI4}THa2u8@wxtaT#1sW^@a+jDE)BJ@ +z=5S{jRR3GEeL8(Fb!ycx*yx_IW{p5&+MA(S?oRg`vkcpkUAQh_s5o=` +z0qt4Ji~4Vr1-GMo0h`AM+=o{8UjvcgScB~X-V%sF#4eesS^>!Gg#&-mfG<{bk%(n6=md{T_m*=T +z_>1%6km2aR`-!%zc%j=B0@+D1&gbb0xt)eiuA)jEl7& +z<}YoME+-hqb>FD~Bs3yOw`Ba@%{6*v$5-)dPEVIhBIoPh$9~DC4KS|!ST`lJt%ejK +zy_5&<$qhCp?yjL3QHg4<0)W#L=bD#7NvHyBTt6-$ty0b>1V6S{gtEL|R|x%@kDWbZ +zUtgbl+M`4KD3+)mnDSU%&JQ0X-p1geArYTQ^rdMf)-WAf*I=i*OE$qNT(h3PzMEf| +zb>2tb9U(%5L%5~SJVt$GZk=+Il@-9{ieL7=@nw_o^fMZXV%99iAF9CynXI);BTBhm +zZZ6czai95Bc0A47@(NS#J;-gsQ;*1%bYJeeXBZJ(G{z?BHMft(3 +z83d6bOpSx`fFZD{>K%d-zq0Iw+Je8EM$rl_#13+1r%Bp7IK3)sf3oN` +zKhc$3U1Z>xq-NSk^%E18GE~Nr$slIJCRhJ7(&l?H($wU>D0I6t^OI=PgM>x--Jz<; +zaO#;FcYXH!awN(*Q(Jv3>ikuIX?*(_6EC%h9?Iy5dHubka>Y$LDMOg0vBDWMEx*OU +zUslOw$rK9EW4$T1Z_A&ABaUNB`Cx`Fpwug_b~4htr)~TycfjF-O}%6zWMN5&>J+x7J5aG#@$y8sPJyzwR&ONEc;oAXUjtJ +zD?KD#9;oN +zYdMxw-(7I)Ao_TYd`{5hFQTv~FC7k6xp(2q%15u=Px;r(q0@JP{_Cz=eRh$3R@spY +ztanR<%0KAn`|EqKj$Y0rZ`=>f8W|=U)-_rcYa21v6W!~M&Xv|iUIs`E5qT*>uvdzoEr$kOZ#UjrtbPfY +z#4J7)s-qB{y}{U*xl9f%Zy@tRKmF1OiSq1H;MTJ*Udss)^@%^r(w5_8rSKMs5pz`u +z^8Kx)+MN<35hRK?OfxQv%0IqEvyal%0wRZ=M(Hz1)=zCmIg) +zCjzzFi+%Cvr5A6w%n|5mlzH0oR^RLPd{HJRR8DlTsEQX=HUN2%nnJz(g`6}N9P;}8 +zQLHP!$CtOZ6SfD!EGCVBr_0j>S}0!66je6f9=4%(6_)&uc`TLmo%um>Q4Tu7HN<4U +zStD%t?y|iC5p{PWV?g#0_A$t37%U5d<2I3svc3~}d#B$B+J7~)HS=!H)@YiNMVvYB +zO5>+}ov+`W1fTi4ho&@kYC3{8%jw(JpWU8>LK~%a614xQLF+$yycx~i#hK1K(ZYUF +zx2s+U$aCzThc8EDv2)IMtbKbp(CU +z!y`*{$ns07LwN`$wBm$P>VK$BlJ|>U4nRwwRT|&6uQhl-xVL4cANP#pdGH|kftr&1 +zoAi220WfB^!==eACfx1`IX=;=B +z__hd*bt=+-c5y7@9e5%g%TG`j)iqsF$~a_0%%|m6hP?}xR!2)jVIPg{HI<=6OsBxa +zZ4wP=qf5}P%h%JwbDLja!R7C$hAwF|$L=9`1W{<`;h9 +zbUh)7rcs?*1hrX>QKw3dFQ}a2BiPXK51O@3fGRa>h!-v5?gx(Y0iR2+Jxe>ZnhK`B +zF?IQdQkN>vgS-c$GZvLt17r +zQ-pALwE!6R6H$i=LWI{6)pbwwKJWfwagk-ly;PI(ZEs^?C+)V#h;?!9Y +z$s_A8N5jh7n~XNkBp-5mO_e84f8RR~aH@m7EKyKid{o4D+c(RrcShG2c{;FA?^c!v +z`W>9;IdMkd2UBD#doRoP&lhb8ZcdF^s%x~gvSIw)Gu#`&< +zPHB=KFWo#*t=LLGqFCv2l$+&O$<{`{#wjml9SJU|9;~6-!oKUUf{*IIoZyZ +zX+F&{NS8<5m-38@W7EmpZvW8chJwad;nn2&3W5fMU7ZKQFT`oSMVt8RnH?9HuTFbY +zkYb#(Nh=P?{iA@`!y>ej4#6I&TqO^eq&c=&;OL_8;t +zk%!EaasdcuU@=Qs%$cpu@hAvehVPXrp9S|_Cu@qYxWzyW85^!RrTrGA(WE&9w^eJ8 +zq>O8LqSirlx1#8m{`7`GUMYacjE}tad2td_6N{ZaMBqTADvfZ|lYtDQ8X1Z6%stx& +z|2#`^2y57M5SrV6)YMTR{vym0pJ(5yb}AD|-ImE~*h(cmZ9*2()A9!}RcHyCUL2B} +zp0%At*WPN^^*pA984}z63jE>S#>|IwwId!N@-DoY2U;-C9u)Qk)6!P{9ehsKP8*=t +zs^)_o+Z?NX3MoSA%bEW~R#k0NQoi}K>6w!G!u>s77n)ye7qa5fgzT9x&C!t!-&j|{@oZd4u +z=iLoZ9xJianfI)Ak}q!-!piVZ5`*D?Q>l!Cly>|NAwCxq$6<3@@Us+F!C#wsc=Vih +z4ll#bKaREG61<7kKg0$sFJR}G(>MZd;@-aa`pNVX)8XOSg5_?ctk7S}G1f3U+##Y* +z#wMh_S?;);d*RGOb1H;%F`VaWAp%=&!~)S7$#ISLI};0zw?nh{Cy4G0})Lg +zR$r{=2igyny_xPVD~5PzH=4iZ=iSNe5~%40ryQgxTKY_Atvn-o!xt%@GFsz;d;No^ +zfJMFH4*++PkGO;kG>hgmwlN`7{5+}*OCa<8jR{f +zwshj!n_*&3Wv10pSYsjfZ4Q%{Ij+z6S|1e76Tp?LKlT<77ZmlNLq6%%81rnsZgSZ| +zsh`nrd#aocSB3VjX-I4N?{b=u5PKg#M$ji?nd(j5$qkKnDJnaVZS3s=kF-(OHs^+e8eK59Z!8`)Uo#;l9#-$TU0?x?M4 +zfAD<%V0I5x@|_eE;|?l6vqCf&UtGWd@=eW-?Q*j%h0vD#`|)k8&h+n8ytNPhnOURm5(=7F8e#~SDoOa;o +zIX>zNeBIZO^d+oWDCd}tw +zz3g$|_#wk*q61fYT!W|bys|Y*eMt=~=K7J7o%ekHT7{W5cW02~q+Q|KHJ{k}qcPS^ +zQz*aGnsJBi1zKT-!v7EVyh!wQkWenQaoQv>R@W^yT|iGe$n>=98!C$)GtJ(vfDhhW +zfW2~#YzN^XzbyVujC?YAja3|DAFe-~A?ZoI67czxX0eXq{nfYKKd9qOs{qNAO7XL^ +z)%-E|>!`a`_>$8@yhhO5F5)AeK)h0c`tGY +z0s+!$7IKcbDQ2Jh($}-c+7*)>+@*FDt($_sZNevuDe-ui2CJq_z>PK3QH>F47`@w?9^Lqn`z#zBjcxOrx}Efm#;J7dRe{8Ev_63wusp=zV!VxdxLQ3&Rof9=~0%eXrX +zj6dB)O;R6MTN9<-wtD*f);GOB)=kg=%D1~rHH2kzzvTq>?cJi;{X`!0;I@d~^&3I@ +zNIk6SqJlli~@@b`xhV-0q=4z{Rw!GGNP-;E9yY=2sMwg`aM +z2CzSaIb0YUu&gW$bQ)%4Dfep^`rY!o@u9uIZi(Z6o@RN-sy1$RJGtfg?W1KaJWbp_ +zX?9&3nf_gmAtunj*Xa#K`E71f-~Lgcf8HV4L#inGj>IRXNDw +zpP)Q7CXxo(IbE|nV8 +zt^PqOQM&uTfg^}z+}((KoRq?Qc0-~Mu};fAt;0#9 +zN{eHJ$$brKzAFuHvPPArv)qb}41DI5wS3GL?S7A=*P60pRWEGFjfX?}O($31uXPGS +zW#whJvgz%2O~OYu<+e`8@&ndPKYLuckd{3Ur`?v%|6654F{7{eK%HjM^`BkP{tghINX*^`^+T^+a4=&=M&+a-cv33~6B${Ua?aR$g%arH}FM{ogUog8DF6$bV%UPOS?eP34Q?9k__mYjjtA{!SmCY5VQZ$`hGwouL7ac;Bv? +z=jKbvH8++M_s<~L)Li!|5njwt=&By_n<=*WEIKdl7yAPB>3teoQ(tEp9zMDyJ~b4I +z>Kmy0mi(kBC(x(mtVuKRUb(!3DGr$IF<-a9QJs-%e{(jqe&G3`SNB#rKqpyUdlm9^ +zM4O=xBRcA3dg{%?y3$DoXC_0Q?;4v-FR#gqVAPv*Y6T^uKg#%BLpppwns#MS=3|d` +z{rk@xz7q=0BlR>q08ecQnR+M=UNdv-ci +z4AZ-9kFF`L=gguEEel-L!-f35RI0~r9J4WwvRRCGc$Yj5KW1&l^DME?M|K|u^y4Kl@gO_ +za_Znp%UEvM{>Iq&p{D{zxrg~yo4pmOWX2#mrvP>guo-!u=dd>+zDC99BtQ$f6vO`I +zX#nAcDpo}UD=9R_bm6fGDoi@YlsGv#w&%8dTtGHqw2(1$!F|Yg{%w22i8idMIu&`$ +z#Wmw$ZTqaeaAUAvQFQddU?lh`CAzQ{ZRYwJ24Facz7^(z$xhhgr$9FfteBXN{@#}x%`h#VjKM( +zUs9VZJL0BuB$qElc}Ow=!?tmv2qq>Qp6%Bi$mUFzbl8+lM@uxIG??%TG!kYnEzoro +z3TYGV><4BCyxD)Y;2EN3{zcbje&#}PAuXrIs`bFoPpJ}Jnm?;_!;79xKjHIXhA=u! +zhRAg%^3*B<<7c~Bqxw=$IoJ&Y%+!mC4Lr(`kfcSFO}HtZc$C16TNF*<1MiOJK^yO# +z12Tz%7Q=_h<}I&VLwYC`>!VwQon=~~t}rcKty_PC^l2LS+!5;`rcj&KDW2I`B@Fng +zFZaM~-fF=SYlw!qZ#-6^Pe^#IoKRWPdVfd%Ti+S)LqtThP^Ql|lT!~QxL;u*Ug{mR +z%KD4uQ^03fvFh?<-2zh}jr>a%i)(o0vgr~R8S|!vfuF^@@qZL^?}0c8-Cpf4G{AXE +znN;C;O#J8lO6wkhS0wt8Yn7KoM8p%Oy{sXAAV@8AxNm){B;IOwMu;#rN{hsR_P6o* +zEp8%k{@C#Y@bEo>YR%OC7UhVUp5rt86l>-X+ul2Ayt&w* +z%r>Kedh#}t%U`$rzNZSPkBVpZ#=xZrnxWbKtJnN)&R%IgtaW2}qLHbh)wm_IhuvqD +zBMDbLKT`ba>{cp}yKVAu(y$NrG2Tq*w#R88+fSlW>O_HhPiu&e%c^mT9qf +zqrYmpJ>eFkc=#!JSL$im%|)+$PMdCfL}JR5Sv|cGy6G-Nxji$TkBN +zTg8pW$B|3U!fmFjV(!;!WYreSc2~1K>`93K^;HCXiJRa$yM%-%OkL8K_9x=eWGV7n +z{e+f84Ok8eR-Empa7X!BU~to_6FBc&Dfu`Tnaaym*v46)a=0f_VXuL(C{?Q*Fyr$h +zvAwC()4w*ZiaWh+UV=P9tbURQ)Uium?L0E&)O1T|bsX|$F*|}FFj7r$0;U_z7)$`hWVRu>LiQRkVxpWs2Dqm@8g9xob4at +z^mkLsmrv$9$J)P05-HdqEO1gL+jceQd|r@H#R(FD-Bj`g5%JQ{YbZ<@=B@~C?{3Cp +z)GrEH=>EPL9ZdTeImzLOnV`#EH3Bs9Ph +z#kvrz)>D+_r&d2UH+%j!H!6qn?PqzZKjq*(v1oSpbYE>%qm=5SFP~r+W1|7^EwwfA +zF>VQCb}Flt_vI#C@k|wj!ygVa(5JJKgxnuih-}G&5+cN`cn_c7B^PVMtVch8gGD3u +z9|x2@ZSC$&?PT!hOuzO9>`B}TyepCHY@iu-Q1Vd~8bB&IAfF_Jc*K6EVCZF{-PBu( +z`-Y->r@g7-KLJSZyM`a|0a(W0%-ybYK3GxLAIsEEZ*)Ji33lz+zRzJl{8SFH4@kq; +z{kh!jm4WMAu2ZMqgIFjFXYAFbl0gWf?y!*^lZYw@=iI#rD>}nC$?y~FFzPTt^^%?4 +zw}O#0OpjKd8nKFeZ0TmhQ#k0UIys-3#?Sg(}GDe>h(piQ2gwClH3G6Q}|O9wOGCEVxB>yvKK^CgyM^WZ@Fmy#2m0CXoiOmyoiX&GeZ|a&_Zr%{9!owP;6!N +z1HKbkmFDW{fXUGdm8>PQx+e>cEtI;R{8`h)ig3O@RCj?t$Y1}|v!*6}<9vS6|D5of +zUxj%Aa;vSQ1e9$8CC+PmaCt#tWui%&T!TQqE!XuY3Vh(lLiSqM&~1GtPz7C20|0SThMT_c=R{P2A_2aCoo|JRc-w9actuAyd@#8}fRCZkzIu;zJ1;PS +z;NKa928hh)m-P30bZ`l|VnG9V#wpcV3IFU1K8L})Ly5b!=3kgemcNxa@Wpp9AHOsF +z_%{*|*+W9K-LO|CvjpXDdMd#`Rb7gK7(xB55?0tUv(_@L-Yhzwwu=GrDMA;*V}j5F +z%;$fCp7hT{44`ZO^_}k{L`m)GaNd_hECm}iKd?*BK7s7B+Pbmmh+pNY?Pa8DukQmRk5KXlw`Mi<*x +zv+z`|SV>)n_}-*zSN|-GNxubCZB@MGlimw^KdR(t=0b3 +zMbL#H%oufAU(IlOi%_d{z7YHQ0$7yqfR5A0%#d!s)74$`C;0FE +zVV2;>#!==3>gFU(W_-_t7VK}Ld;G!^q=aD{=Z#ce*NJBQ-s@$=VaqDw90cm~7iD|b +z0IPKEvh*d;ck_T|JXb>+?$OvOAgviWO(;beb&0=2WeRttqy@w&e4>-&H3m#-lLJNOUBlAS6%8uDC4|I6;^#AMNb(#Y+>c9 +zCFHKVW+@L|#pwK%8IX#h%*>(c9y{bK+ZDfJh!YV?C_W;8pZIqIMJ?+YO>c;=<`8Qp +zi&cZ#kMS+amY=UZr!E$^25j^g{Q1_pF5%@M3vo;?mV^>o6t)1Qv9%kOd_V?ImTeXycn8NYswH7 +zxl!Tg#+1E$=-~duw+GGmGH_gS;IfMtl;1p?a0~q()F?$utp4Jfk;{_`cGT3%efkRb +zdUYLIKTeL3>ANIC;>GzVK9EEv5g!e{J!fH0L<&Op73KMh^I78pt0P-xiABSqt7}dS +z-wq3TVJr?x>_6f|)cY5F?R`;S4PJ +zper`482>OsB{+XBWK)0?b*VkBzx^#oB5vxjv%TTNDFxTZCf~nrcFa6t3?(lqVGAm1 +z-OptGy-e&S^{MdipBMHhV5#yS$>>=FGyghb!J29*|I}jHwbFE$T30@riTA$RuIS+dH+_oCl^l2V`Q;u|Ct9!bN(I +zekix4qpx;cFohqZaxDdJ5dF+N)bKOo1C`H|ZL)FeG|eM*$rezl|LTYJF)+)#%1}f_ +zt^e}^bxQ;uuI1qmro52tR6JhVdxt)OHubkS-wH+c_Lu*zA0O|3D~5=OtGrnW{kor| +z?-^^dIBGS%*k(=zvb%sc$4utVrv2sjHqaU~On-H;U(>5xT3`3#vi^J6D$1n@My*zP +z8*vjhX;+MJ==gloGfaO(&C=zN<-1r$#gy_)Kn24yAbFM1DN&5RFL7k3mWKx|_es&S +zbqcLZ?%nzI`BEx5(o<)gJFLuZmD7yQ_TEy@3KK89ZLT{w;UUm^IGhT7cFR!TjH!xx +z_acS5r$VWB!i>wCLjLk*qKm#bCbzZroJ5_Oi@@5(Gi8(WU!F9jF7*mi3YH#C92eHV +z#_$)KP=2+d<^;8Rv~bpE&H69e{u9LB;D6Z?bOPY@fDudPa*x_K=?h27zFU +z5b7jd&xsjsRRpLviYkcShk$W;XXi_K0L+`;-r7LNK^6YkxAIb3=%?nSvXwzKGyc$S +zEB^>aD+5*jFN34lZuDDSB;W$=o#!p#+G;#W+;Hvgu~cDz_V;`+gsX3B$vOHEpE5za +z$$?Nt%O)W^%S7zWV3^0In&oaEcc>iLN^@Uhbo#Q;suRmD?KiPcu3FVfw@JDU6_N +zabKag-4h#ilo^LfetbFBt+&r)G;NbsYY7%z$-4_}Jh}X$sTItR4gB=)VqhzxzJRMr +z>F@8_x)faEMmC*%Mdpc>H^-qB0KrdYHY_SVTWYm-E{(vrysEJB3C!;>y8hZVYo+G< +z)>K0}jDO^H#V?;T`yfQk9}8iLyvxCoeRdrQ@!XEKDhKmru7@kWqzLFT8dCi!jXj}O +zl@9wN1a!FrxpX{oMuxlb8|8fiB*)!lEwUOyLW;h5bJXE)rMjyIFr>G@pngq%xjsIjtKn{>BM2pz +zD4V9aqN((<#zPL(G;iP>iPpVRxFPFU+u$uR#K)l5OmDogCFOTu3Q3MB!R%t4O5G|r_y85=f`5TuI@SmvAsK7(*;G#5f7~R&R9xQ +zrSXuhjYD}B*Ek^(y%Y>|3_WbZ9$+^R6Z#=ZCDsy|DYH%mvn~ +zx0J!TsCDXG*?w%D7e5adJWcJ*rOuaI&)H%~o8Ec~z*rM5fHA%fHs+)c8NKR2Sp>w9aF5x$@Jytv1ey9MLt0_3f?wVSE~mAM@H^ +z2>qwq6mz3iXLqvaEbj`K@0y?#>TAX~b256K7qplm45V!1$`C>4#X_6Aao&2R9#q%5 +ztoK9R`2p3Tk^mit>uY9#AQKf%2zjc3n9+&v<&or0RvRfMZrE{KvM}NEjltg3#}%(j +zADfP`&rJ@8YiYn3!k*Eu^7t*3X!=fsyF%!s-#d)5{2$iWxba$9Uc8G8j_jk}5zRlR +zz7_`3gDx;JOrfnCt6oT2JiQYW!ugLfH7Eh1K^^=4_6Hxk6FzvEsAvzG6kkoQj-wk^ +zWCN-kn0gQ+@LwZ#z9Z8Suk`CgM#6|d8>Z9R>!lnB?So=sm87nRuurSDoByZcnmYdF +zL}7$n%h~Imu96AC3T=3313+eqSWjK7etUt$CF+qN`ZcEJJmaWeia)pL3yi9_S+HEr +zH#k0|XK1L_#;WO?eS7)wzpELKp12v +z8;Ce~hov7H(|~SYFz5Xi6oa{VWimhL(w{8gu)09vd>yP}u%m3d19jgya)O5+N|0nR +z7xm8Lv%hbgA~I+-?$k=ghR|qN88_Yboy|2szF}3a#-SkJVcf#%G>=e6IN%Sq*kT1~ +z$j3UtcT%c~&6P?D)@R`HIr71W+S!XBVyT+%?#yn?H3d*%gnm##>~XqVB$Uh$Hz4$GVA@{=~>%xsoMb}Va~Y}i`5N^7k^+W~YW6v{Q +zy$)1BdfI9~>NckqEW^^t&Ktk+p5o11D}?Qqkfral^6MYV&3I{U2eNFLk~uXfq2-C^ +zf_L$DYeFbR&PND`>ti=?%?6l^pkcCv)%#Rfd_fK%4TxuQ-8h-hbvBV2l@zNqI%=l* +zG`bKi$oA7QaJ0PL0L@1KLBJPbY-=1|-zMiH0djKMm6DNaOfUT{gjF5R-$i!9PvY57 +zsixQ}V)y~(-+2)C(bsLT9MJ{Pq6{#93FtD*MAS)>X@~g +z;Zb7K6P#2mhTL>#vuh)qMgt}LY!k$w^(sfR6IE(`M^z;vx|96=YmLf+J=^KJWX9_q +zqbRMjOzr3gU$9dg-n02^UYBZDe#%4aI|y(0{xP36s(TOz6S%Kw}FdM|zB_UJ4>>fHL+20uG>pgYz}Sh7-S%H-yt?MmPfvt7*h +z*Fq_tIE>#ColauXSN2lC-!#by^vzJOfHK0XSRL{%JFLTrUdp~6upiU%A{9B*KrNL> +zN1(7zEzW4ZNoZq1GAL1)MTdvQMRVeYj!&z{hhhKMe%$dRH)A3EqT$K^B6r_rRSHFzRWx& +z=lLM?u5f{*Lon+@B1855n+SZO3nIH4sWu^^@gm2=pJU8+0|i7Stqvyr=n=?&^NTZIcbe9u=3x0Ro(c04v()|mFn_nq!#j}etHk`NnHSZ#h8v6u +zf5eM0zo6-Z(u0WkiMv8XJ>^r%^Qj92cNH~l<)JP1_%g&1INvu?xX!LPX1+^{qBqS8 +z7pXqKe8+rPRc{9=0hWPv%Qvc&LEwjby>e9ESjiSu0|D+22i8zeU5!JB#p}tgi~YmP +z!*jayAD7`inz0!PZNCM7n1g$0z9jI8Bwb!PXCCevtlUHr$ +zP9fP}>oZYnIzzJR0wn?zhXl<+j0|JTaqySeSsvtTfBD2ttBmqVV*8=$3VF78dEs;( +zzkBWJ(;~+<01m&mf;M>-Zrc2gs;Z*YK+*1|SkdwLBIH|5a`>3>*=XpPIq4(|Y$F~a +z4rTw*UjP8eVEzDB(Jrobn=c23>)7^3KR!sej9uQbCf#1_7mKd^c3D0O +z<`|)M7-6*$DsFsN19WVIi09OBRMs~p_=xKzrnw+5dK><^(NGDoZ{Qitc0DbrL=JzHTKi>c6qn&G^rflme>@omkWvpJ +z)<>-Su~>8hs{3=Z+{f!rd$FR1ds717i~U&MKVFm)S5P9o!GvEZiHQFb-Id+R6YZAg +ztC&K$msHB{g1$PckFrOd|Fe9@rG9FHo2w=O0HH#Q@MZdl$Wgf{MB>}nvGwjO#UCsI +zIaJ*7h`Q`YdqbqN3f0Xr{o=WnUho~f)_NO6B*yIx1zheE7Fa$7Ck%O_{U(@-Cp_1| +z_&*j10npN;4W%t*vGsi9*z3;=M$Kb~B8`SijS~%m@cY*X%`w*ONyR;Edx^`si-Mn6 +z9cfljADKW`y4U-){Mv*u*~P_v=IYP=<4^&;!GBP=st9Q8?{p(f{2UxCrhi3FhH*j0 +zf#Ieq)FJku@J<3Jd(REJle+NOjf}bDb&3s&jwgM~?t%9}e=)p(4*ahH^1`hm%3XXE +ze%2`KcUFpqx`T +zroIttkUfT$l&P(UX_tG--w}8IwwS*KgIRB$o>U5i|9Omh-HF%%cD2W~=$Y0!h3ON# +zZ^|v;k^PkQ#8o6XMhrMIV!;)n9)^euK%%GC#zx%e{J-)4uqH8;yqD16ujjd8JjnJJ +z8<^MSr@o=S{r6apELUUxCvBZsu2PkZK(YFGMFzSFA_Xd3IVw`6m-uq3@>6W1|CdqK +zYhCbq#ZH9sm_H{3faP4p!vrB?*{&Iwm=3D=cdlERJ2LFloc&x>vuqDSMGM +z))QyGnGkJ^7?6Gp3TAq#BDTsAtDG~O#D%%x^?yg{rM5;Lo*!Pq9CWjM`FwoU(A*Zw +z*s066{VCxV5%>hT?mIbN_3ai7c#40Vb#I8177;>$?O>epub$k9B7fY^nIT^^C+{ +z5{it|vHyZ9zk7hKMHDt8%ed0*TRmQnZ4A&RHJ6c$8xo?W)6@YVmROXqu`6fxh9X6%5sHKt{_54r^sjrgS=P<#8e35BRye4|2JYzkLod6?Av;{v2i=ueG#Nk3FaY^X8 +zg-wCUKh-b*A0RCCC#<{%#Y;)jNAt%AVTqe#(2WDbOzy$F>ssL@eydMGZrk@v_ulJn +ze*K~yzC%ysM@lk4Tfo})3OtH~hrarx{}I8B2PIJYnVjzqK!$??1VcTvIN5*6!e#=U +z27eTUKR5(UzbAB{mmpHy9`2x^krdlB6}x`fDZ6zidjkCuyVjtGyt~a;7S;#EFvG#z +z-s0q>(ckEb5Gtqg`l#RRX7WE$&}_ISDP(vgkuzDEMM3LWppt)X1G~bI`+X#tx);p0 +z*WpS +zlkoISDBuh3*D56HHw_FH#1ys(^Un!7Z)}J8dG^4nqlEuER#ABs0DxF%7IBi|xh29G +zcHR?g4?twF!rgHdNxq(Ha`?3k`7B}Tnm_+465-bQ^!%xYCWOR^J^!T#HJ)_PB)`;~ +zdjknjV%^gmws3)bTjCcqj$jybZ>xpRYb`PAH-iGk;arDt+PuoTgj~Qqt0r@9g#!r* +zc>D6I%e-Wq1?RT>>Yr+Avv%t_Gt^ZdPLJs}KV2Xq)_ye1Oc87qn|{;w^blNordsu| +zjH}wf<-G^Z!cHk_#n`Z(22V}t&vM$7L#%h337f$O4H|qDd1n*^@&ucnaHy#FrnkCR +zbc5RCz|NA1XOFUbXxY1UetzTkubc7i>8=;o+d52p@FU_8sRn)&;wUM`rS`uhGdnIG +zL}lSFQKWZ?!yjsoN!SOSP6^nA@o#>5T^T#)`?w +zf(9zM)|-?8-Q*I=)E{UDemY$ao%sFdt0`&B4T;a#{*5ogm8hlgWXS8}L_ +zqJ17^wa_Zus!V8>vBQaU_3QrQPhZAM$g~rSMYGmP&f%pPpCa23VZZ&POU|rK=IZLx +z%ve?angKbVM!#*_u8%c{bdLe=ww^rkrttCYMILvpv;F`avXFX^@q +z4%{DHdj-mwo9_8rfLYNxi$z^vEhch)lEU*kowi?}%0xd}KGt~rI!^nfr2jCT)RDhY +zJxZ6tU;ft3`q29jyjbILceh`sTB~f_&e(Ux`6*G4xi4K$ +zW?v959=w}iJ!Dcr89`u3Y81zU6yiPg0S%j%bJD9=iJ84k>=%2nyIBm{Pob_aV +zr5305+C;&px1jKfyzIen+!Oj^Y`<9PK2+`5m?mxfa5X6YOEYukXPYzr%cuTj)dA}g +zEaP#vMTTQKNbEo7TS{ykQ)@ +zF)e7^S@4S}%3=Ypc7z_JbPW0OiQ?L$*pAXH^0s8J8c%+2tif5y?&;z>SE}V0lGy9n +z{SFaPM~BUW(mg_`v>t-3Xuk@OaLKW$Eq2_8K+Li>KCJmbr4v>P +z$d|;>Z^{gNwGf%qC4Z3HRIDhu$gt3dpX13ss-qO5-u#^KqEVHfB{OX=*35VA?qaHJ +zn>#vM-RtXt;brr8<+6kQ!x9GH8k_IlmMK2lv@sdGI@Bz2Wixi(K0cW_MCZ9}EX)g9 +zS;@z-OxUb`GeU(1U0m10R)&PAYb{oVnDF)y79gXp!zHzHx(97~&W^rjl-? +zE~8U6A(a3x5qW=Hqp)h&DQ1e-iu1(xsJjbZKhAr?Fn=_2r@L*kJ?rl8J4rLaw_2B{ +z-V&yxoYhXhOm)sJMMXI#gB8(yd>HmDxfQi1cZX~->T6ei)zQ@9*lTPQfac} +zoi#_>_e%Sx$@){-*W{*|@9+{2o|{eok%p;2K+JTPku9P!>nP1Y&j@7yAloSeI0NjU +z5HY>lU@LfA(RN|a73G|2FWl`f>60b?ibvNbG7k-j=ik9csq#W64y}!qH?LUf$kjmd +z=}^f+Mn9!xUWZ|iLJ!xESNfjXN>vZSTck*cwX`SFZ1l72fg+gA%?}vu?%ts^<|U#} +z_k2%;#3SIeXB_G~8b5nf@EL{aKjG^%#zW~f8$V>}y5vqz1}f-M=m>11B7Cb(d?ZF5 +zr%KX+P<>_b5GL9~C>!XWVtzyZd3UW9^}6ZRVmNO)<;ZzPd;1|Ng-&ualTD`ZeY%mD +z=#TBWzmt_72@Xy7+3c&|=?WnF%&6Jo +zJuTc_cox^Vt$#WXoMwPyE0<ZO>MQ7WR|IU@Jn*5%F?x-%AcVA-l +zeQsj0l1U?IZmo9eF(`lcBT<&c2RwtoP%O0$6E-8KI;zWP_NTEMIc6FdUQF42=lKR +z2~_u(IX1Qe9oDXS41ZjR$eoC`_&HNWS}T=BOEF2j0qgsN$mI>f+lC$$GtO7Z>{bHZ +z0@{Lo<-G8p3bqtT70rOL{g*E<l*}$h1kr>t~QxIAAWGxl=Sdxo4eWikthKY-8AM@|mAT0$h +zORXVd@M+~Mk%C99ss=NTD&48H(Yp@05_wk<-@uS#w=7$CDYRMg7C0EuSc-8rV$ADE +z+=H#O@2v@V+Nr3h7AmzNwR(gYZ<#4yOjxFPIQZeLwb;S{(8_$d2zDB!0-ZGKSC}K& +zK5UxBGeP?=>o8jo6q{$3Ez$RO=4dgnqAeBAHd9VM6>hKbE&5P^M-yr}&OfzW#&~j@ +z>%r0sJl6A(%!}n6%5}?ui%0jbY~#J_mgGe?9Lh@AB02bt`}wv +zPuyRNQuv-pdp!=t=Ft0VQSYK;OeMzoHLC3qY=6o+@~j*NOEhQ=Hp((gufmVmdN4To +z)AV4IPffKUW<9KHovyxAUGopkN*XIVUF~{H?yg-s)Ots8i!9=Dj8G5DTaLA-ynlMA +zU)xrtOgv5y_?*&CBu$Z=GtJ&M3yk?(;1ewMfpNt()6h08)bq%b$k6A6sc%G99ywb +zb^WnCu83BI;-t08>*)64;Z3Xe8&;b$_+=4hjG(C-EP{9)K@=n(O^5Z#r?C}J$M(>GI0sTtnL7e%X2xRCr7d=gTwk=h{6ccd=tsoI#_g*}|)tnr3oRS>lp?VQl7cpQ_0&;1*rUa2&QjgonYs$k=u+jd?a=*!)%V +z$9yh8WiqT_LX>D-+{DlrAhLa7h#x+czJC1Wl}>XwFo1rAE_idpRn4+y63GtvVFj9U +zF??!DwXeGKWsCV%Mk)xp^Mna4Qbd}^cXsbT6i3Xg +zx^E11MMq@lV_PbcGbx&r2HYQj^yrH`P%AbYr#S-(3XX +zBObNKbKb9|4S!gT<1)>SxMf{O?8b{!48q!Q*O2lnlG%L0QOanGI0!FwcyfAfO+Cx# +zAW<0f%}cJ8za{7_;Y%>vnl=AySF?+T^7~dTfSeVIHzdS}h&x6ay^; +zO70_%E&3R}I*|zGcw~W#&3cT{S)_R5vkRmZR64j{?h14nLkAIID23m{bKjzG61W$EAUgxYK=wqT`QFm8S8v%Rw +zP>{e-A!3jBz-^g{&V`o+%sbh +zJvfZR=ST(I_XkS3U$k9NN|;_<)_L?%F4R0xA1=93(YR9ZNtZMn-b#E(DC|PVrpTs; +zqlerlbfmA~v>s6(&Up&a)E@K8cw%Q1BwImu7A{ey7p}x=Pt9%AJLlp}M>f%u5=T0Y +zKXZyAok{&*C?%BTc)RBa>5to>8-l4;^m_L8Ae=$iiV%S;&G7Vxv`{h)w_DRqnBXPp!Z)jW6IMV;XO3WM+(tnoNrt1v@W7Iilq1{ +zCaQ*^yCe|Zi<)zrTgg@LTdIQs;>TJKq5ILZ8X(1b(JL`N;Hem5&@`ARx)o(Xu=#M6 +z<{6jD7iK?Mv8~nD@~7xh&%(l<_43vU62Zey2Kf0wrF+D}-1!mNFKJXWIQ27LlFOj( +zn!|}pN=j}tT+hG<#Ay)TNcK2CY`^@vQk`_m{$EhztH=Dq!PW@|~0k#d; +zI~8?Onz9Qt!j`ST$QHpPCvz6`Ue`M!-@{l9X^Rh|WMGPL{PTleajubE0n2RCzd3M|5V{KYa?w{<(I +z+jr3wzZiX&;l`^Ty~~;i8c+_3cLE@}f6RX?U=$YBPLrbLFT{Bi?AW1aT*3sJqPC|< +zBGk)>7u8G2EDZwCNHl9N`_j3``1uJ)>Hh%#RF!kIxbH)YuMN|E;MeO}f|Mp82=x)N`>Yw(9jK)G--0z*up +zPC8wDqQMEsWr64q{x2aG54|m1ZjY95Qb9thV=zg+8;$%2iTxdxTYZ(k5tV|7#MVU-#x(I%!ezC +zjc?2aR?j;&a`#-Va|cNw205sIR7XJ2m6T%>!W%wC3yHOjYAzYrB?%)0hRjA6iTS+v +z6`LjFf~o$^c1h!V7F&j;OjNuDmX6ps>~sD6ng<@X{T4LR8*&Wfcg;IDZFGS$;1xwC4fT?R-Td%i+VMKa6!6)O +zAf*tn6yjFNgi(W*aZHJ9&F^9}a{~LuaNrq~Ga6FmXWz0Pn2qYQylm@NT`ZM#t~zI6 +z)@_jnFqm*mV~|LWq>qj?W%Zk{&FJ^d?X-2tyeNFzt#>1S+i|_KSg`n7Y{Z{D=%) +zXvZkn%_ETeL~y74p2c*a9>t#u9ru~bCxZBtopXWI&_F$ +zbOC8__at0Vqcezdkv!jDk|dvu{_IX82$hZ`3uG{)X%ioRF$G!7FAIszq<-1er&6c3 +zA1x~(g-y}+O^VhmH6m5}aK$rXd(;q58qf4G0dq1WCf!;E7b_;cnTpRV_sgOhlZ)^* +zhdfu<+U&*2fL+gn!P4xSNu~tfPN^Souq49cB|ep{_)mKv3o$)_K$G*!3p^a4h%zo_ +z)ER-E2qF*aAG^jr2+`fE@uxx6$aLbVtDor75IgDS->19egrcM8VJ}G7x +za?9^i7_6Zew%K~B95nV;jPi&?V)U^Z954SAODEYKXlHHtMdoMQP#ZYcPDr%Scd%|k +zp*(cS%SRI=_59HXk?nHe6?s?@%3!&eHQ-#B`xc7<36b^&QyrqVoc}`Npijz3$tny* +zDrhaQw*Yh(&W4DnWbEhnOmKpNB}c4YBl-OW=W1}a5OA6+bd#&bFFPkE=Tz?^+|Nkb +zND5`d=;gPD8uh5mknpOWIP%xL(ovW93H}^E5dw=p$qBzpmC9-zXN2$Y6VLnh5wU;K +z#vx02=!)Zop>(Y10~y}Ih$E?m%GXx4$)Ns~ON&NPvWEJFWYXE~?p7<&tpPFb-w4N-@8>T*b|jrCi`o3hYWCT +zzn;xwb#IcBf!8WqlvfqGL(yrpEZHnRoRNGz30564cC61lz7y1GHmSm7E6F&J7w*>)Jq@FMdUuca8!q}52jEg>W+jzNV-##bl}aU-X|4~7z~EkR1Pl- +zdftRYsc2!4j`a0K`Zs$Yv+reXZJX(jZCygE&2MNMcrkL71+gY8IY-tp6OuOyUDBX~ +z&6gLX*wN&IpO1uZ-YByUNW2OmEWgu=u|+~0cEfV;P{}26%dB`kdPz(#PYDXjgYVqK +z1OnlyR=Mh2UqIfD?xu#3hF^B6dvtJTtonG%1TOtuL%A6fmFpcG30OZ-B`~@XgT(O}yUop2;RC3Q +z+V>ei^rx%8Pyuz)_}x`$J%KBzfH^IyYahg*sWBX2MY-7IFr;=C3NJD5gcP(snRDM- +zqkw@KhtOS^DqvL^XP5M1WdeobDH)5rMXJk+E_@vBEJk3yai88-eXrQzOz4JujU@24 +zn^W@o1#hvukae6{e&}sGS9DA0C`^ya;bg2itYK?~Wmi}}u;2{I&ZutLEfNX1-ZGL2 +z_aK4|sU+ZUvoh{fdiDha`5oUidH-|1(bB`fSS)pVkF?fqS9woKP?7j8$VK#_ +zRuqN&xYW-U<@eqxkBX<@$z46xEnT+HK~onV18dL(FhOK^yt|#!#A`N)R-T=4lAv5G +z&Q68vJ=fWGN&=Ifk^$}QatBHU8DgwzdwSWaU;&E6hQ^kE`!~DvLSY~m>~jkb))Zs& +zITmiP=ry$HSom=`T1&a6Y +zIW5v854~E343(6XYIRaU8-a1jhuJfOvbm)WEs&}9?p~)uR2;K +zi_FI;!-#G7vTQcEBuz9}3|j(!D7lt_y+F2Za%O8kqn#TrvED5=Tu?#?S&H0_BthCz +z2$nLhKa3FZV6aY}a%%Bsi7$9wa}{|f{6D(hGOn$*>mDQ!Ab4XZqla +z0qqbOL;f+jF6G2*MR)~tw#(wX<#TzOYxY?=`tqf1rbX|&B5*Xis;Ca)yGC{o$_5k9 +ztVXYL5_y=umbl_X%P;RFWRzzd6gXzAsSJA=X=tO{AMHDCP6Wo)x9~+Tyw{pU=8Yub +zJ@niuEpr|%D-e+|Vh@}4tY13FYYwdqB7nYaZ*WG?am;fFkz*aCPQMTlevyzp=5>h< +zH|tS;@CB%#3`wf+453SE%H~06LgTv?0QTxR9a*zfYWa{b;hoXBRF+ZEm?e){NfyZ1 +z$Ic9!NwWueIuIj7QX~!&`=51H1p3m;^0RYN5`7%eG8D)=kHf^G#1Q+58b$^-r3Wy` +z9EQ4{|9SIBx4hL+mssCiv(S9ZPb==0(kTY`gdF|_N1Ub*vk%6;@qqAi@4!c`s!KQt +z-w3q?n|-s-3{bt_0C;BKO_qIbbfvvxNiaTne>k9?=3~3n818u@h#KyFCPcd@U8!YB +z)P;tecGkEuXg^i!7OWh+wl9)7``+{VTKXfY2$#XkUzfA4kGZ>uy`|~ +z-S?7=_=lGx%F4P%057q<#Mh;IePweb5MYejjiwZ#>bHKhnIBZk;9_4nENY;^{E7Px +z+{@M3tgM_mDcI7rtT(n69l~#M7~p?@(3zTI-t7AOrZ=T~{_mvwRNl>KHoxbP%3NFj +z(ZN`m2*de-9{E~{v5La+0*7!hS?Mju4PumiWwSUaTZBqGx-DRYKF$}=&E>H>Y0|1O +zI`6n8$y6PV-3+TD;`(($Qf~zU!7B{xrq9Pc4+B0gUoo+sx(4?P14-HO6{{5DU&qUa +z&RIOwKQmGU54m4k8}>#Y=uM=l7FUOdBP)9u{n;1t5tDS(eyZpm?>6E-D$j#0^{-leXGn3HS2v* +zWMA7QVuO6(K6{~sAziU+%CaixR!|HpY_v_~kLt9V!h5-vSRB@ga!X^*9e^6;oD3ifFUWzZe=xFa|s7W(d6e?Stlq- +zky-+s`Z0CjG1M@DNB|wk3i;&X1rlcMreOXfJnZxTmOh*0%&Iy;3ao_8k|uZ0>yZ_6b73L>f4##6KA +zUI!oayDSegcSr!1E4zV$hNK|!esv2i0q|d1ah`2fsX^6;n*i{$=FhxF2^1h3f{1a; +z>+V`$H~sSlQ^sk10!l9B%6>``Mob+P#xQdzOgf1JJu3=@QSws|<6cQ#?5pTnw|kC+ +zPq~Wp7o!{QVVFG?f=yQ?xNL#RQRi8jUq$IaVj)*(VX@t4a)GC9o6kZy@E~Q>O|<5M +z0jYvKAl8_Wcn8vGwK$>kFM)`ZKf0x0@OorOp=H*9``4|GIVMT`wopcqW%yKeV8swB +zQmG;XLd6Z3>kcc+Wu5?x&?!qP)4Zd$QMpGQbBTgQATaRWC2zUP;FGbR$KOo9)dq|r +zEM6)?ard-CX5Ia<*+q@zF7wES_LW)DitagES0AQRN!WgXF?&Yj{RZ}dS-`SB*JR{ZMtq6~iNjuBIKLAvG;zHd +zU!RUv+&#tnk2EWQityA6N0N;Z74??gR}NVg+`Q>h#qf)Yme1hH^!_8RG0x+|t}VMS +z*1UX$=S$LsgOU%@FiWP9{>~ZpVOZ)h4hB9wdjju-{JFa)aoy7m88t7GihCT+RZc&{ +zDSqrs<-n=`9k#`$m@UIre~&IRJEcv^QP(;!Vg!z&2=v+(M_4#7o4}LKdCZ}(hVRhKy+&e=?uIat +z;12tX;(`alW_RJ9$6^XzDPO5?D8|_gBlmcRV5t{JWz)l_QF@a{CJD)?S(Ga|ET2j+ +zqSxg}lkR)S60qe!cB=F;+!TLkyd5A^6&+B>aC6ifeJ&l#vjq!@*@e;+m{Shob8lVW +zzo7spbl3tc3V~(9K%inErlJaX0|hmVh!6sKO;DYApi&2=*~)N_M_t9zgm(NDi=QYD +zIiK)*hCEu*&}S%(%&tiI4pm*c)j3h(fM8-~&FaDTkg0&Lwif8n*&k4vjnJtp)SfuL +zEf68YB5^mUcwq@jQ)8;jDCc8$W^2g8msbAyz{##eED;5d +za$44>Iqw}t(g#w)DVaiChVzX@Ogw6?HR6UfWTJigjKQ$Oo!ZMgP+8ZOySzK#9;N9e +zaH;=C_<;&1%o2w6Xw!PP@A1hgi}4u66xB{O3ma1a?E&l7lA9;aJ-T9U=W~tijq|er4qC6k!IhEZz +zggf5L23+ls#hAR6aE>)husHw1JPn~sIF(-aV8Af{S$dM;5l;?|htdG7s@~8HcBSU` +z{HHp?Go6Wy0DySpq;0B_ul5GsBF!5baMF6OKXWW1Ikgvsag$!!48%LSw$;--BIhpN +z(iThzlAGSLIlf+vZ+kqMh>aj@0Kh(>U`DZG!i)(hnc5YqHei^onE5Rwa@~tV-;^{fIO*AtQ#FcN-1+H{yA>o{RDvFHNpP?}{Y`D`lVV=T +zD55R$-9APIj&Vy+^nhl*dUgP9zQW1qJW?uWe6I6a*X@oWb5M%@a?(u%tzC+f0KhFLR +zT<1=UaxkoSB&NO=wZ?j>*4-`m5h9F}SzUtcINIZYmZe$s!>qLbEC;Kt64NZbKW#`R +zZ&U61+4#QR+DdZj?{s6WB>QrGO4CV*_G@3|2x2W-1-TEt5N1nR61i&3IrL&%D@sXj +z51;!B(pKLcRuj7phNZairu|K5B+j@fi*&yjuO_e+jZ!t?n>!73)3)BxL(pxzHI)a( +zbT|%-{t+@P&(IY8{4{uRvJ|*S!k;4JUuwiPV>blogk!I(sn*zPsv9jXSOoz$7`n; +zus-*V{#!$)V=z3J(V(DU7|hNiVSvrA;{tMT8de|jVShI`(>j=Q+(8lGi6?U=PSf_kou +zLc>~p3agoFJ>Dn#(kg}bZ@xaO=4nx*s8oEA0z;kP8$ie40bhcd@7FM{@jvLruiHw>QH#PeI{>;Tznm9% +z)<`ec_#XPSPrlVDGFK1kbi8%Ku%sy$>Qddh@25_=y$%I|@?=1mZ=(q504>$>=N`24 +zm_5Qh8)-WN56(N<8=+08^H*wd36Gh}NiZ6jGM7AfOvvfj8UE9wP_f!$0fOmvuctuQ +zcHyN3FI#4X3JhxF%(Itz +zR|YjLLY{}L&%*rH$9ahOfGBt8s-J&~y{%&4km7;ZA%BwOfgF&^mWv>)*of~fk?G>H +z=_{6Pv7r%&KMoO+qlZkNnYc0BaLLu32p6!iP(Y`rRGf^qLm}4&{t6#um!v*DgFU^i+~N31 +z6n)KZp-84kJivMq{hmQ;n>s9Xr7Hc*QQ{oO{8Na&awoQ`K#x%otQ~#u)L3PW3l&o( +zB&~o)ih31EP+>AQMT5!b#qF?Yn>rJM*82g3sJe8u^jCF1P%Zzc=HFz0Q<6O$VQ1Xb +z`SP~uO#~_k^c-`(_>Xnfs~@kHj0@HzkT}GGnLm5+{eFD>h(-Fu^(i&eS0%N^sCkXT +zZYFRtQ7B_CqxgnlvtaLfv@J@8ak|$!bPjtys-^AKVl8EN#8>6t^bx2uy(BRx^g`!_ +z-u#cE&MR!kQ~VCDydcVxgv*2Qo1Jq<7>1MVqm>NPSSKRxQx?eT +zO+@JZ4{MbyNqs|n*1jAXVCFO1V%1F_fXDhWzxP9;bojaB}_Yc)tC +zQZr{P(zB~#hbg;kd28qWU&zv>Yaf(kl`7*K&?+ +zReKrp%of?0j5CiwXg7f!2rEek83qdP&#J?c*ZtGqV3}bGQ15;6F^l1Y46>ua@OUSZ +zWSWd`QL#WrZ|pJUcTzF8Ngo0s5$~(&qqV{XKd(#vc<661Kf5K7BN=DjzeEKB3oz?I +zvZvuhk^*CEO=&HWb7A7AhQnk-AEf7w*bB1_L~iNBA7E|pDbZ@5{iM~aeQGv6<7*>- +zoA*iVTeOUWYf~~Lew70|kW6X;0&MFtOjXFx6&lz`E|AX?+&_ME+2KL2Lv +z`+z`B2*2AzT{wl%4Z1abA8I)N=Zl}xD2Ou#s^hU-AB7L>2@ +zm~hTNlh`>Q$1q*{+E3o#({0iYPX-@H@I>xxUd?kABUs+MIVfict};CLHJt?hN@UvwFa +zSPE?F!cG9Z@K8!8)q=jwftImJOFR_ZNaviWRXD2Iy1uvb&-nQGRVKUx|6WBBpWP@1 +zWG1o3^6Dw_K4{lJ>2VKLdj7a9J+1^ +z0cf?v1IgJmi4SfXgSEW_gCStpHA)@IXVLfqUPe@N7}0av{YtkO_-ENm>i_$-=75;@ +z(J%c036O^Il$<%x_eE0F>G!?;8(ylQ)zvX{b7-vOti&Gz3bAreB!Z+TgQNF{g11H2 +z*hWua_j;(|jF-|>{FQpko2P^v+%*>RC#2H~9~8DbQt^pBjQ3z{j}B!RSdw;we$681 +zWA1n-Saq^Hs%6BPqE_z`h%w_mcCzE^yO3LH!V%*n&pg%LYr3Car>wGK3f8V#Zc2FS +zc_K;Ysu-^%fp72!r11DKQEuS5N!Dv}epy-QK!YU-EyeG6y8T}Ch^T}uI*M3FT#6(0 +z7f)MM44ARpl_{N|gVc9FgMc=I)F;z-0}8nK(0d&Fg!qhJt$zQXms{nZ2;VJBoR|*% +zu-IRJ+jcfwz5eM}N9g}1gEK21MltZPebZAf7~)^RbS(S##VSCP-5W~84c1sjLaw@#_OlT@{sVW9H9c4M +z9{g$diQub1rM&zk?jT=wveZcvMmdscHvje(M$?6OAN8&G;VqNO68wtqeG{`LMbS{h +zN>hq?i91%VgDAs*jadU-fjNLCfEXPgH|0SNI`Es*x%<>V<;DT(NNJ=ocrxeL*LS8o +zxM9v1lBhG^{)COeY~^q{&N<|}5$49WAe=S6^{V6y`!uH!TO!YaIM +zX~PO(L(9L4zH}ti*_HbeE;)irqHB`)@p1SDhw*CZq5x^t%CZ-D3(upUrhh<^Y67E& +z&Uv-}A|K1KQ%vaT*k5ACL8A*ZM6i;64=<-@^AOc(+W20kY>og!;@VSl{f+Z&aKbzm +z)>^o!-1o`rK%#vCV=CJFSB=`Hxc~VqeM?Rcfk3s%R^Ja@cq&6xy6?y7HOq6Ra^5U>MbTM(*A?2Cdc%LZ|I>KL +z|FhN}Fu0iaCt;I$0zHQRIjS!yP)1t4r3tw<@6djjIlVDX0)O=oKe!6*WUTL0pLG1^ +zEHX?Zn`>e&)AMvu)O4V?ZW|RBW9M`!$CLZP*ar5oOqr-6q>i&_*RjKNAhs1vg{>u& +zZZ#;jf^|;CqO_m(fB1V_3CNC^u&Msr+FNuzW|-oi`D=^rw7Q5Gp=l-jbbN39K0*DTnVPlIm%tM_g|y9eqln +zx|u?89cT9u`HQP5C^6Q7Mta7DU(7!3Q1NtbTgmLMxy*J(NEZ`_8e>~q8rZO1UhS&* +zXD5Q85`orD2QCrHW&kHF@93~aRzHLP@ksvZ5eo~Ol5*Ey>WHU3K5CGE_^VnG>{(9S +z60BzZKDxuLO*QzmokfmlIJPTBhGpc@!{6qC6b{!+deA%HYaY%ZhHz!5CVo-g9{!s= +z?oe7&bapHMpVpWQ$-hhg5BK8}N|+1uj&XdWB*|EVDlbFs=`&Zt&>54zO>lI+SUL=% +z`t0{wdCiJT)d9{Td^&hSvtm}Pvvf25YWU;Ng4-pNiz2SfFM0$m9&LGzne{X^OM-n= +zR5AbktkM6)Hb{8W#t92CztkyTQGKA%pi0YDdg{L;l=A3M|7ZWJf6U5X6foJ~i>ZIg +zd3uT%dL&Ch@F0skybp1EnHO*qd(V*+r4o;N_OW>h56AS~3+otBh%;AhGpG1WGPQlw^j@CPt~cf_O3j|3@MFXodDt~#1x+=E`NJr7nhy9?HM;yLBIO)4Td5VhckRreP_`V6JK}-NZc-Z{w0$!ox72V^>BTfRCLp?5D+*UpRY{J +z1edK$jHb;?D{5N_rw2kChOlT7b#214!6E`bjRN3y}Uwl$HDSH==j +zSV6K&#`vE`>yr5fbI|loV&4cct>2tIhIb=J&n_$FLI=@M(n<~ax8VFMF*WLaHDlI>>J)%lt2@Q5=lD^iev_nC!wl&S7Z>gZ^|<55&T +zBI$uN!uwMq3w>rIcb5x9`r%XQIlsb3+YV=xo!vLdGOu5)E^h*T@0GUCNmFNYHSsm+ +z#PS&Gf1{$}Fb%E>aVHupCp%UnZ&}k>rh=NgMAr2F+t~x+=5fP)mEbifxdH;8ncneY0l8Z)Yb6RA9APsyP<7scZ*X->hf}T@4$!x +zI)lfYwEOPy!n`_dm8m1}Q$4PS6lPCZI`lYqf$G0GLLoXZVV*Q9WF4C%xi@prx^n$P +z&yDeR+Uy!y>V#&*O^}RmR>9yu%P(2;Wo)n0**EABv6UR~fJn@p73$sT_lw-N@kQ&G +zAKC(RLkRn~v1C=Wx}MRQr5GnQUlg@nkq*5EW9$iR~$ucTca9V(Mj8X3j#FY<_&~R1jn2h +z2*2Gen?k7<8=4)DH}I#=N;48E7eUNsh>Cdfnu6xr-A53kqqAxD0^?#JxKQ{1*^~a0 +zhCt|$-ibe_u6329(|kl6AC$Hu&B2_i%FbeFMwd&YMlQ}A#6GJS>qUi)&V(qYBvG#5 +zBvfotz3H)*(JK25hweP|&(;z?(^NTef!C8+l7WD|I!Ps}x{I`en;#gk-lkA7sf3Qm +zmNrrv=}YG3yP@W`0?bHk-;iI0ZbQ)-WZ-iC|eeZcRFfy>LPuolFquzuvj@bC}BLm=8X +zR4xN{Z867ozM6xBzJ-L?i=L^R^6#Uk9{q)`1t-8!>0JIS9bWYP)V_y5f3EvSSIcRf +zjSfTz=yuF)1O9iP+IRy3EfNm7eC9fT>X)$CYVFVoS@Tr!voLW?_u?BJt*l7jTWX(f +zjnBU_*%U(-<4`EfYWHrd1?kSq +zZrcox!Jj-@!^LVeC|b2M)n;$bCczv=TfeU*72yd8buaJ_X{B3^P(cCkOD3HY4*0u! +zP3^0-XRi%DwMj@p2st~k{1O`(N*u@g2G!D&P2nmrOJ>P`+|0uaxK$$4QNx~g6p@}6 +zS{BdL3%=#;a;uOL6f^dqD#)_$y-S_dM#}P+ye|az^FzEbo0|9m^^+Cl$wRM`ueVYj +zuy`*xX4KFtDw_P__~=V`vI&EijWJe=n)bYXOqYYQ9WYA8MWvInbj;Pbw01J?+rX$( +znaV&Ef{|C<-L*duB-d|U%*jCcm{tzzU;Yr^)w~U+JZ$WQ>-GiKq$XaXDw~lW>#x0b +z74!3m8yf8Th4Ppi`k;{{hWAq=f#cuweo7^@gA`1%5l^V;Xmyu538!SFqd@IZ1%ZPd +zU>TVp&?Fxp5K}Pu6-4W9>j-j_ryt7WwsW#3S?TQaGSt;CW=SaD^Bh#~~wDf2pM)Mc0$B;^wPs)mAy~RbX;nkYF+& +zLdDOsPco68YsOJJA9YvIk0I+$UubM^Fyw$6+|n98XjZxD20G2B=$2M3KJ5pb3DTSd +zgc4z2H+}8`(r`h3RMhILID4$f^=*;RatDRQHlDIJyYA{33+m}+qUXe7FE85UpKsd|1SeZnP6WP +zz`(SmWN)DHYSdFp+~p2I_EhKWk#Wf~(-_{@`aJki`lzpy4`$o-IVh_6{P4}Txx^-} +z;vq|9m(`AEZKu9{J2o>G$PMG2Cr;0}@^k8Jy%$=?S0T)wQ>1dam_cwZW!5vBZRsTdP6W#k;uL*J@w1!aPZ +zLqB2*>Mx=^BD|3DvLcf^%}@1l+s`_JR?hrXQg(f?wke&a@#`o1ES^$48m!=MpT1O; +zo$2n*#Fs;bEe<7F4_P==urj~vcau_%YcR7^9Un3^fw0d2(q)?QN1fmX{^(9oT^ruw +zjK&|YwWU%K#A^P*cT`cJQ$bXe>N*d)I+F66go9azx%~198yP%Ew6~MX6EcIYn>M +z^U)nXo~y;>hTT9tH^-nk{7|wo+eP9;HZyXC^Yobf$)t}~dA9c=6;{%k-Kj`_s4p7y +z!EM?TX9u~nY8|{cdCn8xO4^Lm^ZxI{uLN&j=q7)+j;}C-@D< +z%ecRw@m-euiuP&tu%CRi5-#?pfo?D8V&k5{q-ADA8&hsIs^d(NfW6NyE^q;Frqg+g +zPCVeAh*ruk+Z!$+H*3pBSkwi13Uw9ZG5Q`kVd79RyJrOD6~UEf;f}ajWSs%9hLY#F +zOy#3h*iB6dI5fm1B~%y4FZs|j$x!#I;FMVA`p^{LsQx01KiGNKINVN9(5(1E!?f6{ +zBG-Pa5lz1IXh6NBigZauz87+i6o6ipTc(xbZTTNf^diN^ +zfEI#tEg9$5yrj0{1&=%oT$=tHood7iU=n>R0B6tR&viCqrFjG~wAOAh%C05i9CK>q +z`_jSv4!$#&zCA0k8C#m+a#N`#``$5kL8tZ%LP%hx@d&n!7ZJR$kJSJ_H4yi_FP0<@ +zIj7f_kh>-PMo7jq492spb*Cht5YB0(3zC~{Uada{oGC9f-mxdp2}Q1DLb??!hhxY8 +zkv_8oDZW^O&B?kPzDuR5z%zN>kja=l+= +zb8=KO^q9$T*^Ul-`qDCLHy-tok(F(2blHcw_hdlS1r_O6%o6y{QieP@Q1e0iQ{eE` +ztuzWXz!D`RF#mZVFJ8@_3sKtuDmv529E +z(z*m$eb}QoTS~IBE$dXnLMI#Mr&2N_4XPIxyYD_adhTi)CXU!PlHMRrF`uc>HqpcU#hXr~ZNjZDa +zHdr3x|L%7xMQrFdIIQ5i*A)S+?0?>_c;;dqp8qtmpUr6Zd>XE~$<|k)O@EV6*l3ir +zzuoeVi05&+4Bx^2hbDSiIC7{S5ygqkysp9Is^w(az)O$)jkL9yg#m+YORLA_NPGw9 +zmR^h%Uc)AP@~>-c(PZ2PgHhI5Q9lz)E{(3HTF38@nrF6qy5-_$)IQUO%5ID1lyx0f +z*Yk7UcQN2vixReJB@TC|*&69!Tc}+xsT|p#o!uY>N)K7OLH7eD+0amMgiUHeSPE0UYfcF|9E7xp#c; +z+EtQy-mj)*|C*xUD6Ux!!&yL>z +zpr#V9eXsy>>wB|`a}{WnuJ7lsnCW!!$!;C&J{A^As5XrP@~?MI27z0)r`@bPbAs8h +z=#^ID?LMa<%#nN((+g&ZZ1WKZteBEBcvmt=f>{O;C;x#E)qoWTvnnM|*?~O_Ck2vy +zAzvyGWMp!~2&Py%2@y8iev~R|3QHmar<0?CpCk?=!R)%I@?geLBz+SCIty761X75~ +zG$F)HL5y_foC+Oz1t>5Kb_LN1g_X+=)6@xKzDkv~U4Za5rFVEPSm`?KoYZ~Q#YBdJ%pPiw<%g~BmJ(+ +zFy6`FoTq3h)D1)Nt=@w}hi~GBaVh-|pZsMmslh>!AqH`;4wDYIeI}*!U2nW;$>`aU +zl>6v@H2jB>m4vMkVr-~WO3*Cd{9>N3hQgf2gUM|*gDyoZ2F#}$c8g`b*Y-Xv +zI}pdEF97>oEFl125u>kG1=sNlIuH+8BHk-@0Ro$l)|I?#);jeGG8#dY77eGE7|QR- +zDiR0?cX2j*bp_2q$hj@Um}J(!v)JsDJ^u{8pklIF6zTMQyq*f<-wKZ<1W?@VeCWM? +z%TQ)(6`aNS>4nxoNRgmFmK+K*`CT1rYl6)6^9(a%+7`aOY?&$ki8F@2O76t)2AgeP +zcH`$E$S(S;4!H?8^{9Rp`uEEfwGB~|{X^}VeJEh(c#_SVS2s83v7fdn@(KJF8jrwf +zD`x#dw00dGz?zK$F?qiQ)YTHBc=B>1<6~vmnB#UEo_jC*! +z-{i}r6|f{7eRZEwsjiWK!%JAFa6-&k78_dPYt#cR8yN%p%?ZQE9nFxc)3;Z2s6E9y +zMt%HY8{h19^a#1SD+wSZRzx*S~?;RR)#%emo7axnDJH|)kGC;npH76a>MHLE`TmKGi*4~XJw6oRm +z-QC19Y`17iHp!*ltC~JS3UsU55gZAWGPxOFfGLkz6Q-?lFvm{MpJWd0XCcD|ESc@m +zQHeRS9|<}N>k?*LOn^GEApVeLwTwXHK^|HHyI=QjX2`U>!HnMYl1$d(nMr0Y!79ih +zF&ldqvD1F%%U($On3}b9V0`km_k?EPmfBf^tm&k28;7!RJTzUmwlHT@pmf!ZqC1t1 +zA|C?B)NGo-jI`h(C{Q5grBE#@gX~@Ej}c5RU&7K +zjYf=6Bqals0tV~IWq4Z4%4+3P4t&BHwx?8KoF-(T0Zs_DlBmEmG1@t;p_4M-tCfBta?GF5@ +zr2X5Lspq#u70;|HYDRpB8IS+?joM#iy*yu`5-(1nVw)t(MY*CEtWQBS=f4_p3*Nya +z8=~%78;=wYK6B`>MN23Uj>aH(myEja@MF`IwL-G|Hc`Tn}7Bvh*i4%`oO|h@4@|J|7MGX!VCe7~QSic56(mgY>UKny-j^lLk{^}S&52D7Yvh?1jS>Wvh{r)BiSQ-V +zvo^)O9gPlz5YrSPB-1+$lxUSMPedj>G#`b1x8I%_4{hG&2nC~?Z26;CulaK`?VUn6 +zYN!}aZd?1I@h=3(O-R7|;5Vdg4mC=drToixkWY+fEy@*0*e% +zGyZTpl0)ul>bN|C#fZYrET@w~UZO33(0xSZwX0 +zS$l*mxIY%lP4E{8l4P@)Ayo~Mgi{*U5wUnHNS*I#JN;Ch(0oKQy^P0)L;;3K6ifv# +zw5B^D@|rX*lwOenye4dW58?`8q-hiZh+~~crgPkj?m1XUvc54>sdBvO99Dk9 +z9U;X85(%b=ueBYAs8uYqJ)pe#8BgQ}@KZ9F6H!Y$zyYIJGbbLA3?P;IA(-;TAt6WP +zd`pmdAubcieP(5R4{Yv6K)MUUF=dIW6jA=;cj4lmGL +z5GSJ>pQbq)GKoYee`jn5u< +z>G@M*N28R{!}3ww(A;peg8ll#OUEp`q|x&bl4mG=*nW}hXmZYiO?X1J^rl>R#5OTQ +z78<>{k_y^4{G8PkbV~Lu@&e$+Xb&VkYvc6K@5Y9*7F3u|LV1*NMe|Hzif2j#<8p6sw>C7oq!rwXhkgGa9aK +z9|)H5zyeu1{y`m2U(@?J(R_(IJly2zci=#YCFavdd*|a~^(BH4Q9xj@b8{H4X$#Am +zky`eTIrq2|_h2EnLrF2rhrL5DiOQqyJx8DA!bGDa-}h;+Zk9jyC=(gj|o?6 +z%?JULt&T6BY#J80j+$taAJwr~TabuF=IIr&UJ9c)>+~gH$PfQYWxL$j#>G|f!Svl4SLv{ +zHE@8W=JSSi@vweEnwjiuc|8mS=12z81un=$5BE6upI^YURUC=f)?d))Jv?iDmJ9chw);_ +zhOl#wW9!p3m-zcSbr}3~nm8iavJ{1WUnq0Qey?aY-Ap7RDlwnEWCcN1Q+9Q?WPNvi +z0N)PUxOXtDwUb(De=6I?;)O5WY`wqj@#19voMJ1eCyfGO?b>+&rafC`V_H(k%?gkM +zVbynI6Iw|VXM?E)g>3jp(MtVYzsA2>wYH48$ESXle~$ebO5YcT!Rik4##2BA;vAUI +z6(Ke%UO%%YX)my`C~kxy-OT9in206gClV=T5>7k=84x>imA}RFr%J1E_IodAc%RVc +zRxi_)lM3XzHywW0I?0%{tSX(74D(SkZ-P|BQ2j#NdQd_opbyRVmWs$Cix@Af0^f9a +zNJz_LX0`Wpe)!(C!QRxE-)QBBBO*7n<@>WMOx1;Je_3_aUvA<6_xezqW*5V}p)Ksz +zfn#iCB~MQpIv`zh2u9?6h~tTrXQH8tOwmdksasb`7(u>dpt__ma)ylBpoVeLR)t|@ +z#xVaXOLJZiDQVQR^0!}B8j=#EpS>cmz@P&;HPOW(?LMAS1Z3F&y28~Y^c`Mr +z2Nb>V3*)zb>ux{Npfu-qCB$i4kLrT!`&t&%;|ET`AfV=g-wCreQ^+aGxF;hLJq3!z +z6V|@FrR}P`4|~qUJ7zqP^$GQyjXC@!8jubgYy4P1O;s4ia{Fpd1bvD=@QK=oEuRn>=S$(QNInR&^Vqm;RmvRy==-S?FJw*r~VKGDQM_B*(4bLAX(e4d(on +zV09V`A$LO8b*EwltT{Z=c1|gfZP6NNXq)(#%Vk0#ehsVO+Jv5)FMbp;e3^)Mr%9KmpD(fy7 +z%siFKcWs>oy_--@Ti1f%77e3MEkLwdFgxLe>Yd4=>AN3%m+$YbaegL!lq+$lnA&fx +zdfD}uQo=UtSzR(Z@2so3)J_M0^ETWeL$6nOuRD#C<#~KRBh{eQ$XNet3GaGbyDnWE +zOXfahUydqY9t-R!u7mRr5}xUIN(Sm-cOn6^nje3#q6ToxZyZ-^E_yF=&vJgk{E4`_ +zBR1>_bZX=o+8G;!2P)%aI+0KVdH|9%{5*WKn{n76`$f~3Zyrx)nS%UGx{S{W!1Pnq +zr>i!L?)C@gJm(V;=b!r0uq@%_$1fJ}50W48@?+^_%B^>1sDjouVlV{Ib#D#_##1}0 +zc5Y2#)OAPv!fORw&Zy3+sl`8~bRL<_oPPi#?P72cq6c@fuLvRrsy0KlmiV)s#_W-L +z95svornIWXH9%nVa)IRh0B~L4#f +zc_PNSGOxZ7Tk?d|hG_jC%DytHt+4A7CwM5O1P@Z6#oe7E#Y=H3P~1JZODR$c4bbB5 +zl;W-pPS8Secemh7`hGLtH}iAWx<9h+dXklU?{m&Qvd`XO;S0@A!m61y$)#zK_XLy- +znzN_nE*4TxTL(ll3_g$wGzrARX;o}~}q(iB^)fJ`Dm|XKTszP-WlC8C!HzNkFY}rG= +zot;RMpf}IXk;F6^cUc3)L;OyYJh(?)ZRfdY-4V)yw$tA;*35ooDggi`-zOBst_MGr+w-G01T6nNQL11P30T&DOyyeIKbSNEk0N9 +zPa-W4N1fNFvVABuw(Wdn&pQdxR24dPmk#HS93;!)aLDF9J}PPl9e81-by{E;7#2j= +z5C9?NH*HZ)Et0gnMSGQUX-E`}J-$cKVzZR1&4a15hJSKyA%}Nv&F=7gdzOoqteQAE +z1#O9_)>VvXB678C3(Gx&G}(1N4gl~{F#~)K{wYZh20xwH?D(HF0drs7vq16ah`TBy0(XW!=YGeS~5 +zqF8lXcEBkl(JX)tpNlz^c(bWMco*;MV&)_y4ZYj1G=W1QHo4xT*NUmS95ON(cC=`J +zb4;e6e|^sdUFF5YvI=I?{WCRmk%r{L`-JiX7eV9yM$lQ3bd?Y-Go(j>b9}Y&sZ-iX +zbl#gLuR?{e;iM%NK`uw!9ePSRPq2{5y@{5|2)n>jao5azY4^E=Vo1k1>AowB@YqGY +zv1}+x{zUdHAgDRDZuuA%zo*Efy-I>DfSusc%kV?L_<=j#z(M~< +zwc*1TaSMn%+RipQ`eb|%^x27sRQdDTR}i*U(HUxRuB-!0G&Yct6sU`m8L|$8=lHfu +zhKIi|uE>&iPu*B*M<~6cIPGlXdg*{b1#mLAU}As(IMEhpQZ{^n>wes3Z49u +zC8{6|>4boo3uklhm}X$CpZoSpRySf +zwda@a862gYGgjisT_l}!yO+MQ-l?Gr21AM3Q6bxK!R5l7Jf`vd-Os;=&?c`-v8G*x +z>5mES7f*%+ajJcgI)qFn`c)SA7yv*tYlt)&1dKs%==@Xhx6~NTf6kcx%yMDX^H!K-6|;)6mRICYk2n>>0lMBYrA&)??VVFZ62=c6aDFwVo%lJ_c>B +zMPCf$Mt}Dqxfyf}etu-nXq)Abw|&AFMOGW2n%5|l@DmTkpu2i4Q28cBZuBt>GgN-&fz#%`w`pw-2Q{bN?0YhII1Y#pDRs&; +z`eNPK*w6F~7x5s9kSu_`s@ZkE-HZ;w(3Pho$yXUwkmP8wt= +zbg{@`JgHlH5Haim28%GlTj;?K$Ul(6DetU_yfH_O=dSOixI&$NGc2L6zP$@z5@U_v +zW1z)Oc&TodI*36o&%8BhUi&At8wC~Bhf-5e|A>GA#9-e414A?RbI2D*1^jT0j~H(; +z_b9X_6QPw0D#|I&$YQMEC!x?1RNxA@@&hi`oDjZWIWig_LXL)P)w(W8EyaOj?g&kR +zs9n3dA>8LNrMBdh6+8A)V{^`uW<7zGv11iR4O-EQ?LFI@&EogNqF0&OrCoIknPBj6 +zvULxy`Ez1<`aeYqVni3rH^0$sZ@wRX6##%IoYEH^{ +zpqdXTkTv8Ln7pmou6&mB;V9Sh6}R)MfkC}FYeok|w$RKI`wJBPz7@&hX&0_2&4Th* +z1ryk%dS7|%He{LU^UkFi1El4>GM7QI2a*2z;e-5psm4pW@X$&>(|nBh90FSLrE2Vq%iGeGlHJ +zsfGKj!uSX<9>ar!>k(p7Tv8G(J%^*}micWBNIZi*028*yY_$TC!vdU&qtc`45QRv6 +zctOw}7aS~rnJA^hw1WjKqBCEAi2Cvc7knQrJUJ#Ls`QToqNXV;4C1K9K+{OpYaWeZ +zmGA=+9Gu4vT9X#D;$tszy)JYM($*7@3-`vS$R;A~5<<#X&Y|fvb|Ivo1)J@dGB4XH +zZM6FF+K;c*x)@MGhyG+Juzcbay+db$kQA_Oa8-$ +z*SK;3jH@@24gkP6e$Vg{BB10iVa}hbD~Bl31ZchcNZBAQ(1~h@wdG>ttM^05LeI#F +zFYfbX`~QMU%xA+fFj|m~%m9f#EbGPZd=~<)za|Lvcytq`^L4R`XO`JGuF2u~j8iNN +zebwTB$dU9-Rpy{HGWG>LTn@F1{v%B#EOipU`a@@@8L~CVjAW6Kjuw$$Ta_m;k3+F* +zJl1Erdxo!hWXRzZVnJaYjop1oNo2X?Wl;DG_`PZ|aq}ziGfL^x?l}shG7;*knTG)^lHD(F3D>Rwo78^{K=sKwLOL`-2yMtL +ze7(m5o6==a+pWLf^(%)WAx08T($>S%L7T3-_Fw4n4Yf#a+1^Jc@G(0Y4h|g=AVdlr +zA_ZA(LkK8+<@>Ufh?AE8vQ5~>D5F$M+Zd6P$8ejoF7MH?u+ylH*loXL28jn*R5l^z +z?46aqC^9y|A=kf`{L*%3=IrmcQ!REWH1={&35_V>uC~u&iF>^JGx~1NXrFdQsj0>r +z#C{ScW7BeipO+1Bc$RuQp0xI)Oaei}h7u1~>8XS5yxj%oBb5_OYrzj;`m +zVtKiQ?Ec%YB81VQUaX!(dX0`9S6AZfPk%C^rhu?Zs;=PdRI%Y|0Z9zaeCodmy4XYJ +zCO!}GLoNCQ-9R~FPEwks_gw4JF~Nc34o$GPV=wUm0D{p@bv%IB1wELH#8e(DkfQA~<`QhLSQK(B*K>E}L;y_-+M2(H~Q7LJJm +zvc^7=oo8n2n=?nnbP?WircZXBHPl8C_LT{}0b|CK!*1tNvDh+3QezF|a?;OU&3nyM +z-p&nsOKL6oBD4auKR%t+wI5UlaPe?K{8!ppe&%wRft?~uCAtg^fxxBa08`on{n1WJ +z9Ec(_k|YRNvn!^OBq7t6vrabMmR|Q4;|BxJ{Q}e}u3L2(zhbcvrhNQs`Vlpw`gsU8 +z>U!E0k!{UAh9Po9^>Vamk34)E)daVPVSW=zxjsDT2KVcUQ;KE4e*;}zsc%P&>65!# +zhz4UnqcLb9N`LD^3uXj^KjdQ)g$+Q?FvA_jSCpOsBNh5;Qn*_3;xsB+bHjQw?@qZcp{qNHq|3#Jl;bH+g-}D?BylJpx#8UZ +z>Ad6Z(+U!qR)!GGG=&Kqk3Y$@%0nheXRc2tXBIq91TTH@p2Q*3E)Vp3rBT}GuDs0= +zC{V~BVjH38o%?N|-1lotHZ!4sM_)#8tpZxNyf*>x$DU}3qruLg#LnR3G0#ssl_x>K +z$sKnWMBV7h#6TE2{Xo$x4umSu0JI6Bg<89brTh90#z6ox9NBOGM0Y?$BOZ8wa?#%v +z0n;a`mE;?3Js|yi;=q&fl%=s)6L>!lI+Sww^+#y4LEiMaujpg{b&Z8QHR|elRiGD< +zryoC7_oJ*y^og@Vu}2ocXmBI*ar;w~yd(|9Qn_?&rIDYu6}%)7JqkW>N4qp5>2fy4 +zzp{L<6p2)=wYfF*e^6jY4@K?sza9|OW!jf6M0>k%B#>h^+gGW)ze-?@j5qRs~&uj-eLgTl7B^-`yck +zjReO$CS+jqV1)Z+J>(}I0(`dvXm&!m~>$FhwI7kY#A)`$#D3t9^@4kdZkpd@ShqlTtaF +zQD`M-8r#t5&hW+6u1X5Ug1?S+Xrkf%;OAf|pfZ#QoV(}&`+Y9BtYwiA^=@!vUsPYA +z#r8hff1k$gG}{as==wx)xJ`=Wy>@DhiZ>dwl&yV*dI|w=ug+1DxA0+ggJt?fb +z_r#m1Icr@~Wk`kVsz+oN4F7s4qp%g@yW81%>>?VEnGsm~#3$4$U4O9gE%B-3^c^T6 +zI+hL#Z4Ga~R`T`$o1ifU8+DDyR?FLJm@D*?yYz{c74D}HDV3yl%d&!eT+}Y?dd6d4 +z#L9hqOBA?}jdm`7zr&|u>vI)k6>~M}9K?&9h2X-f3d1G^q9LK&BjQH}-;JbZ+m36g +zB)tAE>b6gzR4A>ktYU`y5Yd5K^X(gH8-hEVPfTBfR~&(@M1a^K??#VJD2Ewff8l1s +zq<;bvUtU7rlm^NbKBJDc3VuLZ${;!pz%=MC7_Hk7!RHl;1K@oLzJ7^sA_W$}qL%?j +zV?TPolLD-vc2Az%Qrl^`arzw|`uP%dyI09zt}CKsG$=li;5hZW2m8kC29d4Ps{c~s +z@3&B?7RcL1!+j*z# +zuRy0CYF-et^{VeQCOWZXCPi`8@U|@L*Gng|KVn;4+X5%mjOlZR7)DKJ9?gW;u@SA< +zXs)ypxA%S}84ViW4O*u2jtg{`y%Akzh^HXwz}v~7y`bm2Jlx5ATR{ZxXaE2}*^IIh +z!D;mO@EO*9>$(Kl-e-}TtRXAIbltQTY +zCO+%v{Qi(STH-ijG@Yu&@*(V#_j;}Pi0LfL`Oz?uyipbLpl%58=kSVm&D%xv9^$4; +z5v(v%+)g*t*x8w-_A+j4uHr0rB1{lvG#UnZ^T6r&1lpg@#rC_V_V(z@u;ZZX2&`GN +zLaMHn5pT(f;J6L=af*qLf~lcgEQVDHF>8OATf56Sdz`0~1jeTgN5F>c!}U7dJ8q6e +z2>4`BiJq6?^^PG3cM|4dreE+JdfT9G6m;rI%5;q`dy@Ug7Ib|!FWGP1xNGXWL{5U3 +zCQZn8^F9jL#UJOcpgimnm?J%nO#9@11PMA)Kf1$vT?jV6U)OrcvlVNHCT863rIq4& +zqJtN|EoSw8sbzgWW9H-8TUc9^+K_eSl4;|`QQ&RG9de4L#yE3$i$=&xCypPgyl``E +z|F_XGOKfw_r(lZbT=Wd~EQK&SX~G6cN+HZ0VTx5{TD2R-Z)&)Vu)jNeaYDf&$d%19^g<<~d;;)?V} +zJ8T`i`91}I3>%J>JQ>Che`sgZ8866l=lWz@57)#7XX^|@*HwW*yuWiJFE(H3ee*=RU1-f8-Krgf-Rs +zWMlLa#JV3bhy_Qyqzk-afA|1XAw?t1q?SXz{L>-iw~uEOCVg9HrXu+=8G^l!xiI6N +zdqQKAE?KTnOrpPC8W!?|(wrBsh&B-tPtyBuehNU9KlHv6q?xltnCL3>CURw>keRkR +zD*mXJnPZz@t#}@bV-$$<4mLqgbD@+ZME5xD`FM!C%I{&j8hlJ#_ZWxut?uy<6XFsM +z0whWQ6kW49(a3sExQ2!Xa}Tg6;=ShHBimqX3-7|NQO3o7h1zx4au9oPE8h`Lmtplz +zOfgLA%X31WGy|#Z4By*7{SQ@-Y?AjE$7c@-(EwK_GN7Aui;>TjHYpHD?0qZdTLcOf%W= +zk`CV-gbnOYiv_cDf`RC(hshejsX(-McAXHKb|2VHMJ=H+>kFIa9( +z>W%E@_xv0hCvxaw&$c)ZlzuB|n9=3k|5Ir-QyK8U0caR377JPsKy>ycKly0rnUu*h +zVF4pqB6)6d=GNtl=BY4VOa)6>1{%&DC6vuPyd=F8nWwlyhw;igyW2PLy+OU>TKk(K +zVuDWg)jMNL+BaV{O)rId;_dck(0RPY!{I2iYCGKf=Hx2sO9Q8F +zjg5 +z!6?t|etb+wQyO>%yr=a;k^RnJxb(qN-1_rq_K8%a^k5hXmbX22)rG+`@f~jasq#Dj +z-~mGYsTw0pv_HSGEnZlpL!PaA`%>~2jEIOTn}7~V)V;;%L$X|DK8<~5mGVILVDRh-_u1uB((&Ezz^iIA}UQZ;hL9NF1D(FCXgJb}^gNhb`;lsUzX^1nEI4gNV=55j +zAYEOhrN}vrm7^M(t*P=5(7qdpYgsZC{OO_uzTDu~`a~1MhDw*jH8gS%pCAql>}x;Q +z2$8O=K~$%m0_12g(oUeku>rTclQG*xCm;I46wsc2?$FA}+&)5{(h}7?pZCXDXl)lI +zXD1IP8Z_9s(CpVeEMnV>BH!bUg<+(wPg3UxG6H$(s&!t2Pr); +znfmbR`LP;;=;*I4gqX<~scX&5#!4^mQR(X^*Y_(fmB|oBq)2(}FjM{f_B9B{BpeW` +z0sP7n7mZ=ip#gVr31dQ`$Q1C_mbw;UrftT)NpV_)elH|^MaRPIO7$$c`#CqT+Yn^Z +zcoSbM+QY?`f^(UlZq^j~l&Ms2^W`}Z@@<__g$}vR*E{wMc>wfxn}R_TGn%}uH>yL4mZa9Dzek>v^i+6mfKByrt@#ifsPyVFi$2p8QlpV}@ +z%S_&0f2N)CJ4Vtq{xY{|n;LPP+@`>AIXYe?X*`aAOWg}1m|iZ-Uo5R3qeC7m40xf< +z4IdYKy2V-hB!Y;}mmkC#1K_xKJrAG!4ENfz?{e?t*r)N +zH*>n>Je= +zf3vU`)YsYaE9kdksrZC#+CSNI?w6CVz$=keN7wPWXQfjJFcVS=-uN!d)u~Q!V(1x` +z$M7{i%tmN5i;UMxR)iRYuv_76Q5Sx>J4%6Zu~RHAx@}D%~S_?C^EZ`>pB&_egukvk)_ry&TR!r-(W**E)rZcW$zQJPXk4pDi{E52?bj>bf67&qd>6Iyu?fhsc6k!| +zrd2ifcGorS@ENBy?9|MV?GW<76zUK8Qbf3W=#PCi`dou=@x9@l`@4XgtS&%8MvzK?sNmp; +zZWwl`;2-5_25=HG=7EiPd)TVFVNr6OMena!_=J01J|vZ+->q{c?JC7!(^&B_Gd%;o +zsDf6tKh)e`L@sf +zi}fAIZ`J3?IpmtcLA&m*@UDAu3yDI@ipFRyCC-eXtYSS44uMRpL?%S9oQ6tM8O{0( +z5pp}vfukTm3=FGla;;tfJ?K!b^&Hh1?Q99n%t!j!+SN*+Lk`1{zUW8hn$ye}2m +z5&L+X?xbRXIxX3iW%A2Ma&=}e&iCeob +z^_Lh2){E$U9^W&UGydnHpbhklr +zOTR>W>UesXM827H_yec#1x{huhkm33(2-3EPUIApO@yOhT|}_b?c|7JAIDLxNQ~HX +z;q3YRC7*CTaa%p)1VN44!f+Cq{}PeN>hp?|w$Ah2=vEKs#TU@9(z;{k7CGeRblqT3;)un~duoP>XuL#4`~b +zWPK~>^tTMveljbQ8$l`SiD*xv5R43NIu@}I3Y!%OAQ?D>U+953;D7Pp_Kz)ytf_{y +zliTq}(*5_0A8-_}etqYwC6%^a=WDpM42e>KuW5gG3f5!A;AQIH+1WCt*vC8#o~3I% +zIv@CfilyuPzK1Gxu}_0mPq@^Ci~aDkC%GO6umIY~w-Z)2W2!|IRVJL>p%?gOexcn@ +z61{%a^W-Q4e(4DV(6Ji(usls{+Vh_{Si^CRO~wxP)LabLR$K_^GjGXnOv8*2B?^{j +zPbC{qJ?jiLY_F&Ck4XM*`Y&wU9^YGDKWW3rxNF`^MKhJtVE)sz?`JL4l{&G1V&2l6 +z41&-M1oIbuWSVcdro#-sE`RHEucCBywM(a;8AUg9LzLMbw1P9l9?1S{h~FoGso(|K +za^nw9&(Bb;k6rD3`Unl^@EPO&4@S`pY^lXh+xzcGVjb2TfC?{r{{VnbdHJ={G0J+S#B%%@*7ATzSNIxEhVLVs)Di%opb$bL~S_*8(v#;UvB^{rpo0yU&{*!T3&B|=~ +z`Q$F+(0MWAIo?r0(}3pEE>qbkHy_~}FKV8>B*yW;K)>P|KS<>dvUwWCD`|(IUn(Tj +zg2ejU8}w;v&0>E8zAAYcBqe4f6cYOhT@f{F#u6D{t(-|ujj3OaAR=`yO61)(NHt7I +z#W1+~b1eay9~75OA9;JChhAI6${A043VF2)sxYLi#aiUne~KOZz~VRFg)F#$I9SV4 +zCMHB)y>fj%^VYSxg^aYx?@i*&1CA4)=`r#!~Q~_$}b6}uYKzM;SW>% +z1CKV#Dqp{umWw5bNhIy|(a|pVwOU5nqg~U~+ddV|F+z1P5(lPTQ-aPX9?hl1^8-Cmx!PZ4u$a1HOcbi|oHZ!VpBN>jp^=U9I~swj4422g*57uTi;e +z*7M!{H`QK29!#w6PCWZx_?gVt9p2Jx%`%D{krm8`x#r%jtuq@-EI+1Qs>b*2T9non +z%{_7DSM|W4V=o3QXrQ}_3}k7$Rtb51>R#lu$s4H`yVHVoH;snEnd&qz{^Y>=c)y;btTSf>8lry}Q}cHm!cnIz_12ub%Lk38jK%u)(`_3xdQsGIsPY%}|J +z@P_@DasB6A_wNL3_ydyr1LPC&pN`AFMclmqmdl%%{X74kZw=%B|NIOrsV6U9ay01q +z*IiZSfmXLfF>4VefB3Yga>{TIdo^&1vfHwX!7j^$B|Vo&RembD?%!vvZvjEp}YZKKYGEIze6{#FaEgP0QkFKHkGW +zibmEZ&1c~UvdZ5$=fp;u{Xh5Tbm$eE-}`l!*RJNd0jsT_;6#>wU(Uq)xz(`-eEaq# +z#;+7MH7EU&-3fZGD=hxcy@xr6bbsXwG@Q_YOkv+Uc%T07%d#!a&#sl(dY-a9OP1G5 +zt!tF{jpOUFG|kh8=5Z?z{9vAUUDG +zc{ph?_ZGgmwZ}lYZ|=pQg#(STphyueUx4Z +zC7cNyS&H6*j81wDvT5~m*0GDgQuvccwkZcG=VH7{$}^Y`jg{Zc`->)8)xDn@KHp?5 +z8!1&VL&7c9O$k*cH>g=_;DWE!S#)#LcWWQo>*FO#935OY89A3z=HwHrX`G+NXy^_H +zzc)=R{K+UK%fA!LP`9Em^Gntt_qkO!O`v#H;o$M0%^8S)cRy=-1Z!H>DZ3*_7>pJV +zzp?c6b@K5{>UK7*(p&ORzyXjyP_WwD@II)-ee!4-c0Qb{4*T-uFY{?F&)sN)x>ZX) +zLRVSoO+~WypT2lA)4DRPu&~2+e9~!mEk~Nz6OXp-CccG-?A~SOn1#kO#Ak{blk-|t +znHKu)2noe${?~rK_>)$rfq2V3m&W^Zx$_NgN;xMkH`6^Ri}uq;KTDMUdAxXPx(c!f +z1#$kjai4x%i$k*_zq{2Ps@6)`=uKmFyT7p1Z?~H|R{n%-OH2pCxdqdb({%+C$m3># +zS_1uUr6s9IVq07sZto5FQ7E=Kzb+M&)jFKs3(FJavZymkO>6r<`5i|M(qbw^Q!Jr2^LA+2PbhV1+Vct-8tCYo(Z +z%6c%Z#RQ50$wx_J*b+}4eqyd+eJplwUG5fOcP~HgcFos(r^)O`l%8BL6lLi}=XnBS +z$_j*u823Sz46Frp=0si$t*&p^9e_fYY~lvV25I(4>g-GXg-^R@Z&gJtr%&V`Dj#-k +zi)y^nu5>@~@>!GV4D7%%em$L#bMhrmHMdtza4VW>F$qB~%3IWQ3I>gUb-L--jgww# +z(=!PnCyR)nDFMsNU)!BHDOX?OGu~(IR%^E9UGVgAUn-d6rn3NIrJ8iD@W}g(fyQ&-)5DmPIp%Mw~ntsT2SPGOXwfo`e@D%dl$hw1#R$Q9ZqA_X&St37>6NV@4gcUucb@1>#w2*!ll_?$ +z9(VYSDh&vQN|dWnElwczMG)1DYDOVhETZveXWbiNEJ|rQSdL+4p*jo#yJ|O%@K@ahuO#T +z`Wup64iQ_Qd}}$=p01aJK~CfWSoRrLtPctg$gy~aT6j+FxavaIL=mU-1?)qo^FyBv)KSFx1f9(&u@f|4&CTCoo +z?&haxXqblMAFVDz~i=2S!;4lc6FB8mF7Hc;L>5)AntJIjUBqIs@N54LAR8n9KsWgQSAa$`nu0sPV3HB|ED7T~Hjla}vo4UYmpDn-m&1U*a +z{jy+LCxZrF{i{S!p@i1HYhUTEI@La3?TS{SVx2rBul4)saRf=Losb}Z{M(bv69i4W +z`|*zPuQ_8Pc(bo5)KQd3KJ$H~*}eFGM*0%-*WG%-ti;wFD +zS78e9UA7uQjy7Ti3FEdQ3IF5^{27OEqqmaDEqEXF`ns;fBgVSBMSLseJsAAVC +z*q7HVdwlWNF%y!X9|jme3%3Y`R|X@l--v2(;=A>JJ8U6TaP*pvn|7@9XR9bq5qC=W +z<}}%zb851m513Q0y=$&^)!&^DN!OPQYaZJTtPmbSM5c5lV6Qr+l8i +z#VCvJYWQ`r$L0Jc{(g_hqw?alFyx}OoMMShLEXq}(_ULPLqFhb*R)zK^Sd$K(}hP9 +z+Y#-!ebgrJ0uc}#Pt(DM_u@7%Yi8~h-kfcMiDMMz5eYZpX~bi9e-ZnMN46<(#|wpS +z!gYpN>A`qxnFh-9Xxx@7>gkul7lw-qIt|CQG3l5`jXA`+M@cDs1al>w(B`B^L2V6WZ|2&iDAP*~ax5brKaT{x3|yKkEZyZ-J(WlD9$o#p&d4^J7Wx&S`C3N81*!gZ{* +z@@SzF_R)X)`bX^{w5c`ID{{WxSmEQtmiW;iBAl6Bax;JJB-uA3Oi_5TWRis03zRBL!XBb;dHy65(z#V=d`tYlG(TEE@5-V5&U)?<4xD<6nI+9=2WZ49WQnOw&o +z(b}$3R<&r^U*B{an6ApRjzC7`rHwVdhgSS2Hw!;~yg$leJO6kqDDZl5_Aj#qQomXY +z)6&U-U?{}Gy4J+q=)wC)G{+*dI`CqO^e_mvA|_}u;O*p|GS%WL_R}sbz3n1Yv{F}G +zedic`6lFPRBR#Z5672=r^GF}~S33SqVjr%PQ*)3>$0B*>d0oJ$QESfbs2Obel0=4B +z87Wi*TmupSN#y@HYZ2ksCzR8YiM`Olk1L5H>_g_agG*t|`6a*o@FVOgtAb%Shp%gq +zEf8HeAc?r^^9qsWnG!Fr|b_lz1pei^H7s8 +zBWJdo;&NLZSL+bUJV-Qn2dv{tHTI%Rp`p>fTADCJsLJzh@!)L=jIM3;6slk6g-kJy +zyl-ag4J(&x4G98!F<`thjGi0Kaw@hiGrLOOhc2`R8@E}_Q~d01mJZ*LUb{G0pe$RA +z0?*LWe@2s3ty`Q^_v9@yH^Tm(+=z;&?}MJgFdKbMn7!b^7`D6B3^&aSqS*eXA;5Nt1K6oQ-;HGz=2=F3me37CKFxGj0Cb;^^S&9$_T-^gX)&W48qVYi=}do}mo72+hfz*kM;|IlAZ% +zU#cDZGD$e#i#+w7S-kD0&?a`{(d_*(@|{JFh +zen#RHTizn;SHGHF*7p*0(T_GDVPGsS-lySoT-!sv(ts%9yJX%%tf)Tv^>(J0;4zbZ +z!Nad4MOohneq5D;yFi^Ij~h*tm8J;;{j-GR^YscGVt2ViEwXbh*v8AI@3^tAXvG<* +z1BN!UtWreQ{btG2`=)?$QagKw>}B%x{YZFoy7V5t8Z1S$s5p1DZbnSl^{AAu&Z`8V +z9XxcK$ou+FZ7rvlk5O{ztqy4gg<-1#{u)lCul4fl-5rx0-qkk+1bgDhE?J|)LUUis +zaLUSDCgGm$`=d6;)HWC%N?1d!T_rCxVPNGuH-D}@1=-$pVVmdzNZjuf^TEIHr@>a; +z)nG&!XIZgSkV(^HX$(U&VMGBN-G>3SSQaPrC7j`4-bpx{qzPWd=NI;r7T0dnjtljs +zzX2Gut*BGzU2*zi?4a#@L~Ki;KudKePZ+M?5^$~lSR{7_WeLd^(VkDkx}j^Nmd5jB +z=r1*F=S_`Fe9-L|J*(yRL$9SDSkNs#^ME0A8Y8bRA0mVxM`r}=chmb-O#-3?7S!lj +ze5aKk(xl3ul5Fb5?v7ejmtC^K5=9Bb^q@(AY_F;m$Rd44pw3oOgEqXSLNoy|l-wHp +zlSftxQrL+Hi2ZCnQ@9WF+V!rqolCO+D)ge)CRJPSUp48T+Wf!k9Z9&6-j3^N8PvZ& +zq-iU*1~c)yxtduS3-E3Q3`Kfw(aK`DqBQJN`^5{dTm9hOSdeWIB};LzQgMBf^t=+* +z7jM;uUL@??o`Jzs;(>1_`#RTEvDh?cckvI~3woMi-(M19{`UM%RtXPe_fGPqa&Uq1 +zVZErSEw}HxzS3MImeFf4;jo`cD08j$y#InPln0cJ<sE@|5RRW&|A%Z!jDkcv*z#d(5FZQ%!VPM7Fb+Wo}m{Ml^fm +zMd=^O{J9d`dl%PUfSjR`Fd17ab2Jw8&cUU=c)3|Hl2T<*whnQ*l;1UN*XbjE_}qnz +zLtm--xgxuBve$0wSG&U66hDIP@w;$Q%^-_j)O~DuDQNJRxo%y#SwR;r#0#}Lnng}EPgc&(bS#mj%2`u^enN?>BLGK|ZmNqjUo-6OdZ@40p6 +zGO8o*u1acV7Puem_$=<($U&}=N);5F(dfNuHg0{V8J7n`x)n3nPiKQS6I-!UpLI>4 +zl}plb;w~4xq$B97xvkBbqsFbPu?J?>Yb52Uc(RKUiaEg0Mv5>mcAO?NER5UkH;l<-rr(Z+$$Q9yqYEgyeDwuz@>~DnYeGperkJj_V}`UE|<4uJbb_2Ow!OS9^vP(QX~XlAB;2{_Wz#zz{2UVB*nLJg_w^9uNO0b|-;}jjG3}PsLD6%Nx7#ifKywtO5)+bRDHB&sj3OQ_!I>Fc`R>Ydw%1 +zk7+W#YdX2x7lpu`Wznr{dyO2VopxK9NC4J2xMcWpp~!POs(8WQa)lk#D#@`AyF-1W<*K&@mhb|}US2P)Sm<6smS3P98xp +zyr%R<|8@|YEw(xhpPh@S~wIRuUIyR7Dt2eSb}3|OJa9zo*r{}d1z*Om>BP1|59 +zby(i+Fcz4g(l<87O3O>OJMM?bN@fBm +zgj>lZP(D+apF_vuo5&Z=0Pmx(L7%Z4Rc^a>s9mt;BS$c*FP50hu+q+`B;RNZ;MoZY +zj2hJvX1ny&E2Lqr<-B<&R+IN5ZPkr5u0Hm7CLk8a__K*##Wv57 +z3QCXbjQ1md@bq_VpJyyu=!_y3iJq6`Mb@ocM|w4I?pj*&=e84-t=(T+U!h-8x;^xk +zi_injyX!Ns%{?zI=-DQ7;z8Yz9t~Y{ji2{n)%?sn9!e=!`zEb2>)2FczVh>5+t)8M +zqvA8CC72|aN{K@uSl#=WuPFDg$z?GF<%)mIOytbqURwRhI#qR1yZsUz1mFRk*KY2f +z&ma}JDl>RR{!B_gSKsplD5sncj=hcJl#%$h7YFp&$VX1QdZCHw38l=W(AT;WAi)gp +zox^_~ga&>z(!{DH;H+HztI5Es<2YvDQTvrU3cE+lm}2}{TV1zb0OR94WFGHSDdx2{ +zOYSg+%v6I*qlA^e`ym79V9ER4ALorn&8keX4Do0!(uwgFnfcG|r!1Xe(qSynDO=#1 +z%8H*=&4D3Tyqh^AjoA4nr#2w@rT?p7DbK|flS@8#@|NEy_2$BUa8mOwZL=4pS8)SX +z9JB|@jkUkBm6A=NImO`mU)=rmSCrouKY&W3)X?1^AUS}9bR(dobR(d2hcuEhlpx)$ +zAl=;~A>EAwjO5Tm&mHys{e0K`2ku(myVm?P>v_&O`#JmUeC>VueK6?jwkR;{97N9> +zLMO!_v(@f{#7CnPwpk;b%X}|p(%gMD$WFCBCZvzI`Z`fp9i4S0T|7}_fbM|;1?%E_ +zh)q$TW{yA9{IjV<4e?WCoBat1?m!O3A4D{Sa6e4VctNOm&h|TyK~Ym}a#Kw(aPt+G +z40aygojqQFq&vVt<(`u#K4Ng5 +zD0nNs&y+J4QJNA`UHJCu6PN-z@9(c3JNq8XlHHHkOCvu^gT!!e!_G*dl@UyWORwOt +z&|Yk5{Ax;XK=FjF_flj1EdrU^?E1gbs3?&LfGt?Me1|4d7eg`a`tcLQGgfj?`7L(e +zZ^bz)flQ+k=+_WKI*^Vtiw|1Zxrh`$ogmE3yCV1goZ!Awyw%AhHtR?AVD(mprd(7& +zk9{w-=iRqapboYu--WfSi` +z*?U9G$qBVd?rU-tA0NGE{O%!kb-yb(ZK)P_l6pfImS2{Y%%RI;<{cFHBEGDO7<_%G +zdBd8&dUv@PzRlnKLGaUd53q-tiAa)(c`d$Yf-U{MeP)DEw0yURSVDxl(rt=gL<+0V +z2m^yWYVJ<@53%PvyV^IvluFDF(4O|*G~60Q50QdESn7lJ()4`^S$|nEFG?Xn8@&HZ +z6pOzvqCe=+%yR3&2N!`Vegr>;79V3i=9zr(^Wb7a!F+qLkbFWz=`8s|@*0)k+M=;Z +zRso~SOCm?FwY`TOTT$1mLs?9P**aP7+$6SP^ju(ndxQwYYKHj1Fv%b3Zl!4U4DV9G +zw-{76k&;itxayO0;--iyH)EP^&HOzfk`2;CwGa&v7}m&^ +zCgJ@fE0(7tlBiVkjZYet0|xi$mdWf(JzhNATwY)gelX-x!J+8np1Q#CR57dTQGL2aT3Myj +z^*)nzNkXYDtoHihn2vBL5Qxi+0HXBpf!ce~(IHJCv9BOaz0MXD4`jn4w09jd(|jPd +z|C=Dk?c+oKKF*?l~2QVMR`RkPiR{^xNebIY>-lMbp=1u&Eb*gieEDp +zsrp&R&p%&)j^rQtHd@k;=zb9reOi_C2M|9|S?R1bOmz+j#YDbph7KnQ3W}CTQ_5fa +zSPN7AgN-<8{`YAupJJc~_&~*aWq`7>-;w!dZ(lFv{el5xlE47-Py+dAR~U-7xry}s+P +zZb2gKl;5oS!(#K`1qZiDf6y7?`2UgXg~GTaWV|uO#Z{G*(c)MWd3z8Uh#Y;lsL;8E +zYmYXewL}+&G9dAlgqQtDQ}oE_FVsWspZEl3qsqFFUfRdm7w1*zB7-w;C03}BxCW7q +zDD08qnie^j!iV^&G_IY*KR6O`$^XqMV-oQ=MZYiyYi<9Lu8q_N@+CF$j+Q1@f^XSv +zd>vVgM1-H=ynmSPe<&d@l%5_VvSIt3mws?*HR&HGE(p$vwGMInWr$^M`?9;k$iUn8 +z;=A?Uw$6U~bj1(xZb#14d)ZJ=nCE>ASjNqd0JT@;@b**2(+*+6%z;0wjtg=93s&(R +zI7ikC3s-*e{VqX3dzSazU7C!`u%zA#_br7#Vs}}Rt$QW2iV#T~tdZNsZv}%||L^slJR^KaSCj}r2h{|DSkiZ; +zj9X9Kzpnp?pgryV4u5;ny#xxFaXwt&@jdwhFLqvZd5s1PJV1uZx0Z06%TTC<`I4^H +zy6G}2CYx6%vE#uR6u{*CmtWdtT=Z~|0Z^ncS +z110Cr+3}rqTn2QY0e{OGdYp{MHb$PaIMsPl`i^E&gI1oQBAt`$^GE2Rk +zO>oq9;%|D(=DQ-_Kk{O+(bz@(TV|66M%Y%s$R7!4^H*OWF|j7nE`m+)7tFj(t~uyd +zXDi#}+ta)VhDI^m>c@JJ7$sh;t)hfFmF1$AA +z;4vWpsiTJpuiwhU3BIp>Lsd;nEX{tGWW0TTD3AwsSwogBKa{T8XWIQFcz;KYk=`^lkZ|NlcQ`R3x|fs0@#Whn7J8>Mh

  • vSG%e$FYCt!utT@ThDzLz|rhJc6*%H0P*e&tZS~Hi&LV2wT$q%?zCI6|mR*bM;wN$nst|3}N +z2^4}miV1JYhu32!fuem0=DI{y#1s(cxLLoGGB?m +zZg1k$ewTB5pH08AaeY@A%zn~|CT3Yr>&JG*2qU*C6+MFR22Sx;E;lRfyFCn# +z@Ql-6_%FGxZ=Tt)FnuyN)^b|PbzAqpM|q%>0-?!TZO&? +zuDkbsDK#4}Tm{PM+xz{an34vf8cJ(|@W}WtVN(;J}U%!?ge|#TX +z6Lj4&Uyt0(LGDnL&}C{^V1}JQXBE|Ai6ue6!;W8W%jJXYNQ|^Rm2qZ~jDbk*N#OR) +zers%v46SI&<S;f5_qPer +zkAiN=pX&p!(trcMqj9jwD?2L`**rx)>={S|A`)?)>AI}3!4Y{-i*qXyt7LAo4ucO_ +zj2C0sw{DQRwpz=diUw`2P3`w6gB$90E67SN&s%}-)V_WnFT|d|fHrX_N??r->#ZZ+ +zK;&^NmWnVr9m8Z|OCMpLAlnpaZ@+!q8&lO@x{kbcsRHb@=qf+GLo@XRRaRdce^lBm +z;K9>2FQ^D?nZ`^up||WsJMB0_EtkxH*(@}e(Eh-GvBcCj=Hj0z6nOF~pih4OGvi@A +zt)D?}f>8SqF)uMXIqoGhTa{fppT_m89EzbR@?`RP1YZo|dOqlLN95S=1CbGK$xehH +zrANRr^k$=xnr*z}sxien+dgP`km9#1q-?(7tZl40;HbFCB-g>inl{z@D3DRlxt)0wpx*VTvlA?pxiZe|{+QS8WU1VcuAgPl;5ZFgZ94;*ZGye0%v??#9y${~Z=WVN +z)7cF9V3ABrlWf&_wKHCM0QESSimT_8eFVYk%hYGUrm`|3D2uAT?WVji-w;6$BX#21 +z?C4$)ft+ffFnaww$+Qo-EO1t)F3jEs*@p`P-uyu8%jI_w2Vd_)I;q8~_O2AVuMUuf +zv4)oVyfAP_SkiW*+=_fNiX^II-8;~JxOi^&P|GPpebqYfY +ziBUVtBJ&>>Q5K^vJc?w4HyylzDfddeL%&nd_*;Fu1g6$gGo9CxSU*~gRbQ_ayPJ*= +zu%GTvz=~H80pjRgUes3Ei@Lb1AgNPESW$4WOhj<^359WTqML;=GewwzLm0-CJ +ziZ>4hoNq#`F^(5QNnSKN^*!e-UV4I}8*YLgOXg +zlk>BjvFx5Aa7Ecf=4hXBA@MjG7oArJSO#s@xfQGJddoi7$0<4nmvkL@w_+)kiJnIA +z{fCg52iDrN4G!P&FP@XX7)4!v)vmZ%StW9g$IeS8*bA?J(JMGL4%(%4Bwym?N;SW$ +zZ|xNKI%;jdx}p6EDHc8&Zb)5sg}~(Wyi>=@Vf$%T;^u11?=-w_0BBh!vR%jkeBX<6 +zsQ~p%hKw5LvnvZng0bR&D9QpL%lUV*-EDCs#G`okNX);dfk2f4AfZ24ZM+}S{W5~+Q6%+p)7p)Gj=Qx1y-5%jvXinO9KpbB +z{$Z;v()d@>$`5g< +zM?grl<`!ObJ*Pm#i~TZSNjlJ$>;h?5*aJFsK~}b-e3_#lD1MV;hd`s`^`83J@vXem +zi;w#y0yd_8Fo6~~2)DrwsB{@w!gbiDUUotP3Usa?#0cw|!39Px8Y#GXAS$Mc5iH&B +z0*>mR2tRkjiYxBW{ni}}e-z*W-d+`})?JF4-&nHfYeXBC(rJpbN@i-leeT=K>+JgNyP_jR2Y +zzUC%J3U#A>^nyF4(Nz^1T@JNX((SK6GmozFCtW3+XTIzrK_{LDXcyAbd1e7CkGlF< +za@eM#PLz0vMtmC$5f?5cLHpg<5N +zq=X(rk+-ptXaRtfUmLq0)8yz$%B$8fn7=d#rU6Nt&3 +zeu&#f35yk4&EC+SPTsGNLMI}XJPbZ?=u*GY}CSK(T +zB&TVXtdxVj-lw0)o7>Q6TF|TF93d*`uq|F<3G`5W#c9K)5Gim3^rK#Cmu$QQ*hHNq +z=XC%WdnIuPsD+*FB>?Dr03g>(961uJ=h)aM57Fb|kp_0TWFw3|%;;5XG`fYvO@S#9 +zVQ$lk(!uR;sT-Bn)XKR>0g_R_z9yf7c;QV)-!KoH;Z&*g~~ip%)bAO$87yVZ#7n +zE>)cEr7g_utGxr#fT!{!{`e33-Q3yRF*MX$>C>ko +z8&bJq!c`5G;x=O+=O2uYoo0Z==+B;v5(sK+Z9`2b? +zoyFuPLK6Cu;_hilj~M-Nh7_*a-0ie2d42YKcU-#_s2rl&A*}H&W^OC)cWUo|hiIy1 +zY=s#(sjyQT9@HPzkcf-0DvD2Fyu!;*IO3Y0n{9jZP4w2dJ?bua$YyjXH8>YruzxFW +zw4TxVSQ@}cJ7r^R7PR}lOdS9?_PNqu&*|Z(Gv&M<&h%~|c)zbY@3gmIj(cqQYKa|W +z_3Oz**HG=mV1Ce7X6u4DAJz@V?pnf1W^n>`PD|1eg?BjptYHb&4m&f~5`W8 +zbc~&y%MCniW2UOHs(U1BpJh!rIBzMY`vDgI$Qx&tSq&aoWTf+GLwx|lJL(iX+}7q1S*(vZ +zBZbRRND1O1EP^%1Y9EDR#+fxvb#+NT+_bug528mXE{t7Mz&HH>n7xfG4$w67jpAwj +zVxGm(Ms26S$kSWcgZ~)Ld*ZkFR-;(CIPbq}jAQL}my-ggM5L0r{Vge2@k*M@dL+ZX +z6v-&8)ytI00fgR(1qg%)Rc;Fb~2&{>emtuZ^xI$ +z`p{b1wEQ~E7-f_G!1&!KH{3Bf-0hfhk4$PEjsm@^J$plaEloXixh{Csh|J?apdptnE<>$`FgyOpJ89es2S`aF$OBTIdRx~RpbzVsVd{?^^Gv_oHR+(EqV +zw7G&f8F?2B8lT?7?)jA(j9{1T3>&<)CQPelJZLPgEK0YPCP3p1j4fsa9y>zNU0*+6)6xA7 +z|9Wtn`X|7lCiO4Cp;j8KL6c2y92SBF|J0f?lL4*3y61Qdy&8W!oyI@cTQ!vT!9twF +zTCFkP43Zm=WpUTIadR4GjkD*waU_FL-0s_x!t>`N#MA#vER|?WD-+SprC%^Vm9=s& +z?7WKT+h>t%M0JC61N?3 +zPW;qQJKz3Y4$l8lZ0w-WFDH-8Ml-C8qkUtk66p6o;~`%}IpSC`Y6c%?`n|?ZQx2Zr +zx*z(Ro5T9|VMdFB0o4s)WGIMhrL8#AcuR0`W;%|8ChF1mj48oU9Io-2*q`>z)Je)~WQ*NY`0F)=goGade +zXLq{LF*L;Jpe+E!Mlbov__8`SQ2IKC!LeZHySmvZ{w +zsu&%^Y^nDJob%M5(slR$`g0inoX2XPz79i`7;0;;7U6SM)htG{C5F0@W3ul3M`&Gx +zj%VW;J!tt8_k&{j&Gk&DB*(;>?|DO2B;q|f$YsBXn;7tn>m;|WuQ6M^_We#mKK_M2 +zHOQcb9OqbBdQ=+3&T|slUlskIbmwF!8X^PGefc_)LP8)*S<=PzX9X4%nB%6!tQv); +zm=y4EY|LUjE&Zk6je06C4c?!Fc5v^AazsxgH(Efp8dKt*&p)vff@Jp;?cgkPiw95Uq!?Z3RU> +zdaH5wzc3_H1?ljazcPH>D-*kxym+bVJoa#h#1I@vCEe18X<*`YYVS<3m%DElI>hVh +z{S|_sF7GEQcH3WjKcp9{V)zVNK1S71*7OY*e`Cyy(4S#I4X1#buHp;>K#-hg$ +zrynD%+P{ZZ+mN@8)9CH%3serzTbnn6Di)5B$;k<(9d{qac6L7rrEAYKkLw7&rwiZi +zmo5X)%TMA^pPZh8zKw@-)lI;NrOIoiS`JhH6V}KyuENSeO!-?~nPPE?Qez}!okyKq +zr>w^Gcw!Cvx!rH_)Ol6w!#ibZpR`lgz=lztiYb0}_h)at63LnABqjcmg3GM%GCX2`OT&(ADV_5ae|WNpj}Cy +zO5U8RJ(i)3X&2d?VvJ&98Tnoy!@_bbrmY;+kN&j3z$ytU#q~cO&eztgVzZ!siFW-mz +z?bs(9j6Fh_c$PYJozuJhxW1kY^60@ +z#I<#pyExw^=bONOG{yua8Mr?ys&W5bu^MFbY@d7tp{38uT-udNtM{OtA=acyn1emw +z^E`RK|4sg&k;{1O!1m&{*D{o(&5exMD*wf{M2rt#X<4t|)VH!th48KG|I^VY{uu+h +zbz}Ex7&W=a@Upk$m#V<47;*To%+p_-upy*V`=+wvyd-&cOuZ$P5q*@S;`3to7oa8s +zdh1Cq%59)6(Ine76VCaFM3>B`#n&-G(_zc*>4@-kw9?q?g%7RBq1JQg>y*spf5}Jh9~=p +zM5T1o_2GeS?wo*^`1Y(@q{j$0$<}g$)%0yUn^JqW06dak!awMbDTEZcSm6rRT9b`0 +z8@o;rLp{QXwi)k=q~2a4c-9m8Hj|`S-<$SNc=y%rj^8J1k2^qPnW{98HS7Rv{Aj_|Z#w86Dxh)|SIshigoORtKGL +z`{ZSs#yVqo?Z$YBmY)**e1gx;mOc~(xu|t|CebC99U^EZ3Ql-wHhoA*lR~8bjJ%(ATXY&76Aqgm+73D{*5A$^wH>v5lS$>clQSr;cyZct;D3$yvMP+={pw(Fm{T=e +zRPP$=&~^-~gT;Xf-cw2I4eEH;CE{hia{;lHJPnpj5iuvSs&4f +zWU#}~u>nNGqEH&mIOQmYXclRr;iV~juq4=VUCg1W>TtOc)EVeg9KlWH +z)zLpssOduP+`T7&i0TKCmaPg$M8qL*%Eu7S`iN-UG#}F~N?OFf>@T6c4PPiyy9wMafxQK?o9V>Mv5i@nDtz}(1#KAP$C;&m +zVPRtC4cEDgszJf~RC*Sg_fKv<9wl=> +zmVnb+x?Ewe3!5UQy#*y$h3Xt@!|Gr +zh)ohcEA*jrHxAw{rU6GI^sg?O`>mh~z6%9Nh$NcxE{ejpNehg>a +z(vG~sD>O_+Q^5GdG6LI{ccr%@1lT`Vs%=KYo@MmX(O=wj{zO3UYXHq;51i4QTCsb4 +zooVZ}CW(S(a?m=(ItOUprt;Alg|kJ}YqL+=r3ie3@}s<#`zrIZpK>Ar1Rym|@F#Z! +zp>M2&e!w2R`4Br8AuZeaVVSv}_8#pZ{fmbahn=vb;@NI4fG`tZY4S-$^Ft6r*Ab04 +z@Z6-_LwL^gCP=gE_IX8qzB0Srz&Ce9*g~^P7+5EN_liDoeI>lAaJP*vyX)Wz25j0U +z;>wB~(GJ^YLtvhf<VN!;Uj3nr=b@n+~GA2{Rm#3tap=en3TL88}k3{ +z4cXN|lhI5z7ywgX@8s0*5&donFQ)K4CBq?aO4#t~RRMnwY|22mL>d%qUY6w5*+n1p +zrpNwW|B3K@p+T25J@Rtr7|QNR^ejg`A|b;LWV-)ZPA|s|4J2$1<|w*m*SodtM9{?M_RzJD6_;E&JX?NShkBEJ09> +zP!_ZFrz9eE@!kLmIjzQ2$vNoPISJ=<-Ko}BtOSQ8JFt}OY-}x-mpL1tm9mjOS6x0n +z-X&n?GKosN5?rD750wM+Du>DH%1}CaT&hYH27GlHcnTR?B|H#QL#ROn5>2E%@!~A* +z=Op;YLf~?ZvJ>~!q8-&QB+wB$NWLIZxW3S*;p&Wq1=u(}ioioxH$)T4dce~EG+*#= +zZpPp|={kBXgzp#DZVeE%x~)mDpA_ScS8-y#B(d>Vj_R|>PTOEIf7_5HHyurHUf0iDV@|7Uc4ynqqD#FjyDes4??Ag<6x-gMoTvv{#frCt +zKJ+aFowH;t&-LuZJ!g`C9$iqtK78&E3Ne4M!29y%)C~mc7`PiQKHWl#lipB*Rwn{p +z7D$BY-NukxUQOD+g{E4*Zl+xiQ_L@DrTfUt%W@#q)r?vgk8GLkEZG?w)vIT*S@zv5 +zdJ`ycH07lM;M+wX=Q>B81N@cmuKRZ}$^=?14Qen#9K>nEzXJ}L`2 +zon$n^bUgilUD)REJDJ5#%VnzqpIpjoFnS!C+N>-*eKb3( +z2?^*r`Q1>XqPz>5>#3MoQj(^h0K01O`wG{4G81C?GHKllxQHD21aBl71meToQypxP +z+lxb9c?no;c01#Fysy|Nogn1P64`@90Zw{4lQjO84>O2m5h2uh>TJCMmQ~ajFRO4W +zxNHxbPvu!g@!6Lfx}(CC`Vc-dib@1AHSY2xaNPGYH@((cOaUCeA}2<+cBXf|9yGB- +zYbuU0h5y=^^wK|gtl#BYWo+v{Fnoz0&x^-4Zd}|bReZUobX&{=WcarKo8I^hZA3HP +zjP05;+%XIdr_4W2+8iwe4w3k9iN<3BZEYP8-)Kb^yQ4*l1%3<3WB*jZ{^^u5fd-pW +zwR&$wis3~A*;vtl{^A4thuONSR8ZKe-Xq%`qI#1%vyGP((mEdab2$&xMbEPaVN6`r +zVIqLc4l4Vq2V&#h*&k(Mu{(7LY*i0npe-I4@pfGLa4ommVcb3}QZ&JMB4QSbK#2=$ +zxtdE1pJNUR?;UinmRIakSQz!GaYPC+i}cl +z&CFRvQfRequ@*|@`>RiHCTDPae?nL=BCVOkpV|ieY1T;ooLav@?GlD&K4^hMr1EM#(G+a9ZgkXcLPYu2#|MTd2!raqgYZf6Y=t`Vw8%vF}NL +z>5h5Y;M;+WLrV|+)o2m7(vhChY +zh+R3VSkZZ06*X2i04w;VS>s8u|KWi?xoWp3C=X`WxbhUay|Ck_^Gt$FR5*sMH76@w +zj14>3ma0uw*OOiSQyuYyuh-L!zk2HXZFc2Vh|CVq8)_cZ4B(?mO`w~2@;|< +zaiiy+)s!I_ak%0a1Q#8lKJ;-z-q-VE)Fl%1n(t!TAWv`%p`n+&ren2^k>ARd2tr$h +zD{5*t5vfwonVVl>T+)chfQ?z2JDPo3sJ%bX=YMz^6LtN7RR!d~M6>**4Xi4_@3V5Q +z_61vzq|PEQ{x1BS%qQW^)60T|Dm6HH#bzG!GQkmpfUYCoO^B(h`pI|4WfoV+WD+^6 +zR!FYfL|_e?Ne8ojy${A-MwtZjc#Zu!Fj{$I1&@PL;q#2^)=>OR?u(&Zb5C&L{i&me +z2VTLqM#@>wk;z3d5VA`h6U$08p3&tm8{1rYQx{Z4A+^Drd%E&DSoaEsr=Xaed6*E`bAK^Q8hNHC=6W{9@XS7=AJF3NjJ^z~ +zlL@olllZ&E3fDC9@}htw;}FtY#m(2*F;$`lo;@lRk|&ral&`Qi72-#qaETqGABACH +zH;9}-Cl}d`Jc;eq@0BVX5K1~QnsDa)U$NYb;~t( +z#c_z=KN`)j)58`eX+9&ZL1L|YqKPTn9mtPg43&?5)9{L=;&%HhY5+?puNjOLr9&s~ +zg#NU-A2&-R;7ubMAF}ga#7GkR5MCdbS5Xo9ua#mkmb}~zJFrQ`x|_5tT;lQ9)Be!n +zSLLqIBfcZNxXqixTb42-fYbKjV<&wU%>q@>lR&`&VMb!)jU9dWSmS(xf6mHabpRCj +za_5Kd?cue}V#%c^vpF6pSFjd|l~Nd!I2g8xWMd_u>^K@kkgTd62{LM-Ug>*(~g~ +z#WCX(zP@?=!$^_42D{16yg}5w=>>@G?WDIr<`>ryvRQ!S&dbog%vbB>M9N!Ygm&@y +zZdvEFBO%IqOtj+)A^})9HDbpNoGHVXngKP~t}UC@BGx_Z{hP)1#J|%bD{Yw>2ssAC +zs;_@_F_idl1W4f)Lp%=01N2U*v9sicN6+}Sv*P9P&=oXoNa?K@(5+Z>SY*_KerpSy +zeLYZbVl!OoQ@2Zd7k_S_A&z!8gXNUOCW+zX +zTu>kLDmvO7m99#8+cgI}RXAr)IWOp)VlMMMqoczoMRktA4i=gRAGr*3}}W8QYgBRO*|Kh$AM6lckzw=(v!72vJ04zqsRK-1mWZqIx7 +z(G@r*6;vY8V~s()Ft)K@%G|6zXiwl{kHzMI8}0IBGpSKlt>YnJU;D2~n*P%z5MwMG +z@IE$tfCwvd7L8g;Wo0jZqaLx_bo_tj*Qh^Q$g(y6^pfy$8U;kTO_gs=81^x) +z>yHd$ACAoX$BgNY4D7k&f8O}>cA)BioWY8`NNk|uboZ*|I}>u`$(r=t-*<12Pdxlb +zsbvW@c8M?Gy*IO)G5Tn`*E57E5Xu}B7#hzZ5ZTR6b$^?WoX21C-*`ZGyf6y+DRm({ +zwdUBwrR??0$m1+^-=Qg3!^YK6gb`LU6wey}@NWkZ;&5pM%!SU#BP7u`>Wu@m9oo7O0ai;A``p3C#;3kf2|F86h9(02NUj8G+XtERb6s6 +z-p^g0&Z+ +z#uZM71(RCGPHNu6qSeOb)834fY-g)o#OMhoqF$*f6cc9O^H_a!m +zfo37Mq82EQX{a!^Q)DAhk_2GX<4EmFCj~-@|;rqqMa7r0UF)VqhsM9+gmfNmkXLO@oC{VSg$Z +zeP&Y<(tUVrzMb)X!EZxQ$YCiSFOwjzq!I(@%JJ}~pTOC<=(!^LdW$M;S*1xtwX&(_ +zj+>ffMSD6}-mKYqpcu#X`U82a!8g$2?Rh*#(kg8{j-FGYnp9kEkiDq+}$dDLxsoA~s&ja>3fm)ZJY88WXW-i6?1$dL06f +zgff}A-<`8^y*K8te$`=fdLa(|85OyD8xL6-h;vP{HI%6U^}z=TvxBt{^>$&(bwTZH +z>+V+g13y-3MJzW?So`)ifZjiwum2G2FZ%CPS=vxy+uXBM($qAsN<=;6Lq*&x(a5V- +zdNzKz8}B#ATqV;GhP0njDWax6ZaOBvi{`!NLfyJ&kBCJj!vxOg(I4T757ut)1E_j9 +z7wBFbrToAwe4r3rZ%)x;KwBpB0rh1S-dn3@5z8E6$9gbu4O?In7_V&oPFT=F~;M3!QI=pi9aDd +z7gx7dn~wN9dtRuR5xEw-wQsO3(0a_8L~{t+Wk)O!wWQs@Q$U +zuHoNesiY!DHc~C)is?2jbVyf5X?f{;^Xx3Itm%<@a*cI=g|==@F;}06`N{bF*-D*= +z|L=o6(C)uz$9z6s=+#7#iuC8|#K98Pntb +zahZMjGiZGG3P>q+<}P$%z!&eHKK+y6OuUdwSeA*#rI~nQS@=`QoM#|fkJYGsP}|ko +zSpa_VxYKJHju%JgjLdENmCwO}{$rh1rq7TEK!xw0z)my3?3Jgkz~_BjCsVOpkF_3V +zL}*#V$Xu#}8;-Yp3E?C%##d)LRec8Zmf^M}t|feWdyfaS%HMa#8R0{Q42E{EnOZ4? +z03UAX^7F0HI`M2Rek<77FcN9I-$DBsqQ`j4jYB~zTKdLZxGj!AoldNmlo)AUE*w+t +zJH3QSp<5u!uQ5sD5tA1+FAdnszsXs=$2b5=euzg;h;?OtUY-`mtzhSsh;1OJUpv7B +zd@pQT=i~k%smP4=%ziU^oW$aX><`FDFfS`L@Z>AIK{d8-IH`_z*12%+jy!vIQkrS7 +zo1K}KAUih +zHl{Sti&MOY|8{{V$d4Ny0hlq0GOBUmgb!LevbKRRsAH5C)q1qJrvk0ay?_URxx6B6t|h= +zw~ZP8rfF;#Lt!RVxlY%8I!paNPH8A@sQ{9wrzK;h8y}Wz+_ym37Xbr8?fFjW`>#F| +zrmDGrTq0NX>}R#@Q_T(W-S+%3##{LC${{AIK0R9bC5ut^OsB{e`E&rUnVRR@TMY4W +zXW^fV=v$=P_c+wMIiw@^8-XlSzR^S15kuxyGKaFVZahSvPv3GCjkb^?;cF8#Ka2b} +zKVMrE!+npM+lRDl=~mr=nq{5I6|xzHs&60UPi%h-J6=q=LLPX_DwVXJnq{iw4#{SC +zKU~YrvlVS?Qtu~f3+5ihs!KBO*bezv;=}~3)h0xIt~Dh2qKjNKs}O4{w;3lUDm5=n +zwzhF=RGFe+2Y$5M1V#V!idk%84NSAngVhou0P4c?AegR&4ntX=mxuj-m^urnsJ>|7 +zD~;p~-8Gcb-5}ivD2Oy8Jv7oG-QBH(prDkLbT>mO-7z#n$? +zsioas_4Hqi-R#f>#J6EWot@y=+f!sWiC7s2F3Y}f0Y$p@=PTp7xXF>rRhE^J&dXgL +z7_$CnK7KTNvio~q$tnHgzhwIW81{t@lV9B|2lbQtoKr?<=4RN-G8(&U^3(W6-6vV^ +z90rE|R`YZY+@XjXbsveUidJqChlkFeQ4{RUPmwux?W&Lh>RtAwd>+`37HK*k{cLcU +zF$dCzi)`H2JTgZG@q2QK{kZuY51SPJR{Fg~Pp^5#abE7uFSfkFXJYDsFhP0sK-yie +z$4j}GW=n9GN5+3^C&*-PqiNW(k-Q=atGbC#@1w%=mhy=sn!hn>yQ@=M)d(o^(T^IiA!q#oduF +zW^2yqubZH)-2#2wRbdGeui;*_#eCuYCL7D+B8hZ5}3J!iDU! +z^yK9J;in35pq5%2-2C;)I!E^vsx;*c6G~Ik)5Sn4>>;`S1zuTj@0soUBejM3Lj=oo +zP@4U$%uZ=QwRxS)e2!^L2c5xMUlf0a3j!eutmSp?|Mcwr&<)z;AttPrRg%N;I!>x>Tdw?1@aef*+LS?nA__lIX0M^bUsZgx4x^v79JP~pET{D75tJGHNMm9YT`=wGa#hBGUpo-SEJf=*yK<8VkI)J% +z_KCgfXIaY+DdiEl)`^$=(A4%0#sGG_ViG~xys2{$wzTm5%LP7Bl7F@}GOr_+`1TBh +z-rYm8WA9xvyxEBZR7I_lm! +z8?GJ!lW~-*T1&2fX-iIk-eG=ufItePPDz`H3G^{MayS_azB{7leLV=zAAUMl>?f?uYs|JR#5Mih;iuMau00(q8 +zvY|b`6{zp@@15DV#nxnNYqbrp=f5j{GyGosO5V#I68%shvH@i9Ma(J~wM=RpYGwSq +zBeD00+{x2NPDw5*)BPIcB~t0BoQ!L>nV1q7E$g%ynGA}Amjy)i<84j+H77pg6(MmTL$w9cX5~;EP4VH^6%ac&9h{asURKJ!iu!*g#pgRXgLIZ +zzrzx-goPX=-+x3F0MgfsczwMHnXO4Uy2u>Kx-i6#O(Ue=Z^(&oUNHIP+Jv5!jHInnMGXEJ)W6=h` +z(^7xdb9iKal?MYV3FA&!TOS3B@e=6pxs{CKkP3~%=mDGp5lWv&DMWiM{4_4Zl8M~^ +zSc0Zfq?k&wid2S=?-meJX#Xpf=E%3=xVWceJ%feZbPfsW=Zf@3Hc!4Q0+vW_5oF94 +z$oqw?Az7v=d;yf1$IC9V6hPFkX#)7j?7Q&U$-vufjUj3DS(>MhlJnNKKTv)pihkAE +z3cMm2xO+MHD{Ls+3LwNmZJ$xE8hI9?d@~{ViZ)@Mn3UFZ)qasHJ +zQ>2kNni!il*h)KIN+ddvxG@?rni=4_ZYr!Ts!GsT=Vlu--fjOpVIKZv5jk%yPoKFy +z#I$w@<67zqDuTf|^M=c3sb%xi#Pwx=801m`upAgaaE^IVq0x$Rev%JXT*~-%spCCtiL3u@Nd*O=wAXI| +z*lD>QThAN~neBmKCke|~Y1#I=*%N;g4H{q;NTWLKkavL4?=F62@4y%>*ay^`j +zUYj5GEeWvnJ(bEKgv|FatCM3%EQb_Lm;N9%aDbHt{K+#vZ|1_XzApIma+P9Gx5H3f +zL^W80s#*N;GkvFCBmlBHev=z~1#C?;H8Lm168HrPpceDcJf$((C+|tmi|Ai~owF1= +zA+pG^An8U;-ui`aMqCV9&4p>Bf_gEqj+=zHv83}td#lN@v`v7m)^GC_W+nNAMUkn} +zx-*jHQ`~^g?cNKh&_k+*WAO0rZa!Lw9h@|CWD$4`DPEud`WP}FpgbZrhT5U!vvcUH +zyzVF2#VGNzUlOwH61m-mDt<|Iv0$WfY9QveRzE_JUu!R{-CW?$z2FR-Jl%2=%8)9R +zsIT!4YRFE+f*tbv1uTdO*|Zh?)*G~TSnb4V+?WJiZ}F}Vk8<Gt)?4OS)mpBPC7aWZzCkzx^YoszwWis;%V2?KIf(J?kXQp*cR0O(R*#*&XBZ +z_%_XP&aQK#uL)D6@)3>}?K7?v^p#)65dcIgDztP +zVf8B|K6liD31ES`)BK=X_*6PFi09UCQHGq{I|^MOelaj!Z?^5G4C}40JKtzEpkl5u +zLcZbW0F#-^T23OHpCs!O2m6LqE0{l^p-JJ4SGgW6Qd=M$>~$&keO(_6Z-i&&Hv2{Nm8kWFELgHw2;ibIp)f3VeY~)? +z{k9Aw3UecXt4b>^rl#uMRDy{{*#~)jZxnA;&<-c9j)rF#lBS^|?kjX>FbtP*ghKcF +zrUVG}wVMCKo?$bx%M~%O4Qgr7_$4>oECaZ +z#ZVL~`qn!wmutTWa;&ft9Lu9>^A0ywQ6@Sv$MX;R^CNEQ3%B!5^y+l~dFEsPPh12s +zyMDy^p1g`QZks-n3N-Pyns6t2{9~i@Us^da$E>xo_z5lFj%_DJ!e|+P#AD}lzMcK_p +zJO{m8_@L0WwWfD%hkd#7Ua@Px_6(-_9VKEbki;S-ZPmu~3rKR)8hSJ96&V}!Gfjrk +zm8s98#;OE8|24G~X(0lR0q=uFg)ZSKcr)EW>A?O-~EQXm>>2&c3K_@*eIGifu(` +zp8}kzMEB@HlT2N4ZW{2}?TnXeSQp2LEc=(03k3s{N*AP^D&W0SP>c!@WpC6~@}?Mj +zZ6ozw?qj^c58rI7o{zuiownAM-fn@boj|N75%af9h6Dj3_d@*-NL7q_Se4d>dL+upRLK>*7-EeeHat_ +zNof35YXrz-;d1wc*JEAxbbRWP9J%UzK69td6vdR`U5Dd~pPOdiNC$aa<%0BOGcuRU +zxN5+StUr!wV?9%8bR&Lz87JK=8urR!=R}Voetp$Jv?VT5L7G+QZAuUx+Wmp6tS0I2 +z+?Mll!qO22CqGiZsJZ(S-VvZ?m(RtvF>jX0n~46LO7p42hJNRlcqkaPf7Rn!HYkVB6Yhnnvpx6)0x33!B2E@tZBP{~e +znn5~-kI)I!!Ch}gSuer7GrCX%3zze0=5uK;SrnlNH1)U&$8U3uWM^L}oo^R4Bp!|` +zl}9R)5|3Z)dq!op`RsqIc(dFjt}1)ew5hr3KS_N*@@$B@x}*HE#nX3qOm2XSjp^e< +zstMh5t7SiIR2a$C<_Jm@DNipZ6&A-6r$vg?Yr|g(FWMyCqRXuGn_>OTzmJ_s- +zT+95j^@~+j1&b+twNFl8h1HyC!p>z9{F!sJK%yD4ZdyE~vxpnNaIx9Nzk=w~5Tn#e3jAp~~k#wIcuAqMGh2pZH}`;;T9{Xvq= +zvsjg-SJXE2&SjE0j|mD|I=2rowVz+Eb)|`0lr^=o-(Roj+}&t5m7$Yyz5AH6Edu5D +ze#w&7?cj5=U0Hnzr^pIn(DoQ)=eb4rMkTU;ea(n5k>}eDA15z;n-)2!XY__iRH_kA +zEu(G_7Ry8_qPo3a*X(t@N06yxk;^3PaVW`-jmy2Vc_qPCz!X1H)x2JoH+bs3%E==U +z?sqI2{r)&aLL>M=6HZF@rL^fCDy`(*^W+4*pRSUvUMd^LS~GAW5|QZOBD(D~?zDR8 +zfZx8^E7UojaQOB+@)GFtjUf#I>ugK${@LLIkU{aEIfZQq`thHS +z&XOLDMf-deu{&&I(usX7Qk*S!h+Mndczr+6*ydSp21$tad$w}E5}jLjwUtXYB1D%?xx*ig9||FsrkUQYji>Iv@rQNUxl9iG +zTCO-w7i$_GsF7I79tQST({{@Q)1J6!w(W7Bc%N0apuR0C=d-a75S2#t!uSRq<@N(w +zf%?;GhTSt}I|4eKTIWx9HggfnbK)Satksj%;&+fwRifWxkFMgr1-M_FynaV@n23AR +zw`i6`u6Z~zkgaBMk5RY(_{nxUSTi(a$T)5+p8-3e-Svz%TDdVixhmEfL6>CPMpD6=$p#GaM64Al)3-$RK3SDB#gp*xVFEaqragE#X4Fpy^`-LK(F4+vaw0A +z!H=I`ElWg}uUNnSdc%h)a<#d(LDs{w%DYx=RBOu`_fw7{g;P9U)m_fv&ee~~oFu5N +z#ewL;`gPyddYw4P(LSzU1;bBM_H#k`j6?Ub94oVQ4`p`k?MqZ4LXqp>B4xxqY{Dox +zlIGzynf1lE$f?kyF=j@sy$@H$FO0TUT%TW5&U2hSI6e@02R}E_Zc6P;aKeppoxC*- +zHZ&%X7pe8`=%ba^tA2MOG|1U?w|!;E7YOioTuR10pRA+ZBz6|nlz(w!lrQhSCkwsP +z9^9?=m@+Z?z{|<8wg_-+Y_(0+3W6QCMEBa*ErAxy5GR=&zeFcDI7Xq(s0p_Cl8LN= +z@OmoV$lC;$&6|-XDIv%lx5K)F8YH`_sn*%0c9Vyl(=@gmiPL_o;_VuwL)Oz`@Q-ut +zFMa}KW_&7f4B~-zpTiIrSF(Re^M3+P!4tIGb +zea{6z79Z^2!}A3RfuU}a$_m+DmX!7#yY^k6LXP{YbG#uzgFzd(pB_c{(a{m?EbWM?Zy)H +z@q9&kFGX1D)t*%+`(4)d^{O|a9|v4AUwQ*<$vaQw$2rq^Hcp-Z0v{<;-N9T$-SZZP +z4F!R>m#iP~?tOB3r6|aj=YdCoK7H=7u+YqHH{flvL7R3&51UuXnNmcBnUr@6{vvoDb8&D)SI-l`&I}w`)q$l +zJ#aZ2oNk8M?)~B1Q~yzEyeW90rd`C4zWvXp+X`ifgm!{Y9enDjfxiNzTf~|x-Z5R% +zwj`2hm@pju5RUaGsjIQh*0;D4a))=%3^EZeAnp8{Se%sBN=C;O{}MT7 +ze8yG&)fKLk{vRyke+F|K(wfR5d^|rM@P@r4#F>;kKTU=z%~o6{I8b&81G2-kN5CjOv}N@5mka +zhcl1fct6nG3HErLBV72O$d7IDxSqZ1fPUV>G^=N?I=*-93k*QxBfT$}syGVe!!~6vTbO(6wSoZjl<;tXN@mu>5ahJm__hjV4N+UkE1vD(oMKbx$ +zFJ9;{Tv2otA#$UH)M?9i_isv0@SaKb +zXDZ!nxC+?j$@p%(20K``M?;d-=>Dy)g)7CBDL2~p0_3{N{)fF3hVB7IgVU+#W0~=cesD(qFTUz8ix_EXu7%#lz3`~S!%@d5Jh-q;8si8Vsot*^Mc?Uii=|L4u` +z3E(%?VX43;VBEdKwm=T~KgU)Ik$GHrHx0L9@Lz){zzVr1RSg}#)rNTpk#N<29e{i{fBVJu1?pNic7p4Z&f|Y)YEl2Z2<=bL@j@?I +zCy!)PB`IyQt_B9UkgjY^QKi2PQMfwQ{zI`@BB3L%Kt>4H>>};?ZvykT2k7g-NkVfM +z4N$X3u@|F1C} +z>ye7K2Ix=Azi4YWC6+Nh)n!9UbdES+!AOhl4x#E-9Rhf`OlCi%Z>1A#=`&xKK_Z4@eEsFf +z{m*gRLIvcn<4_$VUIkF1Wt9Ba0sqlZ(R_HO^+6r|xE!ZbBS}1034Wn@Sw{qcJko*q +zIYhVQ{Wo$!mHwVt|Acqs +zpYUd?V048JB%DxT@fR5%_l$)HQ!%UwO$&eR$;Mj%(aDDXWbXgR+95R-;1525>~A~z +zTc5syQ<(>|u0m?$HM8{{eX{VYfA`M#3+1s*@zmml63lhMugp_AW`A`^<8Ar}?1i-G +z5&a`w)&A1jskWI}7|3KlLGY`n;y`L@7qd((mTlj2tzNMU%KwI-qEBsAJ{%WT +zV^jRSDs$u|YG+0&UV||DSSFhEiTpaZS-yQ+z|4s!!0$c+XteC4;q2duRzRl?zoJuh +z_Wt*=T^@e%=jY$A1v#s~{n*+SDE@v)W=~Ja-vA}xz=$AW4KgMw=ot(9`yNwzc+2h> +z0f)U~Gwet7jqDQV(+2T>tYiB1g)x#DL6hCbv-L(jTqp1&7-VhD>qeD^y+y4$k>Q=i>$#ZbHJN;) +zkB{U!7wwSlv=@aWUc+*Z0Ar#YF%^lJopW_8JAOBRGlScCK+9bxxP^6FM#g)pj+xL~ +zdR1(r?nJkz=BoL@uXdR|GE)5s;fe}qdqSto6s9 +zW=_N2^PvQ0qt$!r_`f90Sc}u@|Cu9MW;}^f&hMeKf;vpm?+4i`3rC +zB^vJI1*4OVb@c#G&6fUl%;;vB*N@Iz%BMIA!af2X1H4+(hH9H*`*k!Um4%*<{P4>p +z{EmUIk77)QWTCe?QKCT8{1ca}?CEviSUYzF_tL(i1~Ce#e5?M^s9B_ug7%rt^ +ze52a`QX9Fa9X1BL;pP40>eNW5LrcZocS8!3Z?$TQ#ft-)*kMfmJ3+eUX*s+P8(sFEjd?H0SL{IKeHkLcLeRd~_;%bZW9k08=s^ZZ8N0 +zS4@J6Z?2;Fh+DCyHx$7EZ^CBxeqC)xg-_l3zMGW`yfuHQJjjvkOD$pjR}Q^}{|Piw +zHxrWsum%HAgXAu}ecxY2EP0Kt+pc_^^18S)1w%V)f&Qx>z@LIvKe*-5f@9>~{5Ex{@o!3HO1av!m}W~<#Nsjs-gxJ8i>tih0}Msc3-Pb5^ImtE~VW6l$XH(CHzOC=;~ +zWn*ar>}*6qw%Hg&+X@mod?WW60|>ZC<_z>%Wp}b=xi(LOoy5lygErp$;SysP6;VE` +z+>z7!ab1D$6=|Hznp#husD4Sz;wNZXhu#eiWW~JM(^7bcilhol+9{eWMcmee9G59% +zT@3JA1X#G7zQn$=c=K3wd#14F6L$6eWs~Wvx=~j;)P+>tMw!0M{_KTRL;17!V+EPQ +zeRry(Y_e94pU?C9{f~lsU7?K}wGwUhR<`AffwVYW1yJ7kRPGt)oJJ@T6MZdnSrr{% +z4S@Qu-}Fm2C0M3i@MZ9ojdh3_2|%0t+S-i@yklWzKAN9nPYRiz*^F#66x^gHa&8Pb +z>rBW}8d7xIbT6(qxx;tRCH$_56bo+OBWu`9kh}P@Kn?y*r~&y<3;Nf^gk)`H(W#24h1euHb~>OW)%o855@Jtmo@r +zW%gQ-skvJ361D{COy|F+-?zbk#e_Awd4}$Jv%I^M_KFV!#d^QW{cQ9d7quI@`vW$8 +zl+n1vFT3u=fDLtbUkQx0Z@}2Oj%XA{yu5y8vcx8vkN3wXubb0W1lHm~hVG?v*z*^n +zza?3rvs^njAC8yP@wmK;;8R3$J*rp8(mJK}Z6zKF$VHGia#oO$=gxq3H5Dufj%-{} +z(=3HB&U=rD@^u~1IXzoD`(l^gDcfG#k-lF>OI!InreP-lX4b!)^0MFX%Q4$881MFt^R7?AdBku5 +z;W+hkGkNjFQr`U?Y8iqH-}6C3WQ4KiHh%3*$7hwxGk8?YdB7K}o6d=h#(5Xh`JS1) +z=IJOR>u*DM{%|;nqfVA>R&SLUk^FdpXkbd;L){`~n-Vgq#Q?dh*#VKTW!C +zoh1w)mL7?$$S@>%`02U{FJ>=&IJ2%|5!4qOc3tHL?WxWLDcZNd_XV<8W`SN-nESBLDt|h;Yq0iCzGWn+i?u +znViLh71pYAGv*q;9JwK96qe^a;n9wFTkq!) +zQQxq$n~Of>vPur(F|YDLt>=M9i&9~c)SS}gWCrOT&Qv*0nOsA9nE0I@A4z*Lbk#zh +z<5k4J=A{`)vmXqSkricUCTj(C#pkww%I9q7&YZ6lE3wo~4vp2eB>nF(1?nL37OJ#0 +zuEXQP2x?4%0woif +z`B*jmG&&*FwW*ZL)k)D0*3r_o_#Pnw@oyhHFO#W{_Cx4>E2 +zkj2Gs@9JbC+k6*bP9oME&MSX>1#~N`{PbQkUCL>}g&3?A*tO2=0PnH1K#y_}LY;lN +z9+`^7I#mU*SD?Lk{KWgWkmS}*v1GteK`)lg74u@~{MjK7ZX~aC1+vB>G#fB1izkcxj4!+brSPs;|%|X09YFGTT)J6uk!3}WHkV4X8 +z;HNm-%2<`j1)CAQ +zXToNDdy%LA1F!piT-XaG+`0v$IGB)c3Bj6B!Fkler7a#>_u +z?eVC;Rd)wxp;6+^TlsM3w|%7tbW~W7@G4P@Qtmv+W~f-)Rp(hWOuej|Cut;-(9tk6 +zB(SC$ZVL2_yIVwA5O +z*v7c^)xC)elFnZYF^_gE(nv}qIuJh~X&FEKjWD6*C=^ss$aabL}c7dk2)ubijp +z^n`=@qm0Ex*H2ocBmr7#paAM`v&8HJoWkxqiP&gba8STxN+^ +zx3OJN^;b^5uV(1k65v{SAGd(NesZ=loQ|=~*Y#;Eth~=fNWF1ILJ!hc)!C~87^Qwf +z9oKcn!dV(SmdBbz2y|go5x6t-*$Lp-f&boM7k@!*Wb2GGr1;-;VBIi?-tm)7lM;ldv}0gq`Kh^;qb;2#ECCm +zLt3Z9;xn!Y{fnH&`i?P8&kVcVSTwJ@Snw|sjvrI0ZXbt7b#G%+QPbL` +zg!JX%m6O}%XUEP~9-R*DVWpRL9aMU#6vBXV9JRK<8xqni|W_{+H7(4 +z6uDx539<%?Z9l-7G%x`7!&FM_@QiP@+lmUW>2-vuDhQ<^C5jO+7CcW>z_|f`k9IjQ +zdx=bUk`pUT#Z3M2WM@vxrqqMnVUO5l)KW~+78nJvp@-iG9zAsReq{{x$2MqOct9I| +zBN|)KwAf`9ei)Y-Oh(^hLSy^=b=pNbEk={Kfa`Qn0Nw<(OXsJk>M(OyK2f9W +zWwaX^TrVkh@k&P@nIS2QHN$B!W#DmJ!J=kc0ff>St!s7X*~_2xkrX|m6khhS#&T5H +z-a@pK6W*R^qtAo~8{rmOZ-JzT|(TCsY)S{hI@tGLc7U!8il +zzOi=Qh!<|N?UCe_#(e>a7BQ=?>ew2<0>@wwWCKJ2tn0qAkNhaT)H!~$0mcCMr4=I! +zUmbY?UJ55*CV?)?oT#SW&<26V+fRu&=DxWCD@U4i(m|4<@!A +zZ#t)mfj{k*1KD})#uy=fK_kdQfMY0T=?03-t47S_l)Xv3>cMT)5G^ +z<)FJQhp(GmDnoZN^o0laBw{S%jaI)t`lt;F?TZoZGE{A(6(aPq$7X$&%R^`~q=#Z+ +zY5my0A%RHAu~4X&alT)%|AV|AaXLrwkGB5cp_222rzi@TQ*wzt)a7-I_^_cdqqxD%!2z@Gf3MHPh1 +zToHkATZsK_V8g+wN7Y)-kjw;geikOUa@{ej=b88HUTI0Z3KP4!ar|49cs>O!jP?p5UjoeW9B?o +zAc6O>FomB{RaL1!>o_4I(>9*M3WzN-7eKfH;n4S>=oZ)g-=9|9Z#0lm5OQk!EEOgV +zYK*w`$6{7kI}Qn8p24s_W0JP~DI=7c2Uw-(usL$3<_OQ{;WW +zx1+X@-8YbZuYs9%ZM4<^#}qMho-m|?LYrZoBsUKYp0tjxFZJhPbco+`b*&$Grap^l +zl@5x+a^3JLRrpo;heS2;W$6j&4I=4-c_%Qal&T(ssaSEn93Yj4y+Ycegckvqi+lOw +zlCnUctg;gH%=b&P<4=uOf_rDbnqZsY_*dlLjvyzJ+UK2&3V@UoogR7-;ZD%^$hL9n +zlt@B#j5>ZIAN$-Les3qVd^d)~(#WKB4)R?MKrd;P%9>&xj-Xn|gJ=d&VWjjIDb81M +zx$iVjVsZ_E+0!EA)0l58WY;mUl_^~U(PM5R()kmsoJIM}dGyKjkyWY~nb_1nH8ByO +z`S?PLrh^%@t8~Iq%v-X)rfSoV?B{C^Kbd|Rpo@+?t+ur`!9H69R;Tnt)^f#q_6tIS +z;g{>@yRz7ZWmumu{J^&Jny*8`e30#_t?XrC9Uq~wH%-%_Ga7N-7Zvb*bnu+{{N+im|z-yoDD?eLh<1*K}osrST$^V{6rpT2^ZN65aA +z%emm~<@~6!K7KyeK;^WYa9jAsTjD2xVcX4cMbj2{rTX|Whheq$6P{E@H!nYeA-4VC +z9U@C~@;UFE%wx{e#hpgnbG)OzAYa;~jgR{>?0mLyffjb(*qX6D4+0;FUZtJl-OYXz +zM+Pi0Z_5gfMJWFMQ1eM6n%YB4pdPt{JBzaUPG*@c_2=5R5Q`S!!<6S}$8E>F-qI6- +zODypv{;8jfz|Sii!@FJE=V+rdL)9mKnQRijR4wvXIrcdahvS)NO21b?27sOSi5GWj +zu~Y6gwiHr|9)GJsbD?om&^E}UMqQE<$M95n9*Krw7n9DvSp5(nny}|(S4=L0qTQT% +zV|d)CznDTW%#4!?Oc~oZ`Js4Tr}czM>hbev@cPZCLog8dwrq4LoAjXV@DX24j6&-d +zrQnAI`o9UXxnhSI>0kJv`h;jW9fkBHWX*PM4P-Zv?dJI=F1S#+E?gFU*id8x7+{@J +z2HUO5vTi?R7_g(1bDPmLs&u3O1mPqlxPLSjX|zKk9Ngtke)*)yy`<1nr!dFEK_{0JYF@#yQd)}A3YVpV +z2T{A|RXQ|$pmtVB(@#z44 +zX~4uJPDa{`+_34tfI=v0e@hfc}L#ASVSrLVk(&J^(LbozwO4LztPX7%?d5li_@@D6-8PT2;8;~vhl<62< +zEZ)Lk@nJkgN9KcQy_W*Z7efrz?^oL)7EFk1mAe-qq}S=6U(kJ~bN}X(|0d&Dqy1Q7 +zj`cM6~UD%iY3BeN|+{PYBb#O*kFmWr(%|?#W%DWTJlMhgV +z(&>E46$Yb2B8RQ&zvV~#^tfW)uNtDre7mhpeev=_BJ|sdzIOGk_VjP5*wa@8g+5yW +zuZyC2XXN)0g5&#~&n~lbloMbnz|BSHPE;;=li`>0{mVf{cy^jaSgVPGA*@NGX4$4+fQCG0mDGLq)IO +zerUuYUYv-c_s3cv(gUyU^_vl%@#=rSTv^Ao_E_Qp@`(c=d@}YW8@+yeUkkZ~M1@=* +zSP4*0(|MwsgvdXSaP-@~nD)YSKV<6*2}*y)=6KT0fxgrgsl{>B$B~C>6_!pI*%p0O +z5}Vef{A>s(ZU;+_4M_y1$<8n@PP-7cE%kp-rw$ +zuIOl|GW2o@v7eMFGUL~ufIgA=vOkygNdif=zJTB)aYzxVe3xd{vSR!oT)%gUA&Bx2 +z${=JOoAi8_Kl-Q4yDw|B@!R#iB3=*3jINTc6D;~yx060U6VHDMex(JX7Ow)1@Wja3 +z!PxRWCF^6pbh59E%XvtbK0qD;ZC{ilerq?smH6WSD$e(!R=BIJz`Ts|hCdd1Gbb!% +zTChn4sa#XDwDYwSr6}4HZIeoLp=R#Iv0a^N!=CY~b3&fTv%`45`~RBy%CI(*@9pBj +zg9mrFKq)T4p-{XOcPJFs;;zA6ibH|Y;!>RAQd|nf9g4d{-t4}+`}=d{3s;hPGBZc+ +zbDuMFl&>`@$Ll{cZL#t?k6HNb3(a(7r8&8@t-Fv0FQhF582eci|Gdm`$(=E0;fLtH +zt@AxlrNZgJLg}cv+rTKBn+Pzni7Q!RU*Tsw7hQd&F`nmr>{bR#%>PErYi$|%5v${3 +zPuQ~xWtsiimFc_xgWnY%!Q9BZ-IR;Jx~zmZ3&Ir#axgoa5}{%PeD}>0d0Pw(taOls +z>Q}88FVr1bw?`L7Yu{ye7Sf&E_5u%Od7}BJ+QRjlnMyCv9H#WNS+t~r1zvrlpx`vI +z+4=-^LZ_3&;Up0_Yk4Zw@MM$_5`^;}GvpNu2PU5GS72tt3u09qw|lTDgEu%G>sF2J +zKvMZw15`K_YO2aZZU?Ag?jq1dsVD^=kA$6CxSLcmqeI8A>x57nFIDQXU{ty}XeN=Y +zA!hE~R7Ti!FHu}MtwxCZt61BR6TN<}DcFYLCt=7dQ|`2gc2{3Tj&Fm#G-%9)B6$v^ +zr%mE3Qn>ECO$k}fWUE`Cz}!WY_sVQLAi8xx6-4J5ziDbAe?B>kmSbl&IXSUW=enDB6jb4XVg!F@$Pl)in_{Qv-YW09w0@NiFQhQhh +zE~8w1brX6DYWWWtwYFslyrV9p5d^OG#1lyr4cX0EO}r?`rG&u29RCGs7v849+Z?NE +zZ){Po5v>q|SND%F`X=BWr?Z={unPuTLh3vIv+Mcy)$n^H0_Vz#OcGd03ERM3i@QUY +z@guuv%Sco_yYL&E2;^E}Jb#;Z@cYeL0ATB!`Y?8@ilpDneteU~*oFF8lORkN6*suv +z0r`M(3Y=|^E$_c8w)gw$T|F)g5%arvM9I;3k>|4CZGg9cSzVgh-H&Le{*Q$=v*sC{ +z2jG?nSj_b{uH=cz4~l3-Z(znK6f9oh)T>?KH-M?bfOUAhZd_owmF~373AA4@jsGeo +zzxq;#@0_hA+@Cw{4!e75ce69hl*+IP57VO~yycCmxzVcY9x2Rt#?BW}G8Z{xS;$)V +z!La^NJb33!qR8qQ@_S@Z%&8zT)rOr8THjcZcpJ^=MT!c +z7df1yIZvI^8hzd@-q9HZ^!NvIlM65`tKVjLx=mE}tbA`*XsbmN0eWI<1?SQUU;(bGv} +zes9cYe}@;NZUgwG2h5$L-?1MBXl?3DDBWDdSO8$5gzCSG~O$O;b#FVQ!WGSk?djAKY#8Ux1@*8-Y< +zyszoZHurqQVV_UpG#!?(J|P9lvQEpxD~6PBlGW@ulS0cnZWqStwL1up>HHL?%}Sx_ +z_nLP*eje7uAHw~kV)`YKbqm3B#KA%y1bo^d)=KfZ*PeXe{Hcx(azkl(sZmu?hkcHv +z4qAK)oSBLoMqk3Ek1~VK1uVdU&8S4x6->!3X|5VlVa?fc(-Cype+6iiqv5ru)dm!a +zS6U~6C&Cc^3`q^dr6(G6VoNe%WP^gTj>aQ9VoVzu5ia;UB6o;zyFgi(Z**|VNhHEn +z#9&gM$|!`k_DQW|$mBRqondmIX1SHiCC$7$#BV|$mdb((j@EM|^Ef90U(yu|Lq +z{77G9CbpxqY4Z9QGB?&q-_b@=cgbZjvl713mDfd)gBj?0t_xa|s%)M!vWNbL>>^Ui0jRZ$I3`c_Co~1#?YCSpPy0E0D|c)u6BNBfc3sd#+ybW+l|72BipjVz +zzfS49LO%e83q7yTtCn!%;34{jn*Jh-Ne2e)%H9Bdd{Y|Lluys0y&HXisg{H#VXn$i +zRTB@QjjzOLbkKJN5B=$QsV$&74${c-rMP3mD#@q{8I30g4pLuiGF8SMy(IMo(eOjP +zN#kTnt7LShFvP%e8zGzDXkx$=E=?{=HI`WN+l&HFZN6(s(Ac +z#Q<^@p5U%5mM#Vq|G4F=n&|^``8%Y`Q2ooE)@$b>vQFu_>2)WpJ`iTf8%Yn=}zPab;n_ZR{@&5$D3q5rLsa>E>j7b?(W>j2O#(@lFP3gy*w$ +z>sfr{=wT?UPLNSm1W$nqs6HT&y%KAz&@w~f8G|ENbnlWufuYFq!7*W_3AK(jEtnJG +z;`HdN~ot0j}D2=)?e9$9vv +zZYyeqd8f2d4_Ag&bE2L*JsFyD;6_ZU?Jah{lh##x*Zf63d$QJI&KsTt2BhtK +zWDiI?y$sL;B8Htp>)S}FlP)rHw3MU6 +z5DjJ`DaLDUKO~fPpEGK2ksS?fBqZ7Kxs9(@B5WPEvb6ZrZO_U%_$GvQOPj1pRqKK(ux+uk^-2%~n55(o?Dk +zJYzqhBNghr_k$yrl-7APBVd|(UGZDm02!9Uo>@6hUd+*qT;nMh=x0LLGhZ45 +zpB=Mf#1(*7$F#0Ey3tt1Y?S{jr~i*sPg-7nz6bCZz!bE4Ndn^CH=S89{k3NjZ$HrX +z35FvtC4@y=v$2$(sM|;;93p!;6uMU;$!8D4+nL`0>Nnx#NK1sQB-0=D)X0FLE+eU6tvDKR$7Du-Zp*ECueYlbE=abQcPkSJD&5K5^3)c^6?#SOYIC=UQdk +zwMTG9<#yM`6Lgx1VYK(#${z+ca$u0r@RK6%KFuq7)cA+*jpCr%v*HE>{%@=62Y<={3k4g8pJ!-7t2nrv2gtxIpvAu3*O&*b7^Ch?9QW*;S**>a6-; +z2w>goZaX6}J>S9z5h6b7o*(^2-HT0D3O@gaSd_J)ip4UR91_;OH&#yBedUcA?1s_3 +z@hQQm5w&GdkAvc#QsV>va4mVg%Dn4$ll9nuKuJ*j`kgH};P(B-fH&$TRC0zqdZ(g_ +zrokW0L5%|Z8-McG4W@-Q);cPWcr2Lh;JbLpQn>$Fhy*3(y^-A5lq1bx8fpb2o);yq +zY1ffsccvOb2qPADNGHT>6TI^QPH&vilwMHP+xt-I*&^d#2;}?iF^>|!_E|Y~ +zJbtd?r7>bHHZvhFx=b{0st($c^}Tb`ws;#Gb={jSl!RwTgBvPX@90P6^>$N@qI-|p +z4|`}^-?@D(CcoLCcgs%drgBx1+$eNn>dKnxj&)?r4TLRcEmb(V#J%&a7T2|}q0kP6 +zx4LKutB4)b@TpVYY!vme6g(4ain2#_N1x<6XLf8S5X?v0w336s$b?&$0uGJlUySgF +zchVyAu0ZD@jqW|`40}}%K&`-8kK)`T^P_xifoBHgFZg&PvG&?JsfN_*kC|H2DMG!Y +z7g-(lB;cnZ^z%~{{HEX_<_7GaeZGQIBurly7&1X+3}KXKnA` +zyb68bW+2sb735vX}2TyfZx#=yr@9p%9e^ +z6^`XQa?{&*ni@ND?|c7<@Rg@qC=2_CcI@Z@BT2nwfY~sAk>#a8nV~l|U$_$kVMSNl +z*(%hd){;#oSqQ^kq_<{N|CA^i&)f8y0Cs2S;fHho>P_2UELt~D_(G#1Ve9r%1}k(N +zEoVgaxT4WZLmk2kz>okYFEjk}|ym#)J?j2IV2=HVlL +ziTd)gQO51M6S`C`{^v>3hUpb=P5}X#sp$78@kVo3o3$45SB7Yh| +z_j4U1;SzlrE;)kkLf_#=nRax?Jf-gtBrb~i>z2ta9)^(`O%+~b!s#M{Yx3uId|kOr +zPGytp4-uYjBVOFdto6#7E02J&e8N~*^*mesr9||68`nsKe1YkefI6=up|E|KXqE3` +zNVN!l!9;B$>9NnwPfSd}|Bs`w2nH?jfh?w!+vL*;A2>j>(>gAs&jyt^OG3|xD21Sh1zMrH*j$oFw1v^L5iAW%UWqw>bwe|l%zBS>*T+Fmx;W{?-~lJsUoOcSf|)txPX>QFF_TD7W|_v9LAIfe +zh30EQGvE9PGml!;+xI+1Y9r|RTCppB9*6a$G4z|^yGPJ>c;e`?ZdtdJbOc}z8^;+$-ZUu2DSPi}XCI^TXs +za*xs!&^SQ%{`tt7)RXw)!3Z)N?B~={S1{=fr>3HIaz#vCdQB_4!Y8)K_$u1qT#U{L +z(Gx~g1>|7=!Nj$cw$f7s9@@qgr;$(A9zA4qZjubzkcFzm7}Cdhee^C8yiv(8tjRF< +zplvh5%=-QevPT-Hsc;9mD^W|nKI3)%MnL?I>@~@u?A)(B8u2O=*9MPJt5Nys66(l1 +zf2vq=YI)^?4CW#*Fh`N)xDDxOgUWjG84=og^8-JP=@^&&ux8eJ{+6N8WhdH)2ERe+ +zM9S2))4C`2TNZ*>fKE2L8ev}0fB|Q9;E#W7VGukp?xM~yi|`=BL2utKL2ZBvjva>U +zf~LuIts+!VzYz?8W-tDXHtD6mNeP40v}~tcKMVuf1y$*>J`{0c$=POH66bGY&){I| +z*#Z(DMrrW?CA^N*OpBS>AY*;a0oq0WPuqH)_u_MAzvq_ +z5{C_WZ4oIY8mjn=vNW@h2u8n1F7JK{`Ev1xoH=$C)`HjJ5lejHs|l4?7noJLnx-fh +zNOq)1?r#t-ZH#K1@z>)~;-PQ+hKm{IT!pR}(zg|Cc4I%D+Y?+b+`%SPt(}yc5S5s) +zV+pd8t!xdy>62GG`#~tke@AzqaP%!qJRqVvwn_sJ@2!F|P0?g4w%`hG=IfKg{v{d1 +zsaws(4Cn`w#RAXReoSoWd+qFMvTn;p^qjZsSjOaqyxtRa+wb>CABoZ~qmF*%(eu22 +zwWCIEl`K#h2v&YG9eblH?MO$T786Bkurq2glUu>%9wRK9g9$rBqYA)jyEu~0F>att +z#mtlt3C3dTC8_ESw#SurX6m+cQcz8z`9wC}?U^P%eD~?BA3HMx_v#|$;q~=PC!FGh +zjer2pu!Jw0nf#sEVe92#|7S`Vc)$|+c0gPI>FyOZ~oTsjbwzsYse3`nH{LQn>H +z$BN}b{?-1N1BGhE`ux0yyy$ZzEpVxzHNzb@}G2xG@Q0u;8+@-)T_H2<$w*1!oa%a%{~ +zVP?GLU)bKrk;k}(vpE)sZuBu*kiR+6$kQ;UTa_4*_HJic?16kW5~ucJ+-? +zb0Y906{vQx^|fUb2d)M6OUXUGuSz!+Xa7G42u@HSUT#rw`)?hL61$Izqod%@g3}6v +zhA=`h)l909zvjJh$jmhB&p@36$~hN(xv)s>hME;JVmZu;^lW`v{BZ%=xd{HBFodFn +zD!t$|p3pm-cugC0EmS-Zq<%Jd8L^d`PljLtj8NeLEV+E?*}AbMsK`VnMpNbu4>lYt +zRpB~N+kNBMSR47n03M(Iw?jIRCgg;Opcc0vETd>}U@YQ7IK-T`CmoSdFTY<8knV6- +zuWI-!zr*sT>OwDQgJ4jIjvHAhfPp5%(n(Rsm|!Ca3cJH{J-ba?Q^qGw_3Q%x*!HQ- +ziGlx!0C&`A}#)T(9dp0Jm96A2>joU +zY(YdCgf8nV&(4Sur?Ksj=fV4XcMpW1Yo3u*g*9AUxUJanJ%#xUgWn-C(Dc4WkSs0h +zR9A$Y!k)ajD(mK#;mdP(zK$3AriTMQ&`wNB-mLlKc&l%QeLayM;EYC8n6z?2M{sFU +zGZJq?e1Uf)6?OR&d;%k3FRxC@V5{2UcZHE*A{QUz(YhPRcIGJ;GHng@t{QKuI +zpsqzJWYC=6{)}d(Q?c9ZpV0jEw4Ln#>Cj*EhIfPcH`s)~o#ER3*G~U4ZVz}j^gsFiYkFho|83}B +zAIeTvqN6Fh8!h#{S!B!W-V(Zw+Wh|wso@E_qypb_3$(Jm60kdLHp)-VZHmd|hlxk| +zVxW+SfF16e4M+1_TxiC}qDh24GdKp5t2PT?F$pyTcR}_>C!&sKF=NLg`l3IMK+(PH +zbeHn9S>%}AtX9SLfqApdxgYZKVhoJeNhDZAP$(t7sR**nuFRvl*J>5cV4cHK`+jX6}qe(R?Lh +zV8|Ggu5V^QL^Ra#k{mT+)ZlaQ2<;{yir)*xVkq9C<9UVatoS=J23{k>{CWw<#mZal +z`7bCjCLWx}#_M-sn5u;)rWyy0w7H}{@)F(UG|s(f7Jr?{xf6Uyg|Dq3qt?zhW}lE4 +zq}=ZkDbmd@Cu49iAu1@!N3vcXE7a3A8fw;P*0>Q)Z;zeD-yVFQsA&y)MsA>|riGPA +zvn+aP`HFj8@p>aRn8R^`hqAADC0)l{9JSo+o{z$TKmJ03+a3{GFy~fC{|J=*&0#Vhdt=&eoaMPQsx~6hN#B +z;ciQDuoq9I_`h=J`#kYes&7N`gH<9pi#;L5D6+R6k^9c5#J^*K6(UvM@@9-h%0rT| +zaPuV-N+{_L_=RgEA#QQap|J<&hM!Oc0BLSLDtH!;%yUF{f^>8uY^T6^n%$aQDEK^a +zvkY|qVph6TUDz8y)H9$iLc6VPatZE4|_eX$u1=A@-EZ>uscBakWc%*{7Zj +zpC?@PO@Kmohk=l|;0h-DC*OcbNg^&(PG;l6zSRl54585U}b +zomZhrW1L-r@esbUQ_h2jxH~~Ts}`3MUKjAY0jlcBv$#A(-_`kOr#6CQZpOYJq5AqP +z-+I?HC6-G5X-#nkoeWi6E*4z=HA2O}eFnZ~DEP`BkQ<~=imcWMKcJyV4@?T4ix2zc +z_ERLIkeLpU6QvUZFA0&l_JYRDFf%j*xC|htoFb>yTkI{W@-15BK_QGdLq*lFQy3TO*zjPN9h@I7si{+XbGZbG05HoCl=L-U-c +zcEE|TK4S0*iVY1bX`d0o`ePb5P_EgRHt8srG+zh6K$Bf%SoSW)srOZp=^ninj$dGH +z;wB5FP*C}LQ5i@G%4Vwe4tsL(?kmB$_$MNyjTbMdeksaGYUMlQr_v7~*As0LYsFJY +z9_`7q9D1R;#t_dQK)CePHmN}|4%VEz9q%PV0>Pw%QikyxGRkJeweXy7%K8;J7x +z4Va!puc$EeX&pf;wIybWSEnr(z@VU9-}R3T86iSI2Uy8BT{7f4D5E|1cM>ZOv6vyO +zj`Bh##S~r8mR)icA?aTOI5^n-Lt(&C`T(~r9&(-w| +z7Oogv;!SMXRj82GMDT~mR-ZB*c`@AECg52JPfrY? +z1}ZJAJefa(tfpFdx7mVGwX732nQ>>UAFs_N$dj+WxNAe5-(lt9FKU)i1yF~zuZ9~@ +z7h<4lv-1&W>8E3rl->D?0?RaBEe6920}0Op)dFQ$0m|rw`03fP=UG-uZST3MydD*O +z4~9yePyBGr`?B6B6h^4_OS2H}Dz}JQ?xfN4vFXZrW$U3!^fJXZ`ZH=zLoO +z^HH@g^QM(JY4(_Tx1uV{LR2!bVoDo%TqU%8EnlWT+$+?ekMMt=;Meg;?7{7b!RiJgM_#?_O*9CSEdqnD +zS^c7s3KEA_&|;`S=S(a~YQz2AQ*2A_t52TiBC7*IttUN{9}fqL;>Cg-{dQY5WFku$ +zvE5b(Cxb!(-g_*y2VScT|JqsuVqYF0i+e%m8OQ2Gasg#^fH3+-(`|~V){KWq&o3UV +zlcng2CxHULXO}w{^{W>d&n`*(B_{-uzx~`lC(Q*y14Q=k{0A@}=v9Du;-qUiqF?^F +z+G*hYav8jO(IlkXjmDMQs26R^=F^U +z&UWD4#ufMC!eMD|RikS3S6!AQt-PGy0;Y`7%mh*zqep`ic<@y}~9iF<>Q?RH~ +zgMd$qZ7Evu0fCPzC&L_((8({*Hyt{FjRkcBWSG#^E*$4Ks|1Tj%MB(&xWp20W5^(3 +z$f5AN;=ipfMVvgJlE$q1iqx{$n}1*;_TcO{j}IWfS%HDipfsFk(5 +ze*e5GH25k|RO3#?_JKVAELFYTsOE=)REXIH6eGy54fwapC{H03Xt$ggdm|Jm&u9wz +zgFc%Ns4=F?WXl}=_~d!&120STjp?L#LYDB~CU`u)!aeO`{z2{-5PJ2!BcztHwB{*S +zz!~F5BN_~sXb~2+?E{PJ?6|>O5yh~3x;NFrjuehiL8(l7yfNK9Mp7960t^=+ +zy>H&rX+lKnJY(3;BB5jA(MQ#*`gQ#pZ2vnX6Fw{_aqEu0{gTFNSC!^;ctbZEBgl9?kySbaTy5i>Myx4rx??XPyXj%YkFoW=zL!|^o%DP8WP3zY~+qcE^pG +z%$M^{Cot1z&VarHn0T;&D&i+x3aO{K?`k~vIOXBo$!Oi%guvn?1_!Lxm-wVYmI_(t +zKg!Oqv>Lc*8PU#$bR(G?RDI)>hNa-w3>^#I0;#+&SyQz5#xhcA-#ixNn+sw74@$uI +z0hpd0k=2jVv>U{%zFG^YQQV@;CwsJzPUun%mmqx%G50yWYVC~DKNgHcLZeR4#juzs +zKq~U7e!u$EWHY{8_-M4`2ZL+5)u%aITIy&Yfi<{J5e+OwJl%-KB$D6IwdP~g3^nf= +z)u0rF2Gin?1?V!$Nf?mK^LtEQ)m67rj^FJi|-edSeqTX7*i^fzKoSIo{1PBULOB~|Ro +z>;W)2gRUK~9u<>~?O>wWs=+{gPX^V^#*FZY(7m>QR^xK#2G9K3GbA8)zkPp@rE9=dMS1f4*Z^z!P$ +z9>}TETVcTYV~BJ4O_yEn|lvpb4p=D9Z90IE-!IUx-LYmy~(~>cuA+W7>$%v +zw^~oSI4~hC7rPyNN>oHtQIVrl|8AC3ypI)+cI_3#|1AU(tUPPj#!F3 +zhuS&~JU^UPoIC=^8!zdFdPw#dP7xRog4#$}szU-E{xeC=f0-m_)Rwf$WHX47^Kx$z +z5^elNqwK3?K0ViRu`^TRB)I{kxPgFC!PW$|q +zZoI4*`DYWz1_C;QjE5vPuM7KRgB{X!ly^%n4`%u|>GZWY5$dEa3Pni1aOVgjVgX*b +zuh|Dw@AN7eWo3%P?lNY{{Lk*We7A}aYumi|`*3Yiw~6mnuFq10=_x-MF+O-jKPqp_ +zCF>b!XdqOm7I>rhTjT$b%odi1}ay|vw8@=vOL+GM=c7oqzl+D;Q$WTvV +z+LMML`qYzWX0j#c{PMl@=nu6>owFBAw_>4$NVLM@lna4qasLoI8Qe+HM8Lt;{2F`o +zW3RQvvhtIZ;9w4=@pK!)mlr=1QVe5~7Ce^PZfG1@Hfb8;SD=*U-%VHB;|sqvnoB|E +zo9qq9?wyD3%4`f+#PAg$dxq_ltq$$v9W*CfT@X5HEnj|b_&M9>5LJ`Kgfh))zfbZ!u5nT>UKR>Q&MY=c6r@7?Q5E@SNP^7ZD`g$?%N<>lnX^1&k+ +z`4b%~tQcxu`VWYt0+Lrc8Aiw4G))GkIt$WN6lW}4RS+dxG%Puc76pXcw-1w@dAmQ~ +z;{j(lu=3&q)p&gIEsK~f8$+`SlFLAzXg6X$WYj@v^ld?O=gOeZmT)&ww|U}ud?W94 +z$ye=mgJ;)q!08*K>NJN7i^4{Og=%2Wj>`o*9by84nX{Dgd5sbvxtO@yow)@<|XE$i^!Q6sgjR{9hT8) +z9eb%^tzV7kBfF@J(hwOrDy3|eE6V)6Oa%k5k01W2grnkgV09&PjD(9Ol0PTc=?{vU +znoe+JBuGZKkG7v&jYpu$+AONOZOkDfw5x2I+_X-_(o4l5h!8~qk}=n;VC6!?jr?hu +zbNC&NNGY&OkDlz4h}yPccmBJPvagO(6(%d)4#g$A5NQN*IO=DkL%zQ9ZBj#!d=Ks- +zUX968f4#|qwL`0i?}V62fY{x~g(n!hF9{8B0%3;~J_pgyd#-U(h#-kR-1G2Z!*so= +zQciBk8!3D#(oX&H{L;c%hH6~YU +zd0^LEoU@xJMGy=?O20Z0mM;}x|K`VEAQu^#+QmM0l8LI;?N8VmY)zRF?x=anzR6lj +zlTW!oEe}J6BB~U;X(ptPl!lTL^)aI9v;QLyo4cEI5Q1nz@>XtFZzet#M?$F34VC6L +z8~tz<#)Ik)UGA1i1_I5P4m&o*|1_-WcZ1tpx%ur^Sr6#&f{;T+PW%j@sDx-iQ_>oS +z7cJVR%R6{pTn))-hC$2UDSK~6KMr$XvH!YOtSPX6cKK5PS0SrI5FKu)YrHvA2b__5 +zUroo|2@q@lYJz(P*jz;nnU^<0dOPc`GvA=VC2#Fu)3g8TKuv!xa;6@O)Xw=&tx?mG +zDVT*=eI+RMYgrp9>la;@J0v*8rHUMU#l(|-p~gBkOUf~&M9ag9{Z+e4ey)(pIUv+a +zQ&L0C7besMhJ)e!wZxrJCaqY?)L_Z0HJd6=$QhS=29*+B`CHxT@Msqf3Z(3Lt6-3p(Uy5;-YNfnajYQjZ +zP5zHd^=cwS49;>dfPik;Yg}(~n*3k_7A0ZW`tphTTI{ROrMoHSv-kdo9ZyCv%a`t? +zFKQfbi3!tQ4J+fY&s~{1pUCOrQX1>cLZ1`$Ee*mldMOA~_y;%tJWd)uSU?qo)5bU} +ztIwlFOo+aHZzd+g8}S{r_u#7^eUS~pzdf{?M5q%g}r2RdmG0YlaO|G=?88Kw5L +zpmvNJdBi066hZhQe!Vh}swXebH+1s^=>*_^H5U)|JT*+grjm*ibhvx&+C3M{t +z4>&k#dhX-08XHWteje~2sP +zEuVXrUS&j`!j^P>7x($1Y-N1k52B0#7$c3@9XE#1)OVN7dBu9#wB9vZwJO&sAP_8y +z5Es^e)8l%dZ;62sg$=Kx?6Q!y9Y9P;c`M029P5G}{1yWrRWHe;Fnwst6ki9ShBvL@r@CHM5JUKp2qoJ=#hf8qe;Mns{knji3csfw>U0ZxNHTt?QqP{~dH;K}?_;J=4V +z_OK#vxXS9BEBKB=|D`eydD+Sj4{oAabL#y38y^6FQ?IXZLBm9Y`{02CT|A!a+S8x$ +z_dp;MH8l~`$R4}1+{yk4;?_L +z8kEKZry^4&32jj&5YdUY=_62>)kS?VQ0V2=#352WiZi&`5LXy$7uv?{Gz+N+^n6m< +z8YCE-6%?-~>Vf##JVIT-O$$n3N&kFgsQ00NMcx$Gi>4|6WpjS2E^js%dSHt`r*h*3 +z06Zt$Z=K3SOVBZ%o6_RZo(5t1o-$L-T>3gYd>Q#=0;JULL#q0(s$zr)Zc0;PV>i-J +z;C!kR!ukh##%Xq%1T@6ax%#^YHw4KJ(N_Uf9usb~L5BjZ>=fGadoieY&BLX1A^hR8 +z-9&hdvZ&3)rs137?0K?~>b_S++Mu)99=Lm7H|Nc#>*M&#+oVY*0M&(DbY=qJ{C6qx +z3Fam&WyzR*-;qj?X__|;JAfr!RqfMEd{!5(jIqkFFxkT@NnU)R$s@~axKx?0G3?8D +z{T}SQIx(1u%a(H<|CrBH#{XRvqSo8+l8klD@-MlO#RQ0>4AXMi$*J&6xw1Bxul4W| +ze97#CxAc}*mLelOTbmdi-az=56wg#Z=%Vr%5`~}9uvV0NEf?g=x?h*fw0pvV7(Azt +z#)D*)ae--m|8xG2E^vm|xU+tE8NA?oIEzr3vGN?D^^~SO+k}pn>WF=R$fOjg3Nvi2#Ny;QqDKNTBy+2sp`CROA +zNjU7?!6>F`{RP@@7Z$fOqx|z7S29}ge!p3AzXa#|{0y`|8@L!1tQwl4Dyq{8xoE7J +zj0)qd(jT;WL=(|#sanpsp8rcW!gJ%u$aS1r7y!aNlY19LxK(HLOUgjY3k3T;TM+~I +zWZBRG3=FsFhJw4aG@!TKNavn_LwZB=S0UZMho9mWpY{J6(&qU@YdV +zY~@Q6A3QZlUj{r%DpM0UEtq>)E4$RopGRk~n9u;=)3HUU6nO-;fm9TN=<-LE6qavo +zQaHSjQQJ|Vzu_4@QoTQSQV0U}8-JS-cc|!|-Rn45OHO9~C#81vE!kJXy-2{2j9!jE +z-O^H0Rz@hb=RZGxC5S@39AM;QLzg=nkUwb0dz{<*&&z&Zhod#hNX0>jhva9hl3cds +z>*!f~?e&zCIYd5f_qLRrI6JEM)`z;a!*2wR{U&($-!~EBanMqc3ZB+&UKjccEMzCy +zSn_jgQ8{%ebzH$_TqO2DQKI8>zuwC?vhLRkclR2|{q*bb5>5{^0cSdgydwT$xGMD! +z8v$>@gYZFKLTdIlqKvTL^=Mx4Ea%!!4$eq>7|igv$i0|}@#lC+t`&X%Ry<+#Ku%n) +zJzpjFreKX=CPw`?x|e_Ko@VS4tUz$b58bZ{e|O3m;V4&NCVkpS#?WP?bgSv0x6yeI +zyJ?#E=gcQUP;xDJPl1J3lLD)CKh{aRHCvP@yNRopk~8)>)4)orVc(8uB<7@hpM@r( +zkCnS^nTVAd;0@j@ps9fnk&ji-{d3k1%k?wFu{uX>gez2@nVyvscPPo1`+2p>qgQA8 +zA~7S2d5{RzaYShO@F}s2c$84ToBoJ#SonvGg2%UU+f4!P1CY0o%h9%x~z^Sx^MO)k)DWm!60ziy8B +zhD$Tk6sL5fl?fV_dj*u~`@G&V1zq^l|AfVC#L^-B=!gpQd*1zTK`o1()84Jte&z2i +zrgT19m++WnDH?yi@Zc|@Jh;N>Ogrl{%DDAM*z0>lA?RM4)sf17rD+0bkxMC`Kq;C; +zk+q_1B0G!alySJ)5TV_40I|MdUjDTIyok{%W6pb@4bX(Fsa +zO&QR57co2o744m)pw+YwNZg#steALxIFVt``7qO@@yY8a^z6JinAHX-c +zl4B41(cmjl7x8PwJs%+z{)+uERflIedUULJ%uqtLnUoldMe;YAGANn$v`%9sFU2$g +zIkxy0WwWe`+#NKv6FF=IlkKh898>m-u4w4T>qBUSCs%+K6_7faF^CC||(7AIAay +zZh|%>3*jjlu{#mY8*0ESqhsmbqGCX$f|#_%%>&uU96Qp1_rdQi2Z`Y!0_hZ9{{5{(ztfyX!^TfFVWPQAyZ8 +Sj6z<({}g3aWGbahg8v`qQO6Jf + +literal 0 +HcmV?d00001 + +diff --git a/docs/en/Debug_In_Idea.md b/docs/en/Debug_In_Idea.md +new file mode 100644 +index 000000000..996798067 +--- /dev/null ++++ b/docs/en/Debug_In_Idea.md +@@ -0,0 +1,55 @@ ++## How to Debug RocketMQ in Idea ++ ++### Step0: Resolve dependencies ++1. To download the Maven dependencies required for running RocketMQ, you can use the following command:`mvn clean install -Dmaven.test.skip=true` ++2. Ensure successful local compilation. ++ ++### Step1: Start NameServer ++1. The startup class for NameServer is located in `org.apache.rocketmq.namesrv.NamesrvStartup`. ++2. Add runtime `ROCKETMQ_HOME=` parameters in `Idea-Edit Configurations`. ++![Idea_config_nameserver.png](../cn/image/Idea_config_nameserver.png) ++3. Run NameServer and if the following log output is observed, it indicates successful startup. ++```shell ++The Name Server boot success. serializeType=JSON, address 0.0.0.0:9876 ++``` ++ ++### Step2: Start Broker ++1. The startup class for Broker is located in`org.apache.rocketmq.broker.BrokerStartup` ++2. Create the `/rocketmq/conf/broker.conf` file or simply copy it from the official release package. ++```shell ++# broker.conf ++ ++brokerClusterName = DefaultCluster ++brokerName = broker-a ++brokerId = 0 ++deleteWhen = 04 ++fileReservedTime = 48 ++brokerRole = ASYNC_MASTER ++flushDiskType = ASYNC_FLUSH ++namesrvAddr = 127.0.0.1:9876 # name server地址 ++``` ++3. Add the runtime parameter `ROCKETMQ_HOME=` and the environment variable `-c /Users/xxx/rocketmq/conf/broker.conf` in `Idea-Edit Configurations`. ++![Idea_config_broker.png](../cn/image/Idea_config_broker.png) ++4. Run the Broker and if the following log is observed, it indicates successful startup. ++```shell ++The broker[broker-a,192.169.1.2:10911] boot success... ++``` ++ ++### Step3: Send or Consume Messages ++RocketMQ startup is now complete. You can use the examples provided in `/example` to send and consume messages. ++ ++### Additional: Start the Proxy locally. ++1. RocketMQ 5.x introduced the Proxy mode. Using the `LOCAL` mode eliminates the need for `Step2`. The startup class is located at `org.apache.rocketmq.proxy.ProxyStartup`. ++2. Add the runtime parameter `ROCKETMQ_HOME=` in `Idea-Edit Configurations`. ++3. Create a new configuration file named `rmq-proxy.json` in the `/conf/` directory. ++```json ++{ ++ "rocketMQClusterName": "DefaultCluster", ++ "nameSrvAddr": "127.0.0.1:9876", ++ "proxyMode": "local" ++} ++``` ++4. Run the Proxy, and if the following log is observed, it indicates successful startup. ++```shell ++Sat Aug 26 15:29:33 CST 2023 rocketmq-proxy startup successfully ++``` +\ No newline at end of file +-- +2.32.0.windows.2 + + +From 3a035d75d7c600419f855541f4d3ac6c8804ab29 Mon Sep 17 00:00:00 2001 +From: Ao Qiao +Date: Mon, 16 Oct 2023 15:56:48 +0800 +Subject: [PATCH 2/8] [ISSUE #7464] Polish the pop logger format (#7465) + +* Doc: How to debug in Idea + +* en + +* polish + +* polish log +--- + .../broker/processor/PopReviveService.java | 24 +++++++++---------- + 1 file changed, 12 insertions(+), 12 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +index 4f80752e1..3fb689ed6 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopReviveService.java +@@ -129,7 +129,7 @@ public class PopReviveService extends ServiceThread { + PutMessageResult putMessageResult = brokerController.getEscapeBridge().putMessageToSpecificQueue(msgInner); + PopMetricsManager.incPopReviveRetryMessageCount(popCheckPoint, putMessageResult.getPutMessageStatus()); + if (brokerController.getBrokerConfig().isEnablePopLog()) { +- POP_LOGGER.info("reviveQueueId={},retry msg , ck={}, msg queueId {}, offset {}, reviveDelay={}, result is {} ", ++ POP_LOGGER.info("reviveQueueId={},retry msg, ck={}, msg queueId {}, offset {}, reviveDelay={}, result is {} ", + queueId, popCheckPoint, messageExt.getQueueId(), messageExt.getQueueOffset(), + (System.currentTimeMillis() - popCheckPoint.getReviveTime()) / 1000, putMessageResult); + } +@@ -319,7 +319,7 @@ public class PopReviveService extends ServiceThread { + // offset self amend + while (true) { + if (!shouldRunPopRevive) { +- POP_LOGGER.info("slave skip scan , revive topic={}, reviveQueueId={}", reviveTopic, queueId); ++ POP_LOGGER.info("slave skip scan, revive topic={}, reviveQueueId={}", reviveTopic, queueId); + break; + } + List messageExts = getReviveMessage(offset, queueId); +@@ -351,7 +351,7 @@ public class PopReviveService extends ServiceThread { + noMsgCount = 0; + } + if (System.currentTimeMillis() - startScanTime > brokerController.getBrokerConfig().getReviveScanTime()) { +- POP_LOGGER.info("reviveQueueId={}, scan timeout ", queueId); ++ POP_LOGGER.info("reviveQueueId={}, scan timeout ", queueId); + break; + } + for (MessageExt messageExt : messageExts) { +@@ -373,7 +373,7 @@ public class PopReviveService extends ServiceThread { + } else if (PopAckConstants.ACK_TAG.equals(messageExt.getTags())) { + String raw = new String(messageExt.getBody(), DataConverter.CHARSET_UTF8); + if (brokerController.getBrokerConfig().isEnablePopLog()) { +- POP_LOGGER.info("reviveQueueId={},find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); ++ POP_LOGGER.info("reviveQueueId={}, find ack, offset:{}, raw : {}", messageExt.getQueueId(), messageExt.getQueueOffset(), raw); + } + AckMsg ackMsg = JSON.parseObject(raw, AckMsg.class); + PopMetricsManager.incPopReviveAckGetCount(ackMsg, queueId); +@@ -465,15 +465,15 @@ public class PopReviveService extends ServiceThread { + + protected void mergeAndRevive(ConsumeReviveObj consumeReviveObj) throws Throwable { + ArrayList sortList = consumeReviveObj.genSortList(); +- POP_LOGGER.info("reviveQueueId={},ck listSize={}", queueId, sortList.size()); ++ POP_LOGGER.info("reviveQueueId={}, ck listSize={}", queueId, sortList.size()); + if (sortList.size() != 0) { +- POP_LOGGER.info("reviveQueueId={}, 1st ck, startOffset={}, reviveOffset={} ; last ck, startOffset={}, reviveOffset={}", queueId, sortList.get(0).getStartOffset(), ++ POP_LOGGER.info("reviveQueueId={}, 1st ck, startOffset={}, reviveOffset={}; last ck, startOffset={}, reviveOffset={}", queueId, sortList.get(0).getStartOffset(), + sortList.get(0).getReviveOffset(), sortList.get(sortList.size() - 1).getStartOffset(), sortList.get(sortList.size() - 1).getReviveOffset()); + } + long newOffset = consumeReviveObj.oldOffset; + for (PopCheckPoint popCheckPoint : sortList) { + if (!shouldRunPopRevive) { +- POP_LOGGER.info("slave skip ck process , revive topic={}, reviveQueueId={}", reviveTopic, queueId); ++ POP_LOGGER.info("slave skip ck process, revive topic={}, reviveQueueId={}", reviveTopic, queueId); + break; + } + if (consumeReviveObj.endTime - popCheckPoint.getReviveTime() <= (PopAckConstants.ackTimeInterval + PopAckConstants.SECOND)) { +@@ -483,12 +483,12 @@ public class PopReviveService extends ServiceThread { + // check normal topic, skip ck , if normal topic is not exist + String normalTopic = KeyBuilder.parseNormalTopic(popCheckPoint.getTopic(), popCheckPoint.getCId()); + if (brokerController.getTopicConfigManager().selectTopicConfig(normalTopic) == null) { +- POP_LOGGER.warn("reviveQueueId={},can not get normal topic {} , then continue ", queueId, popCheckPoint.getTopic()); ++ POP_LOGGER.warn("reviveQueueId={}, can not get normal topic {}, then continue", queueId, popCheckPoint.getTopic()); + newOffset = popCheckPoint.getReviveOffset(); + continue; + } + if (null == brokerController.getSubscriptionGroupManager().findSubscriptionGroupConfig(popCheckPoint.getCId())) { +- POP_LOGGER.warn("reviveQueueId={},can not get cid {} , then continue ", queueId, popCheckPoint.getCId()); ++ POP_LOGGER.warn("reviveQueueId={}, can not get cid {}, then continue", queueId, popCheckPoint.getCId()); + newOffset = popCheckPoint.getReviveOffset(); + continue; + } +@@ -520,7 +520,7 @@ public class PopReviveService extends ServiceThread { + + private void reviveMsgFromCk(PopCheckPoint popCheckPoint) { + if (!shouldRunPopRevive) { +- POP_LOGGER.info("slave skip retry , revive topic={}, reviveQueueId={}", reviveTopic, queueId); ++ POP_LOGGER.info("slave skip retry, revive topic={}, reviveQueueId={}", reviveTopic, queueId); + return; + } + inflightReviveRequestMap.put(popCheckPoint, new Pair<>(System.currentTimeMillis(), false)); +@@ -646,7 +646,7 @@ public class PopReviveService extends ServiceThread { + consumeReviveMessage(consumeReviveObj); + + if (!shouldRunPopRevive) { +- POP_LOGGER.info("slave skip scan , revive topic={}, reviveQueueId={}", reviveTopic, queueId); ++ POP_LOGGER.info("slave skip scan, revive topic={}, reviveQueueId={}", reviveTopic, queueId); + continue; + } + +@@ -662,7 +662,7 @@ public class PopReviveService extends ServiceThread { + currentReviveMessageTimestamp = System.currentTimeMillis(); + } + +- POP_LOGGER.info("reviveQueueId={},revive finish,old offset is {}, new offset is {}, ckDelay={} ", ++ POP_LOGGER.info("reviveQueueId={}, revive finish,old offset is {}, new offset is {}, ckDelay={} ", + queueId, consumeReviveObj.oldOffset, consumeReviveObj.newOffset, delay); + + if (sortList == null || sortList.isEmpty()) { +-- +2.32.0.windows.2 + + +From d73b6013825db9124e39a37db67094e34b9c3d88 Mon Sep 17 00:00:00 2001 +From: Zhouxiang Zhan +Date: Mon, 16 Oct 2023 19:06:40 +0800 +Subject: [PATCH 3/8] [ISSUE #7330] Fix channel connect issue for goaway + (#7467) + +* add waitChannelFuture for goaway + +* add body for retry channel +--- + .../remoting/netty/NettyRemotingClient.java | 41 +++++++++++++------ + 1 file changed, 28 insertions(+), 13 deletions(-) + +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +index 4bc51bd83..340daee67 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/netty/NettyRemotingClient.java +@@ -716,20 +716,25 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + } + + if (cw != null) { +- ChannelFuture channelFuture = cw.getChannelFuture(); +- if (channelFuture.awaitUninterruptibly(this.nettyClientConfig.getConnectTimeoutMillis())) { +- if (cw.isOK()) { +- LOGGER.info("createChannel: connect remote host[{}] success, {}", addr, channelFuture.toString()); +- return cw.getChannel(); +- } else { +- LOGGER.warn("createChannel: connect remote host[" + addr + "] failed, " + channelFuture.toString()); +- } ++ return waitChannelFuture(addr, cw); ++ } ++ ++ return null; ++ } ++ ++ private Channel waitChannelFuture(String addr, ChannelWrapper cw) { ++ ChannelFuture channelFuture = cw.getChannelFuture(); ++ if (channelFuture.awaitUninterruptibly(this.nettyClientConfig.getConnectTimeoutMillis())) { ++ if (cw.isOK()) { ++ LOGGER.info("createChannel: connect remote host[{}] success, {}", addr, channelFuture.toString()); ++ return cw.getChannel(); + } else { +- LOGGER.warn("createChannel: connect remote host[{}] timeout {}ms, {}", addr, this.nettyClientConfig.getConnectTimeoutMillis(), +- channelFuture.toString()); ++ LOGGER.warn("createChannel: connect remote host[{}] failed, {}", addr, channelFuture.toString()); + } ++ } else { ++ LOGGER.warn("createChannel: connect remote host[{}] timeout {}ms, {}", addr, this.nettyClientConfig.getConnectTimeoutMillis(), ++ channelFuture.toString()); + } +- + return null; + } + +@@ -818,8 +823,14 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + long duration = stopwatch.elapsed(TimeUnit.MILLISECONDS); + stopwatch.stop(); + RemotingCommand retryRequest = RemotingCommand.createRequestCommand(request.getCode(), request.readCustomHeader()); +- Channel retryChannel = channelWrapper.getChannel(); +- if (channel != retryChannel) { ++ retryRequest.setBody(request.getBody()); ++ Channel retryChannel; ++ if (channelWrapper.isOK()) { ++ retryChannel = channelWrapper.getChannel(); ++ } else { ++ retryChannel = waitChannelFuture(channelWrapper.getChannelAddress(), channelWrapper); ++ } ++ if (retryChannel != null && channel != retryChannel) { + return super.invokeImpl(retryChannel, retryRequest, timeoutMillis - duration); + } + } +@@ -994,6 +1005,10 @@ public class NettyRemotingClient extends NettyRemotingAbstract implements Remoti + this.lastResponseTime = System.currentTimeMillis(); + } + ++ public String getChannelAddress() { ++ return channelAddress; ++ } ++ + public boolean reconnect() { + if (lock.writeLock().tryLock()) { + try { +-- +2.32.0.windows.2 + + +From 82b2f8eefac157843c6ccec80d94f202c06bd195 Mon Sep 17 00:00:00 2001 +From: rongtong +Date: Wed, 18 Oct 2023 13:51:47 +0800 +Subject: [PATCH 4/8] AddBroker removes parsing configuration from body (#7472) + +--- + .../rocketmq/container/BrokerContainerProcessor.java | 9 ++++----- + 1 file changed, 4 insertions(+), 5 deletions(-) + +diff --git a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java +index 2ac69112d..5b825fe81 100644 +--- a/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java ++++ b/container/src/main/java/org/apache/rocketmq/container/BrokerContainerProcessor.java +@@ -91,11 +91,10 @@ public class BrokerContainerProcessor implements NettyRequestProcessor { + LOGGER.error("addBroker load config from {} failed, {}", configPath, e); + } + } else { +- byte[] body = request.getBody(); +- if (body != null) { +- String bodyStr = new String(body, MixAll.DEFAULT_CHARSET); +- brokerProperties = MixAll.string2Properties(bodyStr); +- } ++ LOGGER.error("addBroker config path is empty"); ++ response.setCode(ResponseCode.SYSTEM_ERROR); ++ response.setRemark("addBroker config path is empty"); ++ return response; + } + + if (brokerProperties == null) { +-- +2.32.0.windows.2 + + +From f0f15b5e21acd3caf9141375be5db3ef726a2173 Mon Sep 17 00:00:00 2001 +From: guyinyou <36399867+guyinyou@users.noreply.github.com> +Date: Thu, 19 Oct 2023 10:14:29 +0800 +Subject: [PATCH 5/8] [ISSUE #7454] Utilizing cache to avoid duplicate parsing + (#7455) + +* Utilizing cache to avoid duplicate parsing + +* add a method argument to decide cacheable + +* Renaming variable names from cacheAble to isCached + +--------- + +Co-authored-by: guyinyou +Co-authored-by: RongtongJin +--- + .../broker/processor/PopMessageProcessor.java | 2 +- + .../remoting/protocol/RemotingCommand.java | 14 ++++++++++++-- + .../protocol/header/FastCodesHeaderTest.java | 2 +- + 3 files changed, 14 insertions(+), 4 deletions(-) + +diff --git a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +index f5d07c5aa..7ed4d53ab 100644 +--- a/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java ++++ b/broker/src/main/java/org/apache/rocketmq/broker/processor/PopMessageProcessor.java +@@ -204,7 +204,7 @@ public class PopMessageProcessor implements NettyRequestProcessor { + RemotingCommand response = RemotingCommand.createResponseCommand(PopMessageResponseHeader.class); + final PopMessageResponseHeader responseHeader = (PopMessageResponseHeader) response.readCustomHeader(); + final PopMessageRequestHeader requestHeader = +- (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class); ++ (PopMessageRequestHeader) request.decodeCommandCustomHeader(PopMessageRequestHeader.class, true); + StringBuilder startOffsetInfo = new StringBuilder(64); + StringBuilder msgOffsetInfo = new StringBuilder(64); + StringBuilder orderCountInfo = null; +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java +index d27135132..e93072adf 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/RemotingCommand.java +@@ -89,6 +89,7 @@ public class RemotingCommand { + private String remark; + private HashMap extFields; + private transient CommandCustomHeader customHeader; ++ private transient CommandCustomHeader cachedHeader; + + private SerializeType serializeTypeCurrentRPC = serializeTypeConfigInThisServer; + +@@ -260,10 +261,19 @@ public class RemotingCommand { + + public CommandCustomHeader decodeCommandCustomHeader( + Class classHeader) throws RemotingCommandException { +- return decodeCommandCustomHeader(classHeader, true); ++ return decodeCommandCustomHeader(classHeader, false); + } + +- public CommandCustomHeader decodeCommandCustomHeader(Class classHeader, ++ public CommandCustomHeader decodeCommandCustomHeader( ++ Class classHeader, boolean isCached) throws RemotingCommandException { ++ if (isCached && cachedHeader != null) { ++ return cachedHeader; ++ } ++ cachedHeader = decodeCommandCustomHeaderDirectly(classHeader, true); ++ return cachedHeader; ++ } ++ ++ public CommandCustomHeader decodeCommandCustomHeaderDirectly(Class classHeader, + boolean useFastEncode) throws RemotingCommandException { + CommandCustomHeader objectHeader; + try { +diff --git a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/FastCodesHeaderTest.java b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/FastCodesHeaderTest.java +index 6bb100f57..b6a0d6311 100644 +--- a/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/FastCodesHeaderTest.java ++++ b/remoting/src/test/java/org/apache/rocketmq/remoting/protocol/header/FastCodesHeaderTest.java +@@ -73,7 +73,7 @@ public class FastCodesHeaderTest { + + private void check(RemotingCommand command, List fields, + Class classHeader) throws Exception { +- CommandCustomHeader o1 = command.decodeCommandCustomHeader(classHeader, false); ++ CommandCustomHeader o1 = command.decodeCommandCustomHeaderDirectly(classHeader, false); + CommandCustomHeader o2 = classHeader.getDeclaredConstructor().newInstance(); + ((FastCodesHeader)o2).decode(command.getExtFields()); + for (Field f : fields) { +-- +2.32.0.windows.2 + + +From dbc633d92b6c8c35922234611d698d3cb0a1a234 Mon Sep 17 00:00:00 2001 +From: Ji Juntao +Date: Thu, 19 Oct 2023 14:12:33 +0800 +Subject: [PATCH 6/8] Check the input yaml and the path in ACL (#7475) + +* check the input yaml and the path. + +* only modify the path, no yaml. + +* remove useless import +--- + .../org/apache/rocketmq/acl/plain/PlainPermissionManager.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +index f6699fa13..345aed06c 100644 +--- a/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java ++++ b/acl/src/main/java/org/apache/rocketmq/acl/plain/PlainPermissionManager.java +@@ -484,7 +484,7 @@ public class PlainPermissionManager { + return false; + } + +- if (!fileName.startsWith(fileHome)) { ++ if (!file.getAbsolutePath().startsWith(fileHome)) { + log.error("Parameter value " + fileName + " is not in the directory rocketmq.home.dir " + fileHome); + return false; + } +-- +2.32.0.windows.2 + + +From 3968c186a59db96701ade8c343bc6a5d31ee2d24 Mon Sep 17 00:00:00 2001 +From: weihubeats +Date: Fri, 20 Oct 2023 14:49:00 +0800 +Subject: [PATCH 7/8] [ISSUE #7231] Fix: proxy client language error (#7200) + +* Adding null does not update + +* add langeuga code + +* add langeuga code + +* add langeuga code + +* add langeuga code + +* add langeuga code + +* Rerun ci + +* Rerun ci + +* Rerun ci + +* remove redundant package imports + +* redundant line + +* modify the parameter passed as proxyContext to language + +* format +--- + .../proxy/service/message/LocalMessageService.java | 12 ++++++------ + .../proxy/service/message/LocalRemotingCommand.java | 8 ++++++-- + .../rocketmq/remoting/protocol/LanguageCode.java | 11 +++++++++++ + 3 files changed, 23 insertions(+), 8 deletions(-) + +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +index ca7dcc9eb..aaa688fee 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalMessageService.java +@@ -104,7 +104,7 @@ public class LocalMessageService implements MessageService { + body = message.getBody(); + messageId = MessageClientIDSetter.getUniqID(message); + } +- RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader); ++ RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.SEND_MESSAGE, requestHeader, ctx.getLanguage()); + request.setBody(body); + CompletableFuture future = new CompletableFuture<>(); + SimpleChannel channel = channelManager.createInvocationChannel(ctx); +@@ -162,7 +162,7 @@ public class LocalMessageService implements MessageService { + ConsumerSendMsgBackRequestHeader requestHeader, long timeoutMillis) { + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); +- RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader); ++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CONSUMER_SEND_MSG_BACK, requestHeader, ctx.getLanguage()); + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand response = brokerController.getSendMessageProcessor() +@@ -181,7 +181,7 @@ public class LocalMessageService implements MessageService { + CompletableFuture future = new CompletableFuture<>(); + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); +- RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, requestHeader); ++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.END_TRANSACTION, requestHeader, ctx.getLanguage()); + try { + brokerController.getEndTransactionProcessor() + .processRequest(channelHandlerContext, command); +@@ -196,7 +196,7 @@ public class LocalMessageService implements MessageService { + public CompletableFuture popMessage(ProxyContext ctx, AddressableMessageQueue messageQueue, + PopMessageRequestHeader requestHeader, long timeoutMillis) { + requestHeader.setBornTime(System.currentTimeMillis()); +- RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader); ++ RemotingCommand request = LocalRemotingCommand.createRequestCommand(RequestCode.POP_MESSAGE, requestHeader, ctx.getLanguage()); + CompletableFuture future = new CompletableFuture<>(); + SimpleChannel channel = channelManager.createInvocationChannel(ctx); + InvocationContext invocationContext = new InvocationContext(future); +@@ -307,7 +307,7 @@ public class LocalMessageService implements MessageService { + ChangeInvisibleTimeRequestHeader requestHeader, long timeoutMillis) { + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); +- RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader); ++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.CHANGE_MESSAGE_INVISIBLETIME, requestHeader, ctx.getLanguage()); + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand response = brokerController.getChangeInvisibleTimeProcessor() +@@ -346,7 +346,7 @@ public class LocalMessageService implements MessageService { + AckMessageRequestHeader requestHeader, long timeoutMillis) { + SimpleChannel channel = channelManager.createChannel(ctx); + ChannelHandlerContext channelHandlerContext = channel.getChannelHandlerContext(); +- RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader); ++ RemotingCommand command = LocalRemotingCommand.createRequestCommand(RequestCode.ACK_MESSAGE, requestHeader, ctx.getLanguage()); + CompletableFuture future = new CompletableFuture<>(); + try { + RemotingCommand response = brokerController.getAckMessageProcessor() +diff --git a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java +index 73048dbbc..915cafcd5 100644 +--- a/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java ++++ b/proxy/src/main/java/org/apache/rocketmq/proxy/service/message/LocalRemotingCommand.java +@@ -16,16 +16,19 @@ + */ + package org.apache.rocketmq.proxy.service.message; + +-import java.util.HashMap; + import org.apache.rocketmq.remoting.CommandCustomHeader; + import org.apache.rocketmq.remoting.exception.RemotingCommandException; ++import org.apache.rocketmq.remoting.protocol.LanguageCode; + import org.apache.rocketmq.remoting.protocol.RemotingCommand; + ++import java.util.HashMap; ++ + public class LocalRemotingCommand extends RemotingCommand { + +- public static LocalRemotingCommand createRequestCommand(int code, CommandCustomHeader customHeader) { ++ public static LocalRemotingCommand createRequestCommand(int code, CommandCustomHeader customHeader, String language) { + LocalRemotingCommand cmd = new LocalRemotingCommand(); + cmd.setCode(code); ++ cmd.setLanguage(LanguageCode.getCode(language)); + cmd.writeCustomHeader(customHeader); + cmd.setExtFields(new HashMap<>()); + setCmdVersion(cmd); +@@ -37,4 +40,5 @@ public class LocalRemotingCommand extends RemotingCommand { + Class classHeader) throws RemotingCommandException { + return classHeader.cast(readCustomHeader()); + } ++ + } +diff --git a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java +index 19280f996..2df9fbf02 100644 +--- a/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java ++++ b/remoting/src/main/java/org/apache/rocketmq/remoting/protocol/LanguageCode.java +@@ -17,6 +17,11 @@ + + package org.apache.rocketmq.remoting.protocol; + ++import java.util.Arrays; ++import java.util.Map; ++import java.util.function.Function; ++import java.util.stream.Collectors; ++ + public enum LanguageCode { + JAVA((byte) 0), + CPP((byte) 1), +@@ -50,4 +55,10 @@ public enum LanguageCode { + public byte getCode() { + return code; + } ++ ++ private static final Map MAP = Arrays.stream(LanguageCode.values()).collect(Collectors.toMap(LanguageCode::name, Function.identity())); ++ ++ public static LanguageCode getCode(String language) { ++ return MAP.get(language); ++ } + } +-- +2.32.0.windows.2 + + +From 8f020b397a3afdd75429ae91e3624812b8ffc9e1 Mon Sep 17 00:00:00 2001 +From: Ao Qiao +Date: Mon, 23 Oct 2023 16:34:10 +0800 +Subject: [PATCH 8/8] [ISSUE #7489] Code comment enhancement in example (#7490) + +* Doc: How to debug in Idea + +* en + +* enhance code comment +--- + .../java/org/apache/rocketmq/example/simple/PullConsumer.java | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java b/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java +index 5ac8d247d..e1a02aa26 100644 +--- a/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java ++++ b/example/src/main/java/org/apache/rocketmq/example/simple/PullConsumer.java +@@ -75,7 +75,7 @@ public class PullConsumer { + + if (msgs != null && !msgs.isEmpty()) { + this.doSomething(msgs); +- //update offset to broker ++ //update offset to local memory, eventually to broker + consumer.updateConsumeOffset(messageQueue, pullResult.getNextBeginOffset()); + //print pull tps + this.incPullTPS(topic, pullResult.getMsgFoundList().size()); +-- +2.32.0.windows.2 + diff --git a/rocketmq.spec b/rocketmq.spec index d377d58..0983840 100644 --- a/rocketmq.spec +++ b/rocketmq.spec @@ -5,7 +5,7 @@ Summary: Cloud-Native, Distributed Messaging and Streaming Name: rocketmq Version: 5.1.3 -Release: 23 +Release: 24 License: Apache-2.0 Group: Applications/Message URL: https://rocketmq.apache.org/ @@ -32,6 +32,7 @@ Patch0019: patch019-backport-some-bugfix.patch Patch0020: patch020-backport-add-goaway-mechanism.patch Patch0021: patch021-backport-some-enhancements.patch Patch0022: patch022-backport-Support-KV-Storage-for-ConsumeQueue.patch +Patch0023: patch023-backport-some-bugfixes.patch BuildRequires: java-1.8.0-openjdk-devel, maven, maven-local, git Requires: java-1.8.0-openjdk-devel @@ -66,6 +67,9 @@ exit 0 %changelog +* Tue Dec 8 2023 ShiZhili - 5.1.3-24 +- backport some bugfixes + * Tue Dec 8 2023 ShiZhili - 5.1.3-23 - backport support kv storage -- Gitee