From 9ff6f43611014f3ab7e5544f3e1b6c45eda12ce5 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 10 Oct 2022 17:16:29 +0800 Subject: [PATCH] feat(cec): Upgrade cec to support heartbeat monitoring This update includes the following changes: 1. Refactor the code and also change all comments to English; 2. Ensure all cec code is PEP8 compliant; 3. cec-redis supports heartbeat monitoring to transfer unprocessed messages from consumers in the same consumer group to itself for processing and remove them from the consumer group in a timely manner after detecting that they are offline. --- sysom_api/sdk/cec_base/__init__.py | 9 +- sysom_api/sdk/cec_base/admin.py | 367 +++--- sysom_api/sdk/cec_base/base.py | 87 +- sysom_api/sdk/cec_base/consumer.py | 318 ++--- sysom_api/sdk/cec_base/event.py | 50 +- sysom_api/sdk/cec_base/exceptions.py | 80 ++ sysom_api/sdk/cec_base/log.py | 41 +- sysom_api/sdk/cec_base/meta.py | 74 +- sysom_api/sdk/cec_base/producer.py | 122 +- sysom_api/sdk/cec_base/url.py | 41 +- sysom_api/sdk/cec_redis/__init__.py | 5 +- sysom_api/sdk/cec_redis/admin_static.py | 782 ++++++++++++ sysom_api/sdk/cec_redis/common.py | 163 ++- .../sdk/cec_redis/consume_status_storage.py | 122 +- sysom_api/sdk/cec_redis/heartbeat.py | 269 ++++ sysom_api/sdk/cec_redis/redis_admin.py | 1082 +++++------------ sysom_api/sdk/cec_redis/redis_consumer.py | 444 ++++--- sysom_api/sdk/cec_redis/redis_producer.py | 105 +- sysom_api/sdk/cec_redis/utils.py | 137 ++- 19 files changed, 2726 insertions(+), 1572 deletions(-) create mode 100644 sysom_api/sdk/cec_base/exceptions.py create mode 100644 sysom_api/sdk/cec_redis/admin_static.py create mode 100644 sysom_api/sdk/cec_redis/heartbeat.py diff --git a/sysom_api/sdk/cec_base/__init__.py b/sysom_api/sdk/cec_base/__init__.py index df029668..1429ca8e 100644 --- a/sysom_api/sdk/cec_base/__init__.py +++ b/sysom_api/sdk/cec_base/__init__.py @@ -1 +1,8 @@ -name = "cec_base" +# -*- coding: utf-8 -*- # +""" +Time 2022/07/24 11:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py.py +Description: +""" \ No newline at end of file diff --git a/sysom_api/sdk/cec_base/admin.py b/sysom_api/sdk/cec_base/admin.py index b40eb738..6a479908 100644 --- a/sysom_api/sdk/cec_base/admin.py +++ b/sysom_api/sdk/cec_base/admin.py @@ -1,95 +1,136 @@ # -*- coding: utf-8 -*- # """ +Time 2022/7/26 14:32 Author: mingfeng (SunnyQjm) -Created: 2022/07/24 +Email mfeng@linux.alibaba.com +File admin.py Description: """ import importlib import json +from typing import List from abc import ABCMeta, abstractmethod -from .base import Connectable, Disconnectable -from .base import Registrable, ProtoAlreadyExistsException -from .base import ProtoNotExistsException, CecException +from .base import Connectable +from .exceptions import CecProtoAlreadyExistsException, \ + CecProtoNotExistsException from .event import Event from .meta import TopicMeta, \ ConsumerGroupMemberMeta from .url import CecUrl -from loguru import logger +from .log import LoggerHelper -class ConsumeStatusItem(object): - """ - 消费状态 => 表征了单个消费者组对特定主题的消费情况 +class ConsumeStatusItem: + """Consume status + + Consume status => which indicate consume status of a particular topic + by a particular consumer group. + + Args: + topic(str): Topic name + consumer_group_id(str): Consumer group ID + partition(int): Topic partition ID + + Keyword Args + min_id(str): Minimum ID/offset + max_id(str): Maximum ID/offset + total_event_count(int): Total number of events stored in the partition + (both consumed and unconsumed) + last_ack_id(str): ID of the last acknowledged event of the current + consumer group in the partition. + (ID of the last consumer-acknowledged event) + lag(int): Number of messages stacked in the partition LAG + (number of events that have been submitted to the partition, + but not consumed or acknowledged by a consumer in the current + consumer group) - 1. 最小ID（最小 offset） - 2. 最大ID（最大 offset） - 3. 分区中存储的事件总数（包括已消费的和未消费的） - 4. 最后一个当前消费组在该分区已确认的事件ID（最后一次消费者确认的事件的ID） - 5. 分区的消息堆积数量 LAG（已经提交到该分区，但是没有被当前消费者消费或确认的事件数量） """ + # pylint: disable=too-many-instance-attributes + # Eight is reasonable in this case. def __init__(self, topic: str, consumer_group_id: str, partition: int, - min_id: str = "", max_id: str = "", - total_event_count: int = 0, last_ack_id: str = "", - lag: int = 0): + **kwargs): self.topic = topic self.consumer_group_id = consumer_group_id self.partition = partition - self.min_id = min_id - self.max_id = max_id - self.total_event_count = total_event_count - self.last_ack_id = last_ack_id - self.lag = lag + self.min_id = kwargs.get("min_id", "") + self.max_id = kwargs.get("max_id", "") + self.total_event_count = kwargs.get("total_event_count", 0) + self.last_ack_id = kwargs.get("last_ack_id", "") + self.lag = kwargs.get("lag", 0) - def tojson(self): + def __repr__(self): return json.dumps(self.__dict__) - def __repr__(self): - return self.tojson() + def __str__(self): + return json.dumps(self.__dict__) -class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): +class Admin(Connectable, metaclass=ABCMeta): """Common Event Center Management interface definition - 通用事件中心管理接口定义 + This interface defines the generic behavior of the CEC Admin. + """ - protoDict = { + + proto_dict = { } @abstractmethod def create_topic(self, topic_name: str = "", num_partitions: int = 1, - replication_factor: int = 1, - ignore_exception: bool = False, - expire_time: int = 24 * 60 * 60 * 1000) -> bool: + replication_factor: int = 1, **kwargs) -> bool: """Create one topic - 创建主题 + Create a topic in the Event Center. Args: - topic_name: 主题名字（主题的唯一标识） - num_partitions: 该主题的分区数 - - 1. 该参数指定了在分布式集群部署的场景下，同一个主题的数据应该被划分为几个分区，分别存储在不同的集群节点上； - 2. 如果底层的消息中间件支持分区（比如：Kafka），则可以依据该配置进行分区； - 3. 如果底层的消息中间件不支持分区（比如：Redis），则忽略该参数即可（认定为只有一个分区即可），可以通过 - Admin.is_support_partitions() 方法判定当前使用的消息中间件实现是否支持该特性； - - replication_factor: 冗余因子（指定该主题的数据又几个副本） - - 1. 该参数制定了在分布式集群部署的场景下，同一个主题的分区存在副本的数量，如果 replication_factor == 1 - 则表示主题下的所有分区都只有一个副本，一旦丢失不可回复； - 2. 如果底层的消息中间件支持数据副本，则可以依据该配置进行对应的设置； - 3. 如果底层的消息中间件不支持数据副本，则忽略该参数即可（即认定只有一个副本即可），可以通过 - Admin.is_support_replication() 方法判定当前使用的小心中间件实现是否支持该特性； - - ignore_exception: 是否忽略可能会抛出的异常 - expire_time: 事件超时时间（单位：ms，默认：1day） - - 1. 该参数指定了目标 Topic 中每个事件的有效期； - 2. 一旦一个事件的加入到 Topic 的时间超过了 expire_time，则cec不保证该事件 - 的持久性，cec应当在合适的时候删除超时的事件； - 3. 不强制要求超时的事件被立即删除，可以对超时的事件进行周期性的清理。 + topic_name(str): Topic name (unique identification of the topic) + num_partitions(int): Number of partitions of the topic + 1. This parameter specifies that in a distributed cluster + deployment scenario, data on the same topic should be + partitioned into several partitions, stored on separate + cluster nodes. + 2. If the underlying message queue supports partition + (e.g., Kafka), then partition can be performed based + on this param. + 3. If the underlying messaging queue does not support partition + (e.g. Redis), this parameter is ignored (it is assumed that + there is only one partition). + The Admin.is_support_partitions() method can be used to + determine whether the underlying message queue currently + in use supports this feature. + + replication_factor(int): Redundancy factor (specifies how many + copies of the data for the subject should + be kept in the event center) + + 1. This parameter specifies the number of copies of partitions + of the same topic that exist in a distributed cluster + deployment scenario; if replication_factor == 1, it + indicates that all partitions under the topic have only one + copy and are not reversible in case of loss. + 2. If the underlying message queue supports data replication, + the corresponding settings can be made based on this param. + 3. If the underlying message queue does not support data + replication, this parameter is ignored (i.e. it is assumed + that only one replica is available). + The Admin.is_support_replication() method can be used to + determine whether the underlying message queue currently + in use supports this feature. + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown + expire_time: Event timeout time (in ms, default: 1day) + + 1. This parameter specifies the validity of each event in the + target Topic. + 2. Once an event has been added to Topic for longer than the + expire_time, CEC does not guarantee persistence of the + event and CEC shall remove the timed out event when + appropriate. + 3. Instead of forcing time-out events to be deleted immediately + , time-out events can be cleaned up periodically. Returns: bool: True if successful, False otherwise. @@ -101,18 +142,18 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin.create_topic("test_topic") True """ - pass @abstractmethod - def del_topic(self, topic_name: str, - ignore_exception: bool = False) -> bool: + def del_topic(self, topic_name: str, **kwargs) -> bool: """Delete one topic - 删除主题 + Delete a topic in the Event Center. Args: - topic_name: 主题名字（主题的唯一标识） - ignore_exception: 是否忽略可能会抛出的异常 + topic_name(str): Topic name (unique identification of the topic) + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown Returns: bool: True if successful, False otherwise. @@ -125,15 +166,16 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin.del_topic("test_topic") True """ - pass @abstractmethod - def is_topic_exist(self, topic_name: str) -> bool: - """Judge whether one specific topic is exists - 判断某个主题是否存在 + def is_topic_exist(self, topic_name: str, **kwargs) -> bool: + """Determine whther one specific topic exists + + Determines whether the target topic exists in the currently used event + center. Args: - topic_name: 主题名字（主题的唯一标识） + topic_name(str): Topic name (unique identification of the topic) Returns: bool: True if topic exists, False otherwise. @@ -143,36 +185,35 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin.is_topic_exist("test_topic") True """ - pass @abstractmethod - def get_topic_list(self) -> [TopicMeta]: + def get_topic_list(self, **kwargs) -> List[TopicMeta]: """Get topic list - 获取主题列表 + Get a list of topics contained in the event center currently in use. Args: Returns: - [str]: The topic name list + [str]: The topic meta info list Examples: >>> admin = dispatch_admin("redis://localhost:6379") >>> admin.get_topic_list() [TopicMeta(faeec676-60db-4418-a775-c5f1121d5331, 1)] """ - pass @abstractmethod - def create_consumer_group(self, consumer_group_id: str, - ignore_exception: bool = False) -> bool: + def create_consumer_group(self, consumer_group_id: str, **kwargs) -> bool: """Create one consumer group - 创建一个消费组 + Create a consumer group in the Event Center Args: - consumer_group_id: 消费组ID，应当具有唯一性 - ignore_exception: 是否忽略可能会抛出的异常 + consumer_group_id: Consumer group ID, which should be unique + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown Returns: bool: True if successful, False otherwise. @@ -186,18 +227,18 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin.create_consumer_group("test_group") True """ - pass @abstractmethod - def del_consumer_group(self, consumer_group_id: str, - ignore_exception: bool = False) -> bool: + def del_consumer_group(self, consumer_group_id: str, **kwargs) -> bool: """Delete one consumer group - 删除一个消费组 + Delete a consumer group in the Event Center Args: - consumer_group_id: 消费组ID - ignore_exception: 是否忽略可能会抛出的异常 + consumer_group_id: Consumer group ID, which should be unique + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown Returns: bool: True if successful, False otherwise. @@ -210,16 +251,19 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin.del_consumer_group("test_group") True """ - pass @abstractmethod - def is_consumer_group_exist(self, consumer_group_id: str) -> bool: - """Judge whether one specific consumer group exists + def is_consumer_group_exist(self, consumer_group_id: str, + **kwargs) -> bool: + """Determine whther one specific consumer group exists - 判断某个消费组是否存在 + Determines whether the target consumer group exists in the currently + used event center. Args: - consumer_group_id: 消费组ID + consumer_group_id: Consumer group ID, which should be unique + + Keyword Args: Returns: bool: True if consumer group exists, False otherwise. @@ -229,13 +273,14 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin.is_consumer_group_exist("test_group") True """ - pass @abstractmethod - def get_consumer_group_list(self) -> [ConsumerGroupMemberMeta]: + def get_consumer_group_list(self, **kwargs) -> List[ + ConsumerGroupMemberMeta]: """Get consumer group list - 获取消费组列表 + Get a list of consumer groups contained in the event center currently + in use. Returns: [str]: The consumer group list @@ -244,31 +289,35 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin = dispatch_admin("redis://localhost:6379") >>> admin.get_consumer_group_list() """ - pass @abstractmethod - def get_consume_status(self, topic: str, consumer_group_id: str = "", - partition: int = 0) -> [ConsumeStatusItem]: + def get_consume_status( + self, topic: str, consumer_group_id: str = "", + partition: int = 0, **kwargs + ) -> List[ConsumeStatusItem]: """Get consumption info for specific - 获取特定消费者组对某个主题下的特定分区的消费情况，应包含以下数据 - 1. 最小ID（最小 offset） - 2. 最大ID（最大 offset） - 3. 分区中存储的事件总数（包括已消费的和未消费的） - 4. 最后一个当前消费组在该分区已确认的事件ID（最后一次消费者确认的事件的ID） - 5. 分区的消息堆积数量 LAG（已经提交到该分区，但是没有被当前消费者消费或确认的事件数量） + Get the consumption info of a particular topic by a particular consumer + group. Args: - topic: 主题名字 - consumer_group_id: 消费组ID - 1. 如果 consumer_group_id 为空字符串或者None，则返回订阅了该主题的所有 - 消费组的消费情况；=> 此时 partition 参数无效（将获取所有分区的消费数据） - 2. 如果 consumer_group_id 为无效的组ID，则抛出异常； - 3. 如果 consumer_group_id 为有效的组ID，则只获取该消费组的消费情况。 - partition: 分区ID - 1. 如果 partition 指定有效非负整数 => 返回指定分区的消费情况 - 2. 如果 partition 指定无效非负整数 => 抛出异常 - 3. 如果 partition 指定负数 => 返回当前主题下所有分区的消费情况 + topic(str): Topic name + consumer_group_id(str): Consumer group ID + 1. If consumer_group_id == '' or None, returns the consumption + info of all consumer groups subscribed to the topic; + => In this case the partition parameter is invalid + (will get consumption info for all partitions) + 2. Throws an exception if consumer_group_id is an invalid group + ID; + 3. If consumer_group_id is a valid group ID, then only get + consumption info of the specified consumption group. + partition: Partition ID + 1. If partition specifies a valid non-negative integer + => returns the consumption info of the specified partition; + 2. Throws an exception if partition specifies an invalid + non-negative integer; + 3. If partition specifies a negative number => returns the + consumption info of all partitions under the current topic. Raises: CecException @@ -300,32 +349,30 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): Returns: """ - pass @abstractmethod def get_event_list(self, topic: str, partition: int, offset: str, - count: int) -> [Event]: + count: int, **kwargs) -> List[Event]: """ Get event list for specific - 获取特定主题在指定分区下的消息列表 - 1. offset 和 count 用于分页 + Get a list of messages for a specific topic under a specified partition + 1. offset and count for paging Args: - topic: 主题名字 - partition: 分区ID - offset: 偏移（希望读取在该 ID 之后的消息） - count: 最大读取数量 + topic(str): Topic name + partition: Partition ID + offset: Offset (want to read messages after this ID) + count: Maximum number of reads Returns: """ - pass @abstractmethod - def is_support_partitions(self) -> bool: + def is_support_partitions(self, **kwargs) -> bool: """Is current execution module support partitions - 返回当前执行模块是否支持分区 + Returns whether the current execution module supports partitioning Returns: bool: True if current execution module support partitions, False @@ -336,13 +383,12 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin.is_support_partitions() False """ - pass @abstractmethod - def is_support_replication(self) -> bool: + def is_support_replication(self, **kwargs) -> bool: """Is current execution module support replication - 返回当前的执行模块是否支持数据副本 + Returns whether the current execution module supports data replication Returns: bool: True if current execution module support replication, False @@ -353,18 +399,21 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> admin.is_support_replication() False """ - pass @staticmethod def register(proto, sub_class): """Register one new protocol => indicate one execution module - 注册一个新的协议 => 一个新的执行模块的 Admin 实现要生效，需要调用本方法注册（通常执行 - 模块按规范编写的话，是不需要开发者手动调用本方法的，抽象层会动态导入） + Register a new protocol => This function is called by the executing + module to register its own implementation of Admin for the executing + module to take effect. + (Usually when the execution module is implemented according to the + specification, there is no need for the developer to call this method + manually, the abstraction layer will dynamically import) Args: - proto: 协议标识 - sub_class: 子类 + proto(str): Protocol identification + sub_class: Implementation class of Admin Returns: @@ -372,23 +421,26 @@ class Admin(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> Admin.register('redis', RedisAdmin) """ - if proto in Admin.protoDict: - err = ProtoAlreadyExistsException( + if proto in Admin.proto_dict: + err = CecProtoAlreadyExistsException( f"Proto '{proto}' already exists in Cec-base-Admin." ) - logger.error(err) + LoggerHelper.get_lazy_logger().error(err) raise err - Admin.protoDict[proto] = sub_class - logger.success(f"Cec-base-Admin register proto '{proto}' success") + Admin.proto_dict[proto] = sub_class + LoggerHelper.get_lazy_logger().success( + f"Cec-base-Admin register proto '{proto}' success" + ) def dispatch_admin(url: str, **kwargs) -> Admin: """Construct one Admin instance according the url - 根据传入的 URL，构造对应类型的 Admin 实例 + Construct an Admin instance of the corresponding type based on the URL + passed in. Args: - url: CecUrl + url(str): CecUrl Returns: Admin: One Admin instance @@ -397,42 +449,25 @@ def dispatch_admin(url: str, **kwargs) -> Admin: >>> admin = dispatch_admin("redis://localhost:6379") """ cec_url = CecUrl.parse(url) - if cec_url.proto not in Admin.protoDict: - # 检查是否可以动态导入 + if cec_url.proto not in Admin.proto_dict: + # Check if dynamic import is possible target_module = f"sdk.cec_{cec_url.proto}.{cec_url.proto}_admin" try: module = importlib.import_module(target_module) - Admin.protoDict[cec_url.proto] = \ + Admin.register( + cec_url.proto, getattr(module, f'{cec_url.proto.capitalize()}Admin') - except ModuleNotFoundError: - logger.error( - f"Try to auto import module {target_module} failed.") - err = ProtoNotExistsException( - f"Proto '{cec_url.proto}' not exists in Cec-base-Admin." ) - raise err - admin_instance = Admin.protoDict[cec_url.proto](cec_url, **kwargs) - logger.success( + except ModuleNotFoundError as exc: + LoggerHelper.get_lazy_logger().error( + f"Try to auto import module {target_module} failed." + ) + raise CecProtoNotExistsException( + f"Proto '{cec_url.proto}' not exists in Cec-base-Admin." + ) from exc + admin_instance = Admin.proto_dict[cec_url.proto](cec_url, **kwargs) + LoggerHelper.get_lazy_logger().success( f"Cec-base-Admin dispatch one admin instance success. " - f"proto={cec_url.proto}, url={url}") + f"proto={cec_url.proto}, url={url}" + ) return admin_instance - - -class TopicAlreadyExistsException(CecException): - """在创建 Topic 的过程中，如果当前 Topic 已经存在，则应当抛出本异常""" - pass - - -class TopicNotExistsException(CecException): - """在删除 Topic 的过程中，如果不存在目标 Topic，则应当抛出本异常""" - pass - - -class ConsumerGroupAlreadyExistsException(CecException): - """在创建消费组的过程中，如果当前消费组已经存在，则应当抛出本异常""" - pass - - -class ConsumerGroupNotExistsException(CecException): - """在删除消费组的过程中，如果不存在目标消费组，则应当抛出本异常""" - pass diff --git a/sysom_api/sdk/cec_base/base.py b/sysom_api/sdk/cec_base/base.py index 07ea6026..1d7845fe 100644 --- a/sysom_api/sdk/cec_base/base.py +++ b/sysom_api/sdk/cec_base/base.py @@ -1,92 +1,51 @@ # -*- coding: utf-8 -*- # """ +Time 2022/9/25 13:02 Author: mingfeng (SunnyQjm) -Created: 2022/07/24 +Email mfeng@linux.alibaba.com +File base.py Description: + +This file defines some common interfaces which represents the common behavior +of Consumer, Admin and Producer """ + from abc import ABCMeta, abstractmethod -from loguru import logger class Connectable(metaclass=ABCMeta): - """可连接对象接口，定义了一个可以连接到远端服务的客户端对象的通用行为""" + """An abstract class defines the general behavior of connectable objects + + """ @abstractmethod def connect(self, url: str): """Connect to remote server by url - 通过一个 URL 连接到远端的服务，不限制 URL 的格式，可以自行约定 + Connecting to a remote Server via a URL, the format of the URL can be + agreed upon. Args: - url: + url(str): An identifier for connecting to the server Returns: """ - pass - - -class ConnectException(Exception): - """连接过程中发生任何错误都会抛出本异常""" - pass - - -class Disconnectable(metaclass=ABCMeta): - """可断开连接对象接口，定义了一个可以主动断开远程连接的客户端对象的通用行为""" + raise NotImplementedError @abstractmethod def disconnect(self): - pass - + """Disconnect from remote server -class Registrable(metaclass=ABCMeta): - """可注册对象接口，定义了一个可以注册扩展的类的通用行为""" - - @staticmethod - @abstractmethod - def register(proto, instance): - pass + Disconnecting from the remote server. + Returns: -class Dispatchable(metaclass=ABCMeta): - """可分发对象接口，定义了一个可以通过统一的分发接口产生不同类型实例的对象的通用行为""" - - @staticmethod - @abstractmethod - def dispatch(url: str, *args, **kwargs): - pass - - -class CecException(Exception): - pass - - -class ProtoAlreadyExistsException(CecException): - """协议已经存在异常 - - 1. 在注册一个新的协议时，该协议名已经被注册，则会抛出本异常 - """ - pass - - -class ProtoNotExistsException(CecException): - """协议不存在异常 - - 1. 使用URL分发方式创建实例时，如果对应的协议并没有被注册，则会抛出本异常 - """ - pass - + """ + raise NotImplementedError -def raise_if_not_ignore(is_ignore_exception: bool, exception: Exception): - """工具函数，根据配置选择是否抛出异常 + def __enter__(self): + return self - Args: - is_ignore_exception: Is ignore exception while `exception` be raised. - exception: The exception want to check - """ - if is_ignore_exception: - # 选择忽略异常，则在日志中采用 exception 的方式记录该忽略的异常 - logger.exception(exception) - return False - else: - raise exception + def __exit__(self, exc_type, exc_val, exc_tb): + self.disconnect() diff --git a/sysom_api/sdk/cec_base/consumer.py b/sysom_api/sdk/cec_base/consumer.py index b53eac90..402efce1 100644 --- a/sysom_api/sdk/cec_base/consumer.py +++ b/sysom_api/sdk/cec_base/consumer.py @@ -1,65 +1,80 @@ # -*- coding: utf-8 -*- # """ +Time 2022/7/29 10:21 Author: mingfeng (SunnyQjm) -Created: 2022/07/24 +Email mfeng@linux.alibaba.com +File consumer.py Description: """ import importlib import uuid from abc import ABCMeta, abstractmethod from enum import Enum +from typing import List from .event import Event -from .base import Connectable, Disconnectable -from .base import Registrable -from .base import ProtoAlreadyExistsException, ProtoNotExistsException +from .base import Connectable +from .exceptions import CecProtoAlreadyExistsException, \ + CecProtoNotExistsException +from .log import LoggerHelper from .url import CecUrl -from loguru import logger class ConsumeMode(Enum): """Consume mode enum definition - 消费模式枚举值定义 - - CONSUME_FROM_NOW：在指定的topic上，消费从连接上事件中心开始以后产生的事件（扇形广播模式） - CONSUME_FROM_EARLIEST：在指定的topic上，从最早有记录的事件开始从头消费（扇形广播模式） - CONSUME_GROUP：以组消费的模式进行消费，属于同一个消费组的所有的消费者共同消费一组事件（事件会在多个消费者之间进行负载均衡） + Consume mode enumeration value definition + + CONSUME_FROM_NOW: Consume of events generated from after the moment of + access(Fan Broadcast Mode) + CONSUME_FROM_EARLIEST: Consume from the earliest events(Fan Broadcast Mode) + CONSUME_GROUP: Consumption in a group consumption model, where all + consumers belonging to the same consumption group consume a + set of events together (events are load balanced across + multiple consumers) """ CONSUME_FROM_NOW = 1 CONSUME_FROM_EARLIEST = 2 CONSUME_GROUP = 3 -class Consumer(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): +class Consumer(Connectable, metaclass=ABCMeta): """Common Event Center Consumer interface definition - 通用事件中心，消费者接口定义 + This interface defines the generic behavior of the CEC Consumer. Args: - topic_name(str): 主题名字（主题的唯一标识） - consumer_id(str): 消费者ID，唯一标识一个消费者 - group_id(str): 消费者ID，唯一标识一个消费组 - start_from_now(bool): 是否从最早的事件开始消费 - default_batch_consume_limit(int): 默认批量消费限制 + topic_name(str): Topic name (unique identification of the subject) + consumer_id(str): Consumer ID, which uniquely identifies a consumer + group_id(str): Consumer ID, which uniquely identifies a consumer group + start_from_now(bool): Does consumption begin with the earliest events + + Keyword Args: + default_batch_consume_limit(int): Default batch consume limit + auto_convert_to_dict(bool): Whether to automatically treat the event as + json and convert it to dict Attributes: - topic_name(str): 主题名字（主题的唯一标识） - consumer_id(str): 消费者ID，唯一标识一个消费者 - group_id(str): 消费者ID，唯一标识一个消费组 - default_batch_consume_limit(int): 默认批量消费限制 - consume_mode(ConsumeMode): 消费模式 + topic_name(str): Topic name (unique identification of the subject) + consumer_id(str): Consumer ID, which uniquely identifies a consumer + group_id(str): Consumer ID, which uniquely identifies a consumer group + start_from_now(bool): Does consumption begin with the earliest events + default_batch_consume_limit(int): Default batch consume limit + auto_convert_to_dict(bool): Whether to automatically treat the event as + json and convert it to dict """ - protoDict = { + proto_dict = { } def __init__(self, topic_name: str, consumer_id: str = "", - group_id: str = "", start_from_now: bool = True, - default_batch_consume_limit: int = 10): + group_id: str = "", start_from_now: bool = True, **kwargs): self.topic_name = topic_name self.consumer_id = consumer_id - self.default_batch_consume_limit = default_batch_consume_limit + self.default_batch_consume_limit = kwargs.get( + "default_batch_consume_limit", 10 + ) + self.auto_convert_to_dict = kwargs.get("auto_convert_to_dict", True) if consumer_id is None or consumer_id == "": self.consumer_id = Consumer.generate_consumer_id() self.group_id = group_id @@ -71,30 +86,38 @@ class Consumer(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): @abstractmethod def consume(self, timeout: int = -1, auto_ack: bool = False, - batch_consume_limit: int = 0) -> [Event]: - """Start to consume the event from event center according to the - corresponding ConsumeMode + batch_consume_limit: int = 0, **kwargs) -> List[Event]: + """Consuming events from the Event Center - 根据对应消费模式的行为，开始从事件中心消费事件 + Start to consume the event from event center according to the + corresponding ConsumeMode Args: - timeout(int): 超时等待时间（单位：ms），<= 表示阻塞等待 - auto_ack(bool): 是否开启自动确认（组消费模式有效） - - 1. 一旦开启自动确认，每成功读取到一个事件消息就会自动确认； - 2. 调用者一定要保证消息接收后正常处理，因为一旦某个消息被确认，消息中心不保证下次 - 仍然可以获取到该消息，如果客户端在处理消息的过程中奔溃，则该消息或许无法恢复； - 3. 所以最保险的做法是，auto_ack = False 不开启自动确认，在事件被正确处理完 - 后显示调用 Consumer.ack() 方法确认消息被成功处理; - 4. 如果有一些使用组消费业务，可以承担事件丢失无法恢（只会在客户端程序奔溃没有正确 - 处理的情况下才会发生）的风险，则可以开启 auto_ack 选项。 - - batch_consume_limit(int): 批量消费限制 - - 1. 该参数指定了调用 consume 方法，最多一次拉取的事件的数量； - 2. 如果该值 <= 0 则将采用 self.default_batch_consume_limit 中指定的缺省 - 值； - 3. 如果该值 > 0 则将覆盖 self.default_batch_consume_limit，以本值为准。 + timeout(int): Blocking wait time + (Negative numbers represent infinite blocking wait) + auto_ack(bool): Whether to enable automatic confirmation + (valid for group consumption mode) + + 1. Once automatic acknowledgement is turned on, every event + successfully read will be automatically acknowledged; + 2. Caller must ensure that the event is processed properly + after it is received, because once a event is acknowledged, + the event center does not guarantee that the event will + still be available next time, and if the client runs down + while processing the message, the message may not be + recoverable; + 3. So it is safest to leave auto_ack = False and explicitly + call the Consumer.ack() method to acknowledge the event + after it has been processed correctly; + + batch_consume_limit(int): Batch consume limit + + 1. This parameter specifies the number of events to be pulled + at most once by calling the consume method; + 2. If the value <= 0 then the default value specified in + self.default_batch_consume_limit will be used； + 3. If this value > 0 then it will override + self.default_batch_consume_limit, use current passed value. Returns: [Message]: The Event list @@ -107,19 +130,21 @@ class Consumer(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): ... , start_from_now=False) >>> consumer.consume(200, auto_ack=False, batch_consume_limit=20) """ - pass @abstractmethod - def ack(self, event: Event) -> int: + def ack(self, event: Event, **kwargs) -> int: """Confirm that the specified event has been successfully consumed - 对指定的事件进行消费确认 - 1. 通常应当在取出事件，并成功处理之后对该事件进行确认； + Acknowledgement of the specified event + 1. The event should normally be acknowledged after it has been taken + out and successfully processed. Args: - event(Event): 要确认的事件 - 1. 必须是通过 Consumer 消费获得的 Event 实例； - 2. 自行构造的 Event 传递进去不保证结果符合预期 + event(Event): Events to be confirmed + 1. Must be an instance of the Event obtained through Consumer + interface; + 2. Passing in a self-constructed Event does not guarantee that + the result will be as expected. Returns: int: 1 if successfully, 0 otherwise @@ -130,61 +155,34 @@ class Consumer(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): ... , 'this_is_a_test_topic' ... , consumer_id=Consumer.generate_consumer_id() ... , start_from_now=False) - >>> msgs = consumer.consume(200, auto_ack=False, batch_consume_limit=1) + >>> msgs = consumer.consume(200, auto_ack=False) >>> msg = msgs[0] >>> consumer.ack(msg) """ - pass - - # @abstractmethod - # def convert_to_broadcast_consume_mode(self): - # """Change consume mode to broadcast mode - # - # 从组消费模式切换到广播消费模式 - # 1. 如果当前已经处于广播消费模式，则本函数什么也不做 - # 2. 如果当前处于组消费模式，将切换到广播消费模式，并且： - # a. 如果当前已经在组消费模式下消费若干事件了，则接下来会消费当前主题最后一次消费的 - # 事件之后的所有事件； - # b. 如果当前在组消费模式下还没有消费任何事件，则只是简单的切换到广播消费，接下来会 - # 消费当前接入时间之后产生的事件；（不建议，还不如直接dispatch_consumer的时 - # 候就指定广播消费） - # - # 提供本接口的考虑： - # 假设我们现在有一个场景：一个系统 S 由前端（S-Web）+后台（S-Server）组成，S-Server - # 和S-Web之间使用Websocket通信建立了一套通知系统，S-Server会在某些事件产生时，将通知 - # 推送给特定的用户。并且当前系统允许多登陆，即可以在多个设备上登录同一个账号，那么当后台 - # 产生事件时，应该推送给哪个设备呢？ - # => 解决方案：对于登录同一个账号并且同时在线的多个设备，认定第一个登录的为主设备，S-Server - # 会将所有产生的通知（包括没有设备登录时累积的通知） - # - # Returns: - # - # """ - # pass @abstractmethod def __getitem__(self, item): - """ Require subclass to implement __getitem__ to support for-each + """Require subclass to implement __getitem__ to support for-each - 要求客户端实现 __getitem__ 以支持使用for循环直接遍历事件 + Args: + item: + + Returns: - :param item: - :return: """ - pass @staticmethod def generate_consumer_id() -> str: """Generate one random consumer ID - 随机生成一个消费者ID + Generate a random consumer ID Returns: str: The generated consumer ID Examples: >>> Consumer.generate_consumer_id() - UUID('30e2fda7-d4b2-48b0-9338-78ff389648e7') + 30e2fda7-d4b2-48b0-9338-78ff389648e7 """ return str(uuid.uuid4()) @@ -192,12 +190,16 @@ class Consumer(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): def register(proto, sub_class): """Register one new protocol => indicate one execution module - 注册一个新的协议 => 一个新的执行模块的 Consumer 实现要生效，需要调用本方法注册（通常执行 - 模块按规范编写的话，是不需要开发者手动调用本方法的，抽象层会动态导入） + Register a new protocol => This function is called by the executing + module to register its own implementation of Consumer for the executing + module to take effect. + (Usually when the execution module is implemented according to the + specification, there is no need for the developer to call this method + manually, the abstraction layer will dynamically import) Args: - proto: 协议标识 - sub_class: 子类 + proto(str): Protocol identification + sub_class: Implementation class of Consumer Returns: @@ -205,53 +207,77 @@ class Consumer(Connectable, Disconnectable, Registrable, metaclass=ABCMeta): >>> Consumer.register('redis', RedisConsumer) """ - if proto in Consumer.protoDict: - err = ProtoAlreadyExistsException( + if proto in Consumer.proto_dict: + err = CecProtoAlreadyExistsException( f"Proto '{proto}' already exists in Cec-base-Consumer." ) - logger.error(err) + LoggerHelper.get_lazy_logger().error(err) raise err - Consumer.protoDict[proto] = sub_class - logger.success(f"Cec-base-Consumer register proto '{proto}' success") + Consumer.proto_dict[proto] = sub_class + LoggerHelper.get_lazy_logger().success( + f"Cec-base-Consumer register proto '{proto}' success" + ) def dispatch_consumer(url: str, topic_name: str, consumer_id: str = "", group_id: str = "", start_from_now: bool = True, - default_batch_consume_limit: int = 10, **kwargs) -> Consumer: """Construct one Consumer instance according the url - 根据传入的 URL，构造对应类型的 Consumer 实例 + Construct a Consumer instance of the corresponding type based on the URL + passed in. Args: - url: CecUrl - topic_name: 主题名字（主题的唯一标识） - consumer_id: 消费者ID，唯一标识一个消费者 - - 1. 如果是组消费模式，唯一标识一个消费组中的消费者; - 2. consumer_id 建议使用 Consumer.generate_consumer_id() 方法生成； - 3. 如果没有指定 consumer_id，则内部会使用 Consumer.generate_consumer_id() - 自动填充该字段。 - - group_id: 消费者ID，唯一标识一个消费组 - - 1. 如果不传递该字段，则默认采用广播消费模式（可以搭配 start_from_now 指定从什 - 么位置开始消费事件）； - 2. 如果传递了 group_id，则开启组消费模式。 - - start_from_now: 是否从最早的事件开始消费 - - 1. 如果 group_id 字段已指定，则本字段会直接忽略（因为本字段在组消费模式下无效）； - 2. 否则，start_from_now == True 表示从该Topic的最早的有记录的事件开始消费。 - - default_batch_consume_limit: 默认批量消费限制 - - 1. 该参数指定了默认情况下，调用 consume 方法，最多一次拉取的事件的数量； - 2. 由于客户端每次从事件中心拉取消息都会经历一次往返延迟，在网络延迟较大的时，如 - 果每次只能拉取一个消息，会极大的限制消费速率（message per second），因此 - 在网络延迟较大的情况下，可以适当增大每次批量拉取消息的上限； - 3. 本参数指定的是缺省情况下的默认值，可以在调用 consume 方法的时候，传递 - 'batch_consume_limit' 参数覆盖该值。 + url(str): CecUrl + topic_name: Topic name (unique identification of the topic) + consumer_id: Consumer ID, which uniquely identifies a consumer + + 1. consumer_id is recommended to be generated using the + Consumer.generate_consumer_id() method; + 2. If consumer_id is not specified, the field is automatically + populated internally using Consumer.generate_consumer_id(). + + group_id: Consumer group ID, which uniquely identifies a consumer group + + 1. If this field is not passed, the broadcast consumption mode is + used by default (which can be paired with start_from_now to + specify where to start consuming events from); + 2. If group_id is passed, group consumption mode is enabled. + + start_from_now: Does consumption begin with the earliest events + + 1. If the group_id field is specified, this field is simply ignored + (as this field is not valid in group consumption mode); + 2. Otherwise, start_from_now == True means that consumption starts + from the earliest recorded event for that Topic. + + Keyword Args: + default_batch_consume_limit: Default batch consume limit + + 1. This parameter specifies the number of events that will be + pulled at most once by calling the consume method by default; + 2. Since the client will experience a round-trip delay each time it + pulls an event from the event center, if it can only pull one + event at a time when the network latency is high, it will + greatly limit the consumption rate (messages per second), so the + upper limit of events per bulk pull can be appropriately + increased in the case of high network latency; + 3. This parameter specifies the default value, which can be + overridden by passing the 'batch_consume_limit' parameter to the + consume method when it is called. + + auto_convert_to_dict: Whether to automatically treat the event as json + and convert it to dict + 1. CEC supports delivering bytes or dict when using Producer for + message production; + 2. If the corresponding prodcuer chooses to deliver a dict, the + consumer may set this parameter to 'True' to ensure that the + message returned via Consumer.consume() is automatically json + decoded to a dict. + 3. If the corresponding producer chooses to deliver bytes, the + consumer must set this parameter to 'False', because the + underlying event content is in bytes format and can not be + automatically decoded to dict. Returns: Consumer: One Consumer instance @@ -264,30 +290,32 @@ def dispatch_consumer(url: str, topic_name: str, consumer_id: str = "", ... , start_from_now=False) """ cec_url = CecUrl.parse(url) - if cec_url.proto not in Consumer.protoDict: - # 检查是否可以动态导入包 + if cec_url.proto not in Consumer.proto_dict: + # Check if dynamic import is possible target_module = f"sdk.cec_{cec_url.proto}.{cec_url.proto}_consumer" try: module = importlib.import_module(target_module) - Consumer.protoDict[cec_url.proto] = \ + Consumer.register( + cec_url.proto, getattr(module, f'{cec_url.proto.capitalize()}Consumer') - except ModuleNotFoundError: - logger.error( - f"Try to auto import module {target_module} failed.") - err = ProtoNotExistsException( - f"Proto '{cec_url.proto}' not exists in Cec-base-Consumer." ) - raise err - consumer_instance = Consumer.protoDict[cec_url.proto]( + except ModuleNotFoundError as exc: + LoggerHelper.get_lazy_logger().error( + f"Try to auto import module {target_module} failed." + ) + raise CecProtoNotExistsException( + f"Proto '{cec_url.proto}' not exists in Cec-base-Consumer." + ) from exc + consumer_instance = Consumer.proto_dict[cec_url.proto]( cec_url, - topic_name, - consumer_id, - group_id, - start_from_now, - default_batch_consume_limit, + topic_name=topic_name, + consumer_id=consumer_id, + group_id=group_id, + start_from_now=start_from_now, **kwargs ) - logger.success( + LoggerHelper.get_lazy_logger().success( f"Cec-base-Consumer dispatch one consumer instance success. " - f"proto={cec_url.proto}, url={url}") + f"proto={cec_url.proto}, url={url}" + ) return consumer_instance diff --git a/sysom_api/sdk/cec_base/event.py b/sysom_api/sdk/cec_base/event.py index 9ffb08e4..3912d7da 100644 --- a/sysom_api/sdk/cec_base/event.py +++ b/sysom_api/sdk/cec_base/event.py @@ -1,38 +1,68 @@ # -*- coding: utf-8 -*- # """ +Time 2022/07/25 12:16 Author: mingfeng (SunnyQjm) -Created: 2022/07/25 +Email mfeng@linux.alibaba.com +File event.py Description: + +This file define the Event object used in CEC(Common Evcent Center) """ +from typing import Union, Any + -class Event(object): +class Event: """Common Event Center Event definition - 通用事件中心 Event 定义 + This class define the Event model used in CEC Args: - value(dict): The event content + value(bytes | dict): The event content（support bytes and dict） event_id(str): Event ID Attributes: - value(dict): The event content + value(bytes | dict): The event content event_id(str): Event ID """ - def __init__(self, value=None, event_id: str = ""): + def __init__(self, value: Union[bytes, dict] = None, event_id: str = ""): if value is None: - value = dict() + value = {} self.value = value self.event_id = event_id - # cache 用于底层实现缓存数据，用户代码不应当依赖该属性 - self._cache = dict() + # An inner storage, used to cache some internal data related to + # specific event objects + self._cache = {} + + def put(self, key: str, value: Any): + """Store something in inner cache + + The current interface is only used inside the module，used cached some + internal data. + + Args: + key(str): + value(any): - def put(self, key: str, value): + Returns: + + """ self._cache[key] = value def get(self, key): + """Get something from inner cache + + The current interface is only used inside the module，used get some + internal data cached in current Event object. + + Args: + key: + + Returns: + + """ return self._cache.get(key) def __repr__(self): diff --git a/sysom_api/sdk/cec_base/exceptions.py b/sysom_api/sdk/cec_base/exceptions.py new file mode 100644 index 00000000..a6dfaf93 --- /dev/null +++ b/sysom_api/sdk/cec_base/exceptions.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/9/25 13:07 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File exceptions.py +Description: + +This file defines the exceptions that may be thrown in the CEC +""" + + +class CecException(Exception): + """CEC base exception + + This class defines the base exception for CEC, and all exceptions thrown + by CEC should inherit from this class. + """ + + +class CecConnectionException(CecException): + """An exception that may be thrown during the connection phase + + This exception should be thrown if any error occurs while a connectable + object is connecting to the server. + """ + + +class CecProtoAlreadyExistsException(CecException): + """Exceptions thrown for duplicate proto(submodule) registration + + This exception should be thrown if the proto(submodule) already exists while + registering a submodule. + """ + + +class CecProtoNotExistsException(CecException): + """Exceptions thrown for trying to use a non-existent proto(submodule) + + Exceptions that will be thrown when trying to use a non-existent proto + (submodule). + """ + + +class CecNotValidCecUrlException(CecException): + """Exception thrown when an invalid CecUrl format is parsed. + + """ + + +class TopicAlreadyExistsException(CecException): + """Exception for trying to add a topic that already exists + + During the creation of a Topic, this exception should be thrown if the + current Topic already exist. + """ + + +class TopicNotExistsException(CecException): + """Exception thrown when trying to manipulate a topic that does not exist + + This exception should be thrown if the target Topic does not exist during + access, update or deletion of a Topic. + """ + + +class ConsumerGroupAlreadyExistsException(CecException): + """Exception thrown when trying to add a consumer group that already exists + + During the creation of a consumer group, this exception should be thrown + if the current consumer group already exists. + """ + + +class ConsumerGroupNotExistsException(CecException): + """Exception thrown when trying to manipulate a non-existent consumer group. + + This exception should be thrown if the target consumer group does not exist + during access, update or deletion of a consumer group. + """ diff --git a/sysom_api/sdk/cec_base/log.py b/sysom_api/sdk/cec_base/log.py index 247842b1..ac1cc220 100644 --- a/sysom_api/sdk/cec_base/log.py +++ b/sysom_api/sdk/cec_base/log.py @@ -1,25 +1,25 @@ # -*- coding: utf-8 -*- # """ -Time 2022/8/2 17:40 +Time 2022/7/26 10:20 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File log.py Description: + +This file provides a log printing interface to the CEC, providing basic +printing functionality and masking the log library used by the underlying """ import sys from enum import Enum from loguru import logger -# 移除默认的输出到终端的 sink => 默认不打印日志 +# Remove default output to terminal sink => no logs printed by default logger.remove() -# 使用 lazy_logger 打印的日志会根据当前 sink 的日志等级过滤 -# 1. 比如当前日志等级为 INFO，则所有日志等级小于 INFO 的日志都将被过滤； -lazy_logger = logger.opt(lazy=True) - class LoggerLevel(Enum): - """日志等级枚举类""" + """ An enum class that defines the log level + """ LOGGER_LEVEL_TRACE = "TRACE" LOGGER_LEVEL_DEBUG = "DEBUG" LOGGER_LEVEL_INFO = "INFO" @@ -30,13 +30,17 @@ class LoggerLevel(Enum): class LoggerHelper: - """日志辅助类""" + """A logging helper class + + """ - # 使用 lazy_logger 打印的日志会根据当前 sink 的日志等级过滤 - # 1. 比如当前日志等级为 INFO，则所有日志等级小于 INFO 的日志都将被过滤 + # Logs printed with lazy_logger are filtered according to the current + # sink's log level + # 1. for example, if the current log level is INFO, then all logs with a + # log level less than INFO will be filtered. _lazy_logger = logger.opt(lazy=True) - # 记录 stdout 日志的句柄 + # Handle to log stdout logs _stdout_logger_handle_id = None @staticmethod @@ -47,7 +51,7 @@ class LoggerHelper: @staticmethod def update_sys_stdout_sink(level: LoggerLevel): - """Update the level of sys.stdout + """Update the level of 'sys.stdout' Args: level(LoggerLevel): New Level @@ -56,9 +60,11 @@ class LoggerHelper: """ logger.remove(LoggerHelper._stdout_logger_handle_id) - LoggerHelper._stdout_logger_handle_id = logger.add(sys.stdout, - colorize=True, - level=level.value) + LoggerHelper._stdout_logger_handle_id = logger.add( + sys.stdout, + colorize=True, + level=level.value + ) LoggerHelper._update_lazy_logger() return LoggerHelper @@ -66,12 +72,13 @@ class LoggerHelper: def add(sink: str, level: LoggerLevel, **kwargs): """Add new sink - 详情参考：https://github.com/Delgan/loguru - Args: sink(str): New sink path level(LoggerLevel): The log level of new sink kwargs: Other params define in loguru + + References: + https://github.com/Delgan/loguru """ kwargs['level'] = level.value logger.add(sink, **kwargs) diff --git a/sysom_api/sdk/cec_base/meta.py b/sysom_api/sdk/cec_base/meta.py index 1e76a598..a21a495b 100644 --- a/sysom_api/sdk/cec_base/meta.py +++ b/sysom_api/sdk/cec_base/meta.py @@ -1,23 +1,25 @@ # -*- coding: utf-8 -*- # """ -Time 2022/7/27 15:55 +Time 2022/7/25 12:34 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File meta.py Description: -""" +This file define some meta info objects which used to manage CEC +""" +from typing import List class TopicMeta: - """Common topic meta info definition + """Common Topic meta info definition - 通用 Topic 元数据信息定义 + This class define the Topic's meta info. Args: - topic_name(str): 主题名字 + topic_name(str): Topic name Attributes: - topic_name(str): 主题名字 + topic_name(str): Topic name """ def __init__(self, topic_name: str = ""): @@ -27,12 +29,10 @@ class TopicMeta: self.error = None def __repr__(self): - if self.error is not None: - return f"TopicMeta({self.topic_name}, {len(self.partitions)} " \ - f"partitions, {self.error})" - else: - return f"TopicMeta({self.topic_name}, {len(self.partitions)} " \ - f"partitions)" + return f"TopicMeta(" \ + f"{self.topic_name}, {len(self.partitions)} partitions" \ + f"{f', {self.error}' if self.error is not None else ''}" \ + f")" def __str__(self): return self.topic_name @@ -41,10 +41,13 @@ class TopicMeta: class PartitionMeta: """Common Partition meta info definition - 通用 Partition 元数据定义 + This class define the Partition's meta info. Args: + partition_id(int): Partition ID + Attributes: + partition_id(int): Partition ID """ def __init__(self, partition_id: int = -1): @@ -52,10 +55,10 @@ class PartitionMeta: self.error = None def __repr__(self): - if self.error is not None: - return f"PartitionMeta({self.partition_id}, {self.error})" - else: - return f"PartitionMeta({self.partition_id})" + return f"PartitionMeta(" \ + f"{self.partition_id}" \ + f"{f', {self.error}' if self.error is not None else ''}" \ + f")" def __str__(self): return f"{self.partition_id}" @@ -64,22 +67,25 @@ class PartitionMeta: class ConsumerGroupMeta: """Common Consumer Group meta info definition - 通用 ConsumerGroup 元数据定义 + This class define the ConsumerGroup's meta info. + Args: + group_id(str): Group ID + + Attributes: + group_id(str): Group ID """ def __init__(self, group_id: str = ""): self.group_id = group_id - self.members: [ConsumerGroupMemberMeta] = [] + self.members: List[ConsumerGroupMemberMeta] = [] self.error = None def __repr__(self): - if self.error is not None: - return f"ConsumerGroupMeta({self.group_id}, {len(self.members)} " \ - f"members, {self.error})" - else: - return f"ConsumerGroupMeta({self.group_id}, {len(self.members)} " \ - f"members)" + return f"ConsumerGroupMeta(" \ + f"{self.group_id}, {len(self.members)} members" \ + f"{f', {self.error}' if self.error is not None else ''}" \ + f")" def __str__(self): return self.group_id @@ -88,8 +94,13 @@ class ConsumerGroupMeta: class ConsumerGroupMemberMeta: """Common Consumer Group Member meta info definition - 通用 ConsumerGroupMember 元数据定义 + This class define the ConsumerGroupMember's meta info + Args: + client_id(str): Client ID + + Attributes: + client_id(str): Client ID """ def __init__(self, client_id: str = ""): @@ -97,7 +108,10 @@ class ConsumerGroupMemberMeta: self.error = None def __repr__(self): - if self.error is not None: - return f"ConsumerGroupMemberMeta({self.client_id}, {self.error})" - else: - return f"ConsumerGroupMemberMeta({self.client_id})" + return f"ConsumerGroupMemberMeta(" \ + f"{self.client_id}" \ + f"{f', {self.error}' if self.error is not None else ''}" \ + f")" + + def __str__(self): + return self.client_id diff --git a/sysom_api/sdk/cec_base/producer.py b/sysom_api/sdk/cec_base/producer.py index 3e4e34d6..b4c0a95c 100644 --- a/sysom_api/sdk/cec_base/producer.py +++ b/sysom_api/sdk/cec_base/producer.py @@ -1,64 +1,71 @@ # -*- coding: utf-8 -*- # """ +Time 2022/7/27 9:21 Author: mingfeng (SunnyQjm) -Created: 2022/07/25 +Email mfeng@linux.alibaba.com +File producer.py Description: """ import importlib from abc import ABCMeta, abstractmethod -from typing import Callable -from .base import Connectable, Disconnectable -from .base import Registrable -from .base import ProtoAlreadyExistsException, ProtoNotExistsException +from typing import Callable, Union +from .base import Connectable +from .exceptions import CecProtoAlreadyExistsException +from .exceptions import CecProtoNotExistsException +from .log import LoggerHelper from .url import CecUrl from .event import Event -from loguru import logger -class Producer(Connectable, Disconnectable, Registrable, - metaclass=ABCMeta): +class Producer(Connectable, metaclass=ABCMeta): """Common Event Center Producer interface definition - 通用事件中心，生产者接口定义 + This interface defines the generic behavior of the CEC Producer. """ - protoDict = {} + proto_dict = {} @abstractmethod - def produce(self, topic_name: str, message_value: dict, - callback: Callable[[Exception, Event], None] = None, - partition: int = -1, - **kwargs): - """Generate one new event, then put it to event center + def produce(self, topic_name: str, message_value: Union[bytes, dict], + callback: Callable[[Exception, Event], None] = None, **kwargs): + """Generate a new event, then put it to event center - 生成一个新的事件，并将其注入到事件中心当中（本操作默认是异步的，如果想实现同步， - 请搭配 flush 方法使用） + Generate a new event and inject it into the event center (this + operation is asynchronous by default, if you want to be synchronous, + use it with the flush method) Args: - topic_name: 主题名称 - message_value: 事件内容 - callback(Callable[[Exception, Event], None]): 事件成功投递到事件中心回调 - partition(int): 分区号 - 1. 如果指定了有效分区号，消息投递给指定的分区（不建议）； - 2. 传递了一个正数分区号，但是无此分区，将抛出异常； - 3. 传递了一个负数分区号（比如-1），则消息将使用内建的策略均衡的投 - 递给所有的分区（建议）。 + topic_name(str): Topic name + message_value(bytes | dict): Event value + callback(Callable[[Exception, Event], None]): Event delivery + results callback + + Keyword Args: + partition(int): Partition ID + 1. If a valid partition number is specified, the event is + deliverd to the specified partition (not recommended); + 2. A positive partition ID is passed, but no such partition is + available, an exception will be thrown. + 3. A negative partition number is passed (e.g. -1), then the + event will be cast to all partitions in a balanced manner + using the built-in policy (recommended). Examples: >>> producer = dispatch_producer( ..."redis://localhost:6379?password=123456") >>> producer.produce("test_topic", {"value": "hhh"}) """ - pass @abstractmethod - def flush(self, timeout: int = -1): + def flush(self, timeout: int = -1, **kwargs): """Flush all cached event to event center - 将在缓存中还未提交的所有事件都注入到事件中心当中（这是一个阻塞调用） + Deliver all events in the cache that have not yet been committed into + the event center (this is a blocking call) Args: - timeout: 超时等待时间（单位：ms），<= 表示阻塞等待 + timeout(int): Blocking wait time + (Negative numbers represent infinite blocking wait) Examples: >>> producer = dispatch_producer( @@ -66,18 +73,21 @@ class Producer(Connectable, Disconnectable, Registrable, >>> producer.produce("test_topic", {"value": "hhh"}) >>> producer.flush() """ - pass @staticmethod def register(proto, sub_class): """Register one new protocol => indicate one execution module - 注册一个新的协议 => 一个新的执行模块的 Producer 实现要生效，需要调用本方法注册（通常执行 - 模块按规范编写的话，是不需要开发者手动调用本方法的，抽象层会动态导入） + Register a new protocol => This function is called by the executing + module to register its own implementation of Producer for the executing + module to take effect. + (Usually when the execution module is implemented according to the + specification, there is no need for the developer to call this method + manually, the abstraction layer will dynamically import) Args: - proto: 协议标识 - sub_class: 子类 + proto(str): Protocol identification + sub_class: Implementation class of Producer Returns: @@ -85,20 +95,23 @@ class Producer(Connectable, Disconnectable, Registrable, >>> Producer.register('redis', RedisProducer) """ - if proto in Producer.protoDict: - err = ProtoAlreadyExistsException( + if proto in Producer.proto_dict: + err = CecProtoAlreadyExistsException( f"Proto '{proto}' already exists in Cec-base-Producer." ) - logger.error(err) + LoggerHelper.get_lazy_logger().error(err) raise err - Producer.protoDict[proto] = sub_class - logger.success(f"Cec-base-Producer register proto '{proto}' success") + Producer.proto_dict[proto] = sub_class + LoggerHelper.get_lazy_logger().success( + f"Cec-base-Producer register proto '{proto}' success" + ) def dispatch_producer(url: str, **kwargs) -> Producer: """Construct one Producer instance according the url - 根据传入的 URL，构造对应的 Producer 实例 + Construct a Producer instance of the corresponding type based on the URL + passed in. Args: url(str): CecUrl @@ -111,22 +124,25 @@ def dispatch_producer(url: str, **kwargs) -> Producer: ..."redis://localhost:6379?password=123456") """ cec_url = CecUrl.parse(url) - if cec_url.proto not in Producer.protoDict: - # 检查是否可以动态导入包 + if cec_url.proto not in Producer.proto_dict: + # Check if dynamic import is possible target_module = f"sdk.cec_{cec_url.proto}.{cec_url.proto}_producer" try: module = importlib.import_module(target_module) - Producer.protoDict[cec_url.proto] = \ + Producer.register( + cec_url.proto, getattr(module, f'{cec_url.proto.capitalize()}Producer') - except ModuleNotFoundError: - logger.error( - f"Try to auto import module {target_module} failed.") - err = ProtoNotExistsException( - f"Proto '{cec_url.proto}' not exists in Cec-base-Producer." ) - raise err - producer_instance = Producer.protoDict[cec_url.proto](cec_url, **kwargs) - logger.success( + except ModuleNotFoundError as exc: + LoggerHelper.get_lazy_logger().error( + f"Try to auto import module {target_module} failed." + ) + raise CecProtoNotExistsException( + f"Proto '{cec_url.proto}' not exists in Cec-base-Producer." + ) from exc + producer_instance = Producer.proto_dict[cec_url.proto](cec_url, **kwargs) + LoggerHelper.get_lazy_logger().success( f"Cec-base-Producer dispatch one producer instance success. " - f"proto={cec_url.proto}, url={url}") - return producer_instance + f"proto={cec_url.proto}, url={url}" + ) + return producer_instance \ No newline at end of file diff --git a/sysom_api/sdk/cec_base/url.py b/sysom_api/sdk/cec_base/url.py index 1dc6a8b2..037d95c6 100644 --- a/sysom_api/sdk/cec_base/url.py +++ b/sysom_api/sdk/cec_base/url.py @@ -1,28 +1,33 @@ # -*- coding: utf-8 -*- # """ -Time 2022/7/25 16:03 +Time 2022/7/25 14:02 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com -File url_test.py +File url.py Description: """ + import urllib.parse +from .exceptions import CecNotValidCecUrlException class CecUrl: """CecUrl definition - Cec URL 格式定义，其由三部分组成（proto, connect_url, params） + Cec URL format definition, which consists of three parts + (proto, netloc, params) Args: - proto(str): 协议标识（例如：redis） - netloc(str): 连接地址，主要用于连接低层的消息中间件（例如：localhost:6379） - params(dict): 连接参数（例如：{"password": "123456"}） + proto(str): Protocol identifier (e.g., redis) + netloc(str): Connection address, mainly used to connect to low-level + messaging middleware (e.g., localhost:6379) + params(dict): Connection parameters (e.g., {"password": "123456"}) Attributes: - proto(str): 协议标识（例如：redis） - netloc(str): 连接地址，主要用于连接低层的消息中间件（例如：localhost:6379） - params(dict): 连接参数（例如：{"password": "123456"}） + proto(str): Protocol identifier (e.g., redis) + netloc(str): Connection address, mainly used to connect to low-level + messaging middleware (e.g., localhost:6379) + params(dict): Connection parameters (e.g., {"password": "123456"}) """ def __init__(self, proto: str, netloc: str, params: dict): @@ -36,21 +41,25 @@ class CecUrl: @staticmethod def parse(url: str): + """Parses a string into a CecUrl object + + Args: + url(str) + + Returns: + CecUrl + """ parse_result = urllib.parse.urlparse(url) proto, netloc = parse_result.scheme, parse_result.netloc - query_str, params = parse_result.query, dict() + query_str, params = parse_result.query, {} if proto == '' or netloc == '': - raise NotValidCecUrlException(url) + raise CecNotValidCecUrlException(url) for param in query_str.split('&'): if param.strip() == '': continue param_split = param.split('=') if len(param_split) != 2: - raise NotValidCecUrlException( + raise CecNotValidCecUrlException( f"params error: {param}, url: {url}") params[param_split[0]] = param_split[1] return CecUrl(proto, netloc, params) - - -class NotValidCecUrlException(Exception): - pass diff --git a/sysom_api/sdk/cec_redis/__init__.py b/sysom_api/sdk/cec_redis/__init__.py index 4575954f..3a43939e 100644 --- a/sysom_api/sdk/cec_redis/__init__.py +++ b/sysom_api/sdk/cec_redis/__init__.py @@ -1,9 +1,8 @@ # -*- coding: utf-8 -*- # """ -Time 2022/7/25 14:47 +Time 2022/9/26 17:01 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File __init__.py.py Description: -""" -name = "cec_redis" +""" \ No newline at end of file diff --git a/sysom_api/sdk/cec_redis/admin_static.py b/sysom_api/sdk/cec_redis/admin_static.py new file mode 100644 index 00000000..eb2a9ef2 --- /dev/null +++ b/sysom_api/sdk/cec_redis/admin_static.py @@ -0,0 +1,782 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/9/26 21:13 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File admin_static.py +Description: +""" +import sys +from typing import Optional, List +from itertools import chain +from redis import Redis +from loguru import logger +import redis.exceptions +from ..cec_base.exceptions import TopicNotExistsException, \ + TopicAlreadyExistsException, ConsumerGroupNotExistsException, \ + ConsumerGroupAlreadyExistsException +from ..cec_base.meta import TopicMeta, PartitionMeta, ConsumerGroupMeta, \ + ConsumerGroupMemberMeta +from ..cec_base.exceptions import CecException +from ..cec_base.log import LoggerHelper +from .utils import raise_if_not_ignore +from .consume_status_storage import ConsumeStatusStorage +from .common import StaticConst + + +#################################################################### +# Static function implementation of the management interface +#################################################################### + +@logger.catch(reraise=True) +def static_create_topic(redis_client, topic_name: str = "", + num_partitions: int = 1, + replication_factor: int = 1, **kwargs) -> bool: + """A static method to create one topic + + Args: + redis_client(Redis): Redis client + topic_name(str): Topic name + num_partitions(int): Number of partitions + replication_factor(int): Number of replications + + Keyword Args: + ignore_exception(bool): Whether ignore exception + expire_time(int): Event expire time + + Returns: + bool: True if create successful else failed + + """ + ignore_exception = kwargs.get("ignore_exception", False) + expire_time = kwargs.get("expire_time", 24 * 60 * 60 * 1000) + LoggerHelper.get_lazy_logger().debug( + f"{redis_client} try to create_topic .") + + # The Key of the Stream that internally characterizes Topic, spliced with + # a special prefix as a namespace + inner_topic_name = StaticConst.get_inner_topic_name( + topic_name) + result = True + try: + if not _lock_topic(redis_client, topic_name, + ignore_exception): + return False + # 1. Determine whether Topic exists + if static_is_topic_exist(redis_client, + topic_name): + raise TopicAlreadyExistsException( + f"Topic {topic_name} already " + f"exists." + ) + # 2. Use xadd to trigger stream creation + event_id = redis_client.xadd(inner_topic_name, { + "test": 1 + }) + + pipeline = redis_client.pipeline() + # 3. Delete the test event just added and empty the stream. + pipeline.xdel(inner_topic_name, event_id) + + # 4. add the new Topic to the Topic collection (for easy access to + # the list of all Topics) + pipeline.sadd(StaticConst.REDIS_ADMIN_TOPIC_LIST_SET, + inner_topic_name) + pipeline.execute() + except redis.exceptions.RedisError as exc: + raise_if_not_ignore(ignore_exception, exc) + finally: + _unlock_topic(redis_client, topic_name) + + LoggerHelper.get_lazy_logger().success( + f"{redis_client} create_topic '{topic_name}' successfully.") + return result + + +@logger.catch(reraise=True) +def static_del_topic(redis_client: Redis, topic_name: str, **kwargs) -> bool: + """A static method to delete one topic + + Args: + redis_client(Redis): Redis client + topic_name(str): Topic name + + Keyword Args: + ignore_exception(bool): Whether ignore exception + + Returns: + bool: True if delete successful else failed + + """ + ignore_exception = kwargs.get("ignore_exception", False) + LoggerHelper.get_lazy_logger().debug( + f"{redis_client} try to del_topic .") + + inner_topic_name = StaticConst.get_inner_topic_name( + topic_name) + + try: + if not _lock_topic(redis_client, topic_name, ignore_exception): + return False + # 1. Determine whether topic exists. + if not static_is_topic_exist(redis_client, + topic_name): + raise_if_not_ignore(ignore_exception, + TopicNotExistsException( + f"Topic {topic_name} not exists." + )) + pipeline = redis_client.pipeline() + + # 2. Delete the corresponding stream (topic) + pipeline.delete(inner_topic_name) + + # 3. Remove the current topic from the topic list + pipeline.srem(StaticConst.REDIS_ADMIN_TOPIC_LIST_SET, + inner_topic_name) + + pipeline.execute() + + # 4. Clear topic-related metadata information + del_topic_meta(redis_client, topic_name) + + # 5. Delete the structure associated with topic for storing consumption + # status + ConsumeStatusStorage.destroy_by_stream(redis_client, topic_name) + except redis.exceptions.RedisError as exc: + raise_if_not_ignore(ignore_exception, exc) + except CecException as exc: + raise_if_not_ignore(ignore_exception, exc) + finally: + _unlock_topic(redis_client, topic_name) + + LoggerHelper.get_lazy_logger().success( + f"{redis_client} del_topic '{topic_name}' successfully.") + return True + + +@logger.catch(reraise=True) +def static_is_topic_exist(redis_client: Redis, topic_name: str, + **kwargs) -> bool: + """A static method to determine whether specific topic exists + + Args: + redis_client(Redis): Redis client + topic_name(str): Topic name + **kwargs: + + Returns: + + """ + ignore_exception = kwargs.get("ignore_exception", False) + res = False + try: + res = redis_client.type( + StaticConst.get_inner_topic_name(topic_name)) == 'stream' + LoggerHelper.get_lazy_logger().debug( + f"Is topic {topic_name} exists? => {res}, {kwargs}.") + except redis.exceptions.RedisError as exc: + raise_if_not_ignore(ignore_exception, exc) + return res + + +@logger.catch(reraise=True) +def static_get_topic_list(redis_client: Redis, **kwargs) -> List[TopicMeta]: + """A static method to get topic list + + Args: + redis_client(Redis): Redis client + + Keyword Args: + + Returns: + + """ + ignore_exception = kwargs.get("ignore_exception", False) + topics = [] + try: + res = redis_client.smembers(StaticConst.REDIS_ADMIN_TOPIC_LIST_SET) + for inner_topic_name in res: + topic_meta = TopicMeta( + StaticConst.get_topic_name_by_inner_topic_name( + inner_topic_name)) + topic_meta.partitions = { + 0: PartitionMeta(0) + } + topics.append(topic_meta) + LoggerHelper.get_lazy_logger().debug( + f"get_topic_list => {res}.") + except redis.exceptions.RedisError as exc: + raise_if_not_ignore(ignore_exception, exc) + return topics + + +@logger.catch(reraise=True) +def static_create_consumer_group(redis_client: Redis, + consumer_group_id: str, + **kwargs) -> bool: + """A static method to create a consumer group + + Args: + redis_client(Redis): Redis client + consumer_group_id(str): Consumer group ID + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown + + Returns: + bool: True if successful, False otherwise. + """ + ignore_exception = kwargs.get("ignore_exception", False) + LoggerHelper.get_lazy_logger().debug( + f"{redis_client} try to create_consumer_group " + f".") + + try: + if not _lock_consumer_group(redis_client, + consumer_group_id, + ignore_exception): + return False + if static_is_consumer_group_exist(redis_client, + consumer_group_id): + if ignore_exception: + return False + raise ConsumerGroupAlreadyExistsException( + f"Consumer group {consumer_group_id} already exists.") + # Add to the consumer group key collection + redis_client.sadd( + StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LIST_SET, + consumer_group_id) + except redis.exceptions.RedisError as exc: + raise_if_not_ignore(ignore_exception, exc) + except CecException as exc: + raise_if_not_ignore(ignore_exception, exc) + finally: + _unlock_consumer_group(redis_client, + consumer_group_id) + LoggerHelper.get_lazy_logger().debug( + f"{redis_client} create_consumer_group " + f"'{consumer_group_id}' successfully.") + return True + + +@logger.catch(reraise=True) +def static_del_consumer_group(redis_client: Redis, + consumer_group_id: str, + **kwargs) -> bool: + """A static method to delete a consumer group + + Args: + redis_client(Redis): Redis client + consumer_group_id(str): Consumer group ID + + Keyword Args: + ignore_exception: 是否忽略可能会抛出的异常 + + Returns: + + """ + ignore_exception = kwargs.get("ignore_exception", False) + LoggerHelper.get_lazy_logger().debug( + f"{redis_client} try to del_consumer_group " + f".") + + try: + if not _lock_consumer_group( + redis_client, consumer_group_id, ignore_exception + ): + return False + + # 1. First determine if the consumer group exists, and if not, throw + # an exception as appropriate. + if not static_is_consumer_group_exist(redis_client, + consumer_group_id): + raise_if_not_ignore(ignore_exception, + ConsumerGroupNotExistsException( + f"Consumer group {consumer_group_id} " + f"not exists." + )) + + # 2. Removal from the set of consumer groups. + redis_client.srem( + StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LIST_SET, + consumer_group_id) + + # 3. Destroy all consumer group structures of the same name in all + # streams associated with the current consumption group. + streams = redis_client.lpop( + get_sub_list_key(consumer_group_id), + sys.maxsize + ) + if streams is None: + streams = [] + pipeline = redis_client.pipeline() + for stream in streams: + # Unsubscribe from topics + pipeline.xgroup_destroy(stream, consumer_group_id) + + # Delete the corresponding zset + ConsumeStatusStorage.destroy_by_stream_group(pipeline, stream, + consumer_group_id) + pipeline.execute() + for stream in streams: + # Clear metadata information related to topic-consumer groups + del_topic_consumer_group_meta(redis_client, stream, + consumer_group_id) + except ConsumerGroupNotExistsException as exc: + raise_if_not_ignore(ignore_exception, exc) + except redis.exceptions.RedisError: + # Ignore errors that may be generated by Pipeline performing cleanup + # operations here + pass + finally: + _unlock_consumer_group(redis_client, consumer_group_id) + + LoggerHelper.get_lazy_logger().debug( + f"{redis_client} del_consumer_group " + f"'{consumer_group_id}' successfully.") + return True + + +@logger.catch(reraise=True) +def static_is_consumer_group_exist(redis_client: Redis, + consumer_group_id: str, + **kwargs) -> bool: + """A static method to determine whether the specific consumer group exists + + Args: + redis_client(Redis): Redis client + consumer_group_id(str): Consumer group ID + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown + + Returns: + [ConsumerGroupMeta]: The consumer group list + """ + ignore_exception = kwargs.get("ignore_exception", False) + res = False + try: + res = redis_client.sismember( + StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LIST_SET, + consumer_group_id) + + LoggerHelper.get_lazy_logger().debug( + f"{redis_client} Is consumer group '{consumer_group_id}' " + f"exists => {res}") + except redis.exceptions.RedisError as exc: + raise_if_not_ignore(ignore_exception, exc) + return res + + +@logger.catch(reraise=True) +def static_get_consumer_group_list(redis_client: Redis, **kwargs) \ + -> List[ConsumerGroupMeta]: + """A static method to get consumer group list + + Args: + redis_client(Redis): Redis client + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown + + Returns: + + """ + ignore_exception = kwargs.get("ignore_exception", True) + res = redis_client.smembers( + StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LIST_SET) + group_metas = [] + for group_id in res: + group_meta = ConsumerGroupMeta(group_id) + try: + # Get information about all subscribed topics for this consumer + # group + sub_topics = redis_client.lrange( + get_sub_list_key(group_id), 0, -1 + ) + + # Iterate through all topics to get all members + pipeline = redis_client.pipeline(transaction=True) + for topic in sub_topics: + pipeline.xinfo_consumers(topic, group_id) + + # {"name":"Alice","pending":1,"idle":9104628} + for consumer in chain.from_iterable(pipeline.execute()): + group_meta.members.append( + ConsumerGroupMemberMeta(consumer['name'])) + except redis.exceptions.RedisError as exc: + raise_if_not_ignore(ignore_exception, exc) + group_meta.error = exc + except CecException as exc: + raise_if_not_ignore(ignore_exception, exc) + group_meta.error = exc + else: + group_metas.append(group_meta) + + LoggerHelper.get_lazy_logger().debug( + f"get_consumer_group_list => {res}.") + return group_metas + + +def static_del_consumer(redis_client: Redis, topic: str, group: str, + consumer: str): + """A static method to remove consumer from consumer group + + Args: + redis_client(Redis): + topic(str): + group(str): + consumer(str): + + Returns: + + """ + return redis_client.xgroup_delconsumer(topic, group, consumer) == 1 + + +def _lock_consumer_group(redis_client: Redis, consumer_group_id: str, + ignore_exception: bool = False) -> bool: + """Lock specific consumer group + + Lock a consumer group to prevent repeated operation problems in concurrent + scenarios. + + Args: + redis_client: + consumer_group_id: + ignore_exception: + + Returns: + + """ + if redis_client.set( + f"{StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LOCKER_PREFIX}" + f"{consumer_group_id}", + consumer_group_id, nx=True, ex=10) == 0: + return raise_if_not_ignore(ignore_exception, + CecException( + "Someone else is creating or" + " deleting this consumer group." + )) + return True + + +def _unlock_consumer_group(redis_client: Redis, + consumer_group_id: str) -> bool: + """Unlock specific consumer group + + Releasing a lock placed on a consumer group should be used in conjunction + with lock_consumer_group + + Args: + redis_client: + consumer_group_id: + + Returns: + + """ + if redis_client.get( + f"{StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LOCKER_PREFIX}" + f"{consumer_group_id}" + ) == consumer_group_id: + return redis_client.delete( + f"{StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LOCKER_PREFIX}" + f"{consumer_group_id}") == 1 + return False + + +def _lock_topic(redis_client: Redis, topic: str, + ignore_exception: bool = False) -> bool: + """Lock specific topic + + Lock a topic to prevent repeated operation problems in concurrent scenes. + + Args: + redis_client: + topic: + ignore_exception: + + Returns: + + """ + if redis_client.set( + f"{StaticConst.REDIS_ADMIN_TOPIC_LOCKER_PREFIX}{topic}", + topic, nx=True, ex=10) == 0: + return raise_if_not_ignore(ignore_exception, + CecException( + "Someone else is creating or deleting " + "this topic." + )) + return True + + +def _unlock_topic(redis_client: Redis, topic: str) -> bool: + """Unlock specific topic + + Releasing a lock placed on a topic should be used in conjunction with + lock_topic + + Args: + redis_client: + topic: + + Returns: + + """ + # 释放锁 + if redis_client.get( + f"{StaticConst.REDIS_ADMIN_TOPIC_LOCKER_PREFIX}{topic}" + ) == topic: + return redis_client.delete( + f"{StaticConst.REDIS_ADMIN_TOPIC_LOCKER_PREFIX}{topic}") == 1 + return False + + +#################################################################### +# 一些辅助函数 +#################################################################### + +def get_topic_consumer_group_meta_info_key( + topic: Optional[str], group_id: Optional[str], key: Optional[str] +): + """Get topic-group meta info + + Get meta info key + + Args: + topic: + group_id: + key: + + Returns: + + """ + return f"{StaticConst.REDIS_ADMIN_TOPIC_CONSUMER_GROUP_META_PREFIX}" \ + f"{topic + ':' if topic is not None else ''}" \ + f"{group_id + ':' if group_id is not None else ''}" \ + f"{key + ':' if key is not None else ''}" + + +def get_topic_meta_info_key(topic: str, key: Optional[str]): + """Get topic meta info + + Get meta info key + + Args: + topic: + key: + + Returns: + + """ + return f"{StaticConst.REDIS_ADMIN_TOPIC_META_PREFIX}" \ + f"{topic + ':' if topic is not None else ''}" \ + f"{key + ':' if key is not None else ''}" + + +def get_sub_list_key(group_id: str) -> str: + """Get sub list + + Get sub list key => Each topic is associated with a sub list containing the + IDs of all consumer groups that are subscribed to the topic + + Args: + group_id: + + Returns: + + """ + return f"{StaticConst.REDIS_ADMIN_CONSUMER_GROUP_SUB_LIST_PREFIX}" \ + f"{group_id}" + + +def store_meta(redis_client: Redis, key: str, value: str): + """Store meta info + + Store metadata + + Args: + redis_client: + key: + value: + + Returns: + + """ + return redis_client.set(key, value) + + +def get_meta(redis_client: Redis, key: str): + """Get meta info + + Get metadata + + Args: + redis_client: + key: + + Returns: + + """ + return redis_client.get(key) + + +def del_meta(redis_client: Redis, prefix: str): + """Delete meta info + + Delete metadata + + Args: + redis_client: + prefix: + + Returns: + + """ + next_cursor = 0 + while True: + next_cursor, key_list = redis_client.scan( + next_cursor, + match=f"{prefix}*", + count=100 + ) + if len(key_list) > 0: + redis_client.delete(*key_list) + if next_cursor == 0: + break + return True + + +def store_topic_consumer_group_meta(redis_client: Redis, topic: str, + key: str, group_id: str, value): + """Store topic-group meta info + + Store metadata + + Args: + redis_client: + topic: + key: + group_id: + value: + + Returns: + + """ + return store_meta( + redis_client, + get_topic_consumer_group_meta_info_key( + topic, group_id, key + ), + value + ) + + +def store_topic_meta(redis_client: Redis, topic: str, key: str, value): + """Store topic meta info + + Store metadata + + Args: + redis_client: + topic: + key: + value: + + Returns: + + """ + return store_meta( + redis_client, + get_topic_meta_info_key(topic, key), + value + ) + + +def get_topic_consumer_group_meta(redis_client: Redis, topic: str, + group_id: str, key: str): + """Get topic-group meta info + + Get metadata + + Args: + redis_client: + topic: + group_id: + key: + + Returns: + + """ + return get_meta( + redis_client, + get_topic_consumer_group_meta_info_key( + topic, group_id, key + ) + ) + + +def get_topic_meta(redis_client: Redis, topic: str, key: str): + """Get topic meta info + + Get topic-related metadata information + + Args: + redis_client: + topic: + key: + + Returns: + + """ + return get_meta( + redis_client, + get_topic_meta_info_key(topic, key) + ) + + +def del_topic_consumer_group_meta(redis_client: Redis, + topic: str, group_id: str): + """Delete topic-group meta info + + Delete metadata information + + Args: + redis_client: + topic: + group_id: + + Returns: + + """ + return del_meta( + redis_client, + get_topic_consumer_group_meta_info_key( + topic, group_id, None + ) + ) + + +def del_topic_meta(redis_client: Redis, topic: str): + """Delete all meta info for specific topic + + Delete all metadata information for a specific topic + + Args: + redis_client: + topic: + + Returns: + + """ + res1 = del_meta( + redis_client, + get_topic_consumer_group_meta_info_key(topic, None, None) + ) + res2 = del_meta( + redis_client, get_topic_meta_info_key(topic, None) + ) + return res1 and res2 diff --git a/sysom_api/sdk/cec_redis/common.py b/sysom_api/sdk/cec_redis/common.py index 298f1f23..02ea037c 100644 --- a/sysom_api/sdk/cec_redis/common.py +++ b/sysom_api/sdk/cec_redis/common.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # """ -Time 2022/8/31 23:38 +Time 2022/7/29 13:33 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File common.py @@ -12,86 +12,147 @@ from ..cec_base.url import CecUrl class StaticConst: + """Static consts + + This class defines all the static constant values in the cec-redis module + """ CEC_REDIS_PREFIX = "CEC-REDIS:" REDIS_CEC_EVENT_VALUE_KEY = "redis-cec-event-value-key" _REDIS_ADMIN_META_PREFIX = f"{CEC_REDIS_PREFIX}META:" - # 指示一个集合 => 保存了所有的 Stream 的key + # Indicates a collection => holds the keys of all Streams REDIS_ADMIN_TOPIC_LIST_SET = f"{_REDIS_ADMIN_META_PREFIX}" \ f"TOPIC-LIST-SET" - # 指示一个集合 => 保存了所有的 Consumer Group 的key + # Indicates a collection => holds the keys of all Consumer Groups REDIS_ADMIN_CONSUMER_GROUP_LIST_SET \ = f"{_REDIS_ADMIN_META_PREFIX}" \ f"CONSUMER-GROUP-LIST-SET" - # 消费组订阅列表前缀 => 消费组的订阅列表里面存储了该消费组订阅的所有主题 + # Consumer group subscription list prefix => The consumer group's + # subscription list stores all topics to which the consumer group is + # subscribed REDIS_ADMIN_CONSUMER_GROUP_SUB_LIST_PREFIX \ = f"{_REDIS_ADMIN_META_PREFIX}" \ f"SUB-LIST-PREFIX:" - # 指定一个所有的 STREAM key 共用的前缀，方便获取 stream 列表 + # Specify a prefix that is common to all STREAM keys to make it easier to + # get the stream list REDIS_ADMIN_STREAM_KEY_PREFIX \ = f"{CEC_REDIS_PREFIX}" \ f"STREAM-PREFIX:" - # 主题元数据信息前缀 + # Topic metadata information prefix REDIS_ADMIN_TOPIC_META_PREFIX \ = f"{_REDIS_ADMIN_META_PREFIX}TOPIC-META:" - # 主题—消费组元数据信息前缀 + # Topic-Consumer Group Metadata Information Prefix REDIS_ADMIN_TOPIC_CONSUMER_GROUP_META_PREFIX \ = f"{_REDIS_ADMIN_META_PREFIX}TOPIC-CONSUMER-GROUP-META:" - # 主题-消费者元数据 key + # Last ack ID key TOPIC_CONSUMER_GROUP_META_KEY_LAST_ACK_ID \ - = f"LAST-ACK-ID" + = "LAST-ACK-ID" - # 主题锁前缀 + # Topic lock prefix REDIS_ADMIN_TOPIC_LOCKER_PREFIX \ = f"{_REDIS_ADMIN_META_PREFIX}TOPIC-LOCKER:" - # 消费组锁前缀 + # Consumer group lock prefix REDIS_ADMIN_CONSUMER_GROUP_LOCKER_PREFIX \ = f"{_REDIS_ADMIN_META_PREFIX}CONSUMER-GROUP-LOCKER:" - # 特化参数列表： + # Heartbeat monitoring related configurations + REDIS_HEARTBEAT_CHANNEL_PREFIX = f"{CEC_REDIS_PREFIX}HEARTBEAT:" + + # Heartbeat locker prefix + REDIS_HEARTBEAT_LOCKER_PREFIX \ + = f"{CEC_REDIS_PREFIX}HEARTBEAT-LOCKER:" + + # List of specialization parameters REDIS_SPECIAL_PARM_CEC_DEFAULT_MAX_LEN = 'cec_default_max_len' REDIS_SPECIAL_PARM_CEC_AUTO_MK_TOPIC = 'cec_auto_mk_topic' + REDIS_SPECIAL_PARM_CEC_ENABLE_PENDING_LIST_TRANSFER = \ + "cec_enable_pending_list_transfer" REDIS_SPECIAL_PARM_CEC_PENDING_EXPIRE_TIME = 'cec_pending_expire_time' + REDIS_SPECIAL_PARM_CEC_ENABLE_HEART_BEAT = 'cec_enable_heartbeat' + REDIS_SPECIAL_PARM_CEC_HEARTBEAT_INTERVAL = 'cec_heartbeat_interval' + REDIS_SPECIAL_PARM_CEC_HEARTBEAT_CHECK_INTERVAL = \ + 'cec_heartbeat_check_interval' _redis_special_parameter_list = [ - # cec_default_max_len => 默认最大队列长度限制 - # 1. 有效范围：Producer - # 2. 含义：该参数指定了 Producer 将事件投递到某个具体的 Stream 中，期望该 Stream - # 最大保持的队列长度，由于 Redis stream 底层使用树形结构，精确裁剪会很影响 - # 性能，所以该参数限制的是一个大致长度，实际队列可能会稍大于该值 + # cec_default_max_len => Default maximum queue length limit + # 1. Effective range：[Producer] + # 2. Meaning: This parameter specifies the maximum queue length that + # the Producer expects the Stream to hold when it + # delivers events to a specific Stream. REDIS_SPECIAL_PARM_CEC_DEFAULT_MAX_LEN, - # cec_auto_mk_topic => 自动创建主题 - # 1. 有效范围：Consumer - # 2. 含义：该参数指定了 Producer 在投递消息到某个 Topic 时，如果 Topic 不存在 - # 是否需要自动创建 Topic。 + # cec_auto_mk_topic => Automatic topic creation + # 1. Effective range：[Consumer] + # 2. Meaning: This parameter specifies whether the Producer needs to + # create a Topic automatically if it does not exist when it + # delivers a message to a Topic. REDIS_SPECIAL_PARM_CEC_AUTO_MK_TOPIC, - # cec_pending_expire_time =>超期转换时间间隔 - # 1. 有效范围：Consumer - # 2. 含义：该参数指定了一个事件再待确认列表（pending list）中长时间未确认被自动流 - # 转到组内其它消费者的的时间间隔，每个消费者在每批次消费时都会尝试将 pending - # list 中的超期事件流转给自己。 + # cec_enable_pending_list_transfer => Whether enable pending list + # transfer + # 1. Effective range: [Consumer] + # 2. Meaning: This parameter specifies whether enable pending list + # transfer mechanisms, if enabled, the consumer will try to + # transfer long unacknowledged messages from the same group's + # pending list to itself for processing at each consumption. + REDIS_SPECIAL_PARM_CEC_ENABLE_PENDING_LIST_TRANSFER, + # cec_pending_expire_time => expire conversion interval + # 0. Require enable 'cec_enable_pending_list_transfer' + # 1. Effective range：[Consumer(broadcast mode)] + # 2. Meaning: This parameter specifies the time interval after which + # an event in the pending list that has been unacknowledged for a + # long time will be automatically streamed to other consumers in + # the group, and each consumer will try to stream the overdue + # events in the pending list to itself at each batch. REDIS_SPECIAL_PARM_CEC_PENDING_EXPIRE_TIME, + # cec_enable_heartbeat => Enable heartbeat mechanisms + # 1. Effective range: [Consumer(group consume mode)] + # 2. Meaning: This parameter specifies whether enable heartbeat + # mechanisms, if enabled, the current consumer periodically sends + # heartbeat data to the channel shared by the consumer group, + # subscribes to and monitors the health of other consumers in the + # group, and tries to transfer its unacknowledged messages to + # itself for processing when it detects an offline consumer. + REDIS_SPECIAL_PARM_CEC_ENABLE_HEART_BEAT, + # cec_heartbeat_interval => Automatic switching of inspection + # intervals + # 0. Require enable 'cec_enable_heartbeat' + # 1. Effective range：[Consumer(group consume mode)] + # 2. Meaning: This parameter specifies the heartbeat interval. + REDIS_SPECIAL_PARM_CEC_HEARTBEAT_INTERVAL, + # cec_heartbeat_check_interval => Heartbeat check interval + # 0. Require enable 'cec_enable_heartbeat' + # 1. Effective range：[Consumer(group consume mode)] + # 2. Meaning: This parameter specifies the heartbeat checkout + # interval. + REDIS_SPECIAL_PARM_CEC_HEARTBEAT_CHECK_INTERVAL, ] + _redis_special_parameters_default_value = { REDIS_SPECIAL_PARM_CEC_DEFAULT_MAX_LEN: (int, 1000), REDIS_SPECIAL_PARM_CEC_AUTO_MK_TOPIC: (bool, False), - REDIS_SPECIAL_PARM_CEC_PENDING_EXPIRE_TIME: (int, 5 * 60 * 1000) + REDIS_SPECIAL_PARM_CEC_ENABLE_PENDING_LIST_TRANSFER: (bool, False), + REDIS_SPECIAL_PARM_CEC_PENDING_EXPIRE_TIME: (int, 5 * 60 * 1000), + REDIS_SPECIAL_PARM_CEC_ENABLE_HEART_BEAT: (bool, True), + REDIS_SPECIAL_PARM_CEC_HEARTBEAT_INTERVAL: (int, 5), + REDIS_SPECIAL_PARM_CEC_HEARTBEAT_CHECK_INTERVAL: (int, 3) } @staticmethod def parse_special_parameter(params: dict) -> dict: - """ - 解析特化参数，并将特化参数从参数列表中移除 + """Parse specialization parameters + + Parse the specialization parameters and remove the specialization + parameters from the parameter list + Args: - params: + params(dict): CecUrl.params Returns: @@ -107,13 +168,14 @@ class StaticConst: def get_inner_topic_name(topic_name) -> str: """Get inner topic name by topic name - 通过 topic_name 获取对应的 inner_topic_name - 1. 事件主题在 Redis 中对应一个 STREAM； - 2. 本模块为所有由 cec-redis 创建的 STREAM 添加一个公共前缀作为命名空间； + Get the corresponding inner_topic_name by topic_name + 1. event topic corresponds to a STREAM in Redis.； + 2. This module adds a common prefix as a namespace to all STREAMs + created by cec-redis； 3. inner_topic_name = ALI-CEC-REDIS-STREAM-PREFIX:{topic_name} Args: - topic_name: 主题名称 + topic_name(str): Topic name Returns: """ @@ -121,10 +183,12 @@ class StaticConst: @staticmethod def get_topic_name_by_inner_topic_name(inner_topic_name: str) -> str: - """ - 将 inner_topic_name => topic_name + """Get topic name by inner topic name + + inner_topic_name => topic_name + Args: - inner_topic_name: + inner_topic_name(str): inner topic name Returns: @@ -135,7 +199,8 @@ class StaticConst: class ClientBase: """ - cec 客户端基类，Redis* 需要集成本类，本类提供一些通用的实现 + cec-redis client base class, Redis* requires inherit from this class, + which provides some generic implementation """ def __init__(self, url: CecUrl): @@ -143,11 +208,23 @@ class ClientBase: self._special_params = StaticConst.parse_special_parameter(url.params) def get_special_param(self, key: str, default=''): + """Get specialization parameter by key + + Args: + key(str): specialization parameter key + default(Any): default value if key not exists + + Returns: + + """ return self._special_params.get(key, default) def is_gte_6_2(self, redis_client: Redis): - """ - 判断redis版本是否 >= 6.2 + """Determine if redis version >= 6.2 + + Args: + redis_client(Redis): Redis client + Returns: """ @@ -156,10 +233,10 @@ class ClientBase: return self._redis_version >= '6.2' def is_gte_7(self, redis_client: Redis): - """ - 判断redis版本是否 >= 7 + """Determine if redis version >= 7 + Args: - redis_client: + redis_client(Redis): Redis client Returns: diff --git a/sysom_api/sdk/cec_redis/consume_status_storage.py b/sysom_api/sdk/cec_redis/consume_status_storage.py index 75964734..aeaf0f01 100644 --- a/sysom_api/sdk/cec_redis/consume_status_storage.py +++ b/sysom_api/sdk/cec_redis/consume_status_storage.py @@ -1,62 +1,75 @@ # -*- coding: utf-8 -*- # """ -Time 2022/8/31 17:24 +Time 2022/8/3 17:24 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File consume_status_storage.py Description: """ +from typing import Union from redis import Redis from redis.client import Pipeline from ..cec_base.event import Event from .common import StaticConst -from typing import Union class ConsumeStatusStorage: """A consumer group consume status storage - 一个存储消费组消费状态的存储器 - 1. 设计该存储器主要是为了计算 LAG（消息堆积数）； - 2. Redis 7.0 支持使用 xinfo group 得到： - - 'lag' => stream 中任然等待被交付给指定消费组中消费者的消息数量（包括已交付未确认的） - - 'pending' => stream 中已经交付给指定消费组中消费者但是未确认的消息数量 - - 'lag' + 'pending' => 即可得到主题中还未被消费者消费或者确认的消息数 - 3. Redis 版本 < 7 没有办法获取上述数据，因此需要使用本类来兼容实现 - - 兼容的思路如下： - 1. 使用 Redis 提供的 zsets（有序列表）数据结构，为每个维护一个 zsets - 将 ID 转换成分数存储； - 2. 每次 RedisConsumer 使用 consume 获取消息时，调用 xinfo stream 获得： - - 'length' => 队列当前的长度 - - 'fist-entry' => 取出队列中最早的消息的 ID - 3. 然后调用 ZREMRANGEBYSCORE 删除掉所有比最早的消息 ID 还小的消息；（防止过度堆积） - 4. 每个消息 ACK 的时候，将对应的 ID 插入到对应的 zsets 当中； - - 5. 接着本类将提供下列静态方法： - - get_lag(topic, group) => 得到主题的 LAG 值 + A storage to store the consumption status of the consumption group + + 1. This class is designed primarily to calculate the LAG; + 2. Redis 7.0 supports using the xinfo group to get LAG: + - 'lag' => Number of messages in stream that are still waiting to be + delivered to consumers in the specified consumer group + (including delivered unacknowledged ones); + - 'pending' => The number of messages in stream that have been + delivered to consumers in the specified consumer group + but not acknowledged; + - 'lag' + 'pending' => That gives the number of messages in the topic + that have not yet been consumed or confirmed by + the consumer. + 3. Redis version < 7 does not have a way to fetch the above data, so you + need to use this class for a compatible implementation + + The idea of compatibility is as follows. + 1. Maintain a zsets for each to store IDs as scores using + the zsets (ordered list) data structure provided by Redis. + 2. Each time a RedisConsumer uses consume to get a message, call xinfo + stream to get: + - 'length' => Current length of the queue + - 'fist-entry' => Fetch the ID of the earliest message in the queue + 3. Then call ZREMRANGEBYSCORE to remove all messages smaller than the + earliest message ID; (to prevent excessive stacking) + 4. When each message is acknowledged, the corresponding ID is inserted into + the corresponding zsets; + 5. Then this class will provide the following static methods: + - get_lag(topic, group) => Get the LAG value of the subject """ _CEC_REDIS_CONSUME_STATUS_STORAGE_PREFIX = \ f"{StaticConst.CEC_REDIS_PREFIX}CONSUME_STATUS_STORAGE:" - max_float = float("inf") # 无限大 比所有数大 - min_float = float("-inf") # 无限小 比所有数小 + max_float = float("inf") # Infinity. Bigger than all the numbers. + min_float = float("-inf") # Infinitely small. Smaller than all numbers def __init__(self, _redis_client: Redis, stream: str, group_id: str): self._redis_client = _redis_client - # 得到 Redis 服务器的版本号 + # Get the version number of the Redis server self._version = self._redis_client.info('server')['redis_version'] - # 判断是 Redis 版本是否大于 7 + # Determine if the Redis version is greater than 7 self._is_gt_version_7 = self._version >= '7' self.stream = stream self.inner_stream_key = StaticConst.get_inner_topic_name(stream) self.group_id = group_id def update(self): - """ - 使用 xinfo stream 得到 stream 里面最早的消息的 ID，并据此删除对应 zsets 中的数据 + """Update inner zsets + + Use the xinfo stream to get the ID of the oldest message in the stream + and delete the data in the corresponding zsets accordingly + Returns: """ @@ -77,10 +90,13 @@ class ConsumeStatusStorage: def do_after_ack_by_pl(self, pipeline: Pipeline, event: Event): """ - 在某个消息被 ACK 后，执行本方法，将其 ID 存储到 zset 当中 + + After a message is acknowledged, execute this method to store its ID + in zset + Args: - pipeline: - event: + pipeline(Pipeline): Redis pipeline + event(Event): CEC event References: https://redis.io/commands/zadd/ @@ -98,17 +114,21 @@ class ConsumeStatusStorage: event.event_id) }, ) + return True @staticmethod def get_already_ack_count(redis_client: Union[Redis, Pipeline], stream: str, group_id: str, ): """ - 得到指定 目前已经确认的消息数量，可以用来计算 LAG + + Gets the number of messages currently acknowledged for a given + , which can be used to calculate the LAG + Args: - redis_client: - stream: - group_id: + redis_client(Redis): Redis client + stream(str): Topic / stream + group_id(str): Consumer group ID References https://redis.io/commands/zcount/ @@ -127,11 +147,14 @@ class ConsumeStatusStorage: stream: str, group_id: str): """ - 删除 对应的 zset => 通常在某个消费组离开 stream 时调用 + + Delete the zset => corresponding to usually called when + a consumer group leaves the stream + Args: - redis_or_pl: - stream: - group_id: + redis_or_pl(Redis | Pipeline): Redis client or Redis pipeline + stream(str): Topic / stream + group_id(str): Consumer group ID Returns: @@ -142,17 +165,21 @@ class ConsumeStatusStorage: @staticmethod def destroy_by_stream(redis_client: Redis, stream: str): """ - 删除 对应的所有 zset => 通常在某个 stream 被删除时调用 + + Delete all zset corresponding to => usually called when a + stream is deleted + Args: - redis_or_pl: - stream: + redis_client(Redis): Redis client + stream(str): Topic / stream Returns: """ keys = redis_client.keys( - f"{ConsumeStatusStorage._CEC_REDIS_CONSUME_STATUS_STORAGE_PREFIX}" \ - f"{stream}:*") + f"{ConsumeStatusStorage._CEC_REDIS_CONSUME_STATUS_STORAGE_PREFIX}" + f"{stream}:*" + ) if len(keys) > 0: return redis_client.delete(*keys) return 0 @@ -160,7 +187,9 @@ class ConsumeStatusStorage: @staticmethod def _get_score_by_id(message_id: str): """ - 根据 Redis 自动生成的 ID 转换成浮点数：'1526985054069-0' => 1526985054069.0 + + Convert to floating point based on Redis auto-generated ID: + '1526985054069-0' => 1526985054069.0 Args: message_id: @@ -173,7 +202,10 @@ class ConsumeStatusStorage: @staticmethod def _get_z_set_key(stream: str, group_id: str): """ - 获取对应 用于存储 ID 的 zset 的 key + + Get the key of the zset corresponding to the used to + store the ID + Args: stream: group_id: diff --git a/sysom_api/sdk/cec_redis/heartbeat.py b/sysom_api/sdk/cec_redis/heartbeat.py new file mode 100644 index 00000000..8c438435 --- /dev/null +++ b/sysom_api/sdk/cec_redis/heartbeat.py @@ -0,0 +1,269 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/9/22 15:24 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File heartbeat.py +Description: +""" +import threading +from threading import Thread, Event as ThreadEvent +from typing import Optional +from collections import deque + +import redis +from redis import Redis +from redis.client import PubSub +from atomic import AtomicLong +from schedule import Scheduler +from ..cec_base.exceptions import CecException +from ..cec_base.log import LoggerHelper +from .common import StaticConst +from .admin_static import static_del_consumer + + +class Heartbeat: + """A daemon thread/process use to send and listen the heartbeat of consumer + + A daemon thread/process to generate and listen to the consumer's heartbeat + 1. First, the CEC creates a pub/sub channel for each consumer group. + 2. Then, when each member of the consumer group accesses the CEC, a + Heartbeat instance is initiated that periodically sends heartbeat + data to the "heartbeat channel" of the corresponding consumer group. + 3. Consumers also subscribe to the heartbeat channel of the consumer + group and monitor the heartbeat data of other members of the group + in real time. + 4. If the current consumer detects that a group member has not sent a + heartbeat for a long time, it assumes that the member is dead and + does the following. + 4. If the current consumer detects that a member of the group has not + sent heartbeat data for some time, it assumes that the member is + offline and does the following: + - If the member does not have a pending message, move it out of the + consumer group; + - If the member has a pending message, it attempts to transfer the + message to the current consumer, and moves the member out of the + consumer group after a successful transfer. + + Args: + redis_client(Redis): Redis client + topic(str): Topic + consumer_group(str): Consumer group ID + consumer_id(str): Consumer ID + + Keyword Args + heartbeat_interval(int): Heartbeat interval in seconds + heartbeat_process_mod(bool): Whether to use a separate process to run + heartbeat_check_interval(bool): Time interval to check if a consumer in + a group is online + + Attributes: + + """ + + # pylint: disable=too-many-instance-attributes + # Eleven is reasonable in this case. + def __init__(self, redis_client: Redis, topic: str, consumer_group: str, + consumer_id: str, **kwargs): + self._process_mod = kwargs.get("heartbeat_process_mod", False) + self._check_heart_beat_interval = 3 + check_interval = kwargs.get("heartbeat_check_interval", 0) + if check_interval > 0: + self._check_heart_beat_interval = check_interval + self._redis_client = redis_client + self._topic = topic + self._consumer_group = consumer_group + self._consumer_id = consumer_id + self._heartbeat_interval = kwargs.get("heartbeat_interval", 5) + self._stop_event: ThreadEvent = ThreadEvent() + self._heartbeat_listen_thread: Optional[Thread] = None + self._channel_name = f"{StaticConst.REDIS_HEARTBEAT_CHANNEL_PREFIX}" \ + f"{self._topic}:{self._consumer_group}" + self._ps: PubSub = self._redis_client.pubsub() + self._heartbeat_timeline = AtomicLong(0) + self._heartbeat_check_schedule = Scheduler() + # -> heartbeat timeline + self._consumers = { + + } + + self._offline_consumers = deque() + + def _send_heart_beat(self): + """Current send heartbeat to consumer group's heartbeat channel + + The current consumer sends a heartbeat to the heartbeat channel of the + consumer group + + Returns: + + """ + self._heartbeat_timeline += 1 + self._redis_client.publish(self._channel_name, self._consumer_id) + + def _deal_recv_heartbeat(self, message: dict): + """Deal received heartbeat + + Processing incoming heartbeat data + + Returns: + + """ + msg_type = message.get("type", "") + channel, consumer = message.get('channel', ''), message.get('data', '') + if msg_type == "message" and channel == self._channel_name and len( + consumer) > 0: + self._consumers[consumer] = self._heartbeat_timeline.value + + def _check_offline_consumer(self): + """Check if there are offline consumers + + Returns: + + """ + cur, offline_consumers = self._heartbeat_timeline.value, [] + for consumer, time in self._consumers.items(): + if time + self._check_heart_beat_interval < cur: + # Detect some consumer offline + self._offline_consumers.append(consumer) + offline_consumers.append(consumer) + for consumer in offline_consumers: + self._consumers.pop(consumer) + + def get_next_offline_consumer(self) -> Optional[str]: + """Get next offline consumer + + Returns: + + """ + if self._offline_consumers: + return self._offline_consumers[0] + return None + + def remove_consumer(self, consumer: str): + """Remove monitoring of a consumer + + Removal of monitoring of a specific consumer requires the following + conditions to be met: + 1. Consumer offline + 2. All messages inside the PEL have been transferred; + + Args: + consumer(str): Consumer ID + + Returns: + + """ + try: + self._offline_consumers.remove(consumer) + static_del_consumer(self._redis_client, self._topic, + self._consumer_group, + consumer) + except redis.exceptions.RedisError: + pass + + def run(self): + """Listen and send heartbeat + + 1. Periodically sends heartbeat data to the heartbeat channel of the + consumer group; + 2. Monitor and record the heartbeat information of consumers in the + group from the heartbeat channel of the consumer group; + Returns: + + """ + timeout = 1 if self._heartbeat_interval >= 2 \ + else self._heartbeat_interval / 2 + try: + while not self._stop_event.is_set(): + message = self._ps.get_message( + timeout=timeout) + if message is not None: + self._deal_recv_heartbeat(message) + self._heartbeat_check_schedule.run_pending() + except redis.exceptions.ConnectionError: + # ignore clo + pass + + def get_consumers(self) -> dict: + """Get consumer list + + Returns: + + """ + return self._consumers + + def start(self) -> bool: + """Start heartbeat thread + + Returns: + + Examples: + > XINFO CONSUMERS mystream mygroup + 1) 1) name + 2) "Alice" + 3) pending + 4) (integer) 1 + 5) idle + 6) (integer) 9104628 + 2) 1) name + 2) "Bob" + 3) pending + 4) (integer) 1 + 5) idle + 6) (integer) 83841983 + + """ + if self._heartbeat_listen_thread is not None and \ + self._heartbeat_listen_thread.is_alive(): + return False + self._ps.subscribe(self._channel_name) + self._heartbeat_timeline.get_and_set(0) + self._consumers = {} + + # On startup, get a list of consumers in the group + try: + consumers = self._redis_client.xinfo_consumers( + self._topic, self._consumer_group) + for consumer in consumers: + self._consumers[consumer.get('name', '')] = 0 + except redis.exceptions.ResponseError as error: + LoggerHelper.get_lazy_logger().error(error) + + # Add a schedule task to send heartbeats periodically. + self._heartbeat_check_schedule.every(self._heartbeat_interval) \ + .seconds.do(self._send_heart_beat) + self._heartbeat_check_schedule.every( + self._check_heart_beat_interval * self._heartbeat_interval) \ + .seconds.do(self._check_offline_consumer) + + self._heartbeat_listen_thread = threading.Thread( + target=self.run, + name=f"CEC-{self._consumer_group}:{self._consumer_id}-HEARTBEAT" + ) + self._heartbeat_listen_thread.start() + return True + + def stop(self) -> bool: + """Stop heartbeat thread + + Returns: + + """ + try: + if self._heartbeat_listen_thread is None: + return False + if not self._heartbeat_listen_thread.is_alive(): + self._heartbeat_listen_thread = None + return False + self._heartbeat_check_schedule.clear() + self._stop_event.set() + self._heartbeat_listen_thread.join() + self._stop_event.clear() + self._heartbeat_listen_thread = None + self._ps.unsubscribe() + self._ps.close() + return True + except (redis.RedisError, CecException): + # Ignoring errors arising from the stop phase + return False diff --git a/sysom_api/sdk/cec_redis/redis_admin.py b/sysom_api/sdk/cec_redis/redis_admin.py index a89397f5..4abb817f 100644 --- a/sysom_api/sdk/cec_redis/redis_admin.py +++ b/sysom_api/sdk/cec_redis/redis_admin.py @@ -1,144 +1,82 @@ # -*- coding: utf-8 -*- # """ -Time 2022/7/25 14:48 +Time 2022/7/29 11:25 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File redis_admin.py Description: """ + import json -import sys -from typing import Optional +from typing import Optional, List +import redis.exceptions from ..cec_base.admin import Admin, ConsumeStatusItem -from ..cec_base.admin import TopicNotExistsException, TopicAlreadyExistsException -from ..cec_base.admin import ConsumerGroupNotExistsException -from ..cec_base.admin import ConsumerGroupAlreadyExistsException -from ..cec_base.base import raise_if_not_ignore, CecException +from ..cec_base.exceptions import CecException from ..cec_base.event import Event -from ..cec_base.meta import TopicMeta, PartitionMeta, ConsumerGroupMeta, \ - ConsumerGroupMemberMeta +from ..cec_base.meta import ConsumerGroupMeta, TopicMeta from ..cec_base.log import LoggerHelper from ..cec_base.url import CecUrl from redis import Redis from redis.exceptions import ResponseError - -from .utils import do_connect_by_cec_url from loguru import logger -from itertools import chain +from .utils import raise_if_not_ignore, do_connect_by_cec_url from .consume_status_storage import ConsumeStatusStorage from .common import StaticConst, ClientBase +from .admin_static import static_create_topic, static_del_topic, \ + static_is_topic_exist, static_get_topic_list, \ + static_create_consumer_group, static_del_consumer_group, \ + static_is_consumer_group_exist, static_get_consumer_group_list, \ + get_topic_consumer_group_meta, get_sub_list_key class RedisAdmin(Admin, ClientBase): """A redis-based execution module implement of Admin - 一个基于 Redis 实现的执行模块中的 Admin 实现 + Admin implementation in an execution module based on the Redis. """ def __init__(self, url: CecUrl): Admin.__init__(self) ClientBase.__init__(self, url) - self._redis_client: Redis = None + self._redis_client: Optional[Redis] = None self._current_url: str = "" self.connect_by_cec_url(url) #################################################################### - # 事件中心接口实现 + # Event Center Admin Interface Implementation #################################################################### - @staticmethod - @logger.catch(reraise=True) - def static_create_topic(redis_client: Redis, topic_name: str = "", - num_partitions: int = 1, - replication_factor: int = 1, - ignore_exception: bool = False, - expire_time: int = 24 * 60 * 60 * 1000) -> bool: - LoggerHelper.get_lazy_logger().debug( - f"{redis_client} try to create_topic .") - # 内部表征 Topic 的 Stream 的 Key，拼接了特殊的前缀作为命名空间 - inner_topic_name = StaticConst.get_inner_topic_name( - topic_name) - result = True - try: - # 加锁 - if not RedisAdmin._lock_topic(redis_client, topic_name, - ignore_exception): - return False - # 1. 判断 Topic 是否存在 - if RedisAdmin.static_is_topic_exist(redis_client, - topic_name): - raise TopicAlreadyExistsException( - f"Topic {topic_name} already " - f"exists." - ) - else: - # 2. 使用 xadd 触发 stream 创建 - event_id = redis_client.xadd(inner_topic_name, { - "test": 1 - }) - - pl = redis_client.pipeline() - # 3. 删除刚才添加的测试事件，清空 stream - pl.xdel(inner_topic_name, event_id) - - # 4. 将新建的 Topic 加入到 Topic 集合当中（便于获取所有 Topic 列表） - pl.sadd(StaticConst.REDIS_ADMIN_TOPIC_LIST_SET, - inner_topic_name) - pl.execute() - except Exception as e: - raise_if_not_ignore(ignore_exception, e) - finally: - # 解锁 - RedisAdmin._unlock_topic(redis_client, topic_name) - - LoggerHelper.get_lazy_logger().success( - f"{redis_client} create_topic '{topic_name}' successfully.") - return result @logger.catch(reraise=True) def create_topic(self, topic_name: str = "", num_partitions: int = 1, - replication_factor: int = 1, - ignore_exception: bool = False, - expire_time: int = 24 * 60 * 60 * 1000) -> bool: + replication_factor: int = 1, **kwargs) -> bool: """Create one topic - 创建一个 Topic => 对应到 Redis 应该是创建一个 Stream： - 1. 首先判断 Topic 是否已经存在，如果已经存在，则抛出异常； - 2. 接着使用 xadd 命令触发 stream 的创建过程； - 3. 最后将刚才插入的测试数据删掉，清空 stream - 4. 将 topic_name 加入到特定的 set 集合当中 - （该集合包含了所有通过 CEC 创建的 Topic 名称列表） - - TODO: 此处需要进一步考虑是否使用事务，防止中间某一步执行出错，状态不一致 + Creating a Topic => Corresponding to Redis should be creating a Stream: + 1. First determine whether Topic already exists and, if so, throw + an exception; + 2. Then use the xadd command to trigger the stream creation + process; + 3. Thirdly, delete the test data you just inserted and clear + the stream; + 4. Finally, add the topic_name to a specific set + (This collection contains a list of all the Topic names created + through the CEC) + + TODO: This is where further consideration needs to be given to the use + of transactions to prevent errors in execution at an intermediate + step and inconsistent state Args: - topic_name: 主题名字（主题的唯一标识） - num_partitions: 该主题的分区数 - - 1. 该参数指定了在分布式集群部署的场景下，同一个主题的数据应该被划分为几个分区，分别存储在不同的集群节点上； - 2. 如果底层的消息中间件支持分区（比如：Kafka），则可以依据该配置进行分区； - 3. 如果底层的消息中间件不支持分区（比如：Redis），则忽略该参数即可（认定为只有一个分区即可），可以通过 - Admin.is_support_partitions() 方法判定当前使用的消息中间件实现是否支持该特性； - - replication_factor: 冗余因子（指定该主题的数据又几个副本） - - 1. 该参数制定了在分布式集群部署的场景下，同一个主题的分区存在副本的数量，如果 replication_factor == 1 - 则表示主题下的所有分区都只有一个副本，一旦丢失不可回复； - 2. 如果底层的消息中间件支持数据副本，则可以依据该配置进行对应的设置； - 3. 如果底层的消息中间件不支持数据副本，则忽略该参数即可（即认定只有一个副本即可），可以通过 - Admin.is_support_replication() 方法判定当前使用的小心中间件实现是否支持该特性； - - ignore_exception: 是否忽略可能会抛出的异常 - expire_time: 事件超时时间（单位：ms，默认：1day） - - 1. 该参数指定了目标 Topic 中每个事件的有效期； - 2. 一旦一个事件的加入到 Topic 的时间超过了 expire_time，则cec不保证该事件 - 的持久性，cec应当在合适的时候删除超时的事件； - 3. 不强制要求超时的事件被立即删除，可以对超时的事件进行周期性的清理。 + topic_name(str): Topic name (unique identification of the topic) + num_partitions(int): Number of partitions of the topic + replication_factor(int): Redundancy factor (specifies how many + copies of the data for the subject should + be kept in the event center) + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown + expire_time: Event timeout time (in ms, default: 1day) Returns: bool: True if successful, False otherwise. @@ -152,73 +90,27 @@ class RedisAdmin(Admin, ClientBase): >>> admin.create_topic("test_topic") True """ - return RedisAdmin.static_create_topic(self._redis_client, - topic_name, - num_partitions, - replication_factor, - ignore_exception, - expire_time) - - @staticmethod - @logger.catch(reraise=True) - def static_del_topic(redis_client: Redis, topic_name: str, - ignore_exception: bool = False) -> bool: - LoggerHelper.get_lazy_logger().debug( - f"{redis_client} try to del_topic .") - - inner_topic_name = StaticConst.get_inner_topic_name( - topic_name) - - try: - # 加锁 - if not RedisAdmin._lock_topic(redis_client, topic_name, - ignore_exception): - return False - # 1. 判断是否存在 - if not RedisAdmin.static_is_topic_exist(redis_client, - topic_name): - raise_if_not_ignore(ignore_exception, - TopicNotExistsException( - f"Topic {topic_name} not exists." - )) - pl = redis_client.pipeline() - - # 2. 删除对应的 stream（topic） - pl.delete(inner_topic_name) - - # 3. 将当前 topic 从 topic 列表中移除 - pl.srem(StaticConst.REDIS_ADMIN_TOPIC_LIST_SET, - inner_topic_name) - - pl.execute() - - # 4. 清除 TOPIC 相关的元数据信息 - RedisAdmin.del_topic_meta(redis_client, topic_name) - - # 5. 删除 TOPIC 关联的用于存储消费状态的结构 - ConsumeStatusStorage.destroy_by_stream(redis_client, topic_name) - except Exception as e: - raise_if_not_ignore(ignore_exception, e) - finally: - # 解锁 - RedisAdmin._unlock_topic(redis_client, topic_name) - - LoggerHelper.get_lazy_logger().success( - f"{redis_client} del_topic '{topic_name}' successfully.") - return True + return static_create_topic( + self._redis_client, + topic_name, + num_partitions, + replication_factor, + **kwargs + ) @logger.catch(reraise=True) - def del_topic(self, topic_name: str, - ignore_exception: bool = False) -> bool: + def del_topic(self, topic_name: str, **kwargs) -> bool: """Delete one topic - 删除一个 Topic => 对应到 Redis 应该是删除一个 Stream - 1. 直接删除 Stream 对应的key即可 - 2. 清楚一些相关的元数据信息 + Deleting a Topic => Corresponding to redis should be deleting a stream + 1. Just delete the key corresponding to the stream + 2. Clear some relevant metadata information Args: - topic_name: 主题名字（主题的唯一标识） - ignore_exception: 是否忽略可能会抛出的异常 + topic_name(str): Topic name + + Keyword Args + ignore_exception: Whether to ignore exceptions that may be thrown Returns: bool: True if successful, False otherwise. @@ -231,28 +123,21 @@ class RedisAdmin(Admin, ClientBase): >>> admin.del_topic("test_topic") True """ - return RedisAdmin.static_del_topic(self._redis_client, topic_name, - ignore_exception) + return static_del_topic(self._redis_client, topic_name, **kwargs) - @staticmethod @logger.catch(reraise=True) - def static_is_topic_exist(redis_client: Redis, - topic_name: str) -> bool: - res = redis_client.type( - StaticConst.get_inner_topic_name(topic_name)) == 'stream' - LoggerHelper.get_lazy_logger().debug( - f"Is topic {topic_name} exists? => {res}.") - return res - - @logger.catch(reraise=True) - def is_topic_exist(self, topic_name: str) -> bool: + def is_topic_exist(self, topic_name: str, **kwargs) -> bool: """Judge whether one specific topic is exists - 判断 Topic 是否存在 => 对应到 Redis 应该是判断是否存最对应stream - 1. 使用 type 命令判断key对应的类型是否是 stream + Determine if Topic exists => Corresponds to Redis should be to + determine if the most corresponding stream exists + 1. Use the type command to determine whether the type of the key + is 'stream' Args: - topic_name: 主题名字（主题的唯一标识） + topic_name(str): Topic name + + Keyword Args: Returns: bool: True if topic exists, False otherwise. @@ -262,37 +147,22 @@ class RedisAdmin(Admin, ClientBase): >>> admin.is_topic_exist("test_topic") True """ - return RedisAdmin.static_is_topic_exist(self._redis_client, - topic_name) + return static_is_topic_exist(self._redis_client, topic_name, **kwargs) - @staticmethod @logger.catch(reraise=True) - def static_get_topic_list(redis_client: Redis) -> [str]: - res = redis_client.smembers(StaticConst.REDIS_ADMIN_TOPIC_LIST_SET) - topics = [] - for inner_topic_name in res: - topic_meta = TopicMeta( - StaticConst.get_topic_name_by_inner_topic_name( - inner_topic_name)) - topic_meta.partitions = { - 0: PartitionMeta(0) - } - topics.append(topic_meta) - - LoggerHelper.get_lazy_logger().debug( - f"get_topic_list => {res}.") - return topics - - @logger.catch(reraise=True) - def get_topic_list(self) -> [str]: + def get_topic_list(self, **kwargs) -> List[TopicMeta]: """Get topic list - 获取 Topic 列表 => 对应到 Redis 应该是获取所有 Stream 的列表 - 1. 本实现创建了一个特殊的 Set，保存了由本套接口创建的所有 Stream 的key； - 2. 所以只需要查询该 Set，获取到所有的key即可 + Getting the Topic list => corresponding to Redis should be a list of + all Streams. + 1. This implementation creates a special Set that holds the keys + of all Streams created by this set of interfaces; + 2. So just query the Set and get all the keys. Args: + Keyword Args: + Returns: [str]: The topic name list @@ -301,14 +171,14 @@ class RedisAdmin(Admin, ClientBase): >>> admin.get_topic_list() ['test_topic'] """ - return RedisAdmin.static_get_topic_list(self._redis_client) + return static_get_topic_list(self._redis_client, **kwargs) @staticmethod @logger.catch(reraise=True) def get_meta_info(client: Redis, topic_name: str) -> Optional[dict]: """Get topic's meta info - 获取特定 Topic 的元数据信息 + Get metadata information for a specific topic Args: client(Redis): Redis client @@ -329,60 +199,27 @@ class RedisAdmin(Admin, ClientBase): **res } - @staticmethod - @logger.catch(reraise=True) - def static_create_consumer_group(redis_client: Redis, - consumer_group_id: str, - ignore_exception: bool = False) -> bool: - LoggerHelper.get_lazy_logger().debug( - f"{redis_client} try to create_consumer_group " - f".") - - # 使用set给create/del操作加锁（防止并发场景下重复创建删除问题） - try: - # 加锁 - if not RedisAdmin._lock_consumer_group(redis_client, - consumer_group_id, - ignore_exception): - return False - if RedisAdmin.static_is_consumer_group_exist(redis_client, - consumer_group_id): - if ignore_exception: - return False - else: - raise ConsumerGroupAlreadyExistsException( - f"Consumer group {consumer_group_id} already exists.") - # 添加到消费组key集合当中 - redis_client.sadd( - StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LIST_SET, - consumer_group_id) - except Exception as e: - raise_if_not_ignore(ignore_exception, e) - finally: - # 解锁 - RedisAdmin._unlock_consumer_group(redis_client, - consumer_group_id) - LoggerHelper.get_lazy_logger().debug( - f"{redis_client} create_consumer_group " - f"'{consumer_group_id}' successfully.") - return True - @logger.catch(reraise=True) - def create_consumer_group(self, consumer_group_id: str, - ignore_exception: bool = False) -> bool: + def create_consumer_group(self, consumer_group_id: str, **kwargs) -> bool: """Create one consumer group - 创建一个消费组 - 1. Redis 中消费组的概念是对每个 Stream 来讲的，同一个消费组不能消费多个 - Stream； - 2. 本实现创建了一个特殊的 Set，保存了由本套接口创建的所有消费组，在对某个 - Stream 进行组消费时，如果组不存在则创建。 - 3. 再以 ConsumerId 为 key，创建一个 List，保存和该消费者相关的所有的 - Stream，用于保障删除一个消费组时，可以删除所有 Stream 中的同名消费组 + Create a consumer group + 1. The concept of a consumption group in Redis is for each Stream, + the same consumer group cannot consume multiple Streams; + 2. This implementation creates a special Set that holds all + consumer groups created by this set of interfaces, and creates + them if the group does not exist when group consumption is + performed on a Stream; + 3. Then create a list with ConsumerId as the key to store all the + Streams related to that consumer, to ensure that when a consumer + group is deleted, this consumer group should be removed from all + subscribed streams Args: - consumer_group_id: 消费组ID，应当具有唯一性 - ignore_exception: 是否忽略可能会抛出的异常 + consumer_group_id(str): Consumer group ID + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown Returns: bool: True if successful, False otherwise. @@ -396,87 +233,26 @@ class RedisAdmin(Admin, ClientBase): >>> admin.create_consumer_group("test_group") True """ - return RedisAdmin.static_create_consumer_group(self._redis_client, - consumer_group_id, - ignore_exception) + return static_create_consumer_group(self._redis_client, + consumer_group_id, + **kwargs) - @staticmethod @logger.catch(reraise=True) - def static_del_consumer_group(redis_client: Redis, - consumer_group_id: str, - ignore_exception: bool = False) -> bool: - LoggerHelper.get_lazy_logger().debug( - f"{redis_client} try to del_consumer_group " - f".") - - try: - # 加锁 - if not RedisAdmin._lock_consumer_group( - redis_client, consumer_group_id, ignore_exception - ): - return False - - # 1. 首先判断消费组是否存在，不存在则根据情况抛出异常 - if not RedisAdmin.static_is_consumer_group_exist(redis_client, - consumer_group_id): - raise_if_not_ignore(ignore_exception, - ConsumerGroupNotExistsException( - f"Consumer group {consumer_group_id} " - f"not exists." - )) - - # 2. 从消费组集合中移除 - redis_client.srem( - StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LIST_SET, - consumer_group_id) - - # 3. 销毁当前消费组关联的所有stream中的同名消费组结构 - streams = redis_client.lpop( - RedisAdmin.get_sub_list_key(consumer_group_id), - sys.maxsize - ) - pl = redis_client.pipeline() - for stream in streams: - # 取消订阅主题 - pl.xgroup_destroy(stream, consumer_group_id) - - # 删除对应的 zset - ConsumeStatusStorage.destroy_by_stream_group(pl, stream, - consumer_group_id) - pl.execute() - for stream in streams: - # 清除主题-消费组相关的元数据信息 - RedisAdmin.del_topic_consumer_group_meta(redis_client, stream, - consumer_group_id) - except ConsumerGroupNotExistsException as e: - raise_if_not_ignore(ignore_exception, e) - except Exception as e: - print(e) - # 此处忽略 Pipeline 执行清理操作可能产生的错误 - pass - finally: - # 解锁 - RedisAdmin._unlock_consumer_group(redis_client, - consumer_group_id) - - LoggerHelper.get_lazy_logger().debug( - f"{redis_client} del_consumer_group " - f"'{consumer_group_id}' successfully.") - return True - - @logger.catch(reraise=True) - def del_consumer_group(self, consumer_group_id: str, - ignore_exception: bool = False) -> bool: + def del_consumer_group(self, consumer_group_id: str, **kwargs) -> bool: """Delete one consumer group - 删除消费组 - 1. 首先判断消费组是否存在，不存在则根据情况抛出异常 - 2. 首先在消费组 key 集合中移除当前消费组； - 3. 然后找到消费组关联的所有的stream，执行destroy操作 + Delete consumer group + 1. First determine if the consumer group exists, and if not, + throw an exception as appropriate; + 2. Removal from the set of consumer groups; + 3. Destroy all consumer group structures of the same name in all + streams associated with the current consumption group. Args: - consumer_group_id: 消费组ID - ignore_exception: 是否忽略可能会抛出的异常 + consumer_group_id(str): Consumer group ID + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown Returns: bool: True if successful, False otherwise. @@ -489,34 +265,25 @@ class RedisAdmin(Admin, ClientBase): >>> admin.del_consumer_group("test_group") True """ - return RedisAdmin.static_del_consumer_group(self._redis_client, - consumer_group_id, - ignore_exception) - - @staticmethod - @logger.catch(reraise=True) - def static_is_consumer_group_exist(redis_client: Redis, - consumer_group_id: str) -> bool: - res = redis_client.sismember( - StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LIST_SET, - consumer_group_id) - - LoggerHelper.get_lazy_logger().debug( - f"{redis_client} Is consumer group '{consumer_group_id}' exists => {res}") - return res + return static_del_consumer_group(self._redis_client, consumer_group_id, + **kwargs) @logger.catch(reraise=True) - def is_consumer_group_exist(self, consumer_group_id: str) -> bool: + def is_consumer_group_exist(self, consumer_group_id: str, + **kwargs) -> bool: """Judge whether one specific consumer group exists - 判断指定的消费组是否存在 - 1. 判断存储所有消费组 key 的 Set 中是否包含指定的消费者id； - 2. 同时判断 consumer_group_id 是否是一个key，如果被占用，也报已存在，不允许 - 创建 - TODO: 此处要考虑是否需要并发安全 + Determines whether the specific consumer group exists + 1. Determine whether the set storing all consumer group keys + contains the specified consumer id; + 2. Also determine if consumer_group_id is a key, and if it is + occupied, also report it as existing and not allowed to create. Args: - consumer_group_id: 消费组ID + consumer_group_id(str): Consumer group ID + + Keyword Args: + ignore_exception: Whether to ignore exceptions that may be thrown Returns: bool: True if consumer group exists, False otherwise. @@ -526,204 +293,214 @@ class RedisAdmin(Admin, ClientBase): >>> admin.is_consumer_group_exist("test_group") True """ - return RedisAdmin.static_is_consumer_group_exist( - self._redis_client, - consumer_group_id) - - @staticmethod - @logger.catch(reraise=True) - def static_get_consumer_group_list(redis_client: Redis) \ - -> [ConsumerGroupMeta]: - - res = redis_client.smembers( - StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LIST_SET) - group_metas = [] - for group_id in res: - group_meta = ConsumerGroupMeta(group_id) - try: - # 得到改消费组所有订阅的主题信息 - sub_topics = redis_client.lrange( - RedisAdmin.get_sub_list_key(group_id), 0, -1 - ) - - # 遍历所有的主题，得到所有的成员 - pl = redis_client.pipeline(transaction=True) - for topic in sub_topics: - pl.xinfo_consumers(topic, group_id) - - # {"name":"Alice","pending":1,"idle":9104628} - for consumer in chain.from_iterable(pl.execute()): - group_meta.members.append( - ConsumerGroupMemberMeta(consumer['name'])) - except Exception as e: - group_meta.error = e - else: - group_metas.append(group_meta) - - LoggerHelper.get_lazy_logger().debug( - f"get_consumer_group_list => {res}.") - return group_metas + return static_is_consumer_group_exist(self._redis_client, + consumer_group_id) @logger.catch(reraise=True) - def get_consumer_group_list(self) -> [ConsumerGroupMeta]: + def get_consumer_group_list(self, **kwargs) -> List[ConsumerGroupMeta]: """Get consumer group list - 获取消费组列表 - 1. 由于在 Redis 中，消费组是属于 Stream 的，不同 Stream 的消费组是独立的， - 为了实现出同一个消费组可以消费不同 Topic 的目的，使用一个特殊的 set - _REDIS_ADMIN_CONSUMER_GROUP_LIST_SET => 存储了所有的消费组的名字 - 2. 当某个消费组试图消费一个 Stream 时，cec-redis 会自动判断 Stream 中是否 - 包含该消费组，如果不包含则自动创建； - 3. 因此可以直接通过 _REDIS_ADMIN_CONSUMER_GROUP_LIST_SET 获取消费组列表 + Get consumer group list + 1. Since in Redis, consumer groups belong to Streams, and consumer + groups of different Streams are independent, a special set + _REDIS_ADMIN_CONSUMER_GROUP_LIST_SET stores the names of all + consumer groups in order to achieve the purpose that the same + consumer group can consume different Topics; + 2. When a consumer group tries to consume a Stream, cec-redis + automatically determines if the Stream contains the consumer + group and automatically creates it if it does not; + 3. So you can get the list of consumer groups directly from + _REDIS_ADMIN_CONSUMER_GROUP_LIST_SET. Returns: - [str]: The consumer group list + [ConsumerGroupMeta]: The consumer group list Examples: >>> admin = dispatch_admin("redis://localhost:6379") >>> admin.get_consumer_group_list() ['test_group'] """ - return RedisAdmin.static_get_consumer_group_list( - self._redis_client) + return static_get_consumer_group_list(self._redis_client, **kwargs) @logger.catch(reraise=True) - def get_consume_status(self, topic: str, consumer_group_id: str = "", - partition: int = 0) -> [ConsumeStatusItem]: + def get_consume_status( + self, topic: str, consumer_group_id: str = "", partition: int = 0, + **kwargs) -> List[ConsumeStatusItem]: """Get consumption info for specific - 获取特定消费者组对某个主题下的特定分区的消费情况，应包含以下数据 - 1. 最小ID（最小 offset）=> xinfo stream (first-entry) - 2. 最大ID（最大 offset）=> xinfo stream (last-entry) - 3. 分区中存储的事件总数（包括已消费的和未消费的）=> xlen / xinfo stream (length) - 4. 最后一个当前消费组在该分区已确认的事件ID（最后一次消费者确认的事件的ID） - => 使用 Redis 的一个主题相关的 key 存储了最后一次ack的ID，从中提取即可 - 5. 分区的消息堆积数量 LAG（已经提交到该分区，但是没有被当前消费者消费或确认的事件数量） - => xinfo stream (entries-added) 可以得到历史加入到主题的事件数量 - => xinfo group (entries-read) 可以得到当前消费组已经读取的事件数量 - => 两者相减能得到消息堆积数量 + Get the consumption info of a particular topic by a particular consumer + group. Args: - topic: 主题名字 - consumer_group_id: 消费组ID - 1. 如果 consumer_group_id 为空字符串或者None，则返回订阅了该主题的所有 - 消费组的消费情况；=> 此时 partition 参数无效（将获取所有分区的消费数据） - 2. 如果 consumer_group_id 为无效的组ID，则抛出异常； - 3. 如果 consumer_group_id 为有效的组ID，则只获取该消费组的消费情况。 - partition: 分区ID（Redis不支持分区，因此此参数在 cec-redis 的实现里面只有一个合法值0） - 1. 如果 partition 指定有效非负整数 => 返回指定分区的消费情况 - 2. 如果 partition 指定无效非负整数 => 抛出异常 - 3. 如果 partition 指定负数 => 返回当前主题下所有分区的消费情况 + topic(str): Topic name + consumer_group_id(str): Consumer group ID + 1. If consumer_group_id == '' or None, returns the consumption + info of all consumer groups subscribed to the topic; + => In this case the partition parameter is invalid + (will get consumption info for all partitions) + 2. Throws an exception if consumer_group_id is an invalid group + ID; + 3. If consumer_group_id is a valid group ID, then only get + consumption info of the specified consumption group. + partition: Partition ID + 1. If partition specifies a valid non-negative integer + => returns the consumption info of the specified partition; + 2. Throws an exception if partition specifies an invalid + non-negative integer; + 3. If partition specifies a negative number => returns the + consumption info of all partitions under the current topic. Raises: CecException + Examples: + >>> admin = dispatch_admin("redis://localhost:6379") + >>> admin.get_consume_status("topic1") + [ + { + "topic":"topic1", + "consumer_group_id":"c78e8b71-45b9-4e11-8f8e-05a98b534cc0", + "min_id":"1661516434003-0", + "max_id":"1661516434004-4", + "total_event_count":10, + "last_ack_id":"1661516434003-4", + "lag":5 + }, + { + "topic":"topic1", + "consumer_group_id":"d1b39ec3-6ae9-42a6-83b5-257d875788e6", + "min_id":"1661516434003-0", + "max_id":"1661516434004-4", + "total_event_count":10, + "last_ack_id":"1661516434003-1", + "lag":8 + } + ] + Returns: """ - inner_topic_name = StaticConst.get_inner_topic_name(topic) - # 使用 xinfo stream 获取主题信息 - try: - stream_info = self._redis_client.xinfo_stream(inner_topic_name) - min_id, max_id, length, entries_added = None, None, 0, 0 - if 'first-entry' in stream_info: - min_id = stream_info['first-entry'][0] - if 'last-entry' in stream_info: - max_id = stream_info['last-entry'][0] - if 'length' in stream_info: - length = stream_info['length'] - groups = self._redis_client.xinfo_groups(inner_topic_name) - if consumer_group_id != '' and consumer_group_id is not None: - select_group = None - # 尝试获取指定消费组的消费信息 - for group in groups: - if group.get('name') == consumer_group_id: - select_group = group - break - if select_group is None: - # 消费组不存在 - raise CecException( - f"Consumer group {consumer_group_id} not exists or did " - f"not subscribe topic {topic}") - - # 由于目前 cec-redis 的实现不支持分区，因此每个主题有且只有一个分区号 - # 并且分区号为0，如果在指定了消费组的情况下，传入的分区号 <= 0视为有效； - # 传入的分区号 > 0 视为无效 - if partition > 0: - raise CecException( - f"Topic {topic} did not contains partition {partition}" + def _get_one_group_consume_status(_groups: List[dict]) \ + -> List[ConsumeStatusItem]: + """Get the consumption of the specified consumer group""" + select_group = None + # Attempt to obtain consumption information for the specified + # consumer group + for _group in _groups: + if _group.get('name') == consumer_group_id: + select_group = _group + break + if select_group is None: + # Consumer group not exists + raise CecException( + f"Consumer group {consumer_group_id} not exists or did " + f"not subscribe topic {topic}") + + # Since the current implementation of cec-redis does not support + # partitioning, each topic has one and only one partition number + # and the partition number is 0. If a consumption group is + # specified, the partition number passed in <= 0 is considered + # valid; the partition number passed in > 0 is considered invalid. + if partition > 0: + raise CecException( + f"Topic {topic} did not contains partition {partition}" + ) + + # Here it is sufficient to return the consumption of the + # specified consumer group + last_ack_id = get_topic_consumer_group_meta( + self._redis_client, topic, select_group.get('name'), + StaticConst.TOPIC_CONSUMER_GROUP_META_KEY_LAST_ACK_ID + ) + + # Get LAG + if self.is_gte_7(self._redis_client): + lag = select_group['lag'] + select_group['pending'] + else: + lag = ConsumeStatusStorage.get_already_ack_count( + self._redis_client, topic, consumer_group_id + ) + + return [ + ConsumeStatusItem( + topic, consumer_group_id, 0, + min_id=min_id, + max_id=max_id, + total_event_count=length, + last_ack_id=last_ack_id, + lag=lag + ) + ] + + def _get_all_group_consume_status(_groups: List[dict]) \ + -> List[ConsumeStatusItem]: + """Get the consumption of the all consumer group""" + # 获取所有消费组的消费情况（此时 partition 参数无效） + res, counts_map = [], {} + if not self.is_gte_7(self._redis_client): + # 如果 Redis 版本小于7，将使用 ConsumeStatusStorage 获取 lag + pipeline = self._redis_client.pipeline() + for _group in _groups: + ConsumeStatusStorage.get_already_ack_count( + pipeline, topic, _group.get('name') ) + counts = pipeline.execute() + for i, _group in enumerate(_groups): + counts_map[_group.get('name')] = counts[i] - # 此处只需将指定消费组的消费情况返回即可 - last_ack_id = self.get_topic_consumer_group_meta( - self._redis_client, topic, select_group.get('name'), + for _group in _groups: + last_ack_id = get_topic_consumer_group_meta( + self._redis_client, topic, _group.get('name'), StaticConst.TOPIC_CONSUMER_GROUP_META_KEY_LAST_ACK_ID ) # 获取 LAG - if self.is_gte_7(self._redis_client): - lag = select_group['lag'] + select_group['pending'] + if 'lag' in _group and 'pending' in _group: + lag = _group['lag'] + _group['pending'] else: - lag = ConsumeStatusStorage.get_already_ack_count( - self._redis_client, topic, consumer_group_id - ) + lag = length - counts_map[_group.get('name')] + + res.append(ConsumeStatusItem( + topic, consumer_group_id, 0, + min_id=min_id, + max_id=max_id, + length=length, + last_ack_id=last_ack_id, + lag=lag + )) + return res - # 返回指定消费组的消费情况 - return [ - ConsumeStatusItem( - topic, consumer_group_id, 0, - min_id, max_id, length, - last_ack_id, lag - ) - ] - else: - # 获取所有消费组的消费情况（此时 partition 参数无效） - res, counts_map = [], {} - if not self.is_gte_7(self._redis_client): - # 如果 Redis 版本小于7，将使用 ConsumeStatusStorage 获取 lag - pl = self._redis_client.pipeline() - for group in groups: - ConsumeStatusStorage.get_already_ack_count( - pl, topic, group.get('name') - ) - counts = pl.execute() - for i in range(len(groups)): - counts_map[groups[i].get('name')] = counts[i] - - for group in groups: - last_ack_id = self.get_topic_consumer_group_meta( - self._redis_client, topic, group.get('name'), - StaticConst.TOPIC_CONSUMER_GROUP_META_KEY_LAST_ACK_ID - ) + inner_topic_name = StaticConst.get_inner_topic_name(topic) - # 获取 LAG - if 'lag' in group and 'pending' in group: - lag = group['lag'] + group['pending'] - else: - lag = length - counts_map[group.get('name')] - - res.append(ConsumeStatusItem( - topic, group['name'], 0, - min_id, max_id, length, - last_ack_id, lag - )) - return res - except Exception as e: - raise CecException(e) + # Use 'xinfo stream' to get topic information + try: + stream_info = self._redis_client.xinfo_stream(inner_topic_name) + min_id = stream_info.get("first-entry", [None])[0] + max_id = stream_info.get("last-entry", [None])[0] + length = stream_info.get("length", 0) + groups = self._redis_client.xinfo_groups(inner_topic_name) + if consumer_group_id != '' and consumer_group_id is not None: + return _get_one_group_consume_status(groups) + return _get_all_group_consume_status(groups) + except redis.exceptions.RedisError as exc: + raise CecException(exc) from exc @logger.catch(reraise=True) def get_event_list(self, topic: str, partition: int, offset: str, - count: int) -> [Event]: + count: int, **kwargs) -> List[Event]: """ Get event list for specific - 获取特定主题在指定分区下的消息列表 => Redis 中使用 xrange 命令获取 stream 中的消息 - 1. offset 和 count 用于分页 + Get a list of messages for a specific topic under a specified partition + => Use the xrange command in Redis to get the messages in a stream + 1. offset and count for paging + Args: - topic: 主题名字 - partition: 分区ID => Redis 中无分区，因此次参数无效 - offset: 偏移（希望读取在该 ID 之后的消息） - count: 最大读取数量 + topic(str): Topic name + partition(int): Partition ID => There is no partition in Redis, so + this parameter is invalid + offset(int): Offset (want to read messages after this ID) + count(int): Maximum number of reads References: https://redis.io/commands/xrange/ @@ -731,18 +508,22 @@ class RedisAdmin(Admin, ClientBase): Returns: """ + ignore_exception = kwargs.get("ignore_exception", False) inner_topic_name = StaticConst.get_inner_topic_name(topic) - messages = self._redis_client.xrange( - inner_topic_name, - min=f"({offset}", - max='+', - count=count - ) res = [] - for message in messages: - message_content = json.loads( - message[1][StaticConst.REDIS_CEC_EVENT_VALUE_KEY]) - res.append(Event(message_content, message[0])) + try: + messages = self._redis_client.xrange( + inner_topic_name, + min=f"({offset}", + max='+', + count=count + ) + for message in messages: + message_content = json.loads( + message[1][StaticConst.REDIS_CEC_EVENT_VALUE_KEY]) + res.append(Event(message_content, message[0])) + except redis.exceptions.RedisError as exc: + raise_if_not_ignore(ignore_exception, exc) return res @staticmethod @@ -751,7 +532,7 @@ class RedisAdmin(Admin, ClientBase): consumer_group_id: str) -> bool: """Add one consumer group to stream - 将消费组添加到对应的stream中 + Add the consumer group to the corresponding stream Args: redis_client(Redis): Redis client @@ -770,17 +551,13 @@ class RedisAdmin(Admin, ClientBase): inner_topic_name, consumer_group_id, id="0-0") except ResponseError: - # 消费组已存在 LoggerHelper.get_lazy_logger().debug( f"Consumer group '{consumer_group_id}" f"' already exists.") return False - except Exception as e: - raise e else: - # 消费组创建成功，进行关联 redis_client.lpush( - RedisAdmin.get_sub_list_key(consumer_group_id), + get_sub_list_key(consumer_group_id), inner_topic_name ) LoggerHelper.get_lazy_logger().debug( @@ -789,18 +566,18 @@ class RedisAdmin(Admin, ClientBase): return True @logger.catch(reraise=True) - def is_support_partitions(self) -> bool: + def is_support_partitions(self, **kwargs) -> bool: return False @logger.catch(reraise=True) - def is_support_replication(self) -> bool: + def is_support_replication(self, **kwargs) -> bool: return False @logger.catch(reraise=True) def connect_by_cec_url(self, url: CecUrl): """Connect to redis server by CecUrl - 通过 CecUrl 连接到 Redis 服务器 + Connecting to the Redis server via CecUrl Args: url(str): CecUrl @@ -808,7 +585,7 @@ class RedisAdmin(Admin, ClientBase): LoggerHelper.get_lazy_logger().debug( f"{self} try to connect to '{url}'.") self._redis_client = do_connect_by_cec_url(url) - self._current_url = url.__str__() + self._current_url = str(url) LoggerHelper.get_lazy_logger().success( f"{self} connect to '{url}' successfully.") @@ -816,7 +593,8 @@ class RedisAdmin(Admin, ClientBase): def connect(self, url: str): """Connect to redis server by url - 连接到远端的消息中间件 => 对应到本模块就是连接到 Redis 服务器 + Connecting to the remote message queue => Corresponding to this module + is connecting to the Redis server. Args: url(str): CecUrl @@ -828,7 +606,9 @@ class RedisAdmin(Admin, ClientBase): def disconnect(self): """Disconnect from redis server - 断开连接 => 对应到本模块就是断开 Redis 服务器连接 + Disconnect from remote server => Corresponds to this module as + disconnecting the Redis server. + """ if self._redis_client is None: return @@ -839,251 +619,15 @@ class RedisAdmin(Admin, ClientBase): LoggerHelper.get_lazy_logger().success( f"{self} disconnect from '{self._current_url}' successfully.") - #################################################################### - # 一些辅助函数 - #################################################################### - - @staticmethod - def get_topic_consumer_group_meta_info_key(topic: str, group_id: str, - key: str): - return f"{StaticConst.REDIS_ADMIN_TOPIC_CONSUMER_GROUP_META_PREFIX}" \ - f"{topic + ':' if topic is not None else ''}" \ - f"{group_id + ':' if group_id is not None else ''}" \ - f"{key + ':' if key is not None else ''}" - - @staticmethod - def get_topic_meta_info_key(topic: str, key: str): - return f"{StaticConst.REDIS_ADMIN_TOPIC_META_PREFIX}" \ - f"{topic + ':' if topic is not None else ''}" \ - f"{key + ':' if key is not None else ''}" - - @staticmethod - def get_sub_list_key(group_id: str) -> str: - return f"{StaticConst.REDIS_ADMIN_CONSUMER_GROUP_SUB_LIST_PREFIX}" \ - f"{group_id}" - - @staticmethod - def store_meta(redis_client: Redis, key: str, value: str): - return redis_client.set(key, value) - - @staticmethod - def get_meta(redis_client: Redis, key: str): - return redis_client.get(key) - - @staticmethod - def del_meta(redis_client: Redis, prefix: str): - next_cursor = 0 - while True: - next_cursor, key_list = redis_client.scan( - next_cursor, - match=f"{prefix}*", - count=100 - ) - if len(key_list) > 0: - redis_client.delete(*key_list) - if next_cursor == 0: - break - return True - - @staticmethod - def store_topic_consumer_group_meta(redis_client: Redis, topic: str, - key: str, group_id: str, value): - return RedisAdmin.store_meta( - redis_client, - RedisAdmin.get_topic_consumer_group_meta_info_key( - topic, group_id, key - ), - value - ) - - @staticmethod - def store_topic_meta(redis_client: Redis, topic: str, key: str, value): - """Store topic meta info - - 存储主题相关的元数据信息 - - Args: - redis_client: - topic: - key: - value: - - Returns: - - """ - return RedisAdmin.store_meta( - redis_client, - RedisAdmin.get_topic_meta_info_key(topic, key), - value - ) - - @staticmethod - def get_topic_consumer_group_meta(redis_client: Redis, topic: str, - group_id: str, key: str): - return RedisAdmin.get_meta( - redis_client, - RedisAdmin.get_topic_consumer_group_meta_info_key( - topic, group_id, key - ) - ) - - @staticmethod - def get_topic_meta(redis_client: Redis, topic: str, key: str): - """Get topic meta info - - 获取主题相关的元数据信息 - - Args: - redis_client: - topic: - key: - - Returns: - - """ - return RedisAdmin.get_meta( - redis_client, - RedisAdmin.get_topic_meta_info_key(topic, key) - ) - - @staticmethod - def del_topic_consumer_group_meta(redis_client: Redis, - topic: str, group_id: str): - return RedisAdmin.del_meta( - redis_client, - RedisAdmin.get_topic_consumer_group_meta_info_key( - topic, group_id, None - ) - ) - - @staticmethod - def del_topic_meta(redis_client: Redis, topic: str): - """Delete all meta info for specific topic - - 删除特定主题的所有元数据信息 - - Args: - redis_client: - topic: - - Returns: - - """ - res1 = RedisAdmin.del_meta( - redis_client, - RedisAdmin.get_topic_consumer_group_meta_info_key(topic, None, - None) - ) - res2 = RedisAdmin.del_meta( - redis_client, - RedisAdmin.get_topic_meta_info_key(topic, None) - ) - return res1 and res2 - - @staticmethod - def _lock_topic(redis_client: Redis, topic: str, - ignore_exception: bool = False) -> bool: - """ - - 给某个主题加锁，防止并发场景下重复操作问题 - - Args: - redis_client: - topic: - ignore_exception: - - Returns: - - """ - # 使用set给create/del操作加锁（防止并发场景下重复创建删除问题） - if redis_client.set( - f"{StaticConst.REDIS_ADMIN_TOPIC_LOCKER_PREFIX}{topic}", - topic, nx=True, ex=10) == 0: - return raise_if_not_ignore(ignore_exception, - CecException( - f"Someone else is creating or" - f" deleting this topic." - )) - return True - - @staticmethod - def _unlock_topic(redis_client: Redis, topic: str) -> bool: - """ - - 释放给某个主题加的锁，应当和 lock_topic 配套使用 - - Args: - redis_client: - topic: - - Returns: - - """ - # 释放锁 - if redis_client.get( - f"{StaticConst.REDIS_ADMIN_TOPIC_LOCKER_PREFIX}{topic}" - ) == topic: - return redis_client.delete( - f"{StaticConst.REDIS_ADMIN_TOPIC_LOCKER_PREFIX}{topic}") == 1 - else: - return False - - @staticmethod - def _lock_consumer_group(redis_client: Redis, consumer_group_id: str, - ignore_exception: bool = False) -> bool: - """ - - 给某个消费组加锁，防止并发场景下重复操作问题 - - Args: - redis_client: - consumer_group_id: - ignore_exception: - - Returns: - - """ - # 使用set给create/del操作加锁（防止并发场景下重复创建删除问题） - if redis_client.set( - f"{StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LOCKER_PREFIX}" - f"{consumer_group_id}", - consumer_group_id, nx=True, ex=10) == 0: - return raise_if_not_ignore(ignore_exception, - CecException( - f"Someone else is creating or" - f" deleting this consumer group." - )) - return True - - @staticmethod - def _unlock_consumer_group(redis_client: Redis, - consumer_group_id: str) -> bool: - """ - - 释放给某个消费组加的锁，应当和 lock_consumer_group 配套使用 - - Args: - redis_client: - consumer_group_id: - - Returns: - - """ - # 释放锁 - if redis_client.get( - f"{StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LOCKER_PREFIX}" - f"{consumer_group_id}" - ) == consumer_group_id: - return redis_client.delete( - f"{StaticConst.REDIS_ADMIN_CONSUMER_GROUP_LOCKER_PREFIX}" - f"{consumer_group_id}") == 1 - else: - return False - @logger.catch() def __del__(self): self.disconnect() @logger.catch(reraise=True) def client(self): + """Get inner redis client + + Returns: + + """ return self._redis_client diff --git a/sysom_api/sdk/cec_redis/redis_consumer.py b/sysom_api/sdk/cec_redis/redis_consumer.py index a7ccf787..2afe603a 100644 --- a/sysom_api/sdk/cec_redis/redis_consumer.py +++ b/sysom_api/sdk/cec_redis/redis_consumer.py @@ -1,95 +1,192 @@ # -*- coding: utf-8 -*- # """ -Time 2022/7/26 16:45 +Time 2022/8/15 16:45 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File redis_consumer.py Description: """ import json +from queue import Queue +from typing import List, Optional +import redis.exceptions +from redis import Redis +from loguru import logger from ..cec_base.consumer import Consumer, ConsumeMode from ..cec_base.event import Event from ..cec_base.url import CecUrl from ..cec_base.log import LoggerHelper -from redis import Redis from .utils import do_connect_by_cec_url from .redis_admin import RedisAdmin -from loguru import logger -from queue import Queue from .consume_status_storage import ConsumeStatusStorage from .common import StaticConst, ClientBase +from .admin_static import static_create_consumer_group, \ + get_topic_consumer_group_meta_info_key +from .heartbeat import Heartbeat +from .utils import RedisLocker, transfer_pending_list class RedisConsumer(Consumer, ClientBase): """A redis-based execution module implement of Consumer - 一个基于 Redis 实现的执行模块中的 Consumer 实现 + Consumer implementation in an execution module based on the Redis. """ - def __init__(self, url: CecUrl, topic_name: str, consumer_id: str = "", - group_id: str = "", start_from_now: bool = True, - default_batch_consume_limit: int = 10): - Consumer.__init__(self, topic_name, consumer_id, group_id, - start_from_now, default_batch_consume_limit) + # pylint: disable=too-many-instance-attributes + # Eight is reasonable in this case. + def __init__(self, url: CecUrl, **kwargs): + Consumer.__init__(self, **kwargs) ClientBase.__init__(self, url) - # 特化参数1：pending_expire_time => pending 消息超时时间 - # - 在pending列表中超过指定时间的消息，当前消费者会尝试将其获取下来消费 - self.pending_expire_time = self.get_special_param( + # Specialized parameter 1: cec_auto_transfer_interval + self._enable_pending_list_transfer = self.get_special_param( + StaticConst.REDIS_SPECIAL_PARM_CEC_ENABLE_PENDING_LIST_TRANSFER + ) + # Specialized parameter 2: cec_pending_expire_time + self._pending_expire_time = self.get_special_param( StaticConst.REDIS_SPECIAL_PARM_CEC_PENDING_EXPIRE_TIME ) + # Specialized parameter 3: cec_enable_heartbeat + self._enable_heartbeat = self.get_special_param( + StaticConst.REDIS_SPECIAL_PARM_CEC_ENABLE_HEART_BEAT + ) + + # Specialized parameter 4: cec_heartbeat_interval + self._heartbeat_interval = self.get_special_param( + StaticConst.REDIS_SPECIAL_PARM_CEC_HEARTBEAT_INTERVAL + ) + self._current_url = "" - self._redis_client: Redis = None - self.connect_by_cec_url(url) - self._last_event_id: str = None # 最近一次消费的ID - self._message_cache_queue = Queue() # 消息缓存队列 + self._redis_client: Optional[Redis] = None + self._heartbeat: Optional[Heartbeat] = None + self._last_event_id: Optional[str] = None # Last consume ID + self._event_cache_queue = Queue() # Event cache queue self.consume_status_storage = None self.inner_topic_name = StaticConst.get_inner_topic_name( - topic_name) + self.topic_name) - # 如果是组消费模式，检查消费组是否存在 - if self.consume_mode == ConsumeMode.CONSUME_GROUP: - # 尝试创建消费组，如果已经存在就忽略，如果不存在则创建 - RedisAdmin.static_create_consumer_group(self._redis_client, - group_id, - True) - self.consume_status_storage = ConsumeStatusStorage( - self._redis_client, - topic_name, - group_id + # Identifies by this field whether it is a message that needs to be + # pulled from the pending list + self._is_need_fetch_pending_message = True + self.connect_by_cec_url(url) + + def _pending_list_transfer(self, batch_consume_limit: int): + """Do pending list transfer + + Check if there are any messages in the pending list of the same + group that have not been acknowledged for a long time, and if so, + try to transfer them to the current consumer for processing + + Returns: + + """ + _message_ret = [[[], [], []]] + # First process pending list transfers + # 1. Try to filter out events that have not been acked for a long + # time from the pending list of the consumer group as a whole, + # i.e. events that have been in the pending list for longer + # than 'pending_expire_time'; + # 2. Then transfer the overdue events to the current consumer for + # processing. + if self.is_gte_6_2(self._redis_client): + # Redis versions greater than or equal to 6.2 support the use + # of 'xautoclaim' to merge 'xpending + xclaim' operations, so + # it is straightforward to use 'xautoclaim' to attempt to + # transfer overdue messages from the global pending list of the + # consumer group to the current consumer + _message_ret = [ + self._redis_client.xautoclaim( + self.inner_topic_name, self.group_id, + self.consumer_id, + min_idle_time=self._pending_expire_time, + count=batch_consume_limit, + ) + ] + else: + # If Redis version is less than 6.2, 'xautoclaim' is not + # supported, so you need to use 'xpending + xclaim ' to try to + # transfer the overdue message from the pending list of the + # consumer group global to the current consumer + max_range = '+' if self._last_event_id is None else \ + self._last_event_id + _message_ret = transfer_pending_list( + self._redis_client, self.inner_topic_name, self.group_id, + batch_consume_limit, self.consumer_id, + min_id="-", max_id=max_range, + min_idle_time=self._pending_expire_time ) + return _message_ret - # 通过本字段标识是否是需要拉取 pending 列表中的消息 - self._is_need_fetch_pending_message = True + def _heartbeat_checkout(self, batch_consume_limit: int): + """Offline consumer detect + + Check if there is an offline consumer in the group and if so try to + transfer its unprocessed events to the current consumer + """ + _message_ret, transfer_count = [[[], [], []]], 0 + if not self._enable_heartbeat or self._heartbeat is None: + return _message_ret + next_consumer = self._heartbeat.get_next_offline_consumer() + while next_consumer is not None: + with RedisLocker( + self._redis_client, + f"{StaticConst.REDIS_HEARTBEAT_LOCKER_PREFIX}" + f"{self.group_id}", + ) as result: + if not result: + continue + _message_ret = transfer_pending_list( + self._redis_client, self.inner_topic_name, + self.group_id, + batch_consume_limit, self.consumer_id, + min_id="-", max_id="+", + min_idle_time=0, filter_consumer=next_consumer + ) + transfer_count = len(_message_ret[0][1]) + if transfer_count < batch_consume_limit: + self._heartbeat.remove_consumer(next_consumer) + if transfer_count > 0: + break + next_consumer = self._heartbeat.get_next_offline_consumer() + return _message_ret @logger.catch(reraise=True) def consume(self, timeout: int = -1, auto_ack: bool = False, - batch_consume_limit: int = 0) -> [Event]: - """Consume some event from cec + batch_consume_limit: int = 0, **kwargs) -> List[Event]: + """Consuming events from the Event Center - 从事件中心尝试消费一组事件 + Start to consume the event from event center according to the + corresponding ConsumeMode Args: - timeout(int): 超时等待时间（单位：ms），<=0 表示阻塞等待 - auto_ack(bool): 是否开启自动确认（组消费模式有效） - - 1. 一旦开启自动确认，每成功读取到一个事件消息就会自动确认； - 2. 调用者一定要保证消息接收后正常处理，因为一旦某个消息被确认，消息中心不保证下次 - 仍然可以获取到该消息，如果客户端在处理消息的过程中奔溃，则该消息或许无法恢复； - 3. 所以最保险的做法是，auto_ack = False 不开启自动确认，在事件被正确处理完 - 后显示调用 Consumer.ack() 方法确认消息被成功处理; - 4. 如果有一些使用组消费业务，可以承担事件丢失无法恢（只会在客户端程序奔溃没有正确 - 处理的情况下才会发生）的风险，则可以开启 auto_ack 选项。 - - batch_consume_limit(int): 批量消费限制 - - 1. 该参数指定了调用 consume 方法，最多一次拉取的事件的数量； - 2. 如果该值 <= 0 则将采用 self.default_batch_consume_limit 中指定的缺省 - 值； - 3. 如果该值 > 0 则将覆盖 self.default_batch_consume_limit，以本值为准。 + timeout(int): Blocking wait time + (Negative numbers represent infinite blocking wait) + auto_ack(bool): Whether to enable automatic confirmation + (valid for group consumption mode) + + 1. Once automatic acknowledgement is turned on, every event + successfully read will be automatically acknowledged; + 2. Caller must ensure that the event is processed properly + after it is received, because once a event is acknowledged, + the event center does not guarantee that the event will + still be available next time, and if the client runs down + while processing the message, the message may not be + recoverable; + 3. So it is safest to leave auto_ack = False and explicitly + call the Consumer.ack() method to acknowledge the event + after it has been processed correctly; + + batch_consume_limit(int): Batch consume limit + + 1. This parameter specifies the number of events to be pulled + at most once by calling the consume method; + 2. If the value <= 0 then the default value specified in + self.default_batch_consume_limit will be used； + 3. If this value > 0 then it will override + self.default_batch_consume_limit, use current passed value. Returns: [Message]: The Event list @@ -102,113 +199,93 @@ class RedisConsumer(Consumer, ClientBase): ... , start_from_now=False) >>> consumer.consume(200, auto_ack=False, batch_consume_limit=20) """ - if timeout <= 0: - timeout = 0 - batch_consume_limit = self.default_batch_consume_limit if \ - batch_consume_limit <= 0 else batch_consume_limit - LoggerHelper.get_lazy_logger().debug( - f"{self} try to consume one message from " - f"{self.topic_name} in {self.consume_mode}.") - - if self.consume_mode == ConsumeMode.CONSUME_GROUP: - message_ret = [[[], [], []]] + def group_consume(): + """Group consumption mode""" + _message_ret = [[[], [], []]] if self._last_event_id is None: - # 确保消费组存在 + # Ensuring the presence of consumption groups RedisAdmin.add_group_to_stream( self._redis_client, self.topic_name, self.group_id) - # 首先处理 pending list transfer - # 1. 尝试从消费组整体的 pending list 中过滤出长时间未 ACK 的事件，即在 - # pending list 中停留时间超过 'pending_expire_time' 的事件； - # 2. 并将超期的事件 transfer 到当前消费者进行处理 - if self.is_gte_6_2(self._redis_client): - # Redis 版本大于等于 6.2 支持使用 xautoclaim 来合并 xpending + xclaim 操 - # 作，因此直接使用 xautoclaim 即可 - # 尝试从消费组全局的 pending list transfer 超期的消息到当前消费者 - message_ret = [ - self._redis_client.xautoclaim( - self.inner_topic_name, self.group_id, - self.consumer_id, - min_idle_time=self.pending_expire_time, - count=batch_consume_limit - ) - ] - else: - # 如果Redis版本小于 6.2，则不支持 xautoclaim，需要使用 xpending + xclaim - # 尝试从消费组全局的 pending list transfer 超期的消息到当前消费者 - pending_list = self._redis_client.xpending_range( - self.inner_topic_name, self.group_id, - min='-', - max='+' if self._last_event_id is None else self._last_event_id, - count=batch_consume_limit, - ) - if len(pending_list) > 0: - pending_list = list(filter( - lambda item: item.get('time_since_delivered', - 0) > self.pending_expire_time, - pending_list - )) - pending_ids = list(map( - lambda item: item.get('message_id', '0-0'), - pending_list - )) - if len(pending_ids) > 0: - message_ret = [[[], self._redis_client.xclaim( - self.inner_topic_name, self.group_id, - self.consumer_id, self.pending_expire_time, - pending_ids - )]] - - if len(message_ret[0][1]) <= 0: - # 判断是否需要从 pending list 拉取消息 - # 1. 实例创建后，第一次消费会尝试获取 pending 列表的消息，即当前消费者（由 - # consumer_id 区分不同的消费者，使用相同的 consumer_id 创建的RedisConsumer - # 实例表征的是相同的消费者）从事件中心拉取了，但是没确认的消息列表； - # 2. 考虑到没确认的消息列表可能较多，一次拉取不完，所以如果成功从 pending - # list 拉到消息则 _is_need_fetch_pending_message 保持不变，下次仍然尝 - # 试从 pending list 继续拉取消息； - # 3. 如果从 pending list 没有拉取到消息，则将 _is_need_fetch_pending_message - # 设置为 False，本 RedisConsumer 之后将不会尝试从 pending list 拉取消息 + if self._enable_heartbeat: + _message_ret = self._heartbeat_checkout(batch_consume_limit) + + if self._enable_pending_list_transfer: + _message_ret = self._pending_list_transfer(batch_consume_limit) + + if len(_message_ret[0][1]) <= 0: + # Determine whether the message needs to be pulled from the + # pending list + # 1. After the instance is created, the first consumption will + # try to get the message of the pending list, that is, the + # current consumer (consumer_id distinguishes different + # consumers, and the RedisConsumer instance created with the + # same consumer_id represents The same consumer) pulled the + # message list from the event center, but the unconfirmed + # message list; + # 2. Considering that there may be more unconfirmed message + # lists, you can't pull it all at once, so if you + # successfully pull from the pending list to the message, + # '_is_need_fetch_pending_message' will remain unchanged, + # and try to get from pending list next time. + # 3. If the message is not pulled from the pending list, set + # '_is_need_fetch_pending_message' to False, and this + # RedisConsumer will not attempt from pending List pull + # message if self._is_need_fetch_pending_message: - message_ret = self._redis_client.xreadgroup( + _message_ret = self._redis_client.xreadgroup( self.group_id, self.consumer_id, { self.inner_topic_name: '0-0' }, count=batch_consume_limit, block=timeout, noack=auto_ack ) - if len(message_ret[0][1]) == 0: + if len(_message_ret[0][1]) == 0: self._is_need_fetch_pending_message = False - message_ret = self._redis_client.xreadgroup( + _message_ret = self._redis_client.xreadgroup( self.group_id, self.consumer_id, { self.inner_topic_name: '>' }, count=batch_consume_limit, block=timeout, noack=auto_ack ) else: - # 组消费模式单独处理 - message_ret = self._redis_client.xreadgroup( + _message_ret = self._redis_client.xreadgroup( self.group_id, self.consumer_id, { self.inner_topic_name: '>' }, count=batch_consume_limit, block=timeout, noack=auto_ack ) - # 更新状态，执行必要的清除任务 + # Update status and perform necessary clearance tasks self.consume_status_storage.update() - else: - # 下面处理扇形广播消费 + return _message_ret + + def broadcast_consume(): + """Broad consumption mode""" if self._last_event_id is None: - # 表示自从这个 Consumer 被实例化后第一次调用消费方法，做一些初始化操作 - message_ret = self._redis_client.xread({ - self.inner_topic_name: '$' if self.consume_mode == - ConsumeMode.CONSUME_FROM_NOW else '0-0' - }, count=batch_consume_limit, block=timeout) - else: - # 按序依次取出消息 - message_ret = self._redis_client.xread({ - self.inner_topic_name: self._last_event_id + # Indicates the first call to the consumer method since the + # Consumer was instantiated, doing some initialization + return self._redis_client.xread({ + self.inner_topic_name: '$' + if self.consume_mode == + ConsumeMode.CONSUME_FROM_NOW else '0-0' }, count=batch_consume_limit, block=timeout) + # Take out the messages in sequential order + return self._redis_client.xread({ + self.inner_topic_name: self._last_event_id + }, count=batch_consume_limit, block=timeout) + + timeout = 0 if timeout <= 0 else timeout + batch_consume_limit = self.default_batch_consume_limit if \ + batch_consume_limit <= 0 else batch_consume_limit + + LoggerHelper.get_lazy_logger().debug( + f"{self} try to consume one message from " + f"{self.topic_name} in {self.consume_mode}.") + if self.consume_mode == ConsumeMode.CONSUME_GROUP: + message_ret = group_consume() + else: + message_ret = broadcast_consume() if len(message_ret) < 1 or len(message_ret[0]) < 2 or len( message_ret[0][1]) < 1: LoggerHelper.get_lazy_logger().warning( @@ -216,16 +293,18 @@ class RedisConsumer(Consumer, ClientBase): f"{self.topic_name}, but its invalid. => " f"{message_ret}") return [] - messages: [Event] = [] + messages: List[Event] = [] for message_tuple in message_ret[0][1]: self._last_event_id = message_tuple[0] - # 过滤掉不是通过 cec 接口投递的事件 + # Filter out events that are not cast through the cec interface if StaticConst.REDIS_CEC_EVENT_VALUE_KEY not in message_tuple[1]: continue - message_content = json.loads( - message_tuple[1][StaticConst.REDIS_CEC_EVENT_VALUE_KEY]) + message_content = message_tuple[1][ + StaticConst.REDIS_CEC_EVENT_VALUE_KEY] + if self.auto_convert_to_dict: + message_content = json.loads(message_content) msg = Event(message_content, message_tuple[0]) messages.append(msg) LoggerHelper.get_lazy_logger().debug( @@ -234,15 +313,19 @@ class RedisConsumer(Consumer, ClientBase): return messages @logger.catch(reraise=True) - def ack(self, event: Event) -> int: + def ack(self, event: Event, **kwargs) -> int: """Confirm that the specified event has been successfully consumed - 事件确认，在接收到事件并成功处理后调用本方法确认 + Acknowledgement of the specified event + 1. The event should normally be acknowledged after it has been taken + out and successfully processed. Args: - event(Event): 要确认的事件 - 1. 必须是通过 Consumer 消费获得的 Event 实例； - 2. 自行构造的 Event 传递进去不保证结果符合预期 + event(Event): Events to be confirmed + 1. Must be an instance of the Event obtained through Consumer + interface; + 2. Passing in a self-constructed Event does not guarantee that + the result will be as expected. Returns: int: 1 if successfully, 0 otherwise @@ -260,23 +343,23 @@ class RedisConsumer(Consumer, ClientBase): LoggerHelper.get_lazy_logger().debug( f"{self} try to ack => {event.event_id}" ) - # 使用流水线来加速 - # 1. 记录当前主题-消费组最新确认的ID - pl = self._redis_client.pipeline() - key = RedisAdmin.get_topic_consumer_group_meta_info_key( + # Use pipeline to speed up + # 1. Record the latest acked ID of the current topic-consumer group + pipeline = self._redis_client.pipeline() + key = get_topic_consumer_group_meta_info_key( self.topic_name, self.group_id, StaticConst.TOPIC_CONSUMER_GROUP_META_KEY_LAST_ACK_ID) - pl.set( + pipeline.set( key, event.event_id ) - # 2. 对事件进行确认 - pl.xack(self.inner_topic_name, self.group_id, event.event_id) + # 2. Acknowledgement of the event + pipeline.xack(self.inner_topic_name, self.group_id, event.event_id) - # 3. 记录确认的ID - self.consume_status_storage.do_after_ack_by_pl(pl, event) - rets = pl.execute() + # 3. Record acked ID + self.consume_status_storage.do_after_ack_by_pl(pipeline, event) + rets = pipeline.execute() LoggerHelper.get_lazy_logger().info( f"{self} ack '{event.event_id}' successfully" ) @@ -286,22 +369,25 @@ class RedisConsumer(Consumer, ClientBase): def __getitem__(self, item): msg = None try: - if not self._message_cache_queue.empty(): - msg = self._message_cache_queue.get() + if not self._event_cache_queue.empty(): + msg = self._event_cache_queue.get() else: for new_msg in self.consume(): - self._message_cache_queue.put(new_msg) - if not self._message_cache_queue.empty(): - msg = self._message_cache_queue.get() - except Exception as e: - raise StopIteration() + self._event_cache_queue.put(new_msg) + if not self._event_cache_queue.empty(): + msg = self._event_cache_queue.get() + except redis.exceptions.ConnectionError: + pass + finally: + if msg is None: + raise StopIteration() return msg @logger.catch(reraise=True) def connect_by_cec_url(self, url: CecUrl): """Connect to redis server by CecUrl - 通过 CecUrl 连接到 Redis 服务器 + Connecting to the Redis server via CecUrl Args: url(str): CecUrl @@ -309,7 +395,37 @@ class RedisConsumer(Consumer, ClientBase): LoggerHelper.get_lazy_logger().debug( f"{self} try to connect to '{url}'.") self._redis_client = do_connect_by_cec_url(url) - self._current_url = url.__str__() + self._current_url = str(url) + + # If it is group consumption mode, check if the consumer group exists + if self.consume_mode == ConsumeMode.CONSUME_GROUP: + # Try to create a consumer group, ignore if it already exists, + # create if it doesn't + static_create_consumer_group(self._redis_client, + self.group_id, + ignore_exception=True) + RedisAdmin.add_group_to_stream( + self._redis_client, self.topic_name, self.group_id) + self.consume_status_storage = ConsumeStatusStorage( + self._redis_client, + self.topic_name, + self.group_id + ) + else: + self._enable_heartbeat = False + + if self._enable_heartbeat: + self._heartbeat = Heartbeat( + self._redis_client, self.inner_topic_name, self.group_id, + self.consumer_id, + heartbeat_interval=self.get_special_param( + StaticConst.REDIS_SPECIAL_PARM_CEC_HEARTBEAT_INTERVAL + ), + heartbeat_check_interval=self.get_special_param( + StaticConst.REDIS_SPECIAL_PARM_CEC_HEARTBEAT_CHECK_INTERVAL + ), + ) + self._heartbeat.start() LoggerHelper.get_lazy_logger().success( f"{self} connect to '{url}' successfully.") return self @@ -318,7 +434,8 @@ class RedisConsumer(Consumer, ClientBase): def connect(self, url: str): """Connect to redis server by url - 连接到远端的消息中间件 => 对应到本模块就是连接到 Redis 服务器 + Connecting to the remote message queue => Corresponding to this module + is connecting to the Redis server. Args: url(str): CecUrl @@ -334,15 +451,18 @@ class RedisConsumer(Consumer, ClientBase): def disconnect(self): """Disconnect from redis server - 断开连接 => 对应到本模块就是断开 Redis 服务器连接 + Disconnect from remote server => Corresponds to this module as + disconnecting the Redis server. """ if self._redis_client is None: return LoggerHelper.get_lazy_logger().debug( f"{self} try to disconnect from '{self._current_url}'.") - self._redis_client.quit() - self._redis_client.connection_pool.disconnect() + if self._heartbeat is not None: + self._heartbeat.stop() + self._heartbeat = None self._redis_client.close() + self._redis_client.connection_pool.disconnect() self._redis_client = None LoggerHelper.get_lazy_logger().success( f"{self} disconnect from '{self._current_url}' successfully.") diff --git a/sysom_api/sdk/cec_redis/redis_producer.py b/sysom_api/sdk/cec_redis/redis_producer.py index 5c936519..df261708 100644 --- a/sysom_api/sdk/cec_redis/redis_producer.py +++ b/sysom_api/sdk/cec_redis/redis_producer.py @@ -1,30 +1,31 @@ # -*- coding: utf-8 -*- # """ -Time 2022/7/26 18:32 +Time 2022/8/11 14:30 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File redis_producer.py Description: """ import json -from typing import Callable +from typing import Callable, Union, Optional +from redis import Redis +from loguru import logger from ..cec_base.producer import Producer from ..cec_base.event import Event from ..cec_base.url import CecUrl -from ..cec_base.admin import TopicNotExistsException +from ..cec_base.exceptions import TopicNotExistsException from ..cec_base.log import LoggerHelper -from redis import Redis from .utils import do_connect_by_cec_url from .redis_admin import RedisAdmin -from loguru import logger from .common import StaticConst, ClientBase +from .admin_static import static_create_topic class RedisProducer(Producer, ClientBase): """A redis-based execution module implement of Producer - 一个基于 Redis 实现的执行模块中的 Producer 实现 + Producer implementation in an execution module based on the Redis. """ @@ -33,7 +34,8 @@ class RedisProducer(Producer, ClientBase): ClientBase.__init__(self, url) self._current_url = "" - # 处理 Redis 实现的事件中心的特化参数 + # Handles Redis implementation of event-centric specialization + # parameters self.default_max_len = self.get_special_param( StaticConst.REDIS_SPECIAL_PARM_CEC_DEFAULT_MAX_LEN ) @@ -41,33 +43,39 @@ class RedisProducer(Producer, ClientBase): StaticConst.REDIS_SPECIAL_PARM_CEC_AUTO_MK_TOPIC ) - # 1. 首先连接到 Redis 服务器 - self._redis_client: Redis = None + # 1. Connect to the Redis server + self._redis_client: Optional[Redis] = None self.connect_by_cec_url(url) - # 2. 新建一个 dict，用于保存 topic_name => TopicMeta 的映射关系 + # 2. Create a new dict to hold the topic_name => TopicMeta mapping + # relationship self._topic_metas = { } @logger.catch(reraise=True) - def produce(self, topic_name: str, message_value: dict, + def produce(self, topic_name: str, message_value: Union[bytes, dict], callback: Callable[[Exception, Event], None] = None, - partition: int = -1, **kwargs): """Generate one new event, then put it to event center 发布一个事件到事件中心 => 对应到 Redis 就是生产一个消息注入到 Stream 当中 Args: - topic_name: 主题名称 - message_value: 事件内容 - callback(Callable[[Exception, Event], None]): 事件成功投递到事件中心回调 - partition(int): 分区号 - 1. 如果指定了有效分区号，消息投递给指定的分区（不建议）； - 2. 传递了一个正数分区号，但是无此分区，将抛出异常； - 3. 传递了一个负数分区号（比如-1），则消息将使用内建的策略均衡的投 - 递给所有的分区（建议）。 + topic_name(str): Topic name + message_value(bytes | dict): Event value + callback(Callable[[Exception, Event], None]): Event delivery + results callback + + Keyword Args + partition(int): Partition ID + 1. If a valid partition number is specified, the event is + deliverd to the specified partition (not recommended); + 2. A positive partition ID is passed, but no such partition is + available, an exception will be thrown. + 3. A negative partition number is passed (e.g. -1), then the + event will be cast to all partitions in a balanced manner + using the built-in policy (recommended). Examples: >>> producer = dispatch_producer( @@ -80,18 +88,20 @@ class RedisProducer(Producer, ClientBase): topic_exist = False inner_topic_name = StaticConst.get_inner_topic_name(topic_name) - # 判断是否有目标主题的元数据信息 + # Determine whether there is metadata information for the target + # topic if inner_topic_name not in self._topic_metas or \ self._topic_metas[inner_topic_name] is None: - # 拉取元数据信息 + # Pulling metadata information self._topic_metas[inner_topic_name] = RedisAdmin.get_meta_info( self._redis_client, topic_name) - # 如果元数据信息无效，说明主题不存在 + # If the metadata information is invalid, the topic does not exist if self._topic_metas[inner_topic_name] is None: if self.auto_mk_topic: - # 如果设置了主题不存在时自动创建，则尝试创建主题 - topic_exist = RedisAdmin.static_create_topic( + # If you set the theme to be created automatically if it + # does not exist, try to create the theme + topic_exist = static_create_topic( self._redis_client, topic_name) LoggerHelper.get_lazy_logger().debug( @@ -102,40 +112,49 @@ class RedisProducer(Producer, ClientBase): else: topic_exist = True - e, event_id = None, None + err, event_id = None, None if not topic_exist: LoggerHelper.get_lazy_logger().error( f"{self} Topic ({topic_name}) not exists.") - # Topic 不存在 - e = TopicNotExistsException( + # Topic not exists + err = TopicNotExistsException( f"Topic ({topic_name}) not exists.") else: - # 将消息放到对应的 topic 中 + # Deliver the message in the corresponding topic if 'maxlen' not in kwargs: kwargs['maxlen'] = self.default_max_len event_id = self._redis_client.xadd(inner_topic_name, { StaticConst.REDIS_CEC_EVENT_VALUE_KEY: json.dumps( - message_value) + message_value) if isinstance(message_value, + dict) else message_value }, **kwargs) - # 主题不存在则额外处理 + # Additional processing if topice does not exist if event_id is None: - e = TopicNotExistsException( + err = TopicNotExistsException( f"Topic ({topic_name}) not exists.") else: LoggerHelper.get_lazy_logger().info( - f"{self} produce one message '{event_id}'=>{message_value} " - f"successfully." + f"{self} produce one message '{event_id}'=>" + f"{message_value} successfully." ) if callback is not None: - callback(e, Event(message_value, event_id)) + callback(err, Event(message_value, event_id)) @logger.catch(reraise=True) - def flush(self, timeout: int = -1): + def flush(self, timeout: int = -1, **kwargs): """Flush all cached event to event center - TODO: 目前 RedisProducer 的produce实现为阻塞，所以 flush 实现可以为空 + Deliver all events in the cache that have not yet been committed into + the event center (this is a blocking call) + + Args: + timeout(int): Blocking wait time + (Negative numbers represent infinite blocking wait) + + Notes: The RedisProducer's produce func is currently blocking, so the + flush func can be empty Examples: >>> producer = dispatch_producer( @@ -143,13 +162,12 @@ class RedisProducer(Producer, ClientBase): >>> producer.produce("test_topic", {"value": "hhh"}) >>> producer.flush() """ - pass @logger.catch(reraise=True) def connect_by_cec_url(self, url: CecUrl): """Connect to redis server by CecUrl - 通过 CecUrl 连接到 Redis 服务器 + Connecting to the Redis server via CecUrl Args: url(str): CecUrl @@ -157,7 +175,7 @@ class RedisProducer(Producer, ClientBase): LoggerHelper.get_lazy_logger().debug( f"{self} try to connect to '{url}'.") self._redis_client = do_connect_by_cec_url(url) - self._current_url = url.__str__() + self._current_url = str(url) LoggerHelper.get_lazy_logger().success( f"{self} connect to '{url}' successfully.") return self @@ -166,7 +184,8 @@ class RedisProducer(Producer, ClientBase): def connect(self, url: str): """Connect to redis server by url - 连接到远端的消息中间件 => 对应到本模块就是连接到 Redis 服务器 + Connecting to the remote message queue => Corresponding to this module + is connecting to the Redis server. Args: url(str): CecUrl @@ -182,7 +201,9 @@ class RedisProducer(Producer, ClientBase): def disconnect(self): """Disconnect from redis server - 断开连接 => 对应到本模块就是断开 Redis 服务器连接 + Disconnect from remote server => Corresponds to this module as + disconnecting the Redis server. + """ if self._redis_client is None: return diff --git a/sysom_api/sdk/cec_redis/utils.py b/sysom_api/sdk/cec_redis/utils.py index f9ac44f1..b9c70bc4 100644 --- a/sysom_api/sdk/cec_redis/utils.py +++ b/sysom_api/sdk/cec_redis/utils.py @@ -1,31 +1,156 @@ # -*- coding: utf-8 -*- # """ -Time 2022/7/26 17:04 +Time 2022/7/29 11:28 Author: mingfeng (SunnyQjm) Email mfeng@linux.alibaba.com File utils.py Description: """ from redis import Redis +import redis from ..cec_base.url import CecUrl -from ..cec_base.base import ConnectException -from redis.exceptions import ConnectionError +from ..cec_base.exceptions import CecConnectionException +from ..cec_base.log import LoggerHelper def do_connect(url: str) -> Redis: + """Connect to remote redis by url + + Args: + url(str): CecUrl format url + + Returns: + + """ cec_url = CecUrl.parse(url) return do_connect_by_cec_url(cec_url) def do_connect_by_cec_url(cec_url: CecUrl) -> Redis: + """Connect to remote redis by CecUrl + + Args: + cec_url(CecUrl): + + Returns: + + """ host_port = cec_url.netloc.split(":") if len(host_port) != 2: - raise ConnectException( + raise CecConnectionException( f"Not valid host:port => {host_port[0]}:{host_port[1]}") host, port = host_port[0], int(host_port[1]) try: redis_client = Redis(host=host, port=port, db=0, decode_responses=True, **cec_url.params) - except ConnectionError as e: - raise ConnectException(e) + except redis.ConnectionError as exc: + raise CecConnectionException(exc) from exc return redis_client + + +def raise_if_not_ignore(is_ignore_exception: bool, exception: Exception): + """Raise or ignore for specific exception + + Args: + is_ignore_exception: Is ignore exception while `exception` be raised. + exception: The exception want to check + """ + if is_ignore_exception: + # If you choose to ignore the exception, the ignored exception is + # logged in the log as an exception + LoggerHelper.get_lazy_logger().exception(exception) + return False + raise exception + + +class RedisLocker: + """ + This is a simple redis lock implement + + Args: + redis_client(Redis): Redis client + key(str): Locker key + ex(int): Expire time, seconds + """ + + def __init__(self, redis_client: Redis, key: str, ex: int = 10): + self._redis_client = redis_client + self._key = key + self._ex = ex + self._get_locker = False + + def __enter__(self): + return self.lock() + + def __exit__(self, exc_type, exc_val, exc_tb): + return self.unlock() + + def unlock(self): + """Unlock + + Returns: + + """ + if self._get_locker and self._redis_client.get(self._key) == self._key: + return self._redis_client.delete(self._key) == 1 + return False + + def lock(self) -> bool: + """Lock + + Returns: + + """ + self._get_locker = self._redis_client.set( + self._key, self._key, nx=True, ex=self._ex + ) == 1 + return self._get_locker + + +def transfer_pending_list(redis_client: Redis, topic: str, group: str, + count: int, target_consumer: str, **kwargs): + """ + + Args: + redis_client(Redis): Redis client + topic(str): Topic + group(str): Consumer group + count(str): max count + target_consumer(str): + + Keyword Args: + min_id(str): Min ID + max_id(str): Max ID + min_idle_time(int): filter messages that were idle less than this + amount of milliseconds. + filter_consumer(str): name of a consumer to filter by (optional). + + Returns: + + """ + min_id = kwargs.get("min_id", "-") + max_id = kwargs.get("max_id", "+") + min_idle_time = kwargs.get("min_idle_time", 0) + filter_consumer = kwargs.get("filter_consumer", None) + + _message_ret = [[[], [], []]] + pending_list = redis_client.xpending_range( + topic, group, count=count, min=min_id, max=max_id, + consumername=filter_consumer + ) + if len(pending_list) > 0: + pending_list = list(filter( + lambda item: item.get('time_since_delivered', + 0) > min_idle_time, + pending_list + )) + pending_ids = list(map( + lambda item: item.get('message_id', '0-0'), + pending_list + )) + if len(pending_ids) > 0: + _message_ret = [[[], redis_client.xclaim( + topic, group, target_consumer, min_idle_time, + pending_ids + )]] + return _message_ret -- Gitee