From 0c3f409b75c87febb2a4b56711ec7a3ca655e5e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sat, 4 Jul 2026 16:29:58 +0800 Subject: [PATCH 01/86] =?UTF-8?q?=E5=88=9D=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitattributes | 1 + .gitignore | 9 +++++++++ .python-version | 1 + README.md | 0 pyproject.toml | 38 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 0 src/database/__init__.py | 6 ++++++ 7 files changed, 55 insertions(+) create mode 100644 .gitattributes create mode 100644 .gitignore create mode 100644 .python-version create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 src/database/__init__.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..fcadb2c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text eol=lf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..d08fcaf --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +.idea/ + +__pycache__/ + +/build/ +/src/*.egg-info/ +/src/database/_version.py + +/test.py diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..44677e5 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10.20 diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b8c57ab --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = [ + "setuptools>=61.0", + "wheel", + "setuptools-scm>=8.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "database" +dynamic = ["version"] +description = "database SDK" +readme = "README.md" +license = { text = "MIT" } +authors = [ + { name = "MSR", email = "msr20666@foxmail.com" }, +] +requires-python = ">=3.10" +keywords = [ + "database", + "MongoDB", + "MySQL", + "Redis", +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", +] +dependencies = [ + +] + +[project.urls] +Homepage = "https://gitee.com/loongspears/database" +Repository = "https://gitee.com/loongspears/database" + +[tool.setuptools_scm] +write_to = "src/database/_version.py" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/src/database/__init__.py b/src/database/__init__.py new file mode 100644 index 0000000..b3e50b1 --- /dev/null +++ b/src/database/__init__.py @@ -0,0 +1,6 @@ +try: + from ._version import __version__ +except ImportError: + __version__ = '0.0.0+unknown' + +__description__ = 'database SDK' -- Gitee From 26a28006c6c9d5afc37663c81496ddb329f8ee95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 5 Jul 2026 17:59:26 +0800 Subject: [PATCH 02/86] =?UTF-8?q?MongoDB=E6=8F=92=E5=85=A5=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 12 +++++ requirements.txt | 3 ++ src/database/mongodb/types.py | 30 +++++++++++ src/database/mongodb/utils.py | 91 ++++++++++++++++++++++++++++++++++ src/database/utils/__init__.py | 8 +++ src/database/utils/logger.py | 55 ++++++++++++++++++++ 6 files changed, 199 insertions(+) create mode 100644 src/database/mongodb/types.py create mode 100644 src/database/mongodb/utils.py create mode 100644 src/database/utils/__init__.py create mode 100644 src/database/utils/logger.py diff --git a/pyproject.toml b/pyproject.toml index b8c57ab..c6278fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,18 @@ classifiers = [ ] dependencies = [ +] +[project.optional-dependencies] +all = [ + "database[mongodb]", +] +mysql = [ +] +mongodb = [ + "pymongo>=4.6.0", + "motor>=3.3.0", +] +redis = [ ] [project.urls] diff --git a/requirements.txt b/requirements.txt index e69de29..fe37823 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,3 @@ +# MongoDB +pymongo>=4.6.0 +motor>=3.3.0 diff --git a/src/database/mongodb/types.py b/src/database/mongodb/types.py new file mode 100644 index 0000000..10b29c1 --- /dev/null +++ b/src/database/mongodb/types.py @@ -0,0 +1,30 @@ +"""MongoDB 常用类型. + +从 bson 重新导出,用户不需要 import bson。 +""" + +from bson import ( + ObjectId, + Binary, + Code, + DBRef, + Decimal128, + Int64, + MaxKey, + MinKey, + Regex, + Timestamp, +) + +__all__ = [ + "ObjectId", + "Binary", + "Code", + "DBRef", + "Decimal128", + "Int64", + "MaxKey", + "MinKey", + "Regex", + "Timestamp", +] diff --git a/src/database/mongodb/utils.py b/src/database/mongodb/utils.py new file mode 100644 index 0000000..7bfb389 --- /dev/null +++ b/src/database/mongodb/utils.py @@ -0,0 +1,91 @@ +import pymongo.errors +from pymongo import UpdateOne + +from ..utils.logger import get_logger + +logger = get_logger(__name__) + + +class Batch: + + def __init__(self, collection, step=1000, ordered=True, on_error='raise'): + """初始化批量操作. + + Args: + collection: pymongo Collection 对象 + step: 批次大小,默认 1000 + ordered: 是否有序操作,默认 True + on_error: 错误处理策略,默认 'raise' + - 'raise': 抛出异常(默认) + - 'log': 记录日志但继续执行 + - 'ignore': 忽略错误 + """ + self.ops = [] + self.count = 0 + self.collection = collection + self.step = step + self.ordered = ordered + self.on_error = on_error + + def add(self, doc_or_op): + self.ops.append(doc_or_op) + self.count += 1 + if self.count >= self.step: + self.execute() + + def extend(self, docs_or_ops): + self.ops.extend(docs_or_ops) + self.count += len(docs_or_ops) + if self.count > self.step: + self.execute() + + def execute(self): + pass + + def close(self): + if self.count: + self.execute() + + +class BatchInsert(Batch): + + def execute(self): + try: + self.collection.insert_many(self.ops, ordered=self.ordered) + except pymongo.errors.BulkWriteError as error: + if self.on_error == 'raise': + raise + elif self.on_error == 'log': + logger.warning(f'BatchInsert error: {error.details}') + finally: + self.ops.clear() + self.count = 0 + + +class BatchUpdate(Batch): + def add(self, query_or_op, update=None): + if isinstance(query_or_op, UpdateOne): + super().add(query_or_op) + elif update: + super().add(UpdateOne(query_or_op, update)) + else: + raise pymongo.errors.InvalidOperation('Missing update parameter.') + + def extend(self, ops): + for op in ops: + if isinstance(op, UpdateOne): + super().add(op) + else: + raise pymongo.errors.InvalidOperation('Operation type Error.') + + def execute(self): + try: + self.collection.bulk_write(self.ops, ordered=self.ordered) + except pymongo.errors.BulkWriteError as error: + if self.on_error == 'raise': + raise + elif self.on_error == 'log': + logger.warning(f"BatchUpdate error: {error.details}") + finally: + self.ops.clear() + self.count = 0 diff --git a/src/database/utils/__init__.py b/src/database/utils/__init__.py new file mode 100644 index 0000000..b3eef63 --- /dev/null +++ b/src/database/utils/__init__.py @@ -0,0 +1,8 @@ +"""工具函数模块.""" + +from .logger import get_logger, setup_logging + +__all__ = [ + "get_logger", + "setup_logging", +] diff --git a/src/database/utils/logger.py b/src/database/utils/logger.py new file mode 100644 index 0000000..4cfbde0 --- /dev/null +++ b/src/database/utils/logger.py @@ -0,0 +1,55 @@ +"""全项目统一的日志管理. + +提供统一的 logger 获取和配置功能。 +""" + +import logging +from typing import Optional + + +def get_logger(name: str) -> logging.Logger: + """获取指定名称的 logger. + + Args: + name: logger 名称,通常使用 __name__ + + Returns: + logging.Logger 实例 + + Example: + >>> from database.utils.logger import get_logger + >>> logger = get_logger(__name__) + >>> logger.info("Something happened") + """ + return logging.getLogger(name) + + +def setup_logging( + level: int = logging.WARNING, + format: Optional[str] = None +) -> None: + """配置全项目的日志. + + Args: + level: 日志级别,默认 WARNING + format: 日志格式,默认使用标准格式 + + Example: + >>> from database.utils.logger import setup_logging + >>> import logging + >>> setup_logging(level=logging.DEBUG) + """ + if format is None: + format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + + logging.basicConfig( + level=level, + format=format, + datefmt='%Y-%m-%d %H:%M:%S' + ) + + +__all__ = [ + "get_logger", + "setup_logging", +] -- Gitee From fbba3e196ad5dda92b8234f5a13cf1f8db5be200 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 5 Jul 2026 19:13:01 +0800 Subject: [PATCH 03/86] =?UTF-8?q?monodb=E8=BF=9E=E6=8E=A5=E5=AF=B9?= =?UTF-8?q?=E8=B1=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/__init__.py | 5 +++++ src/database/mongodb/client.py | 20 ++++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 src/database/mongodb/__init__.py create mode 100644 src/database/mongodb/client.py diff --git a/src/database/mongodb/__init__.py b/src/database/mongodb/__init__.py new file mode 100644 index 0000000..cdea1c3 --- /dev/null +++ b/src/database/mongodb/__init__.py @@ -0,0 +1,5 @@ +from .client import client + +__all__ = [ + 'client', +] diff --git a/src/database/mongodb/client.py b/src/database/mongodb/client.py new file mode 100644 index 0000000..7f0984a --- /dev/null +++ b/src/database/mongodb/client.py @@ -0,0 +1,20 @@ +from pymongo import MongoClient + + +# from motor.motor_asyncio import AsyncIOMotorClient + + +class Client: + + def __init__(self): + pass + + @staticmethod + def client(*args, **kwargs): + return MongoClient(*args, **kwargs) + + # async def client_async(self, *args, **kwargs): + # return AsyncIOMotorClient(**args, **kwargs) + + +client = Client().client -- Gitee From 7e506cd3023ad68d95e5ffc77dc6dffbec6ada10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 6 Jul 2026 10:47:13 +0800 Subject: [PATCH 04/86] =?UTF-8?q?=E8=A1=A5=E5=85=85=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/mongodb/client.py b/src/database/mongodb/client.py index 7f0984a..8151327 100644 --- a/src/database/mongodb/client.py +++ b/src/database/mongodb/client.py @@ -1,4 +1,4 @@ -from pymongo import MongoClient +import pymongo # from motor.motor_asyncio import AsyncIOMotorClient @@ -10,8 +10,8 @@ class Client: pass @staticmethod - def client(*args, **kwargs): - return MongoClient(*args, **kwargs) + def client(*args, **kwargs) -> pymongo.MongoClient: + return pymongo.MongoClient(*args, **kwargs) # async def client_async(self, *args, **kwargs): # return AsyncIOMotorClient(**args, **kwargs) -- Gitee From d219346f4fd54c7ab154bf59c2a2f56e7572fbfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 6 Jul 2026 10:49:44 +0800 Subject: [PATCH 05/86] =?UTF-8?q?redis=E5=AE=A2=E6=88=B7=E7=AB=AF=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 3 ++- requirements.txt | 3 +++ src/database/redis/__init__.py | 5 +++++ src/database/redis/client.py | 17 +++++++++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 src/database/redis/__init__.py create mode 100644 src/database/redis/client.py diff --git a/pyproject.toml b/pyproject.toml index c6278fd..c1bae66 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,7 @@ dependencies = [ ] [project.optional-dependencies] all = [ - "database[mongodb]", + "database[mongodb,redis]", ] mysql = [ ] @@ -40,6 +40,7 @@ mongodb = [ "motor>=3.3.0", ] redis = [ + "redis>=5.0.0", ] [project.urls] diff --git a/requirements.txt b/requirements.txt index fe37823..e9f807f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ # MongoDB pymongo>=4.6.0 motor>=3.3.0 + +# Redis +redis>=5.0.0 diff --git a/src/database/redis/__init__.py b/src/database/redis/__init__.py new file mode 100644 index 0000000..cdea1c3 --- /dev/null +++ b/src/database/redis/__init__.py @@ -0,0 +1,5 @@ +from .client import client + +__all__ = [ + 'client', +] diff --git a/src/database/redis/client.py b/src/database/redis/client.py new file mode 100644 index 0000000..f1ab5e8 --- /dev/null +++ b/src/database/redis/client.py @@ -0,0 +1,17 @@ +import redis + + +class Client: + + def __init__(self): + pass + + @staticmethod + def client(*args, **kwargs) -> redis.Redis: + return redis.Redis(*args, **kwargs) + + def client_async(*args, **kwargs) -> redis.asyncio.Redis: + return redis.asyncio.Redis(*args, **kwargs) + + +client = Client().client -- Gitee From 4fe8f5c1277ea1af722c8b02f08eb1420abeecdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 7 Jul 2026 10:17:13 +0800 Subject: [PATCH 06/86] =?UTF-8?q?redis=20cluster=E7=9A=84=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/redis/__init__.py | 2 ++ src/database/redis/client.py | 10 ++++++++++ 2 files changed, 12 insertions(+) diff --git a/src/database/redis/__init__.py b/src/database/redis/__init__.py index cdea1c3..2eeb2c5 100644 --- a/src/database/redis/__init__.py +++ b/src/database/redis/__init__.py @@ -1,5 +1,7 @@ from .client import client +from .client import cluster __all__ = [ 'client', + 'cluster', ] diff --git a/src/database/redis/client.py b/src/database/redis/client.py index f1ab5e8..e54b123 100644 --- a/src/database/redis/client.py +++ b/src/database/redis/client.py @@ -10,8 +10,18 @@ class Client: def client(*args, **kwargs) -> redis.Redis: return redis.Redis(*args, **kwargs) + @staticmethod def client_async(*args, **kwargs) -> redis.asyncio.Redis: return redis.asyncio.Redis(*args, **kwargs) + @staticmethod + def cluster(*args, **kwargs) -> redis.cluster.RedisCluster: + return redis.cluster.RedisCluster(*args, **kwargs) + + @staticmethod + def cluster_async(*args, **kwargs) -> redis.asyncio.RedisCluster: + return redis.asyncio.RedisCluster(*args, **kwargs) + client = Client().client +cluster = Client().cluster -- Gitee From afe97728059dc0068575c069f80ba40e7a93fab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 7 Jul 2026 10:24:53 +0800 Subject: [PATCH 07/86] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=9B=A0=E4=B8=BAclien?= =?UTF-8?q?t=E5=91=BD=E5=90=8D=E8=A6=86=E7=9B=96=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/__init__.py | 4 ++-- src/database/mongodb/client.py | 3 --- src/database/redis/__init__.py | 6 ++---- src/database/redis/client.py | 4 ---- 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/src/database/mongodb/__init__.py b/src/database/mongodb/__init__.py index cdea1c3..bcf7009 100644 --- a/src/database/mongodb/__init__.py +++ b/src/database/mongodb/__init__.py @@ -1,5 +1,5 @@ -from .client import client +from .client import Client __all__ = [ - 'client', + 'Client', ] diff --git a/src/database/mongodb/client.py b/src/database/mongodb/client.py index 8151327..ba45774 100644 --- a/src/database/mongodb/client.py +++ b/src/database/mongodb/client.py @@ -15,6 +15,3 @@ class Client: # async def client_async(self, *args, **kwargs): # return AsyncIOMotorClient(**args, **kwargs) - - -client = Client().client diff --git a/src/database/redis/__init__.py b/src/database/redis/__init__.py index 2eeb2c5..bcf7009 100644 --- a/src/database/redis/__init__.py +++ b/src/database/redis/__init__.py @@ -1,7 +1,5 @@ -from .client import client -from .client import cluster +from .client import Client __all__ = [ - 'client', - 'cluster', + 'Client', ] diff --git a/src/database/redis/client.py b/src/database/redis/client.py index e54b123..7bfc264 100644 --- a/src/database/redis/client.py +++ b/src/database/redis/client.py @@ -21,7 +21,3 @@ class Client: @staticmethod def cluster_async(*args, **kwargs) -> redis.asyncio.RedisCluster: return redis.asyncio.RedisCluster(*args, **kwargs) - - -client = Client().client -cluster = Client().cluster -- Gitee From 1ee576781ed9acec055541cd4193a5ba3dbb95ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 7 Jul 2026 11:39:55 +0800 Subject: [PATCH 08/86] =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E5=90=8D=E7=A7=B0=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/client.py | 2 +- src/database/redis/client.py | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/database/mongodb/client.py b/src/database/mongodb/client.py index ba45774..a6b86f7 100644 --- a/src/database/mongodb/client.py +++ b/src/database/mongodb/client.py @@ -10,7 +10,7 @@ class Client: pass @staticmethod - def client(*args, **kwargs) -> pymongo.MongoClient: + def create(*args, **kwargs) -> pymongo.MongoClient: return pymongo.MongoClient(*args, **kwargs) # async def client_async(self, *args, **kwargs): diff --git a/src/database/redis/client.py b/src/database/redis/client.py index 7bfc264..4cd8116 100644 --- a/src/database/redis/client.py +++ b/src/database/redis/client.py @@ -6,12 +6,20 @@ class Client: def __init__(self): pass + @classmethod + def create(cls, *args, **kwargs) -> redis.Redis: + return cls.single(*args, **kwargs) + + @classmethod + def create_async(cls, *args, **kwargs) -> redis.asyncio.Redis: + return cls.single_async(*args, **kwargs) + @staticmethod - def client(*args, **kwargs) -> redis.Redis: + def single(*args, **kwargs) -> redis.Redis: return redis.Redis(*args, **kwargs) @staticmethod - def client_async(*args, **kwargs) -> redis.asyncio.Redis: + def single_async(*args, **kwargs) -> redis.asyncio.Redis: return redis.asyncio.Redis(*args, **kwargs) @staticmethod -- Gitee From b2f32a64d3fff1819136715dd6349d8588888a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 7 Jul 2026 11:42:04 +0800 Subject: [PATCH 09/86] =?UTF-8?q?=E5=93=A8=E5=85=B5=E6=A8=A1=E5=BC=8F?= =?UTF-8?q?=E5=92=8C=E7=BB=9F=E4=B8=80=E5=85=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/redis/client.py | 50 ++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/src/database/redis/client.py b/src/database/redis/client.py index 4cd8116..a6303b3 100644 --- a/src/database/redis/client.py +++ b/src/database/redis/client.py @@ -23,9 +23,55 @@ class Client: return redis.asyncio.Redis(*args, **kwargs) @staticmethod - def cluster(*args, **kwargs) -> redis.cluster.RedisCluster: - return redis.cluster.RedisCluster(*args, **kwargs) + def sentinel(*args, **kwargs) -> redis.Sentinel: + return redis.Sentinel(*args, **kwargs) + + @staticmethod + def sentinel_async(*args, **kwargs) -> redis.asyncio.Sentinel: + return redis.asyncio.Sentinel(*args, **kwargs) + + @staticmethod + def cluster(*args, **kwargs) -> redis.RedisCluster: + return redis.RedisCluster(*args, **kwargs) @staticmethod def cluster_async(*args, **kwargs) -> redis.asyncio.RedisCluster: return redis.asyncio.RedisCluster(*args, **kwargs) + + @classmethod + def connect(cls, mode='single', is_async=False, *args, **kwargs): + """统一连接入口 + :param mode: single[default], sentinel, cluster; + "param mode type: str; + :param is_async: False[default]; + "param is_async type: bool; + """ + if mode == 'single': + return cls._connect_single(is_async, *args, **kwargs) + elif mode == 'sentinel': + return cls._connect_sentinel(is_async, *args, **kwargs) + elif mode == 'cluster': + return cls._connect_cluster(is_async, *args, **kwargs) + else: + raise ValueError(f"Invalid mode: '{mode}'. Expected one of: 'single', 'sentinel', 'cluster'") + + @classmethod + def _connect_single(cls, is_async, *args, **kwargs): + if is_async: + return cls.single_async(*args, **kwargs) + else: + return cls.single(*args, **kwargs) + + @classmethod + def _connect_sentinel(cls, is_async, *args, **kwargs): + if is_async: + return cls.sentinel_async(*args, **kwargs) + else: + return cls.sentinel(*args, **kwargs) + + @classmethod + def _connect_cluster(cls, is_async, *args, **kwargs): + if is_async: + return cls.cluster_async(*args, **kwargs) + else: + return cls.cluster(*args, **kwargs) -- Gitee From a15b3db6d644645177f05719a71f3314cbc58cdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 8 Jul 2026 17:39:13 +0800 Subject: [PATCH 10/86] =?UTF-8?q?=E5=BC=83=E7=94=A8mongodb=20motor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 3 +-- requirements.txt | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c1bae66..d189511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,8 +36,7 @@ all = [ mysql = [ ] mongodb = [ - "pymongo>=4.6.0", - "motor>=3.3.0", + "pymongo>=4.9.0", ] redis = [ "redis>=5.0.0", diff --git a/requirements.txt b/requirements.txt index e9f807f..9ca11e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ # MongoDB pymongo>=4.6.0 -motor>=3.3.0 # Redis redis>=5.0.0 -- Gitee From 72a148cc901972649b1107921d819bd3af678960 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 8 Jul 2026 17:43:42 +0800 Subject: [PATCH 11/86] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/types.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/database/mongodb/types.py b/src/database/mongodb/types.py index 10b29c1..e4db8b0 100644 --- a/src/database/mongodb/types.py +++ b/src/database/mongodb/types.py @@ -1,8 +1,3 @@ -"""MongoDB 常用类型. - -从 bson 重新导出,用户不需要 import bson。 -""" - from bson import ( ObjectId, Binary, -- Gitee From bd4fc64076bdda2b0d11816946c5026ad28419d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 8 Jul 2026 17:57:45 +0800 Subject: [PATCH 12/86] =?UTF-8?q?=E5=BC=82=E6=AD=A5=E5=AE=A2=E6=88=B7?= =?UTF-8?q?=E7=AB=AF=E7=9A=84=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/client.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/database/mongodb/client.py b/src/database/mongodb/client.py index a6b86f7..724875c 100644 --- a/src/database/mongodb/client.py +++ b/src/database/mongodb/client.py @@ -1,9 +1,6 @@ import pymongo -# from motor.motor_asyncio import AsyncIOMotorClient - - class Client: def __init__(self): @@ -13,5 +10,6 @@ class Client: def create(*args, **kwargs) -> pymongo.MongoClient: return pymongo.MongoClient(*args, **kwargs) - # async def client_async(self, *args, **kwargs): - # return AsyncIOMotorClient(**args, **kwargs) + @staticmethod + async def create_async(*args, **kwargs): + return pymongo.AsyncMongoClient(*args, **kwargs) -- Gitee From 039debdddf9e7344ba62a487b71468c6cb392bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 8 Jul 2026 18:30:02 +0800 Subject: [PATCH 13/86] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E7=AE=A1=E7=90=86=E5=99=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/database/mongodb/utils.py b/src/database/mongodb/utils.py index 7bfb389..f1627a9 100644 --- a/src/database/mongodb/utils.py +++ b/src/database/mongodb/utils.py @@ -46,6 +46,12 @@ class Batch: if self.count: self.execute() + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + class BatchInsert(Batch): -- Gitee From 6559cdea36e89a50438d867fd501635d6620ee89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 8 Jul 2026 18:48:38 +0800 Subject: [PATCH 14/86] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/utils.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/database/mongodb/utils.py b/src/database/mongodb/utils.py index f1627a9..2a3ec41 100644 --- a/src/database/mongodb/utils.py +++ b/src/database/mongodb/utils.py @@ -27,6 +27,9 @@ class Batch: self.ordered = ordered self.on_error = on_error + self.success_count = 0 + self.error_count = 0 + def add(self, doc_or_op): self.ops.append(doc_or_op) self.count += 1 @@ -57,8 +60,12 @@ class BatchInsert(Batch): def execute(self): try: - self.collection.insert_many(self.ops, ordered=self.ordered) + result = self.collection.insert_many(self.ops, ordered=self.ordered) + self.success_count += len(result.inserted_ids) except pymongo.errors.BulkWriteError as error: + # 部分成功的情况 + self.success_count += error.details.get('nInserted', 0) + self.error_count += len(self.ops) - error.details.get('nInserted', 0) if self.on_error == 'raise': raise elif self.on_error == 'log': @@ -86,8 +93,13 @@ class BatchUpdate(Batch): def execute(self): try: - self.collection.bulk_write(self.ops, ordered=self.ordered) + result = self.collection.bulk_write(self.ops, ordered=self.ordered) + self.success_count += result.modified_count except pymongo.errors.BulkWriteError as error: + # 部分成功的情况 + self.success_count += error.details.get('nModified', 0) + self.error_count += len(self.ops) - error.details.get('nModified', 0) + if self.on_error == 'raise': raise elif self.on_error == 'log': -- Gitee From 98471dd985f4c7e7b4a92186da27a23ae6afd763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 8 Jul 2026 18:55:40 +0800 Subject: [PATCH 15/86] =?UTF-8?q?=E4=BD=BF=E7=94=A8while=E6=9B=BF=E6=8D=A2?= =?UTF-8?q?extend()=E4=B8=AD=E7=9A=84if=E9=80=BB=E8=BE=91=EF=BC=8C?= =?UTF-8?q?=E9=98=B2=E6=AD=A2=E5=8D=95=E6=AC=A1=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/database/mongodb/utils.py b/src/database/mongodb/utils.py index 2a3ec41..5b9394b 100644 --- a/src/database/mongodb/utils.py +++ b/src/database/mongodb/utils.py @@ -39,7 +39,7 @@ class Batch: def extend(self, docs_or_ops): self.ops.extend(docs_or_ops) self.count += len(docs_or_ops) - if self.count > self.step: + while self.count >= self.step: self.execute() def execute(self): @@ -63,7 +63,6 @@ class BatchInsert(Batch): result = self.collection.insert_many(self.ops, ordered=self.ordered) self.success_count += len(result.inserted_ids) except pymongo.errors.BulkWriteError as error: - # 部分成功的情况 self.success_count += error.details.get('nInserted', 0) self.error_count += len(self.ops) - error.details.get('nInserted', 0) if self.on_error == 'raise': @@ -96,10 +95,8 @@ class BatchUpdate(Batch): result = self.collection.bulk_write(self.ops, ordered=self.ordered) self.success_count += result.modified_count except pymongo.errors.BulkWriteError as error: - # 部分成功的情况 self.success_count += error.details.get('nModified', 0) self.error_count += len(self.ops) - error.details.get('nModified', 0) - if self.on_error == 'raise': raise elif self.on_error == 'log': -- Gitee From 3f813c5601a29fe52388c7866c5bc3d2201373be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 8 Jul 2026 19:16:02 +0800 Subject: [PATCH 16/86] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E5=BC=82?= =?UTF-8?q?=E6=AD=A5=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/_async_utils.py | 106 ++++++++++++++++++++++++ src/database/mongodb/_sync_utils.py | 106 ++++++++++++++++++++++++ src/database/mongodb/utils.py | 117 +++------------------------ 3 files changed, 223 insertions(+), 106 deletions(-) create mode 100644 src/database/mongodb/_async_utils.py create mode 100644 src/database/mongodb/_sync_utils.py diff --git a/src/database/mongodb/_async_utils.py b/src/database/mongodb/_async_utils.py new file mode 100644 index 0000000..6a55e06 --- /dev/null +++ b/src/database/mongodb/_async_utils.py @@ -0,0 +1,106 @@ +import pymongo.errors +from pymongo import UpdateOne + +from ..utils.logger import get_logger + +logger = get_logger(__name__) + + +class AsyncBatch: + + def __init__(self, collection, step=1000, ordered=True, on_error='raise'): + """初始化异步批量操作. + + Args: + collection: pymongo AsyncCollection 对象 + step: 批次大小,默认 1000 + ordered: 是否有序操作,默认 True + on_error: 错误处理策略,默认 'raise' + - 'raise': 抛出异常(默认) + - 'log': 记录日志但继续执行 + - 'ignore': 忽略错误 + """ + self.ops = [] + self.count = 0 + self.collection = collection + self.step = step + self.ordered = ordered + self.on_error = on_error + + self.success_count = 0 + self.error_count = 0 + + async def add(self, doc_or_op): + self.ops.append(doc_or_op) + self.count += 1 + if self.count >= self.step: + await self.execute() + + async def extend(self, docs_or_ops): + self.ops.extend(docs_or_ops) + self.count += len(docs_or_ops) + while self.count >= self.step: + await self.execute() + + async def execute(self): + pass + + async def close(self): + if self.count: + await self.execute() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + +class AsyncBatchInsert(AsyncBatch): + + async def execute(self): + try: + result = await self.collection.insert_many(self.ops, ordered=self.ordered) + self.success_count += len(result.inserted_ids) + except pymongo.errors.BulkWriteError as error: + self.success_count += error.details.get('nInserted', 0) + self.error_count += len(self.ops) - error.details.get('nInserted', 0) + if self.on_error == 'raise': + raise + elif self.on_error == 'log': + logger.warning(f'AsyncBatchInsert error: {error.details}') + finally: + self.ops.clear() + self.count = 0 + + +class AsyncBatchUpdate(AsyncBatch): + async def add(self, query_or_op, update=None): + if isinstance(query_or_op, UpdateOne): + await AsyncBatch.add(self, query_or_op) + elif update: + await AsyncBatch.add(self, UpdateOne(query_or_op, update)) + else: + raise pymongo.errors.InvalidOperation('Missing update parameter.') + + async def extend(self, ops): + for op in ops: + if isinstance(op, UpdateOne): + await AsyncBatch.add(self, op) + else: + raise pymongo.errors.InvalidOperation('Operation type Error.') + + async def execute(self): + try: + result = await self.collection.bulk_write(self.ops, ordered=self.ordered) + self.success_count += result.modified_count + except pymongo.errors.BulkWriteError as error: + self.success_count += error.details.get('nModified', 0) + self.error_count += len(self.ops) - error.details.get('nModified', 0) + if self.on_error == 'raise': + raise + elif self.on_error == 'log': + logger.warning(f"AsyncBatchUpdate error: {error.details}") + finally: + self.ops.clear() + self.count = 0 diff --git a/src/database/mongodb/_sync_utils.py b/src/database/mongodb/_sync_utils.py new file mode 100644 index 0000000..5b9394b --- /dev/null +++ b/src/database/mongodb/_sync_utils.py @@ -0,0 +1,106 @@ +import pymongo.errors +from pymongo import UpdateOne + +from ..utils.logger import get_logger + +logger = get_logger(__name__) + + +class Batch: + + def __init__(self, collection, step=1000, ordered=True, on_error='raise'): + """初始化批量操作. + + Args: + collection: pymongo Collection 对象 + step: 批次大小,默认 1000 + ordered: 是否有序操作,默认 True + on_error: 错误处理策略,默认 'raise' + - 'raise': 抛出异常(默认) + - 'log': 记录日志但继续执行 + - 'ignore': 忽略错误 + """ + self.ops = [] + self.count = 0 + self.collection = collection + self.step = step + self.ordered = ordered + self.on_error = on_error + + self.success_count = 0 + self.error_count = 0 + + def add(self, doc_or_op): + self.ops.append(doc_or_op) + self.count += 1 + if self.count >= self.step: + self.execute() + + def extend(self, docs_or_ops): + self.ops.extend(docs_or_ops) + self.count += len(docs_or_ops) + while self.count >= self.step: + self.execute() + + def execute(self): + pass + + def close(self): + if self.count: + self.execute() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + +class BatchInsert(Batch): + + def execute(self): + try: + result = self.collection.insert_many(self.ops, ordered=self.ordered) + self.success_count += len(result.inserted_ids) + except pymongo.errors.BulkWriteError as error: + self.success_count += error.details.get('nInserted', 0) + self.error_count += len(self.ops) - error.details.get('nInserted', 0) + if self.on_error == 'raise': + raise + elif self.on_error == 'log': + logger.warning(f'BatchInsert error: {error.details}') + finally: + self.ops.clear() + self.count = 0 + + +class BatchUpdate(Batch): + def add(self, query_or_op, update=None): + if isinstance(query_or_op, UpdateOne): + super().add(query_or_op) + elif update: + super().add(UpdateOne(query_or_op, update)) + else: + raise pymongo.errors.InvalidOperation('Missing update parameter.') + + def extend(self, ops): + for op in ops: + if isinstance(op, UpdateOne): + super().add(op) + else: + raise pymongo.errors.InvalidOperation('Operation type Error.') + + def execute(self): + try: + result = self.collection.bulk_write(self.ops, ordered=self.ordered) + self.success_count += result.modified_count + except pymongo.errors.BulkWriteError as error: + self.success_count += error.details.get('nModified', 0) + self.error_count += len(self.ops) - error.details.get('nModified', 0) + if self.on_error == 'raise': + raise + elif self.on_error == 'log': + logger.warning(f"BatchUpdate error: {error.details}") + finally: + self.ops.clear() + self.count = 0 diff --git a/src/database/mongodb/utils.py b/src/database/mongodb/utils.py index 5b9394b..2890e74 100644 --- a/src/database/mongodb/utils.py +++ b/src/database/mongodb/utils.py @@ -1,106 +1,11 @@ -import pymongo.errors -from pymongo import UpdateOne - -from ..utils.logger import get_logger - -logger = get_logger(__name__) - - -class Batch: - - def __init__(self, collection, step=1000, ordered=True, on_error='raise'): - """初始化批量操作. - - Args: - collection: pymongo Collection 对象 - step: 批次大小,默认 1000 - ordered: 是否有序操作,默认 True - on_error: 错误处理策略,默认 'raise' - - 'raise': 抛出异常(默认) - - 'log': 记录日志但继续执行 - - 'ignore': 忽略错误 - """ - self.ops = [] - self.count = 0 - self.collection = collection - self.step = step - self.ordered = ordered - self.on_error = on_error - - self.success_count = 0 - self.error_count = 0 - - def add(self, doc_or_op): - self.ops.append(doc_or_op) - self.count += 1 - if self.count >= self.step: - self.execute() - - def extend(self, docs_or_ops): - self.ops.extend(docs_or_ops) - self.count += len(docs_or_ops) - while self.count >= self.step: - self.execute() - - def execute(self): - pass - - def close(self): - if self.count: - self.execute() - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - -class BatchInsert(Batch): - - def execute(self): - try: - result = self.collection.insert_many(self.ops, ordered=self.ordered) - self.success_count += len(result.inserted_ids) - except pymongo.errors.BulkWriteError as error: - self.success_count += error.details.get('nInserted', 0) - self.error_count += len(self.ops) - error.details.get('nInserted', 0) - if self.on_error == 'raise': - raise - elif self.on_error == 'log': - logger.warning(f'BatchInsert error: {error.details}') - finally: - self.ops.clear() - self.count = 0 - - -class BatchUpdate(Batch): - def add(self, query_or_op, update=None): - if isinstance(query_or_op, UpdateOne): - super().add(query_or_op) - elif update: - super().add(UpdateOne(query_or_op, update)) - else: - raise pymongo.errors.InvalidOperation('Missing update parameter.') - - def extend(self, ops): - for op in ops: - if isinstance(op, UpdateOne): - super().add(op) - else: - raise pymongo.errors.InvalidOperation('Operation type Error.') - - def execute(self): - try: - result = self.collection.bulk_write(self.ops, ordered=self.ordered) - self.success_count += result.modified_count - except pymongo.errors.BulkWriteError as error: - self.success_count += error.details.get('nModified', 0) - self.error_count += len(self.ops) - error.details.get('nModified', 0) - if self.on_error == 'raise': - raise - elif self.on_error == 'log': - logger.warning(f"BatchUpdate error: {error.details}") - finally: - self.ops.clear() - self.count = 0 +from ._async_utils import AsyncBatchInsert +from ._async_utils import AsyncBatchUpdate +from ._sync_utils import BatchInsert +from ._sync_utils import BatchUpdate + +__all__ = [ + 'AsyncBatchInsert', + 'AsyncBatchUpdate', + 'BatchInsert', + 'BatchUpdate', +] -- Gitee From ca190212d2651dc6b9e0ce0d9e2e070da0701259 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Thu, 9 Jul 2026 13:46:53 +0800 Subject: [PATCH 17/86] =?UTF-8?q?=E8=8E=B7=E5=8F=96=E5=BC=82=E6=AD=A5?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E5=AF=B9=E8=B1=A1=E7=9A=84=E6=96=B9=E6=B3=95?= =?UTF-8?q?=E6=9C=AC=E8=BA=AB=E4=B8=8D=E9=9C=80=E8=A6=81=E5=BC=82=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/database/mongodb/client.py b/src/database/mongodb/client.py index 724875c..34391c0 100644 --- a/src/database/mongodb/client.py +++ b/src/database/mongodb/client.py @@ -11,5 +11,5 @@ class Client: return pymongo.MongoClient(*args, **kwargs) @staticmethod - async def create_async(*args, **kwargs): + def create_async(*args, **kwargs): return pymongo.AsyncMongoClient(*args, **kwargs) -- Gitee From 6be92d8948f1a60f69c833d8efa82673e1583644 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Fri, 17 Jul 2026 19:25:41 +0800 Subject: [PATCH 18/86] =?UTF-8?q?=E4=BD=BF=E7=94=A8loongspears.utils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 +- pyproject.toml | 2 +- requirements.txt | 3 ++ src/database/utils/__init__.py | 8 ----- src/database/utils/logger.py | 55 ---------------------------------- 5 files changed, 5 insertions(+), 65 deletions(-) delete mode 100644 src/database/utils/__init__.py delete mode 100644 src/database/utils/logger.py diff --git a/.gitignore b/.gitignore index d08fcaf..e963d7d 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,4 @@ __pycache__/ /src/*.egg-info/ /src/database/_version.py -/test.py +/test*.py diff --git a/pyproject.toml b/pyproject.toml index d189511..c255aaf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ classifiers = [ "License :: OSI Approved :: MIT License", ] dependencies = [ - + "loongspears.utils @ git+https://gitee.com/loongspears/utils", ] [project.optional-dependencies] all = [ diff --git a/requirements.txt b/requirements.txt index 9ca11e2..788ff9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ +# common +loongspears.utils @ git+https://gitee.com/loongspears/utils + # MongoDB pymongo>=4.6.0 diff --git a/src/database/utils/__init__.py b/src/database/utils/__init__.py deleted file mode 100644 index b3eef63..0000000 --- a/src/database/utils/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""工具函数模块.""" - -from .logger import get_logger, setup_logging - -__all__ = [ - "get_logger", - "setup_logging", -] diff --git a/src/database/utils/logger.py b/src/database/utils/logger.py deleted file mode 100644 index 4cfbde0..0000000 --- a/src/database/utils/logger.py +++ /dev/null @@ -1,55 +0,0 @@ -"""全项目统一的日志管理. - -提供统一的 logger 获取和配置功能。 -""" - -import logging -from typing import Optional - - -def get_logger(name: str) -> logging.Logger: - """获取指定名称的 logger. - - Args: - name: logger 名称,通常使用 __name__ - - Returns: - logging.Logger 实例 - - Example: - >>> from database.utils.logger import get_logger - >>> logger = get_logger(__name__) - >>> logger.info("Something happened") - """ - return logging.getLogger(name) - - -def setup_logging( - level: int = logging.WARNING, - format: Optional[str] = None -) -> None: - """配置全项目的日志. - - Args: - level: 日志级别,默认 WARNING - format: 日志格式,默认使用标准格式 - - Example: - >>> from database.utils.logger import setup_logging - >>> import logging - >>> setup_logging(level=logging.DEBUG) - """ - if format is None: - format = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - - logging.basicConfig( - level=level, - format=format, - datefmt='%Y-%m-%d %H:%M:%S' - ) - - -__all__ = [ - "get_logger", - "setup_logging", -] -- Gitee From d5d8433c2796a94afbe571dd0ddc662b5791ed5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sat, 14 Nov 2026 21:34:40 +0800 Subject: [PATCH 19/86] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=20MongoDB=20=E5=BC=82?= =?UTF-8?q?=E6=AD=A5=E5=AE=A2=E6=88=B7=E7=AB=AF=E5=9C=A8=E6=97=A7=E7=89=88?= =?UTF-8?q?=20PyMongo=20=E4=B8=8A=E7=9A=84=E9=94=99=E8=AF=AF=E6=8F=90?= =?UTF-8?q?=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mongodb/client.py | 5 ++++- tests/test_mongodb_client.py | 39 ++++++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 tests/test_mongodb_client.py diff --git a/src/database/mongodb/client.py b/src/database/mongodb/client.py index 34391c0..72bad8c 100644 --- a/src/database/mongodb/client.py +++ b/src/database/mongodb/client.py @@ -12,4 +12,7 @@ class Client: @staticmethod def create_async(*args, **kwargs): - return pymongo.AsyncMongoClient(*args, **kwargs) + async_client = getattr(pymongo, "AsyncMongoClient", None) + if async_client is None: + raise ImportError("MongoDB async client requires PyMongo >= 4.9. Upgrade PyMongo to use create_async().") + return async_client(*args, **kwargs) diff --git a/tests/test_mongodb_client.py b/tests/test_mongodb_client.py new file mode 100644 index 0000000..f2b0b53 --- /dev/null +++ b/tests/test_mongodb_client.py @@ -0,0 +1,39 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from database.mongodb import Client + + +class MongoDBClientTests(unittest.TestCase): + + def test_driver_without_async_client(self): + native_client = object() + driver = SimpleNamespace(MongoClient=Mock(return_value=native_client)) + + with patch("database.mongodb.client.pymongo", driver): + self.assertIs(Client.create("mongodb://localhost", connect=False), native_client) + with self.assertRaisesRegex(ImportError, r"PyMongo >= 4\.9"): + Client.create_async("mongodb://localhost") + + driver.MongoClient.assert_called_once_with("mongodb://localhost", connect=False) + + def test_async_factory_returns_native_client(self): + native_client = object() + driver = SimpleNamespace(AsyncMongoClient=Mock(return_value=native_client)) + + with patch("database.mongodb.client.pymongo", driver): + result = Client.create_async("mongodb://localhost", serverSelectionTimeoutMS=500) + + self.assertIs(result, native_client) + driver.AsyncMongoClient.assert_called_once_with("mongodb://localhost", serverSelectionTimeoutMS=500) + + def test_async_factory_propagates_driver_error(self): + error = AttributeError("driver constructor failed") + driver = SimpleNamespace(AsyncMongoClient=Mock(side_effect=error)) + + with patch("database.mongodb.client.pymongo", driver): + with self.assertRaises(AttributeError) as caught: + Client.create_async("mongodb://localhost") + + self.assertIs(caught.exception, error) -- Gitee From cbfe0badc1b0b2153778ae8d4a97a079a6b68d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sat, 14 Nov 2026 21:34:47 +0800 Subject: [PATCH 20/86] =?UTF-8?q?feat(mysql):=20=E6=8F=90=E4=BE=9B?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=92=8C=E5=BC=82=E6=AD=A5=E8=BF=9E=E6=8E=A5?= =?UTF-8?q?=E5=B7=A5=E5=8E=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/mysql/__init__.py | 5 ++++ src/database/mysql/client.py | 14 ++++++++++ tests/test_mysql_client.py | 50 ++++++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+) create mode 100644 src/database/mysql/__init__.py create mode 100644 src/database/mysql/client.py create mode 100644 tests/test_mysql_client.py diff --git a/src/database/mysql/__init__.py b/src/database/mysql/__init__.py new file mode 100644 index 0000000..19790f0 --- /dev/null +++ b/src/database/mysql/__init__.py @@ -0,0 +1,5 @@ +from .client import Client + +__all__ = [ + "Client", +] diff --git a/src/database/mysql/client.py b/src/database/mysql/client.py new file mode 100644 index 0000000..38f75c1 --- /dev/null +++ b/src/database/mysql/client.py @@ -0,0 +1,14 @@ +from mysql import connector +from mysql.connector import aio + + +class Client: + """Factory for native MySQL Connector/Python connections.""" + + @staticmethod + def create(*args, **kwargs): + return connector.connect(*args, **kwargs) + + @staticmethod + async def create_async(*args, **kwargs): + return await aio.connect(*args, **kwargs) diff --git a/tests/test_mysql_client.py b/tests/test_mysql_client.py new file mode 100644 index 0000000..fcdb437 --- /dev/null +++ b/tests/test_mysql_client.py @@ -0,0 +1,50 @@ +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, patch + +from mysql.connector import Error + +from database.mysql import Client + + +class MySQLClientTests(TestCase): + @patch("database.mysql.client.connector.connect") + def test_create_returns_the_driver_connection(self, connect): + connection = object() + connect.return_value = connection + + result = Client.create("option", host="127.0.0.1", port=3306) + + self.assertIs(result, connection) + connect.assert_called_once_with("option", host="127.0.0.1", port=3306) + + @patch("database.mysql.client.connector.connect") + def test_create_propagates_the_driver_error(self, connect): + error = Error("connection failed") + connect.side_effect = error + + with self.assertRaises(Error) as result: + Client.create() + + self.assertIs(result.exception, error) + + +class AsyncMySQLClientTests(IsolatedAsyncioTestCase): + @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_create_async_returns_the_driver_connection(self, connect): + connection = object() + connect.return_value = connection + + result = await Client.create_async("option", host="127.0.0.1", port=3306) + + self.assertIs(result, connection) + connect.assert_awaited_once_with("option", host="127.0.0.1", port=3306) + + @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_create_async_propagates_the_driver_error(self, connect): + error = Error("connection failed") + connect.side_effect = error + + with self.assertRaises(Error) as result: + await Client.create_async() + + self.assertIs(result.exception, error) -- Gitee From e5f3079bca5dc86722f90de305418d8206e8ddf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sat, 14 Nov 2026 21:36:18 +0800 Subject: [PATCH 21/86] =?UTF-8?q?=E5=AE=8C=E5=96=84=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E5=AE=A2=E6=88=B7=E7=AB=AF=E5=AE=89=E8=A3=85=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E5=92=8C=E4=BD=BF=E7=94=A8=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 83 ++++++++++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 6 ++-- requirements.txt | 3 ++ 3 files changed, 90 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e69de29..6099a71 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,83 @@ +# database + +使用同一套工厂接口获取 MongoDB、MySQL 和 Redis 的原生客户端。工厂把连接参数传给驱动,查询、事务和连接关闭继续使用各驱动的 API。 + +## 安装 + +在仓库根目录运行以下命令。Python 需要 3.10 或更高版本,下载共享依赖时需要 Git。 + +```bash +python -m pip install ".[all]" +``` + +只使用一种数据库时,可以将 `all` 换成 `mongodb`、`mysql` 或 `redis`。 + +## MongoDB + +PyMongo 4.6–4.8 支持 `create()`。`create_async()` 需要 PyMongo 4.9 或更高版本,`all` 会安装异步 API 已正式发布的 PyMongo 4.13 或更高版本。 + +```python +import asyncio + +from database.mongodb import Client + +with Client.create("mongodb://127.0.0.1:27017/") as client: + client.admin.command("ping") + + +async def ping_mongodb(): + async with Client.create_async("mongodb://127.0.0.1:27017/") as client: + await client.admin.command("ping") + + +asyncio.run(ping_mongodb()) +``` + +## MySQL + +使用 MySQL Connector/Python。`create_async()` 需要 `await`,驱动完成建连后返回原生异步连接。示例从 `MYSQL_USER` 和 `MYSQL_PASSWORD` 环境变量读取凭据。 + +```python +import asyncio +import os + +from database.mysql import Client + +options = { + "host": "127.0.0.1", + "user": os.environ["MYSQL_USER"], + "password": os.environ["MYSQL_PASSWORD"], +} + +with Client.create(**options) as connection: + connection.ping() + + +async def ping_mysql(): + async with await Client.create_async(**options) as connection: + await connection.ping() + + +asyncio.run(ping_mysql()) +``` + +## Redis + +`create()` 和 `create_async()` 默认获取单机客户端。`connect()` 还支持 `mode="sentinel"` 和 `mode="cluster"`,可传 `is_async=True` 获取异步对象。Sentinel 模式返回 `Sentinel`,可用 `master_for()` 获取主节点的数据客户端。 + +```python +import asyncio + +from database.redis import Client + +with Client.create(host="127.0.0.1", port=6379) as client: + client.ping() + + +async def ping_redis(): + async with Client.create_async(host="127.0.0.1", port=6379) as client: + await client.ping() + + +asyncio.run(ping_redis()) +``` diff --git a/pyproject.toml b/pyproject.toml index c255aaf..c620aad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,12 +31,14 @@ dependencies = [ ] [project.optional-dependencies] all = [ - "database[mongodb,redis]", + "database[mongodb,mysql,redis]", + "pymongo>=4.13", ] mysql = [ + "mysql-connector-python>=8.3.0", ] mongodb = [ - "pymongo>=4.9.0", + "pymongo>=4.6.0", ] redis = [ "redis>=5.0.0", diff --git a/requirements.txt b/requirements.txt index 788ff9b..f8d66c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,8 @@ loongspears.utils @ git+https://gitee.com/loongspears/utils # MongoDB pymongo>=4.6.0 +# MySQL +mysql-connector-python>=8.3.0 + # Redis redis>=5.0.0 -- Gitee From 8f1352295217973a1b8624a8a01cd39cf88439a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sat, 14 Nov 2026 23:38:38 +0800 Subject: [PATCH 22/86] =?UTF-8?q?chore(git):=20=E5=AE=8C=E5=96=84=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=B1=9E=E6=80=A7=E5=92=8C=E5=BF=BD=E7=95=A5=E8=A7=84?= =?UTF-8?q?=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitattributes | 44 +++++++++++++++++++++++++++++++++++++++++++- .gitignore | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/.gitattributes b/.gitattributes index fcadb2c..34359e9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,43 @@ -* text eol=lf +# 自动识别文本,入库和检出均使用 LF +* text=auto eol=lf + +# 图片 +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.webp binary +*.avif binary + +# 字体 +*.woff binary +*.woff2 binary +*.ttf binary +*.otf binary + +# Word +*.[dD][oO][cC] binary +*.[dD][oO][cC][mMxX] binary +*.[dD][oO][tT][mMxX] binary + +# Excel +*.[xX][lL][sS] binary +*.[xX][lL][sS][bBmMxX] binary +*.[xX][lL][tT] binary +*.[xX][lL][tT][mMxX] binary + +# PowerPoint +*.[pP][pP][tT] binary +*.[pP][pP][tT][mMxX] binary +*.[pP][pP][sS] binary +*.[pP][pP][sS][mMxX] binary +*.[pP][oO][tT][mMxX] binary + +# PDF +*.[pP][dD][fF] binary + +# 数据库 +*.db binary +*.sqlite binary +*.sqlite3 binary diff --git a/.gitignore b/.gitignore index e963d7d..c185862 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,54 @@ +# Secrets +# Environments +.env +.env.dev +.env.develop +.env.test +.env.production +# database files +*.db +*.sql + +# macOS +.DS_Store + +# editor or IDE .idea/ +.run/ +.vscode/ + +# AI agent +.claude/ +.codex/ +# GIT +.gitee/ +.github/ + +# python venv +.venv/ +.uv-cache/ +uv.lock + +# Python cache __pycache__/ +*.py[cod] +.pytest_cache/ +.pytest-tmp/ +.ruff_cache/ +.mypy_cache/ +# Python build /build/ +*.egg-info/ /src/*.egg-info/ -/src/database/_version.py +# test scripts +!/tests /test*.py +src/database/_version.py + +# logs +/logs/ +/outputs/ +*.log -- Gitee From cb1847026cc4b068762cdf6c0f813a84ad958b07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:40:27 +0800 Subject: [PATCH 23/86] =?UTF-8?q?docs(database):=20=E6=98=8E=E7=A1=AE?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E8=BF=9E=E6=8E=A5=E7=AE=A1=E7=90=86=E8=AE=BE?= =?UTF-8?q?=E8=AE=A1=E4=B8=8E=E5=B9=B6=E8=A1=8C=E5=BC=80=E5=8F=91=E5=A5=91?= =?UTF-8?q?=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/connection-management-design.md | 109 +++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 docs/connection-management-design.md diff --git a/docs/connection-management-design.md b/docs/connection-management-design.md new file mode 100644 index 0000000..7491227 --- /dev/null +++ b/docs/connection-management-design.md @@ -0,0 +1,109 @@ +# database 统一连接管理设计与开发流程 + +日期:2026-11-15。状态:实施中,测试结果见验收记录。 +基线 develop:7d38b1ffe13a17b37d0bffc15e175085dcf0bb19。 +参考 temp/save:89497823698812e1c85e6c4c12af86d4281543cf。 + +## 目标和边界 +公共 SDK 支持同一项目同时使用 MongoDB、MySQL、Redis,以及多个同类型连接。提供统一配置、显式检查、保留原始信息的异常转换。保留现有 database..Client 的参数、返回类型、异步方式与原生异常,不封装 CRUD、事务、数据模型,不增加自动重试、后台检查、故障转移或自建连接池。 +SDK 只接收调用方显式参数;不读取环境变量、不加载 .env、不搜索配置文件、不提供环境读取助手。调用方可自行读取私密配置并传入。原生驱动自身已有行为按显式配置及驱动规则执行。 + +## 草稿复用 +| temp/save 内容 | 处理 | +| --- | --- | +| DatabaseType、配置校验 | 复用三种类型与有效规则,以标准库数据类实现,不新增 Pydantic | +| ConfigManager.load | 纯参数校验转换 | +| _process_env_vars、来源合并 | 不迁移,配置来源由调用方决定 | +| PoolConfig、HAConfig、重试、故障转移 | 不迁移,不公开未实现能力 | +| 异常基类 | 复用分类理念,保存原始对象、因果链、上下文 | +| BaseClient、BaseConnection | 提取生命周期理念,实现轻量同步异步句柄 | +| ConnectionState | 不提供持续 is_connected 状态 | +| CRUD、事务、Pipeline 草稿 | 不迁移 | +| 配置和异常测试 | 迁移有效行为场景,采用 unittest | +不整体合并或删除 temp/save,不改写用户原工作区。 + +## 公共接口 +from database import ConnectionConfig, Connections +ConnectionConfig(backend, host=None, port=None, username=None, password=None, + connect_timeout_s=None, options={}, mode="single", nodes=(), service_name=None) +Connections({name: config}) 保存配置,不连接、不缓存客户端。create(name) 每次返回独立 Connection;await create_async(name) 返回独立 AsyncConnection。name 必须是非空字符串;未知名称为 ConfigurationError。 +connection.native 为真实原生对象;check()/await check() 返回 CheckResult;close()/await close() 清理自有资源。支持 with/async with,上下文进入不增加探测。connection.errors(operation=...) 为普通 with,可在其中 await 原生方法。 + +### 参数契约 +backend 支持 mongodb/mysql/redis;port 为非 bool 整数 1..65535;connect_timeout_s 为非 bool 正整数秒。新高层配置必须提供显式目标,不能仅凭凭据或空配置连接 localhost。默认端口仍由驱动决定。 +host 对 MongoDB 可为 URI 或非空地址列表,其他后端为非空字符串。 +username 映射 MongoDB/Redis username、MySQL user;password 同名;connect_timeout_s 映射 MongoDB connectTimeoutMS(乘1000)、MySQL connection_timeout、Redis socket_connect_timeout。 +options 保留原生专有参数。MySQL database、Redis db、MongoDB authSource 等保持各自含义;MongoDB 业务数据库由 native 选择。 +共同字段与 options 中原生同义字段重复时报 ConfigurationError,不静默覆盖。同一个字段的原生别名重复也拒绝。节点与原生节点参数重复也拒绝。只校验 SDK 自身字段,不丢弃原生专有选项。 +复制标准配置容器,外部池/回调等对象保持引用。默认 repr 不打印密码、完整 URI、options。 +Redis mode 为 single/sentinel/cluster;nodes 为(host,port)列表。Sentinel 必须有 nodes 或显式 sentinels 原生参数,并提供 service_name;Cluster 可用 nodes 或 host 或原生 startup_nodes。 +Sentinel 节点凭据只通过 sentinel_kwargs 显式配置,不自动复制数据节点密码。 +外部 connection_pool 与共同连接参数混用拒绝,避免参数被池静默忽略。 +无 URI 环境变量替换;无隐式环境配置。原生工厂原来的默认行为不变。 + +### 创建和生命周期 +配置注册和校验不得有网络请求。创建保留各驱动原生初始化时机(MySQL 及同步 Redis Cluster 本身可能连网),不额外 ping。 +新高层 create_async 都需要 await;内部只等待原生 MySQL 工厂,不因 MongoDB/Redis 原生对象可等待而自动等待它。 +每次创建独立实例;没有全局单例、跨连接事务或额外线程安全保证。用户显式指定驱动池时允许原生资源复用。 +异步检查和关闭必须使用创建句柄的事件循环。成功关闭后幂等;关闭后 native/check 拒绝访问。已取出的原生引用无法收回。 +关闭仅释放自有资源,不额外提交、回滚或重连。外部池不关闭;MySQL 池化连接按原生语义归还池。原生游标/额外客户端由调用方管理。 +Redis 5.0 同步 close 不一定断开池,需显式处理自有池;异步优先 aclose,旧版本兼容 close。 +清理尝试所有资源;有主异常时保留主异常对象并附加 database_cleanup_errors;无主异常时抛首个转换后错误,附加其他清理错误。 + +### 检查 +CheckResult 冻结数据类字段:connection_name, backend, mode, checked_at(带时区UTC datetime), elapsed_ms(float), scope(str), targets(tuple[str,...])。 +MongoDB:native.admin.command("ping", read_preference=native.read_preference),异步等待;scope=selected_server。 +MySQL:native.ping(reconnect=False),异步等待;scope=current_connection。 +Redis single:PING;scope=single。 +Sentinel:native 仍为 Sentinel,通过 master_for(service_name) 创建临时客户端,PING 数据主节点并清理临时客户端/自有池;scope=sentinel_master。 +Cluster:异步必要时 initialize,再取 get_primaries() 快照逐个定向 PING;空列表失败;首个失败即停止,未检查节点不标正常;scope=cluster_primaries。 +失败抛带阶段/目标的统一异常,不返回 False。目标字符串不含凭据;不可无依据声称已检查所有节点。 +检查不写业务数据,不证明 CRUD 权限、持续健康、全部副本健康。无统一整体硬超时:MySQL 8.3 不支持后来9.2新增的 read_timeout;按实际原生参数约束各阶段。 + +### 异常 +DatabaseError 基类及 ConfigurationError, DependencyError, AuthenticationError, +PermissionDeniedError, DatabaseTimeoutError, DatabaseConnectionError, DriverError, ConnectionClosedError。 +字段 message, connection_name, backend, phase(config/create/check/operation/close), operation, target, original, database_cleanup_errors。 +original 保存原对象,raise unified from original 保留原因链、回溯、错误码、标签和详情。已有 DatabaseError 不重复包装。无法恢复驱动内部丢弃的信息。 +优先认证/权限/超时,再通用连接错误。MongoDB code18认证/code13权限;MySQL1045认证,1044/1142/1143/1227权限;Redis 专门子类优先。未知驱动错误为 DriverError。不按自然语言错误消息猜根因,不声明安全重试或确定写入结果。 +errors 上下文只转换所属驱动的异常;业务 ValueError/TypeError、取消、KeyboardInterrupt、SystemExit 原样传播。惰性游标迭代和 await 必须在上下文内。 +SDK 直接调用驱动的边界可以识别明确来源的底层超时/传输异常。 + +## 内部协作契约 +- database.config: DatabaseType, ConnectionConfig;snapshot() 返回配置的隔离副本;driver_kwargs() 返回共同字段映射后与 options 合并的全新参数字典。Redis nodes 转换由适配器负责,service_name 不传给驱动构造器。 +- database.errors: 上述异常;convert_error(error, *, connection_name, backend, phase, operation=None, target=None, classifier=None, owned=False) 返回统一异常或原异常;error_context 使用相同关键字并抛 from original;attach_cleanup_errors(primary, errors) 附加元组。 +- database.connections: Connections, Connection, AsyncConnection, CheckResult,运行时才 import database._adapters.。 +- database._adapters..Adapter(config, name, is_async=False):create()、async create_async() 返回原生对象;check(native)、async check_async(native) 返回(scope, targets);close(native)、async close_async(native);classify(error) 返回统一异常类或 None(非所属驱动错误)。 +- 适配器应在需要标注具体目标的检查异常处调用 convert_error(... owned=True, classifier=self.classify) 并 raise from;核心同样处理创建/检查/关闭外围异常。已统一错误不重复包装。 +- errors 实现不导入驱动;配置层也不导入驱动。可选驱动只在实际使用时加载。 +- 公共 __init__ 导出由协调者统一修改。worker 不改现有原生工厂文件。 +- 新测试独立 tests/test_connection_config.py、test_connection_errors.py、test_connections.py、test_managed_.py。 + +## 开发、分支和历史 +总线 feature/database-connection-management 从 develop 已提交HEAD创建,独立 worktree;develop 只读,由用户最终审核合入。 +禁止 codex/Codex 前缀。功能 feature/,修复 fix/,重构 refactor/。 +配置 feature/database-connection-config 与异常 feature/database-connection-errors 并行;异常先集成,配置如需依赖从总线检查点合入。 +公共层 feature/database-connection-core 基于已集成配置异常;随后三个数据库分支从同一公共层检查点并行: +feature/database-mongodb-connections、feature/database-mysql-connections、feature/database-redis-connections。 +每worker独立worktree、分支、暂存区,负责自己的提交、测试、推送。协调者串行合入总线,维护导出/README/本文/验收。 +必要依赖只从总线明确检查点合入,不交叉复制或合并其他worker草稿。 +中文 Conventional Commits,按功能自然拆分并附必要测试;不把全部工作压成单提交;已推送历史不rebase/amend。 +总线使用 --no-ff 合并保留模块边界,不squash、不用单次cherry-pick替代分支。核对交付SHA和测试后合并并复验。 +允许向 origin https://gitee.com/loongspears/database 推送本任务明确分支;禁止推送develop、无关分支、--all、--mirror、force,禁止删除远端分支。worker推自己的分支,协调者推总线;推后核对SHA。鉴权失败仍完成本地可做工作并记录。 +不改历史worktree、不删除temp/save、不处理无关文件;保留任务分支和worktree供审核。 + +## 验收 +沿用 unittest 与 IsolatedAsyncioTestCase。覆盖:无环境读取、显式目标、配置隔离/冲突/映射、多实例及同名同步异步独立、原生对象身份、检查语义、异常对象和链保真、保守分类、取消透传、外部池保护、清理失败、可选依赖、旧工厂回归。 +保持 Python>=3.10、pymongo>=4.6.0(异步>=4.9,all>=4.13)、mysql-connector-python>=8.3.0、redis>=5.0.0。 +验证最低及选定当前版本;PyMongo4.6/4.8同步及异步提示、4.9/4.13异步。 +隔离服务验证 MongoDB/MySQL8.4/Redis单机、Sentinel、Cluster 共10条同步异步路径,以及认证/权限/连接拒绝、未知服务、不可达主节点,多数据库并用和独立关闭。MySQL 原生 SELECT 1。外部测试进程限制故障测试时限,不冒充SDK硬超时。 +README补全显式配置、多实例、检查、原生访问、异常、资源归属和版本限制。 +交付总线/远端/基线及最终SHA/worker合并表/实际命令版本结果/未验证项/推送状态;用户最终合入develop。 + +## 调研依据 +- https://github.com/redis/redis-py/blob/v5.0.0/redis/client.py +- https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlconnection-ping.html +- https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html +- https://www.mongodb.com/docs/manual/reference/error-codes/ +- https://pymongo.readthedocs.io/en/stable/api/pymongo/errors.html + -- Gitee From 33e578a0c579528bfae5d40c4340b41e5d41af8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:43:14 +0800 Subject: [PATCH 24/86] =?UTF-8?q?docs(database):=20=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E5=A4=9A=E6=95=B0=E6=8D=AE=E5=BA=93=E8=BF=9E=E6=8E=A5=E7=AE=A1?= =?UTF-8?q?=E7=90=86=E4=BD=BF=E7=94=A8=E6=8C=87=E5=8D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 100 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/README.md b/README.md index 6099a71..1d3982f 100644 --- a/README.md +++ b/README.md @@ -81,3 +81,103 @@ async def ping_redis(): asyncio.run(ping_redis()) ``` + +## 统一配置和显式连接管理 + +新高层接口只接受调用方传入的参数,不读取环境变量、不加载 `.env`。上文 MySQL 示例中的环境读取发生在调用方代码中。可以自行读取密钥服务或配置文件,再传入 SDK。 + +```python +from contextlib import ExitStack +from database import ConnectionConfig, Connections + +def build_connections(mysql_password): + return Connections({ + "orders": ConnectionConfig( + backend="mysql", host="127.0.0.1", username="app", + password=mysql_password, options={"database": "orders"}, + ), + "documents": ConnectionConfig( + backend="mongodb", host="mongodb://127.0.0.1:27017/", + ), + "cache": ConnectionConfig( + backend="redis", host="127.0.0.1", options={"db": 0}, + ), + }) + +connections = build_connections("由调用方提供的密码") +with ExitStack() as stack: + orders = stack.enter_context(connections.create("orders")) + documents = stack.enter_context(connections.create("documents")) + cache = stack.enter_context(connections.create("cache")) + for connection in (orders, documents, cache): + result = connection.check() + print(result.scope, result.elapsed_ms) + with orders.errors(operation="read_order"): + with orders.native.cursor() as cursor: + cursor.execute("SELECT 1") + print(cursor.fetchone()) +``` + +同名配置每次创建独立对象,由调用方决定何时复用与关闭。可以增加另一个名字来连接第二个 MySQL。新配置必须给出显式目标,重复的共同参数和原生参数会报错;`options` 中使用驱动原生名称,例如 MySQL 的 `database`、Redis 的 `db`、MongoDB 的 `authSource`。 + +创建保持驱动原生初始化时机,不额外探测。显式 `check()` 成功也不代表持续在线或拥有全部 CRUD 权限;SDK 不提供持续有效的 `is_connected` 状态。 + +### 异步 + +```python +async def read_document(connections, document_id): + async with await connections.create_async("documents") as connection: + await connection.check() + with connection.errors(operation="read_document"): + return await connection.native["app"]["items"].find_one( + {"_id": document_id} + ) +``` + +新高层 `create_async()` 均需 `await`。旧 `database..Client` 的调用方式保持不变。异步句柄应在创建它的事件循环内检查和关闭;同步与异步实例相互独立。 + +### Redis 拓扑 + +```python +sentinel_config = ConnectionConfig( + backend="redis", mode="sentinel", + nodes=[("127.0.0.1", 26379)], service_name="cache-primary", + options={"socket_timeout": 2, "sentinel_kwargs": {"socket_timeout": 2}}, +) +cluster_config = ConnectionConfig( + backend="redis", mode="cluster", + nodes=[("127.0.0.1", 7000), ("127.0.0.1", 7001)], + options={"socket_timeout": 2}, +) +``` + +Sentinel 的 `native` 仍是原生 Sentinel 对象,可以调用 `master_for()` 和 `slave_for()`。检查会通过指定服务建立临时主节点数据客户端,执行 PING 后清理。Sentinel 节点凭据通过 `sentinel_kwargs` 显式设置,不自动复制数据节点密码。 + +Cluster 检查客户端当前发现的全部主节点快照,逐个 PING;首次失败即抛出异常,未检查节点不会标记为正常。不会检查全部副本或保证集群持续健康。 + +### 异常和原生访问 + +```python +from database.errors import DatabaseError + +try: + with connections.create("orders") as connection: + connection.check() +except DatabaseError as error: + print(error.connection_name, error.phase) + # 原始对象、错误码、详情及已有异常链保存在 error.original。 + # 输出原始异常前,由调用方决定如何处理其中可能包含的私密信息。 +``` + +SDK 自动转换其自身配置、创建、检查及关闭范围内的已识别错误。原生业务操作只有位于 `connection.errors(...)` 中时才转换所属驱动的异常;上下文外完全遵循原生行为。惰性游标迭代和异步等待也必须位于上下文中。业务异常和取消信号原样传播。 + +统一类别不意味着写入必然失败或可以安全重试。没有可靠分类的驱动错误保留为 `DriverError`,不按错误文本猜测原因。原始对象在 `original` 中,因果链通过 `__cause__` 保留。已有主异常时发生的关闭错误附加在 `database_cleanup_errors` 中,不覆盖主异常。 + +### 资源和超时 + +高层上下文管理器清理 SDK 创建的客户端和自有资源,不关闭调用方传入的外部池;MySQL 池化连接按驱动语义归还池。自行创建的游标、Pipeline 或额外客户端由调用方管理。不要依赖句柄为原生对象增加线程安全或并发安全保证。 + +`connect_timeout_s` 是正整数秒,仅控制建立连接阶段。读取响应、节点选择和等待连接池等限制通过 `options` 使用驱动支持的参数配置。MySQL Connector/Python 8.3 不支持后来 9.2 加入的 `read_timeout`,本 SDK 不承诺统一的整体检查硬超时,也不增加自动重试。 + +详细设计和开发流程见 [连接管理设计](docs/connection-management-design.md)。实际验证情况见 [验收记录](docs/connection-management-validation.md)。 + -- Gitee From 78b40d88f58addbb1a148652864b11fbaef4dd96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:45:24 +0800 Subject: [PATCH 25/86] =?UTF-8?q?test(connections):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E6=98=BE=E5=BC=8F=E6=9C=8D=E5=8A=A1=E9=85=8D=E7=BD=AE=E7=9A=84?= =?UTF-8?q?=E9=9A=94=E7=A6=BB=E9=AA=8C=E6=94=B6=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/verify_connections.py | 64 +++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 scripts/verify_connections.py diff --git a/scripts/verify_connections.py b/scripts/verify_connections.py new file mode 100644 index 0000000..e553761 --- /dev/null +++ b/scripts/verify_connections.py @@ -0,0 +1,64 @@ +"""Run read-only probes against explicitly supplied isolated test services. + +Usage: python scripts/verify_connections.py --config path/to/fixtures.json +Fixture format: {"connection_name": {ConnectionConfig keyword arguments...}}. +Configuration loading belongs to this test caller, never to the SDK. +""" +import argparse +import asyncio +import json +from pathlib import Path + +from database import ConnectionConfig, Connections + + +def verify_sync(connections, name, backend): + with connections.create(name) as first, connections.create(name) as second: + assert first.native is not second.native + result = first.check() + first.close() + second.check() + if backend == "mysql": + with second.errors(operation="select_one"): + with second.native.cursor() as cursor: + cursor.execute("SELECT 1") + assert cursor.fetchone()[0] == 1 + return {"name": name, "mode": "sync", "scope": result.scope, + "target_count": len(result.targets), "result": "passed"} + + +async def verify_async(connections, name, backend): + async with await connections.create_async(name) as first: + async with await connections.create_async(name) as second: + assert first.native is not second.native + result = await first.check() + await first.close() + await second.check() + if backend == "mysql": + with second.errors(operation="select_one"): + async with await second.native.cursor() as cursor: + await cursor.execute("SELECT 1") + assert (await cursor.fetchone())[0] == 1 + return {"name": name, "mode": "async", "scope": result.scope, + "target_count": len(result.targets), "result": "passed"} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + args = parser.parse_args() + fixtures = json.loads(args.config.read_text(encoding="utf-8-sig")) + connections = Connections({ + name: ConnectionConfig(**parameters) + for name, parameters in fixtures.items() + }) + for name, parameters in fixtures.items(): + print(json.dumps(verify_sync(connections, name, parameters["backend"]))) + print(json.dumps(asyncio.run( + verify_async(connections, name, parameters["backend"]) + ))) + + +if __name__ == "__main__": + main() + -- Gitee From 9b3b340eb5be4e835253759212c0d992b19c5033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:47:23 +0800 Subject: [PATCH 26/86] =?UTF-8?q?feat(database):=20=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E8=BF=9E=E6=8E=A5=E5=BC=82=E5=B8=B8=E5=B9=B6=E4=BF=9D=E7=95=99?= =?UTF-8?q?=E9=A9=B1=E5=8A=A8=E5=8E=9F=E5=A7=8B=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/errors.py | 181 +++++++++++++++++++++++++++ tests/test_connection_errors.py | 211 ++++++++++++++++++++++++++++++++ 2 files changed, 392 insertions(+) create mode 100644 src/database/errors.py create mode 100644 tests/test_connection_errors.py diff --git a/src/database/errors.py b/src/database/errors.py new file mode 100644 index 0000000..42f3bf7 --- /dev/null +++ b/src/database/errors.py @@ -0,0 +1,181 @@ +"""Shared database errors and driver-independent conversion.""" + +import asyncio +import errno +import socket +from collections.abc import Callable, Iterable, Iterator +from contextlib import contextmanager + + +class DatabaseError(Exception): + """A database failure with connection context and the original exception.""" + + def __init__( + self, + message: str, + *, + connection_name: str | None = None, + backend: str | None = None, + phase: str | None = None, + operation: str | None = None, + target: str | None = None, + original: BaseException | None = None, + ) -> None: + super().__init__(message) + self.message = message + self.connection_name = connection_name + self.backend = backend + self.phase = phase + self.operation = operation + self.target = target + self.original = original + self.database_cleanup_errors: tuple[BaseException, ...] = () + + +class ConfigurationError(DatabaseError): + """Invalid configuration or connection parameters.""" + + +class DependencyError(DatabaseError): + """A required driver or driver capability is unavailable.""" + + +class AuthenticationError(DatabaseError): + """The server rejected the connection credentials.""" + + +class PermissionDeniedError(DatabaseError): + """The authenticated user lacks a required permission.""" + + +class DatabaseTimeoutError(DatabaseError): + """The driver or transport reported a timeout.""" + + +class DatabaseConnectionError(DatabaseError): + """A connection or transport failure.""" + + +class DriverError(DatabaseError): + """A driver error without a more specific classification.""" + + +class ConnectionClosedError(DatabaseError): + """The managed connection handle is closed.""" + + +_Classifier = Callable[[Exception], type[DatabaseError] | None] + +_CONNECTION_ERRNOS = frozenset( + { + errno.EHOSTUNREACH, + errno.ENETDOWN, + errno.ENETRESET, + errno.ENETUNREACH, + errno.ENOTCONN, + } +) + +_MESSAGES = { + ConfigurationError: "Invalid database configuration.", + DependencyError: "The required database driver or capability is unavailable.", + AuthenticationError: "Database authentication failed.", + PermissionDeniedError: "Database permission denied.", + DatabaseTimeoutError: "Database operation timed out.", + DatabaseConnectionError: "Database connection failed.", + DriverError: "The database driver reported an error.", + ConnectionClosedError: "The database connection is closed.", +} + + +def _classify_owned_error(error: Exception, phase: str) -> type[DatabaseError] | None: + if isinstance(error, (TimeoutError, asyncio.TimeoutError)): + return DatabaseTimeoutError + if isinstance(error, (ConnectionError, socket.gaierror)): + return DatabaseConnectionError + if isinstance(error, OSError): + if error.errno == errno.ETIMEDOUT: + return DatabaseTimeoutError + if error.errno in _CONNECTION_ERRNOS: + return DatabaseConnectionError + if phase == "create" and isinstance(error, (TypeError, ValueError)): + return ConfigurationError + return None + + +def convert_error( + error: BaseException, + *, + connection_name: str | None, + backend: str | None, + phase: str, + operation: str | None = None, + target: str | None = None, + classifier: _Classifier | None = None, + owned: bool = False, +) -> BaseException: + """Convert a driver error while retaining the original object and cause.""" + if not isinstance(error, Exception) or isinstance(error, DatabaseError): + return error + + error_type = classifier(error) if classifier is not None else None + if error_type is None and owned: + error_type = _classify_owned_error(error, phase) + if error_type is None: + return error + + converted = error_type( + _MESSAGES.get(error_type, "Database operation failed."), + connection_name=connection_name, + backend=backend, + phase=phase, + operation=operation, + target=target, + original=error, + ) + converted.__cause__ = error + return converted + + +@contextmanager +def error_context( + *, + connection_name: str | None, + backend: str | None, + phase: str, + operation: str | None = None, + target: str | None = None, + classifier: _Classifier | None = None, + owned: bool = False, +) -> Iterator[None]: + """Translate matching exceptions escaping a synchronous or async operation.""" + try: + yield + except Exception as error: + converted = convert_error( + error, + connection_name=connection_name, + backend=backend, + phase=phase, + operation=operation, + target=target, + classifier=classifier, + owned=owned, + ) + if converted is error: + raise + raise converted from error + + +def attach_cleanup_errors( + primary: BaseException, errors: Iterable[BaseException] +) -> None: + """Record cleanup failures without replacing the primary exception.""" + try: + additions = tuple(errors) + if additions: + existing = getattr(primary, "database_cleanup_errors", ()) + primary.database_cleanup_errors = existing + additions + except Exception: + # An exception subclass can prevent adding attributes. + pass diff --git a/tests/test_connection_errors.py b/tests/test_connection_errors.py new file mode 100644 index 0000000..d533d22 --- /dev/null +++ b/tests/test_connection_errors.py @@ -0,0 +1,211 @@ +import asyncio +import errno +import socket +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import Mock + +from database.errors import ( + AuthenticationError, + ConfigurationError, + ConnectionClosedError, + DatabaseConnectionError, + DatabaseError, + DatabaseTimeoutError, + DependencyError, + DriverError, + PermissionDeniedError, + attach_cleanup_errors, + convert_error, + error_context, +) + + +class ExampleDriverError(Exception): + pass + + +def classify_example(error): + if isinstance(error, ExampleDriverError): + return DriverError + return None + + +class ConnectionErrorTests(TestCase): + def test_converts_driver_error_and_preserves_native_details(self): + low_level = OSError("socket failed") + try: + raise ExampleDriverError("credentials in native message") from low_level + except ExampleDriverError as original: + original.code = 18 + original.details = {"labels": ["example"]} + original_traceback = original.__traceback__ + converted = convert_error( + original, + connection_name="reports", + backend="mongodb", + phase="check", + operation="ping", + target="db.example:27017", + classifier=lambda error: AuthenticationError, + ) + + self.assertIsInstance(converted, AuthenticationError) + self.assertIsInstance(converted, DatabaseError) + self.assertEqual(str(converted), converted.message) + self.assertNotIn("credentials in native message", str(converted)) + self.assertEqual(converted.connection_name, "reports") + self.assertEqual(converted.backend, "mongodb") + self.assertEqual(converted.phase, "check") + self.assertEqual(converted.operation, "ping") + self.assertEqual(converted.target, "db.example:27017") + self.assertEqual(converted.database_cleanup_errors, ()) + self.assertIs(converted.original, converted.__cause__) + self.assertIs(converted.original.__cause__, low_level) + self.assertIs(converted.original.__traceback__, original_traceback) + self.assertEqual(converted.original.code, 18) + self.assertEqual(converted.original.details, {"labels": ["example"]}) + + def test_database_error_and_interruptions_do_not_reach_classifier(self): + classifier = Mock(return_value=DriverError) + for error in ( + DatabaseError("failure"), + ConfigurationError("config"), + DependencyError("dependency"), + PermissionDeniedError("permission"), + ConnectionClosedError("closed"), + asyncio.CancelledError(), + KeyboardInterrupt(), + SystemExit(), + GeneratorExit(), + ): + with self.subTest(error=type(error).__name__): + self.assertIs( + convert_error( + error, + connection_name="reports", + backend="mongodb", + phase="create", + classifier=classifier, + owned=True, + ), + error, + ) + classifier.assert_not_called() + + def test_owned_errors_are_classified_by_type_or_errno(self): + for original, phase, expected in ( + (TimeoutError(), "check", DatabaseTimeoutError), + (asyncio.TimeoutError(), "check", DatabaseTimeoutError), + (OSError(errno.ETIMEDOUT, "timeout"), "check", DatabaseTimeoutError), + (ConnectionRefusedError(), "create", DatabaseConnectionError), + (OSError(errno.ENETUNREACH, "unknown host"), "check", DatabaseConnectionError), + (socket.gaierror(socket.EAI_NONAME, "unknown host"), "create", DatabaseConnectionError), + (TypeError("invalid argument"), "create", ConfigurationError), + (ValueError("invalid argument"), "create", ConfigurationError), + ): + with self.subTest(original=original, phase=phase): + self.assertIsInstance( + convert_error( + original, + connection_name="reports", + backend="mongodb", + phase=phase, + owned=True, + ), + expected, + ) + + def test_owned_unknown_errors_are_preserved(self): + for error in ( + OSError(errno.ENOENT, "connection timed out"), + OSError(errno.EACCES, "permission denied"), + TypeError("invalid argument"), + ValueError("invalid argument"), + RuntimeError("server unreachable"), + ): + with self.subTest(error=error): + self.assertIs( + convert_error(error, connection_name=None, backend=None, phase="check", owned=True), + error, + ) + + def test_context_reraises_unowned_errors_by_identity(self): + native_cause = ExampleDriverError("native error") + chained_business_error = RuntimeError("business error") + chained_business_error.__cause__ = native_cause + for error in ( + ValueError("business error"), + TypeError("business error"), + chained_business_error, + TimeoutError(), + ConnectionResetError(), + KeyboardInterrupt(), + SystemExit(), + ): + with self.subTest(error=type(error).__name__): + with self.assertRaises(type(error)) as result: + with error_context( + connection_name="reports", + backend="mongodb", + phase="operation", + classifier=classify_example, + ): + raise error + self.assertIs(result.exception, error) + self.assertIs(chained_business_error.__cause__, native_cause) + + def test_cleanup_errors_append_to_primary(self): + primary = ValueError("business error") + first = OSError("first cleanup failure") + second = asyncio.CancelledError("second cleanup failure") + attach_cleanup_errors(primary, ()) + self.assertFalse(hasattr(primary, "database_cleanup_errors")) + attach_cleanup_errors(primary, [first]) + attach_cleanup_errors(primary, (second,)) + self.assertEqual(primary.database_cleanup_errors, (first, second)) + + def test_restricted_exception_cannot_replace_primary(self): + class RestrictedError(Exception): + @property + def database_cleanup_errors(self): + return () + + primary = RestrictedError("business error") + attach_cleanup_errors(primary, [RuntimeError("cleanup failed")]) + self.assertEqual(primary.database_cleanup_errors, ()) + + +class AsyncConnectionErrorTests(IsolatedAsyncioTestCase): + async def test_context_translates_across_await(self): + original = ExampleDriverError("native error") + with self.assertRaises(DriverError) as result: + with error_context( + connection_name="reports", + backend="mongodb", + phase="operation", + operation="find", + classifier=classify_example, + ): + await asyncio.sleep(0) + raise original + + self.assertIs(result.exception.original, original) + self.assertIs(result.exception.__cause__, original) + self.assertIsNotNone(original.__traceback__) + self.assertEqual(result.exception.operation, "find") + + async def test_context_leaves_cancellation_unmodified(self): + original = asyncio.CancelledError() + classifier = Mock(return_value=DriverError) + with self.assertRaises(asyncio.CancelledError) as result: + with error_context( + connection_name="reports", + backend="mongodb", + phase="operation", + classifier=classifier, + owned=True, + ): + await asyncio.sleep(0) + raise original + self.assertIs(result.exception, original) + classifier.assert_not_called() -- Gitee From 4f1c9189843dd6074b6b7d6d5748a86d566f51c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:48:58 +0800 Subject: [PATCH 27/86] =?UTF-8?q?fix(database):=20=E4=BF=9D=E7=95=99?= =?UTF-8?q?=E5=8F=AA=E8=AF=BB=E5=BC=82=E5=B8=B8=E4=B8=8A=E7=9A=84=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E6=B8=85=E7=90=86=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/errors.py | 18 ++++++++++-------- tests/test_connection_errors.py | 12 ++++++------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/database/errors.py b/src/database/errors.py index 42f3bf7..644ec03 100644 --- a/src/database/errors.py +++ b/src/database/errors.py @@ -171,11 +171,13 @@ def attach_cleanup_errors( primary: BaseException, errors: Iterable[BaseException] ) -> None: """Record cleanup failures without replacing the primary exception.""" - try: - additions = tuple(errors) - if additions: - existing = getattr(primary, "database_cleanup_errors", ()) - primary.database_cleanup_errors = existing + additions - except Exception: - # An exception subclass can prevent adding attributes. - pass + additions = tuple(errors) + if not additions: + return + + # Use the base descriptor to bypass custom attribute hooks on exceptions. + attributes = BaseException.__dict__["__dict__"].__get__(primary) + existing = attributes.get("database_cleanup_errors", ()) + if not isinstance(existing, tuple): + existing = () + attributes["database_cleanup_errors"] = existing + additions diff --git a/tests/test_connection_errors.py b/tests/test_connection_errors.py index d533d22..5b427b6 100644 --- a/tests/test_connection_errors.py +++ b/tests/test_connection_errors.py @@ -164,15 +164,15 @@ class ConnectionErrorTests(TestCase): attach_cleanup_errors(primary, (second,)) self.assertEqual(primary.database_cleanup_errors, (first, second)) - def test_restricted_exception_cannot_replace_primary(self): + def test_cleanup_errors_attach_when_primary_disallows_attribute_assignment(self): class RestrictedError(Exception): - @property - def database_cleanup_errors(self): - return () + def __setattr__(self, name, value): + raise TypeError("exception attributes are read-only") primary = RestrictedError("business error") - attach_cleanup_errors(primary, [RuntimeError("cleanup failed")]) - self.assertEqual(primary.database_cleanup_errors, ()) + cleanup = RuntimeError("cleanup failed") + attach_cleanup_errors(primary, (cleanup,)) + self.assertEqual(primary.database_cleanup_errors, (cleanup,)) class AsyncConnectionErrorTests(IsolatedAsyncioTestCase): -- Gitee From 8cb9e3e4fb8ef9018679b1a80f3691c80613e6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:53:40 +0800 Subject: [PATCH 28/86] =?UTF-8?q?feat(database):=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E6=98=BE=E5=BC=8F=E8=BF=9E=E6=8E=A5=E9=85=8D=E7=BD=AE=E4=B8=8E?= =?UTF-8?q?=E9=A9=B1=E5=8A=A8=E5=8F=82=E6=95=B0=E6=98=A0=E5=B0=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/config.py | 239 ++++++++++++++++++++++++++++++ tests/test_connection_config.py | 248 ++++++++++++++++++++++++++++++++ 2 files changed, 487 insertions(+) create mode 100644 src/database/config.py create mode 100644 tests/test_connection_config.py diff --git a/src/database/config.py b/src/database/config.py new file mode 100644 index 0000000..34d103b --- /dev/null +++ b/src/database/config.py @@ -0,0 +1,239 @@ +"""Explicit configuration for managed database connections.""" + +from collections.abc import Mapping +from dataclasses import dataclass, field, replace +from enum import Enum +from typing import Any + +from .errors import ConfigurationError + + +class DatabaseType(str, Enum): + """Backends supported by the connection manager.""" + + MONGODB = "mongodb" + MYSQL = "mysql" + REDIS = "redis" + + def __str__(self) -> str: + return self.value + + +_TIMEOUT_OPTIONS = { + DatabaseType.MONGODB: "connectTimeoutMS", + DatabaseType.MYSQL: "connection_timeout", + DatabaseType.REDIS: "socket_connect_timeout", +} +_NATIVE_ALIASES = { + DatabaseType.MONGODB: {"ssl": "tls"}, + DatabaseType.MYSQL: { + "username": "user", + "passwd": "password", + "db": "database", + "connect_timeout": "connection_timeout", + "read_default_file": "option_files", + }, + DatabaseType.REDIS: {"charset": "encoding", "errors": "encoding_errors"}, +} + + +def _copy_containers(value: Any) -> Any: + """Copy built-in containers without copying driver objects or callbacks.""" + value_type = type(value) + if value_type is dict: + return {key: _copy_containers(item) for key, item in value.items()} + if value_type in (list, tuple, set, frozenset): + return value_type(_copy_containers(item) for item in value) + if value_type is bytearray: + return value.copy() + return value + + +def _is_nonempty_string(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _is_port(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535 + + +@dataclass(frozen=True) +class ConnectionConfig: + """Connection inputs supplied by the caller, independent of loaded drivers.""" + + backend: DatabaseType | str + host: str | list[str] | tuple[str, ...] | None = field(default=None, repr=False) + port: int | None = None + username: str | None = field(default=None, repr=False) + password: str | None = field(default=None, repr=False) + connect_timeout_s: int | None = None + options: Mapping[str, Any] = field(default_factory=dict, repr=False) + mode: str = "single" + nodes: tuple[tuple[str, int], ...] | list[tuple[str, int]] = field( + default=(), repr=False + ) + service_name: str | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + try: + backend = DatabaseType(self.backend) + except ValueError: + raise ConfigurationError( + "backend must be mongodb, mysql, or redis", phase="config" + ) from None + object.__setattr__(self, "backend", backend) + + if self.port is not None and not _is_port(self.port): + raise self._error("port must be an integer from 1 to 65535") + timeout = self.connect_timeout_s + if timeout is not None and ( + not isinstance(timeout, int) or isinstance(timeout, bool) or timeout <= 0 + ): + raise self._error("connect_timeout_s must be a positive integer") + for key in ("username", "password"): + value = getattr(self, key) + if value is not None and not isinstance(value, str): + raise self._error(f"{key} must be a string") + if not isinstance(self.options, Mapping) or any( + not isinstance(key, str) for key in self.options + ): + raise self._error("options must be a mapping with string keys") + if not isinstance(self.nodes, (list, tuple)): + raise self._error("nodes must contain (host, port) pairs") + for node in self.nodes: + if ( + not isinstance(node, (list, tuple)) + or len(node) != 2 + or not _is_nonempty_string(node[0]) + or not _is_port(node[1]) + ): + raise self._error("nodes must contain nonempty hosts and valid ports") + + if isinstance(self.host, (list, tuple)): + object.__setattr__(self, "host", self._copy(self.host)) + object.__setattr__(self, "options", self._copy(dict(self.options))) + object.__setattr__(self, "nodes", tuple((host, port) for host, port in self.nodes)) + self._validated_driver_kwargs() + + def snapshot(self) -> "ConnectionConfig": + """Return an independent copy of the configuration containers.""" + return replace(self) + + def driver_kwargs(self) -> dict[str, Any]: + """Return fresh native keyword arguments for this configuration.""" + return self._copy(self._validated_driver_kwargs()) + + def _error(self, message: str) -> ConfigurationError: + return ConfigurationError(message, backend=self.backend, phase="config") + + def _copy(self, value: Any) -> Any: + try: + return _copy_containers(value) + except RecursionError: + raise self._error("configuration containers must be acyclic") from None + + def _native_key(self, key: str) -> str: + if self.backend == DatabaseType.MONGODB: + key = key.lower() + return _NATIVE_ALIASES[self.backend].get(key, key) + + def _validated_driver_kwargs(self) -> dict[str, Any]: + options = dict(self.options) + seen = set() + for key in options: + if not isinstance(key, str): + raise self._error("options must be a mapping with string keys") + native_key = self._native_key(key) + if native_key in seen: + raise self._error("options contains multiple names for the same driver argument") + seen.add(native_key) + + timeout = self.connect_timeout_s + if timeout is not None and self.backend == DatabaseType.MONGODB: + timeout *= 1000 + common = { + "host": self.host, + "port": self.port, + "user" if self.backend == DatabaseType.MYSQL else "username": self.username, + "password": self.password, + _TIMEOUT_OPTIONS[self.backend]: timeout, + } + for key, value in common.items(): + if value is None: + continue + if self._native_key(key) in seen: + raise self._error(f"{key} is specified both directly and in options") + options[key] = value + + host = options.get("host") + if host is not None: + if self.backend == DatabaseType.MONGODB and isinstance(host, (list, tuple)): + if not host or not all(_is_nonempty_string(address) for address in host): + raise self._error("host must contain nonempty MongoDB addresses") + elif not _is_nonempty_string(host): + raise self._error("host must be a nonempty string") + + if self.backend == DatabaseType.REDIS: + self._validate_redis_target(options) + else: + if self.mode != "single" or self.nodes or self.service_name is not None: + raise self._error("mode, nodes, and service_name apply to Redis topology") + socket = self.backend == DatabaseType.MYSQL and _is_nonempty_string( + options.get("unix_socket") + ) + if host is None and not socket: + raise self._error("an explicit connection target is required") + return options + + def _validate_redis_target(self, options: dict[str, Any]) -> None: + if self.mode not in ("single", "sentinel", "cluster"): + raise self._error("Redis mode must be single, sentinel, or cluster") + + if self.mode == "sentinel": + self._validate_sentinel_target(options) + return + + if self.service_name is not None or "sentinels" in options: + raise self._error("service_name and sentinels apply to Redis Sentinel") + + pool = options.get("connection_pool") + if self.mode == "cluster": + if pool is not None: + raise self._error("connection_pool applies to Redis single mode") + if self.nodes and "startup_nodes" in options: + raise self._error("nodes and options.startup_nodes cannot both be set") + if not self.nodes and options.get("host") is None and not options.get("startup_nodes"): + raise self._error("Redis Cluster requires host, nodes, or options.startup_nodes") + return + + if self.nodes or "startup_nodes" in options: + raise self._error("Redis single mode cannot specify nodes") + if pool is not None: + pool_conflicts = { + "host", "port", "username", "password", "socket_connect_timeout", + "unix_socket_path", "credential_provider", + } + if pool_conflicts.intersection(options): + raise self._error("connection_pool cannot be combined with connection parameters") + if not pool: + raise self._error("connection_pool must supply an explicit target") + elif options.get("host") is None and not _is_nonempty_string( + options.get("unix_socket_path") + ): + raise self._error("Redis single mode requires an explicit connection target") + + def _validate_sentinel_target(self, options: dict[str, Any]) -> None: + if not _is_nonempty_string(self.service_name): + raise self._error("Redis Sentinel requires a nonempty service_name") + if {"host", "port", "startup_nodes"}.intersection(options): + raise self._error("Redis Sentinel requires sentinel addresses through nodes or sentinels") + if self.nodes and "sentinels" in options: + raise self._error("nodes and options.sentinels cannot both be set") + if not self.nodes and not options.get("sentinels"): + raise self._error("Redis Sentinel requires nodes or options.sentinels") + sentinel_kwargs = options.get("sentinel_kwargs") + if options.get("connection_pool") is not None or ( + isinstance(sentinel_kwargs, Mapping) + and sentinel_kwargs.get("connection_pool") is not None + ): + raise self._error("connection_pool applies to Redis single mode") diff --git a/tests/test_connection_config.py b/tests/test_connection_config.py new file mode 100644 index 0000000..b9cee0b --- /dev/null +++ b/tests/test_connection_config.py @@ -0,0 +1,248 @@ +import os +from types import MappingProxyType +import unittest +from unittest.mock import patch + +from database.config import ConnectionConfig, DatabaseType +from database.errors import ConfigurationError + + +class ConnectionConfigTests(unittest.TestCase): + def test_common_fields_map_to_native_driver_arguments(self): + for backend, user_key, timeout_key, timeout in ( + (DatabaseType.MONGODB, "username", "connectTimeoutMS", 2000), + (DatabaseType.MYSQL, "user", "connection_timeout", 2), + (DatabaseType.REDIS, "username", "socket_connect_timeout", 2), + ): + with self.subTest(backend=backend): + config = ConnectionConfig( + backend.value, host="db.internal", port=1234, + username="app", password="", connect_timeout_s=2, + ) + self.assertIs(config.backend, backend) + self.assertEqual(config.driver_kwargs(), { + "host": "db.internal", "port": 1234, user_key: "app", + "password": "", timeout_key: timeout, + }) + + def test_explicit_configuration_does_not_read_environment(self): + with patch.object(os, "environ", None), patch.object( + os, "getenv", side_effect=AssertionError("environment read") + ), patch("builtins.open", side_effect=AssertionError("file read")): + host = "mongodb://${DB_HOST}:27017" + config = ConnectionConfig("mongodb", host=host) + self.assertEqual(config.snapshot().driver_kwargs(), {"host": host}) + with self.assertRaises(ConfigurationError) as raised: + ConnectionConfig("mongodb", username="app", password="secret") + self.assertEqual(raised.exception.backend, DatabaseType.MONGODB) + self.assertEqual(raised.exception.phase, "config") + + def test_requires_an_explicit_connection_target(self): + for backend, options in ( + ("mongodb", {}), + ("mysql", {"pool_name": "default"}), + ("mysql", {"option_files": "/my.cnf"}), + ("redis", {"connection_pool": None}), + ): + with self.subTest(backend=backend, options=options): + with self.assertRaisesRegex(ConfigurationError, "explicit"): + ConnectionConfig(backend, options=options) + + def test_accepts_native_host_and_local_sockets(self): + for backend, options in ( + ("mongodb", {"host": ["first:27017", "second:27017"]}), + ("mysql", {"unix_socket": "/tmp/mysql.sock", "db": "app"}), + ("redis", {"unix_socket_path": "/tmp/redis.sock"}), + ): + with self.subTest(backend=backend, options=options): + self.assertEqual( + ConnectionConfig(backend, options=options).driver_kwargs(), options + ) + + def test_rejects_invalid_sdk_field_values(self): + for key, values in ( + ("backend", ("sqlite", None, [])), + ("host", ("", " ", [], ["valid", ""], 10)), + ("port", (True, 0, -1, 65536, 27017.0, "27017")), + ("connect_timeout_s", (True, 0, -1, 1.5, "2")), + ("username", (True, 1)), + ("password", (True, 1)), + ("options", (None, [], {1: "invalid"})), + ("mode", ("sentinel", "invalid")), + ("nodes", ([("host", 1)], None, "host")), + ("service_name", ("sentinel",)), + ): + for value in values: + with self.subTest(key=key, value=value): + values_by_field = {"backend": "mongodb", "host": "mongo"} + values_by_field[key] = value + with self.assertRaises(ConfigurationError): + ConnectionConfig(**values_by_field) + for backend in ("mysql", "redis"): + with self.subTest(backend=backend): + with self.assertRaisesRegex(ConfigurationError, "host"): + ConnectionConfig(backend, host=["host"]) + + def test_native_aliases_alone_are_preserved(self): + options = {"user": "db-user", "passwd": "example", "db": "app", "connect_timeout": 3} + self.assertEqual( + ConnectionConfig("mysql", host="db.internal", options=options).driver_kwargs(), + {"host": "db.internal", **options}, + ) + + def test_rejects_duplicate_common_fields_and_native_aliases(self): + for backend, common_fields, options in ( + ("mongodb", {"host": "mongo"}, {"host": "second"}), + ("mongodb", {"connect_timeout_s": 2}, {"CONNECTTIMEOUTMS": 5}), + ("mongodb", {"username": "app"}, {"UserName": "other"}), + ("mongodb", {}, {"ssl": True, "tls": False}), + ("mongodb", {}, {"socketTimeoutMS": 1, "sockettimeoutms": 5}), + ("mysql", {"username": "app"}, {"username": "other"}), + ("mysql", {"username": "app"}, {"user": None}), + ("mysql", {"password": "secret"}, {"passwd": "other"}), + ("mysql", {"connect_timeout_s": 2}, {"connect_timeout": 5}), + ("mysql", {}, {"user": "first", "username": "second"}), + ("mysql", {}, {"db": "first", "database": "second"}), + ("mysql", {}, {"passwd": "first", "password": "second"}), + ("mysql", {}, {"connect_timeout": 1, "connection_timeout": 5}), + ("redis", {"port": 6379}, {"port": 6380}), + ("redis", {}, {"encoding": "utf-8", "charset": "ascii"}), + ): + with self.subTest(backend=backend, fields=tuple(common_fields), keys=tuple(options)): + settings = {"host": "db.internal", **common_fields} + with self.assertRaises(ConfigurationError): + ConnectionConfig(backend, options=options, **settings) + + def test_snapshot_and_driver_arguments_keep_opaque_objects(self): + class Opaque: + def __deepcopy__(self, memo): + raise AssertionError("must preserve native identity") + + def callback(): + pass + + node = Opaque() + startup_nodes = [node] + nested = {"list": [bytearray(b"A")], "tuple": ({1},), "callback": callback} + config = ConnectionConfig( + "redis", mode="cluster", + options=MappingProxyType({"startup_nodes": startup_nodes, "option": nested}), + ) + startup_nodes.clear() + nested["list"][0].extend(b"B") + snapshot = config.snapshot() + kwargs = config.driver_kwargs() + self.assertIs(kwargs["startup_nodes"][0], node) + self.assertIs(kwargs["option"]["callback"], callback) + self.assertEqual(kwargs["option"]["list"], [bytearray(b"A")]) + kwargs["startup_nodes"].clear() + kwargs["option"]["tuple"][0].add(2) + snapshot.options["option"]["list"][0].extend(b"C") + self.assertEqual(config.driver_kwargs()["option"]["tuple"], ({1},)) + self.assertEqual(config.driver_kwargs()["option"]["list"], [bytearray(b"A")]) + + def test_mongodb_address_list_is_copied(self): + hosts = ["first:27017", "second:27017"] + config = ConnectionConfig("mongodb", host=hosts) + hosts.clear() + kwargs = config.driver_kwargs() + snapshot = config.snapshot() + kwargs["host"].append("third") + config.host.append("changed") + self.assertEqual(snapshot.driver_kwargs()["host"], ["first:27017", "second:27017"]) + + def test_recursive_containers_are_rejected(self): + options = {} + options["recursive"] = options + with self.assertRaisesRegex(ConfigurationError, "acyclic"): + ConnectionConfig("mongodb", host="mongo", options=options) + + def test_repr_hides_connection_values(self): + config = ConnectionConfig( + "mongodb", host="mongodb://user:uri-secret@mongo:27017/", + username="user", password="password-secret", options={"token": "option-secret"}, + ) + representation = repr(config) + for sensitive in ("mongodb://", "uri-secret", "password-secret", "option-secret", "token"): + self.assertNotIn(sensitive, representation) + + def test_redis_sentinel_passes_data_credentials_separately_from_sentinel_credentials(self): + nodes = [["sentinel", 26379]] + config = ConnectionConfig( + "redis", mode="sentinel", nodes=nodes, service_name="primary", + username="app", password="data-secret", + options={"sentinel_kwargs": {"password": "sentinel-secret"}}, + ) + nodes[0][1] = 12345 + self.assertEqual(config.nodes, (("sentinel", 26379),)) + self.assertEqual(config.driver_kwargs(), { + "username": "app", "password": "data-secret", + "sentinel_kwargs": {"password": "sentinel-secret"}, + }) + without_sentinel_auth = ConnectionConfig( + "redis", mode="sentinel", nodes=config.nodes, service_name="primary", + password="data-secret", + ) + self.assertEqual(without_sentinel_auth.driver_kwargs(), {"password": "data-secret"}) + + def test_redis_allows_native_node_arguments(self): + sentinels = [("sentinel", 26379)] + sentinel = ConnectionConfig( + "redis", mode="sentinel", service_name="primary", options={"sentinels": sentinels} + ) + self.assertEqual(sentinel.driver_kwargs(), {"sentinels": sentinels}) + cluster_node = object() + cluster = ConnectionConfig( + "redis", mode="cluster", host="bootstrap", options={"startup_nodes": [cluster_node]} + ) + kwargs = cluster.driver_kwargs() + self.assertEqual(kwargs["host"], "bootstrap") + self.assertIs(kwargs["startup_nodes"][0], cluster_node) + cluster = ConnectionConfig( + "redis", mode="cluster", nodes=[("first", 7000), ("second", 7001)] + ) + self.assertEqual(cluster.driver_kwargs(), {}) + + def test_redis_rejects_incomplete_or_conflicting_topologies(self): + for config in ( + {"mode": "sentinel", "nodes": [("s", 26379)]}, + {"mode": "sentinel", "service_name": "primary", "options": {"sentinels": []}}, + {"mode": "sentinel", "service_name": "primary", "nodes": [("s", 26379)], + "options": {"sentinels": [("other", 26379)]}}, + {"mode": "sentinel", "service_name": "primary", "nodes": [("s", 26379)], + "options": {"sentinel_kwargs": {"connection_pool": object()}}}, + {"mode": "cluster", "port": 7000}, + {"mode": "cluster", "nodes": [("c", 7000)], "options": {"startup_nodes": [object()]}}, + {"mode": "cluster", "options": {"startup_nodes": []}}, + {"mode": "cluster", "nodes": [("c", True)]}, + {"mode": "cluster", "nodes": [("", 7000)]}, + {"mode": "cluster", "nodes": [("c", 7000, "extra")]}, + {"mode": "single", "host": "redis", "nodes": [("r", 6379)]}, + {"mode": "single", "host": "redis", "service_name": "primary"}, + {"mode": "invalid", "host": "redis"}, + ): + with self.subTest(config=config): + with self.assertRaises(ConfigurationError): + ConnectionConfig("redis", **config) + + def test_redis_external_pool_is_a_target_and_cannot_receive_connection_parameters(self): + pool = object() + self.assertIs( + ConnectionConfig("redis", options={"connection_pool": pool}) + .snapshot().driver_kwargs()["connection_pool"], + pool, + ) + for key, value in ( + ("host", "redis"), ("port", 6379), ("username", "app"), ("password", ""), + ("connect_timeout_s", 1), + ): + with self.subTest(field=key): + with self.assertRaisesRegex(ConfigurationError, "connection_pool"): + ConnectionConfig("redis", options={"connection_pool": pool}, **{key: value}) + for key, value in ( + ("host", "redis"), ("port", 6379), ("username", "app"), ("password", ""), + ("socket_connect_timeout", 1), ("unix_socket_path", "/tmp/redis.sock"), + ): + with self.subTest(option=key): + with self.assertRaisesRegex(ConfigurationError, "connection_pool"): + ConnectionConfig("redis", options={"connection_pool": pool, key: value}) -- Gitee From 03af1ceca93bf9f1a458198c703c764220779727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:54:14 +0800 Subject: [PATCH 29/86] =?UTF-8?q?feat(connections):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E7=8B=AC=E7=AB=8B=E5=8E=9F=E7=94=9F=E5=8F=A5=E6=9F=84=E4=B8=8E?= =?UTF-8?q?=E6=98=BE=E5=BC=8F=E6=A3=80=E6=9F=A5=E7=94=9F=E5=91=BD=E5=91=A8?= =?UTF-8?q?=E6=9C=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/_adapters/__init__.py | 1 + src/database/connections.py | 207 +++++++++++++++++++++++++++++ tests/test_connections.py | 177 ++++++++++++++++++++++++ 3 files changed, 385 insertions(+) create mode 100644 src/database/_adapters/__init__.py create mode 100644 src/database/connections.py create mode 100644 tests/test_connections.py diff --git a/src/database/_adapters/__init__.py b/src/database/_adapters/__init__.py new file mode 100644 index 0000000..34b6f58 --- /dev/null +++ b/src/database/_adapters/__init__.py @@ -0,0 +1 @@ +"""Backend adapters loaded only when a connection is created.""" diff --git a/src/database/connections.py b/src/database/connections.py new file mode 100644 index 0000000..bb1894a --- /dev/null +++ b/src/database/connections.py @@ -0,0 +1,207 @@ +"""Named configurations and explicitly managed native clients.""" +import asyncio +import importlib +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from time import perf_counter + +from .config import ConnectionConfig +from .errors import ( + ConfigurationError, ConnectionClosedError, DependencyError, + attach_cleanup_errors, error_context, +) + + +@dataclass(frozen=True) +class CheckResult: + connection_name: str + backend: str + mode: str + checked_at: datetime + elapsed_ms: float + scope: str + targets: tuple[str, ...] + + +class Connection: + """Own a native client without proxying its operations.""" + + def __init__(self, native, adapter, name, backend, mode): + self._native = native + self._adapter = adapter + self.name = name + self.backend = backend + self.mode = mode + self._closed = False + + @property + def closed(self): + return self._closed + + def _ensure_open(self, phase): + if self._closed: + raise ConnectionClosedError( + "The database connection is closed.", connection_name=self.name, + backend=self.backend, phase=phase, + ) + + @property + def native(self): + self._ensure_open("operation") + return self._native + + def _context(self, phase, *, operation=None, owned=True): + return error_context( + connection_name=self.name, backend=self.backend, phase=phase, + operation=operation, classifier=self._adapter.classify, owned=owned, + ) + + def errors(self, *, operation=None): + self._ensure_open("operation") + return self._context("operation", operation=operation, owned=False) + + def check(self): + self._ensure_open("check") + started = perf_counter() + checked_at = datetime.now(timezone.utc) + with self._context("check"): + scope, targets = self._adapter.check(self._native) + return CheckResult( + self.name, self.backend, self.mode, checked_at, + (perf_counter() - started) * 1000, scope, tuple(targets), + ) + + def close(self): + if self._closed: + return + with self._context("close"): + self._adapter.close(self._native) + self._closed = True + + def __enter__(self): + self._ensure_open("create") + return self + + def __exit__(self, exc_type, exc, traceback): + try: + self.close() + except Exception as cleanup_error: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup_error,)) + return False + + +class AsyncConnection(Connection): + """A native async client tied to its creating event loop.""" + + def __init__(self, native, adapter, name, backend, mode, loop): + super().__init__(native, adapter, name, backend, mode) + self._loop = loop + + def _ensure_loop(self, phase): + if asyncio.get_running_loop() is not self._loop: + raise ConfigurationError( + "Use the connection in its creating event loop.", + connection_name=self.name, backend=self.backend, phase=phase, + ) + + async def check(self): + self._ensure_loop("check") + self._ensure_open("check") + started = perf_counter() + checked_at = datetime.now(timezone.utc) + with self._context("check"): + scope, targets = await self._adapter.check_async(self._native) + return CheckResult( + self.name, self.backend, self.mode, checked_at, + (perf_counter() - started) * 1000, scope, tuple(targets), + ) + + async def close(self): + self._ensure_loop("close") + if self._closed: + return + with self._context("close"): + await self._adapter.close_async(self._native) + self._closed = True + + def __enter__(self): + raise TypeError("Use 'async with' for an asynchronous connection.") + + async def __aenter__(self): + self._ensure_loop("create") + self._ensure_open("create") + return self + + async def __aexit__(self, exc_type, exc, traceback): + try: + await self.close() + except Exception as cleanup_error: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup_error,)) + return False + + +class Connections: + """Reuse configuration snapshots, never native client instances.""" + + def __init__(self, configs: Mapping[str, ConnectionConfig]): + if not isinstance(configs, Mapping): + raise ConfigurationError("Expected a mapping of named configurations.", phase="config") + self._configs = {} + for name, config in configs.items(): + if not isinstance(name, str) or not name.strip(): + raise ConfigurationError("Connection names must be nonempty strings.", phase="config") + if not isinstance(config, ConnectionConfig): + raise ConfigurationError( + "Expected a ConnectionConfig.", connection_name=name, phase="config", + ) + self._configs[name] = config.snapshot() + + def _config(self, name): + if not isinstance(name, str) or name not in self._configs: + raise ConfigurationError( + "Unknown connection name.", connection_name=name if isinstance(name, str) else None, + phase="config", + ) + return self._configs[name].snapshot() + + def _adapter(self, config, name, is_async): + backend = config.backend.value + try: + module = importlib.import_module("database._adapters." + backend) + adapter = module.Adapter(config, name, is_async=is_async) + except ImportError as original: + # Imports happen only after a backend is explicitly selected. + raise DependencyError( + "Install the selected database driver and its required capabilities.", + connection_name=name, backend=backend, phase="create", original=original, + ) from original + return adapter + + def create(self, name): + config = self._config(name) + adapter = self._adapter(config, name, False) + with error_context( + connection_name=name, backend=config.backend.value, phase="create", + classifier=adapter.classify, owned=True, + ): + native = adapter.create() + return Connection(native, adapter, name, config.backend.value, config.mode) + + async def create_async(self, name): + config = self._config(name) + adapter = self._adapter(config, name, True) + loop = asyncio.get_running_loop() + with error_context( + connection_name=name, backend=config.backend.value, phase="create", + classifier=adapter.classify, owned=True, + ): + native = await adapter.create_async() + return AsyncConnection( + native, adapter, name, config.backend.value, config.mode, loop, + ) + diff --git a/tests/test_connections.py b/tests/test_connections.py new file mode 100644 index 0000000..913a42f --- /dev/null +++ b/tests/test_connections.py @@ -0,0 +1,177 @@ +import asyncio +import sys +import unittest +from datetime import timezone +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock, patch + +from database.config import ConnectionConfig +from database.connections import Connections +from database.errors import ( + ConfigurationError, ConnectionClosedError, DependencyError, DriverError, +) + + +class NativeFailure(Exception): + pass + + +def adapter_factory(config, name, is_async=False): + return SimpleNamespace( + config=config, name=name, + create=Mock(side_effect=lambda: object()), + create_async=AsyncMock(side_effect=lambda: object()), + check=Mock(return_value=("test_scope", ("test-target",))), + check_async=AsyncMock(return_value=("test_scope", ("test-target",))), + close=Mock(), close_async=AsyncMock(), + classify=lambda error: DriverError if isinstance(error, NativeFailure) else None, + ) + + +class ConnectionTests(unittest.TestCase): + def setUp(self): + self.registry = Connections({ + "main": ConnectionConfig(backend="mysql", host="localhost"), + "cache": ConnectionConfig(backend="redis", host="localhost"), + }) + self.loader = patch( + "database.connections.importlib.import_module", + return_value=SimpleNamespace(Adapter=adapter_factory), + ) + self.loader.start() + self.addCleanup(self.loader.stop) + + def test_independent_instances_explicit_checks_and_close(self): + first = self.registry.create("main") + second = self.registry.create("main") + self.assertIsNot(first.native, second.native) + first._adapter.check.assert_not_called() + result = first.check() + self.assertEqual(result.connection_name, "main") + self.assertEqual(result.targets, ("test-target",)) + self.assertEqual(result.checked_at.tzinfo, timezone.utc) + self.assertGreaterEqual(result.elapsed_ms, 0) + first.close() + first.close() + first._adapter.close.assert_called_once() + self.assertFalse(second.closed) + second.check() + for action in (lambda: first.native, first.check): + with self.assertRaises(ConnectionClosedError): + action() + second.close() + + def test_error_context_preserves_original_and_business_error(self): + with self.registry.create("main") as handle: + original = NativeFailure("detail") + with self.assertRaises(DriverError) as caught: + with handle.errors(operation="read"): + raise original + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.operation, "read") + business = ValueError("business") + with self.assertRaises(ValueError) as caught: + with handle.errors(): + raise business + self.assertIs(caught.exception, business) + + def test_cleanup_failure_does_not_replace_body_exception(self): + handle = self.registry.create("main") + cleanup = NativeFailure("cleanup") + handle._adapter.close.side_effect = cleanup + body = RuntimeError("body") + with self.assertRaises(RuntimeError) as caught: + with handle: + raise body + self.assertIs(caught.exception, body) + self.assertIs(body.database_cleanup_errors[0].original, cleanup) + + def test_check_failure_and_direct_close_failure_are_classified(self): + handle = self.registry.create("main") + original = NativeFailure() + handle._adapter.check.side_effect = original + with self.assertRaises(DriverError) as caught: + handle.check() + self.assertEqual(caught.exception.phase, "check") + self.assertIs(caught.exception.original, original) + handle._adapter.close.side_effect = original + with self.assertRaises(DriverError) as caught: + handle.close() + self.assertEqual(caught.exception.phase, "close") + self.assertFalse(handle.closed) + + def test_unknown_name_and_bad_registrations(self): + with self.assertRaises(ConfigurationError): + self.registry.create("missing") + for configs in ([], {"": ConnectionConfig("mysql", host="localhost")}, {"main": {}}): + with self.subTest(configs_type=type(configs)), self.assertRaises(ConfigurationError): + Connections(configs) + + def test_import_failure_has_context_and_original(self): + original = ModuleNotFoundError("driver") + with patch("database.connections.importlib.import_module", side_effect=original): + with self.assertRaises(DependencyError) as caught: + self.registry.create("main") + self.assertIs(caught.exception.original, original) + self.assertEqual(caught.exception.connection_name, "main") + + +class AsyncConnectionTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.registry = Connections({"main": ConnectionConfig("mysql", host="localhost")}) + self.loader = patch( + "database.connections.importlib.import_module", + return_value=SimpleNamespace(Adapter=adapter_factory), + ) + self.loader.start() + self.addCleanup(self.loader.stop) + + async def test_async_independence_and_lifecycle(self): + async with await self.registry.create_async("main") as first: + async with await self.registry.create_async("main") as second: + self.assertIsNot(first.native, second.native) + first._adapter.check_async.assert_not_awaited() + self.assertEqual((await first.check()).scope, "test_scope") + await first.close() + await first.close() + first._adapter.close_async.assert_awaited_once() + await second.check() + with self.assertRaises(ConnectionClosedError): + await first.check() + self.assertTrue(first.closed) + self.assertTrue(second.closed) + + async def test_async_cleanup_preserves_cancellation(self): + handle = await self.registry.create_async("main") + cleanup = NativeFailure() + handle._adapter.close_async.side_effect = cleanup + cancellation = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError) as caught: + async with handle: + raise cancellation + self.assertIs(caught.exception, cancellation) + self.assertIs(cancellation.database_cleanup_errors[0].original, cleanup) + + async def test_cross_loop_is_rejected_and_creating_loop_can_close(self): + handle = await self.registry.create_async("main") + async def wrong_loop(): + with self.assertRaises(ConfigurationError): + await handle.check() + with self.assertRaises(ConfigurationError): + await handle.close() + await asyncio.to_thread(lambda: asyncio.run(wrong_loop())) + await handle.close() + + async def test_native_errors_across_await_remain_original(self): + handle = await self.registry.create_async("main") + original = NativeFailure() + async def failing(): + await asyncio.sleep(0) + raise original + with self.assertRaises(DriverError) as caught: + with handle.errors(operation="async_read"): + await failing() + self.assertIs(caught.exception.original, original) + await handle.close() + -- Gitee From d0706e7fa4b4c368897c303fdf5f1880a32ba29d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:54:30 +0800 Subject: [PATCH 30/86] =?UTF-8?q?feat(database):=20=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E8=BF=9E=E6=8E=A5=E9=85=8D=E7=BD=AE=E4=B8=8E?= =?UTF-8?q?=E5=8F=A5=E6=9F=84=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/__init__.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/database/__init__.py b/src/database/__init__.py index b3e50b1..01038c3 100644 --- a/src/database/__init__.py +++ b/src/database/__init__.py @@ -4,3 +4,12 @@ except ImportError: __version__ = '0.0.0+unknown' __description__ = 'database SDK' + +from .config import ConnectionConfig, DatabaseType +from .connections import AsyncConnection, CheckResult, Connection, Connections +from .errors import DatabaseError + +__all__ = [ + 'ConnectionConfig', 'DatabaseType', 'Connections', 'Connection', + 'AsyncConnection', 'CheckResult', 'DatabaseError', +] -- Gitee From ce745eae9e9f3ab914d58c567e4190aaad57b34a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:59:50 +0800 Subject: [PATCH 31/86] =?UTF-8?q?feat(mongodb):=20=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E8=BF=9E=E6=8E=A5=E7=AE=A1=E7=90=86=E4=B8=8E?= =?UTF-8?q?=E6=98=BE=E5=BC=8F=E6=8E=A2=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/_adapters/mongodb.py | 82 ++++++++++++++ tests/test_managed_mongodb.py | 175 ++++++++++++++++++++++++++++++ 2 files changed, 257 insertions(+) create mode 100644 src/database/_adapters/mongodb.py create mode 100644 tests/test_managed_mongodb.py diff --git a/src/database/_adapters/mongodb.py b/src/database/_adapters/mongodb.py new file mode 100644 index 0000000..5eb7803 --- /dev/null +++ b/src/database/_adapters/mongodb.py @@ -0,0 +1,82 @@ +"""MongoDB operations for managed connections.""" + +from pymongo import errors as mongo_errors + +from ..config import ConnectionConfig +from ..errors import ( + AuthenticationError, + ConfigurationError, + DatabaseConnectionError, + DatabaseError, + DatabaseTimeoutError, + DependencyError, + DriverError, + PermissionDeniedError, + error_context, +) +from ..mongodb import Client + + +class Adapter: + def __init__(self, config: ConnectionConfig, name: str, is_async: bool = False): + self._config = config + self._name = name + + def create(self): + return Client.create(**self._config.driver_kwargs()) + + async def create_async(self): + try: + return Client.create_async(**self._config.driver_kwargs()) + except ImportError as original: + raise DependencyError( + "Install PyMongo >= 4.9 for asynchronous MongoDB connections.", + connection_name=self._name, + backend="mongodb", + phase="create", + original=original, + ) from original + + @staticmethod + def classify(error: Exception) -> type[DatabaseError] | None: + if not isinstance(error, mongo_errors.PyMongoError): + return None + if isinstance(error, mongo_errors.OperationFailure): + if error.code == 18: + return AuthenticationError + if error.code == 13: + return PermissionDeniedError + if error.timeout: + return DatabaseTimeoutError + if isinstance(error, mongo_errors.ConnectionFailure): + return DatabaseConnectionError + if isinstance(error, mongo_errors.ConfigurationError): + return ConfigurationError + return DriverError + + def _ping_errors(self): + return error_context( + connection_name=self._name, + backend="mongodb", + phase="check", + operation="ping", + target="selected_server", + classifier=self.classify, + owned=True, + ) + + def check(self, native) -> tuple[str, tuple[str, ...]]: + with self._ping_errors(): + native.admin.command("ping", read_preference=native.read_preference) + return "selected_server", () + + async def check_async(self, native) -> tuple[str, tuple[str, ...]]: + with self._ping_errors(): + await native.admin.command("ping", read_preference=native.read_preference) + return "selected_server", () + + def close(self, native) -> None: + native.close() + + async def close_async(self, native) -> None: + await native.close() diff --git a/tests/test_managed_mongodb.py b/tests/test_managed_mongodb.py new file mode 100644 index 0000000..e7e77d3 --- /dev/null +++ b/tests/test_managed_mongodb.py @@ -0,0 +1,175 @@ +import asyncio +from types import SimpleNamespace +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, Mock, patch + +from pymongo import ReadPreference +from pymongo import errors as mongo_errors + +from database import ConnectionConfig, Connections +from database.errors import ( + AuthenticationError, + ConfigurationError, + DatabaseConnectionError, + DatabaseTimeoutError, + DependencyError, + DriverError, + PermissionDeniedError, +) + + +class MongoDBConnectionTests(TestCase): + def setUp(self): + self.connections = Connections({ + "reports": ConnectionConfig( + "mongodb", + host=["server.test", "replica.test"], + port=27018, + username="reporter", + password="sample-password", + connect_timeout_s=2, + options={"authSource": "admin", "read_preference": ReadPreference.SECONDARY}, + ) + }) + self.native = SimpleNamespace( + admin=SimpleNamespace(command=Mock()), + read_preference=ReadPreference.SECONDARY, + close=Mock(), + ) + + @patch("database.mongodb.Client.create") + def test_create_preserves_driver_options_and_check_uses_read_preference(self, create): + create.return_value = self.native + with self.connections.create("reports") as connection: + self.assertIs(connection.native, self.native) + self.native.admin.command.assert_not_called() + + result = connection.check() + self.assertEqual(result.scope, "selected_server") + self.assertEqual(result.targets, ()) + self.native.admin.command.assert_called_once_with( + "ping", read_preference=ReadPreference.SECONDARY + ) + + create.assert_called_once_with( + host=["server.test", "replica.test"], + port=27018, + username="reporter", + password="sample-password", + connectTimeoutMS=2000, + authSource="admin", + read_preference=ReadPreference.SECONDARY, + ) + self.native.close.assert_called_once_with() + + @patch("database.mongodb.Client.create") + def test_check_translates_native_authentication_failure(self, create): + create.return_value = self.native + original = mongo_errors.OperationFailure( + "credentials in original error", code=18, details={"errorLabels": ["test"]} + ) + self.native.admin.command.side_effect = original + with self.connections.create("reports") as connection: + with self.assertRaises(AuthenticationError) as caught: + connection.check() + + error = caught.exception + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + self.assertTrue(error.original.has_error_label("test")) + self.assertEqual(error.original.code, 18) + self.assertEqual(error.phase, "check") + self.assertEqual(error.operation, "ping") + self.assertEqual(error.target, "selected_server") + self.assertNotIn("credentials in original error", str(error)) + + @patch("database.mongodb.Client.create") + def test_native_operations_classify_supported_driver_errors(self, create): + create.return_value = self.native + errors = ( + (mongo_errors.OperationFailure("denied", code=13), PermissionDeniedError), + (mongo_errors.NetworkTimeout("timeout"), DatabaseTimeoutError), + (mongo_errors.OperationFailure("timeout", code=50), DatabaseTimeoutError), + (mongo_errors.ConnectionFailure("unavailable"), DatabaseConnectionError), + (mongo_errors.InvalidURI("invalid"), ConfigurationError), + (mongo_errors.OperationFailure("unknown", code=42), DriverError), + ) + with self.connections.create("reports") as connection: + for original, expected in errors: + with self.subTest(error=type(original), expected=expected): + with self.assertRaises(expected) as caught: + with connection.errors(operation="aggregate"): + raise original + self.assertIs(caught.exception.original, original) + self.assertEqual(caught.exception.phase, "operation") + self.assertEqual(caught.exception.operation, "aggregate") + + @patch("database.mongodb.Client.create") + def test_business_wrappers_are_preserved_in_operation_context(self, create): + create.return_value = self.native + original = RuntimeError("business rule") + with self.connections.create("reports") as connection: + with self.assertRaises(RuntimeError) as caught: + with connection.errors(operation="build_query"): + raise original from mongo_errors.OperationFailure("denied", code=13) + self.assertIs(caught.exception, original) + + +class AsyncMongoDBConnectionTests(IsolatedAsyncioTestCase): + def setUp(self): + self.connections = Connections({ + "reports": ConnectionConfig("mongodb", host="server.test") + }) + + @patch("database.mongodb.Client.create_async") + async def test_native_object_is_returned_directly_and_probe_is_awaited(self, create): + class NativeClient(SimpleNamespace): + def __await__(self): + raise AssertionError("Construction awaited the native client") + + native = NativeClient( + admin=SimpleNamespace(command=AsyncMock()), + read_preference=ReadPreference.SECONDARY, + close=AsyncMock(), + ) + create.return_value = native + async with await self.connections.create_async("reports") as connection: + self.assertIs(connection.native, native) + native.admin.command.assert_not_awaited() + + result = await connection.check() + self.assertEqual(result.scope, "selected_server") + self.assertEqual(result.targets, ()) + native.admin.command.assert_awaited_once_with( + "ping", read_preference=ReadPreference.SECONDARY + ) + + create.assert_called_once_with(host="server.test") + native.close.assert_awaited_once_with() + + @patch("database.mongodb.Client.create_async") + async def test_missing_async_driver_capability_preserves_import_error(self, create): + original = ImportError("missing AsyncMongoClient") + create.side_effect = original + with self.assertRaises(DependencyError) as caught: + await self.connections.create_async("reports") + + error = caught.exception + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + self.assertEqual(error.phase, "create") + self.assertEqual(error.connection_name, "reports") + self.assertIn("PyMongo >= 4.9", str(error)) + + @patch("database.mongodb.Client.create_async") + async def test_ping_cancellation_is_preserved(self, create): + cancellation = asyncio.CancelledError() + create.return_value = SimpleNamespace( + admin=SimpleNamespace(command=AsyncMock(side_effect=cancellation)), + read_preference=ReadPreference.PRIMARY, + close=AsyncMock(), + ) + async with await self.connections.create_async("reports") as connection: + with self.assertRaises(asyncio.CancelledError) as caught: + await connection.check() + self.assertIs(caught.exception, cancellation) -- Gitee From 636f7baa3f656fac42a71f21b4a83d1646f20509 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 00:59:58 +0800 Subject: [PATCH 32/86] =?UTF-8?q?test(database):=20=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E5=A4=96=E9=83=A8=20Redis=20=E6=B1=A0=E4=B8=8D=E4=BC=9A?= =?UTF-8?q?=E5=9B=9E=E8=90=BD=E5=88=B0=E9=BB=98=E8=AE=A4=E5=9C=B0=E5=9D=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/config.py | 1 + tests/test_connection_config.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/src/database/config.py b/src/database/config.py index 34d103b..6d18361 100644 --- a/src/database/config.py +++ b/src/database/config.py @@ -215,6 +215,7 @@ class ConnectionConfig: } if pool_conflicts.intersection(options): raise self._error("connection_pool cannot be combined with connection parameters") + # redis-py creates its default localhost pool when this value is falsy. if not pool: raise self._error("connection_pool must supply an explicit target") elif options.get("host") is None and not _is_nonempty_string( diff --git a/tests/test_connection_config.py b/tests/test_connection_config.py index b9cee0b..e4c3376 100644 --- a/tests/test_connection_config.py +++ b/tests/test_connection_config.py @@ -225,6 +225,14 @@ class ConnectionConfigTests(unittest.TestCase): with self.assertRaises(ConfigurationError): ConnectionConfig("redis", **config) + def test_redis_rejects_a_falsy_pool_to_prevent_implicit_localhost(self): + class FalsyPool: + def __bool__(self): + return False + + with self.assertRaisesRegex(ConfigurationError, "explicit target"): + ConnectionConfig("redis", options={"connection_pool": FalsyPool()}) + def test_redis_external_pool_is_a_target_and_cannot_receive_connection_parameters(self): pool = object() self.assertIs( -- Gitee From 034443b376316f2cbc8ed769884fce14aee07714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:04:29 +0800 Subject: [PATCH 33/86] =?UTF-8?q?feat(mysql):=20=E6=8E=A5=E5=85=A5?= =?UTF-8?q?=E7=BB=9F=E4=B8=80=E8=BF=9E=E6=8E=A5=E6=A3=80=E6=9F=A5=E5=92=8C?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E8=BD=AC=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/_adapters/mysql.py | 138 +++++++++++++++++++++++++++++ tests/test_managed_mysql.py | 148 ++++++++++++++++++++++++++++++++ 2 files changed, 286 insertions(+) create mode 100644 src/database/_adapters/mysql.py create mode 100644 tests/test_managed_mysql.py diff --git a/src/database/_adapters/mysql.py b/src/database/_adapters/mysql.py new file mode 100644 index 0000000..7950c38 --- /dev/null +++ b/src/database/_adapters/mysql.py @@ -0,0 +1,138 @@ +"""MySQL Connector/Python adapter for managed connections.""" + +import asyncio +from ipaddress import IPv6Address + +from mysql.connector import errorcode +from mysql.connector import errors as driver_errors + +from ..config import ConnectionConfig +from ..errors import ( + AuthenticationError, + DatabaseConnectionError, + DatabaseError, + DatabaseTimeoutError, + DriverError, + PermissionDeniedError, + error_context, +) +from ..mysql import Client + + +_PERMISSION_CODES = frozenset({ + errorcode.ER_DBACCESS_DENIED_ERROR, + errorcode.ER_TABLEACCESS_DENIED_ERROR, + errorcode.ER_COLUMNACCESS_DENIED_ERROR, + errorcode.ER_SPECIFIC_ACCESS_DENIED_ERROR, +}) +_CONNECTION_CODES = frozenset({ + errorcode.CR_CONNECTION_ERROR, + errorcode.CR_CONN_HOST_ERROR, + errorcode.CR_IPSOCK_ERROR, + errorcode.CR_UNKNOWN_HOST, + errorcode.CR_SERVER_GONE_ERROR, + errorcode.CR_TCP_CONNECTION, + errorcode.CR_SERVER_HANDSHAKE_ERR, + errorcode.CR_SERVER_LOST, + errorcode.CR_SSL_CONNECTION_ERROR, + errorcode.CR_SERVER_LOST_EXTENDED, +}) +# Connector/Python 8.3 does not define the driver-specific timeout types. +_NATIVE_TIMEOUTS = tuple( + timeout_type for timeout_type in ( + getattr(driver_errors, "ConnectionTimeoutError", None), + getattr(driver_errors, "ReadTimeoutError", None), + getattr(driver_errors, "WriteTimeoutError", None), + ) if timeout_type is not None +) +_TRANSPORT_TIMEOUTS = (TimeoutError, asyncio.TimeoutError) + + +def _connection_target(native) -> str | None: + if getattr(native, "unix_socket", None): + return "unix_socket" + + host = getattr(native, "server_host", None) + port = getattr(native, "server_port", None) + if not isinstance(host, str) or not host or type(port) is not int or not 1 <= port <= 65535: + return None + if any(character.isspace() or character in "@/?#\\" for character in host): + return None + if ":" in host: + try: + host = f"[{IPv6Address(host.removeprefix('[').removesuffix(']'))}]" + except ValueError: + return None + return f"{host}:{port}" + + +class Adapter: + def __init__(self, config: ConnectionConfig, name: str, is_async: bool = False): + self._config = config + self._name = name + + def create(self): + return Client.create(**self._config.driver_kwargs()) + + async def create_async(self): + return await Client.create_async(**self._config.driver_kwargs()) + + @staticmethod + def classify(error: BaseException | None) -> type[DatabaseError] | None: + if not isinstance(error, driver_errors.Error): + return None + if error.errno == errorcode.ER_ACCESS_DENIED_ERROR: + return AuthenticationError + if error.errno in _PERMISSION_CODES: + return PermissionDeniedError + if isinstance(error, _NATIVE_TIMEOUTS): + return DatabaseTimeoutError + if error.errno in _CONNECTION_CODES: + if isinstance(error.__cause__, _TRANSPORT_TIMEOUTS): + return DatabaseTimeoutError + return DatabaseConnectionError + return DriverError + + def _classify_check_error(self, error: Exception) -> type[DatabaseError] | None: + # ping() can replace a transport failure with an unnumbered InterfaceError. + if not isinstance(error, driver_errors.InterfaceError) or error.errno not in (-1, None): + return self.classify(error) + cause = error.__cause__ + if isinstance(cause, _TRANSPORT_TIMEOUTS): + return DatabaseTimeoutError + if cause is None or isinstance(cause, ConnectionError): + return DatabaseConnectionError + if isinstance(cause, driver_errors.OperationalError) and cause.errno in (-1, None): + return DatabaseConnectionError + return self.classify(cause) or DriverError + + def _check_context(self, target): + return error_context( + connection_name=self._name, + backend="mysql", + phase="check", + operation="ping", + target=target, + classifier=self._classify_check_error, + owned=True, + ) + + def check(self, native) -> tuple[str, tuple[str, ...]]: + target = _connection_target(native) + with self._check_context(target): + native.ping(reconnect=False) + return "current_connection", (target,) if target else () + + async def check_async(self, native) -> tuple[str, tuple[str, ...]]: + target = _connection_target(native) + with self._check_context(target): + await native.ping(reconnect=False) + return "current_connection", (target,) if target else () + + @staticmethod + def close(native): + native.close() + + @staticmethod + async def close_async(native): + await native.close() diff --git a/tests/test_managed_mysql.py b/tests/test_managed_mysql.py new file mode 100644 index 0000000..9cb4ea1 --- /dev/null +++ b/tests/test_managed_mysql.py @@ -0,0 +1,148 @@ +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, Mock, patch + +from mysql.connector import errorcode +from mysql.connector import errors as driver_errors + +from database import ConnectionConfig, Connections +from database.errors import ( + AuthenticationError, + DatabaseConnectionError, + DatabaseTimeoutError, + DriverError, + PermissionDeniedError, +) + + +def registry(): + config = ConnectionConfig( + backend="mysql", + host="example.test", + port=3307, + username="test", + connect_timeout_s=3, + options={"database": "sample"}, + ) + return Connections({"mysql": config}) + + +class ManagedMySQLTests(TestCase): + @patch("database.mysql.client.connector.connect") + def test_create_checks_current_endpoint_and_closes_native_connection(self, connect): + native = Mock(server_host="2001:db8::1", server_port=3308, unix_socket=None) + connect.return_value = native + + with registry().create("mysql") as handle: + self.assertIs(handle.native, native) + native.ping.assert_not_called() + result = handle.check() + + connect.assert_called_once_with( + host="example.test", port=3307, user="test", connection_timeout=3, database="sample", + ) + native.ping.assert_called_once_with(reconnect=False) + native.close.assert_called_once_with() + self.assertEqual(result.scope, "current_connection") + self.assertEqual(result.targets, ("[2001:db8::1]:3308",)) + + @patch("database.mysql.client.connector.connect") + def test_check_preserves_wrapped_driver_error(self, connect): + native = Mock(server_host="example.test", server_port=3307, unix_socket=None) + connect.return_value = native + inner = driver_errors.OperationalError(errno=errorcode.CR_SERVER_LOST) + outer = driver_errors.InterfaceError("driver failure") + outer.__cause__ = inner + native.ping.side_effect = outer + + with registry().create("mysql") as handle: + with self.assertRaises(DatabaseConnectionError) as result: + handle.check() + + error = result.exception + self.assertEqual((error.phase, error.operation, error.target), ("check", "ping", "example.test:3307")) + self.assertIs(error.original, outer) + self.assertIs(error.__cause__, outer) + self.assertIs(error.original.__cause__, inner) + native.ping.assert_called_once_with(reconnect=False) + + @patch("database.mysql.client.connector.connect") + def test_classifies_only_structured_driver_failures(self, connect): + cases = ( + (errorcode.ER_ACCESS_DENIED_ERROR, AuthenticationError), + (errorcode.ER_DBACCESS_DENIED_ERROR, PermissionDeniedError), + (errorcode.ER_TABLEACCESS_DENIED_ERROR, PermissionDeniedError), + (errorcode.ER_COLUMNACCESS_DENIED_ERROR, PermissionDeniedError), + (errorcode.ER_SPECIFIC_ACCESS_DENIED_ERROR, PermissionDeniedError), + (errorcode.CR_CONN_HOST_ERROR, DatabaseConnectionError), + (errorcode.ER_PARSE_ERROR, DriverError), + ) + + with registry().create("mysql") as handle: + for errno, expected_type in cases: + with self.subTest(errno=errno): + original = driver_errors.ProgrammingError(errno=errno, msg="driver message") + with self.assertRaises(expected_type) as result: + with handle.errors(operation="read"): + raise original + self.assertIs(type(result.exception), expected_type) + self.assertEqual(result.exception.operation, "read") + self.assertIs(result.exception.original, original) + self.assertIs(result.exception.__cause__, original) + original = driver_errors.OperationalError(msg="connection refused") + with self.assertRaises(DriverError): + with handle.errors(): + raise original + + @patch("database.mysql.client.connector.connect") + def test_classifies_typed_transport_timeout_wrapped_by_driver(self, connect): + original = driver_errors.InterfaceError(errno=errorcode.CR_CONN_HOST_ERROR) + original.__cause__ = TimeoutError() + connect.side_effect = original + + with self.assertRaises(DatabaseTimeoutError) as result: + registry().create("mysql") + + self.assertIs(result.exception.original, original) + + +class AsyncManagedMySQLTests(IsolatedAsyncioTestCase): + @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_awaits_create_check_and_close(self, connect): + native = Mock( + server_host="example.test", server_port=3307, unix_socket=None, + ping=AsyncMock(), close=AsyncMock(), + ) + connect.return_value = native + + async with await registry().create_async("mysql") as handle: + self.assertIs(handle.native, native) + native.ping.assert_not_awaited() + result = await handle.check() + + connect.assert_awaited_once_with( + host="example.test", port=3307, user="test", connection_timeout=3, database="sample", + ) + native.ping.assert_awaited_once_with(reconnect=False) + native.close.assert_awaited_once_with() + self.assertEqual(result.scope, "current_connection") + self.assertEqual(result.targets, ("example.test:3307",)) + + @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_check_failure_omits_a_non_host_target(self, connect): + original = driver_errors.InterfaceError() + native = Mock( + server_host="mysql://user:secret@example.test", + server_port=3307, + unix_socket=None, + ping=AsyncMock(side_effect=original), + close=AsyncMock(), + ) + connect.return_value = native + + async with await registry().create_async("mysql") as handle: + with self.assertRaises(DatabaseConnectionError) as result: + await handle.check() + + self.assertIsNone(result.exception.target) + self.assertIs(result.exception.original, original) + native.close.assert_awaited_once_with() -- Gitee From f0c6d52034f3b36460efd0b06a4bd50f59c88dd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:05:11 +0800 Subject: [PATCH 34/86] =?UTF-8?q?fix(connections):=20=E4=BF=9D=E7=95=99?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E5=85=B3=E9=97=AD=E6=9C=9F=E9=97=B4=E7=9A=84?= =?UTF-8?q?=E4=B8=BB=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/connections.py | 4 +-- tests/test_connections.py | 66 +++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/database/connections.py b/src/database/connections.py index bb1894a..206f7fd 100644 --- a/src/database/connections.py +++ b/src/database/connections.py @@ -86,7 +86,7 @@ class Connection: def __exit__(self, exc_type, exc, traceback): try: self.close() - except Exception as cleanup_error: + except BaseException as cleanup_error: if exc is None: raise attach_cleanup_errors(exc, (cleanup_error,)) @@ -138,7 +138,7 @@ class AsyncConnection(Connection): async def __aexit__(self, exc_type, exc, traceback): try: await self.close() - except Exception as cleanup_error: + except BaseException as cleanup_error: if exc is None: raise attach_cleanup_errors(exc, (cleanup_error,)) diff --git a/tests/test_connections.py b/tests/test_connections.py index 913a42f..c2e6fda 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -87,6 +87,30 @@ class ConnectionTests(unittest.TestCase): self.assertIs(caught.exception, body) self.assertIs(body.database_cleanup_errors[0].original, cleanup) + def test_sync_cleanup_interrupt_preserves_body_exception(self): + handle = self.registry.create("main") + cleanup = KeyboardInterrupt("close interrupted") + handle._adapter.close.side_effect = cleanup + body = RuntimeError("body") + + with self.assertRaises(RuntimeError) as caught: + with handle: + raise body + + self.assertIs(caught.exception, body) + self.assertEqual(body.database_cleanup_errors, (cleanup,)) + + def test_sync_cleanup_interrupt_propagates_when_there_is_no_body_exception(self): + handle = self.registry.create("main") + cleanup = KeyboardInterrupt("close interrupted") + handle._adapter.close.side_effect = cleanup + + with self.assertRaises(KeyboardInterrupt) as caught: + with handle: + pass + + self.assertIs(caught.exception, cleanup) + def test_check_failure_and_direct_close_failure_are_classified(self): handle = self.registry.create("main") original = NativeFailure() @@ -153,6 +177,48 @@ class AsyncConnectionTests(unittest.IsolatedAsyncioTestCase): self.assertIs(caught.exception, cancellation) self.assertIs(cancellation.database_cleanup_errors[0].original, cleanup) + async def test_cancelling_async_cleanup_does_not_replace_body_exception(self): + handle = await self.registry.create_async("main") + cleanup_started = asyncio.Event() + finish_cleanup = asyncio.Event() + cancellations = [] + + async def close_async(native): + cleanup_started.set() + try: + await finish_cleanup.wait() + except asyncio.CancelledError as error: + cancellations.append(error) + raise + + handle._adapter.close_async.side_effect = close_async + body = RuntimeError("body") + + async def enter_handle(): + async with handle: + raise body + + task = asyncio.create_task(enter_handle()) + await cleanup_started.wait() + task.cancel() + with self.assertRaises(RuntimeError) as caught: + await task + + self.assertIs(caught.exception, body) + self.assertEqual(len(cancellations), 1) + self.assertIs(body.database_cleanup_errors[0], cancellations[0]) + + async def test_async_cleanup_cancellation_propagates_without_body_exception(self): + handle = await self.registry.create_async("main") + cancellation = asyncio.CancelledError() + handle._adapter.close_async.side_effect = cancellation + + with self.assertRaises(asyncio.CancelledError) as caught: + async with handle: + pass + + self.assertIs(caught.exception, cancellation) + async def test_cross_loop_is_rejected_and_creating_loop_can_close(self): handle = await self.registry.create_async("main") async def wrong_loop(): -- Gitee From 4aae92a014705ee30ff1aac2b5cd1db855ac7a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:05:11 +0800 Subject: [PATCH 35/86] =?UTF-8?q?fix(database):=20=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E8=B5=84=E6=BA=90=E6=B8=85=E7=90=86=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E5=85=83=E7=BB=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/errors.py | 2 +- tests/test_connection_errors.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/database/errors.py b/src/database/errors.py index 644ec03..a8cf162 100644 --- a/src/database/errors.py +++ b/src/database/errors.py @@ -180,4 +180,4 @@ def attach_cleanup_errors( existing = attributes.get("database_cleanup_errors", ()) if not isinstance(existing, tuple): existing = () - attributes["database_cleanup_errors"] = existing + additions + attributes["database_cleanup_errors"] = tuple.__add__(existing, additions) diff --git a/tests/test_connection_errors.py b/tests/test_connection_errors.py index 5b427b6..a7a3cfb 100644 --- a/tests/test_connection_errors.py +++ b/tests/test_connection_errors.py @@ -164,6 +164,20 @@ class ConnectionErrorTests(TestCase): attach_cleanup_errors(primary, (second,)) self.assertEqual(primary.database_cleanup_errors, (first, second)) + def test_cleanup_error_attachment_bypasses_tuple_add_override(self): + class CustomTuple(tuple): + def __add__(self, other): + raise RuntimeError("custom addition must not run") + + primary = ValueError("primary") + first = OSError("first cleanup failure") + second = RuntimeError("second cleanup failure") + primary.database_cleanup_errors = CustomTuple((first,)) + + attach_cleanup_errors(primary, (second,)) + + self.assertEqual(primary.database_cleanup_errors, (first, second)) + def test_cleanup_errors_attach_when_primary_disallows_attribute_assignment(self): class RestrictedError(Exception): def __setattr__(self, name, value): -- Gitee From 7c39cebf2760dcff997ea93baa67ad19523151c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:06:34 +0800 Subject: [PATCH 36/86] =?UTF-8?q?feat(database):=20=E7=AE=A1=E7=90=86=20Re?= =?UTF-8?q?dis=20=E5=8D=95=E6=9C=BA=E3=80=81=E5=93=A8=E5=85=B5=E5=92=8C?= =?UTF-8?q?=E9=9B=86=E7=BE=A4=E8=BF=9E=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/_adapters/redis.py | 332 +++++++++++++++++++++++++++++ tests/test_managed_redis.py | 367 ++++++++++++++++++++++++++++++++ 2 files changed, 699 insertions(+) create mode 100644 src/database/_adapters/redis.py create mode 100644 tests/test_managed_redis.py diff --git a/src/database/_adapters/redis.py b/src/database/_adapters/redis.py new file mode 100644 index 0000000..0530912 --- /dev/null +++ b/src/database/_adapters/redis.py @@ -0,0 +1,332 @@ +"""Managed Redis clients for standalone, Sentinel, and Cluster deployments.""" + +from copy import copy +from functools import partial + +from redis import asyncio as async_redis +from redis import cluster, exceptions + +from ..config import ConnectionConfig +from ..errors import ( + AuthenticationError, + DatabaseError, + ConfigurationError, + DatabaseConnectionError, + DatabaseTimeoutError, + DriverError, + PermissionDeniedError, + attach_cleanup_errors, + convert_error, + error_context, +) +from ..redis import Client + + +def _endpoint(host, port) -> str | None: + if not isinstance(host, str) or not host or any(char in host for char in "/@?#\r\n"): + return None + if isinstance(port, str) and port.isdecimal(): + port = int(port) + if not isinstance(port, int) or isinstance(port, bool): + return None + if ":" in host and not host.startswith("["): + host = f"[{host}]" + return f"{host}:{port}" + + +def _single_target(native) -> str | None: + settings = native.connection_pool.connection_kwargs + path = settings.get("path") + if isinstance(path, str) and path and "://" not in path: + return path + return _endpoint(settings.get("host"), settings.get("port")) + + +def _master_target(native) -> str | None: + address = native.connection_pool.master_address + if isinstance(address, tuple) and len(address) == 2: + return _endpoint(*address) + return None + + +def _async_closer(native): + close = getattr(native, "aclose", None) + return close if close is not None else native.close + + +class Adapter: + """Dispatch to the native Redis factories and manage owned resources.""" + + def __init__(self, config: ConnectionConfig, name: str, is_async: bool = False): + self._config = config + self._name = name + self._is_async = is_async + self._external_pool = config.options.get("connection_pool") is not None + + @staticmethod + def classify(error: Exception) -> type[DatabaseError] | None: + if isinstance(error, ( + exceptions.AuthenticationError, + exceptions.AuthenticationWrongNumberOfArgsError, + )): + return AuthenticationError + if isinstance(error, (exceptions.NoPermissionError, exceptions.AuthorizationError)): + return PermissionDeniedError + if isinstance(error, exceptions.TimeoutError): + return DatabaseTimeoutError + if isinstance(error, exceptions.ConnectionError): + return DatabaseConnectionError + if isinstance(error, (exceptions.RedisError, exceptions.RedisClusterException)): + return DriverError + return None + + def create(self): + return self._create() + + async def create_async(self): + # Async redis-py clients can be awaitable. Construction leaves I/O to callers. + return self._create() + + def _create(self): + kwargs = self._config.driver_kwargs() + mode = self._config.mode + if mode == "sentinel" and self._config.nodes: + kwargs["sentinels"] = list(self._config.nodes) + if mode == "cluster": + node_type = async_redis.cluster.ClusterNode if self._is_async else cluster.ClusterNode + if self._config.nodes: + kwargs["startup_nodes"] = [node_type(host, port) for host, port in self._config.nodes] + elif not self._is_async and kwargs.get("startup_nodes"): + kwargs["startup_nodes"] = self._sync_startup_nodes(kwargs["startup_nodes"]) + return Client.connect(mode, is_async=self._is_async, **kwargs) + + def _sync_startup_nodes(self, startup_nodes) -> list: + nodes = [] + for node in startup_nodes: + if getattr(node, "redis_connection", None) is not None: + raise ConfigurationError( + "Use Cluster startup nodes without an existing client.", + connection_name=self._name, backend="redis", phase="create", + ) + # redis-py adds a client in place during discovery. Keep reusable + # configured nodes independent of that client's ownership. + if isinstance(node, cluster.ClusterNode): + node = copy(node) + nodes.append(node) + return nodes + + def _context(self, phase: str, *, operation=None, target=None): + return error_context( + connection_name=self._name, + backend="redis", + phase=phase, + operation=operation, + target=target, + classifier=self.classify, + owned=True, + ) + + def _require_pong(self, result, target: str | None) -> None: + if result is not True: + raise DatabaseConnectionError( + "Redis did not acknowledge PING.", + connection_name=self._name, backend="redis", phase="check", + operation="PING", target=target, + ) + + def check(self, native) -> tuple[str, tuple[str, ...]]: + if self._config.mode == "sentinel": + return self._check_sentinel(native) + if self._config.mode == "cluster": + targets = [] + for node in self._primaries(native): + target = _endpoint(node.host, node.port) + with self._context("check", operation="PING", target=target): + self._require_pong(native.ping(target_nodes=node), target) + if target is not None: + targets.append(target) + return "cluster_primaries", tuple(targets) + + target = _single_target(native) + with self._context("check", operation="PING", target=target): + self._require_pong(native.ping(), target) + return "single", (target,) if target is not None else () + + async def check_async(self, native) -> tuple[str, tuple[str, ...]]: + if self._config.mode == "sentinel": + return await self._check_sentinel_async(native) + if self._config.mode == "cluster": + # redis-py makes initialize() idempotent. + with self._context("check", operation="initialize"): + await native.initialize() + targets = [] + for node in self._primaries(native): + target = _endpoint(node.host, node.port) + with self._context("check", operation="PING", target=target): + self._require_pong(await native.ping(target_nodes=node), target) + if target is not None: + targets.append(target) + return "cluster_primaries", tuple(targets) + + target = _single_target(native) + with self._context("check", operation="PING", target=target): + self._require_pong(await native.ping(), target) + return "single", (target,) if target is not None else () + + def _primaries(self, native) -> tuple: + with self._context("check", operation="get_primaries"): + primaries = tuple(native.get_primaries()) + if not primaries: + raise DatabaseConnectionError( + "Redis Cluster reported no primary nodes.", + connection_name=self._name, backend="redis", phase="check", + operation="get_primaries", + ) + return primaries + + def _check_sentinel(self, native) -> tuple[str, tuple[str, ...]]: + with self._context("check", operation="master_for", target="sentinel_master"): + primary = native.master_for(self._config.service_name) + target = None + error = None + try: + try: + result = primary.ping() + except BaseException as caught: + target = _master_target(primary) + self._raise_probe_error(caught, target) + target = _master_target(primary) + self._require_pong(result, target or "sentinel_master") + except BaseException as caught: + error = caught + raise + finally: + errors = self._close_single(primary, True, target=target or "sentinel_master") + self._finish_cleanup(errors, error) + return "sentinel_master", (target,) if target is not None else () + + async def _check_sentinel_async(self, native) -> tuple[str, tuple[str, ...]]: + with self._context("check", operation="master_for", target="sentinel_master"): + primary = native.master_for(self._config.service_name) + target = None + error = None + try: + try: + result = await primary.ping() + except BaseException as caught: + target = _master_target(primary) + self._raise_probe_error(caught, target) + target = _master_target(primary) + self._require_pong(result, target or "sentinel_master") + except BaseException as caught: + error = caught + raise + finally: + errors = await self._close_single_async(primary, True, target=target or "sentinel_master") + self._finish_cleanup(errors, error) + return "sentinel_master", (target,) if target is not None else () + + def _raise_probe_error(self, error: BaseException, target: str | None): + converted = convert_error( + error, connection_name=self._name, backend="redis", phase="check", + operation="PING", target=target or "sentinel_master", + classifier=self.classify, owned=True, + ) + if converted is error: + raise error + raise converted from error + + def _release(self, action, target, errors) -> None: + try: + with self._context("close", target=target): + action() + except BaseException as error: + errors.append(error) + + async def _release_async(self, action, target, errors) -> None: + try: + with self._context("close", target=target): + await action() + except BaseException as error: + errors.append(error) + + @staticmethod + def _finish_cleanup(errors, primary=None) -> None: + if not errors: + return + if primary is not None: + attach_cleanup_errors(primary, errors) + return + first, *remainder = errors + attach_cleanup_errors(first, remainder) + raise first + + def _close_single(self, native, owned: bool, *, target=None) -> list[BaseException]: + errors = [] + target = target or _single_target(native) + driver_closes_pool = getattr(native, "auto_close_connection_pool", False) + if not owned and driver_closes_pool: + # External pools must remain borrowed even if caller code changed + # the driver's ownership flag on the exposed client. + native.auto_close_connection_pool = False + self._release(native.close, target, errors) + if owned and (not driver_closes_pool or errors): + self._release(native.connection_pool.disconnect, target, errors) + return errors + + async def _close_single_async(self, native, owned: bool, *, target=None) -> list[BaseException]: + errors = [] + target = target or _single_target(native) + close = _async_closer(native) + # Keep client release and owned-pool release independent on all versions. + await self._release_async(partial(close, close_connection_pool=False), target, errors) + if owned: + pool = native.connection_pool + cleanup = getattr(pool, "aclose", None) or pool.disconnect + await self._release_async(cleanup, target, errors) + return errors + + @staticmethod + def _cluster_nodes(native): + # Include startup nodes that never became part of the discovered topology. + nodes = (*native.get_nodes(), *native.nodes_manager.startup_nodes.values()) + return {id(node): node for node in nodes}.values() + + def close(self, native) -> None: + errors = [] + if self._config.mode == "sentinel": + for client in tuple(native.sentinels): + errors.extend(self._close_single(client, True)) + elif self._config.mode == "cluster": + topology_clients = {id(node.redis_connection) for node in native.get_nodes()} + clients = { + id(node.redis_connection): node.redis_connection + for node in self._cluster_nodes(native) if node.redis_connection is not None + } + self._release(native.close, None, errors) + cluster_close_failed = bool(errors) + for client in clients.values(): + if cluster_close_failed or id(client) not in topology_clients: + errors.extend(self._close_single(client, True)) + elif not getattr(client, "auto_close_connection_pool", False): + self._release(client.connection_pool.disconnect, _single_target(client), errors) + else: + errors.extend(self._close_single(native, not self._external_pool)) + self._finish_cleanup(errors) + + async def close_async(self, native) -> None: + errors = [] + if self._config.mode == "sentinel": + for client in tuple(native.sentinels): + errors.extend(await self._close_single_async(client, True)) + elif self._config.mode == "cluster": + nodes = self._cluster_nodes(native) + await self._release_async(_async_closer(native), None, errors) + if errors: + # Older and newer drivers can both stop before draining startup nodes. + for node in nodes: + target = _endpoint(node.host, node.port) + await self._release_async(node.disconnect, target, errors) + else: + errors.extend(await self._close_single_async(native, not self._external_pool)) + self._finish_cleanup(errors) diff --git a/tests/test_managed_redis.py b/tests/test_managed_redis.py new file mode 100644 index 0000000..574490d --- /dev/null +++ b/tests/test_managed_redis.py @@ -0,0 +1,367 @@ +"""Managed Redis tests using installed driver classes without Redis servers.""" + +from importlib.util import find_spec +import unittest +from unittest.mock import AsyncMock, Mock, call, patch + +from database import ConnectionConfig, Connections +from database.errors import ( + AuthenticationError, + ConfigurationError, + DatabaseConnectionError, + DatabaseTimeoutError, + DriverError, + PermissionDeniedError, +) + +if find_spec("redis") is not None: + import redis + from database._adapters.redis import Adapter +else: + redis = None + + +def _config(mode="single", **kwargs): + if mode == "single": + kwargs.setdefault("host", "redis.test") + else: + kwargs.setdefault("nodes", [("redis.test", 7000)]) + if mode == "sentinel": + kwargs.setdefault("service_name", "main") + return ConnectionConfig("redis", mode=mode, **kwargs) + + +def _registry(config=None): + return Connections({"test": config or _config()}) + + +def _configure_sync_cluster(manager): + # Supply a discovered topology to the actual Cluster constructor. + nodes = [ + redis.cluster.ClusterNode("first.test", 7001, server_type="primary"), + redis.cluster.ClusterNode("second.test", 7002, server_type="primary"), + redis.cluster.ClusterNode("replica.test", 7003, server_type="replica"), + ] + for node in nodes: + node.redis_connection = redis.Redis(host=node.host, port=node.port) + manager.nodes_cache = {node.name: node for node in nodes} + manager.default_node = nodes[0] + + +def _configure_async_cluster(client): + nodes = [ + redis.asyncio.cluster.ClusterNode("first.test", 7001, server_type="primary"), + redis.asyncio.cluster.ClusterNode("second.test", 7002, server_type="primary"), + redis.asyncio.cluster.ClusterNode("replica.test", 7003, server_type="replica"), + ] + client.nodes_manager.nodes_cache = {node.name: node for node in nodes} + client.nodes_manager.default_node = nodes[0] + + +@unittest.skipIf(redis is None, "redis driver is not installed") +class ManagedRedisTests(unittest.TestCase): + def test_single_returns_native_and_pings_only_when_requested(self): + with patch.object(redis.Redis, "ping", return_value=True) as ping: + with _registry().create("test") as handle: + self.assertIsInstance(handle.native, redis.Redis) + ping.assert_not_called() + result = handle.check() + self.assertEqual(result.scope, "single") + self.assertEqual(result.targets, ("redis.test:6379",)) + ping.assert_called_once_with() + + def test_external_pool_is_preserved_and_owned_pool_is_disconnected(self): + pool = redis.ConnectionPool(host="external.test") + config = ConnectionConfig("redis", options={"connection_pool": pool}) + shared = _registry(config).create("test") + private = _registry().create("test") + self.assertIs(shared.native.connection_pool, pool) + with patch.object(pool, "disconnect") as shared_disconnect, patch.object( + private.native.connection_pool, "disconnect" + ) as private_disconnect: + shared.close() + private.close() + private.close() + shared_disconnect.assert_not_called() + private_disconnect.assert_called_once_with() + + def test_single_false_ping_and_driver_errors_are_not_healthy(self): + handle = _registry().create("test") + self.addCleanup(handle.close) + with patch.object(handle.native, "ping", return_value=False): + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertEqual(failed.exception.target, "redis.test:6379") + self.assertEqual(failed.exception.operation, "PING") + original = redis.exceptions.NoPermissionError("server detail") + with patch.object(handle.native, "ping", side_effect=original): + with self.assertRaises(PermissionDeniedError) as failed: + handle.check() + self.assertIs(failed.exception.original, original) + self.assertIs(failed.exception.__cause__, original) + self.assertEqual(failed.exception.phase, "check") + + def test_sentinel_uses_native_manager_and_closes_temporary_master(self): + config = _config( + "sentinel", username="data", password="data-secret", + options={"sentinel_kwargs": {"username": "sentinel", "password": "sentinel-secret"}}, + ) + with _registry(config).create("test") as handle: + self.assertIsInstance(handle.native, redis.Sentinel) + sentinel_client, = handle.native.sentinels + self.assertEqual(sentinel_client.connection_pool.connection_kwargs["username"], "sentinel") + self.assertEqual(handle.native.connection_kwargs["username"], "data") + master = handle.native.master_for("main") + master.connection_pool.proxy.master_address = ("master.test", 6380) + with patch.object(handle.native, "master_for", return_value=master) as factory, patch.object( + master, "ping", return_value=True + ) as ping, patch.object(master.connection_pool, "disconnect") as disconnect: + result = handle.check() + self.assertEqual(result.scope, "sentinel_master") + self.assertEqual(result.targets, ("master.test:6380",)) + factory.assert_called_once_with("main") + ping.assert_called_once_with() + disconnect.assert_called_once_with() + + def test_sentinel_original_ping_error_survives_cleanup_errors(self): + handle = _registry(_config("sentinel")).create("test") + self.addCleanup(handle.close) + master = handle.native.master_for("main") + master.connection_pool.proxy.master_address = ("master.test", 6380) + original = redis.exceptions.ConnectionError("unreachable") + release_error = redis.exceptions.ResponseError("release failed") + pool_error = redis.exceptions.TimeoutError("pool release timed out") + with patch.object(handle.native, "master_for", return_value=master), patch.object( + master, "ping", side_effect=original + ), patch.object(master, "close", side_effect=release_error), patch.object( + master.connection_pool, "disconnect", side_effect=pool_error + ) as disconnect: + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertEqual(failed.exception.target, "master.test:6380") + self.assertIs(failed.exception.original, original) + self.assertIs(failed.exception.__cause__, original) + errors = failed.exception.database_cleanup_errors + self.assertEqual(tuple(type(error) for error in errors), (DriverError, DatabaseTimeoutError)) + self.assertEqual(tuple(error.original for error in errors), (release_error, pool_error)) + self.assertEqual(tuple(error.phase for error in errors), ("close", "close")) + disconnect.assert_called_once_with() + + def test_sentinel_unknown_service_preserves_driver_error(self): + with _registry(_config("sentinel")).create("test") as handle: + master = handle.native.master_for("main") + error = redis.sentinel.MasterNotFoundError("not found") + with patch.object(handle.native, "master_for", return_value=master), patch.object( + master, "ping", side_effect=error + ): + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertIs(failed.exception.original, error) + self.assertEqual(failed.exception.target, "sentinel_master") + + def test_sentinel_close_attempts_all_node_clients(self): + config = _config("sentinel", nodes=[("first.test", 26379), ("second.test", 26379)]) + handle = _registry(config).create("test") + first, second = handle.native.sentinels + self.addCleanup(handle.close) + first_error = redis.exceptions.ConnectionError("first") + second_error = redis.exceptions.TimeoutError("second") + with patch.object(first, "close", side_effect=first_error), patch.object( + first.connection_pool, "disconnect" + ) as first_disconnect, patch.object(second, "close", side_effect=second_error) as second_close, patch.object( + second.connection_pool, "disconnect" + ) as second_disconnect: + with self.assertRaises(DatabaseConnectionError) as failed: + handle.close() + self.assertIs(failed.exception.original, first_error) + self.assertIs(failed.exception.database_cleanup_errors[0].original, second_error) + first_disconnect.assert_called_once_with() + second_disconnect.assert_called_once_with() + second_close.assert_called_once_with() + + def test_cluster_targets_actual_primaries_in_order_and_stops_on_failure(self): + with patch.object(redis.cluster.NodesManager, "initialize", autospec=True, side_effect=_configure_sync_cluster), patch.object( + redis.cluster.CommandsParser, "initialize" + ): + handle = _registry(_config("cluster")).create("test") + self.addCleanup(handle.close) + self.assertIsInstance(handle.native, redis.RedisCluster) + first, second = handle.native.get_primaries() + with patch.object(handle.native, "_execute_command", return_value=True) as command: + result = handle.check() + self.assertEqual(result.scope, "cluster_primaries") + self.assertEqual(result.targets, ("first.test:7001", "second.test:7002")) + self.assertEqual(command.call_args_list, [call(first, "PING"), call(second, "PING")]) + with patch.object(handle.native, "_execute_command", return_value=False) as command: + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertEqual(failed.exception.target, "first.test:7001") + command.assert_called_once_with(first, "PING") + + def test_cluster_rejects_empty_primary_snapshot(self): + with patch.object(redis.cluster.NodesManager, "initialize", autospec=True, side_effect=_configure_sync_cluster), patch.object( + redis.cluster.CommandsParser, "initialize" + ): + handle = _registry(_config("cluster")).create("test") + self.addCleanup(handle.close) + with patch.object(handle.native, "get_primaries", return_value=[]), patch.object( + handle.native, "ping" + ) as ping: + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertEqual(failed.exception.operation, "get_primaries") + ping.assert_not_called() + + def test_cluster_pool_from_an_existing_node_stays_external(self): + original_client = redis.Redis(host="external.test") + self.addCleanup(original_client.close) + node = redis.cluster.ClusterNode("external.test", 7000, redis_connection=original_client) + config = ConnectionConfig("redis", mode="cluster", options={"startup_nodes": [node]}) + with patch.object(redis.cluster.NodesManager, "initialize") as initialize: + with self.assertRaises(ConfigurationError) as failed: + _registry(config).create("test") + self.assertEqual(failed.exception.phase, "create") + initialize.assert_not_called() + self.assertIs(node.redis_connection, original_client) + + def test_cluster_reuses_configured_host_addresses_without_sharing_driver_nodes(self): + node = redis.cluster.ClusterNode("redis.test", 7000) + config = ConnectionConfig("redis", mode="cluster", options={"startup_nodes": [node]}) + with patch.object(redis.cluster.NodesManager, "initialize", autospec=True, side_effect=_configure_sync_cluster), patch.object( + redis.cluster.CommandsParser, "initialize" + ): + registry = _registry(config) + first = registry.create("test") + second = registry.create("test") + with first, second: + first_node, = first.native.nodes_manager.startup_nodes.values() + second_node, = second.native.nodes_manager.startup_nodes.values() + self.assertEqual((first_node.host, first_node.port), (node.host, node.port)) + self.assertIsNot(first_node, node) + self.assertIsNot(first_node, second_node) + + def test_redis_classification_uses_native_types_and_prioritizes_subclasses(self): + for error_type, expected in ( + (redis.exceptions.AuthenticationError, AuthenticationError), + (redis.exceptions.AuthenticationWrongNumberOfArgsError, AuthenticationError), + (redis.exceptions.NoPermissionError, PermissionDeniedError), + (redis.exceptions.AuthorizationError, PermissionDeniedError), + (redis.exceptions.TimeoutError, DatabaseTimeoutError), + (redis.sentinel.MasterNotFoundError, DatabaseConnectionError), + (redis.exceptions.RedisClusterException, DriverError), + (redis.exceptions.ResponseError, DriverError), + (TypeError, None), + ): + with self.subTest(error_type=error_type): + error = error_type("irrelevant authentication timeout message") + self.assertIs(Adapter.classify(error), expected) + + +@unittest.skipIf(redis is None, "redis driver is not installed") +class ManagedAsyncRedisTests(unittest.IsolatedAsyncioTestCase): + async def test_single_returns_raw_awaitable_client_without_awaiting_it(self): + with patch.object(redis.asyncio.Redis, "initialize", new_callable=AsyncMock) as initialize: + handle = await _registry().create_async("test") + self.assertIsInstance(handle.native, redis.asyncio.Redis) + initialize.assert_not_called() + async with handle: + with patch.object(handle.native, "ping", new_callable=AsyncMock, return_value=True) as ping: + result = await handle.check() + self.assertEqual(result.targets, ("redis.test:6379",)) + ping.assert_awaited_once_with() + + async def test_async_external_pool_is_preserved_and_owned_pool_is_disconnected(self): + pool = redis.asyncio.ConnectionPool(host="external.test") + config = ConnectionConfig("redis", options={"connection_pool": pool}) + shared = await _registry(config).create_async("test") + private = await _registry().create_async("test") + self.assertIs(shared.native.connection_pool, pool) + with patch.object(pool, "disconnect", new_callable=AsyncMock) as shared_disconnect, patch.object( + private.native.connection_pool, "disconnect", new_callable=AsyncMock + ) as private_disconnect: + await shared.close() + await private.close() + await private.close() + shared_disconnect.assert_not_called() + private_disconnect.assert_awaited_once_with() + + async def test_async_sentinel_releases_temporary_master_and_node_pools(self): + handle = await _registry(_config("sentinel")).create_async("test") + self.assertIsInstance(handle.native, redis.asyncio.Sentinel) + master = handle.native.master_for("main") + master.connection_pool.master_address = ("2001:db8::1", 6380) + sentinel_client, = handle.native.sentinels + with patch.object(handle.native, "master_for", return_value=master) as factory, patch.object( + master, "ping", new_callable=AsyncMock, return_value=True + ) as ping, patch.object(master.connection_pool, "disconnect", new_callable=AsyncMock) as disconnect, patch.object( + sentinel_client.connection_pool, "disconnect", new_callable=AsyncMock + ) as sentinel_disconnect: + result = await handle.check() + await handle.close() + self.assertEqual(result.scope, "sentinel_master") + self.assertEqual(result.targets, ("[2001:db8::1]:6380",)) + factory.assert_called_once_with("main") + ping.assert_awaited_once_with() + disconnect.assert_awaited_once_with() + sentinel_disconnect.assert_awaited_once_with() + + async def test_async_sentinel_keeps_cancelled_error_when_cleanup_fails(self): + from asyncio import CancelledError + + handle = await _registry(_config("sentinel")).create_async("test") + self.addAsyncCleanup(handle.close) + master = handle.native.master_for("main") + original = CancelledError("task cancelled") + cleanup = redis.exceptions.ConnectionError("close failed") + with patch.object(handle.native, "master_for", return_value=master), patch.object( + master, "ping", new_callable=AsyncMock, side_effect=original + ), patch.object(master.connection_pool, "disconnect", new_callable=AsyncMock, side_effect=cleanup): + with self.assertRaises(CancelledError) as failed: + await handle.check() + self.assertIs(failed.exception, original) + self.assertIs(failed.exception.database_cleanup_errors[0].original, cleanup) + + async def test_async_cluster_initializes_then_pings_each_primary(self): + with patch.object(redis.asyncio.RedisCluster, "initialize", new_callable=AsyncMock) as initialize: + handle = await _registry(_config("cluster")).create_async("test") + initialize.assert_not_called() + self.assertIsInstance(handle.native, redis.asyncio.RedisCluster) + self.addAsyncCleanup(handle.close) + _configure_async_cluster(handle.native) + first, second = handle.native.get_primaries() + + async def initialize_cluster(): + handle.native._initialize = False + + with patch.object(handle.native, "initialize", new_callable=AsyncMock, side_effect=initialize_cluster) as initialize, patch.object( + handle.native, "_execute_command", new_callable=AsyncMock, return_value=True + ) as command: + result = await handle.check() + initialize.assert_awaited_once_with() + self.assertEqual(result.targets, ("first.test:7001", "second.test:7002")) + self.assertEqual(command.await_args_list, [call(first, "PING"), call(second, "PING")]) + + async def test_async_cluster_close_tries_topology_and_seed_nodes_after_failure(self): + handle = await _registry(_config("cluster")).create_async("test") + self.addAsyncCleanup(handle.close) + _configure_async_cluster(handle.native) + first, second = handle.native.get_primaries() + seed, = handle.native.nodes_manager.startup_nodes.values() + native_close = "aclose" if hasattr(handle.native, "aclose") else "close" + primary = redis.exceptions.ConnectionError("cluster close failed") + fallback = redis.exceptions.TimeoutError("first node close failed") + async def disconnect_node(node): + if node is first: + raise fallback + + with patch.object( + handle.native, native_close, new_callable=AsyncMock, side_effect=primary + ), patch.object( + type(first), "disconnect", autospec=True, side_effect=disconnect_node + ) as disconnect: + with self.assertRaises(DatabaseConnectionError) as failed: + await handle.close() + self.assertIs(failed.exception.original, primary) + self.assertIs(failed.exception.database_cleanup_errors[0].original, fallback) + self.assertIn(call(second), disconnect.await_args_list) + self.assertIn(call(seed), disconnect.await_args_list) -- Gitee From 1064c2a03ceac353a5698ccce99f77109ff70473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:07:13 +0800 Subject: [PATCH 37/86] =?UTF-8?q?test(database):=20=E9=AA=8C=E8=AF=81?= =?UTF-8?q?=E5=BC=82=E6=AD=A5=E9=9B=86=E7=BE=A4=E5=90=84=E5=8F=A5=E6=9F=84?= =?UTF-8?q?=E7=8B=AC=E7=AB=8B=E6=8B=A5=E6=9C=89=E5=90=AF=E5=8A=A8=E8=8A=82?= =?UTF-8?q?=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_managed_redis.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/test_managed_redis.py b/tests/test_managed_redis.py index 574490d..52a20c3 100644 --- a/tests/test_managed_redis.py +++ b/tests/test_managed_redis.py @@ -341,6 +341,30 @@ class ManagedAsyncRedisTests(unittest.IsolatedAsyncioTestCase): self.assertEqual(result.targets, ("first.test:7001", "second.test:7002")) self.assertEqual(command.await_args_list, [call(first, "PING"), call(second, "PING")]) + async def test_async_cluster_recreates_native_startup_nodes_per_handle(self): + node = redis.asyncio.cluster.ClusterNode("seed.test", 7000) + config = ConnectionConfig("redis", mode="cluster", options={"startup_nodes": [node]}) + registry = _registry(config) + first = await registry.create_async("test") + second = await registry.create_async("test") + self.addAsyncCleanup(first.close) + self.addAsyncCleanup(second.close) + first_node, = first.native.nodes_manager.startup_nodes.values() + second_node, = second.native.nodes_manager.startup_nodes.values() + self.assertEqual((first_node.host, first_node.port), (node.host, node.port)) + self.assertIsNot(first_node, node) + self.assertIsNot(second_node, node) + self.assertIsNot(first_node, second_node) + + # Both clients reached initialization independently. Closing one can + # disconnect its private seed without touching the other client's seed. + first.native._initialize = False + second.native._initialize = False + with patch.object(type(node), "disconnect", autospec=True) as disconnect: + await first.close() + disconnect.assert_awaited_once_with(first_node) + self.assertFalse(second.closed) + async def test_async_cluster_close_tries_topology_and_seed_nodes_after_failure(self): handle = await _registry(_config("cluster")).create_async("test") self.addAsyncCleanup(handle.close) -- Gitee From d9d62017e167bc4611d90385338bb5a940ccfd64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:09:15 +0800 Subject: [PATCH 38/86] =?UTF-8?q?test(connections):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=A4=9A=E6=95=B0=E6=8D=AE=E5=BA=93=E5=90=8C=E6=97=B6=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E7=9A=84=E9=9A=94=E7=A6=BB=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/verify_connections.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/scripts/verify_connections.py b/scripts/verify_connections.py index e553761..189df1d 100644 --- a/scripts/verify_connections.py +++ b/scripts/verify_connections.py @@ -7,6 +7,7 @@ Configuration loading belongs to this test caller, never to the SDK. import argparse import asyncio import json +from contextlib import AsyncExitStack, ExitStack from pathlib import Path from database import ConnectionConfig, Connections @@ -43,6 +44,31 @@ async def verify_async(connections, name, backend): "target_count": len(result.targets), "result": "passed"} +def verify_together(connections, names): + with ExitStack() as stack: + handles = [stack.enter_context(connections.create(name)) for name in names] + for handle in handles: + handle.check() + if handles: + handles[0].close() + for handle in handles[1:]: + handle.check() + return {"mode": "sync", "simultaneous_connections": len(handles), "result": "passed"} + + +async def verify_together_async(connections, names): + async with AsyncExitStack() as stack: + handles = [await stack.enter_async_context(await connections.create_async(name)) + for name in names] + for handle in handles: + await handle.check() + if handles: + await handles[0].close() + for handle in handles[1:]: + await handle.check() + return {"mode": "async", "simultaneous_connections": len(handles), "result": "passed"} + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", required=True, type=Path) @@ -57,6 +83,8 @@ def main(): print(json.dumps(asyncio.run( verify_async(connections, name, parameters["backend"]) ))) + print(json.dumps(verify_together(connections, fixtures))) + print(json.dumps(asyncio.run(verify_together_async(connections, fixtures)))) if __name__ == "__main__": -- Gitee From a94add57ddbc5997b96cfabdb355df30e8f64f84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:10:02 +0800 Subject: [PATCH 39/86] =?UTF-8?q?fix(mysql):=20=E6=8B=92=E7=BB=9D=E5=9B=9E?= =?UTF-8?q?=E8=90=BD=E5=88=B0=E9=9A=90=E5=BC=8F=E5=9C=B0=E5=9D=80=E7=9A=84?= =?UTF-8?q?=E5=A5=97=E6=8E=A5=E5=AD=97=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/config.py | 10 +++++++--- tests/test_connection_config.py | 17 +++++++++++++++-- 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/database/config.py b/src/database/config.py index 6d18361..5107f9b 100644 --- a/src/database/config.py +++ b/src/database/config.py @@ -1,5 +1,6 @@ """Explicit configuration for managed database connections.""" +import os from collections.abc import Mapping from dataclasses import dataclass, field, replace from enum import Enum @@ -178,10 +179,13 @@ class ConnectionConfig: else: if self.mode != "single" or self.nodes or self.service_name is not None: raise self._error("mode, nodes, and service_name apply to Redis topology") - socket = self.backend == DatabaseType.MYSQL and _is_nonempty_string( - options.get("unix_socket") + # Connector/Python uses its default TCP host for Unix sockets on non-POSIX platforms. + has_socket_target = ( + self.backend == DatabaseType.MYSQL + and os.name == "posix" + and _is_nonempty_string(options.get("unix_socket")) ) - if host is None and not socket: + if host is None and not has_socket_target: raise self._error("an explicit connection target is required") return options diff --git a/tests/test_connection_config.py b/tests/test_connection_config.py index e4c3376..0c3569d 100644 --- a/tests/test_connection_config.py +++ b/tests/test_connection_config.py @@ -25,6 +25,20 @@ class ConnectionConfigTests(unittest.TestCase): "password": "", timeout_key: timeout, }) + def test_mysql_accepts_a_socket_target_on_posix(self): + options = {"unix_socket": "/tmp/mysql.sock", "db": "app"} + with patch("database.config.os.name", "posix"): + self.assertEqual(ConnectionConfig("mysql", options=options).driver_kwargs(), options) + + def test_mysql_socket_without_a_host_does_not_fall_back_on_windows(self): + options = {"unix_socket": "/tmp/mysql.sock", "db": "app"} + with patch("database.config.os.name", "nt"): + with self.assertRaisesRegex(ConfigurationError, "explicit connection target"): + ConnectionConfig("mysql", options=options) + + config = ConnectionConfig("mysql", host="database.example", options=options) + self.assertEqual(config.driver_kwargs(), {"host": "database.example", **options}) + def test_explicit_configuration_does_not_read_environment(self): with patch.object(os, "environ", None), patch.object( os, "getenv", side_effect=AssertionError("environment read") @@ -48,10 +62,9 @@ class ConnectionConfigTests(unittest.TestCase): with self.assertRaisesRegex(ConfigurationError, "explicit"): ConnectionConfig(backend, options=options) - def test_accepts_native_host_and_local_sockets(self): + def test_accepts_native_mongodb_hosts_and_redis_socket(self): for backend, options in ( ("mongodb", {"host": ["first:27017", "second:27017"]}), - ("mysql", {"unix_socket": "/tmp/mysql.sock", "db": "app"}), ("redis", {"unix_socket_path": "/tmp/redis.sock"}), ): with self.subTest(backend=backend, options=options): -- Gitee From 5c83b61f000e5c61858096fa5df87612f8ad5341 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:11:45 +0800 Subject: [PATCH 40/86] =?UTF-8?q?fix(mysql):=20=E7=BB=9F=E4=B8=80=E7=BC=BA?= =?UTF-8?q?=E5=A4=B1=E9=A9=B1=E5=8A=A8=E8=83=BD=E5=8A=9B=E7=9A=84=E8=BF=9E?= =?UTF-8?q?=E6=8E=A5=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/_adapters/mysql.py | 17 +++++++++++++-- tests/test_managed_mysql.py | 38 +++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/src/database/_adapters/mysql.py b/src/database/_adapters/mysql.py index 7950c38..444bfea 100644 --- a/src/database/_adapters/mysql.py +++ b/src/database/_adapters/mysql.py @@ -12,6 +12,7 @@ from ..errors import ( DatabaseConnectionError, DatabaseError, DatabaseTimeoutError, + DependencyError, DriverError, PermissionDeniedError, error_context, @@ -72,10 +73,22 @@ class Adapter: self._name = name def create(self): - return Client.create(**self._config.driver_kwargs()) + kwargs = self._config.driver_kwargs() + with self._dependency_context(): + return Client.create(**kwargs) async def create_async(self): - return await Client.create_async(**self._config.driver_kwargs()) + kwargs = self._config.driver_kwargs() + with self._dependency_context(): + return await Client.create_async(**kwargs) + + def _dependency_context(self): + return error_context( + connection_name=self._name, + backend="mysql", + phase="create", + classifier=lambda error: DependencyError if isinstance(error, ImportError) else None, + ) @staticmethod def classify(error: BaseException | None) -> type[DatabaseError] | None: diff --git a/tests/test_managed_mysql.py b/tests/test_managed_mysql.py index 9cb4ea1..fe9bae6 100644 --- a/tests/test_managed_mysql.py +++ b/tests/test_managed_mysql.py @@ -9,6 +9,7 @@ from database.errors import ( AuthenticationError, DatabaseConnectionError, DatabaseTimeoutError, + DependencyError, DriverError, PermissionDeniedError, ) @@ -27,6 +28,30 @@ def registry(): class ManagedMySQLTests(TestCase): + @patch("database.mysql.client.connector.connect") + def test_create_wraps_missing_driver_capability(self, connect): + original = ImportError("MySQL C extension is unavailable") + connect.side_effect = original + + with self.assertRaises(DependencyError) as result: + registry().create("mysql") + + error = result.exception + self.assertEqual((error.connection_name, error.backend, error.phase), ("mysql", "mysql", "create")) + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + + @patch("database.mysql.client.connector.connect") + def test_operation_preserves_unrelated_import_error(self, connect): + original = ImportError("unrelated import") + + with registry().create("mysql") as handle: + with self.assertRaises(ImportError) as result: + with handle.errors(operation="read"): + raise original + + self.assertIs(result.exception, original) + @patch("database.mysql.client.connector.connect") def test_create_checks_current_endpoint_and_closes_native_connection(self, connect): native = Mock(server_host="2001:db8::1", server_port=3308, unix_socket=None) @@ -106,6 +131,19 @@ class ManagedMySQLTests(TestCase): class AsyncManagedMySQLTests(IsolatedAsyncioTestCase): + @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_create_async_wraps_missing_driver_capability(self, connect): + original = ImportError("async driver capability is unavailable") + connect.side_effect = original + + with self.assertRaises(DependencyError) as result: + await registry().create_async("mysql") + + error = result.exception + self.assertEqual((error.connection_name, error.backend, error.phase), ("mysql", "mysql", "create")) + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) async def test_awaits_create_check_and_close(self, connect): native = Mock( -- Gitee From b2c23a8454590c705789d571c120f0fb684d3a70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:13:01 +0800 Subject: [PATCH 41/86] =?UTF-8?q?docs(database):=20=E8=AF=B4=E6=98=8E?= =?UTF-8?q?=E7=9B=AE=E6=A0=87=E6=A0=A1=E9=AA=8C=E5=92=8C=E5=8E=9F=E7=94=9F?= =?UTF-8?q?=E9=A9=B1=E5=8A=A8=E6=B8=85=E7=90=86=E8=BE=B9=E7=95=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 4 +++ docs/connection-management-design.md | 37 +++++++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 1d3982f..de0bbc8 100644 --- a/README.md +++ b/README.md @@ -179,5 +179,9 @@ SDK 自动转换其自身配置、创建、检查及关闭范围内的已识别 `connect_timeout_s` 是正整数秒,仅控制建立连接阶段。读取响应、节点选择和等待连接池等限制通过 `options` 使用驱动支持的参数配置。MySQL Connector/Python 8.3 不支持后来 9.2 加入的 `read_timeout`,本 SDK 不承诺统一的整体检查硬超时,也不增加自动重试。 +MongoDB 的检查结果使用 `scope="selected_server"`,`targets` 为空:原生 `ping` 不公开实际执行节点,因此 SDK 不猜测地址。Windows 上 MySQL 需要显式主机地址,不能仅用驱动不支持的 Unix socket 参数指定目标。同步 Redis Cluster 的 `startup_nodes` 不接受已经绑定客户端的节点,以防意外关闭调用方资源。 + +异步创建尚未返回句柄时的资源清理由原生驱动负责。当前受测 Connector/Python 的握手取消存在上游清理限制,详见设计文档。Redis 拓扑已完成驱动级验证,真实服务验证仍待完成。 + 详细设计和开发流程见 [连接管理设计](docs/connection-management-design.md)。实际验证情况见 [验收记录](docs/connection-management-validation.md)。 diff --git a/docs/connection-management-design.md b/docs/connection-management-design.md index 7491227..8362e51 100644 --- a/docs/connection-management-design.md +++ b/docs/connection-management-design.md @@ -1,6 +1,6 @@ # database 统一连接管理设计与开发流程 -日期:2026-11-15。状态:实施中,测试结果见验收记录。 +日期:2026-11-15。状态:实现已集成,实际验证范围和剩余事项见 [验收记录](connection-management-validation.md)。 基线 develop:7d38b1ffe13a17b37d0bffc15e175085dcf0bb19。 参考 temp/save:89497823698812e1c85e6c4c12af86d4281543cf。 @@ -23,9 +23,11 @@ SDK 只接收调用方显式参数;不读取环境变量、不加载 .env、 不整体合并或删除 temp/save,不改写用户原工作区。 ## 公共接口 +```python from database import ConnectionConfig, Connections ConnectionConfig(backend, host=None, port=None, username=None, password=None, connect_timeout_s=None, options={}, mode="single", nodes=(), service_name=None) +``` Connections({name: config}) 保存配置,不连接、不缓存客户端。create(name) 每次返回独立 Connection;await create_async(name) 返回独立 AsyncConnection。name 必须是非空字符串;未知名称为 ConfigurationError。 connection.native 为真实原生对象;check()/await check() 返回 CheckResult;close()/await close() 清理自有资源。支持 with/async with,上下文进入不增加探测。connection.errors(operation=...) 为普通 with,可在其中 await 原生方法。 @@ -100,6 +102,39 @@ feature/database-mongodb-connections、feature/database-mysql-connections、feat README补全显式配置、多实例、检查、原生访问、异常、资源归属和版本限制。 交付总线/远端/基线及最终SHA/worker合并表/实际命令版本结果/未验证项/推送状态;用户最终合入develop。 +## 实现后的边界说明 + +### 显式目标与配置隔离 + +- Windows 上仅提供 MySQL `unix_socket` 会被原生驱动忽略并回落到默认 TCP 地址,因此高层配置拒绝这种输入;POSIX 允许仅提供 socket。Windows 若显式提供 `host`,其余参数仍按驱动语义处理。 +- Redis 外部池必须是有效的真值对象。驱动会将假值池当成未提供池,可能转而使用默认地址,所以高层提前拒绝。 +- 同步 Redis Cluster 的原生 `startup_nodes` 会被驱动原地填入客户端。适配器为普通 `ClusterNode` 复制地址信息,保证复用配置时每个客户端独立;节点已经带有 `redis_connection` 时拒绝创建,防止关闭调用方资源。异步驱动自行从节点地址重建,两个受测版本已验证不共享节点连接。 +- `options` 中其他外部对象继续保留引用。SDK 只隔离标准配置容器,不承诺隔离调用方主动共享的原生资源。 + +### 检查结果的准确含义 + +| 后端或模式 | `scope` | `targets` | +| --- | --- | --- | +| MongoDB | `selected_server` | 空元组。公开 `ping` 返回值不提供实际执行节点,不另读 `address` 触发不同的节点选择 | +| MySQL | `current_connection` | 当前连接公开的主机和端口;socket 使用标记,不暴露完整路径 | +| Redis 单机 | `single` | 客户端池中的目标 | +| Sentinel | `sentinel_master` | 临时数据客户端实际发现的主节点 | +| Cluster | `cluster_primaries` | 本次快照中依次成功检查的全部主节点 | + +无法可靠取得地址时不编造目标。检查失败直接抛错,不返回部分成功的 `CheckResult`。`checked_at` 为本次检查开始时的 UTC 时间,`elapsed_ms` 为单调时钟测得的总耗时。 + +### 取消与关闭 + +高层句柄退出时如已有业务异常或取消信号,清理发生的新异常(包括中断或取消)附加至该原对象的 `database_cleanup_errors`。没有主异常时,清理中的取消或中断原样传播。成功关闭之后再次关闭无额外操作;关闭失败不标记为成功,调用方可再次尝试。 + +句柄尚未产生前的构造失败仍受原生驱动实现约束。对 Connector/Python 26.10.0 的事件控制复现实验发现:异步握手期间取消可能在驱动尚未返回连接时留下未关闭 socket。SDK 无法获取这个未返回对象;本期不通过屏蔽取消、后台任务或无限等待改变驱动行为。已经返回的句柄按本节清理规则管理。此限制独立记录,不宣称覆盖驱动内部无法访问的资源。 + +### 验证状态与交付 + +实现、测试及工作流程均在独立 worktree 完成。原工作目录不切换分支、不写入文档副本;本文在交付分支中的仓库相对位置为 `docs/connection-management-design.md`,审查合并后自然进入主工作目录。 + +MongoDB、MySQL 已完成隔离真实实例验证;Redis 三种拓扑已完成两版真实驱动对象和替身拓扑的测试,真实 Redis 服务及三数据库同时在线的验收仍待可用的隔离 Linux 环境。完整命令、版本、模块提交和合并记录见 [验收记录](connection-management-validation.md)。 + ## 调研依据 - https://github.com/redis/redis-py/blob/v5.0.0/redis/client.py - https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlconnection-ping.html -- Gitee From 9019b3b079c184a57c81479ab94b0badcfa2f32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:17:37 +0800 Subject: [PATCH 42/86] =?UTF-8?q?docs(database):=20=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E7=9F=A9=E9=98=B5=E5=92=8C=E7=9C=9F=E5=AE=9E?= =?UTF-8?q?=E5=AE=9E=E4=BE=8B=E9=AA=8C=E6=94=B6=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/connection-management-validation.md | 130 +++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 docs/connection-management-validation.md diff --git a/docs/connection-management-validation.md b/docs/connection-management-validation.md new file mode 100644 index 0000000..94a5484 --- /dev/null +++ b/docs/connection-management-validation.md @@ -0,0 +1,130 @@ +# 统一连接管理验收与交付记录 + +日期:2026-11-15。实现及可用环境内的验证已完成;Redis 真实拓扑验证仍待补齐,不能将驱动替身测试视为真实服务验收。 + +## 分支与工作区 + +- 远端:`https://gitee.com/loongspears/database`,名称 `origin`。 +- 唯一交付分支:`feature/database-connection-management`。 +- 基线 `develop`:`7d38b1ffe13a17b37d0bffc15e175085dcf0bb19`。 +- 完整源码及安装验收检查点:`b2c23a8454590c705789d571c120f0fb684d3a70`;其后仅补充本验收记录,不改变受测 Python 源码。 +- 集成 worktree:`D:\996-work\loongspears\database-worktrees\connection-management`。 +- 原工作区保留在 `chore/update-git-config`,提交 `8f1352295217973a1b8624a8a01cd39cf88439a2`。 +- `temp/save` 保持 `89497823698812e1c85e6c4c12af86d4281543cf`,仅参考有效设计,没有整体合并。 + +最终交付提交可通过 `git rev-parse feature/database-connection-management` 获取。交付分支由用户审核并合入 `develop`,本次未对 `develop` 提交、合并、重写或推送。 + +## 实际模块交付与合并 + +每个模块在独立 worktree 开发,提交说明使用中文。公共配置与异常先并行,公共连接层建立后,MongoDB/MySQL/Redis 从同一 `d0706e7` 检查点并行。协调者审查后串行使用 `--no-ff` 合并,修复通过新提交记录,未压缩或改写已推送历史。 + +| 工作分支 | 模块完成提交 | 总线合并提交 | 内容 | +| --- | --- | --- | --- | +| `feature/database-connection-errors` | `4f1c918` | `e041e34` | 分类、原始错误及异常链、显式转换上下文、清理错误附加 | +| `feature/database-connection-config` | `636f7ba` | `f43b59c`、`b091724` | 显式配置、参数转换、容器隔离、冲突校验、假值外部池保护 | +| `feature/database-connection-core` | `03af1ce` | `7ccc660` | 命名配置集合、同步/异步句柄、生命周期、CheckResult | +| `feature/database-mongodb-connections` | `ce745ea` | `1c38d78` | MongoDB 创建、读取偏好检查、关闭、版本及错误分类 | +| `feature/database-mysql-connections` | `034443b` | `a41c775` | MySQL 同步/异步、非重连检查、原生错误分类及池化关闭 | +| `feature/database-redis-connections` | `1064c2a` | `0c6a1e8` | 单机、Sentinel、Cluster 的分发、逐主节点检查和资源清理 | +| `fix/database-cleanup-primary-error` | `4aae92a` | `6a99124` | 清理中的取消/中断不覆盖正在传播的业务异常 | +| `fix/database-mysql-explicit-target` | `a94add5` | `a9e8ec5` | Windows socket-only 配置不回落到默认 TCP 目标 | +| `fix/database-mysql-driver-capability` | `5c83b61` | `97a99a6` | 高层创建时缺少 C 扩展等能力转换为 DependencyError | + +公共导出、文档和验收脚本由协调者独立提交。所有任务分支已明确命名推送;没有 `--all`、`--mirror` 或 force push。远端原有 `feature` 分支导致的命名冲突由用户删除该旧分支解决,之后 `fetch --prune` 同步并继续使用原定 `feature/` 前缀。 + +## 功能范围 + +新增 `ConnectionConfig`、`Connections`、`Connection`、`AsyncConnection`、`CheckResult` 和公共异常层。不同数据库及同类数据库均以名称区分;注册只保存配置,每次创建独立句柄。原生对象从 `.native` 获取,CRUD、事务和原生返回值不包装。 + +SDK 不读取环境变量、`.env` 或隐式配置文件。调用方需要自行读取私密配置后传参。配置缺少目标时提前失败;共同参数与原生选项重复时不静默覆盖。标准容器复制,调用方显式提供的外部资源保留引用。 + +原生三个 `Client` 工厂目录、`pyproject.toml` 和 `requirements.txt` 相对基线无修改。既有参数透传、返回类型、异步调用方式和原生异常行为保留。新高层 `create_async()` 统一使用 `await`。 + +## 自动化与驱动矩阵 + +主验收环境为 Windows、CPython 3.10.20。使用隔离虚拟环境和独立 `--target` 驱动目录,运行前断言实际导入的版本,避免误用系统已安装版本。 + +| PyMongo | MySQL Connector/Python | redis-py | 完整 unittest 结果 | +| --- | --- | --- | --- | +| 4.6.0 | 8.3.0 | 5.0.0 | 84 通过 | +| 4.8.0 | 8.3.0 | 5.0.0 | 84 通过 | +| 4.9 | 8.3.0 | 5.0.0 | 84 通过 | +| 4.13.0 | 8.3.0 | 5.0.0 | 84 通过 | +| 4.18.3 | 26.10.0 | 8.1.0 | 84 通过 | +| 干净 `.[all]`:4.18.3 | 26.10.0 | 8.1.0 | 安装后的包 84 通过 | + +命令(从交付 worktree 运行): + +```powershell +$env:PYTHONPATH = "$PWD\src" +python -B -m unittest discover -s tests -q +``` + +最低版本目录分别使用 `python -m pip install --target <独立目录> pymongo==4.6.0` 等安装。将对应 PyMongo、MySQL、Redis 目录置于源码目录之前的 `PYTHONPATH`,在新进程中打印并断言三者版本,再执行相同 unittest 命令。不要在一个解释器中切换已经导入的驱动。 + +覆盖配置来源、参数映射与冲突、独立实例、原生对象身份、显式检查、错误类别/原对象/标签/详情/异常链、业务异常及取消透传、同步异步关闭、清理部分失败、外部池保护、Redis 三拓扑和原工厂回归。Redis 测试使用真实驱动类,替换网络执行或拓扑发现;同步和异步 Cluster 定向 PING 使用真实分发逻辑验证。 + +在未安装三种驱动的基础 Python 3.10.9 上额外验证:导入配置层、构造三种配置和注册集合均无驱动导入;六个创建入口只加载对应驱动,缺失时统一 DependencyError,保留原始 ModuleNotFoundError 及 cause。未修改基础环境。 + +## 干净安装 + +在独立 Python 3.10.20 虚拟环境执行: + +```powershell +python -m pip install ".[all]" +python -I -m pip check +python -I -B -m unittest discover -s tests -q +python -I -B scripts/verify_connections.py --config <调用方准备的隔离服务JSON> +``` + +安装成功,`pip check` 输出 `No broken requirements found.`。显式核对 `database.__file__` 位于该虚拟环境的 site-packages,验收使用安装后的包。 + +生成包版本 `database 0.1.dev55+gb2c23a845`。原有 utils Git 依赖实际解析为 `3f1820ff19ad8d51e83a0133609493897fa9dc2c`,未改写其仓库。安装网络使用 Git 的进程级 OpenSSL 后端配置,没有修改用户全局 Git 配置。 + +## 隔离真实服务 + +使用官网下载并校验哈希的 MongoDB 8.0.24 与 MySQL 8.4.11 Windows 二进制,使用专属临时数据目录、随机凭据和回环监听端口。未使用用户现有数据库;凭据不进入仓库或日志。 + +| 路径 | 真实服务结果 | +| --- | --- | +| MongoDB 同步 | PyMongo 4.6.0、4.8.0、4.9、4.13.0、4.18.3 检查与关闭通过 | +| MongoDB 异步 | 4.9、4.13.0、4.18.3 检查与关闭通过;4.6/4.8 给出升级提示并保留原始 ImportError | +| MySQL 同步 | Connector/Python 8.3.0 纯 Python 实现、26.10.0 纯 Python/C 扩展实现通过 | +| MySQL 异步 | 两版驱动均通过检查、SELECT 1 和关闭 | +| Redis 三模式同步/异步 | 未进行真实服务连接;驱动级测试通过 | + +MySQL 同步连接池验证关闭归还后重新获取相同 connection ID;两个独立连接关闭互不影响。认证错误 1045、库权限 1044、查询权限 1142 和拒绝连接均验证统一分类及原始对象保留。MongoDB 认证错误 18、权限错误 13、不可达目标的 ServerSelectionTimeout 均验证原始信息与分类。 + +集成检查点及安装后的包均运行 `scripts/verify_connections.py`:MongoDB/MySQL 同步和异步共四条真实路径通过,每条创建两个独立实例、关闭一个后检查另一个;另外在同一进程内同时持有 MongoDB 和 MySQL 连接,关闭其中一个后继续检查另一个。MySQL 使用原生游标执行 `SELECT 1`。 + +验收 JSON 格式如下,由脚本调用方读取,SDK 本身不读取文件: + +```json +{ + "orders": { + "backend": "mysql", + "host": "127.0.0.1", + "port": 3306, + "username": "isolated_test_user", + "password": "由调用方提供", + "connect_timeout_s": 3, + "options": {"database": "isolated_test"} + } +} +``` + +可向该 JSON 增加 MongoDB、Redis 三种拓扑和同类多连接名称;脚本逐项检查后再次同时持有全部命名连接。服务故障测试须由外部测试进程限制运行时间,不代表 SDK 提供整体硬超时。 + +## 未验证事项和已知限制 + +1. 本机没有 Docker/Podman;已有 WSL Ubuntu 无法启动,报告需要 WSL2 内核及虚拟机平台支持。没有为本任务修改系统虚拟化设置。Redis 单机、Sentinel、Cluster 的六条真实路径,以及未知 Sentinel 服务、实际不可达主节点、真实 Redis 认证/权限故障仍待隔离 Linux 环境复验。三种数据库同时在线的真实验收也待补齐,当前完成的是 MongoDB/MySQL 同时在线。 +2. Connector/Python 8.3.0 的 C 扩展在本机不可用,最低版本真实同步验收使用纯 Python 实现。高层显式要求缺失 C 扩展时现给出 DependencyError。26.10.0 的异步池在本机 Windows/Python 3.10 组合由驱动报告 NotSupportedError;不宣称支持驱动禁用的能力。 +3. Connector/Python 26.10.0 在异步握手中取消、原生工厂尚未返回对象时,事件控制复现显示已打开 socket 未关闭。SDK 无法获取该内部对象;没有引入屏蔽取消或后台等待改变行为。已返回句柄的清理和取消行为通过回归测试。 +4. 未验证所有 Python 3.10 以上版本、全部操作系统、TLS/复制集/failover 等原生选项组合。原生选项继续透传,其能力以所选驱动为准。 +5. MongoDB ping 不公开实际执行节点,成功结果 targets 为空;不通过额外选择服务器伪造目标。检查成功不证明所有 CRUD 权限或持续可用。统一异常不代表操作可安全重试。 + +## 历史与保护核对 + +使用 `git log --first-parent develop..feature/database-connection-management` 查看集成顺序,进入各 merge 查看模块迭代。使用 `git diff --check` 检查空白问题;核对原工作区状态、develop/temp/save 引用、原工厂和依赖文件差异。 + +任务分支及 worktree 保留供用户审查。验收结束仅关闭本任务启动的临时 MySQL/MongoDB 进程,临时数据和原有用户服务不删除。最终推送只指定总集成分支,并以 `git ls-remote origin <分支完整引用>` 核对远端 SHA。 -- Gitee From 930194245ea77cb2e2d479a8b57d89bf4d984973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 01:18:51 +0800 Subject: [PATCH 43/86] =?UTF-8?q?style(database):=20=E6=B8=85=E7=90=86?= =?UTF-8?q?=E4=BA=A4=E4=BB=98=E6=96=87=E4=BB=B6=E6=9C=AB=E5=B0=BE=E7=A9=BA?= =?UTF-8?q?=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 - docs/connection-management-design.md | 1 - docs/connection-management-validation.md | 2 +- scripts/verify_connections.py | 1 - src/database/connections.py | 1 - tests/test_connections.py | 1 - 6 files changed, 1 insertion(+), 6 deletions(-) diff --git a/README.md b/README.md index de0bbc8..1650ff3 100644 --- a/README.md +++ b/README.md @@ -184,4 +184,3 @@ MongoDB 的检查结果使用 `scope="selected_server"`,`targets` 为空:原 异步创建尚未返回句柄时的资源清理由原生驱动负责。当前受测 Connector/Python 的握手取消存在上游清理限制,详见设计文档。Redis 拓扑已完成驱动级验证,真实服务验证仍待完成。 详细设计和开发流程见 [连接管理设计](docs/connection-management-design.md)。实际验证情况见 [验收记录](docs/connection-management-validation.md)。 - diff --git a/docs/connection-management-design.md b/docs/connection-management-design.md index 8362e51..03bc71c 100644 --- a/docs/connection-management-design.md +++ b/docs/connection-management-design.md @@ -141,4 +141,3 @@ MongoDB、MySQL 已完成隔离真实实例验证;Redis 三种拓扑已完成 - https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html - https://www.mongodb.com/docs/manual/reference/error-codes/ - https://pymongo.readthedocs.io/en/stable/api/pymongo/errors.html - diff --git a/docs/connection-management-validation.md b/docs/connection-management-validation.md index 94a5484..3654f03 100644 --- a/docs/connection-management-validation.md +++ b/docs/connection-management-validation.md @@ -7,7 +7,7 @@ - 远端:`https://gitee.com/loongspears/database`,名称 `origin`。 - 唯一交付分支:`feature/database-connection-management`。 - 基线 `develop`:`7d38b1ffe13a17b37d0bffc15e175085dcf0bb19`。 -- 完整源码及安装验收检查点:`b2c23a8454590c705789d571c120f0fb684d3a70`;其后仅补充本验收记录,不改变受测 Python 源码。 +- 完整源码及安装验收检查点:`b2c23a8454590c705789d571c120f0fb684d3a70`;其后仅补充本验收记录和清理文件末尾空行,不改变受测代码行为。 - 集成 worktree:`D:\996-work\loongspears\database-worktrees\connection-management`。 - 原工作区保留在 `chore/update-git-config`,提交 `8f1352295217973a1b8624a8a01cd39cf88439a2`。 - `temp/save` 保持 `89497823698812e1c85e6c4c12af86d4281543cf`,仅参考有效设计,没有整体合并。 diff --git a/scripts/verify_connections.py b/scripts/verify_connections.py index 189df1d..06a15f5 100644 --- a/scripts/verify_connections.py +++ b/scripts/verify_connections.py @@ -89,4 +89,3 @@ def main(): if __name__ == "__main__": main() - diff --git a/src/database/connections.py b/src/database/connections.py index 206f7fd..bbf2ffa 100644 --- a/src/database/connections.py +++ b/src/database/connections.py @@ -204,4 +204,3 @@ class Connections: return AsyncConnection( native, adapter, name, config.backend.value, config.mode, loop, ) - diff --git a/tests/test_connections.py b/tests/test_connections.py index c2e6fda..aa9bf60 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -240,4 +240,3 @@ class AsyncConnectionTests(unittest.IsolatedAsyncioTestCase): await failing() self.assertIs(caught.exception.original, original) await handle.close() - -- Gitee From ea78e3158d91e6d064290108726ef5c8e9819ca7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 02:09:11 +0800 Subject: [PATCH 44/86] =?UTF-8?q?fix(database):=20=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E9=9B=86=E7=BE=A4=E5=85=B3=E9=97=AD=E9=87=8D=E8=AF=95=E4=B8=8E?= =?UTF-8?q?=E5=AF=86=E7=A0=81=E5=88=AB=E5=90=8D=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 保留异步 Redis Cluster 清理失败的节点,后续关闭继续重试,全部清理成功后才标记关闭。 将 MySQL password1 纳入密码别名冲突检测,保留独立的多因子密码配置。 补充对应回归测试;最低支持版本和较新版本驱动下均通过 87 项测试。 --- src/database/_adapters/redis.py | 14 +++++++--- src/database/config.py | 1 + tests/test_connection_config.py | 26 +++++++++++++++++++ tests/test_managed_redis.py | 46 +++++++++++++++++++++++++++++++++ 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/src/database/_adapters/redis.py b/src/database/_adapters/redis.py index 0530912..d49f07e 100644 --- a/src/database/_adapters/redis.py +++ b/src/database/_adapters/redis.py @@ -62,6 +62,7 @@ class Adapter: self._name = name self._is_async = is_async self._external_pool = config.options.get("connection_pool") is not None + self._pending_cluster_nodes = {} @staticmethod def classify(error: Exception) -> type[DatabaseError] | None: @@ -323,10 +324,15 @@ class Adapter: nodes = self._cluster_nodes(native) await self._release_async(_async_closer(native), None, errors) if errors: - # Older and newer drivers can both stop before draining startup nodes. - for node in nodes: - target = _endpoint(node.host, node.port) - await self._release_async(node.disconnect, target, errors) + # Native close marks the client uninitialized before disconnecting. + # Later closes may be no-ops, so retain nodes until cleanup succeeds. + self._pending_cluster_nodes.update((id(node), node) for node in nodes) + for node_id, node in tuple(self._pending_cluster_nodes.items()): + target = _endpoint(node.host, node.port) + error_count = len(errors) + await self._release_async(node.disconnect, target, errors) + if len(errors) == error_count: + del self._pending_cluster_nodes[node_id] else: errors.extend(await self._close_single_async(native, not self._external_pool)) self._finish_cleanup(errors) diff --git a/src/database/config.py b/src/database/config.py index 5107f9b..52dd0ff 100644 --- a/src/database/config.py +++ b/src/database/config.py @@ -30,6 +30,7 @@ _NATIVE_ALIASES = { DatabaseType.MYSQL: { "username": "user", "passwd": "password", + "password1": "password", "db": "database", "connect_timeout": "connection_timeout", "read_default_file": "option_files", diff --git a/tests/test_connection_config.py b/tests/test_connection_config.py index 0c3569d..a82530a 100644 --- a/tests/test_connection_config.py +++ b/tests/test_connection_config.py @@ -126,6 +126,32 @@ class ConnectionConfigTests(unittest.TestCase): with self.assertRaises(ConfigurationError): ConnectionConfig(backend, options=options, **settings) + def test_mysql_rejects_duplicate_first_factor_passwords(self): + for fields, options in ( + ({"password": "new"}, {"password1": "old"}), + ({"password": ""}, {"password1": "old"}), + ({"password": "same"}, {"password1": "same"}), + ({}, {"password": "first", "password1": "second"}), + ({}, {"password1": "first", "passwd": "second"}), + ): + with self.subTest(fields=tuple(fields), options=tuple(options)): + with self.assertRaises(ConfigurationError) as failed: + ConnectionConfig("mysql", host="db.internal", options=options, **fields) + self.assertEqual(failed.exception.phase, "config") + self.assertEqual(failed.exception.backend, DatabaseType.MYSQL) + + def test_mysql_password1_and_additional_factors_are_preserved(self): + for fields, options in ( + ({}, {"password1": "first", "password2": "second", "password3": "third"}), + ({"password": "first"}, {"password2": "second", "password3": "third"}), + ): + with self.subTest(fields=tuple(fields), options=tuple(options)): + config = ConnectionConfig("mysql", host="db.internal", options=options, **fields) + self.assertEqual( + config.snapshot().driver_kwargs(), + {"host": "db.internal", **options, **fields}, + ) + def test_snapshot_and_driver_arguments_keep_opaque_objects(self): class Opaque: def __deepcopy__(self, memo): diff --git a/tests/test_managed_redis.py b/tests/test_managed_redis.py index 52a20c3..9640c7e 100644 --- a/tests/test_managed_redis.py +++ b/tests/test_managed_redis.py @@ -389,3 +389,49 @@ class ManagedAsyncRedisTests(unittest.IsolatedAsyncioTestCase): self.assertIs(failed.exception.database_cleanup_errors[0].original, fallback) self.assertIn(call(second), disconnect.await_args_list) self.assertIn(call(seed), disconnect.await_args_list) + + async def test_async_cluster_close_retries_failed_nodes_until_disconnected(self): + for failing_node_kind in ("primary", "seed"): + with self.subTest(failing_node_kind=failing_node_kind): + handle = await _registry(_config("cluster")).create_async("test") + self.addAsyncCleanup(handle.close) + native = handle.native + _configure_async_cluster(native) + seed, = native.nodes_manager.startup_nodes.values() + failing_node = native.get_primaries()[0] if failing_node_kind == "primary" else seed + # Use native close(), which marks the client uninitialized before cleanup. + native._initialize = False + failure = redis.exceptions.ConnectionError("node disconnect failed") + should_fail = True + disconnected = set() + + async def disconnect_node(node): + if node is failing_node and should_fail: + raise failure + disconnected.add(id(node)) + + with patch.object( + type(failing_node), "disconnect", autospec=True, side_effect=disconnect_node + ) as disconnect: + with self.assertRaises(DatabaseConnectionError) as first: + await handle.close() + self.assertIs(first.exception.original, failure) + self.assertTrue(native._initialize) + self.assertFalse(handle.closed) + self.assertNotIn(id(failing_node), disconnected) + + disconnect.reset_mock() + with self.assertRaises(DatabaseConnectionError) as retry: + await handle.close() + self.assertIs(retry.exception.original, failure) + self.assertFalse(handle.closed) + disconnect.assert_awaited_once_with(failing_node) + + should_fail = False + disconnect.reset_mock() + await handle.close() + self.assertTrue(handle.closed) + self.assertIn(id(failing_node), disconnected) + disconnect.assert_awaited_once_with(failing_node) + await handle.close() + disconnect.assert_awaited_once_with(failing_node) -- Gitee From b6707125620d1cea28cabbc7a72e72c84ae73bbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Sun, 15 Nov 2026 14:09:55 +0800 Subject: [PATCH 45/86] =?UTF-8?q?chore(database):=20=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E9=A1=B9=E7=9B=AE=E6=96=87=E6=A1=A3=E5=B9=B6=E7=BB=9F=E4=B8=80?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 补齐 README、发布日志、Ruff 配置和开发规范,记录 worktree 并行协作流程。 默认安装三个数据库驱动,统一运行依赖与 requirements.txt 的版本约束。 --- .ruff.toml | 11 +++ AGENTS.md | 70 +++++++++++++++++++ CHANGELOG.md | 40 +++++++++++ CONTRIBUTING.md | 48 +++++++++++++ README.md | 83 ++++++++++++++++++++--- SECURITY.md | 11 +++ docs/README.md | 25 +++++++ docs/assets/database-badges.svg | 19 ++++++ docs/assets/database-banner.svg | 86 ++++++++++++++++++++++++ docs/connection-management-design.md | 6 +- docs/connection-management-validation.md | 2 + pyproject.toml | 13 +--- tests/README.md | 40 +++++++++++ 13 files changed, 428 insertions(+), 26 deletions(-) create mode 100644 .ruff.toml create mode 100644 AGENTS.md create mode 100644 CHANGELOG.md create mode 100644 CONTRIBUTING.md create mode 100644 SECURITY.md create mode 100644 docs/README.md create mode 100644 docs/assets/database-badges.svg create mode 100644 docs/assets/database-banner.svg create mode 100644 tests/README.md diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000..b93a424 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,11 @@ +line-length = 120 +target-version = "py310" + +[lint] +select = ["E", "F", "I"] + +[lint.isort] +force-single-line = true + +[format] +quote-style = "single" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4483040 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,70 @@ +# AGENTS.md + +## 项目概览 + +`database` 是支持 Python 3.10 及以上版本的 MongoDB、MySQL 和 Redis 连接 SDK。本文件适用于整个仓库。进入子目录时先检查是否有该目录的 `AGENTS.md` 或 `AGENTS.override.md`。 + +## 从哪里开始 + +- 先看 `git status --short --branch`、任务相关差异和暂存区,保留工作区已有改动。 +- 使用示例见 [README](README.md),开发和提交规范见 [贡献指南](CONTRIBUTING.md)。 +- `src/database/config.py` 负责配置和参数校验,`connections.py` 负责命名连接和生命周期,`errors.py` 负责异常转换。 +- `src/database/_adapters/` 负责高层连接与驱动之间的适配。原生工厂维护在 `src/database/{mongodb,mysql,redis}/`,离线测试放在 `tests/`。 + +## 修改连接代码 + +- 核对 [连接管理设计](docs/connection-management-design.md)。保持原生 `Client` 的签名、参数透传和返回对象兼容。高层 `Connections.create_async()` 一律通过 `await` 调用,原生工厂的异步调用方式按各后端现有契约处理。 +- 高层配置由调用方显式传入。默认安装包含三个后端的驱动。公共接口、配置构造和连接登记保持独立于驱动导入,创建连接时再加载对应驱动。 +- 按改动覆盖同步和异步路径。检查只报告当次探测结果,关闭遵守自有资源与外部资源的归属,异常转换保留原始驱动错误和因果链。 +- 使用 [.ruff.toml](.ruff.toml) 的风格,为新增公开接口补充类型注解。修改运行依赖时,同步维护 [pyproject.toml](pyproject.toml) 的 `project.dependencies` 和 [requirements.txt](requirements.txt) 中的包名与版本约束。 + +## 运行检查 + +在仓库根目录使用当前工作区的源码。开发环境与工具安装见 [贡献指南](CONTRIBUTING.md)。 + +```bash +python -m pip install -e . +python -B -m unittest discover -s tests -q +git diff --check +``` + +- Python 改动先按 [测试指南](tests/README.md) 运行定向 `unittest`,对改动文件执行 Ruff lint 和格式检查。涉及公共入口、生命周期或共享逻辑时运行完整离线回归。 +- 自动测试使用本地样本和驱动替身。隔离服务验收使用 `scripts/verify_connections.py --config` 和显式测试配置,凭据和日志处理见 [安全说明](SECURITY.md)。 +- 文档改动核对链接、示例和差异。工具配置改动使用对应工具检查。 + +## Git 与 worktree 协作 + +### 分工和基线 + +主智能体负责协调。先用 `git status --short --branch`、`git diff --cached` 和 `git worktree list --porcelain` 核对现状,记录任务基线提交、集成目标和已有工作树。分配阶段目标、文件范围、依赖及验收命令。同一文件由一名工作者负责,公共导出和其他共享文件由协调者安排。 + +需要并行修改时,每个工作者使用独立分支、worktree 和暂存区。协调者在集成工作树负责合并。下面以已提交的 `develop` 为基线,示例中的分支名和路径按任务替换。 + +```powershell +$base = (git rev-parse develop).Trim() +git worktree add -b feature/task-integration ../database-worktrees/task-integration $base +git worktree add -b feature/task-config ../database-worktrees/task-config $base +git worktree add -b feature/task-errors ../database-worktrees/task-errors $base +``` + +### 开发和集成 + +1. 工作者在自己的 worktree 修改和验证,使用 `git -C <路径>` 或进入该目录执行 Git 命令。需要提交时,按分配的路径暂存、核对 `git diff --cached`,使用中文 Conventional Commits。 +2. 有前置依赖时,先由协调者集成公共变更,再从已验证的集成提交建立或同步后续任务分支。工作者在交付时提供改动范围、验证结果和提交 SHA。仅要求工作区改动的任务交付对应差异和验证结果。 +3. 需要合并时,协调者核对交付 SHA 和差异,在集成 worktree 串行用 `--no-ff` 合入。合并后记录新的 HEAD 并重跑相关验证。冲突由协调者在集成 worktree 处理,核对公共接口并保留各分支的原始提交。最终合入目标分支和推送按当次任务要求执行。 + +```powershell +$workerHead = (git rev-parse feature/task-config).Trim() +git -C ../database-worktrees/task-integration diff "HEAD...$workerHead" +git -C ../database-worktrees/task-integration merge --no-ff -m "feat(config): 合入连接配置" $workerHead +git -C ../database-worktrees/task-integration rev-parse HEAD +``` + +### Git 操作记录 + +在任务交付中按执行顺序记录 Git 命令,标明执行者、分支、worktree 与结果。汇总基线 SHA、各任务的交付 SHA 或差异路径,以及验证命令与结果。合并记录写明合入的 SHA 和集成后的 HEAD。推送记录写明远端、分支及核对过的远端 SHA。清理工作树前,用 `git worktree list` 和 `git status` 核对任务已验收且工作区干净。 + +## 文档与交付 + +- 公开用法变更时同步 README。对外可见的功能、修复和兼容性变化登记在 [发布日志](CHANGELOG.md) 的「未发布」部分,版本记录依据实际 Git 标签。 +- 分支、暂存、提交、合并和推送按当次任务要求执行。交付时注明检查结果和待验证事项。 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a381417 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,40 @@ +# 发布日志 + +记录面向使用者的功能、修复与兼容性变化。发布日期以仓库的 Git 标签为准。 + +## 未发布 + +### 新增 + +- 新增 `ConnectionConfig` 和 `Connections`,支持按名称登记 MongoDB、MySQL 和 Redis 配置,按需创建独立的同步或异步句柄。 +- 新增显式连接检查、上下文管理和统一异常。可通过 `.native` 访问原生驱动对象,通过 `DatabaseError.original` 读取原始异常。 +- 统一连接管理覆盖 Redis 单机、Sentinel 与 Cluster。Cluster 的检查范围为当前发现的主节点。 +- 新增基于 MySQL Connector/Python 的同步和异步工厂。 +- 新增隔离服务验证脚本,以及按后端覆盖配置、检查、异常和关闭的离线测试。 + +### 修复 + +- MySQL 目标配置提前拒绝 Windows 上无法使用的纯 Unix socket 配置,并把缺少驱动能力的错误转换为 `DependencyError`。 +- 识别 MySQL 的 `password1` 密码别名,在公共密码参数与该别名同时传入时报告冲突。 +- Redis 异步 Cluster 关闭失败后保留待清理节点,以便后续关闭重试。关闭期间的错误附加到已有业务异常。 + +### 兼容性与依赖 + +- 源码安装统一使用 `python -m pip install .`,默认包含 MongoDB、MySQL 和 Redis 驱动。原有安装 extras 已移除,运行依赖及版本约束与 `requirements.txt` 一致。 +- PyMongo 的默认依赖下限为 `4.6.0`,同步连接支持该版本。异步连接需要 PyMongo 4.9 或以上版本,缺少异步 API 时给出升级提示。 +- 原生工厂继续返回驱动对象。统一入口的 `create_async()` 需要 `await`,原生工厂沿用各自的异步调用方式。 + +### 文档与验证 + +- 补充快速开始、架构示意、贡献和测试指南。 +- 已有隔离验证覆盖 MongoDB 和 MySQL。Redis 三种拓扑已进行驱动级离线测试,真实服务验收待补齐。[连接管理验收记录](docs/connection-management-validation.md) + +## v0.0.1-beta - 2026-07-17 + +初始 Beta 版本,对应 Git 标签 `v0.0.1-beta`。 + +### 新增 + +- 提供 MongoDB 同步和异步原生客户端工厂,以及批量插入和更新辅助工具。 +- 提供 Redis 同步和异步客户端,支持单机、Sentinel 与 Cluster 的统一创建入口。 +- 提供 `mongodb`、`redis` 和包含两者的 `all` 安装选项。 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..1f1d4da --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,48 @@ +# 贡献指南 + +## 本地开发 + +项目支持 Python 3.10 及以上版本。本地参考版本见 [.python-version](.python-version)。安装通过 Git 获取共享依赖,请先准备 Python 与 Git。 + +在仓库根目录执行 `python -m venv .venv`。PowerShell 使用 `.\.venv\Scripts\Activate.ps1` 激活环境,Bash 使用 `source .venv/bin/activate`。在虚拟环境中执行以下命令。 + +```bash +python -m pip install -e . +python -m pip install ruff +``` + +开发安装包含三个后端的驱动,测试使用标准库 `unittest`。运行依赖及版本约束在 [pyproject.toml](pyproject.toml) 的 `project.dependencies` 和 [requirements.txt](requirements.txt) 中同步维护。验证驱动最低版本时,在独立环境使用对应的版本约束。源码构建时由 `setuptools-scm` 生成 `src/database/_version.py`。 + +## 代码与文档风格 + +- [.ruff.toml](.ruff.toml) 统一 Python 3.10 语法目标、120 字符行宽、单引号格式,以及 `E`、`F`、`I` 规则。一行导入一个对象。 +- 新增公开接口写明参数和返回值类型。涉及默认值、异常或资源生命周期时,用 NumPy 风格的 docstring 描述契约,完整用法示例维护在 README 或相关文档。 +- Markdown 普通段落及引用正文采用一段一行、段间空行。列表、表格、代码块和显式换行保留各自结构。 +- 根据 [文档索引](docs/README.md) 更新对应文档。文档中的连接地址、凭据和测试数据使用本地占位值。 + +## 提交前检查 + +涉及 Python 改动时,从仓库根目录运行离线回归。用 `git diff --check` 核对本次差异。定向测试及真实服务探测见 [测试指南](tests/README.md)。 + +```bash +python -B -m unittest discover -s tests -q +git diff --check +``` + +修改 Python 后,对实际改动的文件运行 `python -m ruff check <文件路径>` 和 `python -m ruff format --check <文件路径>`。全库扫描可以作为现有代码风格的基线,检查本次改动时以涉及的诊断为准。 + +修改依赖或构建配置时,在独立环境构建 wheel 和 sdist,验证产物安装、`python -m pip check` 和导入来源。 + +## 分支与提交 + +日常改动从 `develop` 建立任务分支,审查后合入 `develop`。`master` 用于发布。按工作内容使用 `feature/*`、`fix/*`、`refactor/*`、`docs/*` 或 `chore/*`。保留任务分支的提交关系时使用 `--no-ff` 合并。智能体的并行分工、集成和交付记录见 [Git 与 worktree 协作](AGENTS.md#git-与-worktree-协作)。 + +采用中文 Conventional Commits,保留英文类型和可选的 scope,正文使用中文说明背景和验证情况。每次提交包含一组可独立审查的改动,按路径暂存后核对 `git diff --cached`。 + +```text +feat(connections): 支持按名称创建数据库连接 +fix(redis): 保留集群连接关闭失败的原始异常 +docs(readme): 补充异步连接示例 +``` + +提交合并请求时说明变更目的、影响的数据库或驱动版本、兼容性、验证结果及待验证事项。 diff --git a/README.md b/README.md index 1650ff3..ae9ab8d 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,71 @@ -# database +
+

database

+

用一套 Python API 管理 MongoDB、MySQL 和 Redis 连接

+

+ Python 3.10+ · 同步与异步 API · MIT 许可证 +

+

+ 快速开始 · + 原生客户端 · + 连接管理 · + 文档 · + 发布日志 +

+
-使用同一套工厂接口获取 MongoDB、MySQL 和 Redis 的原生客户端。工厂把连接参数传给驱动,查询、事务和连接关闭继续使用各驱动的 API。 +--- -## 安装 + -在仓库根目录运行以下命令。Python 需要 3.10 或更高版本,下载共享依赖时需要 Git。 +## 🎯 快速开始 + +### 安装 + +准备 Python 3.10 或更高版本和 Git。在源码仓库根目录执行以下命令,安装包及三个后端的驱动。 ```bash -python -m pip install ".[all]" +python -m pip install . ``` -只使用一种数据库时,可以将 `all` 换成 `mongodb`、`mysql` 或 `redis`。 +驱动版本约束与 [requirements.txt](requirements.txt) 一致。 + +| 后端 | 默认驱动依赖 | +| --- | --- | +| MongoDB | `pymongo>=4.6.0` | +| MySQL | `mysql-connector-python>=8.3.0` | +| Redis | `redis>=5.0.0` | -## MongoDB +### 第一个连接 -PyMongo 4.6–4.8 支持 `create()`。`create_async()` 需要 PyMongo 4.9 或更高版本,`all` 会安装异步 API 已正式发布的 PyMongo 4.13 或更高版本。 +下面连接到本机 `127.0.0.1:6379` 上的 Redis,检查一次连接并在退出上下文时关闭它。 + +```python +from database import ConnectionConfig +from database import Connections + +connections = Connections({ + 'cache': ConnectionConfig(backend='redis', host='127.0.0.1', port=6379), +}) + +with connections.create('cache') as connection: + result = connection.check() + print(result.backend, result.elapsed_ms) +``` + +## ✨ 核心特性 + +- 🗂️ **按名称管理连接**。在同一个 `Connections` 中登记多个后端或同一后端的多个实例。 +- 🔌 **直接使用原生驱动**。用 `Client` 工厂或句柄的 `.native` 访问驱动对象,继续使用原生查询、事务和命令。 +- ⚡ **同步与异步**。统一入口管理创建和关闭,支持 Redis 单机、Sentinel 与 Cluster。 +- 🩺 **显式检查与统一异常**。用 `check()` 查看一次探测结果,从 `DatabaseError.original` 获取原始异常。 + + + +## 原生客户端 + +### MongoDB + +PyMongo 4.6–4.8 支持同步的 `create()`。异步的 `create_async()` 需要 PyMongo 4.9 或更高版本。使用 MongoDB 异步连接的应用应在自己的依赖中声明 `pymongo>=4.9`。 ```python import asyncio @@ -33,7 +84,7 @@ async def ping_mongodb(): asyncio.run(ping_mongodb()) ``` -## MySQL +### MySQL 使用 MySQL Connector/Python。`create_async()` 需要 `await`,驱动完成建连后返回原生异步连接。示例从 `MYSQL_USER` 和 `MYSQL_PASSWORD` 环境变量读取凭据。 @@ -61,7 +112,7 @@ async def ping_mysql(): asyncio.run(ping_mysql()) ``` -## Redis +### Redis `create()` 和 `create_async()` 默认获取单机客户端。`connect()` 还支持 `mode="sentinel"` 和 `mode="cluster"`,可传 `is_async=True` 获取异步对象。Sentinel 模式返回 `Sentinel`,可用 `master_for()` 获取主节点的数据客户端。 @@ -82,6 +133,8 @@ async def ping_redis(): asyncio.run(ping_redis()) ``` + + ## 统一配置和显式连接管理 新高层接口只接受调用方传入的参数,不读取环境变量、不加载 `.env`。上文 MySQL 示例中的环境读取发生在调用方代码中。可以自行读取密钥服务或配置文件,再传入 SDK。 @@ -183,4 +236,12 @@ MongoDB 的检查结果使用 `scope="selected_server"`,`targets` 为空:原 异步创建尚未返回句柄时的资源清理由原生驱动负责。当前受测 Connector/Python 的握手取消存在上游清理限制,详见设计文档。Redis 拓扑已完成驱动级验证,真实服务验证仍待完成。 -详细设计和开发流程见 [连接管理设计](docs/connection-management-design.md)。实际验证情况见 [验收记录](docs/connection-management-validation.md)。 +## 文档与贡献 + +- [文档索引](docs/README.md) 汇总连接管理设计、验收记录和源码目录说明。 +- [发布日志](CHANGELOG.md) 记录已发布版本和未发布改动。 +- [贡献指南](CONTRIBUTING.md) 说明开发环境、代码检查和提交约定。 +- [测试指南](tests/README.md) 提供离线测试及隔离数据库的验证命令。 +- [安全说明](SECURITY.md) 说明凭据、日志和安全问题反馈方式。 + +本项目采用 [MIT 许可证](LICENSE)。 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..960bab9 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,11 @@ +# 安全说明 + +## 配置与日志 + +应用从自己的配置来源取得连接参数和凭据,再显式传给 SDK。保管好数据库 URI、密码和真实服务的测试配置,文档、测试和共享日志使用脱敏值。 + +`DatabaseError.original`、异常链和 `database_cleanup_errors` 会保留原始异常对象,其中可能包含驱动返回的连接信息。共享诊断信息前请检查连接名称、地址和原始异常详情,按需脱敏。 + +## 问题反馈 + +涉及凭据泄露或其他安全问题时,请通过项目维护邮箱联系维护者。报告附受影响的提交或版本、数据库及驱动版本、影响范围和使用占位值的复现信息。[联系维护者](mailto:msr20666@foxmail.com) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..b17bc7e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,25 @@ +# database 文档索引 + +## 使用与开发 + +- [项目 README](../README.md) 提供安装、原生客户端、统一连接管理和同步、异步用法。 +- [贡献指南](../CONTRIBUTING.md) 提供本地环境、代码与文档风格、检查命令和分支提交约定。 +- [智能体编码工作规范](../AGENTS.md) 提供仓库协作规则和任务阅读入口。 +- [安全说明](../SECURITY.md) 说明连接凭据、异常日志和安全问题反馈方式。 +- [发布日志](../CHANGELOG.md) 记录已发布版本和未发布改动。 + +## 连接管理 + +- [连接管理设计](connection-management-design.md) 记录配置、创建、检查、异常和关闭的接口契约。 +- [测试指南](../tests/README.md) 提供离线回归和隔离服务探测的运行方式。 +- [连接管理验收记录](connection-management-validation.md) 记录交付时实际验证的驱动版本、真实拓扑及待验证项。 + +## 目录职责 + +| 路径 | 职责 | +| --- | --- | +| [src/database/](../src/database/) | 公共导出、连接配置、统一句柄和异常转换 | +| [src/database/_adapters/](../src/database/_adapters/) | 三种后端高层连接的创建、检查、关闭及驱动错误映射 | +| [mongodb/](../src/database/mongodb/)、[mysql/](../src/database/mysql/)、[redis/](../src/database/redis/) | 后端原生客户端工厂及各自的辅助功能 | +| [tests/](../tests/) | 离线自动化测试 | +| [scripts/verify_connections.py](../scripts/verify_connections.py) | 使用显式测试配置验证真实连接 | diff --git a/docs/assets/database-badges.svg b/docs/assets/database-badges.svg new file mode 100644 index 0000000..41909c6 --- /dev/null +++ b/docs/assets/database-badges.svg @@ -0,0 +1,19 @@ + + Python 3.10+, Sync and async API, MIT license + + + + + + + + + + Python + 3.10+ + API + sync + async + License + MIT + + diff --git a/docs/assets/database-banner.svg b/docs/assets/database-banner.svg new file mode 100644 index 0000000..429da3a --- /dev/null +++ b/docs/assets/database-banner.svg @@ -0,0 +1,86 @@ + + database + The database Python SDK connects to MongoDB, MySQL, and Redis through the Connections API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PYTHON SDK + database + Configure · Connect · Check · Close + + + + + + + + + + Connections + sync / async + + + + + + + + + MongoDB + + + + + + + + + + + MySQL + + + + + + + + + + Redis + + + diff --git a/docs/connection-management-design.md b/docs/connection-management-design.md index 03bc71c..f0be27e 100644 --- a/docs/connection-management-design.md +++ b/docs/connection-management-design.md @@ -77,7 +77,7 @@ SDK 直接调用驱动的边界可以识别明确来源的底层超时/传输异 - database.connections: Connections, Connection, AsyncConnection, CheckResult,运行时才 import database._adapters.。 - database._adapters..Adapter(config, name, is_async=False):create()、async create_async() 返回原生对象;check(native)、async check_async(native) 返回(scope, targets);close(native)、async close_async(native);classify(error) 返回统一异常类或 None(非所属驱动错误)。 - 适配器应在需要标注具体目标的检查异常处调用 convert_error(... owned=True, classifier=self.classify) 并 raise from;核心同样处理创建/检查/关闭外围异常。已统一错误不重复包装。 -- errors 实现不导入驱动;配置层也不导入驱动。可选驱动只在实际使用时加载。 +- `errors` 和配置层独立于驱动。创建连接时由对应后端加载驱动。 - 公共 __init__ 导出由协调者统一修改。worker 不改现有原生工厂文件。 - 新测试独立 tests/test_connection_config.py、test_connection_errors.py、test_connections.py、test_managed_.py。 @@ -95,8 +95,8 @@ feature/database-mongodb-connections、feature/database-mysql-connections、feat 不改历史worktree、不删除temp/save、不处理无关文件;保留任务分支和worktree供审核。 ## 验收 -沿用 unittest 与 IsolatedAsyncioTestCase。覆盖:无环境读取、显式目标、配置隔离/冲突/映射、多实例及同名同步异步独立、原生对象身份、检查语义、异常对象和链保真、保守分类、取消透传、外部池保护、清理失败、可选依赖、旧工厂回归。 -保持 Python>=3.10、pymongo>=4.6.0(异步>=4.9,all>=4.13)、mysql-connector-python>=8.3.0、redis>=5.0.0。 +沿用 unittest 与 IsolatedAsyncioTestCase。覆盖:无环境读取、显式目标、配置隔离/冲突/映射、多实例及同名同步异步独立、原生对象身份、检查语义、异常对象和链保真、保守分类、取消透传、外部池保护、清理失败、驱动缺失诊断、旧工厂回归。 +保持 Python 3.10 及以上版本。默认安装 `pymongo>=4.6.0`、`mysql-connector-python>=8.3.0` 和 `redis>=5.0.0`,版本约束与 [requirements.txt](../requirements.txt) 一致。MongoDB 异步连接需要 PyMongo 4.9 或更高版本。 验证最低及选定当前版本;PyMongo4.6/4.8同步及异步提示、4.9/4.13异步。 隔离服务验证 MongoDB/MySQL8.4/Redis单机、Sentinel、Cluster 共10条同步异步路径,以及认证/权限/连接拒绝、未知服务、不可达主节点,多数据库并用和独立关闭。MySQL 原生 SELECT 1。外部测试进程限制故障测试时限,不冒充SDK硬超时。 README补全显式配置、多实例、检查、原生访问、异常、资源归属和版本限制。 diff --git a/docs/connection-management-validation.md b/docs/connection-management-validation.md index 3654f03..f765f19 100644 --- a/docs/connection-management-validation.md +++ b/docs/connection-management-validation.md @@ -68,6 +68,8 @@ python -B -m unittest discover -s tests -q ## 干净安装 +以下命令记录当时交付提交的安装过程。当前分支通过 `python -m pip install .` 安装全部三个后端,版本约束见 [安装说明](../README.md#安装)。 + 在独立 Python 3.10.20 虚拟环境执行: ```powershell diff --git a/pyproject.toml b/pyproject.toml index c620aad..8079248 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,19 +28,8 @@ classifiers = [ ] dependencies = [ "loongspears.utils @ git+https://gitee.com/loongspears/utils", -] -[project.optional-dependencies] -all = [ - "database[mongodb,mysql,redis]", - "pymongo>=4.13", -] -mysql = [ - "mysql-connector-python>=8.3.0", -] -mongodb = [ "pymongo>=4.6.0", -] -redis = [ + "mysql-connector-python>=8.3.0", "redis>=5.0.0", ] diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..88c8206 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,40 @@ +# 测试指南 + +## 离线回归 + +按 [贡献指南](../CONTRIBUTING.md) 在独立环境运行 `python -m pip install -e .`,以 editable 方式安装当前工作区及三个后端的驱动。从仓库根目录运行以下命令。 + +```bash +python -B -c "import database; print(database.__file__)" +python -B -m unittest discover -s tests -q +``` + +源码回归时,第一条命令的路径应指向当前工作区的 `src/database/__init__.py`。自动测试使用固定数据、驱动替身或打桩的原生驱动,保持离线。 + +## 按改动范围定位测试 + +| 变更范围 | 测试文件 | +| --- | --- | +| 显式配置、参数映射和配置隔离 | [test_connection_config.py](test_connection_config.py) | +| 公共连接句柄和同步、异步生命周期 | [test_connections.py](test_connections.py) | +| 异常分类、异常链和清理错误 | [test_connection_errors.py](test_connection_errors.py) | +| MongoDB、MySQL 原生工厂 | [test_mongodb_client.py](test_mongodb_client.py)、[test_mysql_client.py](test_mysql_client.py) | +| 高层后端适配 | [test_managed_mongodb.py](test_managed_mongodb.py)、[test_managed_mysql.py](test_managed_mysql.py)、[test_managed_redis.py](test_managed_redis.py) | + +用 `-p` 指定文件名可以运行定向测试。例如,以下命令覆盖 Redis 单机、Sentinel 和 Cluster 的高层适配。 + +```bash +python -B -m unittest discover -s tests -p "test_managed_redis.py" -q +``` + +新增回归沿用 `unittest.TestCase` 和 `unittest.IsolatedAsyncioTestCase`。涉及后端适配时,按改动覆盖可用的同步和异步路径,并核对失败条件、清理和兼容性。驱动兼容测试在独立环境中运行,记录实际导入的版本。 + +## 隔离服务验证 + +[scripts/verify_connections.py](../scripts/verify_connections.py) 使用 `--config` 读取测试环境的 JSON 配置。JSON 的键为连接名称,值为 `ConnectionConfig` 的关键字参数。配置文件保存在仓库外,并指向隔离测试实例。 + +```bash +python -B scripts/verify_connections.py --config "<测试配置文件路径>" +``` + +脚本探测同步与异步连接,验证实例独立关闭,并对 MySQL 执行只读的 `SELECT 1`。分别记录离线回归与真实服务验证的结果,具体环境和实际结果见 [连接管理验收记录](../docs/connection-management-validation.md)。 -- Gitee From 80fd877360787309bf4a679bd94b1baa76c733ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:12:57 +0800 Subject: [PATCH 46/86] =?UTF-8?q?docs(crud):=20=E7=A1=AE=E7=AB=8B=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E6=93=8D=E4=BD=9C=E4=B8=8E=E8=BF=94=E5=9B=9E=E7=BB=93?= =?UTF-8?q?=E6=9E=9C=E8=AE=BE=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/README.md | 4 + docs/crud-design.md | 545 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 549 insertions(+) create mode 100644 docs/crud-design.md diff --git a/docs/README.md b/docs/README.md index b17bc7e..54b2511 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,10 @@ - [测试指南](../tests/README.md) 提供离线回归和隔离服务探测的运行方式。 - [连接管理验收记录](connection-management-validation.md) 记录交付时实际验证的驱动版本、真实拓扑及待验证项。 +## 待审查设计 + +- [高级 CRUD 设计提案](crud-design.md) 说明批量操作、字段投影、返回结果、Redis 兼容边界、同步与异步及原生客户端访问,状态为待审查。 + ## 目录职责 | 路径 | 职责 | diff --git a/docs/crud-design.md b/docs/crud-design.md new file mode 100644 index 0000000..f385abf --- /dev/null +++ b/docs/crud-design.md @@ -0,0 +1,545 @@ +# database 高级 CRUD 设计提案 + +日期:2026-11-15。 +状态:待用户审查。 +基线:`develop` 的已提交版本 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 +本地文档分支:`docs/database-crud-design`。 + +本文中的 CRUD 模块、方法和结果类型均为拟议接口,尚待实现与验收。 +当前已经实现的连接接口以 [连接管理设计](connection-management-design.md) 和 [项目 README](../README.md) 为准。 +用户表述中的 Redis“金融边界”在本文按前文理解为“兼容与功能边界”,同时纳入事务、原子性、金额精度与持久化约束,供审查确认。 + +## 1. 目标与适用场景 + +为同一项目同时使用 MySQL、MongoDB 和 Redis,或连接多个同类数据库的场景,提供可选的高级记录操作接口。 +开发者通过已经命名的连接绑定表、集合或 Redis Hash 命名空间,再执行单条、批量、字段投影和条件操作。 +设计优先覆盖记录操作的真实语义、批量失败的可诊断性和原生能力的可达性。 + +第一版拟覆盖以下范围。 + +| 范围 | 拟议能力 | +| --- | --- | +| 按键操作 | 单条和多条读取、插入、更新、删除 | +| 集合操作 | MySQL/MongoDB 条件查询、计数、同条件批量更新和删除 | +| 投影 | 明确选择普通顶层字段 | +| 查询规模 | 显式 limit、offset 和字段排序 | +| 批量执行 | 分块执行、后端结果保留、部分失败及结果未知的表达 | +| Redis | 调用方明确选择的 Hash 记录适配、TTL、单机/Sentinel/Cluster | +| 同步与异步 | 相同参数和结果契约,分别使用同步调用与 await | +| 原生能力 | 继续通过连接句柄的 native 访问原生客户端 | + +跨连接事务、自动索引维护、数据模型迁移、JOIN/聚合的跨库翻译、自动重试、Redis 全库扫描模拟查询,留在本期公共 CRUD 范围之外。 +这些范围选择是本提案的建议,随用户审查调整。 + +## 2. 资源绑定与原生客户端 + +拟新增 `database.crud`,提供 `Records` 和 `AsyncRecords` 两个入口。 +绑定方法接收现有 `Connection` 或 `AsyncConnection`,不创建额外主连接,不缓存全局实例,也不关闭调用方的连接。 +绑定只校验输入与后端类型,保持无网络请求。 +公共导入继续保持惰性驱动加载。 + +| 绑定方式 | 必要声明 | +| --- | --- | +| `Records.mysql(connection, table=..., key_fields=...)` | 表名与实际唯一键字段,允许复合键 | +| `Records.mongodb(connection, database=..., collection=..., key_fields=("_id",), session=None)` | 数据库、集合、实际唯一键,可绑定调用方提供的原生 session | +| `Records.redis_hash(connection, namespace=..., key_fields=..., codec=...)` | 显式命名空间、记录身份字段、字段编解码规则 | + +`AsyncRecords` 提供相同的绑定方法,返回异步记录操作对象。 +调用方声明的唯一键应当与真实数据约束一致;绑定时不隐式建索引或修改表结构。 +MySQL 和 MongoDB 保留键的原生类型,MongoDB 的 ObjectId 不自动转换为字符串。 +按键操作接收完整键映射,例如 `{"id": 1001}` 或 `{"tenant_id": 7, "id": 1001}`,不假设所有数据库都存在单列 id。 +查询若发现声明的唯一键对应多条记录,应报数据约束错误,避免静默选择一条。 + +`connection.native` 保持真实原生对象身份,原有 `database..Client` 工厂契约继续适用。 +MySQL 的原生 SQL、事务、游标,MongoDB 的聚合、更新操作符、session,以及 Redis 的 List、Set、Stream、Pipeline 等能力继续可用。 +调用方自行创建的原生对象由调用方管理,CRUD 层负责自己创建的游标、Pipeline 和临时数据客户端。 +连接关闭后,记录操作通过原连接句柄检查并抛出 `ConnectionClosedError`。 + +## 3. 操作集合与后端支持矩阵 + +以下方法名为本提案的统一命名。 + +| 方法 | 意图 | 返回契约 | +| --- | --- | --- | +| `get(key, *, fields=None)` | 按唯一键读取一条 | 记录字典或 None | +| `get_many(keys, *, fields=None, batch_size=500)` | 按输入键批量读取 | 与输入位置对齐的 ReadItem 列表 | +| `find(*, where=None, fields=None, limit, offset=0, order_by=())` | 按条件查询有限条记录 | 记录字典列表 | +| `count(*, where=None)` | 按条件计数 | 非负整数 | +| `insert(record, ...)` | 插入新记录 | WriteResult | +| `insert_many(records, *, batch_size=500, ...)` | 批量插入 | BatchResult | +| `update(key, *, set_values, ...)` | 仅更新指定记录的部分字段 | WriteResult | +| `update_many(*, where, set_values, all_records=False)` | 相同条件、相同修改,作用于全部匹配记录 | WriteResult | +| `bulk_update(items, *, batch_size=500, ...)` | 多个唯一键分别执行不同修改 | BatchResult | +| `delete(key)` | 按键删除 | WriteResult | +| `delete_many(keys, *, batch_size=500)` | 按已知键列表删除 | BatchResult | +| `delete_where(*, where, all_records=False)` | 删除全部条件匹配记录 | WriteResult | + +`insert` 使用仅创建语义,已有身份对应记录时失败。 +`update` 使用仅更新语义,记录不存在时返回未匹配结果,默认不创建记录。 +`set_values` 只修改列出的字段,记录键不可通过该参数修改。 +记录替换、upsert、字段删除、原子自增和混合类型 bulk_write 使用原生接口,后续按明确场景增加公共操作。 + +| 能力 | MySQL | MongoDB | Redis Hash 记录适配 | +| --- | --- | --- | --- | +| 按键单条/批量读取和投影 | 支持 | 支持 | 支持 | +| 按键单条/批量插入、修改、删除 | 支持 | 支持 | 支持,遵守记录格式和单键执行约束 | +| where 条件查询、排序、计数 | 支持约定的公共条件 | 支持约定的公共条件 | 第一版不提供 | +| where 条件 update_many/delete_where | 支持 | 支持 | 第一版不提供 | +| 自动生成身份 | 依赖表定义 | 默认可由驱动生成 _id | 必须显式提供身份 | +| 多记录事务 | 遵循原生连接和表引擎能力 | 使用调用方提供的原生 session | 遵循 Redis 原生事务能力,不能等同 SQL 回滚 | + +调用当前后端不支持的方法时,在发送操作前抛出拟议的 `UnsupportedOperationError`,包含后端和操作名称。 +不把失败返回空列表或零条记录,也不通过隐式扫描或创建索引补足缺失能力。 + +## 4. 公共条件与查询边界 + +第一版普通字典 `where={"status": "pending", "tenant_id": 7}` 表达顶层标量字段的 AND 等值条件。 +扩展比较拟采用明确条件对象 `Eq`、`In`、`Gt`、`Gte`、`Lt`、`Lte`、`And`、`Or`、`IsNull` 和 `Exists`,只提供已定义语义且通过后端测试的操作符。 +条件对象是数据结构,不接收任意 Python 回调或拼接 SQL 字符串。 +MongoDB 原生操作符字典不能直接放入公共 where 冒充统一表达式。 + +| 情况 | 公共契约 | +| --- | --- | +| `Eq(field, None)` 或字典值 None | 采用显式空值语义,等价于 IsNull(field) | +| MySQL IsNull | SQL IS NULL | +| MongoDB IsNull | 字段存在且为 BSON null,避免同时匹配缺失字段 | +| Exists | MongoDB 表达文档字段存在性,MySQL 不提供行级“缺列”语义,报能力错误 | +| In 的空列表 | 恒不匹配,写入影响数可确定为零 | +| 普通标量比较 | 按实际后端类型和排序规则执行,MongoDB 不将数组元素匹配冒充标量相等 | +| 不合法字段、未支持的值类型、运算符冲突 | 发送前报告配置或能力错误 | + +MySQL 值使用驱动参数绑定;表名、字段名、排序方向使用独立校验和标识符引用规则。 +MongoDB 公共标量条件需处理 BSON 类型与数组匹配差异,不能直接透传用户结构形成操作符注入。 +字符串大小写、排序规则、日期时区和数值范围继续取决于后端及字段定义,公共 API 不承诺跨后端查询结果自动等价。 +高级 SQL 表达式、MongoDB 嵌套路径和数组条件由原生接口表达。 + +`find` 要求显式正整数 limit,offset 为非负整数。 +省略 order_by 时返回顺序由数据库决定。 +使用分页时建议调用方给出包含唯一键的完整排序,翻页期间的数据变更仍按后端一致性语义处理。 +一次 find 在返回前取完受 limit 限制的结果并关闭内部游标,不将待消费游标混入列表返回类型。 +第一版不引入跨后端流式游标或可互换的分页 token。 + +## 5. 字段投影与读取结果 + +### 5.1 fields 的语义 + +`fields=None` 表示返回全部业务字段。 +`fields=["name", "email"]` 表示只返回这些顶层字段。 +`fields=[]` 表示只确认记录是否存在,存在记录返回空字典,缺失记录仍返回 None。 +重复字段名、字段表达式和不合法名称在发送前报错。 +公共 fields 第一版采用包含模式,不混用排除模式、SQL 别名或计算表达式。 + +| 后端 | 获取 name/email 的执行方式 | 必须处理的差异 | +| --- | --- | --- | +| MySQL | SELECT 对应列,使用字典结果 | 列名引用与值绑定分离,列不存在时保留驱动异常 | +| MongoDB | 包含投影并按需设置 _id: 0 | 仅在请求 _id 时返回它,避免驱动默认附带字段 | +| Redis Hash | 对指定键读取 HMGET 对应字段 | 业务值解码,内部身份/版本字段不进入记录 | + +投影尽量在服务端执行。 +`fields=[]` 可查询常量、身份或内部标记以确认存在性,但这些内部读取值不进入返回记录。 +Redis String 中存储 JSON 的场景属于另一种数据格式;不将下载完整 JSON 后裁剪伪装成 Hash 的字段读取。 +MongoDB 的包含投影及 _id 特例依据官方接口处理。[MongoDB 字段投影](https://www.mongodb.com/docs/languages/python/pymongo-driver/current/crud/query/project/) + +### 5.2 空值、缺失字段和缺失记录 + +| 情况 | 读取结果 | +| --- | --- | +| 记录不存在 | None | +| 记录存在,字段值是数据库空值 | 字典中该字段为 None | +| MongoDB 文档或 Redis Hash 记录缺失所选字段 | 返回字典省略该字段 | +| 记录存在,但所选字段均缺失 | 空字典 | +| MySQL 选择了表中不存在的列 | 保留原始驱动异常的统一错误 | + +MySQL 已有列的 NULL 与 MongoDB/Redis 的缺失字段保持上述区别,不用 None 填平所有情况。 +日期、Decimal、ObjectId、bytes 等值保留驱动或显式 codec 解码得到的类型,避免强制 JSON 化导致精度或类型丢失。 +Redis 的 HMGET 对“键不存在”和“字段不存在”均可能给出 nil;单凭一组 nil 不能判断记录不存在。 +Redis 适配使用同一次单键原子执行中的记录标记检查和字段读取来区分两者。[Redis HMGET](https://redis.io/docs/latest/commands/hmget//) + +例如请求 fields=["name", "email"] 时,以下结果反映了数据实际差别。 + +| 后端及数据 | get 返回值 | +| --- | --- | +| MySQL 行 name 为 Alice,email 为 SQL NULL | `{"name": "Alice", "email": None}` | +| MongoDB 文档 name 为 Alice,没有 email 字段 | `{"name": "Alice"}` | +| Redis Hash 记录 name 为 Alice,email 编码保存为 null | `{"name": "Alice", "email": None}` | +| 任一后端的目标记录不存在 | `None` | +| 任一后端记录存在,请求 fields=[] | `{}` | + +### 5.3 批量读取 + +`get_many` 输入为有限键序列,结果与输入顺序和数量一致,重复的输入键保留对应结果位置。 +同一批内重复键只读取一次并复用该次观察,避免一次调用中同一个键出现两个矛盾结果。 +后端查询返回乱序时由适配器按输入键重排,额外查询的身份字段不会泄露到投影结果中。 + +```python +# 拟议返回示例,ReadItem 的三个字段为 index、key、record。 +[ + ReadItem(index=0, key={"id": 1001}, record={"name": "Alice"}), + ReadItem(index=1, key={"id": 1002}, record=None), + ReadItem(index=2, key={"id": 1001}, record={"name": "Alice"}), +] +``` + +空输入返回空列表且无网络请求。 +整个批次不承诺单一时间点快照;MySQL/MongoDB 需要更强一致性时使用原生事务或 session。 +批量读取中途失败应抛出带已知部分读取结果的批量异常,尚未读取或无法确认的项标为未知,不能填写 record=None 冒充记录缺失。 +只有完整成功返回时,每个 ReadItem.record 才能使用上述 None/字典约定。 + +## 6. 批量写入 + +### 6.1 相同条件、相同修改 + +```python +result = orders.update_many( + where={"status": "pending"}, + set_values={"status": "closed"}, +) +``` + +MySQL 生成一次参数化 `UPDATE ... SET ... WHERE ...`。 +MongoDB 调用集合的 `update_many(filter, {"$set": values})`。 +Redis Hash 适配第一版对这种集合条件操作报告能力错误。 +条件更新直接在服务端求值,避免先查出全部身份再更新造成语义变化。 +update_many/delete_where 默认拒绝空条件,全量操作需调用方显式传入 `all_records=True`;这是提议的防误用约定,待本次审查确认。 +单次集合操作不会为了套用 batch_size 而擅自拆成多个查询。 + +### 6.2 每条记录分别修改 + +```python +result = orders.bulk_update([ + {"key": {"id": 1001}, "set_values": {"status": "paid"}}, + {"key": {"id": 1002}, "set_values": {"status": "cancelled"}}, +], batch_size=500) +``` + +| 后端 | 第一版执行策略 | 结果限制 | +| --- | --- | --- | +| MySQL | 按连续相同更新字段形状分组,以参数化 executemany 执行 | 通常只能获得组级 rowcount,失败时驱动可能不公开确切执行位置 | +| MongoDB | 集合级 bulk_write,使用 UpdateOne,默认 ordered=True | 保存组级统计、错误索引和已报告的执行范围,普通汇总未必含逐项匹配数 | +| Redis Hash | 单键条件更新命令/脚本组成 pipeline,显式 transaction=False | 成功读取回复时可获得逐项结果,传输中断后有无法确认的项 | + +Connector/Python 的 executemany 通常逐组调用 execute,针对 INSERT 有多行优化,因此一次方法调用不等于只发送一条 SQL。[MySQL executemany](https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-executemany.html) +MongoDB 使用集合级 bulk_write,避免为本期功能额外要求 MongoClient 跨命名空间批量接口及其服务器版本。[PyMongo bulk_write](https://www.mongodb.com/docs/languages/python/pymongo-driver/current/crud/bulk-write/) + +批量写入接收有限序列,在首次发送前完成可由 SDK 判断的结构、键、字段和编码校验。 +同一调用中重复的已知记录键默认报配置错误,避免分组、Pipeline 或不同后端执行策略改变同一记录上的依赖顺序。 +依赖前一次结果的操作由调用方明确顺序执行。 +分块按输入顺序提交,发生异常后停止发送后续分块;已经发送的分块可能继续执行,尤其是 Pipeline 中已经排队的命令。 +本期不提供跨后端统一的“发现第一条错误后其余项保证未执行”承诺。 +batch_size 控制传给原生批量接口的分块规模,后端仍可能按消息大小再次拆分。 + +### 6.3 插入与删除 + +MySQL 批量插入可采用 executemany 的多行优化,MongoDB 使用 insert_many 或集合批量写入,Redis 使用单键仅创建操作组成 Pipeline。 +所有目标后端都将重复身份视为插入冲突,保留原始错误;Redis 需要单键原子存在性判断以避免覆盖已有键。 +批量删除按已知键执行,MongoDB/MySQL 可以使用集合条件组合,但不得把独立复合键展开成各字段的笛卡尔积。 +Redis Cluster 批量删除采用单键命令分发,避免将任意跨槽键列表拼进一个 DEL。 +删除不存在的记录是成功的零删除结果。 +空批次为成功的空结果且不发起网络请求。 +服务端约束和数据竞态仍可能在发送后失败,发送前校验不保证整个批次成功。 + +## 7. 写入返回结果、投影与失败 + +### 7.1 写入结果不隐含回读 + +fields 只用于 get/get_many/find 等读取方法。 +insert/update/delete 及其批量方法第一版返回执行信息,不支持统一的 `returning=fields`。 +这是因为三种后端对“修改并返回前值/后值”的能力和原子性不同。 +调用方可以写入后显式 get/get_many 并投影,但两次操作之间可能有其他写入,回读结果不能冒充本次操作原子返回的后值。 +MongoDB find_one_and_update 等专有原子返回能力通过原生对象使用。 +如后续增加公共 returning,需要单独评审各后端保证。 + +### 7.2 WriteResult + +| 字段 | 含义 | +| --- | --- | +| `backend`、`operation` | 执行后端和操作名称 | +| `acknowledged` | 是否收到可解释的执行确认,无法确定为 None | +| `matched_count` | 后端可确定的匹配记录数,否则 None | +| `modified_count` | 后端可确定的实际修改记录数,否则 None | +| `inserted_count`、`deleted_count` | 后端可确定的插入/删除记录数,否则 None | +| `generated_keys` | 后端明确返回的新身份,缺少完整映射时为 None | +| `native_result` | 原生返回对象,或游标关闭前保存的明确原生统计快照 | + +各计数字段按操作和后端填写,None 表示不适用或无法确定,0 必须有明确依据。 +MySQL UPDATE 的 rowcount 保存在 native_result 中,并附带 changed/matched 模式说明;只有驱动配置和语句语义足以确定时才填入对应公共计数,不通过额外 SELECT 猜测匹配数。 +MongoDB 保留 matched_count 与 modified_count 的区别;w=0 等未确认写入的确认标志为 False,无法获得的计数保持 None。[MySQL rowcount](https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-rowcount.html)、[PyMongo 写入结果](https://pymongo.readthedocs.io/en/stable/api/pymongo/results.html) + +Redis HSET 的整数回复是新增字段数,不能直接赋给 modified_count 作为“修改记录数”。 +记录适配的单键脚本可以明确返回是否匹配已有记录或是否成功创建,未做实际值比较时 modified_count 保持 None。[Redis HSET](https://redis.io/docs/latest/commands/hset/) +MySQL 批量插入只有 lastrowid 时,不推算所有自增身份;触发器、配置和并发等因素都可能使连续推算失真。 +生成或分配的身份不代表记录已经成功写入,尤其是未确认写入和随后失败的批量操作。 +原生返回对象或快照可包含敏感数据,日志处理沿用 [安全说明](../SECURITY.md)。 + +### 7.3 BatchResult 与失败状态 + +BatchResult 保存输入项数量、按输入索引排列的项状态、原生分块结果和能够明确汇总的计数。 +拟议字段为 requested_count、items、counts 和 native_results;每个项包含 index、key、state、result 和驱动提供的 error 信息。 +写入成功项的 result 可为 WriteResult,读取成功项的 result 为 ReadItem,无法获得的明细为 None。 +批量读取异常也使用此结构保存 partial_result,只有状态已确认的 ReadItem.record 才能解释为缺失记录或实际字典。 +分块统计覆盖重叠或明细不足时,公共汇总保持 None,不能将“发送项数”当作“成功修改记录数”。 +成功项可能没有逐项匹配或修改数,这与组级执行已确认并不冲突。 + +| 项状态 | 确定含义 | +| --- | --- | +| `acknowledged` | 所属操作执行已获确认,结果可能是零匹配或零删除 | +| `unacknowledged` | 后端配置有意省略执行确认,不能判为已成功修改 | +| `failed` | 收到该项明确失败回复,保存输入索引和原始异常信息 | +| `not_sent` | SDK 或驱动可靠确认该项未发送 | +| `not_executed` | 已发送但后端明确报告因前项失败等原因未执行 | +| `unknown` | 已可能发送或执行,但证据不足以确定结果 | + +批量调用出现实际错误时抛出拟议 `BatchOperationError`,其 partial_result 保存已知结果。 +original 保留驱动交付的最外层异常对象,cause 保留异常链,驱动提供的错误明细对象也按原样保存,不重新拼接出不存在的逐项异常。 +MySQL executemany 失败而无法确定组内执行位置时,该组相应项可为 unknown,后续未发送分组为 not_sent。 +已确认的 MongoDB 错误索引映射回整个输入序列;有 ordered 停止信息时区分后续未执行项。 +Redis 收到完整 Pipeline 回复时保留每条命令错误及成功回复,连接中断时只填写可以确定的结果。 + +执行确认与事务最终提交分别判断。 +MySQL 操作确认后可能仍处于未提交事务,MongoDB session 内操作可能随后被回滚,记录适配不把 WriteResult/BatchResult 的确认当成提交凭证。 +单个命令也可能因原生表引擎、脚本或事务执行语义出现部分影响;失败状态不是“该项绝对没有产生任何效果”的承诺。 +SDK 不根据批量状态自动重试写入。 + +以三个按键更新为例,第一条记录的新值与旧值相同,第二条发生变化,第三条不存在;全部请求正常返回时,统计可以出现以下差异。 + +| 后端 | matched_count | modified_count | 解释 | +| --- | --- | --- | --- | +| MySQL,已明确确认 rowcount 为 changed 模式 | None | 1 | 保存原生 rowcount=1,不补查询以猜测匹配数 | +| MongoDB,已确认写入 | 2 | 1 | 原生结果分别提供匹配数和修改数 | +| Redis Hash 记录脚本 | 2 | None | 脚本确认两条记录存在,未逐值比较时不报告实际改变记录数 | + +三项的执行状态均可为 acknowledged,但第三项没有匹配记录;批次执行成功和实际修改数量分别表达。 + +## 8. Redis 的兼容、格式和功能边界 + +### 8.1 Hash 记录格式 + +调用方显式启用 redis_hash 适配后,一个记录对应一个 Redis Hash 键。 +namespace 定义独立命名空间;键身份采用确定性、带类型的组合编码,避免字符串拼接造成复合键碰撞。 +默认身份编码仅支持明确约定的字符串与整数,其他类型通过显式编码器支持,不自动将任意对象转成字符串。 + +Hash 保存业务字段和一个保留的格式版本标记,例如 `__database_record__=1`。 +该标记用于记录存在性、空记录和存储格式识别,业务字段名不能占用保留名称,读取结果隐藏内部字段。 +这一存储约定需要用户审查,它会影响使用同一命名空间的其他程序。 +遇到同名的 String/List/其他类型键,或缺少正确格式标记的 Hash,报告数据格式或原生 WRONGTYPE 错误,不自动覆盖或迁移已有数据。 +现有任意 Redis 数据仍通过原生客户端使用;若需要将已有 Hash 纳入记录接口,应先明确格式、codec 和迁移方案。 + +字段 codec 负责逐字段编码与解码。 +拟提供 JSON 字段 codec 处理字符串、整数、布尔值、有限浮点数、null 及 JSON 容器,null 以实际编码值保存,以区分字段不存在。 +Decimal、bytes、datetime、ObjectId 等类型需要显式支持的 codec,遇到不支持的类型在发送前报错,不能隐式降精度或转换为随意字符串。 +字段 JSON 容器可以作为整体字段保存;其内部路径查询和局部修改不属于本期公共能力。 +codec 必须与原连接的编码和 decode_responses 设置兼容,二进制 codec 需要能够取得原始字节;不在绑定时偷偷修改原客户端配置。 + +### 8.2 原子存在性与 TTL + +仅创建、仅更新、记录格式检查和投影读取使用单键 Lua 脚本或经验证等价的原生原子命令组合。 +存在性检查与写入之间不留客户端竞争窗口。 +脚本先进行参数和键类型校验,再执行写入;不把 Lua 自动理解为具有错误回滚能力的事务。 +部署必须允许所需脚本命令,权限不足时给出明确错误;不能静默降级成非原子的先查再写。 + +| 动作 | 建议的 TTL 约定 | +| --- | --- | +| 新建记录未指定 ttl_s | 无自动过期 | +| 新建记录指定正整数 ttl_s | 创建和设置过期在同一次单键原子操作内完成 | +| 更新时省略 ttl_s | 保留现有键 TTL | +| 更新时 ttl_s 为正整数 | 从更新时重新设置 TTL | +| 更新时显式 ttl_s=None | 清除键 TTL,变为持久键 | +| 更新时键已过期或不存在 | 返回未匹配,不复活记录 | + +省略与显式 None 使用内部“未提供”标记区分。 +TTL 为零、负数或布尔值时报告配置错误,避免将非法值解释为删除或永久保存。 +批量 Redis 插入和修改项可分别携带 ttl_s。 +对 MySQL/MongoDB 传入 TTL 选项报告能力错误,不能翻译成临时任务或自动 TTL 索引。 +记录级 TTL 不代表业务字段级过期;本期格式不启用 Hash 字段独立过期。 + +### 8.3 单机、Sentinel 和 Cluster + +| 拓扑 | 数据操作与资源归属 | +| --- | --- | +| 单机 | 复用 connection.native,CRUD 层不关闭它或外部连接池 | +| Sentinel | connection.native 仍为 Sentinel;记录上下文通过已配置 service_name 创建并复用自有主节点数据客户端,退出时关闭其自有资源 | +| Cluster | 使用原生集群客户端路由单键命令和脚本,批量跨键分发遵循驱动能力 | + +Redis 记录对象使用 with/async with 管理其自有临时资源;进入上下文可构造临时客户端,但不额外探测服务。 +所有后端的记录对象均支持同样的上下文用法,MySQL/MongoDB 退出记录上下文时仅清理其自有操作资源。 +第一版约定记录对象在有效上下文内执行操作,退出后拒绝继续使用,调用方可基于仍然打开的连接重新绑定对象。 +记录对象借用的主连接应覆盖整个记录上下文生命周期。 + +跨槽 Pipeline 可以分发多个独立单键操作,但不能提供跨槽整体事务、统一回滚或多键原子性。 +多键事务和脚本受 Redis Cluster 同槽约束,namespace 不会自动强制所有记录落在同一槽。[Redis Cluster](https://redis.io/docs/latest/management/scaling/) +Sentinel 故障切换和 Cluster 重定向遵循驱动策略;对“命令是否已生效”的不确定性按批量结果契约报告。 +原生驱动配置的重试行为需纳入验证,SDK 不额外叠加重试循环。 + +### 8.4 查询、事务及精度 + +普通 Hash 记录接口提供按键能力;任意字段搜索、范围查询、排序、聚合和跨记录条件更新,需要额外索引与查询能力。 +本期不自动维护索引集合或接入 Search,也不通过 KEYS/SCAN 将整个库拉回本地筛选。 +以后若支持索引查询,应单独说明部署要求、索引一致性、写入成本和失败恢复。 + +Redis Pipeline 优化往返开销,Redis MULTI/EXEC 具有自己的执行语义,但执行期错误不提供 SQL 式回滚。[Redis Pipeline](https://redis.io/docs/latest/develop/using-commands/pipelining/)、[Redis 事务](https://redis.io/docs/latest/develop/using-commands/transactions/) +跨 MySQL/MongoDB/Redis 的原子提交需要应用层协议,本期 CRUD 不能保证三个数据库一起成功或一起撤销。 + +如涉及金额,业务可选择明确币种与单位的整数最小单位,或配置支持 Decimal 的精确 codec。 +SDK 不默认把 Decimal 转成 float,也不自动将金额更新映射为 Redis INCRBYFLOAT。 +Redis INCRBYFLOAT 使用浮点运算,数值存储精度、算术原子性与账务事务必须分别评估。[Redis INCRBYFLOAT](https://redis.io/docs/latest/commands/incrbyfloat/) +本期 set_values 表达字段赋值,不自动实现“读取余额后加减再写回”的原子账务操作。 +持久化、复制确认、故障切换期间的数据保证由实际数据库部署及配置决定,CRUD 执行确认不充当业务持久化或账务一致性保证。 + +## 9. 同步与异步契约 + +| 项目 | 同步 | 异步 | +| --- | --- | --- | +| 创建连接 | connections.create(name) | await connections.create_async(name) | +| 绑定记录对象 | Records.mysql/mongodb/redis_hash | AsyncRecords.mysql/mongodb/redis_hash,绑定本身为普通方法 | +| 记录上下文 | with | async with | +| 单条、查询、批量方法 | 普通调用 | await,对应参数与结果含义相同 | +| 多数据库同时使用 | ExitStack 管理多个命名连接与记录对象 | AsyncExitStack 管理多个异步连接与记录对象 | +| 原生异常转换范围 | 普通 with connection.errors(...) | 同样使用普通 with,里面可以 await | + +异步实现调用原生异步驱动,不使用同步数据库调用阻塞事件循环。 +异步记录对象在所属连接的事件循环内使用,所有高层 CRUD 操作都检查连接状态与循环归属。 +批量执行默认顺序提交分块,异步不意味着把同一 MySQL 会话上的操作自动并发执行。 +同步与异步接受相同的输入范围,具有相同的字段、计数、错误状态和 TTL 契约。 +同步线程安全和异步并发能力仍遵循各原生客户端约束。 + +取消信号保持原始 CancelledError,不转换为普通数据库失败。 +若取消时批量操作可能已发送,给原取消对象附加可获取的 `database_partial_result`,保留已知结果与未知范围。 +清理错误沿用 `database_cleanup_errors`,不得覆盖主异常或取消。 +取消或断开不意味着数据库回滚,不通过自动重试推断本次写入是否成功。 + +## 10. 拟议用法 + +以下示例演示审查中的 API 形状,`connections` 是调用方用显式参数创建的现有 Connections 集合。 +示例中的连接名分别绑定业务用途,不形成按数据库类型划分的全局对象。 + +```python +from database.crud import Records + +with connections.create("orders") as connection: + with Records.mysql(connection, table="orders", key_fields=("id",)) as orders: + rows = orders.find( + where={"status": "pending"}, + fields=["id", "status"], + limit=100, + order_by=[("id", "asc")], + ) + result = orders.bulk_update([ + {"key": {"id": 1001}, "set_values": {"status": "paid"}}, + {"key": {"id": 1002}, "set_values": {"status": "cancelled"}}, + ]) + # 应用在自己的事务边界决定提交或回滚。 + with connection.errors(operation="commit_orders"): + connection.native.commit() +``` + +```python +from database.crud import AsyncRecords + +async def read_and_update_orders(connections): + async with await connections.create_async("orders") as connection: + async with AsyncRecords.mysql( + connection, table="orders", key_fields=("id",) + ) as orders: + items = await orders.get_many( + [{"id": 1001}, {"id": 1002}], fields=["status"] + ) + result = await orders.bulk_update([ + {"key": {"id": 1001}, "set_values": {"status": "paid"}}, + {"key": {"id": 1002}, "set_values": {"status": "cancelled"}}, + ]) + with connection.errors(operation="commit_orders"): + await connection.native.commit() + return items, result +``` + +MongoDB 绑定后同样支持单条/批量/条件接口,原生 session 由调用方传入并管理。 + +```python +with connections.create("documents") as connection: + with Records.mongodb( + connection, database="app", collection="orders", key_fields=("_id",) + ) as documents: + result = documents.update_many( + where={"status": "pending"}, set_values={"status": "closed"} + ) + with connection.errors(operation="aggregate_orders"): + # 示例展示原生能力仍可达,完整迭代和关闭位于异常转换范围内。 + with connection.native["app"]["orders"].aggregate([ + {"$group": {"_id": "$status", "count": {"$sum": 1}}} + ]) as cursor: + summary = list(cursor) +``` + +```python +from database.crud import JsonFieldCodec + +with connections.create("cache") as connection: + with Records.redis_hash( + connection, namespace="order", key_fields=("id",), codec=JsonFieldCodec() + ) as cache: + created = cache.insert({"id": 1001, "status": "pending"}, ttl_s=60) + item = cache.get({"id": 1001}, fields=["status"]) + updated = cache.update({"id": 1001}, set_values={"status": "paid"}) + # 该次更新省略 ttl_s,保留已有过期时间。 +``` + +MongoDB 和 Redis 异步用法分别采用 AsyncRecords.mongodb/redis_hash、async with 和 await,返回结构与同步一致。 +原生异步 MongoDB/Redis 工厂是否 await 的既有差异,继续由连接层按现有契约处理。 +高级 CRUD 的异步方法始终通过 await 调用。 + +## 11. 实施阶段与验收计划 + +用户审查通过本文后,再确定实施分支、模块文件和开发排期。 +建议从届时本地已提交 develop 建立独立集成分支,按 [协作规范](../CONTRIBUTING.md) 安排公共契约、MySQL、MongoDB、Redis 和验收工作。 +公共结果与条件语义先形成检查点,后端模块再从同一检查点开展并行实现,最后串行合并。 +分支命名使用 feature/fix/refactor/docs 等项目规范前缀。 +是否提交、合并和推送遵循实施时用户指令;目前 Git 操作限定本地。 + +| 阶段 | 交付与通过条件 | +| --- | --- | +| 契约 | 资源绑定、方法签名、fields/空值/键/结果状态均有明确测试场景 | +| MySQL | 单条与批量、参数绑定、字段引用、复合键、结果消费、原生事务和游标关闭 | +| MongoDB | 原生键类型、投影、空值与缺失、集合更新、批量错误索引、session、未确认写入 | +| Redis | 明确 Hash 格式与 codec、原子存在性、TTL、部分 Pipeline 失败、三拓扑资源管理 | +| 集成 | 同进程三种数据库及同类多实例并用,同步/异步契约一致,原生访问与原工厂回归 | + +必须覆盖以下验收场景。 + +- 相同条件更新多条记录,每条记录不同值的批量更新,批量插入和删除,复合键及重复输入。 +- fields=None、空列表、部分字段、身份字段、缺失字段、NULL/null、未知 SQL 列,原生类型保真。 +- get_many 输入顺序、重复键、缺失记录、后端乱序、跨分块部分读取失败。 +- MongoDB 相同值更新 matched_count 与 modified_count 不同,MySQL rowcount 模式差异,Redis 新增字段数不得误报为修改记录数。 +- 批量前置校验失败时无请求,服务端约束失败、未知执行位置、未发送项、发送后断线、取消后的不确定状态。 +- MySQL 回滚、显式提交和自动提交配置,MongoDB session 事务行为,Redis 原生事务及 Cluster 限制。 +- Redis create-only/update-only 的并发竞争、Hash 类型冲突、无标记旧数据、全字段缺失、codec 错误、TTL 保留/重设/清除和过期竞态。 +- Sentinel 主节点数据路径、临时客户端清理,Cluster 跨槽独立批量和不支持的多键原子要求。 +- 同步与异步逐项对照,事件循环归属,异常原对象、因果链、取消、主异常与清理失败。 +- 多个连接独立关闭,原生客户端对象身份、方法、事务和返回值继续可用。 + +离线测试沿用 unittest 和 IsolatedAsyncioTestCase;对应模块执行 Ruff lint 与格式检查,公共契约改动执行完整回归。 +真实实例验证使用隔离 MySQL、MongoDB、Redis 单机/Sentinel/Cluster,记录客户端与服务端版本,并由外部测试进程控制故障用例时限。 +覆盖项目最低驱动版本和选定当前版本,MongoDB 异步要求 PyMongo >=4.9。 +当前基线的默认安装已包含三种驱动;CRUD 提案不以旧版 extras 安装方案为前提。 +之前的连接管理测试结果不代表这些新 CRUD 行为已验证。 +实现后再更新 README、测试指南和 CHANGELOG 未发布部分,按实际完成范围描述支持能力。 + +## 12. 审查重点与本次文档交付 + +重点审查按键操作与条件操作的边界、Redis Hash 格式及保留标记、TTL 省略/None 语义、fields 的空值规则、批量确认和事务提交的区别。 +同样需要确认 MySQL/MongoDB 公共条件的支持范围,以及第一版写入不包含原子投影回读的范围选择。 +以上均是可调整的设计决策,审核通过后才进入实现。 + +本次由协调者负责两份文档,文档本身和 docs 索引。 +Git 操作在本地依次完成,主目录为 `D:\996-work\loongspears\database`,文档 worktree 为 `D:\996-work\loongspears\database-worktrees\crud-design`。 + +| 顺序 | 执行者与位置 | 命令或操作 | 结果 | +| --- | --- | --- | --- | +| 1 | 协调者,主目录 develop | git status --short --branch;git diff --cached --stat;git diff --stat | 开始时无待提交改动 | +| 2 | 协调者,主目录 | git worktree list --porcelain;git rev-parse HEAD develop | 核对既有 15 个 worktree,基线为 b6707125620d1cea28cabbc7a72e72c84ae73bbd | +| 3 | 协调者,主目录 | git worktree add -b docs/database-crud-design D:\996-work\loongspears\database-worktrees\crud-design b6707125620d1cea28cabbc7a72e72c84ae73bbd | 创建独立文档分支与第 16 个 worktree | +| 4 | 协调者,文档 worktree | 写入 docs/crud-design.md 并补充 docs/README.md 索引 | 交付工作区文档差异,待审查 | +| 5 | 协调者,文档 worktree | python -B -(内联文档检查);git diff --check;git status --short | UTF-8、相对链接、代码围栏、Python 示例语法及空白检查通过 | + +文档验证已检查 UTF-8、相对链接目标、代码块语法和 Git 空白差异;检查脚本对新增未跟踪文档也进行了逐行空白核对。 +Python 示例只做语法核对,因为拟议模块尚未实现;真实驱动行为及新接口测试均留在实施验收阶段。 +本次文档尚未暂存或提交,交付状态以工作区差异为准。 -- Gitee From da0f2d36b368002d3497d9bc763002b5654c418b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:17:40 +0800 Subject: [PATCH 47/86] =?UTF-8?q?feat(crud):=20=E5=AE=9A=E4=B9=89=E5=85=AC?= =?UTF-8?q?=E5=85=B1=E7=BB=93=E6=9E=9C=E7=B1=BB=E5=9E=8B=E4=B8=8E=E9=80=82?= =?UTF-8?q?=E9=85=8D=E5=99=A8=E5=8D=8F=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/crud-adapter-contract.md | 51 +++++++++++ src/database/crud/_conditions.py | 141 +++++++++++++++++++++++++++++++ src/database/crud/_errors.py | 28 ++++++ src/database/crud/_types.py | 98 +++++++++++++++++++++ tests/test_crud_conditions.py | 42 +++++++++ 5 files changed, 360 insertions(+) create mode 100644 docs/crud-adapter-contract.md create mode 100644 src/database/crud/_conditions.py create mode 100644 src/database/crud/_errors.py create mode 100644 src/database/crud/_types.py create mode 100644 tests/test_crud_conditions.py diff --git a/docs/crud-adapter-contract.md b/docs/crud-adapter-contract.md new file mode 100644 index 0000000..2e60329 --- /dev/null +++ b/docs/crud-adapter-contract.md @@ -0,0 +1,51 @@ +# CRUD 适配器实现约定 + +设计语义见 [高级 CRUD 设计](crud-design.md)。公共模块负责校验、资源上下文、分块、错误转换和结果汇总。适配器负责原生驱动操作、后端特有校验和可证实的结果映射。 + +## 工厂和生命周期 + +- 模块 `database.crud._mysql`、`_mongodb`、`_redis_hash` 导出同步 `Adapter` 和异步 `AsyncAdapter`。 +- 构造函数为关键字参数 `connection`、`key_fields`(有序字段元组)及后端参数。MySQL 接收 `table`。MongoDB 接收 `database`、`collection`、`session`。Redis 接收 `namespace`、`codec`。构造阶段只校验配置。 +- 同步类提供 `__enter__()`、`__exit__(exc_type, exc, traceback)`,异步类提供对应异步上下文方法。可返回自身,不能抑制主异常。释放适配器自行持有的资源,借用连接、客户端和 session 归调用方。 +- 入口上下文可构造 Sentinel 主节点数据客户端,不额外探测。Redis 适配模块暂用内部连接适配器配置 `connection._adapter._config.service_name`,该桥接不构成公共接口。 +- 公共层负责连接状态和异步事件循环归属,在每次操作以及进入上下文前检查。适配器不新增自动重试、提交或回滚。 + +## 方法签名 + +| 方法 | 参数及返回值 | +| --- | --- | +| `get(key, fields)` | `dict` 或 `None` | +| `get_many(keys, fields)` | 按传入键位置严格对齐的 `dict\|None` 序列 | +| `find(where, fields, limit, offset, order_by)` | 受 limit 限制的字典列表 | +| `count(where)` | 整数 | +| `insert(item)`、`update(item)`、`delete(item)` | `WriteInput -> WriteResult` | +| `insert_many(items)`、`bulk_update(items)`、`delete_many(items)` | 每次一个分块,返回 `ChunkResult` | +| `update_many(where, set_values)`、`delete_where(where)` | `WriteResult` | + +同步和异步类的操作同名,异步操作使用原生驱动协程。输入键为按声明字段顺序整理的字典。复合键必须整组匹配。 +公共层会消除 `get_many` 中完全相同的键,保持其首次出现顺序。适配器按块返回一项对应一个传入键,并区分未命中与空投影。适配器负责数据库排序规则造成的键比较差异和多匹配检测,公共层不依赖查询结果的自然顺序。 +Redis 不需要实现 `find/count/update_many/delete_where`,公共层会拒绝这些调用。 + +`fields` 为字段元组或 `None`。空元组要求已有记录返回空字典。查询内部携带的键和版本字段不外泄。投影中 BSON/Hash 缺失字段从结果字典省略。 +`where` 为 `_conditions.Condition | None`,普通字典已规范化成等值条件。非条件写入仅当公共层已审核 `all_records=True` 时收到 `None`。排序为 `(field, direction)` 元组序列,方向 `asc` 或 `desc`。 +`Eq(field, None)` 规范化为 `IsNull(field)`。`In` 使用 `values` 元组,空元组明确零匹配,列表中的 null 改用 `Or(In(...), IsNull(...))` 表达。`And/Or` 使用 `conditions` 元组,其他比较节点使用 `field` 和 `value`。MongoDB `Exists` 与 `IsNull` 只有字段参数。 +公共层只校验普通字段名,如无点号、NUL 或开头的美元符。各适配器负责各自的标识符、安全引用、参数绑定和 BSON 兼容性。 + +## 写入准备和结果 + +`_types.WriteInput` 包含 `key`、`values`、`ttl_s` 和可选的不透明 `prepared`。插入的 `values` 是完整输入记录,未提供完整身份时 `key=None`。更新的 `values` 是字段赋值,删操作的 `values={}`。 +`ttl_s is UNSET` 表示省略,`None` 表示显式移除过期时间。TTL 只支持 Redis。 +需要在发送前完成编码或类型校验时,实现同步的 `prepare_write(operation, item) -> WriteInput`(`operation` 固定为 `insert/update/delete`),可利用 `dataclasses.replace` 保存准备好的参数。可实现同步 `prepare_key(key) -> None` 验证读取键的编码。两个准备钩子都不访问网络。 +公共层在批量发送之前对整批执行结构、键、字段及适配器准备。若后端的静态校验(如字段/值编码)可判断是否能执行,放在准备期间。 + +`_types` 定义以下共享类型,写结果尽可能保留原生对象。 + +- `WriteResult(backend, operation, acknowledged=None, matched_count=None, modified_count=None, inserted_count=None, deleted_count=None, generated_keys=None, native_result=None)`。完整生成身份使用字典元组,缺少证据时使用 `None`。 +- `WriteCounts(matched_count=None, modified_count=None, inserted_count=None, deleted_count=None)`。分块只有能准确汇总的计数才填写。 +- `BatchItem(index, key, state, result=None, error=None)`,`index` 是**分块内相对索引**,`state` 使用 `BatchState`。分块返回尽可能完整的项,公共层依据相对索引复原全局顺序。 +- `ChunkResult(items, counts=WriteCounts(), native_results=())` 提供当前块结果,后续块仍由公共层发送。 +- `ChunkError(original, partial_result)` 用于原生错误有已知部分结果的情况。原始错误为驱动交付的对象,`partial_result` 为 `ChunkResult`,包含已知状态和原生错误明细。没有逐项证据时可直接抛驱动异常。 + +已发送或可能发送的块无法判断的项留为 `unknown`。SQL 分组在块内停止时,明确未发送的组可标记 `not_sent`。读分块有已确认明细时,`BatchItem.result` 为 `ReadItem`,其 `index` 同样是分块内索引。 +核心将已发送块的未知项与后续未发送项区分,保留原始异常及因果链。捕获取消时按同样方式附加部分结果后重新抛出同一个对象。适配器有已知明细时可给 ChunkError 或原始取消附加 `database_partial_result`(值为 `ChunkResult`)。 +`UnsupportedOperationError`、`RecordConflictError`、`RecordFormatError`、`RecordIdentityError` 位于 `_errors.py`。适配器已明确分类的异常保留原生对象和 `raise ... from ...` 关系。其他驱动异常由公共层使用连接句柄现有转换。 diff --git a/src/database/crud/_conditions.py b/src/database/crud/_conditions.py new file mode 100644 index 0000000..b51550d --- /dev/null +++ b/src/database/crud/_conditions.py @@ -0,0 +1,141 @@ +"""Structured conditions for ordinary fields.""" + +import re +import sys +from collections.abc import Mapping +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date +from datetime import datetime +from datetime import time +from decimal import Decimal +from typing import Any +from typing import TypeAlias +from uuid import UUID + +from ..errors import ConfigurationError +from ._errors import UnsupportedOperationError + + +@dataclass(frozen=True) +class Eq: + field: str + value: Any + + +@dataclass(frozen=True) +class In: + field: str + values: Sequence[Any] + + +@dataclass(frozen=True) +class Gt: + field: str + value: Any + + +@dataclass(frozen=True) +class Gte: + field: str + value: Any + + +@dataclass(frozen=True) +class Lt: + field: str + value: Any + + +@dataclass(frozen=True) +class Lte: + field: str + value: Any + + +@dataclass(frozen=True) +class IsNull: + field: str + + +@dataclass(frozen=True) +class Exists: + field: str + + +@dataclass(frozen=True, init=False) +class And: + conditions: tuple['Condition', ...] + + def __init__(self, *conditions: 'Condition') -> None: + object.__setattr__(self, 'conditions', conditions) + + +@dataclass(frozen=True, init=False) +class Or: + conditions: tuple['Condition', ...] + + def __init__(self, *conditions: 'Condition') -> None: + object.__setattr__(self, 'conditions', conditions) + + +Condition: TypeAlias = Eq | In | Gt | Gte | Lt | Lte | IsNull | Exists | And | Or +Where: TypeAlias = Condition | Mapping[str, Any] | None +_FIELD_NAME = re.compile(r'[^.\x00]+\Z') +_SCALARS = {str, int, float, bool, bytes, Decimal, date, datetime, time, UUID} + + +def field_name(name: Any) -> str: + """Validate one plain field name; adapters apply their identifier rules.""" + if not isinstance(name, str) or not name.strip() or name.startswith('$') or not _FIELD_NAME.fullmatch(name): + raise ConfigurationError('Expected a plain, nonempty record field name.', phase='config') + return name + + +def is_sequence(value: Any) -> bool: + return isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray, memoryview)) + + +def validate_scalar(value: Any) -> None: + object_id = getattr(sys.modules.get('bson.objectid'), 'ObjectId', None) + if type(value) not in _SCALARS and type(value) is not object_id: + raise ConfigurationError('Expected a supported scalar comparison or key value.', phase='config') + + +def normalize_where(where: Where, backend: str, operation: str) -> Condition | None: + """Normalize dict equality and validate the complete condition tree.""" + if where is None: + return None + if isinstance(where, Mapping): + if not where: + return None + conditions = tuple(_condition(Eq(name, value), backend, operation) for name, value in where.items()) + return conditions[0] if len(conditions) == 1 else And(*conditions) + return _condition(where, backend, operation) + + +def _condition(node: Condition, backend: str, operation: str) -> Condition: + if type(node) in (And, Or): + if not node.conditions: + raise ConfigurationError('And and Or require at least one condition.', phase='config') + return type(node)(*(_condition(child, backend, operation) for child in node.conditions)) + if type(node) not in (Eq, In, Gt, Gte, Lt, Lte, IsNull, Exists): + raise ConfigurationError('Expected a structured record condition.', phase='config') + name = field_name(node.field) + if type(node) is Exists and backend != 'mongodb': + raise UnsupportedOperationError( + 'Field existence conditions require MongoDB.', backend=backend, operation=operation, phase='config' + ) + if type(node) in (IsNull, Exists): + return node + if type(node) is Eq and node.value is None: + return IsNull(name) + if type(node) is In: + if not is_sequence(node.values): + raise ConfigurationError('In requires a finite sequence of scalar values.', phase='config') + values = tuple(node.values) + for value in values: + validate_scalar(value) + return In(name, values) + validate_scalar(node.value) + return node diff --git a/src/database/crud/_errors.py b/src/database/crud/_errors.py new file mode 100644 index 0000000..d4b05f1 --- /dev/null +++ b/src/database/crud/_errors.py @@ -0,0 +1,28 @@ +"""Errors shared by the managed CRUD API.""" + +from ..errors import DatabaseError +from ._types import BatchResult + + +class UnsupportedOperationError(DatabaseError): + """The selected backend does not support the requested operation.""" + + +class RecordConflictError(DatabaseError): + """A create-only operation conflicts with an existing record.""" + + +class RecordFormatError(DatabaseError): + """A stored record uses an unsupported format.""" + + +class RecordIdentityError(DatabaseError): + """A declared unique key matches more than one record.""" + + +class BatchOperationError(DatabaseError): + """A batch operation failed; the confirmed and uncertain results are retained.""" + + def __init__(self, message: str, *, partial_result: BatchResult, **context) -> None: + super().__init__(message, **context) + self.partial_result = partial_result diff --git a/src/database/crud/_types.py b/src/database/crud/_types.py new file mode 100644 index 0000000..14b5011 --- /dev/null +++ b/src/database/crud/_types.py @@ -0,0 +1,98 @@ +"""Public result types and private data exchanged with backend adapters.""" + +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class _Unset(Enum): + TOKEN = 'unset' + + +UNSET = _Unset.TOKEN + + +@dataclass(frozen=True) +class ReadItem: + """The observed record (or an observed absence) for one input position.""" + + index: int + key: dict[str, Any] + record: dict[str, Any] | None + + +@dataclass(frozen=True) +class WriteCounts: + """Counts the backend can establish for a write.""" + + matched_count: int | None = None + modified_count: int | None = None + inserted_count: int | None = None + deleted_count: int | None = None + + +@dataclass(frozen=True) +class WriteResult: + """Write execution information, independent of the final transaction outcome.""" + + backend: str + operation: str + acknowledged: bool | None = None + matched_count: int | None = None + modified_count: int | None = None + inserted_count: int | None = None + deleted_count: int | None = None + generated_keys: tuple[dict[str, Any], ...] | None = None + native_result: Any = None + + +class BatchState(str, Enum): + """States backed by evidence from the SDK or the driver.""" + + ACKNOWLEDGED = 'acknowledged' + UNACKNOWLEDGED = 'unacknowledged' + FAILED = 'failed' + NOT_SENT = 'not_sent' + NOT_EXECUTED = 'not_executed' + UNKNOWN = 'unknown' + + +@dataclass(frozen=True) +class BatchItem: + index: int + key: dict[str, Any] | None + state: BatchState + result: ReadItem | WriteResult | None = None + error: Any = None + + +@dataclass(frozen=True) +class BatchResult: + requested_count: int + items: tuple[BatchItem, ...] + counts: WriteCounts = WriteCounts() + native_results: tuple[Any, ...] = () + + +@dataclass(frozen=True) +class WriteInput: + key: dict[str, Any] | None + values: dict[str, Any] + ttl_s: int | None | _Unset = UNSET + prepared: Any = None + + +@dataclass(frozen=True) +class ChunkResult: + items: tuple[BatchItem, ...] + counts: WriteCounts = WriteCounts() + native_results: tuple[Any, ...] = () + + +class ChunkError(Exception): + """Adapter reports reliable partial data without wrapping the driver cause.""" + + def __init__(self, original: BaseException, partial_result: ChunkResult) -> None: + super().__init__('A database batch chunk failed.') + self.original = original + self.partial_result = partial_result diff --git a/tests/test_crud_conditions.py b/tests/test_crud_conditions.py new file mode 100644 index 0000000..601c103 --- /dev/null +++ b/tests/test_crud_conditions.py @@ -0,0 +1,42 @@ +"""Contract tests for conditions that adapters receive.""" + +import unittest + +from database.crud._conditions import And +from database.crud._conditions import Eq +from database.crud._conditions import Exists +from database.crud._conditions import Gt +from database.crud._conditions import In +from database.crud._conditions import IsNull +from database.crud._conditions import Or +from database.crud._conditions import normalize_where +from database.crud._errors import UnsupportedOperationError +from database.errors import ConfigurationError + + +class ConditionTests(unittest.TestCase): + def test_dict_becomes_explicit_and_null(self): + result = normalize_where({'tenant_id': 7, 'deleted_at': None}, 'mongodb', 'find') + self.assertEqual(result, And(Eq('tenant_id', 7), IsNull('deleted_at'))) + + def test_nested_empty_membership_and_normalized_null(self): + result = normalize_where(Or(Gt('id', 9), In('id', []), Eq('note', None)), 'mysql', 'find') + self.assertEqual(result, Or(Gt('id', 9), In('id', ()), IsNull('note'))) + + def test_exists_has_no_mysql_translation(self): + with self.assertRaises(UnsupportedOperationError) as caught: + normalize_where(Or(Eq('id', 3), Exists('label')), 'mysql', 'delete_where') + self.assertEqual(caught.exception.operation, 'delete_where') + + def test_rejects_operator_dicts_and_ambiguous_values(self): + inputs = [ + {'field': {'$gt': 0}}, + {'$or': []}, + {'extra.path': 7}, + In('field', [None]), + Or(), + Gt('field', None), + ] + for value in inputs: + with self.subTest(value=value), self.assertRaises(ConfigurationError): + normalize_where(value, 'mongodb', 'find') -- Gitee From da1103db99a798ffbbadaf24679d2236632f1e84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:17:56 +0800 Subject: [PATCH 48/86] =?UTF-8?q?docs(crud):=20=E8=AE=B0=E5=BD=95=E5=BC=80?= =?UTF-8?q?=E5=8F=91=E4=B8=BB=E7=BA=BF=E4=B8=8E=E5=AE=9E=E6=96=BD=E5=9F=BA?= =?UTF-8?q?=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/README.md | 4 ++-- docs/crud-design.md | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/README.md b/docs/README.md index 54b2511..4d76c23 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,9 +14,9 @@ - [测试指南](../tests/README.md) 提供离线回归和隔离服务探测的运行方式。 - [连接管理验收记录](connection-management-validation.md) 记录交付时实际验证的驱动版本、真实拓扑及待验证项。 -## 待审查设计 +## CRUD 开发设计 -- [高级 CRUD 设计提案](crud-design.md) 说明批量操作、字段投影、返回结果、Redis 兼容边界、同步与异步及原生客户端访问,状态为待审查。 +- [高级 CRUD 设计](crud-design.md) 说明批量操作、字段投影、返回结果、Redis 兼容边界、同步与异步及原生客户端访问,目前进入实施阶段。 ## 目录职责 diff --git a/docs/crud-design.md b/docs/crud-design.md index f385abf..c308ac8 100644 --- a/docs/crud-design.md +++ b/docs/crud-design.md @@ -1,7 +1,7 @@ # database 高级 CRUD 设计提案 日期:2026-11-15。 -状态:待用户审查。 +状态:已获准进入实施,2026-11-16 起以 `docs/database-crud-design` 作为功能开发与集成主线。 基线:`develop` 的已提交版本 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 本地文档分支:`docs/database-crud-design`。 @@ -489,8 +489,8 @@ MongoDB 和 Redis 异步用法分别采用 AsyncRecords.mongodb/redis_hash、asy ## 11. 实施阶段与验收计划 -用户审查通过本文后,再确定实施分支、模块文件和开发排期。 -建议从届时本地已提交 develop 建立独立集成分支,按 [协作规范](../CONTRIBUTING.md) 安排公共契约、MySQL、MongoDB、Redis 和验收工作。 +用户于 2026-11-16 指定 `docs/database-crud-design` 为功能开发主线;各实现分支从该主线的已提交检查点建立。 +按 [协作规范](../CONTRIBUTING.md) 安排公共契约、MySQL、MongoDB、Redis 和验收工作。 公共结果与条件语义先形成检查点,后端模块再从同一检查点开展并行实现,最后串行合并。 分支命名使用 feature/fix/refactor/docs 等项目规范前缀。 是否提交、合并和推送遵循实施时用户指令;目前 Git 操作限定本地。 @@ -542,4 +542,6 @@ Git 操作在本地依次完成,主目录为 `D:\996-work\loongspears\database 文档验证已检查 UTF-8、相对链接目标、代码块语法和 Git 空白差异;检查脚本对新增未跟踪文档也进行了逐行空白核对。 Python 示例只做语法核对,因为拟议模块尚未实现;真实驱动行为及新接口测试均留在实施验收阶段。 -本次文档尚未暂存或提交,交付状态以工作区差异为准。 +上述记录为 2026-11-15 文档编写阶段。 +2026-11-16 已将设计与索引提交为 `80fd877360787309bf4a679bd94b1baa76c733ea`,并将主目录切换到 `docs/database-crud-design`。 +实施记录及验收状态由后续交付补充。 -- Gitee From b5fc99c9f30bd00785ac6c4757c04d00e49c611b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:25:31 +0800 Subject: [PATCH 49/86] =?UTF-8?q?feat(crud):=20=E5=AE=9E=E7=8E=B0=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E4=B8=8A=E4=B8=8B=E6=96=87=E5=92=8C=E6=89=B9=E5=A4=84?= =?UTF-8?q?=E7=90=86=E7=BB=93=E6=9E=9C=E6=B1=87=E6=80=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/crud-adapter-contract.md | 1 + src/database/crud/__init__.py | 70 ++++ src/database/crud/_batch.py | 103 ++++++ src/database/crud/_conditions.py | 3 + src/database/crud/_records.py | 526 +++++++++++++++++++++++++++++++ src/database/crud/_types.py | 11 + tests/test_crud_core.py | 312 ++++++++++++++++++ 7 files changed, 1026 insertions(+) create mode 100644 src/database/crud/__init__.py create mode 100644 src/database/crud/_batch.py create mode 100644 src/database/crud/_records.py create mode 100644 tests/test_crud_core.py diff --git a/docs/crud-adapter-contract.md b/docs/crud-adapter-contract.md index 2e60329..cf168b7 100644 --- a/docs/crud-adapter-contract.md +++ b/docs/crud-adapter-contract.md @@ -35,6 +35,7 @@ Redis 不需要实现 `find/count/update_many/delete_where`,公共层会拒绝 `_types.WriteInput` 包含 `key`、`values`、`ttl_s` 和可选的不透明 `prepared`。插入的 `values` 是完整输入记录,未提供完整身份时 `key=None`。更新的 `values` 是字段赋值,删操作的 `values={}`。 `ttl_s is UNSET` 表示省略,`None` 表示显式移除过期时间。TTL 只支持 Redis。 +公共 `insert_many` 接收普通记录或 `InsertItem(record, ttl_s=...)`,普通记录中的同名业务字段原样保留。 需要在发送前完成编码或类型校验时,实现同步的 `prepare_write(operation, item) -> WriteInput`(`operation` 固定为 `insert/update/delete`),可利用 `dataclasses.replace` 保存准备好的参数。可实现同步 `prepare_key(key) -> None` 验证读取键的编码。两个准备钩子都不访问网络。 公共层在批量发送之前对整批执行结构、键、字段及适配器准备。若后端的静态校验(如字段/值编码)可判断是否能执行,放在准备期间。 diff --git a/src/database/crud/__init__.py b/src/database/crud/__init__.py new file mode 100644 index 0000000..eb9f14c --- /dev/null +++ b/src/database/crud/__init__.py @@ -0,0 +1,70 @@ +"""Optional record operations that borrow managed database connections.""" + +from typing import TYPE_CHECKING +from typing import Any + +from ._conditions import And +from ._conditions import Eq +from ._conditions import Exists +from ._conditions import Gt +from ._conditions import Gte +from ._conditions import In +from ._conditions import IsNull +from ._conditions import Lt +from ._conditions import Lte +from ._conditions import Or +from ._errors import BatchOperationError +from ._errors import RecordConflictError +from ._errors import RecordFormatError +from ._errors import RecordIdentityError +from ._errors import UnsupportedOperationError +from ._records import AsyncRecords +from ._records import Records +from ._types import BatchItem +from ._types import BatchResult +from ._types import BatchState +from ._types import InsertItem +from ._types import ReadItem +from ._types import WriteCounts +from ._types import WriteResult + +if TYPE_CHECKING: + from ._codecs import FieldCodec + from ._codecs import JsonFieldCodec + +__all__ = [ + 'And', + 'AsyncRecords', + 'BatchItem', + 'BatchOperationError', + 'BatchResult', + 'BatchState', + 'Eq', + 'Exists', + 'FieldCodec', + 'Gt', + 'Gte', + 'In', + 'InsertItem', + 'IsNull', + 'JsonFieldCodec', + 'Lt', + 'Lte', + 'Or', + 'ReadItem', + 'RecordConflictError', + 'RecordFormatError', + 'RecordIdentityError', + 'Records', + 'UnsupportedOperationError', + 'WriteCounts', + 'WriteResult', +] + + +def __getattr__(name: str) -> Any: + if name in ('FieldCodec', 'JsonFieldCodec'): + from . import _codecs + + return getattr(_codecs, name) + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/src/database/crud/_batch.py b/src/database/crud/_batch.py new file mode 100644 index 0000000..c266954 --- /dev/null +++ b/src/database/crud/_batch.py @@ -0,0 +1,103 @@ +"""Preserve fetch ordering and evidence from completed batch chunks.""" + +from collections.abc import Sequence +from dataclasses import replace +from typing import Any + +from ._types import BatchItem +from ._types import BatchResult +from ._types import BatchState +from ._types import ChunkResult +from ._types import ReadItem +from ._types import WriteCounts + +Record = dict[str, Any] +Positions = Sequence[tuple[int, ...]] + + +def identity(key: Record) -> tuple[tuple[type, Any], ...]: + """Compare previously validated scalar keys without conflating Python types.""" + return tuple((type(value), value) for value in key.values()) + + +def unique_keys(keys: Sequence[Record]) -> tuple[list[Record], list[tuple[int, ...]]]: + groups: dict[tuple[tuple[type, Any], ...], list[int]] = {} + unique = [] + for index, key in enumerate(keys): + token = identity(key) + if token not in groups: + unique.append(key) + groups[token] = [] + groups[token].append(index) + return unique, [tuple(group) for group in groups.values()] + + +class Batch: + """Track the original positions, including read keys queried once.""" + + def __init__(self, keys: Sequence[Record | None]) -> None: + self.items = [BatchItem(index, key, BatchState.NOT_SENT) for index, key in enumerate(keys)] + self.native_results: list[Any] = [] + self._counts: list[WriteCounts] = [] + + def absorb(self, positions: Positions, chunk: ChunkResult | None) -> None: + by_index = {item.index: item for item in chunk.items} if chunk is not None else {} + for offset, group in enumerate(positions): + chunk_item = by_index.get(offset) + for index in group: + original = self.items[index] + if chunk_item is None: + self.items[index] = replace(original, state=BatchState.UNKNOWN) + continue + result = chunk_item.result + if isinstance(result, ReadItem): + result = self._read_result(original, result.record) + self.items[index] = replace(original, state=chunk_item.state, result=result, error=chunk_item.error) + if chunk is not None: + self.native_results.extend(chunk.native_results) + self._counts.append(chunk.counts if chunk is not None else WriteCounts()) + + def absorb_reads(self, positions: Positions, rows: Sequence[Record | None]) -> None: + if len(rows) != len(positions): + raise RuntimeError('Adapter read result does not match chunk size.') + for group, row in zip(positions, rows): + for index in group: + item = self.items[index] + self.items[index] = replace(item, state=BatchState.ACKNOWLEDGED, result=self._read_result(item, row)) + + @staticmethod + def _read_result(item: BatchItem, record: Record | None) -> ReadItem: + if item.key is None: + raise RuntimeError('A batch read result must have an input key.') + return ReadItem(item.index, item.key.copy(), None if record is None else record.copy()) + + def read_items(self) -> list[ReadItem]: + results = [] + for item in self.items: + if not isinstance(item.result, ReadItem): + raise RuntimeError('A complete batch read must include every input position.') + results.append(item.result) + return results + + def result(self, operation: str) -> BatchResult: + if not self.items: + counts = WriteCounts( + matched_count=0 if operation == 'bulk_update' else None, + modified_count=0 if operation == 'bulk_update' else None, + inserted_count=0 if operation == 'insert_many' else None, + deleted_count=0 if operation == 'delete_many' else None, + ) + else: + counts = WriteCounts( + matched_count=self._sum_count('matched_count'), + modified_count=self._sum_count('modified_count'), + inserted_count=self._sum_count('inserted_count'), + deleted_count=self._sum_count('deleted_count'), + ) + return BatchResult(len(self.items), tuple(self.items), counts, tuple(self.native_results)) + + def _sum_count(self, name: str) -> int | None: + values = [getattr(counts, name) for counts in self._counts] + if not values or any(value is None for value in values): + return None + return sum(values) diff --git a/src/database/crud/_conditions.py b/src/database/crud/_conditions.py index b51550d..cd21da3 100644 --- a/src/database/crud/_conditions.py +++ b/src/database/crud/_conditions.py @@ -1,5 +1,6 @@ """Structured conditions for ordinary fields.""" +import math import re import sys from collections.abc import Mapping @@ -100,6 +101,8 @@ def validate_scalar(value: Any) -> None: object_id = getattr(sys.modules.get('bson.objectid'), 'ObjectId', None) if type(value) not in _SCALARS and type(value) is not object_id: raise ConfigurationError('Expected a supported scalar comparison or key value.', phase='config') + if (type(value) is float and not math.isfinite(value)) or (type(value) is Decimal and not value.is_finite()): + raise ConfigurationError('Comparison and key values must be finite.', phase='config') def normalize_where(where: Where, backend: str, operation: str) -> Condition | None: diff --git a/src/database/crud/_records.py b/src/database/crud/_records.py new file mode 100644 index 0000000..9110d0f --- /dev/null +++ b/src/database/crud/_records.py @@ -0,0 +1,526 @@ +"""Typed sync and async record operations on borrowed connection handles.""" + +import asyncio +import importlib +from collections.abc import Mapping +from collections.abc import Sequence +from contextlib import AbstractContextManager +from typing import Any +from typing import ClassVar +from typing import NoReturn +from typing import TypeVar + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DatabaseError +from ..errors import DependencyError +from ..errors import attach_cleanup_errors +from ._batch import Batch +from ._batch import Positions +from ._batch import Record +from ._batch import identity +from ._batch import unique_keys +from ._conditions import Condition +from ._conditions import Where +from ._conditions import field_name +from ._conditions import is_sequence +from ._conditions import normalize_where +from ._conditions import validate_scalar +from ._errors import BatchOperationError +from ._errors import UnsupportedOperationError +from ._types import UNSET +from ._types import BatchResult +from ._types import ChunkError +from ._types import ChunkResult +from ._types import InsertItem +from ._types import ReadItem +from ._types import WriteInput +from ._types import WriteResult +from ._types import _Unset + +_Bound = TypeVar('_Bound', bound='_Records') +_Key = Mapping[str, Any] +_Fields = Sequence[str] | None +_Order = Sequence[tuple[str, str]] +_TTL = int | None | _Unset + + +def _names(value: Sequence[str], *, empty: bool = True) -> tuple[str, ...]: + if not is_sequence(value) or (not empty and not value): + raise ConfigurationError('Expected a finite sequence of field names.', phase='config') + names = tuple(field_name(name) for name in value) + if len(set(names)) != len(names): + raise ConfigurationError('Field names must not be repeated.', phase='config') + return names + + +def _integer(value: int, minimum: int, option: str) -> int: + if type(value) is not int or value < minimum: + raise ConfigurationError(f'{option} must be an integer of at least {minimum}.', phase='config') + return value + + +def _fields(value: _Fields) -> tuple[str, ...] | None: + return None if value is None else _names(value) + + +class _Records: + _async: ClassVar[bool] = False + + def __init__(self, connection: Connection, adapter: Any, key_fields: tuple[str, ...]) -> None: + self._connection = connection + self._adapter = adapter + self._key_fields = key_fields + self._state = 'new' + + @classmethod + def mysql(cls: type[_Bound], connection: Connection, *, table: str, key_fields: Sequence[str]) -> _Bound: + """Bind a MySQL table to its declared unique key.""" + return cls._bind(connection, 'mysql', 'mysql', key_fields, table=table) + + @classmethod + def mongodb( + cls: type[_Bound], + connection: Connection, + *, + database: str, + collection: str, + key_fields: Sequence[str] = ('_id',), + session: Any = None, + ) -> _Bound: + """Bind a MongoDB collection and optionally borrow a caller-managed session.""" + return cls._bind( + connection, 'mongodb', 'mongodb', key_fields, database=database, collection=collection, session=session + ) + + @classmethod + def redis_hash( + cls: type[_Bound], + connection: Connection, + *, + namespace: str, + key_fields: Sequence[str], + codec: Any = None, + ) -> _Bound: + """Bind an explicitly named Redis Hash record namespace.""" + return cls._bind(connection, 'redis', 'redis_hash', key_fields, namespace=namespace, codec=codec) + + @classmethod + def _bind( + cls: type[_Bound], + connection: Connection, + backend: str, + adapter_name: str, + key_fields: Sequence[str], + **options: Any, + ) -> _Bound: + if not isinstance(connection, Connection) or cls._async != isinstance(connection, AsyncConnection): + raise ConfigurationError('Record binding requires a connection with the same sync mode.', phase='config') + if connection.backend != backend: + raise ConfigurationError( + 'Record binding does not match the connection backend.', + connection_name=connection.name, + backend=connection.backend, + phase='config', + ) + connection._ensure_open('operation') + names = _names(key_fields, empty=False) + try: + module = importlib.import_module(f'database.crud._{adapter_name}') + except ImportError as error: + raise DependencyError( + 'The CRUD adapter or its database driver is unavailable.', + connection_name=connection.name, + backend=backend, + phase='config', + original=error, + ) from error + factory = module.AsyncAdapter if cls._async else module.Adapter + adapter = factory(connection=connection, key_fields=names, **options) + return cls(connection, adapter, names) + + def _context(self, operation: str) -> AbstractContextManager: + return self._connection._context('operation', operation=operation, owned=True) + + def _error_context(self, operation: str) -> dict[str, str]: + return dict( + connection_name=self._connection.name, + backend=self._connection.backend, + phase='operation', + operation=operation, + ) + + def _check_connection(self) -> None: + if isinstance(self._connection, AsyncConnection): + self._connection._ensure_loop('operation') + self._connection._ensure_open('operation') + + def _start(self) -> None: + self._check_connection() + if self._state != 'new': + raise ConfigurationError('Record contexts can be entered only once.', phase='operation') + self._state = 'closed' + + def _check_active(self) -> None: + self._check_connection() + if self._state != 'active': + raise ConfigurationError('Record operations require an active record context.', phase='operation') + + def _key(self, key: _Key) -> Record: + if not isinstance(key, Mapping) or set(key) != set(self._key_fields): + raise ConfigurationError('A record key must contain exactly the declared key fields.', phase='config') + values = {field: key[field] for field in self._key_fields} + for value in values.values(): + validate_scalar(value) + prepare_key = getattr(self._adapter, 'prepare_key', None) + if prepare_key is not None: + prepare_key(values) + return values + + def _read_batch( + self, keys: Sequence[_Key], fields: _Fields, batch_size: int + ) -> tuple[Batch, list[Record], list[tuple[int, ...]], tuple[str, ...] | None]: + self._check_active() + _integer(batch_size, 1, 'batch_size') + projected = _fields(fields) + if not is_sequence(keys): + raise ConfigurationError('Record keys must be a finite sequence.', phase='config') + prepared = [self._key(key) for key in keys] + unique, positions = unique_keys(prepared) + return Batch(prepared), unique, positions, projected + + def _where(self, where: Where, operation: str) -> Condition | None: + self._check_active() + backend = self._connection.backend + if backend == 'redis': + raise UnsupportedOperationError( + 'Redis Hash records only support keyed operations.', **self._error_context(operation) + ) + return normalize_where(where, backend, operation) + + def _find_parameters( + self, where: Where, fields: _Fields, limit: int, offset: int, order_by: _Order + ) -> tuple[Condition | None, tuple[str, ...] | None, int, int, tuple[tuple[str, str], ...]]: + condition = self._where(where, 'find') + projected = _fields(fields) + _integer(limit, 1, 'limit') + _integer(offset, 0, 'offset') + if not is_sequence(order_by): + raise ConfigurationError('Expected a finite sequence of sort fields.', phase='config') + order = [] + for pair in order_by: + if not is_sequence(pair) or len(pair) != 2 or pair[1] not in ('asc', 'desc'): + raise ConfigurationError('Each sort must provide a field and asc or desc.', phase='config') + order.append((field_name(pair[0]), pair[1])) + _names([field for field, _ in order]) + return condition, projected, limit, offset, tuple(order) + + def _write_where(self, where: Where, operation: str, all_records: bool) -> Condition | None: + condition = self._where(where, operation) + if type(all_records) is not bool or (condition is None and not all_records): + raise ConfigurationError('A write without conditions requires all_records=True.', phase='config') + return condition + + def _values(self, value: Mapping[str, Any], *, updating: bool = False) -> Record: + if not isinstance(value, Mapping): + raise ConfigurationError('Expected a record mapping.', phase='config') + result = {field_name(field): item for field, item in value.items()} + if updating and (not result or set(result).intersection(self._key_fields)): + raise ConfigurationError('Updates require at least one non-key field.', phase='config') + return result + + def _ttl(self, ttl_s: _TTL, operation: str) -> _TTL: + if ttl_s is UNSET: + return UNSET + if self._connection.backend != 'redis': + raise UnsupportedOperationError('Record TTL requires Redis Hash records.', **self._error_context(operation)) + if ttl_s is not None: + _integer(ttl_s, 1, 'ttl_s') + return ttl_s + + def _write_input( + self, + operation: str, + *, + key: _Key | None = None, + values: Mapping[str, Any] | None = None, + ttl_s: _TTL = UNSET, + ) -> WriteInput: + self._check_active() + ttl_s = self._ttl(ttl_s, operation) + if operation == 'insert': + values = self._values(values) + supplied = {field: values[field] for field in self._key_fields if field in values} + for value in supplied.values(): + validate_scalar(value) + if len(supplied) == len(self._key_fields): + key = self._key(supplied) + elif self._connection.backend == 'redis': + raise ConfigurationError('Redis Hash inserts require all key fields.', phase='config') + else: + key = self._key(key) + values = self._values(values, updating=True) if operation == 'update' else {} + item = WriteInput(key, values, ttl_s) + prepare_write = getattr(self._adapter, 'prepare_write', None) + return item if prepare_write is None else prepare_write(operation, item) + + def _insert_item(self, value: Mapping[str, Any] | InsertItem) -> WriteInput: + if isinstance(value, InsertItem): + return self._write_input('insert', values=value.record, ttl_s=value.ttl_s) + return self._write_input('insert', values=value) + + def _update_item(self, value: Mapping[str, Any]) -> WriteInput: + if ( + not isinstance(value, Mapping) + or not {'key', 'set_values'}.issubset(value) + or not set(value).issubset({'key', 'set_values', 'ttl_s'}) + ): + raise ConfigurationError('Bulk updates require key, set_values and optional ttl_s.', phase='config') + return self._write_input( + 'update', key=value['key'], values=value['set_values'], ttl_s=value.get('ttl_s', UNSET) + ) + + def _write_batch(self, operation: str, values: Sequence[Any], batch_size: int) -> tuple[Batch, list[WriteInput]]: + self._check_active() + _integer(batch_size, 1, 'batch_size') + if not is_sequence(values): + raise ConfigurationError('Batch writes require a finite sequence.', phase='config') + if operation == 'insert_many': + items = [self._insert_item(value) for value in values] + elif operation == 'bulk_update': + items = [self._update_item(value) for value in values] + else: + items = [self._write_input('delete', key=value) for value in values] + seen = set() + for item in items: + if item.key is None: + continue + token = identity(item.key) + if token in seen: + raise ConfigurationError('The same record key appears more than once in a write batch.', phase='config') + seen.add(token) + return Batch([item.key for item in items]), items + + def _batch_failed(self, operation: str, error: BaseException, batch: Batch, positions: Positions) -> NoReturn: + if isinstance(error, ChunkError): + chunk, source = error.partial_result, error.original + else: + chunk = getattr(error, 'database_partial_result', None) + source = error + batch.absorb(positions, chunk if isinstance(chunk, ChunkResult) else None) + partial = batch.result(operation) + if isinstance(source, asyncio.CancelledError): + BaseException.__dict__['__dict__'].__get__(source)['database_partial_result'] = partial + if source is error: + raise + raise source + original = source.original if isinstance(source, DatabaseError) and source.original is not None else source + wrapped = BatchOperationError( + 'A database record batch failed.', + partial_result=partial, + original=original, + **self._error_context(operation), + ) + for failure in {id(value): value for value in (original, source, error)}.values(): + attach_cleanup_errors(wrapped, getattr(failure, 'database_cleanup_errors', ())) + raise wrapped from original + + +class Records(_Records): + """Synchronous operations within a borrowed connection's record context.""" + + def __enter__(self) -> 'Records': + self._start() + with self._context('open_records'): + self._adapter.__enter__() + self._state = 'active' + return self + + def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: Any) -> bool: + if self._state != 'active': + return False + try: + self._state = 'closed' + with self._context('close_records'): + self._adapter.__exit__(exc_type, exc, traceback) + except BaseException as cleanup_error: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup_error,)) + return False + + def _invoke(self, operation: str, *args: Any) -> Any: + self._check_active() + with self._context(operation): + return getattr(self._adapter, operation)(*args) + + def get(self, key: _Key, *, fields: _Fields = None) -> Record | None: + self._check_active() + return self._invoke('get', self._key(key), _fields(fields)) + + def get_many(self, keys: Sequence[_Key], *, fields: _Fields = None, batch_size: int = 500) -> list[ReadItem]: + batch, unique, positions, projected = self._read_batch(keys, fields, batch_size) + for start in range(0, len(unique), batch_size): + end = start + batch_size + group = positions[start:end] + try: + rows = self._invoke('get_many', unique[start:end], projected) + batch.absorb_reads(group, rows) + except (Exception, asyncio.CancelledError) as error: + self._batch_failed('get_many', error, batch, group) + return batch.read_items() + + def find( + self, + *, + where: Where = None, + fields: _Fields = None, + limit: int, + offset: int = 0, + order_by: _Order = (), + ) -> list[Record]: + return self._invoke('find', *self._find_parameters(where, fields, limit, offset, order_by)) + + def count(self, *, where: Where = None) -> int: + return self._invoke('count', self._where(where, 'count')) + + def insert(self, record: Mapping[str, Any], *, ttl_s: _TTL = UNSET) -> WriteResult: + return self._invoke('insert', self._write_input('insert', values=record, ttl_s=ttl_s)) + + def update(self, key: _Key, *, set_values: Mapping[str, Any], ttl_s: _TTL = UNSET) -> WriteResult: + return self._invoke('update', self._write_input('update', key=key, values=set_values, ttl_s=ttl_s)) + + def delete(self, key: _Key) -> WriteResult: + return self._invoke('delete', self._write_input('delete', key=key)) + + def update_many(self, *, where: Where, set_values: Mapping[str, Any], all_records: bool = False) -> WriteResult: + condition = self._write_where(where, 'update_many', all_records) + return self._invoke('update_many', condition, self._values(set_values, updating=True)) + + def delete_where(self, *, where: Where, all_records: bool = False) -> WriteResult: + return self._invoke('delete_where', self._write_where(where, 'delete_where', all_records)) + + def insert_many(self, records: Sequence[Mapping[str, Any] | InsertItem], *, batch_size: int = 500) -> BatchResult: + return self._execute_batch('insert_many', records, batch_size) + + def bulk_update(self, items: Sequence[Mapping[str, Any]], *, batch_size: int = 500) -> BatchResult: + return self._execute_batch('bulk_update', items, batch_size) + + def delete_many(self, keys: Sequence[_Key], *, batch_size: int = 500) -> BatchResult: + return self._execute_batch('delete_many', keys, batch_size) + + def _execute_batch(self, operation: str, values: Sequence[Any], batch_size: int) -> BatchResult: + batch, items = self._write_batch(operation, values, batch_size) + for start in range(0, len(items), batch_size): + end = min(start + batch_size, len(items)) + positions = [(index,) for index in range(start, end)] + try: + chunk = self._invoke(operation, items[start:end]) + batch.absorb(positions, chunk) + except (Exception, asyncio.CancelledError) as error: + self._batch_failed(operation, error, batch, positions) + return batch.result(operation) + + +class AsyncRecords(_Records): + """Asynchronous operations within a borrowed connection's record context.""" + + _async = True + + async def __aenter__(self) -> 'AsyncRecords': + self._start() + with self._context('open_records'): + await self._adapter.__aenter__() + self._state = 'active' + return self + + async def __aexit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: Any) -> bool: + if self._state != 'active': + return False + try: + self._connection._ensure_loop('operation') + self._state = 'closed' + with self._context('close_records'): + await self._adapter.__aexit__(exc_type, exc, traceback) + except BaseException as cleanup_error: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup_error,)) + return False + + async def _invoke(self, operation: str, *args: Any) -> Any: + self._check_active() + with self._context(operation): + return await getattr(self._adapter, operation)(*args) + + async def get(self, key: _Key, *, fields: _Fields = None) -> Record | None: + self._check_active() + return await self._invoke('get', self._key(key), _fields(fields)) + + async def get_many(self, keys: Sequence[_Key], *, fields: _Fields = None, batch_size: int = 500) -> list[ReadItem]: + batch, unique, positions, projected = self._read_batch(keys, fields, batch_size) + for start in range(0, len(unique), batch_size): + end = start + batch_size + group = positions[start:end] + try: + rows = await self._invoke('get_many', unique[start:end], projected) + batch.absorb_reads(group, rows) + except (Exception, asyncio.CancelledError) as error: + self._batch_failed('get_many', error, batch, group) + return batch.read_items() + + async def find( + self, + *, + where: Where = None, + fields: _Fields = None, + limit: int, + offset: int = 0, + order_by: _Order = (), + ) -> list[Record]: + return await self._invoke('find', *self._find_parameters(where, fields, limit, offset, order_by)) + + async def count(self, *, where: Where = None) -> int: + return await self._invoke('count', self._where(where, 'count')) + + async def insert(self, record: Mapping[str, Any], *, ttl_s: _TTL = UNSET) -> WriteResult: + return await self._invoke('insert', self._write_input('insert', values=record, ttl_s=ttl_s)) + + async def update(self, key: _Key, *, set_values: Mapping[str, Any], ttl_s: _TTL = UNSET) -> WriteResult: + return await self._invoke('update', self._write_input('update', key=key, values=set_values, ttl_s=ttl_s)) + + async def delete(self, key: _Key) -> WriteResult: + return await self._invoke('delete', self._write_input('delete', key=key)) + + async def update_many( + self, *, where: Where, set_values: Mapping[str, Any], all_records: bool = False + ) -> WriteResult: + condition = self._write_where(where, 'update_many', all_records) + return await self._invoke('update_many', condition, self._values(set_values, updating=True)) + + async def delete_where(self, *, where: Where, all_records: bool = False) -> WriteResult: + return await self._invoke('delete_where', self._write_where(where, 'delete_where', all_records)) + + async def insert_many( + self, records: Sequence[Mapping[str, Any] | InsertItem], *, batch_size: int = 500 + ) -> BatchResult: + return await self._execute_batch('insert_many', records, batch_size) + + async def bulk_update(self, items: Sequence[Mapping[str, Any]], *, batch_size: int = 500) -> BatchResult: + return await self._execute_batch('bulk_update', items, batch_size) + + async def delete_many(self, keys: Sequence[_Key], *, batch_size: int = 500) -> BatchResult: + return await self._execute_batch('delete_many', keys, batch_size) + + async def _execute_batch(self, operation: str, values: Sequence[Any], batch_size: int) -> BatchResult: + batch, items = self._write_batch(operation, values, batch_size) + for start in range(0, len(items), batch_size): + end = min(start + batch_size, len(items)) + positions = [(index,) for index in range(start, end)] + try: + chunk = await self._invoke(operation, items[start:end]) + batch.absorb(positions, chunk) + except (Exception, asyncio.CancelledError) as error: + self._batch_failed(operation, error, batch, positions) + return batch.result(operation) diff --git a/src/database/crud/_types.py b/src/database/crud/_types.py index 14b5011..360677c 100644 --- a/src/database/crud/_types.py +++ b/src/database/crud/_types.py @@ -1,5 +1,7 @@ """Public result types and private data exchanged with backend adapters.""" +from collections.abc import Mapping +from dataclasses import KW_ONLY from dataclasses import dataclass from enum import Enum from typing import Any @@ -21,6 +23,15 @@ class ReadItem: record: dict[str, Any] | None +@dataclass(frozen=True) +class InsertItem: + """An inserted record with explicit per-item options.""" + + record: Mapping[str, Any] + _: KW_ONLY + ttl_s: int | None | _Unset = UNSET + + @dataclass(frozen=True) class WriteCounts: """Counts the backend can establish for a write.""" diff --git a/tests/test_crud_core.py b/tests/test_crud_core.py new file mode 100644 index 0000000..56f0cfa --- /dev/null +++ b/tests/test_crud_core.py @@ -0,0 +1,312 @@ +"""Exercise the public read/write contract without opening database connections.""" + +import asyncio +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import Mock +from unittest.mock import patch + +from database import AsyncConnection +from database import Connection +from database.crud import AsyncRecords +from database.crud import BatchItem +from database.crud import BatchOperationError +from database.crud import BatchState +from database.crud import Eq +from database.crud import InsertItem +from database.crud import ReadItem +from database.crud import Records +from database.crud import UnsupportedOperationError +from database.crud import WriteCounts +from database.crud import WriteResult +from database.crud._types import UNSET +from database.crud._types import ChunkError +from database.crud._types import ChunkResult +from database.errors import ConfigurationError +from database.errors import ConnectionClosedError +from database.errors import DriverError + + +class FakeFailure(Exception): + pass + + +def connection_adapter(): + return SimpleNamespace( + classify=lambda error: DriverError if isinstance(error, FakeFailure) else None, + close=Mock(), + close_async=AsyncMock(), + ) + + +def adapter_result(*items, counts=WriteCounts(), native_results=()): + return ChunkResult(tuple(items), counts, native_results) + + +def bind(connection, adapter, *, asynchronous=False): + factory = Mock(return_value=adapter) + with patch( + 'database.crud._records.importlib.import_module', + return_value=SimpleNamespace(Adapter=factory, AsyncAdapter=factory), + ) as load: + cls = AsyncRecords if asynchronous else Records + if connection.backend == 'redis': + resource = cls.redis_hash(connection, namespace='orders', key_fields=['id']) + else: + resource = cls.mysql(connection, table='orders', key_fields=['id']) + expected_module = '_redis_hash' if connection.backend == 'redis' else '_mysql' + load.assert_called_once_with(f'database.crud.{expected_module}') + assert factory.call_args.kwargs['connection'] is connection + assert factory.call_args.kwargs['key_fields'] == ('id',) + if asynchronous: + adapter.__aenter__.assert_not_called() + else: + adapter.__enter__.assert_not_called() + return resource + + +class RecordsTests(unittest.TestCase): + def setUp(self): + self.native = object() + self.owner = connection_adapter() + self.connection = Connection(self.native, self.owner, 'orders', 'mysql', 'single') + self.adapter = Mock( + spec=[ + '__enter__', + '__exit__', + 'get', + 'get_many', + 'update_many', + 'delete_where', + 'insert', + 'insert_many', + 'bulk_update', + 'delete_many', + 'prepare_write', + 'prepare_key', + ] + ) + self.adapter.__enter__ = Mock() + self.adapter.__exit__ = Mock() + self.adapter.prepare_write.side_effect = lambda operation, item: item + self.records = bind(self.connection, self.adapter) + + def test_lifetime_only_releases_adapter_resources(self): + with self.assertRaises(ConfigurationError): + self.records.get({'id': 1}) + with self.records as records: + self.adapter.get.return_value = {'name': 'Alice'} + self.assertEqual(records.get({'id': 1}, fields=['name']), {'name': 'Alice'}) + with self.assertRaises(ConfigurationError): + self.records.get({'id': 1}) + self.assertIs(self.connection.native, self.native) + self.owner.close.assert_not_called() + self.adapter.__exit__.assert_called_once_with(None, None, None) + + def test_closed_borrowed_connection_is_checked_before_operation(self): + with self.records: + self.connection.close() + with self.assertRaises(ConnectionClosedError): + self.records.get({'id': 1}) + self.adapter.get.assert_not_called() + + def test_converted_error_preserves_driver_cause_and_cleanup(self): + error = FakeFailure('native') + cleanup = RuntimeError('cleanup') + self.adapter.get.side_effect = error + self.adapter.__exit__.side_effect = cleanup + with self.assertRaises(DriverError) as caught: + with self.records: + self.records.get({'id': 1}) + self.assertIs(caught.exception.original, error) + self.assertIs(caught.exception.__cause__, error) + self.assertEqual(caught.exception.database_cleanup_errors, (cleanup,)) + + def test_get_many_deduplicates_and_keeps_observed_missing_distinct_from_empty_record(self): + seen = [] + + def fetch(keys, fields): + seen.extend(keys) + self.assertEqual(fields, ()) + return [None if key['id'] == 2 else {} for key in keys] + + self.adapter.get_many.side_effect = fetch + with self.records: + rows = self.records.get_many([{'id': 3}, {'id': 2}, {'id': 3}, {'id': 1}], fields=[], batch_size=2) + self.assertEqual(seen, [{'id': 3}, {'id': 2}, {'id': 1}]) + self.assertEqual( + rows, + [ + ReadItem(0, {'id': 3}, {}), + ReadItem(1, {'id': 2}, None), + ReadItem(2, {'id': 3}, {}), + ReadItem(3, {'id': 1}, {}), + ], + ) + + def test_partial_reads_restore_global_indices_and_keep_unknown_separate(self): + failure = FakeFailure('transport') + self.adapter.get_many.side_effect = [ + [{'v': 1}, None], + ChunkError(failure, adapter_result(BatchItem(0, None, BatchState.ACKNOWLEDGED, ReadItem(0, {}, {})))), + ] + with self.records: + with self.assertRaises(BatchOperationError) as caught: + self.records.get_many([{'id': value} for value in [1, 2, 3, 4, 3, 5]], batch_size=2) + self.assertIs(caught.exception.original, failure) + self.assertIs(caught.exception.__cause__, failure) + items = caught.exception.partial_result.items + self.assertEqual( + tuple(item.state for item in items), + ( + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.UNKNOWN, + BatchState.ACKNOWLEDGED, + BatchState.NOT_SENT, + ), + ) + self.assertEqual(items[1].result, ReadItem(1, {'id': 2}, None)) + self.assertEqual(items[4].result, ReadItem(4, {'id': 3}, {})) + self.assertIsNone(items[3].result) + self.assertEqual(self.adapter.get_many.call_count, 2) + + def test_write_prepares_all_inputs_before_any_batch_io(self): + invalid = ConfigurationError('invalid codec value') + self.adapter.prepare_write.side_effect = lambda operation, item: ( + (_ for _ in ()).throw(invalid) if item.key == {'id': 2} else item + ) + with self.records: + with self.assertRaises(ConfigurationError) as caught: + self.records.insert_many([{'id': 1}, {'id': 2}], batch_size=1) + self.assertIs(caught.exception, invalid) + self.adapter.insert_many.assert_not_called() + + def test_batch_rejects_duplicate_keys_and_applies_confirmation_rules(self): + with self.records: + with self.assertRaises(ConfigurationError): + self.records.delete_many([{'id': 1}, {'id': 1}]) + with self.assertRaises(ConfigurationError): + self.records.update_many(where={}, set_values={'state': 'done'}) + with self.assertRaises(ConfigurationError): + self.records.delete_where(where=None) + with self.assertRaises(ConfigurationError): + self.records.update_many(where={'id': 1}, set_values={'id': 2}) + self.records.delete_where(where={}, all_records=True) + self.records.update_many(where={'id': 1}, set_values={'state': 'done'}) + self.adapter.delete_many.assert_not_called() + self.adapter.delete_where.assert_called_once_with(None) + self.adapter.update_many.assert_called_once_with(Eq('id', 1), {'state': 'done'}) + + def test_known_counts_sum_and_unknown_batch_work_is_not_invented(self): + ok = BatchItem(0, None, BatchState.ACKNOWLEDGED) + self.adapter.delete_many.side_effect = [ + adapter_result(ok, counts=WriteCounts(deleted_count=1), native_results=('first',)), + adapter_result(ok, counts=WriteCounts(deleted_count=0), native_results=('second',)), + adapter_result(ok, counts=WriteCounts(deleted_count=1)), + FakeFailure('interrupted'), + ] + with self.records: + result = self.records.delete_many([{'id': 1}, {'id': 2}], batch_size=1) + with self.assertRaises(BatchOperationError) as caught: + self.records.delete_many([{'id': 3}, {'id': 4}, {'id': 5}], batch_size=1) + empty = self.records.delete_many([]) + self.assertEqual(result.counts.deleted_count, 1) + self.assertEqual(result.native_results, ('first', 'second')) + self.assertEqual(result.items[1].key, {'id': 2}) + self.assertEqual(empty.counts.deleted_count, 0) + failure = caught.exception.partial_result + self.assertIsNone(failure.counts.deleted_count) + self.assertEqual( + [item.state for item in failure.items], + [BatchState.ACKNOWLEDGED, BatchState.UNKNOWN, BatchState.NOT_SENT], + ) + self.assertEqual(self.adapter.delete_many.call_count, 4) + + def test_ttl_is_explicit_and_redis_conditions_are_unsupported(self): + record = {'id': 1, 'ttl_s': 'business field'} + redis_connection = Connection(object(), self.owner, 'cache', 'redis', 'single') + self.adapter.insert_many.return_value = adapter_result(BatchItem(0, None, BatchState.ACKNOWLEDGED)) + with bind(redis_connection, self.adapter) as cache: + cache.insert_many([InsertItem(record, ttl_s=15)]) + with self.assertRaises(ConfigurationError): + cache.insert(record, ttl_s=False) + with self.assertRaises(UnsupportedOperationError): + cache.find(limit=1) + (prepared,) = self.adapter.insert_many.call_args.args[0] + self.assertEqual(prepared.ttl_s, 15) + self.assertEqual(prepared.values, record) + with self.records: + with self.assertRaises(UnsupportedOperationError): + self.records.insert(record, ttl_s=None) + self.records.insert(record) + self.assertIs(self.adapter.insert.call_args.args[0].ttl_s, UNSET) + + def test_rejects_async_binding_mismatch_before_importing_backend(self): + with patch('database.crud._records.importlib.import_module') as load: + with self.assertRaises(ConfigurationError): + AsyncRecords.mysql(self.connection, table='users', key_fields=['id']) + load.assert_not_called() + + +class AsyncRecordsTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.owner = connection_adapter() + self.native = object() + self.connection = AsyncConnection( + self.native, self.owner, 'orders', 'mysql', 'single', asyncio.get_running_loop() + ) + self.adapter = SimpleNamespace( + __aenter__=AsyncMock(), + __aexit__=AsyncMock(), + get_many=AsyncMock(), + insert=AsyncMock(), + bulk_update=AsyncMock(), + ) + self.records = bind(self.connection, self.adapter, asynchronous=True) + + async def test_awaits_each_native_async_operation_and_keeps_owner_open(self): + self.adapter.get_many.return_value = ({'a': 1},) + self.adapter.insert.return_value = WriteResult('mysql', 'insert', acknowledged=True, inserted_count=1) + async with self.records: + found = await self.records.get_many([{'id': 1}, {'id': 1}]) + inserted = await self.records.insert({'id': 1}) + self.assertEqual([item.record for item in found], [{'a': 1}, {'a': 1}]) + self.assertEqual(inserted.inserted_count, 1) + self.adapter.get_many.assert_awaited_once() + self.adapter.__aexit__.assert_awaited_once() + self.owner.close_async.assert_not_awaited() + self.assertIs(self.connection.native, self.native) + + async def test_cancelled_batch_keeps_the_original_signal_and_partial_status(self): + error = asyncio.CancelledError('cancelled during second chunk') + self.adapter.bulk_update.side_effect = [ + adapter_result(BatchItem(0, None, BatchState.ACKNOWLEDGED)), + error, + ] + cleanup = FakeFailure('driver cleanup failed') + self.adapter.__aexit__.side_effect = cleanup + with self.assertRaises(asyncio.CancelledError) as caught: + async with self.records: + await self.records.bulk_update( + [{'key': {'id': n}, 'set_values': {'value': n}} for n in range(3)], + batch_size=1, + ) + self.assertIs(caught.exception, error) + self.assertEqual( + [item.state for item in error.database_partial_result.items], + [BatchState.ACKNOWLEDGED, BatchState.UNKNOWN, BatchState.NOT_SENT], + ) + self.assertIs(error.database_cleanup_errors[0].original, cleanup) + + async def test_operations_check_event_loop_before_driver_call(self): + async with self.records: + + def another_loop(): + return asyncio.run(self.records.get_many([{'id': 1}])) + + with self.assertRaises(ConfigurationError): + await asyncio.to_thread(another_loop) + self.adapter.get_many.assert_not_awaited() -- Gitee From 0ba757a925247f632d32b9d236457fad5e8fbe00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:26:50 +0800 Subject: [PATCH 50/86] =?UTF-8?q?test(crud):=20=E9=AA=8C=E8=AF=81=E9=A9=B1?= =?UTF-8?q?=E5=8A=A8=E6=83=B0=E6=80=A7=E5=AF=BC=E5=85=A5=E4=B8=8E=E4=BE=9D?= =?UTF-8?q?=E8=B5=96=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/crud-adapter-contract.md | 3 ++- src/database/crud/_errors.py | 4 +++- tests/test_crud_core.py | 41 +++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/docs/crud-adapter-contract.md b/docs/crud-adapter-contract.md index cf168b7..88f89e4 100644 --- a/docs/crud-adapter-contract.md +++ b/docs/crud-adapter-contract.md @@ -9,6 +9,7 @@ - 同步类提供 `__enter__()`、`__exit__(exc_type, exc, traceback)`,异步类提供对应异步上下文方法。可返回自身,不能抑制主异常。释放适配器自行持有的资源,借用连接、客户端和 session 归调用方。 - 入口上下文可构造 Sentinel 主节点数据客户端,不额外探测。Redis 适配模块暂用内部连接适配器配置 `connection._adapter._config.service_name`,该桥接不构成公共接口。 - 公共层负责连接状态和异步事件循环归属,在每次操作以及进入上下文前检查。适配器不新增自动重试、提交或回滚。 +- 导入公共 CRUD 模块不加载数据库驱动。绑定所需的适配模块或驱动缺失时,公共工厂抛出 `DependencyError` 并保留原始导入异常及因果链。 ## 方法签名 @@ -48,5 +49,5 @@ Redis 不需要实现 `find/count/update_many/delete_where`,公共层会拒绝 - `ChunkError(original, partial_result)` 用于原生错误有已知部分结果的情况。原始错误为驱动交付的对象,`partial_result` 为 `ChunkResult`,包含已知状态和原生错误明细。没有逐项证据时可直接抛驱动异常。 已发送或可能发送的块无法判断的项留为 `unknown`。SQL 分组在块内停止时,明确未发送的组可标记 `not_sent`。读分块有已确认明细时,`BatchItem.result` 为 `ReadItem`,其 `index` 同样是分块内索引。 -核心将已发送块的未知项与后续未发送项区分,保留原始异常及因果链。捕获取消时按同样方式附加部分结果后重新抛出同一个对象。适配器有已知明细时可给 ChunkError 或原始取消附加 `database_partial_result`(值为 `ChunkResult`)。 +核心将已发送块的未知项与后续未发送项区分,保留原始异常及因果链。捕获取消时按同样方式附加部分结果后重新抛出同一个对象。适配器有已知明细时通过 `ChunkError.partial_result` 提供;保留原始取消时,可在原对象上附加 `database_partial_result`(值为 `ChunkResult`)。 `UnsupportedOperationError`、`RecordConflictError`、`RecordFormatError`、`RecordIdentityError` 位于 `_errors.py`。适配器已明确分类的异常保留原生对象和 `raise ... from ...` 关系。其他驱动异常由公共层使用连接句柄现有转换。 diff --git a/src/database/crud/_errors.py b/src/database/crud/_errors.py index d4b05f1..65358bd 100644 --- a/src/database/crud/_errors.py +++ b/src/database/crud/_errors.py @@ -1,5 +1,7 @@ """Errors shared by the managed CRUD API.""" +from typing import Any + from ..errors import DatabaseError from ._types import BatchResult @@ -23,6 +25,6 @@ class RecordIdentityError(DatabaseError): class BatchOperationError(DatabaseError): """A batch operation failed; the confirmed and uncertain results are retained.""" - def __init__(self, message: str, *, partial_result: BatchResult, **context) -> None: + def __init__(self, message: str, *, partial_result: BatchResult, **context: Any) -> None: super().__init__(message, **context) self.partial_result = partial_result diff --git a/tests/test_crud_core.py b/tests/test_crud_core.py index 56f0cfa..2d63159 100644 --- a/tests/test_crud_core.py +++ b/tests/test_crud_core.py @@ -1,7 +1,10 @@ """Exercise the public read/write contract without opening database connections.""" import asyncio +import subprocess +import sys import unittest +from pathlib import Path from types import SimpleNamespace from unittest.mock import AsyncMock from unittest.mock import Mock @@ -25,6 +28,7 @@ from database.crud._types import ChunkError from database.crud._types import ChunkResult from database.errors import ConfigurationError from database.errors import ConnectionClosedError +from database.errors import DependencyError from database.errors import DriverError @@ -66,6 +70,43 @@ def bind(connection, adapter, *, asynchronous=False): return resource +class ImportTests(unittest.TestCase): + def test_public_import_does_not_attempt_to_import_drivers(self): + script = """ +import importlib.abc +import sys + +class RejectDriverImports(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.partition('.')[0] in {'mysql', 'pymongo', 'redis'}: + raise AssertionError(fullname) + +sys.meta_path.insert(0, RejectDriverImports()) +sys.path.insert(0, sys.argv[1]) +from database.crud import AsyncRecords, Records, WriteResult +""" + source = Path(__file__).resolve().parents[1] / 'src' + result = subprocess.run( + [sys.executable, '-I', '-B', '-c', script, str(source)], + capture_output=True, + text=True, + check=False, + timeout=10, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_binding_reports_unavailable_driver_with_original_import_error(self): + connection = Connection(object(), connection_adapter(), 'orders', 'mysql', 'single') + original = ModuleNotFoundError('The requested driver is unavailable.') + with patch('database.crud._records.importlib.import_module', side_effect=original): + with self.assertRaises(DependencyError) as caught: + Records.mysql(connection, table='orders', key_fields=['id']) + failure = caught.exception + self.assertIs(failure.original, original) + self.assertIs(failure.__cause__, original) + self.assertEqual((failure.backend, failure.connection_name, failure.phase), ('mysql', 'orders', 'config')) + + class RecordsTests(unittest.TestCase): def setUp(self): self.native = object() -- Gitee From d3a93cf35e531ec7311072c7a6ed4d091911e1e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:29:14 +0800 Subject: [PATCH 51/86] =?UTF-8?q?test(crud):=20=E9=AA=8C=E8=AF=81=E5=A4=9A?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E4=B8=8E=E5=90=8C=E7=B1=BB=E5=A4=9A?= =?UTF-8?q?=E5=AE=9E=E4=BE=8B=E8=B5=84=E6=BA=90=E9=9A=94=E7=A6=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/test_crud_integration.py | 160 +++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tests/test_crud_integration.py diff --git a/tests/test_crud_integration.py b/tests/test_crud_integration.py new file mode 100644 index 0000000..4afd51e --- /dev/null +++ b/tests/test_crud_integration.py @@ -0,0 +1,160 @@ +"""Named connections and record contexts compose without sharing ownership.""" + +import asyncio +import unittest +from contextlib import AsyncExitStack +from contextlib import ExitStack +from types import SimpleNamespace +from unittest.mock import patch + +from database import ConnectionConfig +from database import Connections +from database.crud import AsyncRecords +from database.crud import Records +from database.errors import ConnectionClosedError + + +class Native: + def __init__(self, name): + self.name = name + self.closed = False + + def native_only(self): + return self.name + + +class ConnectionAdapter: + def __init__(self, config, name, is_async=False): + self.name = name + + def create(self): + return Native(self.name) + + async def create_async(self): + await asyncio.sleep(0) + return self.create() + + @staticmethod + def classify(error): + return None + + @staticmethod + def close(native): + native.closed = True + + async def close_async(self, native): + await asyncio.sleep(0) + self.close(native) + + +class RecordAdapter: + def __init__(self, *, connection, key_fields, **options): + self.connection = connection + self.closed = False + + def __enter__(self): + return self + + def __exit__(self, *exc): + self.closed = True + + def get(self, key, fields): + return {'owner': self.connection.native.native_only()} + + +class AsyncRecordAdapter(RecordAdapter): + async def __aenter__(self): + await asyncio.sleep(0) + return self + + async def __aexit__(self, *exc): + await asyncio.sleep(0) + self.closed = True + + async def get(self, key, fields): + await asyncio.sleep(0) + return super().get(key, fields) + + +def modules(): + result = { + f'database._adapters.{name}': SimpleNamespace(Adapter=ConnectionAdapter) + for name in ('mysql', 'mongodb', 'redis') + } + result.update( + { + f'database.crud._{name}': SimpleNamespace(Adapter=RecordAdapter, AsyncAdapter=AsyncRecordAdapter) + for name in ('mysql', 'mongodb', 'redis_hash') + } + ) + return result + + +def connections(): + return Connections( + { + 'orders': ConnectionConfig(backend='mysql', host='explicit.invalid'), + 'documents': ConnectionConfig(backend='mongodb', host='explicit.invalid'), + 'cache': ConnectionConfig(backend='redis', host='explicit.invalid'), + } + ) + + +def bindings(factory, owners): + return ( + factory.mysql(owners[0], table='orders', key_fields=('id',)), + factory.mysql(owners[1], table='orders', key_fields=('id',)), + factory.mongodb(owners[2], database='app', collection='items'), + factory.redis_hash(owners[3], namespace='orders', key_fields=('id',), codec=object()), + ) + + +class ConnectionRecordsIntegrationTests(unittest.TestCase): + def test_multiple_backends_and_same_config_keep_native_identity_and_ownership(self): + with patch.dict('sys.modules', modules()), ExitStack() as stack: + registry = connections() + owners = [stack.enter_context(registry.create(name)) for name in ('orders', 'orders', 'documents', 'cache')] + natives = [owner.native for owner in owners] + self.assertEqual(len({id(native) for native in natives}), 4) + with ExitStack() as records_stack: + records = [records_stack.enter_context(value) for value in bindings(Records, owners)] + for record, owner in zip(records, owners): + key = {'_id': 1} if owner.backend == 'mongodb' else {'id': 1} + self.assertEqual(record.get(key), {'owner': owner.name}) + owners[0].close() + with self.assertRaises(ConnectionClosedError): + records[0].get({'id': 1}) + self.assertEqual(records[1].get({'id': 1}), {'owner': 'orders'}) + self.assertTrue(all(not native.closed for native in natives[1:])) + self.assertTrue(all(record._adapter.closed for record in records)) + self.assertTrue(all(not native.closed for native in natives[1:])) + self.assertIs(owners[2].native, natives[2]) + self.assertTrue(all(native.closed for native in natives)) + + +class AsyncConnectionRecordsIntegrationTests(unittest.IsolatedAsyncioTestCase): + async def test_multiple_async_records_release_only_owned_resources(self): + with patch.dict('sys.modules', modules()): + async with AsyncExitStack() as stack: + registry = connections() + owners = [ + await stack.enter_async_context(await registry.create_async(name)) + for name in ('orders', 'orders', 'documents', 'cache') + ] + natives = [owner.native for owner in owners] + self.assertEqual(len({id(native) for native in natives}), 4) + async with AsyncExitStack() as records_stack: + records = [ + await records_stack.enter_async_context(value) for value in bindings(AsyncRecords, owners) + ] + for record, owner in zip(records, owners): + key = {'_id': 1} if owner.backend == 'mongodb' else {'id': 1} + self.assertEqual(await record.get(key), {'owner': owner.name}) + await owners[0].close() + with self.assertRaises(ConnectionClosedError): + await records[0].get({'id': 1}) + self.assertEqual(await records[1].get({'id': 1}), {'owner': 'orders'}) + self.assertTrue(all(record._adapter.closed for record in records)) + self.assertTrue(all(not native.closed for native in natives[1:])) + self.assertIs(owners[2].native, natives[2]) + self.assertTrue(all(native.closed for native in natives)) -- Gitee From e7326058152f9036c317395b800dbfa44e218b3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:35:55 +0800 Subject: [PATCH 52/86] =?UTF-8?q?feat(crud):=20=E5=A2=9E=E5=8A=A0=E5=8F=AF?= =?UTF-8?q?=E6=98=BE=E5=BC=8F=E9=80=89=E6=8B=A9=E7=9A=84=20Redis=20Hash=20?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E7=BC=96=E8=A7=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_codecs.py | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 src/database/crud/_codecs.py diff --git a/src/database/crud/_codecs.py b/src/database/crud/_codecs.py new file mode 100644 index 0000000..35cb9d2 --- /dev/null +++ b/src/database/crud/_codecs.py @@ -0,0 +1,52 @@ +"""Field codecs used by the Redis Hash record adapter.""" + +import json +import math +from typing import Any +from typing import Protocol + + +class FieldCodec(Protocol): + """Encode a field as Redis data and decode the reply from the native client.""" + + def encode(self, value: Any) -> str | bytes: + """Return the wire representation for a field value.""" + ... + + def decode(self, value: str | bytes) -> Any: + """Return the field value from a native Redis reply.""" + ... + + +def _check_json_value(value: Any) -> None: + kind = type(value) + if value is None or kind in (str, bool, int): + return + if kind is float and math.isfinite(value): + return + if kind is list: + for item in value: + _check_json_value(item) + return + if kind is dict and all(type(key) is str for key in value): + for item in value.values(): + _check_json_value(item) + return + raise ValueError('Expected a JSON value with finite numbers and string object keys.') + + +class JsonFieldCodec: + """Store standard JSON fields in UTF-8 bytes. + + Accept ordinary JSON types and finite numbers. Decoding accepts the + native client's UTF-8 bytes or decoded string, including JSON null. + """ + + def encode(self, value: Any) -> bytes: + _check_json_value(value) + return json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(',', ':')).encode('ascii') + + def decode(self, value: str | bytes) -> Any: + decoded = json.loads(value) + _check_json_value(decoded) + return decoded -- Gitee From 32c990422a3fdd972b5e8d089b13932dc7f33d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:35:56 +0800 Subject: [PATCH 53/86] =?UTF-8?q?feat(crud):=20=E5=AE=9E=E7=8E=B0=E5=8E=9F?= =?UTF-8?q?=E5=AD=90=20Redis=20Hash=20=E8=AE=B0=E5=BD=95=E6=93=8D=E4=BD=9C?= =?UTF-8?q?=E4=B8=8E=E6=89=B9=E9=87=8F=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_redis_hash.py | 502 ++++++++++++++++++++++++++++++ tests/test_crud_redis.py | 512 +++++++++++++++++++++++++++++++ 2 files changed, 1014 insertions(+) create mode 100644 src/database/crud/_redis_hash.py create mode 100644 tests/test_crud_redis.py diff --git a/src/database/crud/_redis_hash.py b/src/database/crud/_redis_hash.py new file mode 100644 index 0000000..a2a20d0 --- /dev/null +++ b/src/database/crud/_redis_hash.py @@ -0,0 +1,502 @@ +"""Atomic single-key CRUD for managed Redis Hash records.""" + +import json +from asyncio import CancelledError +from base64 import urlsafe_b64encode +from contextlib import asynccontextmanager +from contextlib import contextmanager +from dataclasses import replace +from typing import Any + +from redis import exceptions + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DriverError +from ..errors import attach_cleanup_errors +from ._codecs import FieldCodec +from ._errors import RecordConflictError +from ._errors import RecordFormatError +from ._types import UNSET +from ._types import BatchItem +from ._types import BatchState +from ._types import ChunkError +from ._types import ChunkResult +from ._types import ReadItem +from ._types import WriteCounts +from ._types import WriteInput +from ._types import WriteResult + +_MARKER = '__database_record__' +_MAX_TTL_SECONDS = 2**53 - 1 + +# Replies begin with 0 (missing), 1 (success), 2 (wrong format), or 3 (conflict). +# HMGET retains false placeholders in Lua (converted back to null by Redis). +_SCRIPT = """ +local key = KEYS[1] +local operation = ARGV[1] +local qualifier = ARGV[2] +local marker = '__database_record__' + +if redis.call('EXISTS', key) == 0 then + if operation ~= 'insert' then + return {0} + end +else + if redis.call('HGET', key, marker) ~= '1' then + return {2} + end + if operation == 'insert' then + return {3} + end +end + +if operation == 'get' then + if qualifier == 'all' then + return {1, redis.call('HGETALL', key)} + end + local values = {} + for index = 3, #ARGV, 512 do + local last = math.min(index + 511, #ARGV) + local page = redis.call('HMGET', key, unpack(ARGV, index, last)) + for _, value in ipairs(page) do + values[#values + 1] = value + end + end + return {1, values} +end + +if operation == 'delete' then + return {redis.call('DEL', key)} +end + +if operation == 'insert' then + redis.call('HSET', key, marker, '1') +end +-- Limit the Lua unpack size for wide records while keeping one atomic script. +for index = 3, #ARGV, 512 do + local last = math.min(index + 511, #ARGV) + redis.call('HSET', key, unpack(ARGV, index, last)) +end +if qualifier == 'persist' then + redis.call('PERSIST', key) +elseif qualifier ~= 'keep' then + redis.call('EXPIRE', key, qualifier) +end +return {1} +""" + + +def _key_segment(value: bytes) -> bytes: + return urlsafe_b64encode(value).rstrip(b'=') + + +def _queue(pipe, commands) -> None: + # The public eval() method on synchronous ClusterPipeline is a stub. + for command in commands: + pipe.execute_command('EVAL', _SCRIPT, 1, *command) + + +def _pending_chunk(keys, sent: bool) -> ChunkResult: + state = BatchState.UNKNOWN if sent else BatchState.NOT_SENT + return ChunkResult(tuple(BatchItem(index, key, state) for index, key in enumerate(keys))) + + +class _HashRecords: + def __init__( + self, + *, + connection: Connection | AsyncConnection, + key_fields: tuple[str, ...], + namespace: str, + codec: FieldCodec, + ) -> None: + self._connection = connection + self._key_fields = key_fields + self._codec = codec + self._client = None + self._sentinel = connection.mode == 'sentinel' + if not isinstance(namespace, str) or not namespace.strip(): + raise self._error(ConfigurationError, 'Redis Hash namespace must be nonempty.', phase='config') + if _MARKER in key_fields: + raise self._error(ConfigurationError, 'A Redis Hash key field uses the reserved marker.', phase='config') + if not callable(getattr(codec, 'encode', None)) or not callable(getattr(codec, 'decode', None)): + raise self._error(ConfigurationError, 'Redis Hash requires an encode/decode field codec.', phase='config') + try: + self._prefix = b'database:hash:v1:' + _key_segment(namespace.encode('utf-8')) + b':' + except UnicodeError as original: + raise self._error( + ConfigurationError, 'Redis Hash namespace must be valid UTF-8.', phase='config', original=original + ) from original + + def _error(self, kind, message, *, operation=None, phase='operation', original=None): + return kind( + message, + connection_name=self._connection.name, + backend='redis', + phase=phase, + operation=operation, + original=original, + ) + + def _open_client(self) -> None: + native = self._connection.native + if self._sentinel: + service_name = self._connection._adapter._config.service_name + native = native.master_for(service_name) + self._client = native + + def prepare_key(self, key: dict[str, Any]) -> bytes: + """Return this record's canonical Redis key.""" + identity = [[name, key[name]] for name in self._key_fields] + if any(type(value) not in (str, int) for _, value in identity): + raise self._error(ConfigurationError, 'Redis Hash key values must be strings or integers.', phase='config') + try: + encoded = json.dumps(identity, ensure_ascii=True, separators=(',', ':')).encode('ascii') + except ValueError as original: + raise self._error( + ConfigurationError, 'Could not encode the Redis Hash key.', phase='config', original=original + ) from original + return self._prefix + _key_segment(encoded) + + def _field_bytes(self, name: str) -> bytes: + if name == _MARKER: + raise self._error(ConfigurationError, 'The Redis Hash marker is reserved.', phase='config') + try: + return name.encode('utf-8') + except UnicodeError as original: + raise self._error( + ConfigurationError, 'Redis Hash field names must be valid UTF-8.', phase='config', original=original + ) from original + + def prepare_write(self, operation: str, item: WriteInput) -> WriteInput: + """Validate and encode input before any chunk is sent.""" + if item.key is None: + raise self._error(ConfigurationError, 'Redis Hash inserts require a complete key.', phase='config') + key = self.prepare_key(item.key) + if operation == 'delete': + return replace(item, prepared=(key, operation, b'keep')) + ttl_s = item.ttl_s + if ttl_s is UNSET: + ttl = b'keep' + elif ttl_s is None: + ttl = b'keep' if operation == 'insert' else b'persist' + elif type(ttl_s) is int and 1 <= ttl_s <= _MAX_TTL_SECONDS: + # Pass integer seconds verbatim to Redis without a Lua float conversion. + ttl = str(ttl_s).encode('ascii') + else: + raise self._error( + ConfigurationError, + 'Redis Hash ttl_s must be a positive integer within the Redis expiration range.', + phase='config', + ) + command = [key, operation, ttl] + for name, value in item.values.items(): + field = self._field_bytes(name) + try: + data = self._codec.encode(value) + if isinstance(data, str): + data = data.encode('utf-8') + if not isinstance(data, bytes): + raise TypeError('Expected encoded field data as bytes or str.') + except Exception as original: + raise self._error( + ConfigurationError, 'Could not encode a Redis Hash field.', phase='config', original=original + ) from original + command.extend((field, data)) + return replace(item, prepared=tuple(command)) + + def _write_command(self, operation: str, item: WriteInput) -> tuple: + return item.prepared if item.prepared is not None else self.prepare_write(operation, item).prepared + + def _read_command(self, key, fields) -> tuple: + encoded_key = self.prepare_key(key) + if fields is None: + return encoded_key, 'get', b'all' + return (encoded_key, 'get', b'fields', *(self._field_bytes(name) for name in fields)) + + def _unexpected_reply(self, operation): + return self._error(DriverError, 'Unexpected Redis Hash script reply.', operation=operation) + + def _status(self, reply, operation) -> int: + if not isinstance(reply, (list, tuple)) or not reply or type(reply[0]) is not int: + raise self._unexpected_reply(operation) + if reply[0] == 2: + raise self._error(RecordFormatError, 'Invalid Redis Hash record marker.', operation=operation) + if reply[0] == 3 and operation == 'insert': + raise self._error(RecordConflictError, 'Redis Hash record already exists.', operation=operation) + if reply[0] not in (0, 1): + raise self._unexpected_reply(operation) + return reply[0] + + def _read_reply(self, reply, fields, operation) -> dict[str, Any] | None: + if self._status(reply, operation) == 0: + if len(reply) != 1: + raise self._unexpected_reply(operation) + return None + if len(reply) != 2 or not isinstance(reply[1], (list, tuple)): + raise self._unexpected_reply(operation) + values = reply[1] + if fields is None: + if len(values) % 2: + raise self._unexpected_reply(operation) + pairs = zip(values[::2], values[1::2]) + else: + if len(values) != len(fields): + raise self._unexpected_reply(operation) + pairs = zip(fields, values) + + try: + record = {} + for field, value in pairs: + if isinstance(field, bytes): + field = field.decode('utf-8') + if field != _MARKER and value is not None: + record[field] = self._codec.decode(value) + return record + except Exception as original: + raise self._error( + RecordFormatError, 'Could not decode a Redis Hash field.', operation=operation, original=original + ) from original + + def _write_reply(self, reply, operation) -> WriteResult: + status = self._status(reply, operation) + if len(reply) != 1 or (operation == 'insert' and status == 0): + raise self._unexpected_reply(operation) + return WriteResult( + 'redis', + operation, + acknowledged=True, + inserted_count=status if operation == 'insert' else None, + matched_count=status if operation == 'update' else None, + modified_count=0 if operation == 'update' and status == 0 else None, + deleted_count=status if operation == 'delete' else None, + native_result=reply, + ) + + def _batch_reply(self, replies, keys, operation, fields) -> ChunkResult: + if not isinstance(replies, (list, tuple)): + raise ChunkError(self._unexpected_reply(operation), _pending_chunk(keys, True)) + first_error = None + items = [] + for index, key in enumerate(keys): + if index >= len(replies): + item = BatchItem(index, key, BatchState.UNKNOWN, error=self._unexpected_reply(operation)) + else: + item = self._batch_item(replies[index], index, key, operation, fields) + if first_error is None or ( + isinstance(item.error, CancelledError) and not isinstance(first_error, CancelledError) + ): + first_error = item.error + items.append(item) + + if first_error is None and len(replies) != len(keys): + first_error = self._unexpected_reply(operation) + result = ChunkResult( + tuple(items), self._counts(items) if first_error is None else WriteCounts(), native_results=(replies,) + ) + if first_error is not None: + raise ChunkError(first_error, result) from first_error + return result + + def _batch_item(self, reply, index, key, operation, fields) -> BatchItem: + if isinstance(reply, BaseException): + # Only an explicit negative server reply proves failure. Transport, + # routing and other driver failures may follow successful execution. + state = BatchState.UNKNOWN + if isinstance( + reply, (exceptions.ResponseError, exceptions.AuthenticationError, exceptions.AuthorizationError) + ): + state = BatchState.FAILED + return BatchItem(index, key, state, error=reply) + try: + if operation == 'get_many': + result = ReadItem(index, key, self._read_reply(reply, fields, operation)) + else: + result = self._write_reply(reply, operation) + except (RecordConflictError, RecordFormatError) as error: + return BatchItem(index, key, BatchState.FAILED, error=error) + except DriverError as error: + return BatchItem(index, key, BatchState.UNKNOWN, error=error) + return BatchItem(index, key, BatchState.ACKNOWLEDGED, result) + + @staticmethod + def _counts(items: list[BatchItem]) -> WriteCounts: + if not items or not isinstance(items[0].result, WriteResult): + return WriteCounts() + totals = {} + for field in ('inserted_count', 'matched_count', 'modified_count', 'deleted_count'): + values = [getattr(item.result, field) for item in items] + if all(value is not None for value in values): + totals[field] = sum(values) + return WriteCounts(**totals) + + @staticmethod + def _cleanup_primary(primary, cleanup) -> None: + if primary is None: + raise cleanup + original = primary.original if isinstance(primary, ChunkError) else primary + attach_cleanup_errors(original, (cleanup,)) + + +class Adapter(_HashRecords): + """Synchronous Redis Hash adapter.""" + + def __enter__(self): + self._open_client() + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + client, self._client = self._client, None + if self._sentinel: + adapter = self._connection._adapter + errors = adapter._close_single(client, True, target='sentinel_master') + adapter._finish_cleanup(errors, exc) + + def get(self, key, fields): + reply = self._client.execute_command('EVAL', _SCRIPT, 1, *self._read_command(key, fields)) + return self._read_reply(reply, fields, 'get') + + def insert(self, item: WriteInput) -> WriteResult: + return self._write('insert', item) + + def update(self, item: WriteInput) -> WriteResult: + return self._write('update', item) + + def delete(self, item: WriteInput) -> WriteResult: + return self._write('delete', item) + + def _write(self, operation: str, item: WriteInput) -> WriteResult: + reply = self._client.execute_command('EVAL', _SCRIPT, 1, *self._write_command(operation, item)) + return self._write_reply(reply, operation) + + def get_many(self, keys, fields) -> list[dict[str, Any] | None]: + commands = [self._read_command(key, fields) for key in keys] + result = self._run_batch(commands, keys, 'get_many', fields) + return [item.result.record for item in result.items] + + def insert_many(self, items) -> ChunkResult: + return self._write_many('insert', items) + + def bulk_update(self, items) -> ChunkResult: + return self._write_many('update', items) + + def delete_many(self, items) -> ChunkResult: + return self._write_many('delete', items) + + def _write_many(self, operation, items) -> ChunkResult: + commands = [self._write_command(operation, item) for item in items] + return self._run_batch(commands, [item.key for item in items], operation, None) + + @contextmanager + def _pipeline(self, operation): + pipeline = self._client.pipeline(transaction=False) + primary = None + try: + yield pipeline + except BaseException as error: + primary = error + raise + finally: + try: + with self._connection._context('close', operation=operation): + pipeline.__exit__(None, None, None) + except BaseException as cleanup: + self._cleanup_primary(primary, cleanup) + + def _run_batch(self, commands, keys, operation, fields) -> ChunkResult: + sent = False + result = None + try: + with self._pipeline(operation) as pipe: + _queue(pipe, commands) + sent = True + result = self._batch_reply(pipe.execute(raise_on_error=False), keys, operation, fields) + except ChunkError: + raise + except (Exception, CancelledError) as original: + raise ChunkError(original, result or _pending_chunk(keys, sent)) from original + return result + + +class AsyncAdapter(_HashRecords): + """Asynchronous Redis Hash adapter.""" + + async def __aenter__(self): + self._open_client() + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + client, self._client = self._client, None + if self._sentinel: + adapter = self._connection._adapter + errors = await adapter._close_single_async(client, True, target='sentinel_master') + adapter._finish_cleanup(errors, exc) + + async def get(self, key, fields): + reply = await self._client.execute_command('EVAL', _SCRIPT, 1, *self._read_command(key, fields)) + return self._read_reply(reply, fields, 'get') + + async def insert(self, item: WriteInput) -> WriteResult: + return await self._write('insert', item) + + async def update(self, item: WriteInput) -> WriteResult: + return await self._write('update', item) + + async def delete(self, item: WriteInput) -> WriteResult: + return await self._write('delete', item) + + async def _write(self, operation: str, item: WriteInput) -> WriteResult: + reply = await self._client.execute_command('EVAL', _SCRIPT, 1, *self._write_command(operation, item)) + return self._write_reply(reply, operation) + + async def get_many(self, keys, fields) -> list[dict[str, Any] | None]: + commands = [self._read_command(key, fields) for key in keys] + result = await self._run_batch(commands, keys, 'get_many', fields) + return [item.result.record for item in result.items] + + async def insert_many(self, items) -> ChunkResult: + return await self._write_many('insert', items) + + async def bulk_update(self, items) -> ChunkResult: + return await self._write_many('update', items) + + async def delete_many(self, items) -> ChunkResult: + return await self._write_many('delete', items) + + async def _write_many(self, operation, items) -> ChunkResult: + commands = [self._write_command(operation, item) for item in items] + return await self._run_batch(commands, [item.key for item in items], operation, None) + + @asynccontextmanager + async def _pipeline(self, operation): + pipeline = self._client.pipeline(transaction=False) + primary = None + try: + yield pipeline + except BaseException as error: + primary = error + raise + finally: + try: + with self._connection._context('close', operation=operation): + # Avoid extra initialization from __aenter__. redis-py 5.0's + # async ClusterPipeline.reset is the Redis RESET command. + await pipeline.__aexit__(None, None, None) + except BaseException as cleanup: + self._cleanup_primary(primary, cleanup) + + async def _run_batch(self, commands, keys, operation, fields) -> ChunkResult: + sent = False + result = None + try: + async with self._pipeline(operation) as pipe: + _queue(pipe, commands) + sent = True + result = self._batch_reply(await pipe.execute(raise_on_error=False), keys, operation, fields) + except ChunkError: + raise + except (Exception, CancelledError) as original: + raise ChunkError(original, result or _pending_chunk(keys, sent)) from original + return result diff --git a/tests/test_crud_redis.py b/tests/test_crud_redis.py new file mode 100644 index 0000000..b7af54b --- /dev/null +++ b/tests/test_crud_redis.py @@ -0,0 +1,512 @@ +"""Verify Redis Hash wire commands, result state, and native ownership.""" + +import asyncio +import json +import unittest +from base64 import urlsafe_b64decode +from contextlib import ExitStack +from decimal import Decimal +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import Mock +from unittest.mock import call +from unittest.mock import patch + +import redis + +from database import ConnectionConfig +from database import Connections +from database.crud import AsyncRecords +from database.crud import BatchOperationError +from database.crud import Records +from database.crud._codecs import JsonFieldCodec +from database.crud._errors import RecordConflictError +from database.crud._errors import RecordFormatError +from database.crud._errors import UnsupportedOperationError +from database.crud._redis_hash import _SCRIPT +from database.crud._redis_hash import Adapter +from database.crud._redis_hash import AsyncAdapter +from database.crud._types import UNSET +from database.crud._types import BatchState +from database.crud._types import ChunkError +from database.crud._types import WriteInput +from database.errors import ConfigurationError + + +def _open(backend='single', **kwargs): + if backend == 'sentinel': + kwargs.update(nodes=[('sentinel.test', 26379)], service_name='main') + else: + kwargs.setdefault('host', 'redis.test') + return Connections({'cache': ConnectionConfig('redis', mode=backend, **kwargs)}) + + +def _bind(connection, is_async=False, *, key_fields=('id',), namespace='orders', codec=None): + adapter_type = AsyncAdapter if is_async else Adapter + return adapter_type( + connection=connection, + key_fields=key_fields, + namespace=namespace, + codec=JsonFieldCodec() if codec is None else codec, + ) + + +def _write(identity, values=None, ttl_s=UNSET): + return WriteInput({'id': identity}, {'id': identity} if values is None else values, ttl_s) + + +def _encoded(segment): + return urlsafe_b64decode(segment + b'=' * (-len(segment) % 4)) + + +class RedisFieldCodecTests(unittest.TestCase): + def test_json_codec_preserves_nested_values_and_explicit_null(self): + value = {'city': '上海', 'active': True, 'details': [None, 2**60, 0.75]} + codec = JsonFieldCodec() + encoded = codec.encode(value) + self.assertEqual(codec.encode(None), b'null') + self.assertEqual(codec.decode(encoded), value) + self.assertEqual(codec.decode(encoded.decode('ascii')), value) + + def test_json_codec_rejects_types_and_nonfinite_numbers_that_lose_information(self): + codec = JsonFieldCodec() + for value in (Decimal('2.0'), b'bytes', (1, 2), {'nested': {3: 'a'}}, float('nan'), [float('inf')]): + with self.subTest(value=value): + with self.assertRaises(ValueError): + codec.encode(value) + for wire in (b'NaN', b'1e999', 'Infinity'): + with self.subTest(wire=wire): + with self.assertRaises(ValueError): + codec.decode(wire) + + +class RedisHashTests(unittest.TestCase): + def setUp(self): + self.connection = _open().create('cache') + self.addCleanup(self.connection.close) + + def test_binding_has_no_network_and_key_is_canonical_and_typed(self): + with patch.object(self.connection.native, 'execute_command') as execute: + records = _bind(self.connection, key_fields=('shop', 'id'), namespace='order:{用户}') + fields_a = {'shop': 'a:1:b', 'id': 1} + fields_b = {'id': 1, 'shop': 'a:1:b'} + native_key = records.prepare_key(fields_a) + self.assertEqual(records.prepare_key(fields_b), native_key) + self.assertNotEqual(records.prepare_key({'shop': 'a:1:b', 'id': '1'}), native_key) + self.assertNotEqual(records.prepare_key({'shop': 'a', 'id': '1:b:1'}), native_key) + self.assertNotIn(b'{', native_key) + self.assertNotIn(b'}', native_key) + _, _, _, namespace, identity = native_key.split(b':') + self.assertEqual(_encoded(namespace).decode('utf-8'), 'order:{用户}') + self.assertEqual(json.loads(_encoded(identity)), [['shop', 'a:1:b'], ['id', 1]]) + with self.assertRaises(ConfigurationError): + records.prepare_key({'shop': 'a', 'id': True}) + self.assertNotEqual( + _bind(self.connection, key_fields=('shop', 'id'), namespace='other').prepare_key(fields_a), + native_key, + ) + execute.assert_not_called() + + def test_reserved_field_is_rejected_for_binding_writes_and_projection(self): + with patch.object(self.connection.native, 'execute_command') as execute: + with self.assertRaises(ConfigurationError): + _bind(self.connection, key_fields=('__database_record__',)) + with _bind(self.connection) as records: + for operation, outgoing in ( + ('insert', _write(1, {'id': 1, '__database_record__': 1})), + ('update', _write(1, {'__database_record__': 1})), + ): + with self.subTest(operation=operation): + with self.assertRaises(ConfigurationError): + records.prepare_write(operation, outgoing) + with self.assertRaises(ConfigurationError): + records.get({'id': 1}, ('__database_record__',)) + execute.assert_not_called() + + def test_projected_slotted_replies_distinguish_missing_null_and_presence_only(self): + with ( + patch.object( + self.connection.native, + 'execute_command', + side_effect=[ + [0], + [1, [None, b'null', '"ready"']], + [1, []], + [1, [b'__database_record__', b'1', b'id', b'1', b'name', b'"test"']], + ], + ) as execute, + _bind(self.connection) as records, + ): + key = {'id': 1} + self.assertIsNone(records.get(key, ('status',))) + self.assertEqual( + records.get(key, ('missing', 'empty', 'status')), + {'empty': None, 'status': 'ready'}, + ) + self.assertEqual(records.get(key, ()), {}) + self.assertEqual(records.get(key, None), {'id': 1, 'name': 'test'}) + native_key = records.prepare_key(key) + self.assertEqual( + execute.call_args_list, + [ + call('EVAL', _SCRIPT, 1, native_key, 'get', b'fields', b'status'), + call('EVAL', _SCRIPT, 1, native_key, 'get', b'fields', b'missing', b'empty', b'status'), + call('EVAL', _SCRIPT, 1, native_key, 'get', b'fields'), + call('EVAL', _SCRIPT, 1, native_key, 'get', b'all'), + ], + ) + + def test_write_counts_expire_modes_and_script_status(self): + scripts = [[1], [1], [0], [1], [3], [2], [1], [0]] + with ( + patch.object(self.connection.native, 'execute_command', side_effect=scripts) as execute, + _bind(self.connection) as records, + ): + created = records.insert(_write(1, ttl_s=10)) + self.assertEqual(created.inserted_count, 1) + self.assertIs(created.native_result, scripts[0]) + self.assertEqual(execute.call_args.args[-3:], (b'10', b'id', b'1')) + updated = records.update(_write(1, {'status': 'a'})) + self.assertEqual(updated.matched_count, 1) + self.assertIsNone(updated.modified_count) + self.assertEqual(execute.call_args.args[-3:], (b'keep', b'status', b'"a"')) + missing = records.update(_write(8, {'status': 1}, ttl_s=20)) + self.assertEqual((missing.matched_count, missing.modified_count), (0, 0)) + self.assertEqual(execute.call_args.args[-3], b'20') + records.update(_write(1, {'status': 'b'}, ttl_s=None)) + self.assertEqual(execute.call_args.args[-3], b'persist') + with self.assertRaises(RecordConflictError): + records.insert(_write(1)) + with self.assertRaises(RecordFormatError): + records.delete(_write(8, {})) + self.assertEqual(records.delete(_write(1, {})).deleted_count, 1) + self.assertEqual(records.delete(_write(8, {})).deleted_count, 0) + + def test_prepare_all_batch_fields_before_opening_a_pipeline(self): + with patch.object(self.connection.native, 'pipeline') as pipeline, _bind(self.connection) as records: + with self.assertRaises(ConfigurationError): + records.insert_many([_write(1), _write(2, {'id': 2, 'cost': Decimal('2.99')})]) + for ttl in (0, -1, True, 1.0, 2**63): + with self.subTest(ttl=ttl): + with self.assertRaises(ConfigurationError): + records.prepare_write('insert', _write(1, ttl_s=ttl)) + pipeline.assert_not_called() + + def test_custom_field_codec_preserves_binary_data_and_types(self): + codec = SimpleNamespace(encode=Mock(return_value=b'\x00\xff'), decode=Mock(return_value=Decimal('0.10'))) + with ( + patch.object( + self.connection.native, + 'execute_command', + side_effect=[ + [1], + [1, [b'\x00\xff']], + ], + ) as execute, + _bind(self.connection, codec=codec) as records, + ): + source = Decimal('0.10') + records.update(_write(1, {'balance': source})) + codec.encode.assert_called_once_with(source) + self.assertEqual(execute.call_args.args[-1], b'\x00\xff') + self.assertEqual(records.get({'id': 1}, ('balance',)), {'balance': source}) + codec.decode.assert_called_once_with(b'\x00\xff') + + def test_corrupt_payload_retains_decoder_cause(self): + with ( + patch.object(self.connection.native, 'execute_command', return_value=[1, [b'{?']]), + _bind(self.connection) as records, + ): + with self.assertRaises(RecordFormatError) as failed: + records.get({'id': 1}, ('x',)) + self.assertIsInstance(failed.exception.original, json.JSONDecodeError) + self.assertIs(failed.exception.__cause__, failed.exception.original) + + def test_driver_reply_list_proves_items_independently(self): + pipe = Mock() + denied = redis.exceptions.NoPermissionError('denied') + disconnected = redis.exceptions.ConnectionError('connection dropped') + replies = [[1], denied, [0], disconnected] + pipe.execute.return_value = replies + pipe.__exit__ = Mock() + with ( + patch.object(self.connection.native, 'pipeline', return_value=pipe) as factory, + _bind(self.connection) as records, + ): + with self.assertRaises(ChunkError) as failed: + records.delete_many([_write(index, {}) for index in range(1, 5)]) + keys = [records.prepare_key({'id': index}) for index in range(1, 5)] + partial = failed.exception.partial_result + self.assertIs(failed.exception.original, denied) + self.assertIs(partial.native_results[0], replies) + self.assertEqual( + [item.state for item in partial.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.FAILED, + BatchState.ACKNOWLEDGED, + BatchState.UNKNOWN, + ], + ) + self.assertEqual(partial.items[0].result.deleted_count, 1) + self.assertEqual(partial.items[2].result.deleted_count, 0) + self.assertIs(partial.items[1].error, denied) + self.assertIs(partial.items[3].error, disconnected) + factory.assert_called_once_with(transaction=False) + self.assertEqual( + [args.args[:4] for args in pipe.execute_command.call_args_list], [('EVAL', _SCRIPT, 1, key) for key in keys] + ) + pipe.execute.assert_called_once_with(raise_on_error=False) + pipe.__exit__.assert_called_once_with(None, None, None) + + def test_queue_error_leaves_chunk_not_sent_and_execute_error_is_unknown(self): + error = redis.exceptions.ConnectionError('transient') + for step, state in (('execute_command', BatchState.NOT_SENT), ('execute', BatchState.UNKNOWN)): + pipe = Mock() + pipe.__exit__ = Mock() + getattr(pipe, step).side_effect = error + with ( + self.subTest(step=step), + patch.object(self.connection.native, 'pipeline', return_value=pipe), + _bind(self.connection) as records, + ): + with self.assertRaises(ChunkError) as failed: + records.insert_many([_write(1), _write(2)]) + self.assertIs(failed.exception.original, error) + self.assertEqual([item.state for item in failed.exception.partial_result.items], [state, state]) + pipe.__exit__.assert_called_once() + if step == 'execute_command': + pipe.execute.assert_not_called() + + def test_partial_read_saves_only_confirmed_observations(self): + missing = [0] + found = [1, [b'null']] + corrupt = [2] + pipe = Mock() + pipe.__exit__ = Mock() + pipe.execute.return_value = [missing, corrupt, found] + with patch.object(self.connection.native, 'pipeline', return_value=pipe), _bind(self.connection) as records: + with self.assertRaises(ChunkError) as failed: + records.get_many([{'id': index} for index in (1, 2, 3, 4)], ('field',)) + items = failed.exception.partial_result.items + self.assertIsNone(items[0].result.record) + self.assertIsInstance(items[1].error, RecordFormatError) + self.assertEqual(items[2].result.record, {'field': None}) + self.assertIsNone(items[3].result) + self.assertIs(items[3].state, BatchState.UNKNOWN) + + def test_scope_exits_without_closing_borrowed_native(self): + with ( + patch.object(self.connection.native, 'close') as native_close, + Records.redis_hash( + self.connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records, + ): + calls = ( + ('find', lambda: records.find(limit=1)), + ('count', records.count), + ('update_many', lambda: records.update_many(where={'id': 1}, set_values={'x': 1})), + ('delete_where', lambda: records.delete_where(where={'id': 1})), + ) + for operation, invoke in calls: + with self.subTest(operation=operation): + with self.assertRaises(UnsupportedOperationError) as failed: + invoke() + self.assertEqual(failed.exception.operation, operation) + native_close.assert_not_called() + self.assertFalse(self.connection.closed) + + def test_sentinel_scope_opens_one_owned_master_and_releases_its_pool(self): + with _open('sentinel').create('cache') as connection: + sentinel = connection.native + master = sentinel.master_for('main') + with ( + patch.object(sentinel, 'master_for', return_value=master) as factory, + patch.object(master, 'execute_command', return_value=[0]) as command, + patch.object(master.connection_pool, 'disconnect') as disconnect, + ): + records = _bind(connection) + factory.assert_not_called() + with records: + self.assertIsNone(records.get({'id': 1}, None)) + self.assertIsNone(records.get({'id': 2}, None)) + factory.assert_called_once_with('main') + self.assertEqual(command.call_count, 2) + disconnect.assert_called_once_with() + self.assertIs(connection.native, sentinel) + self.assertFalse(connection.closed) + + +class AsyncRedisHashTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.connection = await _open().create_async('cache') + self.addAsyncCleanup(self.connection.close) + + async def test_async_commands_and_batch_await_native_execution(self): + pipe = Mock() + pipe.__aexit__ = AsyncMock() + pipe.execute = AsyncMock(return_value=[[1], [1], [0]]) + with ( + patch.object( + self.connection.native, + 'execute_command', + new_callable=AsyncMock, + side_effect=[ + [1, [b'null']], + [1], + [0], + [1], + ], + ) as command, + patch.object(self.connection.native, 'pipeline', return_value=pipe), + ): + async with _bind(self.connection, is_async=True) as records: + self.assertEqual(await records.get({'id': 1}, ('field',)), {'field': None}) + self.assertEqual((await records.insert(_write(1))).inserted_count, 1) + self.assertEqual((await records.update(_write(2, {'x': 1}))).matched_count, 0) + self.assertEqual((await records.delete(_write(1, {}))).deleted_count, 1) + result = await records.bulk_update([_write(index, {'x': 1}) for index in (1, 2, 3)]) + self.assertEqual(result.counts.matched_count, 2) + self.assertIsNone(result.counts.modified_count) + self.assertEqual(command.await_count, 4) + self.assertEqual(pipe.execute_command.call_count, 3) + pipe.execute.assert_awaited_once_with(raise_on_error=False) + pipe.__aexit__.assert_awaited_once_with(None, None, None) + self.assertFalse(self.connection.closed) + + async def test_async_batch_cancellation_preserves_original_and_cleanup_error(self): + original = asyncio.CancelledError() + cleanup = redis.exceptions.ConnectionError('cleanup failed') + pipe = Mock() + pipe.__aexit__ = AsyncMock(side_effect=cleanup) + pipe.execute = AsyncMock(side_effect=original) + with patch.object(self.connection.native, 'pipeline', return_value=pipe): + async with _bind(self.connection, is_async=True) as records: + with self.assertRaises(ChunkError) as failed: + await records.insert_many([_write(1)]) + self.assertIs(failed.exception.original, original) + self.assertIs(failed.exception.partial_result.items[0].state, BatchState.UNKNOWN) + self.assertIs(original.database_cleanup_errors[0].original, cleanup) + pipe.__aexit__.assert_awaited_once() + + async def test_async_sentinel_context_closes_owned_pool_and_preserves_primary_error(self): + connection = await _open('sentinel').create_async('cache') + self.addAsyncCleanup(connection.close) + sentinel = connection.native + master = sentinel.master_for('main') + original = asyncio.CancelledError() + cleanup = redis.exceptions.ConnectionError('cleanup failed') + with ( + patch.object(sentinel, 'master_for', return_value=master) as factory, + patch.object(master, 'execute_command', new_callable=AsyncMock, side_effect=original), + patch.object( + master.connection_pool, 'disconnect', new_callable=AsyncMock, side_effect=cleanup + ) as disconnect, + ): + with self.assertRaises(asyncio.CancelledError) as failed: + async with _bind(connection, is_async=True) as records: + await records.get({'id': 1}, None) + self.assertIs(failed.exception, original) + self.assertIs(original.database_cleanup_errors[0].original, cleanup) + factory.assert_called_once_with('main') + disconnect.assert_awaited_once_with() + self.assertIs(connection.native, sentinel) + self.assertFalse(connection.closed) + + +def _cluster_topology(manager, nodes, keys=()): + manager.nodes_cache = {node.name: node for node in nodes} + manager.default_node = nodes[0] + slots = [redis.crc.key_slot(key) for key in keys] + manager.slots_cache = {slot: [node] for slot, node in zip(slots, nodes)} + + +def _assert_cluster_partial(test, result, error): + test.assertIs(result.original, error) + test.assertEqual( + [item.state for item in result.partial_result.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.FAILED, + ], + ) + + +class RedisNativeClusterTests(unittest.IsolatedAsyncioTestCase): + def test_sync_native_cluster_pipeline_executes_scripts_across_slots(self): + clients = [redis.Redis(host=f'primary{index}.test') for index in (1, 2)] + nodes = [ + redis.cluster.ClusterNode(f'primary{index}.test', 7000, server_type='primary', redis_connection=client) + for index, client in enumerate(clients, 1) + ] + error = redis.exceptions.NoPermissionError('denied') + with ExitStack() as cleanup: + cleanup.enter_context( + patch.object( + redis.cluster.NodesManager, + 'initialize', + lambda manager, *args, **opts: _cluster_topology(manager, nodes), + ) + ) + cleanup.enter_context(patch.object(redis.cluster.CommandsParser, 'initialize')) + connection = cleanup.enter_context(_open('cluster').create('cache')) + key_encoder = _bind(connection) + keys = [key_encoder.prepare_key({'id': index}) for index in (1, 2)] + self.assertNotEqual(redis.crc.key_slot(keys[0]), redis.crc.key_slot(keys[1])) + _cluster_topology(connection.native.nodes_manager, nodes, keys) + connection.native.commands_parser.commands['eval'] = {'flags': ['movablekeys']} + sockets = [] + for index, client in enumerate(clients): + socket = Mock() + socket.pack_commands.side_effect = lambda commands: commands + cleanup.enter_context(patch.object(client.connection_pool, 'get_connection', return_value=socket)) + cleanup.enter_context(patch.object(client.connection_pool, 'release')) + # Current redis-py queries COMMAND GETKEYS for EVAL in a sync Pipeline. + cleanup.enter_context( + patch.object( + client, + 'execute_command', + side_effect=lambda command, *args: args[3:4], + ) + ) + reply = {'return_value': [1]} if index == 0 else {'side_effect': error} + cleanup.enter_context(patch.object(client, 'parse_response', **reply)) + sockets.append(socket) + with Records.redis_hash( + connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records: + with self.assertRaises(BatchOperationError) as failure: + records.insert_many([{'id': 1}, {'id': 2}]) + _assert_cluster_partial(self, failure.exception, error) + self.assertEqual([sock.send_packed_command.call_args.args[0][0][3] for sock in sockets], keys) + + async def test_async_native_cluster_pipeline_executes_scripts_across_slots(self): + async with await _open('cluster').create_async('cache') as connection: + key_encoder = _bind(connection, is_async=True) + keys = [key_encoder.prepare_key({'id': index}) for index in (1, 2)] + self.assertNotEqual(redis.crc.key_slot(keys[0]), redis.crc.key_slot(keys[1])) + nodes = [ + redis.asyncio.cluster.ClusterNode(f'primary{index}.test', 7000, server_type='primary') + for index in (1, 2) + ] + _cluster_topology(connection.native.nodes_manager, nodes, keys) + connection.native._initialize = False + error = redis.exceptions.NoPermissionError('denied') + routed = {} + + async def execute(node, commands): + for command in commands: + routed[node.host] = command.args[3] + command.result = [1] if node is nodes[0] else error + return node is nodes[1] + + with patch.object( + redis.asyncio.cluster.ClusterNode, 'execute_pipeline', autospec=True, side_effect=execute + ): + async with AsyncRecords.redis_hash( + connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records: + with self.assertRaises(BatchOperationError) as failure: + await records.insert_many([{'id': 1}, {'id': 2}]) + _assert_cluster_partial(self, failure.exception, error) + self.assertEqual(routed, {node.host: key for node, key in zip(nodes, keys)}) -- Gitee From 87abaae6e6c4be69bd095070dac9fa83fcfe6cc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:42:30 +0800 Subject: [PATCH 54/86] =?UTF-8?q?fix(crud):=20=E5=9C=A8=E6=89=B9=E9=87=8F?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E5=88=86=E5=8F=91=E5=89=8D=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E5=90=8E=E7=AB=AF=E6=8A=95=E5=BD=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/crud-adapter-contract.md | 2 ++ src/database/crud/_records.py | 15 +++++++++++---- src/database/crud/_redis_hash.py | 6 ++++++ tests/test_crud_core.py | 18 ++++++++++++++++++ tests/test_crud_redis.py | 20 ++++++++++++++++++++ 5 files changed, 57 insertions(+), 4 deletions(-) diff --git a/docs/crud-adapter-contract.md b/docs/crud-adapter-contract.md index 88f89e4..e2d048c 100644 --- a/docs/crud-adapter-contract.md +++ b/docs/crud-adapter-contract.md @@ -39,6 +39,8 @@ Redis 不需要实现 `find/count/update_many/delete_where`,公共层会拒绝 公共 `insert_many` 接收普通记录或 `InsertItem(record, ttl_s=...)`,普通记录中的同名业务字段原样保留。 需要在发送前完成编码或类型校验时,实现同步的 `prepare_write(operation, item) -> WriteInput`(`operation` 固定为 `insert/update/delete`),可利用 `dataclasses.replace` 保存准备好的参数。可实现同步 `prepare_key(key) -> None` 验证读取键的编码。两个准备钩子都不访问网络。 公共层在批量发送之前对整批执行结构、键、字段及适配器准备。若后端的静态校验(如字段/值编码)可判断是否能执行,放在准备期间。 +读取投影可实现同步的 `prepare_fields(fields) -> None`,公共层在 `get/get_many/find` 分发前调用,包括整个读取批次的首次发送之前。 +该钩子不访问网络,用于 SQL 标识符、Redis 保留字段、字段编码等后端静态校验;配置错误直接传播,不伪装为已发送批次的未知结果。 `_types` 定义以下共享类型,写结果尽可能保留原生对象。 diff --git a/src/database/crud/_records.py b/src/database/crud/_records.py index 9110d0f..f8d32c4 100644 --- a/src/database/crud/_records.py +++ b/src/database/crud/_records.py @@ -178,12 +178,19 @@ class _Records: prepare_key(values) return values + def _prepare_fields(self, fields: _Fields) -> tuple[str, ...] | None: + projected = _fields(fields) + prepare_fields = getattr(self._adapter, 'prepare_fields', None) + if prepare_fields is not None: + prepare_fields(projected) + return projected + def _read_batch( self, keys: Sequence[_Key], fields: _Fields, batch_size: int ) -> tuple[Batch, list[Record], list[tuple[int, ...]], tuple[str, ...] | None]: self._check_active() _integer(batch_size, 1, 'batch_size') - projected = _fields(fields) + projected = self._prepare_fields(fields) if not is_sequence(keys): raise ConfigurationError('Record keys must be a finite sequence.', phase='config') prepared = [self._key(key) for key in keys] @@ -203,7 +210,7 @@ class _Records: self, where: Where, fields: _Fields, limit: int, offset: int, order_by: _Order ) -> tuple[Condition | None, tuple[str, ...] | None, int, int, tuple[tuple[str, str], ...]]: condition = self._where(where, 'find') - projected = _fields(fields) + projected = self._prepare_fields(fields) _integer(limit, 1, 'limit') _integer(offset, 0, 'offset') if not is_sequence(order_by): @@ -357,7 +364,7 @@ class Records(_Records): def get(self, key: _Key, *, fields: _Fields = None) -> Record | None: self._check_active() - return self._invoke('get', self._key(key), _fields(fields)) + return self._invoke('get', self._key(key), self._prepare_fields(fields)) def get_many(self, keys: Sequence[_Key], *, fields: _Fields = None, batch_size: int = 500) -> list[ReadItem]: batch, unique, positions, projected = self._read_batch(keys, fields, batch_size) @@ -456,7 +463,7 @@ class AsyncRecords(_Records): async def get(self, key: _Key, *, fields: _Fields = None) -> Record | None: self._check_active() - return await self._invoke('get', self._key(key), _fields(fields)) + return await self._invoke('get', self._key(key), self._prepare_fields(fields)) async def get_many(self, keys: Sequence[_Key], *, fields: _Fields = None, batch_size: int = 500) -> list[ReadItem]: batch, unique, positions, projected = self._read_batch(keys, fields, batch_size) diff --git a/src/database/crud/_redis_hash.py b/src/database/crud/_redis_hash.py index a2a20d0..5dea28b 100644 --- a/src/database/crud/_redis_hash.py +++ b/src/database/crud/_redis_hash.py @@ -170,6 +170,12 @@ class _HashRecords: ConfigurationError, 'Redis Hash field names must be valid UTF-8.', phase='config', original=original ) from original + def prepare_fields(self, fields: tuple[str, ...] | None) -> None: + """Reject invalid projection names before a read batch is dispatched.""" + if fields is not None: + for name in fields: + self._field_bytes(name) + def prepare_write(self, operation: str, item: WriteInput) -> WriteInput: """Validate and encode input before any chunk is sent.""" if item.key is None: diff --git a/tests/test_crud_core.py b/tests/test_crud_core.py index 2d63159..ade9e30 100644 --- a/tests/test_crud_core.py +++ b/tests/test_crud_core.py @@ -118,6 +118,7 @@ class RecordsTests(unittest.TestCase): '__exit__', 'get', 'get_many', + 'find', 'update_many', 'delete_where', 'insert', @@ -126,6 +127,7 @@ class RecordsTests(unittest.TestCase): 'delete_many', 'prepare_write', 'prepare_key', + 'prepare_fields', ] ) self.adapter.__enter__ = Mock() @@ -164,6 +166,22 @@ class RecordsTests(unittest.TestCase): self.assertIs(caught.exception.__cause__, error) self.assertEqual(caught.exception.database_cleanup_errors, (cleanup,)) + def test_backend_projection_validation_precedes_read_dispatch(self): + original = ConfigurationError('Backend projection rejected.', phase='config') + self.adapter.prepare_fields.side_effect = original + with self.records: + for operation in ( + lambda: self.records.get({'id': 1}, fields=['bad']), + lambda: self.records.get_many([{'id': 1}, {'id': 2}], fields=['bad'], batch_size=1), + lambda: self.records.find(fields=['bad'], limit=1), + ): + with self.assertRaises(ConfigurationError) as caught: + operation() + self.assertIs(caught.exception, original) + self.adapter.get.assert_not_called() + self.adapter.get_many.assert_not_called() + self.adapter.find.assert_not_called() + def test_get_many_deduplicates_and_keeps_observed_missing_distinct_from_empty_record(self): seen = [] diff --git a/tests/test_crud_redis.py b/tests/test_crud_redis.py index b7af54b..18e09f4 100644 --- a/tests/test_crud_redis.py +++ b/tests/test_crud_redis.py @@ -123,6 +123,17 @@ class RedisHashTests(unittest.TestCase): records.get({'id': 1}, ('__database_record__',)) execute.assert_not_called() + def test_public_batch_projection_error_is_configuration_error_before_pipeline(self): + with patch.object(self.connection.native, 'pipeline') as pipeline: + with Records.redis_hash( + self.connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records: + for field in ('__database_record__', '\ud800'): + with self.subTest(field=repr(field)): + with self.assertRaises(ConfigurationError): + records.get_many([{'id': 1}, {'id': 2}], fields=[field], batch_size=1) + pipeline.assert_not_called() + def test_projected_slotted_replies_distinguish_missing_null_and_presence_only(self): with ( patch.object( @@ -342,6 +353,15 @@ class AsyncRedisHashTests(unittest.IsolatedAsyncioTestCase): self.connection = await _open().create_async('cache') self.addAsyncCleanup(self.connection.close) + async def test_public_async_batch_projection_error_precedes_pipeline(self): + with patch.object(self.connection.native, 'pipeline') as pipeline: + async with AsyncRecords.redis_hash( + self.connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records: + with self.assertRaises(ConfigurationError): + await records.get_many([{'id': 1}, {'id': 2}], fields=['__database_record__'], batch_size=1) + pipeline.assert_not_called() + async def test_async_commands_and_batch_await_native_execution(self): pipe = Mock() pipe.__aexit__ = AsyncMock() -- Gitee From 27c3c33f6b2965716ee93af803206761d7fe8f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:43:32 +0800 Subject: [PATCH 55/86] =?UTF-8?q?feat(mongodb):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=92=8C=E5=BC=82=E6=AD=A5=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_mongodb.py | 518 ++++++++++++++++++++++++++++++++++ tests/test_crud_mongodb.py | 404 ++++++++++++++++++++++++++ 2 files changed, 922 insertions(+) create mode 100644 src/database/crud/_mongodb.py create mode 100644 tests/test_crud_mongodb.py diff --git a/src/database/crud/_mongodb.py b/src/database/crud/_mongodb.py new file mode 100644 index 0000000..e53b7a2 --- /dev/null +++ b/src/database/crud/_mongodb.py @@ -0,0 +1,518 @@ +"""MongoDB record operations on a borrowed collection and optional session.""" + +import asyncio +from collections.abc import AsyncIterator +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import Sequence +from contextlib import asynccontextmanager +from contextlib import contextmanager +from dataclasses import replace +from types import TracebackType +from typing import Any + +from bson import BSON +from bson.errors import BSONError +from pymongo import DeleteOne +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError +from pymongo.errors import DuplicateKeyError +from pymongo.errors import InvalidName + +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import attach_cleanup_errors +from ._batch import Record +from ._conditions import And +from ._conditions import Condition +from ._conditions import Eq +from ._conditions import Exists +from ._conditions import Gt +from ._conditions import Gte +from ._conditions import In +from ._conditions import IsNull +from ._conditions import Lt +from ._conditions import Lte +from ._conditions import Or +from ._errors import RecordConflictError +from ._errors import RecordIdentityError +from ._types import BatchItem +from ._types import BatchState +from ._types import ChunkError +from ._types import ChunkResult +from ._types import ReadItem +from ._types import WriteCounts +from ._types import WriteInput +from ._types import WriteResult + +_Fields = tuple[str, ...] | None +_Order = tuple[tuple[str, str], ...] +_BACKEND = 'mongodb' +_COMPARISONS = {Eq: '$eq', Gt: '$gt', Gte: '$gte', Lt: '$lt', Lte: '$lte'} + + +def _scalar_predicate(operator: str, value: Any) -> Record: + return {operator: value, '$exists': True, '$not': {'$type': 'array'}} + + +def _condition_filter(where: Condition | None) -> Record | None: + """Translate a normalized condition. Return None only for a known empty match.""" + if where is None: + return {} + if isinstance(where, (And, Or)): + filters = [_condition_filter(child) for child in where.conditions] + if isinstance(where, And): + return None if None in filters else {'$and': filters} + alternatives = [clause for clause in filters if clause is not None] + return {'$or': alternatives} if alternatives else None + if isinstance(where, Exists): + return {where.field: {'$exists': True}} + if isinstance(where, IsNull): + return {where.field: _scalar_predicate('$type', 'null')} + if isinstance(where, In): + return {where.field: _scalar_predicate('$in', where.values)} if where.values else None + return {where.field: _scalar_predicate(_COMPARISONS[type(where)], where.value)} + + +def _key_filter(key: Record) -> Record: + return {field: _scalar_predicate('$eq', value) for field, value in key.items()} + + +def _projection(fields: _Fields) -> Record | None: + if fields is None: + return None + if not fields: + return {'_id': 1} + options = dict.fromkeys(fields, 1) + if '_id' not in options: + options['_id'] = 0 + return options + + +def _record(row: Mapping[str, Any], fields: _Fields) -> Record: + if fields is None: + return dict(row) + return {field: row[field] for field in fields if field in row} + + +@contextmanager +def _closing(cursor: Any) -> Iterator[Any]: + try: + yield cursor + except BaseException as error: + try: + cursor.close() + except BaseException as cleanup_error: + attach_cleanup_errors(error, (cleanup_error,)) + raise + else: + cursor.close() + + +@asynccontextmanager +async def _async_closing(cursor: Any) -> AsyncIterator[Any]: + try: + yield cursor + except BaseException as error: + try: + await cursor.close() + except BaseException as cleanup_error: + attach_cleanup_errors(error, (cleanup_error,)) + raise + else: + await cursor.close() + + +@contextmanager +def _read_batch(keys: Sequence[Record]) -> Iterator[list[Record | None]]: + rows: list[Record | None] = [] + try: + yield rows + except (Exception, asyncio.CancelledError) as error: + completed = [ + BatchItem(index, keys[index], BatchState.ACKNOWLEDGED, ReadItem(index, keys[index], row)) + for index, row in enumerate(rows) + ] + unsent = [BatchItem(index, keys[index], BatchState.NOT_SENT) for index in range(len(rows) + 1, len(keys))] + partial = ChunkResult(tuple(completed + unsent)) + if isinstance(error, asyncio.CancelledError): + error.database_partial_result = partial + raise + raise ChunkError(error, partial) from error + + +def _update_result(operation: str, result: Any) -> WriteResult: + return WriteResult( + _BACKEND, + operation, + acknowledged=result.acknowledged, + matched_count=result.matched_count if result.acknowledged else None, + modified_count=result.modified_count if result.acknowledged else None, + native_result=result, + ) + + +def _delete_result(operation: str, result: Any) -> WriteResult: + return WriteResult( + _BACKEND, + operation, + acknowledged=result.acknowledged, + deleted_count=result.deleted_count if result.acknowledged else None, + native_result=result, + ) + + +def _batch_state(acknowledged: bool) -> BatchState: + return BatchState.ACKNOWLEDGED if acknowledged else BatchState.UNACKNOWLEDGED + + +def _bulk_result(operation: str, items: Sequence[WriteInput], result: Any) -> ChunkResult: + counts = WriteCounts() + if result.acknowledged: + if operation == 'bulk_update': + counts = WriteCounts(matched_count=result.matched_count, modified_count=result.modified_count) + else: + counts = WriteCounts(deleted_count=result.deleted_count) + return ChunkResult( + tuple(BatchItem(index, item.key, _batch_state(result.acknowledged)) for index, item in enumerate(items)), + counts, + (result,), + ) + + +def _partial_bulk(operation: str, items: Sequence[WriteInput], error: BulkWriteError) -> ChunkResult: + details = error.details + write_errors = details.get('writeErrors', ()) + indexed = { + entry['index']: entry + for entry in write_errors + if type(entry.get('index')) is int and 0 <= entry['index'] < len(items) + } + counts = WriteCounts() + stop = None + if len(write_errors) == len(indexed) == 1 and not details.get('writeConcernErrors'): + stop = next(iter(indexed)) + reported = { + 'insert_many': {'inserted_count': 'nInserted'}, + 'bulk_update': {'matched_count': 'nMatched', 'modified_count': 'nModified'}, + 'delete_many': {'deleted_count': 'nRemoved'}, + }[operation] + counts = WriteCounts( + **{ + field: details[metric] + for field, metric in reported.items() + if type(details.get(metric)) is int and 0 <= details[metric] <= stop + } + ) + outcomes = [] + for index, item in enumerate(items): + if index in indexed: + state = BatchState.FAILED + elif stop is None: + state = BatchState.UNKNOWN + elif index < stop: + state = BatchState.ACKNOWLEDGED + else: + state = BatchState.NOT_EXECUTED + outcomes.append(BatchItem(index, item.key, state, error=indexed.get(index))) + return ChunkResult(tuple(outcomes), counts, (details,)) + + +@contextmanager +def _bulk_errors(operation: str, items: Sequence[WriteInput]) -> Iterator[None]: + try: + yield + except BulkWriteError as error: + raise ChunkError(error, _partial_bulk(operation, items, error)) from error + + +class _MongoRecords: + def __init__( + self, + *, + connection: Connection, + key_fields: tuple[str, ...], + database: str, + collection: str, + session: Any = None, + ) -> None: + self._connection = connection + for name in (database, collection): + if not isinstance(name, str) or not name.strip() or '\x00' in name: + raise self._config_error('Expected explicit MongoDB database and collection names.') + self._database = database + self._collection_name = collection + self._key_fields = key_fields + self._session = session + self._collection: Any = None + + def _open(self) -> None: + try: + self._collection = self._connection.native[self._database][self._collection_name] + except InvalidName as error: + raise self._config_error('Invalid MongoDB database or collection name.', original=error) from error + + def _config_error(self, message: str, *, original: BaseException | None = None) -> ConfigurationError: + return ConfigurationError( + message, connection_name=self._connection.name, backend=_BACKEND, phase='config', original=original + ) + + def _validate_bson(self, document: Record) -> None: + try: + BSON.encode(document, codec_options=self._collection.codec_options) + except (BSONError, OverflowError, RecursionError, TypeError, ValueError) as error: + raise self._config_error( + 'The MongoDB collection cannot encode a record value as BSON.', original=error + ) from error + + def _filter(self, where: Condition | None) -> Record | None: + query = _condition_filter(where) + if query: + self._validate_bson(query) + return query + + def _update_document(self, values: Record) -> Record: + if '_id' in values: + raise self._config_error('MongoDB updates cannot modify the _id field.') + document = {'$set': values} + self._validate_bson(document) + return document + + def prepare_key(self, key: Record) -> None: + self._validate_bson(_key_filter(key)) + + def prepare_fields(self, fields: _Fields) -> None: + projection = _projection(fields) + if projection is not None: + self._validate_bson(projection) + + def prepare_write(self, operation: str, item: WriteInput) -> WriteInput: + if operation == 'insert': + if any(field != '_id' and field not in item.values for field in self._key_fields): + raise self._config_error('MongoDB inserts require every declared key field other than _id.') + self._validate_bson(item.values) + return item + if item.key is None: + raise self._config_error('A keyed MongoDB write requires its complete key.') + query = _key_filter(item.key) + if operation == 'update': + return replace(item, prepared=(query, self._update_document(item.values))) + return replace(item, prepared=query) + + def _unique_record(self, rows: list[Record], fields: _Fields) -> Record | None: + if len(rows) > 1: + raise RecordIdentityError( + 'The declared MongoDB key matched more than one document.', + connection_name=self._connection.name, + backend=_BACKEND, + phase='operation', + operation='get', + ) + return _record(rows[0], fields) if rows else None + + def _find_cursor(self, query: Record, fields: _Fields, limit: int, offset: int = 0, order_by: _Order = ()) -> Any: + return self._collection.find( + query, + projection=_projection(fields), + limit=limit, + skip=offset, + sort=[(field, 1 if direction == 'asc' else -1) for field, direction in order_by] or None, + session=self._session, + ) + + @contextmanager + def _create_only(self) -> Iterator[None]: + try: + yield + except DuplicateKeyError as error: + raise RecordConflictError( + 'A MongoDB insert conflicts with a unique key.', + connection_name=self._connection.name, + backend=_BACKEND, + phase='operation', + operation='insert', + original=error, + ) from error + + def _generated_keys(self, item: WriteInput, inserted_id: Any) -> tuple[Record, ...] | None: + if '_id' not in self._key_fields or '_id' in item.values or inserted_id is None: + return None + return ({field: inserted_id if field == '_id' else item.values[field] for field in self._key_fields},) + + def _inserted( + self, item: WriteInput, inserted_id: Any, acknowledged: bool, native_result: Any = None + ) -> WriteResult: + return WriteResult( + _BACKEND, + 'insert', + acknowledged=acknowledged, + inserted_count=1 if acknowledged else None, + generated_keys=self._generated_keys(item, inserted_id), + native_result=native_result, + ) + + def _insert_batch(self, items: Sequence[WriteInput], result: Any) -> ChunkResult: + outcomes = tuple( + BatchItem( + index, + item.key, + _batch_state(result.acknowledged), + self._inserted(item, inserted_id, result.acknowledged), + ) + for index, (item, inserted_id) in enumerate(zip(items, result.inserted_ids, strict=True)) + ) + return ChunkResult(outcomes, WriteCounts(inserted_count=len(items) if result.acknowledged else None), (result,)) + + +class Adapter(_MongoRecords): + def __enter__(self) -> 'Adapter': + self._open() + return self + + def __exit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> bool: + return False + + def get(self, key: Record, fields: _Fields) -> Record | None: + with _closing(self._find_cursor(_key_filter(key), fields, 2)) as cursor: + return self._unique_record(list(cursor), fields) + + def get_many(self, keys: Sequence[Record], fields: _Fields) -> list[Record | None]: + with _read_batch(keys) as rows: + for key in keys: + rows.append(self.get(key, fields)) + return rows + + def find(self, where: Condition | None, fields: _Fields, limit: int, offset: int, order_by: _Order) -> list[Record]: + query = self._filter(where) + if query is None: + return [] + with _closing(self._find_cursor(query, fields, limit, offset, order_by)) as cursor: + return [_record(row, fields) for row in cursor] + + def count(self, where: Condition | None) -> int: + query = self._filter(where) + return 0 if query is None else self._collection.count_documents(query, session=self._session) + + def insert(self, item: WriteInput) -> WriteResult: + with self._create_only(): + result = self._collection.insert_one(item.values.copy(), session=self._session) + return self._inserted(item, result.inserted_id, result.acknowledged, result) + + def update(self, item: WriteInput) -> WriteResult: + result = self._collection.update_one(*item.prepared, session=self._session) + return _update_result('update', result) + + def delete(self, item: WriteInput) -> WriteResult: + return _delete_result('delete', self._collection.delete_one(item.prepared, session=self._session)) + + def insert_many(self, items: Sequence[WriteInput]) -> ChunkResult: + documents = [item.values.copy() for item in items] + with _bulk_errors('insert_many', items): + result = self._collection.insert_many(documents, ordered=True, session=self._session) + return self._insert_batch(items, result) + + def bulk_update(self, items: Sequence[WriteInput]) -> ChunkResult: + operations = [UpdateOne(*item.prepared) for item in items] + with _bulk_errors('bulk_update', items): + result = self._collection.bulk_write(operations, ordered=True, session=self._session) + return _bulk_result('bulk_update', items, result) + + def delete_many(self, items: Sequence[WriteInput]) -> ChunkResult: + operations = [DeleteOne(item.prepared) for item in items] + with _bulk_errors('delete_many', items): + result = self._collection.bulk_write(operations, ordered=True, session=self._session) + return _bulk_result('delete_many', items, result) + + def update_many(self, where: Condition | None, set_values: Record) -> WriteResult: + query = self._filter(where) + document = self._update_document(set_values) + if query is None: + return WriteResult(_BACKEND, 'update_many', matched_count=0, modified_count=0) + return _update_result('update_many', self._collection.update_many(query, document, session=self._session)) + + def delete_where(self, where: Condition | None) -> WriteResult: + query = self._filter(where) + if query is None: + return WriteResult(_BACKEND, 'delete_where', deleted_count=0) + return _delete_result('delete_where', self._collection.delete_many(query, session=self._session)) + + +class AsyncAdapter(_MongoRecords): + async def __aenter__(self) -> 'AsyncAdapter': + self._open() + return self + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> bool: + return False + + async def get(self, key: Record, fields: _Fields) -> Record | None: + async with _async_closing(self._find_cursor(_key_filter(key), fields, 2)) as cursor: + return self._unique_record([row async for row in cursor], fields) + + async def get_many(self, keys: Sequence[Record], fields: _Fields) -> list[Record | None]: + with _read_batch(keys) as rows: + for key in keys: + rows.append(await self.get(key, fields)) + return rows + + async def find( + self, where: Condition | None, fields: _Fields, limit: int, offset: int, order_by: _Order + ) -> list[Record]: + query = self._filter(where) + if query is None: + return [] + async with _async_closing(self._find_cursor(query, fields, limit, offset, order_by)) as cursor: + return [_record(row, fields) async for row in cursor] + + async def count(self, where: Condition | None) -> int: + query = self._filter(where) + return 0 if query is None else await self._collection.count_documents(query, session=self._session) + + async def insert(self, item: WriteInput) -> WriteResult: + with self._create_only(): + result = await self._collection.insert_one(item.values.copy(), session=self._session) + return self._inserted(item, result.inserted_id, result.acknowledged, result) + + async def update(self, item: WriteInput) -> WriteResult: + result = await self._collection.update_one(*item.prepared, session=self._session) + return _update_result('update', result) + + async def delete(self, item: WriteInput) -> WriteResult: + return _delete_result('delete', await self._collection.delete_one(item.prepared, session=self._session)) + + async def insert_many(self, items: Sequence[WriteInput]) -> ChunkResult: + documents = [item.values.copy() for item in items] + with _bulk_errors('insert_many', items): + result = await self._collection.insert_many(documents, ordered=True, session=self._session) + return self._insert_batch(items, result) + + async def bulk_update(self, items: Sequence[WriteInput]) -> ChunkResult: + operations = [UpdateOne(*item.prepared) for item in items] + with _bulk_errors('bulk_update', items): + result = await self._collection.bulk_write(operations, ordered=True, session=self._session) + return _bulk_result('bulk_update', items, result) + + async def delete_many(self, items: Sequence[WriteInput]) -> ChunkResult: + operations = [DeleteOne(item.prepared) for item in items] + with _bulk_errors('delete_many', items): + result = await self._collection.bulk_write(operations, ordered=True, session=self._session) + return _bulk_result('delete_many', items, result) + + async def update_many(self, where: Condition | None, set_values: Record) -> WriteResult: + query = self._filter(where) + document = self._update_document(set_values) + if query is None: + return WriteResult(_BACKEND, 'update_many', matched_count=0, modified_count=0) + result = await self._collection.update_many(query, document, session=self._session) + return _update_result('update_many', result) + + async def delete_where(self, where: Condition | None) -> WriteResult: + query = self._filter(where) + if query is None: + return WriteResult(_BACKEND, 'delete_where', deleted_count=0) + result = await self._collection.delete_many(query, session=self._session) + return _delete_result('delete_where', result) diff --git a/tests/test_crud_mongodb.py b/tests/test_crud_mongodb.py new file mode 100644 index 0000000..1296af9 --- /dev/null +++ b/tests/test_crud_mongodb.py @@ -0,0 +1,404 @@ +import asyncio +from unittest import IsolatedAsyncioTestCase +from unittest import TestCase +from unittest.mock import AsyncMock +from unittest.mock import Mock +from uuid import uuid4 + +from bson import CodecOptions +from bson import ObjectId +from bson.binary import UuidRepresentation +from pymongo.errors import AutoReconnect +from pymongo.errors import BulkWriteError +from pymongo.errors import DuplicateKeyError +from pymongo.results import BulkWriteResult +from pymongo.results import DeleteResult +from pymongo.results import InsertManyResult +from pymongo.results import InsertOneResult +from pymongo.results import UpdateResult + +from database._adapters.mongodb import Adapter as ManagedMongoDB +from database.connections import AsyncConnection +from database.connections import Connection +from database.crud import And +from database.crud import AsyncRecords +from database.crud import BatchOperationError +from database.crud import BatchState +from database.crud import Eq +from database.crud import Exists +from database.crud import Gt +from database.crud import In +from database.crud import IsNull +from database.crud import RecordConflictError +from database.crud import RecordIdentityError +from database.crud import Records +from database.errors import ConfigurationError +from database.errors import DatabaseConnectionError + + +class _Cursor: + def __init__(self, rows, *, error=None, cleanup=None): + self._rows = rows + self._error = error + self.close = Mock(side_effect=cleanup) + + def __iter__(self): + if self._error is not None: + raise self._error + yield from self._rows + + +class _AsyncCursor: + def __init__(self, rows, *, error=None): + self._rows = rows + self._error = error + self.close = AsyncMock() + + async def __aiter__(self): + if self._error is not None: + raise self._error + for row in self._rows: + yield row + + +def _write_data(**counts): + return { + 'nMatched': 0, + 'nModified': 0, + 'nRemoved': 0, + 'nInserted': 0, + 'nUpserted': 0, + 'upserted': [], + 'writeErrors': [], + 'writeConcernErrors': [], + **counts, + } + + +def _adapter(): + return Mock(classify=ManagedMongoDB.classify) + + +class MongoDBRecordsTests(TestCase): + def setUp(self): + self.collection = Mock(codec_options=CodecOptions()) + self.connection_adapter = _adapter() + self.connection = Connection( + {'app': {'orders': self.collection}}, self.connection_adapter, 'main', 'mongodb', 'single' + ) + self.session = Mock() + + def records(self, *, key_fields=('_id',)): + return Records.mongodb( + self.connection, database='app', collection='orders', key_fields=key_fields, session=self.session + ) + + def test_get_composite_key_and_projection(self): + self.collection.find.side_effect = [ + _Cursor([{'name': None, 'tenant': 7, '_id': 1}]), + _Cursor([{'_id': 1}]), + _Cursor([]), + _Cursor([{'_id': 1, 'name': None}]), + ] + with self.records(key_fields=('tenant', 'number')) as records: + key = {'tenant': 7, 'number': 'A'} + self.assertEqual(records.get(key, fields=['name', 'missing']), {'name': None}) + self.assertEqual(records.get(key, fields=[]), {}) + self.assertIsNone(records.get(key, fields=[])) + self.assertEqual(records.get(key), {'_id': 1, 'name': None}) + + args, kwargs = self.collection.find.call_args_list[0] + self.assertEqual( + args[0], + { + 'tenant': {'$eq': 7, '$exists': True, '$not': {'$type': 'array'}}, + 'number': {'$eq': 'A', '$exists': True, '$not': {'$type': 'array'}}, + }, + ) + self.assertEqual(kwargs['limit'], 2) + self.assertEqual(kwargs['projection'], {'name': 1, 'missing': 1, '_id': 0}) + self.assertIs(kwargs['session'], self.session) + self.assertEqual(self.collection.find.call_args_list[1].kwargs['projection'], {'_id': 1}) + self.assertIsNone(self.collection.find.call_args_list[3].kwargs['projection']) + self.connection_adapter.close.assert_not_called() + self.session.end_session.assert_not_called() + + def test_find_closes_cursor_and_translates_plain_conditions(self): + cursor = _Cursor([{'name': 'Alice'}]) + self.collection.find.return_value = cursor + with self.records() as records: + result = records.find( + where=And(Eq('name', '$literal'), IsNull('state'), Exists('date'), Gt('count', 4)), + fields=['name'], + limit=2, + offset=1, + order_by=[('name', 'desc')], + ) + self.assertEqual(result, [{'name': 'Alice'}]) + args, kwargs = self.collection.find.call_args + scalar = {'$exists': True, '$not': {'$type': 'array'}} + self.assertEqual( + args[0], + { + '$and': [ + {'name': {'$eq': '$literal', **scalar}}, + {'state': {'$type': 'null', **scalar}}, + {'date': {'$exists': True}}, + {'count': {'$gt': 4, **scalar}}, + ] + }, + ) + self.assertEqual( + kwargs, + { + 'projection': {'name': 1, '_id': 0}, + 'session': self.session, + 'limit': 2, + 'skip': 1, + 'sort': [('name', -1)], + }, + ) + cursor.close.assert_called_once() + + def test_find_retains_primary_cursor_failure_and_cleanup(self): + original = AutoReconnect('failed to read') + cleanup = RuntimeError('failed to close') + self.collection.find.return_value = _Cursor([], error=original, cleanup=cleanup) + with self.records() as records, self.assertRaises(DatabaseConnectionError) as caught: + records.find(limit=4) + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + + def test_duplicate_record_identity_is_rejected_and_cursor_closed(self): + cursor = _Cursor([{'_id': 7}, {'_id': 9}]) + self.collection.find.return_value = cursor + with self.records(key_fields=('name',)) as records, self.assertRaises(RecordIdentityError): + records.get({'name': 'Alice'}, fields=[]) + cursor.close.assert_called_once() + + def test_keyed_batch_preserves_read_evidence_and_pending_keys(self): + native_error = AutoReconnect('lost') + self.collection.find.side_effect = [_Cursor([{'name': 'Alice'}]), _Cursor([], error=native_error)] + keys = [{'_id': 1}, {'_id': 2}, {'_id': 1}, {'_id': 3}, {'_id': 4}] + with self.records() as records, self.assertRaises(BatchOperationError) as caught: + records.get_many(keys, fields=['name'], batch_size=3) + + error = caught.exception + self.assertIs(error.original, native_error) + self.assertIs(error.__cause__, native_error) + self.assertEqual( + [item.state for item in error.partial_result.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.UNKNOWN, + BatchState.ACKNOWLEDGED, + BatchState.NOT_SENT, + BatchState.NOT_SENT, + ], + ) + self.assertEqual(error.partial_result.items[2].result.record, {'name': 'Alice'}) + self.assertEqual(self.collection.find.call_count, 2) + + def test_empty_condition_has_no_driver_requests(self): + with self.records() as records: + where = And(Eq('state', 'pending'), In('_id', [])) + self.assertEqual(records.find(where=where, limit=2), []) + self.assertEqual(records.count(where=where), 0) + self.assertEqual(records.update_many(where=where, set_values={'state': 'paid'}).matched_count, 0) + self.assertEqual(records.delete_where(where=where).deleted_count, 0) + self.assertEqual(self.collection.mock_calls, []) + + def test_whole_batch_preflight_uses_collection_codec(self): + supported_id = uuid4() + self.collection.codec_options = CodecOptions(uuid_representation=UuidRepresentation.STANDARD) + self.collection.insert_many.return_value = InsertManyResult([supported_id], True) + with self.records() as records: + self.assertEqual(records.insert_many([{'_id': supported_id}]).counts.inserted_count, 1) + self.collection.insert_many.reset_mock() + with self.assertRaises(ConfigurationError) as caught: + records.insert_many( + [{'_id': 'valid'}, {'_id': 'invalid', 'nested': {'invalid': object()}}], batch_size=1 + ) + self.assertIs(caught.exception.original, caught.exception.__cause__) + self.collection.insert_many.assert_not_called() + + def test_insert_generated_key_preserves_bson_type_and_caller_record(self): + identifier = ObjectId() + + def driver(document, **kwargs): + document['_id'] = identifier + return InsertOneResult(identifier, False) + + self.collection.insert_one.side_effect = driver + record = {'tenant': 5, 'state': 'ready'} + with self.records(key_fields=('tenant', '_id')) as records: + result = records.insert(record) + with self.assertRaises(ConfigurationError): + records.insert({'state': 'missing tenant'}) + self.assertNotIn('_id', record) + self.assertIs(result.generated_keys[0]['_id'], identifier) + self.assertEqual(result.generated_keys[0]['tenant'], 5) + self.assertFalse(result.acknowledged) + self.assertIsNone(result.inserted_count) + + def test_create_only_duplicate_preserves_original(self): + original = DuplicateKeyError('duplicate') + self.collection.insert_one.side_effect = original + with self.records() as records, self.assertRaises(RecordConflictError) as caught: + records.insert({'_id': 1}) + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + + def test_reports_ordered_bulk_failure_and_driver_details(self): + detail = {'index': 1, 'code': 11000, 'errmsg': 'duplicate'} + original = BulkWriteError(_write_data(nInserted=1, writeErrors=[detail])) + before = InsertManyResult([1, 2, 3], True) + self.collection.insert_many.side_effect = [before, original] + with self.records() as records, self.assertRaises(BatchOperationError) as caught: + records.insert_many([{'_id': position} for position in range(1, 8)], batch_size=3) + error = caught.exception + partial = error.partial_result + self.assertIs(error.original, original) + self.assertEqual(partial.counts.inserted_count, 4) + self.assertIs(partial.native_results[0], before) + self.assertIs(partial.native_results[1], original.details) + self.assertEqual( + [item.state for item in partial.items], + [BatchState.ACKNOWLEDGED] * 4 + [BatchState.FAILED, BatchState.NOT_EXECUTED, BatchState.NOT_SENT], + ) + self.assertIs(partial.items[4].error, detail) + self.assertEqual(self.collection.insert_many.call_count, 2) + self.assertTrue( + all( + call.kwargs == {'ordered': True, 'session': self.session} + for call in self.collection.insert_many.call_args_list + ) + ) + + def test_write_concern_failure_does_not_confirm_other_items(self): + detail = {'index': 1, 'code': 11000} + original = BulkWriteError(_write_data(nInserted=1, writeErrors=[detail], writeConcernErrors=[{'code': 64}])) + self.collection.insert_many.side_effect = original + with self.records() as records, self.assertRaises(BatchOperationError) as caught: + records.insert_many([{'_id': position} for position in range(3)]) + partial = caught.exception.partial_result + self.assertEqual( + [item.state for item in partial.items], [BatchState.UNKNOWN, BatchState.FAILED, BatchState.UNKNOWN] + ) + self.assertIsNone(partial.counts.inserted_count) + + def test_writes_keep_acknowledgement_and_native_counts_separate(self): + no_change = UpdateResult({'n': 1, 'nModified': 0, 'upserted': None}, True) + self.collection.update_one.return_value = no_change + self.collection.delete_one.return_value = DeleteResult({}, False) + self.collection.bulk_write.side_effect = [ + BulkWriteResult(_write_data(nMatched=1, nModified=0), True), + BulkWriteResult({}, False), + ] + with self.records() as records: + result = records.update({'_id': 1}, set_values={'name': 'same'}) + self.assertEqual((result.matched_count, result.modified_count), (1, 0)) + self.assertIs(result.native_result, no_change) + self.assertIsNone(records.delete({'_id': 1}).deleted_count) + bulk = records.bulk_update([{'key': {'_id': 1}, 'set_values': {'name': 'same'}}]) + self.assertEqual((bulk.counts.matched_count, bulk.counts.modified_count), (1, 0)) + deleted = records.delete_many([{'_id': 1}, {'_id': 2}]) + self.assertEqual([item.state for item in deleted.items], [BatchState.UNACKNOWLEDGED] * 2) + self.assertIsNone(deleted.counts.deleted_count) + + def test_custom_identity_does_not_allow_modifying_id(self): + with self.records(key_fields=('sku',)) as records: + with self.assertRaises(ConfigurationError): + records.update({'sku': 1}, set_values={'_id': 2}) + with self.assertRaises(ConfigurationError): + records.update_many(where={'sku': 1}, set_values={'_id': 2}) + self.collection.update_one.assert_not_called() + self.collection.update_many.assert_not_called() + + +class AsyncMongoDBRecordsTests(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.collection = Mock(codec_options=CodecOptions()) + self.connection_adapter = _adapter() + self.connection_adapter.close_async = AsyncMock() + self.connection = AsyncConnection( + {'app': {'orders': self.collection}}, + self.connection_adapter, + 'main', + 'mongodb', + 'single', + asyncio.get_running_loop(), + ) + self.session = Mock(end_session=AsyncMock()) + + def records(self): + return AsyncRecords.mongodb(self.connection, database='app', collection='orders', session=self.session) + + async def test_reads_preserve_session_and_close_awaitable_cursor(self): + cursor = _AsyncCursor([{'_id': 1, 'name': None}]) + self.collection.find.return_value = cursor + self.collection.count_documents = AsyncMock(return_value=1) + async with self.records() as records: + self.assertEqual(await records.find(fields=['name'], limit=2), [{'name': None}]) + self.assertEqual(await records.count(where={'name': None}), 1) + cursor.close.assert_awaited_once() + self.assertIs(self.collection.find.call_args.kwargs['session'], self.session) + self.assertIs(self.collection.count_documents.await_args.kwargs['session'], self.session) + self.connection_adapter.close_async.assert_not_awaited() + self.session.end_session.assert_not_awaited() + + async def test_write_operations_use_native_async_methods(self): + updated = UpdateResult({'n': 1, 'nModified': 1, 'upserted': None}, True) + deleted = DeleteResult({'n': 1}, True) + self.collection.insert_one = AsyncMock(return_value=InsertOneResult(ObjectId(), True)) + self.collection.update_one = AsyncMock(return_value=updated) + self.collection.delete_one = AsyncMock(return_value=deleted) + self.collection.insert_many = AsyncMock(return_value=InsertManyResult([1, 2], True)) + self.collection.bulk_write = AsyncMock( + side_effect=[BulkWriteResult(_write_data(nMatched=1), True), BulkWriteResult(_write_data(nRemoved=1), True)] + ) + self.collection.update_many = AsyncMock(return_value=updated) + self.collection.delete_many = AsyncMock(return_value=deleted) + + async with self.records() as records: + self.assertEqual((await records.insert({'name': 'name'})).inserted_count, 1) + self.assertEqual((await records.update({'_id': 1}, set_values={'name': 'new'})).modified_count, 1) + self.assertEqual((await records.delete({'_id': 1})).deleted_count, 1) + self.assertEqual((await records.insert_many([{'_id': 1}, {'_id': 2}])).counts.inserted_count, 2) + self.assertEqual( + (await records.bulk_update([{'key': {'_id': 1}, 'set_values': {'name': 'same'}}])).counts.matched_count, + 1, + ) + self.assertEqual((await records.delete_many([{'_id': 9}])).counts.deleted_count, 1) + self.assertEqual((await records.update_many(where={'_id': 1}, set_values={'name': 'a'})).modified_count, 1) + self.assertEqual((await records.delete_where(where={'_id': 1})).deleted_count, 1) + + for name in ( + 'insert_one', + 'update_one', + 'delete_one', + 'insert_many', + 'bulk_write', + 'update_many', + 'delete_many', + ): + for call in getattr(self.collection, name).await_args_list: + self.assertIs(call.kwargs['session'], self.session) + + async def test_read_batch_cancellation_preserves_identity_and_evidence(self): + original = asyncio.CancelledError() + first = _AsyncCursor([{'name': 'Alice'}]) + interrupted = _AsyncCursor([], error=original) + self.collection.find.side_effect = [first, interrupted] + async with self.records() as records: + with self.assertRaises(asyncio.CancelledError) as caught: + await records.get_many([{'_id': 1}, {'_id': 2}, {'_id': 3}], fields=['name']) + self.assertIs(caught.exception, original) + items = original.database_partial_result.items + self.assertEqual( + [item.state for item in items], [BatchState.ACKNOWLEDGED, BatchState.UNKNOWN, BatchState.NOT_SENT] + ) + self.assertEqual(items[0].result.record, {'name': 'Alice'}) + first.close.assert_awaited_once() + interrupted.close.assert_awaited_once() -- Gitee From 1e7f52ee5f4646498cca57632a99bb2ca2a3704b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:44:39 +0800 Subject: [PATCH 56/86] =?UTF-8?q?feat(mysql):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=BC=82=E6=AD=A5=E8=AE=B0=E5=BD=95=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E5=92=8C=E6=89=B9=E9=87=8F=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_mysql.py | 635 ++++++++++++++++++++++++++++++++++++ tests/test_crud_mysql.py | 474 +++++++++++++++++++++++++++ 2 files changed, 1109 insertions(+) create mode 100644 src/database/crud/_mysql.py create mode 100644 tests/test_crud_mysql.py diff --git a/src/database/crud/_mysql.py b/src/database/crud/_mysql.py new file mode 100644 index 0000000..3ecc1b1 --- /dev/null +++ b/src/database/crud/_mysql.py @@ -0,0 +1,635 @@ +"""MySQL record operations on a borrowed Connector/Python connection.""" + +import asyncio +import re +from collections.abc import AsyncIterator +from collections.abc import Iterator +from collections.abc import Sequence +from contextlib import asynccontextmanager +from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import replace +from datetime import date +from datetime import datetime +from datetime import time +from datetime import timedelta +from decimal import Decimal +from itertools import groupby +from math import isfinite +from typing import Any + +from mysql.connector.constants import ClientFlag +from mysql.connector.errorcode import ER_DUP_ENTRY +from mysql.connector.errors import Error as MySQLError + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import attach_cleanup_errors +from ._conditions import And +from ._conditions import Condition +from ._conditions import Eq +from ._conditions import Exists +from ._conditions import Gt +from ._conditions import Gte +from ._conditions import In +from ._conditions import IsNull +from ._conditions import Lt +from ._conditions import Lte +from ._conditions import Or +from ._errors import RecordConflictError +from ._errors import RecordIdentityError +from ._errors import UnsupportedOperationError +from ._types import BatchItem +from ._types import BatchState +from ._types import ChunkError +from ._types import ChunkResult +from ._types import ReadItem +from ._types import WriteCounts +from ._types import WriteInput +from ._types import WriteResult + +_IDENTIFIERS = re.compile(r'\w{1,64}') +_COMPARISONS = {Eq: '=', Gt: '>', Gte: '>=', Lt: '<', Lte: '<='} +_VALUE_TYPES = {str, bytes, bytearray, int, bool, float, Decimal, date, datetime, time, timedelta} +_COUNT_FIELDS = ('matched_count', 'modified_count', 'inserted_count', 'deleted_count') + +_Row = dict[str, Any] +_Fields = tuple[str, ...] | None +_Order = tuple[tuple[str, str], ...] + + +def _identifier(name: str) -> str: + if not isinstance(name, str) or not _IDENTIFIERS.fullmatch(name): + raise ConfigurationError( + 'MySQL identifiers require 1–64 letters, digits or underscores.', backend='mysql', phase='config' + ) + return f'`{name}`' + + +def _validate_value(value: Any) -> None: + if value is not None and type(value) not in _VALUE_TYPES: + raise ConfigurationError('Expected a native MySQL scalar value.', backend='mysql', phase='config') + if (type(value) is float and not isfinite(value)) or (type(value) is Decimal and not value.is_finite()): + raise ConfigurationError('Expected a finite MySQL numeric value.', backend='mysql', phase='config') + + +def _columns(names: Sequence[str]) -> tuple[str, ...]: + columns = tuple(_identifier(name) for name in names) + if len({name.casefold() for name in names}) != len(names): + raise ConfigurationError('MySQL requires distinct column names.', backend='mysql', phase='config') + return columns + + +def _projection(fields: _Fields) -> str: + if fields is None: + return '*' + return ', '.join(_columns(fields)) if fields else '1' + + +@dataclass(frozen=True) +class _Statement: + sql: str + params: tuple[Any, ...] = () + + +def _predicate(where: Condition) -> _Statement: + if isinstance(where, (And, Or)): + children = tuple(_predicate(child) for child in where.conditions) + joiner = ' AND ' if isinstance(where, And) else ' OR ' + return _Statement( + '(' + joiner.join(child.sql for child in children) + ')', + tuple(value for child in children for value in child.params), + ) + if isinstance(where, Exists): + raise UnsupportedOperationError('Field existence requires MongoDB.', backend='mysql', phase='config') + + field = _identifier(where.field) + if isinstance(where, IsNull): + return _Statement(f'{field} IS NULL') + if isinstance(where, In): + if not where.values: + return _Statement('0 = 1') + for value in where.values: + _validate_value(value) + placeholders = ', '.join('%s' for _ in where.values) + return _Statement(f'{field} IN ({placeholders})', tuple(where.values)) + _validate_value(where.value) + return _Statement(f'{field} {_COMPARISONS[type(where)]} %s', (where.value,)) + + +def _where_clause(where: Condition | None) -> _Statement: + if where is None: + return _Statement('') + predicate = _predicate(where) + return _Statement(f' WHERE {predicate.sql}', predicate.params) + + +def _is_duplicate(error: BaseException | None) -> bool: + return isinstance(error, MySQLError) and error.errno == ER_DUP_ENTRY + + +def _write_result(operation: str, cursor: Any, native: Any) -> WriteResult: + rowcount = cursor.rowcount + count = rowcount if type(rowcount) is int and rowcount >= 0 else None + stats = {'rowcount': rowcount, 'lastrowid': cursor.lastrowid} + counts = {} + if operation in ('insert', 'insert_many'): + counts['inserted_count'] = count + elif operation in ('delete', 'delete_many', 'delete_where'): + counts['deleted_count'] = count + else: + # The Python driver's client_flags can change before the connection is renegotiated. + # The C extension exposes the session's actual flags. + session_flags = getattr(getattr(native, '_cmysql', None), 'st_client_flag', None) + flags = session_flags() if callable(session_flags) else None + stats['rowcount_mode'] = 'unknown' + if type(flags) is int: + if flags & ClientFlag.FOUND_ROWS: + stats['rowcount_mode'] = 'matched' + counts['matched_count'] = count + else: + stats['rowcount_mode'] = 'changed' + counts['modified_count'] = count + return WriteResult('mysql', operation, acknowledged=True, native_result=stats, **counts) + + +def _read_records(rows: Sequence[tuple[Any, ...]], column_names: Sequence[str], fields: _Fields) -> list[_Row]: + columns = column_names if fields is None else fields + if not columns: + return [{} for _ in rows] + return [dict(zip(columns, row, strict=True)) for row in rows] + + +@dataclass(frozen=True) +class _WriteGroup: + sql: str + params: tuple[Any, ...] | list[tuple[Any, ...]] + size: int + many: bool + + +class _WriteProgress: + """Track completed groups separately from a group that may have been sent.""" + + def __init__(self, operation: str, items: Sequence[WriteInput]) -> None: + self.operation = operation + self.items = items + self.acknowledged: list[BatchItem] = [] + self.results: list[WriteResult] = [] + self.sent = 0 + + def complete(self, result: WriteResult) -> None: + start = len(self.acknowledged) + single = self.sent - start == 1 + self.acknowledged.extend( + BatchItem( + index, + self.items[index].key, + BatchState.ACKNOWLEDGED, + replace(result, operation=self.operation) if single else None, + ) + for index in range(start, self.sent) + ) + self.results.append(result) + + def result(self, error: BaseException | None = None) -> ChunkResult: + acknowledged = len(self.acknowledged) + single_conflict = self.operation == 'insert' and self.sent == acknowledged + 1 and _is_duplicate(error) + items = list(self.acknowledged) + for index in range(acknowledged, len(self.items)): + if index >= self.sent: + state = BatchState.NOT_SENT + elif single_conflict: + state = BatchState.FAILED + else: + state = BatchState.UNKNOWN + items.append( + BatchItem(index, self.items[index].key, state, error=error if state is BatchState.FAILED else None) + ) + + totals = WriteCounts() + if acknowledged == self.sent or single_conflict: + counts = {} + for name in _COUNT_FIELDS: + group_counts = [getattr(result, name) for result in self.results] + if group_counts and all(count is not None for count in group_counts): + counts[name] = sum(group_counts) + if not self.results: + if self.operation == 'insert': + counts['inserted_count'] = 0 + elif self.operation == 'delete': + counts['deleted_count'] = 0 + else: + counts.update(matched_count=0, modified_count=0) + totals = WriteCounts(**counts) + return ChunkResult(tuple(items), totals, tuple(result.native_result for result in self.results)) + + +def _chunk_error(error: BaseException, partial: ChunkResult) -> None: + if isinstance(error, asyncio.CancelledError): + BaseException.__dict__['__dict__'].__get__(error)['database_partial_result'] = partial + elif isinstance(error, Exception): + raise ChunkError(error, partial) from error + + +def _read_chunk(keys: Sequence[_Row], records: list[_Row | None] | None) -> ChunkResult: + if records is None: + items = tuple(BatchItem(index, key, BatchState.NOT_SENT) for index, key in enumerate(keys)) + else: + items = tuple( + BatchItem(index, key, BatchState.ACKNOWLEDGED, ReadItem(index, key, record)) + for index, (key, record) in enumerate(zip(keys, records, strict=True)) + ) + return ChunkResult(items) + + +class _MySQLPlan: + def __init__(self, *, connection: Connection | AsyncConnection, table: str, key_fields: tuple[str, ...]) -> None: + self.connection = connection + self.table = _identifier(table) + self.key_fields = key_fields + if not key_fields: + raise ConfigurationError('MySQL requires key fields.', backend='mysql', phase='config') + key_columns = _columns(key_fields) + self.key_names = {name.casefold(): name for name in key_fields} + self.key_where = ' AND '.join(f'{column} = %s' for column in key_columns) + + def prepare_fields(self, fields: _Fields) -> None: + _projection(fields) + + def prepare_key(self, key: _Row) -> None: + for name in self.key_fields: + _validate_value(key[name]) + + def prepare_write(self, operation: str, item: WriteInput) -> WriteInput: + if item.key is not None: + self.prepare_key(item.key) + if operation == 'insert': + statement = self._insert(item.values) + elif operation == 'update': + statement = self._update(item.values, _Statement(f' WHERE {self.key_where}', self._key_params(item.key))) + else: + statement = _Statement(f'DELETE FROM {self.table} WHERE {self.key_where}', self._key_params(item.key)) + return replace(item, prepared=statement) + + def _value_columns(self, values: _Row, *, update: bool) -> tuple[tuple[str, ...], tuple[Any, ...]]: + names = tuple(sorted(values)) + columns = _columns(names) + for name in names: + declared = self.key_names.get(name.casefold()) + if declared is not None and (update or declared != name): + raise ConfigurationError( + 'Use declared MySQL key names and leave keys unchanged.', + backend='mysql', + phase='config', + ) + params = tuple(values[name] for name in names) + for value in params: + _validate_value(value) + return columns, params + + def _key_params(self, key: _Row | None) -> tuple[Any, ...]: + if key is None: + raise ConfigurationError('Expected a complete MySQL record key.', backend='mysql', phase='config') + return tuple(key[name] for name in self.key_fields) + + def _insert(self, values: _Row) -> _Statement: + columns, params = self._value_columns(values, update=False) + placeholders = ', '.join('%s' for _ in columns) + return _Statement(f'INSERT INTO {self.table} ({", ".join(columns)}) VALUES ({placeholders})', params) + + def _update(self, values: _Row, where: _Statement) -> _Statement: + columns, params = self._value_columns(values, update=True) + assignments = ', '.join(f'{name} = %s' for name in columns) + return _Statement(f'UPDATE {self.table} SET {assignments}{where.sql}', params + where.params) + + def _delete_where(self, where: Condition | None) -> _Statement: + tail = _where_clause(where) + return _Statement(f'DELETE FROM {self.table}{tail.sql}', tail.params) + + def _lookup(self, key: _Row, fields: _Fields) -> _Statement: + return _Statement( + f'SELECT {_projection(fields)} FROM {self.table} WHERE {self.key_where} LIMIT 2', + self._key_params(key), + ) + + def _lookups(self, keys: Sequence[_Row], fields: _Fields) -> _Statement: + parts = [] + params = [] + selection = f'{self.table}.*' if fields is None else _projection(fields) + for index, key in enumerate(keys): + parts.append(f'(SELECT %s, {selection} FROM {self.table} WHERE {self.key_where} LIMIT 2)') + params.extend((index, *self._key_params(key))) + return _Statement(' UNION ALL '.join(parts), tuple(params)) + + def _identity_error(self, operation: str) -> RecordIdentityError: + return RecordIdentityError( + 'The declared MySQL key matched more than one record.', + connection_name=self.connection.name, + backend='mysql', + phase='operation', + operation=operation, + ) + + def _lookup_result(self, rows: Sequence[tuple[Any, ...]], columns: Sequence[str], fields: _Fields) -> _Row | None: + if len(rows) > 1: + raise self._identity_error('get') + return _read_records(rows, columns, fields)[0] if rows else None + + def _lookup_results( + self, keys: Sequence[_Row], rows: Sequence[tuple[Any, ...]], columns: Sequence[str], fields: _Fields + ) -> list[_Row | None]: + records: list[_Row | None] = [None] * len(keys) + duplicates = set() + for row in rows: + # Position is a generated integer literal, even if the native cursor is in raw mode. + index = int(row[0]) + if records[index] is not None: + duplicates.add(index) + else: + records[index] = _read_records((row[1:],), columns[1:], fields)[0] + if duplicates: + error = self._identity_error('get_many') + items = tuple( + BatchItem(index, key, BatchState.FAILED, error=error) + if index in duplicates + else BatchItem(index, key, BatchState.ACKNOWLEDGED, ReadItem(index, key, records[index])) + for index, key in enumerate(keys) + ) + raise ChunkError(error, ChunkResult(items)) from error + return records + + def _find( + self, + where: Condition | None, + fields: _Fields, + limit: int, + offset: int, + order_by: _Order, + ) -> _Statement: + tail = _where_clause(where) + sql = f'SELECT {_projection(fields)} FROM {self.table}{tail.sql}' + if order_by: + names, directions = zip(*order_by) + order = ', '.join( + f'{column} {"DESC" if direction == "desc" else "ASC"}' + for column, direction in zip(_columns(names), directions, strict=True) + ) + sql += f' ORDER BY {order}' + return _Statement(f'{sql} LIMIT %s OFFSET %s', (*tail.params, limit, offset)) + + def _count(self, where: Condition | None) -> _Statement: + tail = _where_clause(where) + return _Statement(f'SELECT COUNT(*) FROM {self.table}{tail.sql}', tail.params) + + def _groups(self, operation: str, items: Sequence[WriteInput]) -> Iterator[_WriteGroup]: + if operation == 'delete': + joined = ' OR '.join(f'({self.key_where})' for _ in items) + params = tuple(value for item in items for value in self._key_params(item.key)) + yield _WriteGroup(f'DELETE FROM {self.table} WHERE {joined}', params, len(items), False) + return + for statement, group in groupby(items, key=lambda item: item.prepared.sql): + params = [item.prepared.params for item in group] + yield _WriteGroup(statement, params, len(params), True) + + def _conflict(self, error: MySQLError) -> RecordConflictError: + return RecordConflictError( + 'The MySQL insert conflicts with an existing unique key.', + connection_name=self.connection.name, + backend='mysql', + phase='operation', + operation='insert', + original=error, + ) + + +class Adapter(_MySQLPlan): + """Synchronous records backed by one caller-owned connection.""" + + def __enter__(self) -> 'Adapter': + return self + + def __exit__(self, exc_type: Any, exc: BaseException | None, traceback: Any) -> None: + return None + + @contextmanager + def _cursor(self) -> Iterator[tuple[Any, Any]]: + native = self.connection.native + cursor = native.cursor(buffered=True) + try: + yield cursor, native + except BaseException as error: + try: + cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(error, (cleanup,)) + raise + else: + cursor.close() + + def _read(self, statement: _Statement) -> tuple[Sequence[tuple[Any, ...]], Sequence[str]]: + with self._cursor() as (cursor, _): + cursor.execute(statement.sql, statement.params) + return cursor.fetchall(), cursor.column_names + + def get(self, key: _Row, fields: _Fields) -> _Row | None: + rows, columns = self._read(self._lookup(key, fields)) + return self._lookup_result(rows, columns, fields) + + def get_many(self, keys: Sequence[_Row], fields: _Fields) -> list[_Row | None]: + if not keys: + return [] + statement = self._lookups(keys, fields) + observations = None + sent = False + try: + with self._cursor() as (cursor, _): + sent = True + cursor.execute(statement.sql, statement.params) + rows = cursor.fetchall() + observations = self._lookup_results(keys, rows, cursor.column_names, fields) + except BaseException as error: + if observations is not None or not sent: + _chunk_error(error, _read_chunk(keys, observations)) + raise + return observations + + def find(self, where: Condition | None, fields: _Fields, limit: int, offset: int, order_by: _Order) -> list[_Row]: + rows, columns = self._read(self._find(where, fields, limit, offset, order_by)) + return _read_records(rows, columns, fields) + + def count(self, where: Condition | None) -> int: + rows, _ = self._read(self._count(where)) + return int(rows[0][0]) + + def _write(self, operation: str, statement: _Statement) -> WriteResult: + with self._cursor() as (cursor, native): + try: + cursor.execute(statement.sql, statement.params) + except MySQLError as error: + if operation == 'insert' and _is_duplicate(error): + raise self._conflict(error) from error + raise + return _write_result(operation, cursor, native) + + def insert(self, item: WriteInput) -> WriteResult: + return self._write('insert', item.prepared) + + def update(self, item: WriteInput) -> WriteResult: + return self._write('update', item.prepared) + + def delete(self, item: WriteInput) -> WriteResult: + return self._write('delete', item.prepared) + + def update_many(self, where: Condition | None, set_values: _Row) -> WriteResult: + return self._write('update_many', self._update(set_values, _where_clause(where))) + + def delete_where(self, where: Condition | None) -> WriteResult: + return self._write('delete_where', self._delete_where(where)) + + def _batch(self, operation: str, native_operation: str, items: Sequence[WriteInput]) -> ChunkResult: + progress = _WriteProgress(operation, items) + if not items: + return progress.result() + try: + with self._cursor() as (cursor, native): + for group in self._groups(operation, items): + progress.sent += group.size + if group.many: + cursor.executemany(group.sql, group.params) + else: + cursor.execute(group.sql, group.params) + progress.complete(_write_result(native_operation, cursor, native)) + except BaseException as error: + _chunk_error(error, progress.result(error)) + raise + return progress.result() + + def insert_many(self, items: Sequence[WriteInput]) -> ChunkResult: + return self._batch('insert', 'insert_many', items) + + def bulk_update(self, items: Sequence[WriteInput]) -> ChunkResult: + return self._batch('update', 'bulk_update', items) + + def delete_many(self, items: Sequence[WriteInput]) -> ChunkResult: + return self._batch('delete', 'delete_many', items) + + +class AsyncAdapter(_MySQLPlan): + """Asynchronous records backed by one caller-owned async connection.""" + + async def __aenter__(self) -> 'AsyncAdapter': + return self + + async def __aexit__(self, exc_type: Any, exc: BaseException | None, traceback: Any) -> None: + return None + + @asynccontextmanager + async def _cursor(self) -> AsyncIterator[tuple[Any, Any]]: + native = self.connection.native + cursor = await native.cursor(buffered=True) + try: + yield cursor, native + except BaseException as error: + try: + await cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(error, (cleanup,)) + raise + else: + await cursor.close() + + async def _read(self, statement: _Statement) -> tuple[Sequence[tuple[Any, ...]], Sequence[str]]: + async with self._cursor() as (cursor, _): + await cursor.execute(statement.sql, statement.params) + return await cursor.fetchall(), cursor.column_names + + async def get(self, key: _Row, fields: _Fields) -> _Row | None: + rows, columns = await self._read(self._lookup(key, fields)) + return self._lookup_result(rows, columns, fields) + + async def get_many(self, keys: Sequence[_Row], fields: _Fields) -> list[_Row | None]: + if not keys: + return [] + statement = self._lookups(keys, fields) + observations = None + sent = False + try: + async with self._cursor() as (cursor, _): + sent = True + await cursor.execute(statement.sql, statement.params) + rows = await cursor.fetchall() + observations = self._lookup_results(keys, rows, cursor.column_names, fields) + except BaseException as error: + if observations is not None or not sent: + _chunk_error(error, _read_chunk(keys, observations)) + raise + return observations + + async def find( + self, + where: Condition | None, + fields: _Fields, + limit: int, + offset: int, + order_by: _Order, + ) -> list[_Row]: + rows, columns = await self._read(self._find(where, fields, limit, offset, order_by)) + return _read_records(rows, columns, fields) + + async def count(self, where: Condition | None) -> int: + rows, _ = await self._read(self._count(where)) + return int(rows[0][0]) + + async def _write(self, operation: str, statement: _Statement) -> WriteResult: + async with self._cursor() as (cursor, native): + try: + await cursor.execute(statement.sql, statement.params) + except MySQLError as error: + if operation == 'insert' and _is_duplicate(error): + raise self._conflict(error) from error + raise + return _write_result(operation, cursor, native) + + async def insert(self, item: WriteInput) -> WriteResult: + return await self._write('insert', item.prepared) + + async def update(self, item: WriteInput) -> WriteResult: + return await self._write('update', item.prepared) + + async def delete(self, item: WriteInput) -> WriteResult: + return await self._write('delete', item.prepared) + + async def update_many(self, where: Condition | None, set_values: _Row) -> WriteResult: + return await self._write('update_many', self._update(set_values, _where_clause(where))) + + async def delete_where(self, where: Condition | None) -> WriteResult: + return await self._write('delete_where', self._delete_where(where)) + + async def _batch(self, operation: str, native_operation: str, items: Sequence[WriteInput]) -> ChunkResult: + progress = _WriteProgress(operation, items) + if not items: + return progress.result() + try: + async with self._cursor() as (cursor, native): + for group in self._groups(operation, items): + progress.sent += group.size + if group.many: + await cursor.executemany(group.sql, group.params) + else: + await cursor.execute(group.sql, group.params) + progress.complete(_write_result(native_operation, cursor, native)) + except BaseException as error: + _chunk_error(error, progress.result(error)) + raise + return progress.result() + + async def insert_many(self, items: Sequence[WriteInput]) -> ChunkResult: + return await self._batch('insert', 'insert_many', items) + + async def bulk_update(self, items: Sequence[WriteInput]) -> ChunkResult: + return await self._batch('update', 'bulk_update', items) + + async def delete_many(self, items: Sequence[WriteInput]) -> ChunkResult: + return await self._batch('delete', 'delete_many', items) diff --git a/tests/test_crud_mysql.py b/tests/test_crud_mysql.py new file mode 100644 index 0000000..db19f7c --- /dev/null +++ b/tests/test_crud_mysql.py @@ -0,0 +1,474 @@ +"""Offline tests for MySQL CRUD through the public binding API.""" + +import asyncio +from decimal import Decimal +from types import SimpleNamespace +from unittest import IsolatedAsyncioTestCase +from unittest import TestCase +from unittest.mock import AsyncMock +from unittest.mock import Mock +from unittest.mock import call + +from mysql.connector.constants import ClientFlag +from mysql.connector.errorcode import ER_DUP_ENTRY +from mysql.connector.errors import IntegrityError +from mysql.connector.errors import ProgrammingError + +from database import AsyncConnection +from database import Connection +from database._adapters.mysql import Adapter as ConnectionAdapter +from database.crud import And +from database.crud import AsyncRecords +from database.crud import BatchOperationError +from database.crud import BatchState +from database.crud import Eq +from database.crud import Exists +from database.crud import Gt +from database.crud import In +from database.crud import IsNull +from database.crud import Or +from database.crud import RecordConflictError +from database.crud import RecordIdentityError +from database.crud import Records +from database.crud import UnsupportedOperationError +from database.errors import ConfigurationError +from database.errors import DriverError + + +def key(index, tenant=1): + return {'tenant': tenant, 'id': index} + + +def make_handle(*, is_async=False): + cursor = Mock(rowcount=1, lastrowid=None, column_names=('value',)) + native = Mock() + native._cmysql.st_client_flag.return_value = 0 + adapter = SimpleNamespace(classify=ConnectionAdapter.classify) + if is_async: + for name in ('execute', 'executemany', 'fetchall', 'close'): + setattr(cursor, name, AsyncMock()) + native.cursor = AsyncMock(return_value=cursor) + handle = AsyncConnection(native, adapter, 'orders', 'mysql', 'single', asyncio.get_running_loop()) + else: + native.cursor.return_value = cursor + handle = Connection(native, adapter, 'orders', 'mysql', 'single') + return handle, native, cursor + + +def sync_records(connection): + return Records.mysql(connection, table='sample', key_fields=('tenant', 'id')) + + +def async_records(connection): + return AsyncRecords.mysql(connection, table='sample', key_fields=('tenant', 'id')) + + +class MySQLCRUDTests(TestCase): + def setUp(self): + self.connection, self.native, self.cursor = make_handle() + + def test_binding_validates_mysql_names_without_accessing_native_cursor(self): + with self.assertRaises(ConfigurationError): + Records.mysql(self.connection, table='sample; SELECT 1', key_fields=('id',)) + with self.assertRaises(ConfigurationError): + Records.mysql(self.connection, table='sample', key_fields=('id', 'ID')) + with sync_records(self.connection): + self.native.cursor.assert_not_called() + self.native.close.assert_not_called() + + def test_find_binds_values_and_projects_bounded_buffered_rows(self): + literal = "x' OR 1=1" + self.cursor.fetchall.return_value = [(literal, None)] + where = And(Gt('id', 4), Or(Eq('value', literal), IsNull('note')), In('tenant', (5, 9))) + + with sync_records(self.connection) as records: + result = records.find(where=where, fields=('value', 'note'), limit=2, offset=3, order_by=(('id', 'desc'),)) + + self.assertEqual(result, [{'value': literal, 'note': None}]) + self.native.cursor.assert_called_once_with(buffered=True) + quote = chr(96) + self.cursor.execute.assert_called_once_with( + f'SELECT {quote}value{quote}, {quote}note{quote} FROM {quote}sample{quote} ' + f'WHERE ({quote}id{quote} > %s AND ({quote}value{quote} = %s OR {quote}note{quote} IS NULL) ' + f'AND {quote}tenant{quote} IN (%s, %s)) ORDER BY {quote}id{quote} DESC LIMIT %s OFFSET %s', + (4, literal, 5, 9, 2, 3), + ) + self.cursor.fetchall.assert_called_once_with() + self.cursor.close.assert_called_once_with() + self.native.close.assert_not_called() + + def test_get_distinguishes_empty_projection_absence_and_invalid_cardinality(self): + self.cursor.fetchall.side_effect = [[(1,)], [], [(1,), (1,)]] + + with sync_records(self.connection) as records: + self.assertEqual(records.get(key(1), fields=[]), {}) + self.assertIsNone(records.get(key(2), fields=[])) + with self.assertRaises(RecordIdentityError): + records.get(key(3), fields=[]) + + self.assertEqual(self.cursor.close.call_count, 3) + self.assertEqual(self.cursor.execute.call_args.args[1], (1, 3)) + self.assertTrue(self.cursor.execute.call_args.args[0].endswith('LIMIT 2')) + + def test_get_many_validates_projection_before_sending_any_chunk(self): + with sync_records(self.connection) as records: + with self.assertRaises(ConfigurationError): + records.get_many([key(1), key(2)], fields=('bad%s',), batch_size=1) + + self.native.cursor.assert_not_called() + + def test_get_many_uses_sql_position_for_composite_keys_and_projection(self): + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(2, 'one'), (0, 'other tenant')] + keys = [key(2, tenant=2), key(9), key(1), key(2, tenant=2)] + + with sync_records(self.connection) as records: + result = records.get_many(keys, fields=('value',)) + + self.assertEqual([item.index for item in result], list(range(4))) + self.assertEqual([item.key for item in result], keys) + self.assertEqual( + [item.record for item in result], + [{'value': 'other tenant'}, None, {'value': 'one'}, {'value': 'other tenant'}], + ) + sql, parameters = self.cursor.execute.call_args.args + self.assertEqual(parameters, (0, 2, 2, 1, 1, 9, 2, 1, 1)) + self.assertEqual(sql.count('LIMIT 2'), 3) + self.assertEqual(sql.count(' UNION ALL '), 2) + self.cursor.close.assert_called_once_with() + + def test_get_many_preserves_other_observations_when_key_matches_twice(self): + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(0, 'a'), (0, 'b'), (2, 'c')] + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.get_many([key(1), key(2), key(3), key(1)]) + + error = caught.exception + self.assertIsInstance(error.original, RecordIdentityError) + self.assertIs(error.__cause__, error.original) + self.assertIn('SELECT %s, ' + chr(96) + 'sample' + chr(96) + '.*', self.cursor.execute.call_args.args[0]) + items = error.partial_result.items + self.assertEqual( + [item.state for item in items], + [ + BatchState.FAILED, + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.FAILED, + ], + ) + self.assertIsNone(items[1].result.record) + self.assertEqual(items[2].result.record, {'value': 'c'}) + + def test_read_group_keeps_confirmed_observations_when_cursor_close_fails(self): + original = RuntimeError('close failed') + self.cursor.close.side_effect = original + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(0, 'confirmed')] + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.get_many([key(1), key(9), key(1), key(7)], fields=('value',), batch_size=2) + + error = caught.exception + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + items = error.partial_result.items + self.assertEqual( + [item.state for item in items], + [ + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.NOT_SENT, + ], + ) + self.assertEqual(items[0].result.record, {'value': 'confirmed'}) + self.assertIsNone(items[1].result.record) + self.assertEqual(items[2].result.record, {'value': 'confirmed'}) + self.cursor.execute.assert_called_once() + + def test_preflight_rejects_bad_later_value_and_key_alias_before_any_write(self): + with sync_records(self.connection) as records: + with self.assertRaises(ConfigurationError): + records.insert_many( + [{'tenant': 1, 'id': 1, 'value': 1}, {'tenant': 1, 'id': 2, 'value': ['unsupported']}], + batch_size=1, + ) + for unsupported in (float('inf'), float('nan'), Decimal('NaN')): + with self.subTest(unsupported=unsupported), self.assertRaises(ConfigurationError): + records.insert_many( + [{'tenant': 1, 'id': 1, 'value': 1}, {'tenant': 1, 'id': 2, 'value': unsupported}], + batch_size=1, + ) + with self.assertRaises(ConfigurationError): + records.update(key(1), set_values={'ID': 2}) + with self.assertRaises(ConfigurationError): + records.bulk_update([{'key': key(1), 'set_values': {'ID': 2}}]) + with self.assertRaises(ConfigurationError): + records.update_many(where={'tenant': 1}, set_values={'ID': 2}) + with self.assertRaises(ConfigurationError): + records.find(fields=('value%s',), limit=1) + with self.assertRaises(ConfigurationError): + records.find(fields=('value', 'VALUE'), limit=1) + with self.assertRaises(ConfigurationError): + records.find(order_by=(('value', 'asc'), ('VALUE', 'desc')), limit=1) + with self.assertRaises(UnsupportedOperationError): + records.count(where=Exists('value')) + + self.native.cursor.assert_not_called() + + def test_insert_preserves_lastrowid_without_inferred_identity_or_transaction_control(self): + self.cursor.lastrowid = 37 + + with sync_records(self.connection) as records: + result = records.insert({'value': Decimal('12.3400')}) + + self.assertEqual(result.inserted_count, 1) + self.assertEqual(result.native_result, {'rowcount': 1, 'lastrowid': 37}) + self.assertIsNone(result.generated_keys) + self.assertEqual(self.cursor.execute.call_args.args[1], (Decimal('12.3400'),)) + self.native.commit.assert_not_called() + self.native.rollback.assert_not_called() + + def test_update_preserves_changed_matched_and_unknown_modes(self): + self.native.isset_client_flag.return_value = True + self.native._cmysql.st_client_flag.side_effect = [0, ClientFlag.FOUND_ROWS, None] + self.cursor.rowcount = 2 + + with sync_records(self.connection) as records: + results = [records.update_many(where=In('id', (1, 2)), set_values={'value': 0}) for _ in range(3)] + + self.assertEqual( + [(item.matched_count, item.modified_count, item.native_result['rowcount_mode']) for item in results], + [(None, 2, 'changed'), (2, None, 'matched'), (None, None, 'unknown')], + ) + self.assertEqual(self.native._cmysql.st_client_flag.call_args_list, [call()] * 3) + self.native.isset_client_flag.assert_not_called() + + def test_bulk_update_groups_only_contiguous_equivalent_columns(self): + items = [ + {'key': key(1), 'set_values': {'value': 'a', 'note': 'first'}}, + {'key': key(2), 'set_values': {'note': 'second', 'value': 'b'}}, + {'key': key(3), 'set_values': {'value': 'c'}}, + {'key': key(4), 'set_values': {'value': 'd', 'note': 'last'}}, + ] + + with sync_records(self.connection) as records: + result = records.bulk_update(items) + + groups = self.cursor.executemany.call_args_list + self.assertEqual(len(groups), 3) + self.assertEqual(groups[0].args[1], [('first', 'a', 1, 1), ('second', 'b', 1, 2)]) + self.assertEqual(groups[1].args[1], [('c', 1, 3)]) + self.assertEqual(groups[2].args[1], [('last', 'd', 1, 4)]) + self.assertEqual(result.counts.modified_count, 3) + self.assertTrue(all(item.state is BatchState.ACKNOWLEDGED for item in result.items)) + self.assertIsNone(result.items[0].result) + self.assertEqual(result.items[2].result.modified_count, 1) + self.native.commit.assert_not_called() + + def test_delete_many_uses_whole_composite_keys_in_one_statement(self): + self.cursor.rowcount = 1 + + with sync_records(self.connection) as records: + result = records.delete_many([key(1, tenant=1), key(2, tenant=2)]) + + self.assertEqual(result.counts.deleted_count, 1) + sql, parameters = self.cursor.execute.call_args.args + quote = chr(96) + predicate = f'({quote}tenant{quote} = %s AND {quote}id{quote} = %s)' + self.assertEqual(sql, f'DELETE FROM {quote}sample{quote} WHERE {predicate} OR {predicate}') + self.assertEqual(parameters, (1, 1, 2, 2)) + self.cursor.executemany.assert_not_called() + self.assertTrue(all(item.result is None for item in result.items)) + + def test_failed_subgroup_is_unknown_and_remaining_statements_are_not_sent(self): + original = IntegrityError(errno=ER_DUP_ENTRY) + cleanup = RuntimeError('cursor close failed') + self.cursor.executemany.side_effect = [None, original] + self.cursor.close.side_effect = cleanup + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.insert_many( + [ + {'tenant': 1, 'id': 1, 'value': 'ok'}, + {'tenant': 1, 'id': 2, 'note': 'unknown'}, + {'tenant': 1, 'id': 3, 'note': 'unknown'}, + {'tenant': 1, 'id': 4, 'value': 'not sent'}, + ] + ) + + error = caught.exception + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + self.assertEqual(error.database_cleanup_errors, (cleanup,)) + self.assertEqual( + [item.state for item in error.partial_result.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.UNKNOWN, + BatchState.UNKNOWN, + BatchState.NOT_SENT, + ], + ) + self.assertEqual(error.partial_result.native_results[0]['rowcount'], 1) + self.assertIsNone(error.partial_result.counts.inserted_count) + self.assertEqual(self.cursor.executemany.call_count, 2) + + def test_cursor_acquisition_failure_keeps_precise_preceding_counts(self): + original = ProgrammingError('could not allocate cursor') + self.native.cursor.side_effect = [self.cursor, original] + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.insert_many([{'tenant': 1, 'id': 1}, {'tenant': 1, 'id': 2}], batch_size=1) + + partial = caught.exception.partial_result + self.assertEqual([item.state for item in partial.items], [BatchState.ACKNOWLEDGED, BatchState.NOT_SENT]) + self.assertEqual(partial.counts.inserted_count, 1) + self.assertIs(caught.exception.original, original) + + def test_cursor_acquisition_failure_has_no_sent_writes(self): + original = ProgrammingError('could not allocate cursor') + self.native.cursor.side_effect = original + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.delete_many([key(1), key(2)]) + + self.assertIs(caught.exception.original, original) + self.assertTrue(all(item.state is BatchState.NOT_SENT for item in caught.exception.partial_result.items)) + + def test_record_conflict_preserves_the_driver_error_and_close_failure(self): + original = IntegrityError(errno=ER_DUP_ENTRY) + cleanup = RuntimeError('close failed') + self.cursor.execute.side_effect = original + self.cursor.close.side_effect = cleanup + + with sync_records(self.connection) as records: + with self.assertRaises(RecordConflictError) as caught: + records.insert({'id': 1, 'tenant': 1}) + + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.database_cleanup_errors, (cleanup,)) + + def test_unknown_column_remains_a_driver_error_and_closes_cursor(self): + original = ProgrammingError(errno=1054) + self.cursor.execute.side_effect = original + + with sync_records(self.connection) as records: + with self.assertRaises(DriverError) as caught: + records.get(key(1), fields=('unknown_column',)) + + self.assertIs(caught.exception.original, original) + self.cursor.close.assert_called_once_with() + + +class AsyncMySQLCRUDTests(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.connection, self.native, self.cursor = make_handle(is_async=True) + + async def test_get_many_validates_projection_before_sending_any_chunk(self): + async with async_records(self.connection) as records: + with self.assertRaises(ConfigurationError): + await records.get_many([key(1), key(2)], fields=('bad%s',), batch_size=1) + + self.native.cursor.assert_not_awaited() + + async def test_async_reads_consume_and_close_the_native_cursor(self): + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(1, 'second'), (0, 'first')] + + async with async_records(self.connection) as records: + result = await records.get_many([key(1), key(2)], fields=('value',)) + + self.assertEqual([item.record for item in result], [{'value': 'first'}, {'value': 'second'}]) + self.native.cursor.assert_awaited_once_with(buffered=True) + self.cursor.execute.assert_awaited_once() + self.cursor.fetchall.assert_awaited_once_with() + self.cursor.close.assert_awaited_once_with() + self.native.close.assert_not_called() + + async def test_read_group_preserves_results_if_close_is_cancelled(self): + cancellation = asyncio.CancelledError() + self.cursor.close.side_effect = cancellation + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(1, 'confirmed')] + + async with async_records(self.connection) as records: + with self.assertRaises(asyncio.CancelledError) as caught: + await records.get_many([key(1), key(2), key(3)], fields=('value',), batch_size=2) + + self.assertIs(caught.exception, cancellation) + items = cancellation.database_partial_result.items + self.assertEqual( + [item.state for item in items], + [ + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.NOT_SENT, + ], + ) + self.assertIsNone(items[0].result.record) + self.assertEqual(items[1].result.record, {'value': 'confirmed'}) + self.cursor.execute.assert_awaited_once() + + async def test_async_writes_preserve_rowcount_and_use_native_coroutines(self): + self.cursor.rowcount = 2 + self.native._cmysql = None + self.native.isset_client_flag.return_value = True + + async with async_records(self.connection) as records: + batch = await records.bulk_update( + [ + {'key': key(1), 'set_values': {'value': 'first'}}, + {'key': key(2), 'set_values': {'value': 'second'}}, + ] + ) + delete = await records.delete_where(where={'tenant': 1}) + + self.assertEqual((batch.counts.matched_count, batch.counts.modified_count), (None, None)) + self.assertEqual(batch.native_results[0]['rowcount_mode'], 'unknown') + self.native.isset_client_flag.assert_not_called() + self.assertEqual(delete.deleted_count, 2) + self.cursor.executemany.assert_awaited_once() + self.cursor.execute.assert_awaited_once() + self.assertEqual(self.cursor.close.await_count, 2) + self.native.commit.assert_not_called() + self.native.rollback.assert_not_called() + + async def test_cancelled_write_preserves_partial_progress_and_cleanup_error(self): + cancellation = asyncio.CancelledError() + cleanup = RuntimeError('close failed') + self.cursor.executemany.side_effect = [None, cancellation] + self.cursor.close.side_effect = cleanup + + async with async_records(self.connection) as records: + with self.assertRaises(asyncio.CancelledError) as caught: + await records.insert_many( + [ + {'tenant': 1, 'id': 1, 'value': 'acknowledged'}, + {'tenant': 1, 'id': 2, 'note': 'unknown'}, + {'tenant': 1, 'id': 3, 'value': 'not sent'}, + ] + ) + + self.assertIs(caught.exception, cancellation) + self.assertEqual(cancellation.database_cleanup_errors, (cleanup,)) + partial = cancellation.database_partial_result + self.assertEqual( + [item.state for item in partial.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.UNKNOWN, + BatchState.NOT_SENT, + ], + ) + self.assertEqual(partial.native_results[0]['rowcount'], 1) + self.assertIsNone(partial.counts.inserted_count) + self.cursor.close.assert_awaited_once() -- Gitee From ab20cef20c919bdaeb57aab74a7eef966476273c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:45:45 +0800 Subject: [PATCH 57/86] =?UTF-8?q?fix(crud):=20=E4=BF=9D=E7=95=99=E5=8E=9F?= =?UTF-8?q?=E7=94=9F=20BSON=20Int64=20=E9=94=AE=E7=9A=84=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E8=83=BD=E5=8A=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_conditions.py | 3 ++- tests/test_crud_conditions.py | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/database/crud/_conditions.py b/src/database/crud/_conditions.py index cd21da3..4367389 100644 --- a/src/database/crud/_conditions.py +++ b/src/database/crud/_conditions.py @@ -99,7 +99,8 @@ def is_sequence(value: Any) -> bool: def validate_scalar(value: Any) -> None: object_id = getattr(sys.modules.get('bson.objectid'), 'ObjectId', None) - if type(value) not in _SCALARS and type(value) is not object_id: + int64 = getattr(sys.modules.get('bson.int64'), 'Int64', None) + if type(value) not in _SCALARS and type(value) not in (object_id, int64): raise ConfigurationError('Expected a supported scalar comparison or key value.', phase='config') if (type(value) is float and not math.isfinite(value)) or (type(value) is Decimal and not value.is_finite()): raise ConfigurationError('Comparison and key values must be finite.', phase='config') diff --git a/tests/test_crud_conditions.py b/tests/test_crud_conditions.py index 601c103..82b923b 100644 --- a/tests/test_crud_conditions.py +++ b/tests/test_crud_conditions.py @@ -15,6 +15,20 @@ from database.errors import ConfigurationError class ConditionTests(unittest.TestCase): + def test_native_bson_int64_remains_usable_as_a_key_and_condition(self): + try: + from bson.int64 import Int64 + except ImportError: + self.skipTest('PyMongo is needed to test its native Int64 type.') + from database.crud._batch import unique_keys + + identity = Int64(2**40) + condition = normalize_where({'_id': identity}, 'mongodb', 'find') + self.assertIs(condition.value, identity) + keys, positions = unique_keys([{'_id': identity}, {'_id': identity}]) + self.assertIs(keys[0]['_id'], identity) + self.assertEqual(positions, [(0, 1)]) + def test_dict_becomes_explicit_and_null(self): result = normalize_where({'tenant_id': 7, 'deleted_at': None}, 'mongodb', 'find') self.assertEqual(result, And(Eq('tenant_id', 7), IsNull('deleted_at'))) -- Gitee From c0e4246b54274d3222ed75ab57141e492da6131f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:46:50 +0800 Subject: [PATCH 58/86] =?UTF-8?q?fix(mongodb):=20=E4=BF=9D=E7=95=99?= =?UTF-8?q?=E6=B8=B8=E6=A0=87=E6=B8=85=E7=90=86=E5=89=8D=E5=B7=B2=E7=A1=AE?= =?UTF-8?q?=E8=AE=A4=E7=9A=84=E6=89=B9=E9=87=8F=E8=AF=BB=E5=8F=96=E7=BB=93?= =?UTF-8?q?=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_mongodb.py | 53 ++++++++++++++++++++--------- tests/test_crud_mongodb.py | 63 +++++++++++++++++++++++++++++++++++ 2 files changed, 100 insertions(+), 16 deletions(-) diff --git a/src/database/crud/_mongodb.py b/src/database/crud/_mongodb.py index e53b7a2..a5f3b34 100644 --- a/src/database/crud/_mongodb.py +++ b/src/database/crud/_mongodb.py @@ -7,6 +7,8 @@ from collections.abc import Mapping from collections.abc import Sequence from contextlib import asynccontextmanager from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import field from dataclasses import replace from types import TracebackType from typing import Any @@ -123,18 +125,29 @@ async def _async_closing(cursor: Any) -> AsyncIterator[Any]: await cursor.close() +@dataclass +class _ReadProgress: + rows: list[Record | None] = field(default_factory=list) + dispatched: int = 0 + + @contextmanager -def _read_batch(keys: Sequence[Record]) -> Iterator[list[Record | None]]: - rows: list[Record | None] = [] +def _read_batch(keys: Sequence[Record]) -> Iterator[_ReadProgress]: + progress = _ReadProgress() try: - yield rows + yield progress except (Exception, asyncio.CancelledError) as error: - completed = [ + outcomes = [ BatchItem(index, keys[index], BatchState.ACKNOWLEDGED, ReadItem(index, keys[index], row)) - for index, row in enumerate(rows) + for index, row in enumerate(progress.rows) ] - unsent = [BatchItem(index, keys[index], BatchState.NOT_SENT) for index in range(len(rows) + 1, len(keys))] - partial = ChunkResult(tuple(completed + unsent)) + if isinstance(error, RecordIdentityError) and len(progress.rows) < progress.dispatched: + index = progress.dispatched - 1 + outcomes.append(BatchItem(index, keys[index], BatchState.FAILED, error=error)) + outcomes.extend( + BatchItem(index, keys[index], BatchState.NOT_SENT) for index in range(progress.dispatched, len(keys)) + ) + partial = ChunkResult(tuple(outcomes)) if isinstance(error, asyncio.CancelledError): error.database_partial_result = partial raise @@ -299,24 +312,27 @@ class _MongoRecords: return replace(item, prepared=(query, self._update_document(item.values))) return replace(item, prepared=query) - def _unique_record(self, rows: list[Record], fields: _Fields) -> Record | None: + def _unique_record(self, rows: list[Record], fields: _Fields, *, operation: str = 'get') -> Record | None: if len(rows) > 1: raise RecordIdentityError( 'The declared MongoDB key matched more than one document.', connection_name=self._connection.name, backend=_BACKEND, phase='operation', - operation='get', + operation=operation, ) return _record(rows[0], fields) if rows else None def _find_cursor(self, query: Record, fields: _Fields, limit: int, offset: int = 0, order_by: _Order = ()) -> Any: + sort = [(field, 1 if direction == 'asc' else -1) for field, direction in order_by] + if sort: + self._validate_bson(dict(sort)) return self._collection.find( query, projection=_projection(fields), limit=limit, skip=offset, - sort=[(field, 1 if direction == 'asc' else -1) for field, direction in order_by] or None, + sort=sort or None, session=self._session, ) @@ -379,10 +395,12 @@ class Adapter(_MongoRecords): return self._unique_record(list(cursor), fields) def get_many(self, keys: Sequence[Record], fields: _Fields) -> list[Record | None]: - with _read_batch(keys) as rows: + with _read_batch(keys) as progress: for key in keys: - rows.append(self.get(key, fields)) - return rows + with _closing(self._find_cursor(_key_filter(key), fields, 2)) as cursor: + progress.dispatched += 1 + progress.rows.append(self._unique_record(list(cursor), fields, operation='get_many')) + return progress.rows def find(self, where: Condition | None, fields: _Fields, limit: int, offset: int, order_by: _Order) -> list[Record]: query = self._filter(where) @@ -454,10 +472,13 @@ class AsyncAdapter(_MongoRecords): return self._unique_record([row async for row in cursor], fields) async def get_many(self, keys: Sequence[Record], fields: _Fields) -> list[Record | None]: - with _read_batch(keys) as rows: + with _read_batch(keys) as progress: for key in keys: - rows.append(await self.get(key, fields)) - return rows + async with _async_closing(self._find_cursor(_key_filter(key), fields, 2)) as cursor: + progress.dispatched += 1 + matches = [row async for row in cursor] + progress.rows.append(self._unique_record(matches, fields, operation='get_many')) + return progress.rows async def find( self, where: Condition | None, fields: _Fields, limit: int, offset: int, order_by: _Order diff --git a/tests/test_crud_mongodb.py b/tests/test_crud_mongodb.py index 1296af9..fad42d8 100644 --- a/tests/test_crud_mongodb.py +++ b/tests/test_crud_mongodb.py @@ -200,6 +200,47 @@ class MongoDBRecordsTests(TestCase): self.assertEqual(error.partial_result.items[2].result.record, {'name': 'Alice'}) self.assertEqual(self.collection.find.call_count, 2) + def test_cursor_close_failure_preserves_observed_absence(self): + cleanup = AutoReconnect('could not close cursor') + cursor = _Cursor([], cleanup=cleanup) + self.collection.find.return_value = cursor + with self.records() as records, self.assertRaises(BatchOperationError) as caught: + records.get_many([{'_id': 1}, {'_id': 2}]) + + error = caught.exception + self.assertIs(error.original, cleanup) + observed, pending = error.partial_result.items + self.assertIs(observed.state, BatchState.ACKNOWLEDGED) + self.assertIsNone(observed.result.record) + self.assertIs(pending.state, BatchState.NOT_SENT) + cursor.close.assert_called_once() + + def test_batch_reads_report_nonunique_record_as_failed(self): + self.collection.find.return_value = _Cursor([{'_id': 1}, {'_id': 2}]) + with self.records(key_fields=('name',)) as records, self.assertRaises(BatchOperationError) as caught: + records.get_many([{'name': 'Alice'}, {'name': 'Bob'}]) + + error = caught.exception + self.assertIsInstance(error.original, RecordIdentityError) + self.assertEqual(error.original.operation, 'get_many') + matched, pending = error.partial_result.items + self.assertIs(matched.state, BatchState.FAILED) + self.assertIs(matched.error, error.original) + self.assertIs(pending.state, BatchState.NOT_SENT) + + def test_projection_and_sort_encoding_are_validated_before_driving_reads(self): + invalid_field = 'bad\ud800' + with self.records() as records: + with self.assertRaises(ConfigurationError): + records.get({'_id': 1}, fields=[invalid_field]) + with self.assertRaises(ConfigurationError): + records.get_many([{'_id': 1}], fields=[invalid_field]) + with self.assertRaises(ConfigurationError): + records.find(fields=[invalid_field], limit=1) + with self.assertRaises(ConfigurationError): + records.find(order_by=[(invalid_field, 'asc')], limit=1) + self.collection.find.assert_not_called() + def test_empty_condition_has_no_driver_requests(self): with self.records() as records: where = And(Eq('state', 'pending'), In('_id', [])) @@ -386,6 +427,28 @@ class AsyncMongoDBRecordsTests(IsolatedAsyncioTestCase): for call in getattr(self.collection, name).await_args_list: self.assertIs(call.kwargs['session'], self.session) + async def test_cursor_close_cancellation_preserves_observed_record(self): + original = asyncio.CancelledError() + cursor = _AsyncCursor([{'name': 'Alice'}]) + cursor.close.side_effect = original + self.collection.find.return_value = cursor + async with self.records() as records: + with self.assertRaises(asyncio.CancelledError) as caught: + await records.get_many([{'_id': 1}, {'_id': 2}], fields=['name']) + + self.assertIs(caught.exception, original) + observed, pending = original.database_partial_result.items + self.assertIs(observed.state, BatchState.ACKNOWLEDGED) + self.assertEqual(observed.result.record, {'name': 'Alice'}) + self.assertIs(pending.state, BatchState.NOT_SENT) + cursor.close.assert_awaited_once() + + async def test_batch_projection_encoding_is_validated_before_any_reads(self): + async with self.records() as records: + with self.assertRaises(ConfigurationError): + await records.get_many([{'_id': 1}], fields=['bad\ud800']) + self.collection.find.assert_not_called() + async def test_read_batch_cancellation_preserves_identity_and_evidence(self): original = asyncio.CancelledError() first = _AsyncCursor([{'name': 'Alice'}]) -- Gitee From a2b22276b88e6fdd67c90d82b352048ac26f4407 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:50:12 +0800 Subject: [PATCH 59/86] =?UTF-8?q?fix(crud):=20=E4=BF=9D=E7=95=99=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E6=B4=BE=E5=8F=91=E7=8A=B6=E6=80=81=E4=B8=8E=E5=8F=96?= =?UTF-8?q?=E6=B6=88=E5=BC=82=E5=B8=B8=E9=93=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_records.py | 66 +++++++++++++---- tests/test_crud_core.py | 134 ++++++++++++++++++++++++++++++++++ 2 files changed, 187 insertions(+), 13 deletions(-) diff --git a/src/database/crud/_records.py b/src/database/crud/_records.py index f8d32c4..b659f40 100644 --- a/src/database/crud/_records.py +++ b/src/database/crud/_records.py @@ -309,19 +309,27 @@ class _Records: seen.add(token) return Batch([item.key for item in items]), items - def _batch_failed(self, operation: str, error: BaseException, batch: Batch, positions: Positions) -> NoReturn: + def _batch_failed( + self, operation: str, error: BaseException, batch: Batch, positions: Positions, *, dispatched: bool + ) -> NoReturn: if isinstance(error, ChunkError): chunk, source = error.partial_result, error.original else: chunk = getattr(error, 'database_partial_result', None) source = error - batch.absorb(positions, chunk if isinstance(chunk, ChunkResult) else None) + if dispatched: + batch.absorb(positions, chunk if isinstance(chunk, ChunkResult) else None) partial = batch.result(operation) if isinstance(source, asyncio.CancelledError): BaseException.__dict__['__dict__'].__get__(source)['database_partial_result'] = partial - if source is error: - raise - raise source + if source is not error: + attach_cleanup_errors(source, getattr(error, 'database_cleanup_errors', ())) + context = source.__context__ + try: + raise source + finally: + # The caller may already be handling an unrelated exception. + source.__context__ = context original = source.original if isinstance(source, DatabaseError) and source.original is not None else source wrapped = BatchOperationError( 'A database record batch failed.', @@ -371,11 +379,19 @@ class Records(_Records): for start in range(0, len(unique), batch_size): end = start + batch_size group = positions[start:end] + dispatched = False try: - rows = self._invoke('get_many', unique[start:end], projected) + self._check_active() + with self._context('get_many'): + read = self._adapter.get_many + dispatched = True + rows = read(unique[start:end], projected) batch.absorb_reads(group, rows) except (Exception, asyncio.CancelledError) as error: - self._batch_failed('get_many', error, batch, group) + failure = error + else: + continue + self._batch_failed('get_many', failure, batch, group, dispatched=dispatched) return batch.read_items() def find( @@ -422,11 +438,19 @@ class Records(_Records): for start in range(0, len(items), batch_size): end = min(start + batch_size, len(items)) positions = [(index,) for index in range(start, end)] + dispatched = False try: - chunk = self._invoke(operation, items[start:end]) + self._check_active() + with self._context(operation): + write = getattr(self._adapter, operation) + dispatched = True + chunk = write(items[start:end]) batch.absorb(positions, chunk) except (Exception, asyncio.CancelledError) as error: - self._batch_failed(operation, error, batch, positions) + failure = error + else: + continue + self._batch_failed(operation, failure, batch, positions, dispatched=dispatched) return batch.result(operation) @@ -470,11 +494,19 @@ class AsyncRecords(_Records): for start in range(0, len(unique), batch_size): end = start + batch_size group = positions[start:end] + dispatched = False try: - rows = await self._invoke('get_many', unique[start:end], projected) + self._check_active() + with self._context('get_many'): + read = self._adapter.get_many + dispatched = True + rows = await read(unique[start:end], projected) batch.absorb_reads(group, rows) except (Exception, asyncio.CancelledError) as error: - self._batch_failed('get_many', error, batch, group) + failure = error + else: + continue + self._batch_failed('get_many', failure, batch, group, dispatched=dispatched) return batch.read_items() async def find( @@ -525,9 +557,17 @@ class AsyncRecords(_Records): for start in range(0, len(items), batch_size): end = min(start + batch_size, len(items)) positions = [(index,) for index in range(start, end)] + dispatched = False try: - chunk = await self._invoke(operation, items[start:end]) + self._check_active() + with self._context(operation): + write = getattr(self._adapter, operation) + dispatched = True + chunk = await write(items[start:end]) batch.absorb(positions, chunk) except (Exception, asyncio.CancelledError) as error: - self._batch_failed(operation, error, batch, positions) + failure = error + else: + continue + self._batch_failed(operation, failure, batch, positions, dispatched=dispatched) return batch.result(operation) diff --git a/tests/test_crud_core.py b/tests/test_crud_core.py index ade9e30..1f8c2f7 100644 --- a/tests/test_crud_core.py +++ b/tests/test_crud_core.py @@ -48,6 +48,18 @@ def adapter_result(*items, counts=WriteCounts(), native_results=()): return ChunkResult(tuple(items), counts, native_results) +def raise_cancelled_chunk(error, cause, result, cleanup): + try: + raise cause + except FakeFailure: + try: + raise error from cause + except asyncio.CancelledError: + failure = ChunkError(error, result) + failure.database_cleanup_errors = (cleanup,) + raise failure from error + + def bind(connection, adapter, *, asynchronous=False): factory = Mock(return_value=adapter) with patch( @@ -284,6 +296,65 @@ class RecordsTests(unittest.TestCase): ) self.assertEqual(self.adapter.delete_many.call_count, 4) + def test_connection_closed_between_read_chunks_keeps_unsent_status(self): + def first_chunk(keys, fields): + self.connection.close() + return [None] + + self.adapter.get_many.side_effect = first_chunk + cleanup = FakeFailure('driver cleanup failed') + self.adapter.__exit__.side_effect = cleanup + + with self.assertRaises(BatchOperationError) as caught: + with self.records: + self.records.get_many([{'id': 1}, {'id': 2}, {'id': 3}], batch_size=1) + + failure = caught.exception + self.adapter.get_many.assert_called_once_with([{'id': 1}], None) + self.assertIsInstance(failure.original, ConnectionClosedError) + self.assertIs(failure.__cause__, failure.original) + self.assertIs(failure.database_cleanup_errors[0].__cause__, cleanup) + items = failure.partial_result.items + self.assertEqual( + [item.state for item in items], + [BatchState.ACKNOWLEDGED, BatchState.NOT_SENT, BatchState.NOT_SENT], + ) + self.assertEqual(items[0].result, ReadItem(0, {'id': 1}, None)) + + def test_wrapped_batch_cancellation_preserves_original_chain_and_evidence(self): + error = asyncio.CancelledError('cancelled after receiving a partial write reply') + cause = FakeFailure('driver cancellation cause') + cleanup = FakeFailure('chunk cleanup failed') + completed = adapter_result( + BatchItem(0, None, BatchState.ACKNOWLEDGED), counts=WriteCounts(deleted_count=1), native_results=('reply',) + ) + + def cancel_chunk(items): + if items[0].key == {'id': 1}: + return completed + raise_cancelled_chunk(error, cause, completed, cleanup) + + self.adapter.delete_many.side_effect = cancel_chunk + with self.assertRaises(asyncio.CancelledError) as caught: + with self.records: + try: + raise ValueError('an unrelated caller error') + except ValueError: + self.records.delete_many([{'id': 1}, {'id': 2}, {'id': 3}], batch_size=1) + + self.assertIs(caught.exception, error) + self.assertIs(error.__cause__, cause) + self.assertIs(error.__context__, cause) + self.assertEqual(error.database_cleanup_errors, (cleanup,)) + partial = error.database_partial_result + self.assertEqual( + [item.state for item in partial.items], + [BatchState.ACKNOWLEDGED, BatchState.ACKNOWLEDGED, BatchState.NOT_SENT], + ) + self.assertEqual(partial.counts.deleted_count, 2) + self.assertEqual(partial.native_results, ('reply', 'reply')) + self.assertEqual(self.adapter.delete_many.call_count, 2) + def test_ttl_is_explicit_and_redis_conditions_are_unsupported(self): record = {'id': 1, 'ttl_s': 'business field'} redis_connection = Connection(object(), self.owner, 'cache', 'redis', 'single') @@ -360,6 +431,69 @@ class AsyncRecordsTests(unittest.IsolatedAsyncioTestCase): ) self.assertIs(error.database_cleanup_errors[0].original, cleanup) + async def test_connection_closed_between_write_chunks_keeps_unsent_status(self): + async def first_chunk(items): + await self.connection.close() + return adapter_result( + BatchItem(0, None, BatchState.ACKNOWLEDGED), + counts=WriteCounts(matched_count=1, modified_count=1), + native_results=('reply',), + ) + + self.adapter.bulk_update.side_effect = first_chunk + cleanup = FakeFailure('driver cleanup failed') + self.adapter.__aexit__.side_effect = cleanup + with self.assertRaises(BatchOperationError) as caught: + async with self.records: + await self.records.bulk_update( + [{'key': {'id': n}, 'set_values': {'value': n}} for n in range(3)], + batch_size=1, + ) + + failure = caught.exception + self.adapter.bulk_update.assert_awaited_once() + self.assertIsInstance(failure.original, ConnectionClosedError) + self.assertIs(failure.__cause__, failure.original) + self.assertIs(failure.database_cleanup_errors[0].__cause__, cleanup) + partial = failure.partial_result + self.assertEqual( + [item.state for item in partial.items], + [BatchState.ACKNOWLEDGED, BatchState.NOT_SENT, BatchState.NOT_SENT], + ) + self.assertEqual(partial.counts, WriteCounts(matched_count=1, modified_count=1)) + self.assertEqual(partial.native_results, ('reply',)) + + async def test_wrapped_read_cancellation_keeps_original_and_partial_items(self): + error = asyncio.CancelledError('cancelled during a read chunk') + cause = FakeFailure('driver cancellation cause') + cleanup = FakeFailure('chunk cleanup failed') + completed = adapter_result( + BatchItem(0, None, BatchState.ACKNOWLEDGED, ReadItem(0, {'id': 2}, {'value': 'second'})) + ) + + def cancel_chunk(keys, fields): + if keys[0] == {'id': 1}: + return [None] + raise_cancelled_chunk(error, cause, completed, cleanup) + + self.adapter.get_many.side_effect = cancel_chunk + with self.assertRaises(asyncio.CancelledError) as caught: + async with self.records: + await self.records.get_many([{'id': 1}, {'id': 2}, {'id': 2}, {'id': 3}], batch_size=1) + + self.assertIs(caught.exception, error) + self.assertIs(error.__cause__, cause) + self.assertIs(error.__context__, cause) + self.assertEqual(error.database_cleanup_errors, (cleanup,)) + items = error.database_partial_result.items + self.assertEqual( + [item.state for item in items], + [BatchState.ACKNOWLEDGED, BatchState.ACKNOWLEDGED, BatchState.ACKNOWLEDGED, BatchState.NOT_SENT], + ) + self.assertEqual(items[0].result, ReadItem(0, {'id': 1}, None)) + self.assertEqual(items[2].result, ReadItem(2, {'id': 2}, {'value': 'second'})) + self.assertEqual(self.adapter.get_many.await_count, 2) + async def test_operations_check_event_loop_before_driver_call(self): async with self.records: -- Gitee From a6d778f2a919695ae77012e2426e8c26ce57ea0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:53:01 +0800 Subject: [PATCH 60/86] =?UTF-8?q?fix(mongodb):=20=E4=BF=9D=E7=95=99?= =?UTF-8?q?=E5=8F=AA=E8=AF=BB=E5=8F=96=E6=B6=88=E5=BC=82=E5=B8=B8=E7=9A=84?= =?UTF-8?q?=E6=89=B9=E9=87=8F=E8=AF=BB=E5=8F=96=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_mongodb.py | 2 +- tests/test_crud_mongodb.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/database/crud/_mongodb.py b/src/database/crud/_mongodb.py index a5f3b34..5a499fe 100644 --- a/src/database/crud/_mongodb.py +++ b/src/database/crud/_mongodb.py @@ -149,7 +149,7 @@ def _read_batch(keys: Sequence[Record]) -> Iterator[_ReadProgress]: ) partial = ChunkResult(tuple(outcomes)) if isinstance(error, asyncio.CancelledError): - error.database_partial_result = partial + BaseException.__dict__['__dict__'].__get__(error)['database_partial_result'] = partial raise raise ChunkError(error, partial) from error diff --git a/tests/test_crud_mongodb.py b/tests/test_crud_mongodb.py index fad42d8..04ea7c0 100644 --- a/tests/test_crud_mongodb.py +++ b/tests/test_crud_mongodb.py @@ -450,7 +450,13 @@ class AsyncMongoDBRecordsTests(IsolatedAsyncioTestCase): self.collection.find.assert_not_called() async def test_read_batch_cancellation_preserves_identity_and_evidence(self): - original = asyncio.CancelledError() + class ReadOnlyCancellation(asyncio.CancelledError): + def __setattr__(self, name, value): + if name == 'database_partial_result': + raise AttributeError('read-only cancellation metadata') + super().__setattr__(name, value) + + original = ReadOnlyCancellation() first = _AsyncCursor([{'name': 'Alice'}]) interrupted = _AsyncCursor([], error=original) self.collection.find.side_effect = [first, interrupted] -- Gitee From f32db99a7587dee9a4432bac8abc71a0a91f04a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:53:31 +0800 Subject: [PATCH 61/86] =?UTF-8?q?fix(crud):=20=E4=BF=9D=E7=95=99=20BSON=20?= =?UTF-8?q?=E7=B2=BE=E7=A1=AE=E5=B0=8F=E6=95=B0=E6=9D=A1=E4=BB=B6=E4=B8=8E?= =?UTF-8?q?=E6=89=B9=E9=87=8F=E9=94=AE=E8=BA=AB=E4=BB=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_batch.py | 4 +++- src/database/crud/_conditions.py | 5 ++++- tests/test_crud_conditions.py | 17 +++++++++++++++++ 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/database/crud/_batch.py b/src/database/crud/_batch.py index c266954..a8b1c96 100644 --- a/src/database/crud/_batch.py +++ b/src/database/crud/_batch.py @@ -1,5 +1,6 @@ """Preserve fetch ordering and evidence from completed batch chunks.""" +import sys from collections.abc import Sequence from dataclasses import replace from typing import Any @@ -17,7 +18,8 @@ Positions = Sequence[tuple[int, ...]] def identity(key: Record) -> tuple[tuple[type, Any], ...]: """Compare previously validated scalar keys without conflating Python types.""" - return tuple((type(value), value) for value in key.values()) + decimal128 = getattr(sys.modules.get('bson.decimal128'), 'Decimal128', None) + return tuple((type(value), value.bid if type(value) is decimal128 else value) for value in key.values()) def unique_keys(keys: Sequence[Record]) -> tuple[list[Record], list[tuple[int, ...]]]: diff --git a/src/database/crud/_conditions.py b/src/database/crud/_conditions.py index 4367389..b9b52c1 100644 --- a/src/database/crud/_conditions.py +++ b/src/database/crud/_conditions.py @@ -100,10 +100,13 @@ def is_sequence(value: Any) -> bool: def validate_scalar(value: Any) -> None: object_id = getattr(sys.modules.get('bson.objectid'), 'ObjectId', None) int64 = getattr(sys.modules.get('bson.int64'), 'Int64', None) - if type(value) not in _SCALARS and type(value) not in (object_id, int64): + decimal128 = getattr(sys.modules.get('bson.decimal128'), 'Decimal128', None) + if type(value) not in _SCALARS and type(value) not in (object_id, int64, decimal128): raise ConfigurationError('Expected a supported scalar comparison or key value.', phase='config') if (type(value) is float and not math.isfinite(value)) or (type(value) is Decimal and not value.is_finite()): raise ConfigurationError('Comparison and key values must be finite.', phase='config') + if type(value) is decimal128 and not value.to_decimal().is_finite(): + raise ConfigurationError('Comparison and key values must be finite.', phase='config') def normalize_where(where: Where, backend: str, operation: str) -> Condition | None: diff --git a/tests/test_crud_conditions.py b/tests/test_crud_conditions.py index 82b923b..11d5bdd 100644 --- a/tests/test_crud_conditions.py +++ b/tests/test_crud_conditions.py @@ -15,6 +15,23 @@ from database.errors import ConfigurationError class ConditionTests(unittest.TestCase): + def test_native_decimal128_keeps_precision_and_has_a_hashable_identity(self): + try: + from bson.decimal128 import Decimal128 + except ImportError: + self.skipTest('PyMongo is needed to test its native Decimal128 type.') + from database.crud._batch import unique_keys + + identity = Decimal128('1234567890.123456789012345678') + condition = normalize_where({'amount': identity}, 'mongodb', 'find') + self.assertIs(condition.value, identity) + keys, positions = unique_keys([{'_id': identity}, {'_id': Decimal128(str(identity))}]) + self.assertIs(keys[0]['_id'], identity) + self.assertEqual(positions, [(0, 1)]) + for value in ('NaN', 'Infinity', '-Infinity'): + with self.subTest(value=value), self.assertRaises(ConfigurationError): + normalize_where({'amount': Decimal128(value)}, 'mongodb', 'find') + def test_native_bson_int64_remains_usable_as_a_key_and_condition(self): try: from bson.int64 import Int64 -- Gitee From a64d43861366726176a70cf242c54825d1712049 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 14:53:52 +0800 Subject: [PATCH 62/86] =?UTF-8?q?fix(crud):=20=E4=BF=9D=E7=95=99=E5=8F=AA?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E6=B6=88=E5=BC=82=E5=B8=B8=E7=9A=84=E4=B8=8A?= =?UTF-8?q?=E4=B8=8B=E6=96=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_records.py | 5 +++-- tests/test_crud_core.py | 13 ++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/database/crud/_records.py b/src/database/crud/_records.py index b659f40..4a36936 100644 --- a/src/database/crud/_records.py +++ b/src/database/crud/_records.py @@ -324,12 +324,13 @@ class _Records: BaseException.__dict__['__dict__'].__get__(source)['database_partial_result'] = partial if source is not error: attach_cleanup_errors(source, getattr(error, 'database_cleanup_errors', ())) - context = source.__context__ + context_descriptor = BaseException.__dict__['__context__'] + context = context_descriptor.__get__(source) try: raise source finally: # The caller may already be handling an unrelated exception. - source.__context__ = context + context_descriptor.__set__(source, context) original = source.original if isinstance(source, DatabaseError) and source.original is not None else source wrapped = BatchOperationError( 'A database record batch failed.', diff --git a/tests/test_crud_core.py b/tests/test_crud_core.py index 1f8c2f7..df781b5 100644 --- a/tests/test_crud_core.py +++ b/tests/test_crud_core.py @@ -322,7 +322,11 @@ class RecordsTests(unittest.TestCase): self.assertEqual(items[0].result, ReadItem(0, {'id': 1}, None)) def test_wrapped_batch_cancellation_preserves_original_chain_and_evidence(self): - error = asyncio.CancelledError('cancelled after receiving a partial write reply') + class ReadOnlyCancellation(asyncio.CancelledError): + def __setattr__(self, name, value): + raise AttributeError(f'{name} cannot be assigned') + + error = ReadOnlyCancellation('cancelled after receiving a partial write reply') cause = FakeFailure('driver cancellation cause') cleanup = FakeFailure('chunk cleanup failed') completed = adapter_result( @@ -335,14 +339,17 @@ class RecordsTests(unittest.TestCase): raise_cancelled_chunk(error, cause, completed, cleanup) self.adapter.delete_many.side_effect = cancel_chunk - with self.assertRaises(asyncio.CancelledError) as caught: + caught = None + try: with self.records: try: raise ValueError('an unrelated caller error') except ValueError: self.records.delete_many([{'id': 1}, {'id': 2}, {'id': 3}], batch_size=1) + except (asyncio.CancelledError, AttributeError) as failure: + caught = failure - self.assertIs(caught.exception, error) + self.assertIs(caught, error) self.assertIs(error.__cause__, cause) self.assertIs(error.__context__, cause) self.assertEqual(error.database_cleanup_errors, (cleanup,)) -- Gitee From 09faa013d24e18188042bf951a785a4f4e3c2e0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 15:01:14 +0800 Subject: [PATCH 63/86] =?UTF-8?q?docs(crud):=20=E5=AE=8C=E5=96=84=E7=94=A8?= =?UTF-8?q?=E6=B3=95=E3=80=81=E5=85=BC=E5=AE=B9=E8=BE=B9=E7=95=8C=E4=B8=8E?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E9=AA=8C=E6=94=B6=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 + README.md | 28 ++++ docs/README.md | 6 +- docs/crud-design.md | 4 +- docs/crud-implementation.md | 154 ++++++++++++++++++++ docs/crud-usage.md | 274 ++++++++++++++++++++++++++++++++++++ tests/README.md | 4 + 7 files changed, 470 insertions(+), 3 deletions(-) create mode 100644 docs/crud-implementation.md create mode 100644 docs/crud-usage.md diff --git a/CHANGELOG.md b/CHANGELOG.md index a381417..3a74db3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ ### 新增 +- 新增可选的 `database.crud.Records` / `AsyncRecords`,在已有连接上绑定 MySQL 表、MongoDB 集合或 Redis Hash,支持复合键、字段投影、批量读取及分块写入;MySQL/MongoDB 另支持结构化条件查询和集合更新、删除。 +- CRUD 写入结果区分已知计数与未知计数;批量失败保留逐项执行状态、部分结果和原始异常,异步取消保持原信号。记录层借用连接,不自动提交、回滚或重试。 +- Redis Hash 记录使用显式 namespace、字段 codec、版本标记和单键脚本,支持创建、更新、投影及 TTL;保留原生其他数据结构和客户端能力。 - 新增 `ConnectionConfig` 和 `Connections`,支持按名称登记 MongoDB、MySQL 和 Redis 配置,按需创建独立的同步或异步句柄。 - 新增显式连接检查、上下文管理和统一异常。可通过 `.native` 访问原生驱动对象,通过 `DatabaseError.original` 读取原始异常。 - 统一连接管理覆盖 Redis 单机、Sentinel 与 Cluster。Cluster 的检查范围为当前发现的主节点。 diff --git a/README.md b/README.md index ae9ab8d..8dbcc2b 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ 快速开始 · 原生客户端 · 连接管理 · + 记录操作 · 文档 · 发布日志

@@ -236,6 +237,33 @@ MongoDB 的检查结果使用 `scope="selected_server"`,`targets` 为空:原 异步创建尚未返回句柄时的资源清理由原生驱动负责。当前受测 Connector/Python 的握手取消存在上游清理限制,详见设计文档。Redis 拓扑已完成驱动级验证,真实服务验证仍待完成。 + + +## 高级记录操作 + +`database.crud` 提供可选的记录层:MySQL 表、MongoDB 集合和显式选择的 Redis Hash 格式共用按键读取、字段投影及批量写入接口。它借用已有连接,保留 `connection.native` 的全部能力。 + +```python +from database.crud import Records + +with connections.create('orders') as connection: + with Records.mysql(connection, table='orders', key_fields=('id',)) as orders: + items = orders.get_many([{'id': 1001}, {'id': 1002}], fields=['status']) + result = orders.bulk_update([ + {'key': {'id': 1001}, 'set_values': {'status': 'paid'}}, + {'key': {'id': 1002}, 'set_values': {'status': 'cancelled'}}, + ]) + # 是否提交及如何回滚由调用方决定。 + with connection.errors(operation='commit_orders'): + connection.native.commit() +``` + +异步使用 `AsyncRecords`、`async with` 和 `await`,方法及返回结构一致。MongoDB/MySQL 支持有界条件查询和集合更新;Redis 只提供指定键的 Hash 操作,不扫描整个数据库模拟条件查询。 + +`fields=[]` 对存在记录返回 `{}`,未命中返回 `None`;MongoDB/Redis 的缺失字段与明确空值分开保留。批量失败提供逐项状态,已发送但无法确认的项标为 `unknown`,不会伪装成未执行。写入确认不等于事务提交,SDK 不自动重试、提交或回滚。 + +完整接口、同步/异步示例、Redis 格式与 TTL、原始错误及事务边界见 [高级记录操作使用说明](docs/crud-usage.md)。本功能的实际验证范围见 [CRUD 实施与验收记录](docs/crud-implementation.md)。 + ## 文档与贡献 - [文档索引](docs/README.md) 汇总连接管理设计、验收记录和源码目录说明。 diff --git a/docs/README.md b/docs/README.md index 4d76c23..3631159 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,7 +16,10 @@ ## CRUD 开发设计 -- [高级 CRUD 设计](crud-design.md) 说明批量操作、字段投影、返回结果、Redis 兼容边界、同步与异步及原生客户端访问,目前进入实施阶段。 +- [高级记录操作使用说明](crud-usage.md) 提供批量操作、字段投影、同步/异步和 Redis Hash 的使用边界。 +- [适配器契约](crud-adapter-contract.md) 定义公共层与各后端的模块职责、调用和部分结果协议。 +- [CRUD 实施与验收记录](crud-implementation.md) 记录本地分支、Git 集成过程、测试及待验证项。 +- [高级 CRUD 设计](crud-design.md) 说明已实现接口的设计依据,包括批量操作、字段投影、返回结果、Redis 边界、同步与异步及原生客户端访问。 ## 目录职责 @@ -24,6 +27,7 @@ | --- | --- | | [src/database/](../src/database/) | 公共导出、连接配置、统一句柄和异常转换 | | [src/database/_adapters/](../src/database/_adapters/) | 三种后端高层连接的创建、检查、关闭及驱动错误映射 | +| [src/database/crud/](../src/database/crud/) | 可选的同步/异步记录操作、条件模型、批量结果和后端适配 | | [mongodb/](../src/database/mongodb/)、[mysql/](../src/database/mysql/)、[redis/](../src/database/redis/) | 后端原生客户端工厂及各自的辅助功能 | | [tests/](../tests/) | 离线自动化测试 | | [scripts/verify_connections.py](../scripts/verify_connections.py) | 使用显式测试配置验证真实连接 | diff --git a/docs/crud-design.md b/docs/crud-design.md index c308ac8..77082a3 100644 --- a/docs/crud-design.md +++ b/docs/crud-design.md @@ -1,11 +1,11 @@ # database 高级 CRUD 设计提案 日期:2026-11-15。 -状态:已获准进入实施,2026-11-16 起以 `docs/database-crud-design` 作为功能开发与集成主线。 +状态:已在 `docs/database-crud-design` 实现;同步/异步接口和本地验收记录供用户审查,真实 Redis 拓扑验收仍待补齐。 基线:`develop` 的已提交版本 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 本地文档分支:`docs/database-crud-design`。 -本文中的 CRUD 模块、方法和结果类型均为拟议接口,尚待实现与验收。 +本文中的 CRUD 模块、方法和结果类型已经实现;最终接口细节见 [使用说明](crud-usage.md),实际验证范围见 [实施与验收记录](crud-implementation.md)。 当前已经实现的连接接口以 [连接管理设计](connection-management-design.md) 和 [项目 README](../README.md) 为准。 用户表述中的 Redis“金融边界”在本文按前文理解为“兼容与功能边界”,同时纳入事务、原子性、金额精度与持久化约束,供审查确认。 diff --git a/docs/crud-implementation.md b/docs/crud-implementation.md new file mode 100644 index 0000000..87e0d06 --- /dev/null +++ b/docs/crud-implementation.md @@ -0,0 +1,154 @@ +# CRUD 实施与验收记录 + +日期:2026-11-16。 +集成与交付分支:`docs/database-crud-design`,由用户指定。 +正式功能基线:本地 `develop` 的 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 +本次只操作本地任务分支;没有检查、同步或推送远端,也不将本功能合入 `develop`。 + +使用接口见 [使用说明](crud-usage.md),语义边界见 [设计](crud-design.md)。 + +## 1. 模块和工作树 + +协调者在 `D:\996-work\loongspears\database` 集成。 +各 worker 在 `D:\996-work\loongspears\database-worktrees\` 下独立开发,使用独立分支、目录和暂存区。 +旧工作树及用户其他分支全部保留。 + +| 模块 | 分支 | 工作树目录 | 交付提交 | +| --- | --- | --- | --- | +| 公共协议、记录层 | `feature/database-crud-core` | `crud-core` | `0ba757a925247f632d32b9d236457fad5e8fbe00` | +| MySQL | `feature/database-crud-mysql` | `crud-mysql` | `1e7f52ee5f4646498cca57632a99bb2ca2a3704b` | +| Redis Hash / codec | `feature/database-crud-redis` | `crud-redis` | `32c990422a3fdd972b5e8d089b13932dc7f33d29` | +| MongoDB | `feature/database-crud-mongodb` | `crud-mongodb` | `c0e4246b54274d3222ed75ab57141e492da6131f` | +| MongoDB 取消修复 | `fix/database-mongodb-cancellation` | `crud-mongodb-cancellation` | `a6d778f2a919695ae77012e2426e8c26ce57ea0d` | +| 分块派发与取消链 | `fix/database-crud-batch-evidence` | `crud-batch-evidence` | `a64d43861366726176a70cf242c54825d1712049` | + +公共协议检查点先集成,再启动各数据库模块;公共层完成后,各 worker 可以合入明确的集成检查点进行公开 API 验证。 +各 worker 不交叉合并彼此的分支,不操作总线暂存区。 +协调者串行使用 `--no-ff` 合并核对过的提交,保留分支边界。 + +## 2. Git 操作记录 + +以下按执行顺序记录协调者的 Git 写操作及必要核对;命令均带明确工作目录执行。 +没有使用 squash、rebase、amend、reset、强制推送或删除分支、工作树。 + +| 顺序 | 执行者、分支及工作树 | 操作 | 结果 | +| --- | --- | --- | --- | +| 1 | 协调者,主目录 `develop` | `git status --short --branch`、`git diff --cached`、`git worktree list --porcelain`、`git rev-parse HEAD develop` | 主目录干净;确认本地基线 `b670712`,设计工作树存在待保存文档 | +| 2 | 协调者,`crud-design` 工作树 | 按路径暂存 `docs/crud-design.md` 和 `docs/README.md`,核对暂存差异后 `git commit` | `80fd877360787309bf4a679bd94b1baa76c733ea`,保存既有设计文档 | +| 3 | 协调者,`crud-design` 工作树及主目录 | 旧文档工作树 `git switch --detach 80fd877`;主目录 `git switch docs/database-crud-design` | 主目录切至用户指定开发主线,旧文档工作树保留 | +| 4 | 协调者,主目录 | 按路径暂存设计及索引,核对后中文 `git commit` | `da1103d`,登记实施基线 | +| 5 | 公共层 worker,`crud-core` | `git worktree add -b feature/database-crud-core ... 80fd877`,按模块路径暂存、检查并提交 | `da0f2d36b368002d3497d9bc763002b5654c418b`,共享类型与协议 | +| 6 | 协调者,主目录 | 核对 SHA/范围后 `git merge --no-ff -m "feat(crud): 合入结果类型与适配器协议" da0f2d36...` | 集成 HEAD `63d94d3b10392ef0d9a5514db30e185eb85e22de`;91 项回归通过 | +| 7 | MySQL / Redis worker,各自工作树 | 从 `63d94d3...` 用 `git worktree add -b` 创建各自分支 | 独立并行实现,无共享暂存区 | +| 8 | 公共层 worker,`crud-core` | 模块实现及测试分两次中文提交 | `b5fc99c9f30bd00785ac6c4757c04d00e49c611b`、`0ba757a925247f632d32b9d236457fad5e8fbe00`;106 项回归通过 | +| 9 | 协调者,主目录 | 核对 SHA/范围后 `git merge --no-ff -m "feat(crud): 合入同步异步公共记录层" 0ba757a...` | 集成 HEAD `5140bbee593a323b713674c483faa087feeb7c88` | +| 10 | MongoDB worker,`crud-mongodb` | 从 `5140bbe...` 创建独立分支和工作树 | 使用已集成公共接口开展实现 | +| 11 | 协调者,主目录 | 按路径暂存 `tests/test_crud_integration.py`、检查后中文提交 | `d3a93cf35e531ec7311072c7a6ed4d091911e1e7`;多后端/同类多实例的 2 项测试通过 | + +每次提交前使用 `git diff --cached --stat` / `git diff --cached --check` 检查范围和空白。 +后续后端与审查修复集成按以下顺序执行;每次合并前再次核对交付分支指向的 SHA 及三点差异范围。 + +| 顺序 | 执行者、分支及工作树 | 操作及交付 SHA | 集成结果 | +| --- | --- | --- | --- | +| 12 | Redis worker,`crud-redis` | 提交 codec `e732605`;`git merge --no-ff 5140bbe` 得到 `ac7c5ac`;提交适配与测试 `32c9904` | 工作树干净,19 项定向测试在 redis-py 5.0 / 8.1 通过 | +| 13 | 协调者,主目录 | `git merge --no-ff` 合入 `32c990422a3fdd972b5e8d089b13932dc7f33d29` | `123869eb385da7dcf3307861dbc56b447509ee32`;127 项回归与 Lua 模拟验收通过 | +| 14 | 协调者,主目录 | 投影预检修复按指定源文件、测试及契约路径暂存后提交 | `87abaae6e6c4be69bd095070dac9fa83fcfe6cc9`;130 项回归通过 | +| 15 | MySQL worker,`crud-mysql` | `git merge --ff-only 5140bbe`、`git merge --ff-only 87abaae`;仅暂存 MySQL 模块和测试后提交 `1e7f52e` | 两个驱动版本各 152 项回归通过,真实 MySQL 8.4 CRUD/事务验证通过 | +| 16 | MongoDB worker,`crud-mongodb` | 提交 `27c3c33`;`git merge --no-ff 87abaae` 得到 `23cc533`;提交清理修复 `c0e4246` | 五个驱动版本的真实 MongoDB 验证通过,21 项定向测试通过 | +| 17 | 协调者,主目录 | Int64 修复提交 `ab20cef`;`git merge --no-ff` 合入 `1e7f52ee5f4646498cca57632a99bb2ca2a3704b` | `887cd77305435a17dafd8943e63624a3253f94ab`;153 项回归及两个 MySQL 驱动版本真实复验通过 | +| 18 | 协调者,主目录 | `git merge --no-ff` 合入 `c0e4246b54274d3222ed75ab57141e492da6131f` | `b650039b9a7f130d93a45be62db5e2c237e87f1b`;最低同步、最低异步及当前 MongoDB 真实复验通过 | +| 19 | 审查修复 worker,各自工作树 | 分块修复从 `87abaae`,MongoDB 取消修复从 `b650039`,分别 `git worktree add -b fix/...`;按模块路径暂存、检查、中文提交 | 独立保留审查发现及回归修复历史 | +| 20 | 协调者,主目录 | Decimal128 修复提交 `f32db99`;`git merge --no-ff` 合入 `a6d778f2a919695ae77012e2426e8c26ce57ea0d` | `22586247614e3dd68bfd972ad14fad5ba63f6772`;BSON 精确类型真实回读验证通过 | +| 21 | 分块修复 worker,`crud-batch-evidence` | 对两项可复现问题提交 `a2b2227`,再补只读取消对象的回归修复 `a64d438` | 模块及测试两文件,工作树干净,134 项分支回归通过 | +| 22 | 协调者,主目录 | 核对 SHA、差异后 `git merge --no-ff` 合入 `a64d43861366726176a70cf242c54825d1712049` | `3594f21ab214b144e9d14579697bd5dcf9264fad`,作为最终代码验收基点 | +| 23 | 协调者,主目录 | 使用说明、设计状态、README、CHANGELOG、测试指南和本记录按路径暂存;核对暂存范围及文档检查后中文提交 | 本文所在文档提交;代码保持上述验收基点内容 | + +最终分支 HEAD 随文档提交前进,可用 `git rev-parse docs/database-crud-design` 获取;代码验收绑定于 `3594f21ab214b144e9d14579697bd5dcf9264fad`。 + +## 3. 验证环境和命令 + +源码通过 `PYTHONPATH=<当前工作树>/src` 指向当前工作树。 +验证解释器为独立 QA 环境的 Python 3.10.20。 +当前驱动组合:PyMongo 4.18.3、MySQL Connector/Python 26.10.0、redis-py 8.1.0。 +最低版本分别使用隔离的依赖目录验证,导入后核对实际版本,避免无意回退至当前版本。 +Ruff 使用 0.16.8;运行依赖未增加。 + +```powershell +$env:PYTHONPATH = (Join-Path (Get-Location) 'src') +python -B -m unittest discover -s tests -q +ruff check src/database/crud tests/test_crud_*.py +ruff format --check src/database/crud tests/test_crud_*.py +git diff --check +``` + +`python` 和 `ruff` 在本次实际执行中使用 QA 解释器及工具的绝对路径。 +各后端定向命令使用 `-p test_crud_<模块>.py`。 +标准自动测试使用驱动对象、离线替身和 `unittest.IsolatedAsyncioTestCase`,不接触外部业务数据库。 +最终完整回归为 179 项,三套依赖组合各自全量通过,未跳过测试。 + +| 组合 | PyMongo | MySQL Connector/Python | redis-py | 结果 | +| --- | --- | --- | --- | --- | +| 当前版本 | 4.18.3 | 26.10.0 | 8.1.0 | 179 / 179 | +| 最低同步依赖 | 4.6.0 | 8.3.0 | 5.0.0 | 179 / 179 | +| 最低异步依赖 | 4.9 | 8.3.0 | 5.0.0 | 179 / 179 | + +每个独立进程先断言实际导入版本,再由 `unittest.defaultTestLoader.discover('tests')` 执行完整套件。 +公共接口无驱动导入的独立子进程回归仍通过。 +最终 16 个新增或修改的 Python 文件通过 Ruff lint 和格式检查,Git 空白差异检查通过。 + +## 4. 服务与模拟验证边界 + +真实 CRUD 只在本任务隔离的临时数据库中执行,服务只绑定本机回环地址,不复用用户生产连接。 +测试凭据、数据目录和临时运行日志不进入仓库。 +故障场景的运行时限由测试进程控制,不表示 SDK 提供整体硬超时。 + +Redis 验证包含真实 redis-py 对象的单机 / Sentinel / Cluster 调度和资源清理测试,以及独立 QA 依赖中的 fakeredis / Lua 执行验证。 +Lua 模拟器验证不等于 Redis 服务器、Sentinel 故障切换或 Cluster 多槽网络验收。 +本环境尚无可用 Redis 真实拓扑,相关实测必须独立列为待验证项。 + +| 验证类型 | 环境 | 已验证内容 | +| --- | --- | --- | +| 真实 MySQL | MySQL 8.4.11,Connector 8.3.0 / 26.10.0 | 同步、异步 CRUD;当前版同步 C / pure;复合键、默认/空/包含投影、500 键读取、排序规则、部分失败、原始错误、独立连接的提交/回滚/autocommit | +| 真实 MongoDB | MongoDB 8.0.24 单节点副本集 | PyMongo 4.6 / 4.8 同步,4.9 / 4.13 / 4.18.3 同步及异步;条件与数组/缺失隔离、投影、复合键、排序规则、ordered 部分失败、w=0、原生 session 与事务 abort | +| 原生 BSON 数值 | 同一 MongoDB,PyMongo 4.6 / 4.9 / 4.18.3 | Int64 / Decimal128 读回后重新作为键、条件及重复批量输入使用,保持精确类型 | +| Redis 原生驱动离线 | redis-py 5.0 / 8.1 | 单机 / Sentinel / Cluster 同步与异步调度、所有权和清理;真实 ClusterPipeline 栈使用替代节点及 socket,验证跨槽路由和逐项回复 | +| Redis Lua 模拟器 | fakeredis 2.40.0 / lupa 2.8,redis-py 5.0 / 8.1 | bytes 与 decode_responses、空值/缺失/空投影、560 字段读取、TTL 保留/重置/清除/过期、类型保护、部分失败、8 个并发创建请求仅一个成功 | + +上述 MySQL、MongoDB 各版本及 Redis 模拟器验收均在最终代码验收基点复跑。 +Redis 5.0 的临时验收脚本最初使用了仅新版存在的 `aclose()`;改为适配 `close()` 后完整通过,此处属于测试脚本兼容修正,不是 SDK 清理逻辑变更。 + +临时 QA 根为 `C:\Users\MSR\AppData\Local\Temp\loongspears-db-connection-qa-9d596ec4bf4d`,以下脚本不纳入运行依赖或仓库产品代码。 + +- MySQL:`mysql-crud-runtime/verify_crud.py`、`verify_transactions.py`,参数依次为预期驱动版本、`current` 或隔离驱动目录、QA 根。 +- MongoDB:`mongodb-crud-runtime-4148fab7/verify_mongo_crud.py`,参数为同目录 `fixture.json` 与预期驱动版本;由 `PYTHONPATH` 选当前源码和驱动。 +- BSON 数值:同一目录 `verify_int64_main.py`,同样参数,包含 Int64 与 Decimal128。 +- Redis:`crud-redis-smoke.py`,使用显式 `PYTHONPATH` 加载当前源码、选定 redis-py 及独立 `crud-redis-lua` 模拟器依赖。 + +测试配置仅指向上述隔离服务;凭据不写入本文,也不直接输出配置内容。 +验收完成后,本任务 MySQL 进程与 MongoDB 副本集均已正常关闭,专属回环端口已无监听;临时数据、脚本、配置和日志保留以便复核。 +重跑真实服务脚本需要先按测试夹具恢复已停止的隔离服务,不能替换成业务数据库配置。 + +## 5. 实现选择 + +- MongoDB `get_many` 对去重后的每个键执行原生有界查询,保留数据库的排序规则、数值和时间匹配语义,不用 Python 值重排猜测匹配。 +- MySQL 批量读取由 SQL 标记输入位置;批量写入只报告驱动能证明的组级或逐项信息,不推算连续生成主键。 +- Redis Cluster 的 Pipeline 用原生命令分发单键 EVAL;不调用某些版本未实现的 `pipeline.eval()`,不把异步 Cluster 的服务器 RESET 命令当成客户端资源清理。 +- 新增 `InsertItem(record, ttl_s=...)` 明确区分批量插入的 TTL 控制项与名为 `ttl_s` 的业务字段。 +- `JsonFieldCodec` 是显式的字段编码器,不是自动类型转换层。Redis 身份键只接受字符串或整数;其他身份类型由调用方显式转换。 +- MySQL C 扩展可读取已协商会话标志时映射 UPDATE 统计;纯 Python / 异步连接只保留原始 `rowcount`,统一 matched / modified 计数为 `None`,不依据可变的下次建连标志猜测。 +- BSON `Int64`、`Decimal128` 读回后仍能用作键和条件;精确小数去重使用原生二进制身份,不转成 float。 + +## 6. 最终验收状态 + +本地功能实现、回归、真实 MySQL/MongoDB 以及 Redis 模拟验收完成,代码和使用文档交付用户审查。 +任务工作树保留;用户主目录停在 `docs/database-crud-design`。 +本地共 22 个工作树,包含本任务新增的 7 个独立工作树;既有其他工作树没有被删除、搬移或改写。 +`develop` 仍为初始基线 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`,没有被提交、合并或改写。 +没有执行远端查询、同步、推送或远端分支清理。 + +仍待验证:真实 Redis 单机、Sentinel 与多主节点 Cluster 的网络/故障切换验收,以及这些真实拓扑中与 MySQL/MongoDB 同进程的完整端到端场景。 +多数据库和同类多实例同时使用已由公共层离线集成回归覆盖,这不替代真实三后端联合验收。 +未将 Lua 模拟结果标成真实 Redis 服务验证;此次也未进行吞吐量基准或生产负载测试。 + +复核可使用 `git log --first-parent --oneline b670712..docs/database-crud-design` 查看交付顺序,再进入每个 `--no-ff` 合并查看模块内提交。 +用户审查通过后自行决定何时合入 `develop`。 diff --git a/docs/crud-usage.md b/docs/crud-usage.md new file mode 100644 index 0000000..1c8d0ff --- /dev/null +++ b/docs/crud-usage.md @@ -0,0 +1,274 @@ +# 高级记录操作使用说明 + +`database.crud` 在已有连接之上提供可选的记录操作层。 +MySQL 对应表,MongoDB 对应集合,Redis 对应采用本库格式的 Hash 记录。 +它们共享调用方式、字段投影和结果结构;事务、类型系统和查询能力仍遵循各数据库的真实语义。 + +本功能在 `docs/database-crud-design` 分支开发,具体测试范围与待验收事项见 [实施记录](crud-implementation.md)。 +完整设计背景见 [CRUD 设计](crud-design.md),内部模块协作见 [适配器契约](crud-adapter-contract.md)。 + +## 1. 绑定已有连接 + +安装方式和连接配置见 [README](../README.md)。 +配置始终由调用方显式传入;记录层也不会读取环境变量或 `.env`。 + +```python +from database.crud import Records + +def read_orders(connections): + with connections.create('orders') as connection: + with Records.mysql( + connection, table='orders', key_fields=('tenant_id', 'id') + ) as orders: + return orders.get( + {'tenant_id': 7, 'id': 1001}, fields=['status', 'total'] + ) +``` + +`key_fields` 声明数据库中真实的唯一身份,支持复合键。 +调用方负责对应的唯一索引或主键;SDK 不创建表、不迁移集合、不自动建立索引。 +字段名使用普通字段,不接受 SQL 片段、点路径或 MongoDB 操作符。 +数据库名、表名和集合名是绑定参数,SQL 值使用参数绑定。 +MySQL 的表名和列名限 1–64 个字母、数字或下划线;高级标识符和表达式通过原生 SQL 使用。 +MongoDB 插入必须提供除可由驱动生成的 `_id` 之外的所有声明键字段。 + +| 后端 | 同步绑定入口 | 后端参数 | +| --- | --- | --- | +| MySQL | `Records.mysql(connection, ...)` | `table`、`key_fields` | +| MongoDB | `Records.mongodb(connection, ...)` | `database`、`collection`、`key_fields`,可选原生 `session` | +| Redis | `Records.redis_hash(connection, ...)` | `namespace`、`key_fields`、显式 `codec` | + +记录对象必须在 `with` 中使用,退出后不能继续调用。 +需要继续使用时,可以基于仍然打开的连接重新绑定记录对象。 +记录层只释放自己创建的游标、Pipeline 或 Sentinel 数据客户端,不关闭借用的连接和 session。 +它不改变原生客户端的线程安全或并发约束,也不隐式共享会话。 + +## 2. 同步与异步 + +异步使用 `AsyncRecords`;绑定本身是普通函数,进入上下文和数据库操作使用 `await`。 +异步方法调用原生异步驱动,参数与返回结构和同步方法一致。 + +```python +from database.crud import AsyncRecords + +async def read_documents(connections, document_id): + async with await connections.create_async('documents') as connection: + async with AsyncRecords.mongodb( + connection, database='app', collection='orders', key_fields=('_id',) + ) as documents: + return await documents.get({'_id': document_id}, fields=['status']) +``` + +MongoDB 异步仍要求 PyMongo 4.9 及以上版本。 +异步连接和记录对象在创建连接的事件循环中使用。 +一个项目可以同时打开三种数据库,也可以创建两个同类连接;使用 `ExitStack` 或 `AsyncExitStack` 管理多个上下文即可。 +关闭某个记录对象不会替应用关闭其他连接;跨库成功、回滚和补偿由应用协调。 + +## 3. 方法范围 + +| 方法 | MySQL / MongoDB | Redis Hash | +| --- | --- | --- | +| `get(key, fields=...)` | 唯一键读取 | 指定键读取 | +| `get_many(keys, fields=..., batch_size=500)` | 多键读取 | 多键 Pipeline | +| `find(where=..., fields=..., limit=..., offset=0, order_by=...)` | 有界查询 | 不支持 | +| `count(where=...)` | 条件计数 | 不支持 | +| `insert(record)` / `insert_many(records)` | 创建记录 | 创建专用 Hash,不覆盖已有键 | +| `update(key, set_values=...)` | 按键赋值 | 只更新已有记录 | +| `update_many(where=..., set_values=...)` | 一次集合条件更新 | 不支持 | +| `bulk_update(items, batch_size=500)` | 每条记录分别赋值 | 单键脚本组成 Pipeline | +| `delete(key)` / `delete_many(keys)` | 按键删除 | 单键脚本删除 | +| `delete_where(where=...)` | 条件删除 | 不支持 | + +批量写入接受有限序列,先检查整批可静态判断的结构和编码,再按输入顺序分块。 +默认每块 500 项;一次批量写入不接受重复的已知键。 +空批次返回空结果,不访问数据库。 +Redis 不通过 `KEYS` 或 `SCAN` 模拟条件筛选;不支持的请求在发送前报 `UnsupportedOperationError`。 + +## 4. 字段投影、空值和批量读取 + +| `fields` | 行为 | +| --- | --- | +| `None` | 返回全部业务字段 | +| `['status', 'total']` | 只返回这些字段,保留原生值类型 | +| `[]` | 只确认存在性:存在返回 `{}`,不存在返回 `None` | + +MongoDB 的包含投影只在显式选择 `_id` 时返回 `_id`;`fields=None` 保留完整文档。 +Redis 的内部格式标记不会出现在业务记录中。 +MySQL 选择不存在的列时保留原始驱动错误。 +MongoDB / Redis 缺少被选字段时,返回字典省略该字段;明确存储的空值保留为 `None`。 +因此空字典不等于未找到记录。 + +```python +items = orders.get_many( + [{'tenant_id': 7, 'id': 1001}, + {'tenant_id': 7, 'id': 1002}, + {'tenant_id': 7, 'id': 1001}], + fields=['status'], +) +for item in items: + print(item.index, item.key, item.record) +``` + +返回 `ReadItem` 列表,与输入的位置和数量完全对应。 +完全相同的重复键在一次调用中只读取一次,并复用该次观察;返回结果不依赖数据库返回顺序。 +没有读取成功的项不会用 `record=None` 冒充“不存在”;部分失败通过批量异常表达。 +一次分块读取不是跨节点、跨批次的一致性快照。 +MongoDB 当前对每个去重后的键执行一次 `limit=2` 的有界查询,以保留原生排序规则、数值和时间匹配语义,并发现声明键实际不唯一的情况。 +因此该接口统一多键结果,但不承诺所有后端都只产生一次网络请求;MySQL 使用按输入位置标记的联合查询,Redis 使用 Pipeline。 + +## 5. 条件与批量修改 + +普通字典表示多个字段的等值 AND 条件。 +需要更明确的结构时,使用 `Eq`、`In`、`Gt`、`Gte`、`Lt`、`Lte`、`And`、`Or`、`IsNull`;MongoDB 另支持 `Exists`。 +只支持普通标量条件;原生数组、嵌套路径、SQL 表达式和聚合继续使用原生客户端。 + +```python +from database.crud import And +from database.crud import Gte +from database.crud import In + +pending = orders.find( + where=And(In('status', ['pending', 'paid']), Gte('total', 100)), + fields=['id', 'status'], limit=50, order_by=[('id', 'asc')], +) +changed = orders.update_many( + where={'status': 'pending'}, set_values={'status': 'closed'}, +) +individual = orders.bulk_update([ + {'key': {'tenant_id': 7, 'id': 1001}, 'set_values': {'status': 'paid'}}, + {'key': {'tenant_id': 7, 'id': 1002}, 'set_values': {'status': 'cancelled'}}, +]) +``` + +`update_many` 是服务器端一次集合操作:MySQL 使用一条参数化 UPDATE,MongoDB 使用原生 `update_many`。 +`bulk_update` 则是逐键不同赋值:MySQL 按连续的相同字段形状组织 `executemany`,MongoDB 使用集合级 ordered bulk write,Redis 使用非事务 Pipeline。 +它们不是先读出全部记录,再逐个修改。 +`set_values` 是赋值,不是自动原子加减;不能修改声明的键字段。 + +`Eq('field', None)` / `{'field': None}` 表示明确空值,不包含 MongoDB 的缺失字段。 +`In` 不接受 `None` 元素;可用 `Or(In('field', values), IsNull('field'))` 表达两者。 +`In('field', [])` 匹配零项。 +相同条件在三种数据库上不承诺相同排序规则、数值转换或时区行为。 + +`find` 必须传正整数 `limit`;`offset` 为非负整数。 +分页需要稳定顺序时,应包括唯一键排序;并发写入仍会影响分页观察。 +条件更新和条件删除默认拒绝空筛选;确需全表或全集合时显式使用 `all_records=True`。 + +## 6. 写入结果与部分失败 + +单次写入返回 `WriteResult`,批量写入返回 `BatchResult`。 +写方法不接受 `fields` / `returning`;需要重读时显式调用 `get`,并理解并发写入可能发生在两次操作之间。 + +| 字段 | 解释 | +| --- | --- | +| `acknowledged` | 收到执行确认;不表示事务已经提交 | +| `matched_count` / `modified_count` | 只有可靠依据时填写,否则为 `None` | +| `inserted_count` / `deleted_count` | 确认可知的记录数,不能把未知当作 0 | +| `generated_keys` | 可以映射为完整身份的生成键,否则为 `None` | +| `native_result` | 原生结果对象,或游标关闭前保存的统计快照 | + +MySQL 的 UPDATE `rowcount` 可能表示改变数或匹配数;不能把一个数同时填到两个字段,也不额外查询猜测。 +MySQL C 扩展能够提供当前会话的已协商标志时,填写对应的 matched 或 modified 计数。 +纯 Python / 异步等无法提供该证据的连接,统一计数保留 `None`,原始 `rowcount` 及未知解释模式保存在 `native_result`。 +连接上的可变客户端标志可能是下次建连配置,不能据此证明当前会话采用的计数模式。 +MongoDB 保留 matched / modified 的差别;未确认写入不虚构计数。 +Redis HSET 的新字段数不等于修改的记录数。 +MySQL 不根据 `lastrowid` 推算一批连续主键。 + +`BatchResult.items` 按输入位置保留状态;`counts` 只汇总可证实的统计,`native_results` 保存原生批次结果。 + +| 状态 | 已知事实 | +| --- | --- | +| `acknowledged` | 已收到该项可解释的执行确认 | +| `unacknowledged` | 驱动使用未确认写入 | +| `failed` | 明确报告该项失败 | +| `not_sent` | SDK 尚未发送该项 | +| `not_executed` | 驱动明确报告未执行,例如有序批次失败后的项 | +| `unknown` | 可能已经发送,现有证据无法确定结果 | + +```python +from database.crud import BatchOperationError + +try: + result = orders.bulk_update(changes, batch_size=500) +except BatchOperationError as error: + partial = error.partial_result + # 根据各项状态、业务幂等性和原始错误决定后续处理。 + # error.original 保留最初的驱动或明确的 SDK 错误;不要直接整批重试。 + raise +``` + +失败后 SDK 不再发送后续分块;已进入驱动的命令可能继续执行。 +`asyncio.CancelledError` 保持原类型、原对象,附加 `database_partial_result` 描述已知进度。 +清理失败附加在主异常的 `database_cleanup_errors` 中,不替换业务错误。 +批量异常可以同时包含确定成功、确定失败和不确定项;统一类别本身不能决定是否安全重试。 + +## 7. Redis Hash 格式与边界 + +```python +from database.crud import InsertItem +from database.crud import JsonFieldCodec + +with connections.create('cache') as connection: + with Records.redis_hash( + connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as cache: + cache.insert({'id': 1001, 'status': 'pending'}, ttl_s=60) + cache.update({'id': 1001}, set_values={'status': 'paid'}) + cache.insert_many([ + InsertItem({'id': 1002, 'status': 'pending'}, ttl_s=120), + {'id': 1003, 'status': 'pending'}, + ]) +``` + +只管理本库专用格式,不把已有 String、List 或任意 Hash 自动当成记录。 +键采用带版本的命名空间、声明的键字段顺序及类型编码,字符串 `'1'` 和整数 `1` 不发生碰撞。 +当前键类型限字符串和整数,排除布尔值;其他身份类型由调用方显式转换。 +每个 Hash 保存保留字段 `__database_record__=1`,业务记录不能使用这个名称。 +详细持久键格式在适配模块中定义,升级格式必须显式处理迁移。 +当前格式为 `database:hash:v1::<身份编码>`。 +两段均使用无填充的 URL-safe Base64;namespace 先编码为 UTF-8,身份先按声明顺序编码为紧凑的 ASCII JSON `[[字段名, 值], ...]`,非 ASCII 字符使用 JSON 转义。 + +`JsonFieldCodec` 对每个字段分别编码,支持标准 JSON 值及有限浮点数。 +不将 Decimal、日期、ObjectId 或 bytes 静默转成字符串或 float;需要这些类型时传入自定义 `FieldCodec`。 +自定义 codec 的 `encode(value)` 返回 str / bytes,`decode(value)` 必须兼容连接提供的 bytes / str。 +字段 codec 不负责身份键编码,也不更改连接的 `decode_responses` 设置。 + +| 场景 | `ttl_s` | +| --- | --- | +| 插入时省略或 `None` | 不设置过期时间 | +| 插入或更新传正整数 | 在单键操作中设置相应秒数 | +| 更新时省略 | 保留原 TTL | +| 更新时显式 `None` | 清除 TTL | +| 更新时记录已过期 | 未匹配,不重新创建 | + +0、负数、布尔值和超过 `2**53 - 1` 秒的 TTL 均拒绝。 +批量插入以 `InsertItem` 单独表达 TTL,普通记录中名为 `ttl_s` 的业务字段不会被当作控制参数。 +MySQL 和 MongoDB 不接受这一 TTL 选项。 + +存在性、格式检查、投影、创建或更新通过单键 Lua 脚本完成。 +运行账户需要对应命令权限;SDK 不在权限不足时降级成存在竞态的先查后写。 +Cluster 还需要原生驱动进行节点发现和命令元数据查询所需的权限,具体随 redis-py 版本而异。 +脚本的串行执行不等于错误回滚,也不等于跨键事务。 +Sentinel 使用所配置服务的主节点数据客户端,退出记录上下文后清理;原 Sentinel 对象仍可访问。 +Cluster 按单键路由,批次使用 `transaction=False`,不强制同槽,不承诺整批原子提交。 + +金额应由应用选择精确单位,例如整数最小货币单位,或支持精确小数的 codec。 +本层不把金额映射为浮点增量,不实现资金扣减、防重账、账本、分布式事务或余额一致性协议。 +执行确认也不是 Redis 持久化、复制或故障切换中不丢数据的保证。 + +## 8. 原生能力与事务仍由调用方控制 + +```python +with connections.create('orders') as connection: + with Records.mysql(connection, table='orders', key_fields=('id',)) as orders: + orders.update({'id': 1001}, set_values={'status': 'paid'}) + with connection.errors(operation='commit_orders'): + connection.native.commit() +``` + +SDK 不在记录上下文退出时额外提交、回滚或自动重试。 +MySQL autocommit 和连接关闭行为按原生驱动规则执行;需要明确回滚时调用方自己管理。 +MongoDB 可绑定原生 session,事务及 session 生命周期由调用方管理。 +Redis 其他结构、MongoDB 聚合、SQL JOIN、数据库专有原子操作都通过 `connection.native` 使用。 +上下文外原生异常行为不变;需要转换原生操作错误时显式使用 `connection.errors(operation=...)`。 diff --git a/tests/README.md b/tests/README.md index 88c8206..49fd6c2 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,6 +2,10 @@ ## 离线回归 +新增记录层使用 `tests/test_crud_*.py`:公共条件与生命周期、三个后端适配、以及多数据库/同类多实例资源隔离分别维护。 +批量状态、取消信号、字段投影、原始类型和资源归属均覆盖同步与异步。 +该功能的版本矩阵、Lua 模拟器及隔离服务结果见 [CRUD 实施与验收记录](../docs/crud-implementation.md)。 + 按 [贡献指南](../CONTRIBUTING.md) 在独立环境运行 `python -m pip install -e .`,以 editable 方式安装当前工作区及三个后端的驱动。从仓库根目录运行以下命令。 ```bash -- Gitee From ce740b3aefa554c7e4f0babb7c09b9b53f520ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Mon, 16 Nov 2026 15:08:32 +0800 Subject: [PATCH 64/86] =?UTF-8?q?docs(crud):=20=E8=A7=84=E8=8C=83=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=E5=BC=80=E5=8F=91=E5=88=86=E6=94=AF=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CONTRIBUTING.md | 2 ++ docs/crud-design.md | 7 ++++--- docs/crud-implementation.md | 12 +++++++----- docs/crud-usage.md | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1f1d4da..c372460 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -37,6 +37,8 @@ git diff --check 日常改动从 `develop` 建立任务分支,审查后合入 `develop`。`master` 用于发布。按工作内容使用 `feature/*`、`fix/*`、`refactor/*`、`docs/*` 或 `chore/*`。保留任务分支的提交关系时使用 `--no-ff` 合并。智能体的并行分工、集成和交付记录见 [Git 与 worktree 协作](AGENTS.md#git-与-worktree-协作)。 +`docs/*` 仅用于纯文档任务。包含代码实现的任务按实际类型使用 `feature/*`、`fix/*`、`refactor/*` 等前缀,不以 `docs/*` 分支推进功能开发。 + 采用中文 Conventional Commits,保留英文类型和可选的 scope,正文使用中文说明背景和验证情况。每次提交包含一组可独立审查的改动,按路径暂存后核对 `git diff --cached`。 ```text diff --git a/docs/crud-design.md b/docs/crud-design.md index 77082a3..b914a5f 100644 --- a/docs/crud-design.md +++ b/docs/crud-design.md @@ -1,9 +1,9 @@ # database 高级 CRUD 设计提案 日期:2026-11-15。 -状态:已在 `docs/database-crud-design` 实现;同步/异步接口和本地验收记录供用户审查,真实 Redis 拓扑验收仍待补齐。 +状态:已在 `feature/database-crud` 实现;同步/异步接口和本地验收记录供用户审查,真实 Redis 拓扑验收仍待补齐。 基线:`develop` 的已提交版本 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 -本地文档分支:`docs/database-crud-design`。 +当前功能分支:`feature/database-crud`,由原文档分支 `docs/database-crud-design` 更名,保留全部提交历史。 本文中的 CRUD 模块、方法和结果类型已经实现;最终接口细节见 [使用说明](crud-usage.md),实际验证范围见 [实施与验收记录](crud-implementation.md)。 当前已经实现的连接接口以 [连接管理设计](connection-management-design.md) 和 [项目 README](../README.md) 为准。 @@ -489,7 +489,7 @@ MongoDB 和 Redis 异步用法分别采用 AsyncRecords.mongodb/redis_hash、asy ## 11. 实施阶段与验收计划 -用户于 2026-11-16 指定 `docs/database-crud-design` 为功能开发主线;各实现分支从该主线的已提交检查点建立。 +功能开发主线现为 `feature/database-crud`;前期使用的 `docs/database-crud-design` 已按用户的命名要求更名。各实现分支从主线的已提交检查点建立,代码任务不再使用 `docs/*` 前缀。 按 [协作规范](../CONTRIBUTING.md) 安排公共契约、MySQL、MongoDB、Redis 和验收工作。 公共结果与条件语义先形成检查点,后端模块再从同一检查点开展并行实现,最后串行合并。 分支命名使用 feature/fix/refactor/docs 等项目规范前缀。 @@ -544,4 +544,5 @@ Git 操作在本地依次完成,主目录为 `D:\996-work\loongspears\database Python 示例只做语法核对,因为拟议模块尚未实现;真实驱动行为及新接口测试均留在实施验收阶段。 上述记录为 2026-11-15 文档编写阶段。 2026-11-16 已将设计与索引提交为 `80fd877360787309bf4a679bd94b1baa76c733ea`,并将主目录切换到 `docs/database-crud-design`。 +随后将开发主线更名为 `feature/database-crud`;以上旧分支名称及命令保留为历史记录。 实施记录及验收状态由后续交付补充。 diff --git a/docs/crud-implementation.md b/docs/crud-implementation.md index 87e0d06..27cf582 100644 --- a/docs/crud-implementation.md +++ b/docs/crud-implementation.md @@ -1,7 +1,7 @@ # CRUD 实施与验收记录 日期:2026-11-16。 -集成与交付分支:`docs/database-crud-design`,由用户指定。 +集成与交付分支:`feature/database-crud`。按用户要求,功能代码使用 `feature/*` 前缀,`docs/*` 仅用于纯文档任务。 正式功能基线:本地 `develop` 的 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 本次只操作本地任务分支;没有检查、同步或推送远端,也不将本功能合入 `develop`。 @@ -61,9 +61,11 @@ | 20 | 协调者,主目录 | Decimal128 修复提交 `f32db99`;`git merge --no-ff` 合入 `a6d778f2a919695ae77012e2426e8c26ce57ea0d` | `22586247614e3dd68bfd972ad14fad5ba63f6772`;BSON 精确类型真实回读验证通过 | | 21 | 分块修复 worker,`crud-batch-evidence` | 对两项可复现问题提交 `a2b2227`,再补只读取消对象的回归修复 `a64d438` | 模块及测试两文件,工作树干净,134 项分支回归通过 | | 22 | 协调者,主目录 | 核对 SHA、差异后 `git merge --no-ff` 合入 `a64d43861366726176a70cf242c54825d1712049` | `3594f21ab214b144e9d14579697bd5dcf9264fad`,作为最终代码验收基点 | -| 23 | 协调者,主目录 | 使用说明、设计状态、README、CHANGELOG、测试指南和本记录按路径暂存;核对暂存范围及文档检查后中文提交 | 本文所在文档提交;代码保持上述验收基点内容 | +| 23 | 协调者,主目录 | 使用说明、设计状态、README、CHANGELOG、测试指南和本记录按路径暂存;核对暂存范围及文档检查后中文提交 | `09faa013d24e18188042bf951a785a4f4e3c2e0b`;代码保持上述验收基点内容 | +| 24 | 协调者,主目录 | 核对工作区、暂存区和工作树后执行 `git branch -m feature/database-crud` | 主目录改用功能分支名,HEAD 仍为 `09faa013d24e18188042bf951a785a4f4e3c2e0b`,全部提交保留 | +| 25 | 协调者,主目录 `feature/database-crud` | 按路径暂存贡献指南及三份 CRUD 文档,核对差异后以 `docs(crud): 规范功能开发分支命名` 提交 | 本次仅记录命名规范与当前分支,保留历史命令,产品代码未改动 | -最终分支 HEAD 随文档提交前进,可用 `git rev-parse docs/database-crud-design` 获取;代码验收绑定于 `3594f21ab214b144e9d14579697bd5dcf9264fad`。 +最终分支 HEAD 随文档提交前进,可用 `git rev-parse feature/database-crud` 获取;代码验收绑定于 `3594f21ab214b144e9d14579697bd5dcf9264fad`。本次更名基线为 `09faa013d24e18188042bf951a785a4f4e3c2e0b`,历史记录中的旧分支名称保持原样。 ## 3. 验证环境和命令 @@ -141,7 +143,7 @@ Redis 5.0 的临时验收脚本最初使用了仅新版存在的 `aclose()`; ## 6. 最终验收状态 本地功能实现、回归、真实 MySQL/MongoDB 以及 Redis 模拟验收完成,代码和使用文档交付用户审查。 -任务工作树保留;用户主目录停在 `docs/database-crud-design`。 +任务工作树保留;用户主目录停在 `feature/database-crud`。 本地共 22 个工作树,包含本任务新增的 7 个独立工作树;既有其他工作树没有被删除、搬移或改写。 `develop` 仍为初始基线 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`,没有被提交、合并或改写。 没有执行远端查询、同步、推送或远端分支清理。 @@ -150,5 +152,5 @@ Redis 5.0 的临时验收脚本最初使用了仅新版存在的 `aclose()`; 多数据库和同类多实例同时使用已由公共层离线集成回归覆盖,这不替代真实三后端联合验收。 未将 Lua 模拟结果标成真实 Redis 服务验证;此次也未进行吞吐量基准或生产负载测试。 -复核可使用 `git log --first-parent --oneline b670712..docs/database-crud-design` 查看交付顺序,再进入每个 `--no-ff` 合并查看模块内提交。 +复核可使用 `git log --first-parent --oneline b670712..feature/database-crud` 查看交付顺序,再进入每个 `--no-ff` 合并查看模块内提交。 用户审查通过后自行决定何时合入 `develop`。 diff --git a/docs/crud-usage.md b/docs/crud-usage.md index 1c8d0ff..32c01e1 100644 --- a/docs/crud-usage.md +++ b/docs/crud-usage.md @@ -4,7 +4,7 @@ MySQL 对应表,MongoDB 对应集合,Redis 对应采用本库格式的 Hash 记录。 它们共享调用方式、字段投影和结果结构;事务、类型系统和查询能力仍遵循各数据库的真实语义。 -本功能在 `docs/database-crud-design` 分支开发,具体测试范围与待验收事项见 [实施记录](crud-implementation.md)。 +本功能在 `feature/database-crud` 分支开发,具体测试范围与待验收事项见 [实施记录](crud-implementation.md)。 完整设计背景见 [CRUD 设计](crud-design.md),内部模块协作见 [适配器契约](crud-adapter-contract.md)。 ## 1. 绑定已有连接 -- Gitee From 6d5abb030e239d0887966db714810a6578e0701f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:08:21 +0800 Subject: [PATCH 65/86] =?UTF-8?q?docs(crud):=20=E7=A1=AE=E5=AE=9A=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E4=BD=BF=E7=94=A8=E8=AE=A1=E5=88=92=E5=B9=B6=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E7=9B=B8=E5=AF=B9=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- AGENTS.md | 2 +- CONTRIBUTING.md | 2 + docs/README.md | 2 +- docs/connection-management-design.md | 2 +- docs/connection-management-validation.md | 2 +- docs/crud-adapter-contract.md | 2 +- docs/crud-design.md | 1119 +++++++++++++--------- docs/crud-implementation.md | 26 +- 8 files changed, 687 insertions(+), 470 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 4483040..12e753b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ - 先看 `git status --short --branch`、任务相关差异和暂存区,保留工作区已有改动。 - 使用示例见 [README](README.md),开发和提交规范见 [贡献指南](CONTRIBUTING.md)。 -- `src/database/config.py` 负责配置和参数校验,`connections.py` 负责命名连接和生命周期,`errors.py` 负责异常转换。 +- `src/database/config.py` 负责配置和参数校验,`src/database/connections.py` 负责命名连接和生命周期,`src/database/errors.py` 负责异常转换。 - `src/database/_adapters/` 负责高层连接与驱动之间的适配。原生工厂维护在 `src/database/{mongodb,mysql,redis}/`,离线测试放在 `tests/`。 ## 修改连接代码 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c372460..fe22b56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,6 +18,8 @@ python -m pip install ruff - [.ruff.toml](.ruff.toml) 统一 Python 3.10 语法目标、120 字符行宽、单引号格式,以及 `E`、`F`、`I` 规则。一行导入一个对象。 - 新增公开接口写明参数和返回值类型。涉及默认值、异常或资源生命周期时,用 NumPy 风格的 docstring 描述契约,完整用法示例维护在 README 或相关文档。 - Markdown 普通段落及引用正文采用一段一行、段间空行。列表、表格、代码块和显式换行保留各自结构。 +- 文档正文、表格和命令中的项目路径以仓库根目录为基准,使用相对路径,不写盘符、用户主目录等本机绝对地址;仓库根目录记为 `.`,相邻工作树可写为 `../database-worktrees/<任务目录>`。 +- Markdown 本地链接使用相对目标,按文档所在目录保留必要的 `../` 跳转;链接文字展示路径时使用仓库根目录相对路径。仓库外的历史验收材料只记录文件名与用途,实际存储位置由运行者指定,不将其写成仓库内文件。 - 根据 [文档索引](docs/README.md) 更新对应文档。文档中的连接地址、凭据和测试数据使用本地占位值。 ## 提交前检查 diff --git a/docs/README.md b/docs/README.md index 3631159..90afc65 100644 --- a/docs/README.md +++ b/docs/README.md @@ -28,6 +28,6 @@ | [src/database/](../src/database/) | 公共导出、连接配置、统一句柄和异常转换 | | [src/database/_adapters/](../src/database/_adapters/) | 三种后端高层连接的创建、检查、关闭及驱动错误映射 | | [src/database/crud/](../src/database/crud/) | 可选的同步/异步记录操作、条件模型、批量结果和后端适配 | -| [mongodb/](../src/database/mongodb/)、[mysql/](../src/database/mysql/)、[redis/](../src/database/redis/) | 后端原生客户端工厂及各自的辅助功能 | +| [src/database/mongodb/](../src/database/mongodb/)、[src/database/mysql/](../src/database/mysql/)、[src/database/redis/](../src/database/redis/) | 后端原生客户端工厂及各自的辅助功能 | | [tests/](../tests/) | 离线自动化测试 | | [scripts/verify_connections.py](../scripts/verify_connections.py) | 使用显式测试配置验证真实连接 | diff --git a/docs/connection-management-design.md b/docs/connection-management-design.md index f0be27e..8d89b93 100644 --- a/docs/connection-management-design.md +++ b/docs/connection-management-design.md @@ -79,7 +79,7 @@ SDK 直接调用驱动的边界可以识别明确来源的底层超时/传输异 - 适配器应在需要标注具体目标的检查异常处调用 convert_error(... owned=True, classifier=self.classify) 并 raise from;核心同样处理创建/检查/关闭外围异常。已统一错误不重复包装。 - `errors` 和配置层独立于驱动。创建连接时由对应后端加载驱动。 - 公共 __init__ 导出由协调者统一修改。worker 不改现有原生工厂文件。 -- 新测试独立 tests/test_connection_config.py、test_connection_errors.py、test_connections.py、test_managed_.py。 +- 新测试独立 tests/test_connection_config.py、tests/test_connection_errors.py、tests/test_connections.py、tests/test_managed_.py。 ## 开发、分支和历史 总线 feature/database-connection-management 从 develop 已提交HEAD创建,独立 worktree;develop 只读,由用户最终审核合入。 diff --git a/docs/connection-management-validation.md b/docs/connection-management-validation.md index f765f19..52f9230 100644 --- a/docs/connection-management-validation.md +++ b/docs/connection-management-validation.md @@ -8,7 +8,7 @@ - 唯一交付分支:`feature/database-connection-management`。 - 基线 `develop`:`7d38b1ffe13a17b37d0bffc15e175085dcf0bb19`。 - 完整源码及安装验收检查点:`b2c23a8454590c705789d571c120f0fb684d3a70`;其后仅补充本验收记录和清理文件末尾空行,不改变受测代码行为。 -- 集成 worktree:`D:\996-work\loongspears\database-worktrees\connection-management`。 +- 集成 worktree:`../database-worktrees/connection-management`。 - 原工作区保留在 `chore/update-git-config`,提交 `8f1352295217973a1b8624a8a01cd39cf88439a2`。 - `temp/save` 保持 `89497823698812e1c85e6c4c12af86d4281543cf`,仅参考有效设计,没有整体合并。 diff --git a/docs/crud-adapter-contract.md b/docs/crud-adapter-contract.md index e2d048c..559efcd 100644 --- a/docs/crud-adapter-contract.md +++ b/docs/crud-adapter-contract.md @@ -52,4 +52,4 @@ Redis 不需要实现 `find/count/update_many/delete_where`,公共层会拒绝 已发送或可能发送的块无法判断的项留为 `unknown`。SQL 分组在块内停止时,明确未发送的组可标记 `not_sent`。读分块有已确认明细时,`BatchItem.result` 为 `ReadItem`,其 `index` 同样是分块内索引。 核心将已发送块的未知项与后续未发送项区分,保留原始异常及因果链。捕获取消时按同样方式附加部分结果后重新抛出同一个对象。适配器有已知明细时通过 `ChunkError.partial_result` 提供;保留原始取消时,可在原对象上附加 `database_partial_result`(值为 `ChunkResult`)。 -`UnsupportedOperationError`、`RecordConflictError`、`RecordFormatError`、`RecordIdentityError` 位于 `_errors.py`。适配器已明确分类的异常保留原生对象和 `raise ... from ...` 关系。其他驱动异常由公共层使用连接句柄现有转换。 +`UnsupportedOperationError`、`RecordConflictError`、`RecordFormatError`、`RecordIdentityError` 位于 `src/database/crud/_errors.py`。适配器已明确分类的异常保留原生对象和 `raise ... from ...` 关系。其他驱动异常由公共层使用连接句柄现有转换。 diff --git a/docs/crud-design.md b/docs/crud-design.md index b914a5f..85aaee1 100644 --- a/docs/crud-design.md +++ b/docs/crud-design.md @@ -1,548 +1,763 @@ -# database 高级 CRUD 设计提案 +# database 基础使用封装设计与开发计划 -日期:2026-11-15。 -状态:已在 `feature/database-crud` 实现;同步/异步接口和本地验收记录供用户审查,真实 Redis 拓扑验收仍待补齐。 -基线:`develop` 的已提交版本 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 -当前功能分支:`feature/database-crud`,由原文档分支 `docs/database-crud-design` 更名,保留全部提交历史。 +日期:2026-11-16 -本文中的 CRUD 模块、方法和结果类型已经实现;最终接口细节见 [使用说明](crud-usage.md),实际验证范围见 [实施与验收记录](crud-implementation.md)。 -当前已经实现的连接接口以 [连接管理设计](connection-management-design.md) 和 [项目 README](../README.md) 为准。 -用户表述中的 Redis“金融边界”在本文按前文理解为“兼容与功能边界”,同时纳入事务、原子性、金额精度与持久化约束,供审查确认。 +状态:计划文档已生成,供用户审查;本轮未修改产品代码、未实施新增功能。 -## 1. 目标与适用场景 +文档位置:`docs\crud-design.md`。 -为同一项目同时使用 MySQL、MongoDB 和 Redis,或连接多个同类数据库的场景,提供可选的高级记录操作接口。 -开发者通过已经命名的连接绑定表、集合或 Redis Hash 命名空间,再执行单条、批量、字段投影和条件操作。 -设计优先覆盖记录操作的真实语义、批量失败的可诊断性和原生能力的可达性。 +本文替代上一版仅覆盖记录 CRUD 的设计。当前已有的连接管理与记录 CRUD 可以复用,新增结构管理和 Redis String 能力仍为计划内容。以下开发、提交、合并与验收步骤均为后续工作,不表示本轮已经执行。 -第一版拟覆盖以下范围。 +## 一、目标与实现范围 -| 范围 | 拟议能力 | -| --- | --- | -| 按键操作 | 单条和多条读取、插入、更新、删除 | -| 集合操作 | MySQL/MongoDB 条件查询、计数、同条件批量更新和删除 | -| 投影 | 明确选择普通顶层字段 | -| 查询规模 | 显式 limit、offset 和字段排序 | -| 批量执行 | 分块执行、后端结果保留、部分失败及结果未知的表达 | -| Redis | 调用方明确选择的 Hash 记录适配、TTL、单机/Sentinel/Cluster | -| 同步与异步 | 相同参数和结果契约,分别使用同步调用与 await | -| 原生能力 | 继续通过连接句柄的 native 访问原生客户端 | - -跨连接事务、自动索引维护、数据模型迁移、JOIN/聚合的跨库翻译、自动重试、Redis 全库扫描模拟查询,留在本期公共 CRUD 范围之外。 -这些范围选择是本提案的建议,随用户审查调整。 - -## 2. 资源绑定与原生客户端 - -拟新增 `database.crud`,提供 `Records` 和 `AsyncRecords` 两个入口。 -绑定方法接收现有 `Connection` 或 `AsyncConnection`,不创建额外主连接,不缓存全局实例,也不关闭调用方的连接。 -绑定只校验输入与后端类型,保持无网络请求。 -公共导入继续保持惰性驱动加载。 - -| 绑定方式 | 必要声明 | -| --- | --- | -| `Records.mysql(connection, table=..., key_fields=...)` | 表名与实际唯一键字段,允许复合键 | -| `Records.mongodb(connection, database=..., collection=..., key_fields=("_id",), session=None)` | 数据库、集合、实际唯一键,可绑定调用方提供的原生 session | -| `Records.redis_hash(connection, namespace=..., key_fields=..., codec=...)` | 显式命名空间、记录身份字段、字段编解码规则 | +### 1.1 交付目标 -`AsyncRecords` 提供相同的绑定方法,返回异步记录操作对象。 -调用方声明的唯一键应当与真实数据约束一致;绑定时不隐式建索引或修改表结构。 -MySQL 和 MongoDB 保留键的原生类型,MongoDB 的 ObjectId 不自动转换为字符串。 -按键操作接收完整键映射,例如 `{"id": 1001}` 或 `{"tenant_id": 7, "id": 1001}`,不假设所有数据库都存在单列 id。 -查询若发现声明的唯一键对应多条记录,应报数据约束错误,避免静默选择一条。 +开发者安装本 SDK 后,应当能够仅使用本 SDK 的公开接口,完成以下基础流程: -`connection.native` 保持真实原生对象身份,原有 `database..Client` 工厂契约继续适用。 -MySQL 的原生 SQL、事务、游标,MongoDB 的聚合、更新操作符、session,以及 Redis 的 List、Set、Stream、Pipeline 等能力继续可用。 -调用方自行创建的原生对象由调用方管理,CRUD 层负责自己创建的游标、Pipeline 和临时数据客户端。 -连接关闭后,记录操作通过原连接句柄检查并抛出 `ConnectionClosedError`。 +**显式配置 → 创建连接 → 准备存储结构 → 绑定操作对象 → 单条与批量 CRUD → 关闭资源。** -## 3. 操作集合与后端支持矩阵 +同时满足: -以下方法名为本提案的统一命名。 +- 一个项目同时使用 MySQL、MongoDB、Redis。 +- 同一种数据库可以配置多个连接,分别面向不同业务、实例或数据库。 +- 同步和异步接口覆盖相同的基础能力。 +- 原生客户端始终可用,高级功能继续使用原生驱动。 +- 初始化由调用方显式执行,CRUD 不自动建表、改表、建立索引或迁移结构。 +- SDK 不读取环境变量、`.env` 或其他隐式配置来源。 -| 方法 | 意图 | 返回契约 | -| --- | --- | --- | -| `get(key, *, fields=None)` | 按唯一键读取一条 | 记录字典或 None | -| `get_many(keys, *, fields=None, batch_size=500)` | 按输入键批量读取 | 与输入位置对齐的 ReadItem 列表 | -| `find(*, where=None, fields=None, limit, offset=0, order_by=())` | 按条件查询有限条记录 | 记录字典列表 | -| `count(*, where=None)` | 按条件计数 | 非负整数 | -| `insert(record, ...)` | 插入新记录 | WriteResult | -| `insert_many(records, *, batch_size=500, ...)` | 批量插入 | BatchResult | -| `update(key, *, set_values, ...)` | 仅更新指定记录的部分字段 | WriteResult | -| `update_many(*, where, set_values, all_records=False)` | 相同条件、相同修改,作用于全部匹配记录 | WriteResult | -| `bulk_update(items, *, batch_size=500, ...)` | 多个唯一键分别执行不同修改 | BatchResult | -| `delete(key)` | 按键删除 | WriteResult | -| `delete_many(keys, *, batch_size=500)` | 按已知键列表删除 | BatchResult | -| `delete_where(*, where, all_records=False)` | 删除全部条件匹配记录 | WriteResult | - -`insert` 使用仅创建语义,已有身份对应记录时失败。 -`update` 使用仅更新语义,记录不存在时返回未匹配结果,默认不创建记录。 -`set_values` 只修改列出的字段,记录键不可通过该参数修改。 -记录替换、upsert、字段删除、原子自增和混合类型 bulk_write 使用原生接口,后续按明确场景增加公共操作。 - -| 能力 | MySQL | MongoDB | Redis Hash 记录适配 | +数据库服务的安装、启动、账号创建、权限授予和集群部署属于运行环境准备,不纳入 SDK。 + +### 1.2 三种数据库的基础封装边界 + +统一开发者的使用流程,同时保留各数据库真实存在的概念。 + +| 层次 | MySQL | MongoDB | Redis | | --- | --- | --- | --- | -| 按键单条/批量读取和投影 | 支持 | 支持 | 支持 | -| 按键单条/批量插入、修改、删除 | 支持 | 支持 | 支持,遵守记录格式和单键执行约束 | -| where 条件查询、排序、计数 | 支持约定的公共条件 | 支持约定的公共条件 | 第一版不提供 | -| where 条件 update_many/delete_where | 支持 | 支持 | 第一版不提供 | -| 自动生成身份 | 依赖表定义 | 默认可由驱动生成 _id | 必须显式提供身份 | -| 多记录事务 | 遵循原生连接和表引擎能力 | 使用调用方提供的原生 session | 遵循 Redis 原生事务能力,不能等同 SQL 回滚 | +| 连接 | 连接实例,明确选择业务数据库 | 连接实例或集群 | 连接单机、Sentinel 或 Cluster | +| 初始化 | 创建数据库、声明并创建基础表 | 显式创建普通集合,由此完成数据库初始化 | 选择配置中的编号库,绑定 Hash 命名空间或使用 String 键 | +| 结构查看 | 列出、检查数据库与表,查看表结构 | 列出、检查数据库与集合,查看集合信息 | 查看指定键的存在性和 TTL | +| 基础结构删除 | 显式删除表、数据库 | 显式删除集合、数据库 | 删除指定键或已知记录 | +| 记录操作 | 行记录 CRUD、条件查询与批量操作 | 文档 CRUD、条件查询与批量操作 | Hash 记录 CRUD、按已知键批量操作 | +| 普通键值操作 | 不新增此抽象 | 不新增此抽象 | String 单键与批量读写、删除、过期控制 | +| 高级能力 | 通过原生客户端 | 通过原生客户端 | 通过原生客户端 | -调用当前后端不支持的方法时,在发送操作前抛出拟议的 `UnsupportedOperationError`,包含后端和操作名称。 -不把失败返回空列表或零条记录,也不通过隐式扫描或创建索引补足缺失能力。 +MongoDB 创建第一个真实集合即可完成初始化,不提供没有实际服务端意义的“创建空数据库”操作。[MongoDB 数据库与集合](https://www.mongodb.com/docs/manual/core/databases-and-collections/) -## 4. 公共条件与查询边界 +Redis 不增加虚构的表或集合。编号库由部署提供,Cluster 仅支持数据库 0;Hash 命名空间是 SDK 的键命名约定,不是服务端容器。[Redis SELECT](https://redis.io/docs/latest/commands/select/) -第一版普通字典 `where={"status": "pending", "tenant_id": 7}` 表达顶层标量字段的 AND 等值条件。 -扩展比较拟采用明确条件对象 `Eq`、`In`、`Gt`、`Gte`、`Lt`、`Lte`、`And`、`Or`、`IsNull` 和 `Exists`,只提供已定义语义且通过后端测试的操作符。 -条件对象是数据结构,不接收任意 Python 回调或拼接 SQL 字符串。 -MongoDB 原生操作符字典不能直接放入公共 where 冒充统一表达式。 +### 1.3 当前实现与本轮增量 -| 情况 | 公共契约 | -| --- | --- | -| `Eq(field, None)` 或字典值 None | 采用显式空值语义,等价于 IsNull(field) | -| MySQL IsNull | SQL IS NULL | -| MongoDB IsNull | 字段存在且为 BSON null,避免同时匹配缺失字段 | -| Exists | MongoDB 表达文档字段存在性,MySQL 不提供行级“缺列”语义,报能力错误 | -| In 的空列表 | 恒不匹配,写入影响数可确定为零 | -| 普通标量比较 | 按实际后端类型和排序规则执行,MongoDB 不将数组元素匹配冒充标量相等 | -| 不合法字段、未支持的值类型、运算符冲突 | 发送前报告配置或能力错误 | - -MySQL 值使用驱动参数绑定;表名、字段名、排序方向使用独立校验和标识符引用规则。 -MongoDB 公共标量条件需处理 BSON 类型与数组匹配差异,不能直接透传用户结构形成操作符注入。 -字符串大小写、排序规则、日期时区和数值范围继续取决于后端及字段定义,公共 API 不承诺跨后端查询结果自动等价。 -高级 SQL 表达式、MongoDB 嵌套路径和数组条件由原生接口表达。 - -`find` 要求显式正整数 limit,offset 为非负整数。 -省略 order_by 时返回顺序由数据库决定。 -使用分页时建议调用方给出包含唯一键的完整排序,翻页期间的数据变更仍按后端一致性语义处理。 -一次 find 在返回前取完受 limit 限制的结果并关闭内部游标,不将待消费游标混入列表返回类型。 -第一版不引入跨后端流式游标或可互换的分页 token。 - -## 5. 字段投影与读取结果 - -### 5.1 fields 的语义 - -`fields=None` 表示返回全部业务字段。 -`fields=["name", "email"]` 表示只返回这些顶层字段。 -`fields=[]` 表示只确认记录是否存在,存在记录返回空字典,缺失记录仍返回 None。 -重复字段名、字段表达式和不合法名称在发送前报错。 -公共 fields 第一版采用包含模式,不混用排除模式、SQL 别名或计算表达式。 - -| 后端 | 获取 name/email 的执行方式 | 必须处理的差异 | -| --- | --- | --- | -| MySQL | SELECT 对应列,使用字典结果 | 列名引用与值绑定分离,列不存在时保留驱动异常 | -| MongoDB | 包含投影并按需设置 _id: 0 | 仅在请求 _id 时返回它,避免驱动默认附带字段 | -| Redis Hash | 对指定键读取 HMGET 对应字段 | 业务值解码,内部身份/版本字段不进入记录 | +当前工作分支为 `feature/database-crud`。生成本文之前,工作区没有待提交修改。 -投影尽量在服务端执行。 -`fields=[]` 可查询常量、身份或内部标记以确认存在性,但这些内部读取值不进入返回记录。 -Redis String 中存储 JSON 的场景属于另一种数据格式;不将下载完整 JSON 后裁剪伪装成 Hash 的字段读取。 -MongoDB 的包含投影及 _id 特例依据官方接口处理。[MongoDB 字段投影](https://www.mongodb.com/docs/languages/python/pymongo-driver/current/crud/query/project/) +- 当前实现起点:`ce740b3aefa554c7e4f0babb7c09b9b53f520ea0`。 +- 本地 `develop`:`b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 -### 5.2 空值、缺失字段和缺失记录 +现有连接管理、记录 CRUD、条件表达式、投影、批量结果和异常处理继续复用。本计划主要增加: -| 情况 | 读取结果 | -| --- | --- | -| 记录不存在 | None | -| 记录存在,字段值是数据库空值 | 字典中该字段为 None | -| MongoDB 文档或 Redis Hash 记录缺失所选字段 | 返回字典省略该字段 | -| 记录存在,但所选字段均缺失 | 空字典 | -| MySQL 选择了表中不存在的列 | 保留原始驱动异常的统一错误 | +1. MySQL 基础数据库与表管理。 +2. MongoDB 基础数据库与集合管理,以及必要的记录身份约束。 +3. Redis String 基础操作和批量操作。 +4. 从初始化到 CRUD 的完整示例与验收。 -MySQL 已有列的 NULL 与 MongoDB/Redis 的缺失字段保持上述区别,不用 None 填平所有情况。 -日期、Decimal、ObjectId、bytes 等值保留驱动或显式 codec 解码得到的类型,避免强制 JSON 化导致精度或类型丢失。 -Redis 的 HMGET 对“键不存在”和“字段不存在”均可能给出 nil;单凭一组 nil 不能判断记录不存在。 -Redis 适配使用同一次单键原子执行中的记录标记检查和字段读取来区分两者。[Redis HMGET](https://redis.io/docs/latest/commands/hmget//) +原有记录 CRUD 阶段的测试结果继续作为历史记录,不能替代新增功能的验收。已有实际接口参见 [CRUD 使用说明](crud-usage.md),历史结果参见 [CRUD 实施与验收记录](crud-implementation.md),连接层约定参见 [连接管理设计](connection-management-design.md)。 -例如请求 fields=["name", "email"] 时,以下结果反映了数据实际差别。 +## 二、公共接口与行为约定 -| 后端及数据 | get 返回值 | -| --- | --- | -| MySQL 行 name 为 Alice,email 为 SQL NULL | `{"name": "Alice", "email": None}` | -| MongoDB 文档 name 为 Alice,没有 email 字段 | `{"name": "Alice"}` | -| Redis Hash 记录 name 为 Alice,email 编码保存为 null | `{"name": "Alice", "email": None}` | -| 任一后端的目标记录不存在 | `None` | -| 任一后端记录存在,请求 fields=[] | `{}` | +### 2.1 初始化与基础结构管理 -### 5.3 批量读取 +新增 `database.schema`,提供: -`get_many` 输入为有限键序列,结果与输入顺序和数量一致,重复的输入键保留对应结果位置。 -同一批内重复键只读取一次并复用该次观察,避免一次调用中同一个键出现两个矛盾结果。 -后端查询返回乱序时由适配器按输入键重排,额外查询的身份字段不会泄露到投影结果中。 +```python +from database.schema import AsyncSchema +from database.schema import Column +from database.schema import Schema +from database.schema import TableSpec +``` + +绑定入口为: ```python -# 拟议返回示例,ReadItem 的三个字段为 index、key、record。 -[ - ReadItem(index=0, key={"id": 1001}, record={"name": "Alice"}), - ReadItem(index=1, key={"id": 1002}, record=None), - ReadItem(index=2, key={"id": 1001}, record={"name": "Alice"}), -] +mysql_schema = Schema.mysql(mysql_connection) +mongo_schema = Schema.mongodb(mongo_connection) + +async_mysql_schema = AsyncSchema.mysql(async_mysql_connection) +async_mongo_schema = AsyncSchema.mongodb(async_mongo_connection) ``` -空输入返回空列表且无网络请求。 -整个批次不承诺单一时间点快照;MySQL/MongoDB 需要更强一致性时使用原生事务或 session。 -批量读取中途失败应抛出带已知部分读取结果的批量异常,尚未读取或无法确认的项标为未知,不能填写 record=None 冒充记录缺失。 -只有完整成功返回时,每个 ReadItem.record 才能使用上述 None/字典约定。 +绑定本身不执行初始化。异步绑定不需要 `await`,异步管理方法需要 `await`。 + +管理对象借用连接,不拥有连接,也不要求再嵌套一层管理上下文。方法内部创建的游标由方法负责关闭。 + +#### MySQL + +| 方法 | 行为与返回值 | +| --- | --- | +| `ensure_database(name, *, charset="utf8mb4")` | 不存在则创建;存在则核对声明的字符集;成功返回 `None` | +| `list_databases()` | 返回当前账号可见的数据库名称列表 | +| `database_exists(name)` | 判断数据库是否在当前账号可见目录中 | +| `ensure_table(database, table, spec)` | 创建基础表,或校验已有表的兼容性;成功返回 `None` | +| `list_tables(database)` | 返回普通基础表名称列表 | +| `table_exists(database, table)` | 判断指定基础表是否可见 | +| `describe_table(database, table)` | 返回表信息字典;在可见目录中不存在时返回 `None` | +| `drop_table(database, table)` | 显式删除指定表;不存在时不报错 | +| `drop_database(name)` | 显式删除指定数据库;不存在时不报错 | -## 6. 批量写入 +`describe_table()` 返回数据库名、表名、字段信息、主键顺序、引擎及字符集等基础信息。字段信息保留实际类型、可空性、默认值和生成属性,不将复杂默认表达式伪装成 Python 值。 -### 6.1 相同条件、相同修改 +基础建表模型: ```python -result = orders.update_many( - where={"status": "pending"}, - set_values={"status": "closed"}, +user_table = TableSpec( + columns={ + "id": Column("varchar", length=32), + "name": Column("varchar", length=100), + "age": Column("int"), + }, + primary_key=("id",), ) ``` -MySQL 生成一次参数化 `UPDATE ... SET ... WHERE ...`。 -MongoDB 调用集合的 `update_many(filter, {"$set": values})`。 -Redis Hash 适配第一版对这种集合条件操作报告能力错误。 -条件更新直接在服务端求值,避免先查出全部身份再更新造成语义变化。 -update_many/delete_where 默认拒绝空条件,全量操作需调用方显式传入 `all_records=True`;这是提议的防误用约定,待本次审查确认。 -单次集合操作不会为了套用 batch_size 而擅自拆成多个查询。 +本期约定: + +- `Column` 支持 `int`、`bigint`、`boolean`、`decimal`、`double`、`varchar`、`text`、`date`、`datetime`、`blob`。 +- 支持适用的长度、精度、小数位、可空性、字面量默认值,以及单一整数主键的自增。 +- `nullable` 默认 `False`;未提供默认值表示不声明默认值。 +- 不接受任意 SQL 类型片段、默认值表达式或约束表达式。 +- 新表使用 InnoDB,并继承目标数据库的字符集与排序规则。 +- 必须声明主键,可以是单字段或组合主键。 + +已有表采用已经确认的“基础可用兼容”规则: + +- 声明的字段必须存在,规范化后的类型、长度或精度、可空性及明确声明的默认值、自增属性需要匹配。 +- 主键字段及顺序需要匹配。 +- 允许额外字段,但额外字段必须允许省略,例如可空、有默认值或由数据库生成。 +- 保留已有表的其他索引、引擎及高级约束,不擅自修改。 +- 不兼容时抛出 `SchemaConflictError`,列出差异,不执行 `ALTER TABLE`。 + +这项检查确认声明结构的兼容性,不保证任意业务数据都能绕过已有的外键、触发器或检查约束。 + +MySQL 创建、删除结构可能隐式提交。因此,结构写操作要求连接启用 `autocommit` 且没有活动事务;示例使用独立初始化连接。SDK 不暗中调整事务状态或执行 `USE` 切换业务库。[MySQL 隐式提交规则](https://dev.mysql.com/doc/refman/8.4/en/implicit-commit.html) + +#### MongoDB + +| 方法 | 行为与返回值 | +| --- | --- | +| `ensure_collection(database, collection, *, key_fields=("_id",))` | 显式准备普通集合及必要身份约束;成功返回 `None` | +| `list_databases()` / `database_exists(name)` | 查看当前账号可见的数据库 | +| `list_collections(database)` / `collection_exists(database, collection)` | 查看普通集合 | +| `describe_collection(database, collection)` | 返回集合类型、选项等原生元数据;不可见或不存在时返回 `None` | +| `drop_collection(database, collection)` | 显式删除集合;不存在时不报错 | +| `drop_database(name)` | 显式删除数据库;不存在时不报错 | + +集合准备遵循以下规则: + +- 默认使用 `_id`,不额外创建索引。 +- 声明自定义 `key_fields` 时,仅准备该身份所必需的完整唯一索引。 +- 已有等价约束可复用:索引字段及顺序与声明一致、具有唯一性、不使用 sparse 或 partial 过滤,且排序规则与记录查询一致;不自动删除、转换或重建其他索引。 +- 已有同名视图、时序集合或 capped 集合不作为普通记录集合接受。 +- 集合创建和必要索引建立必须得到确认,不能仅凭目录中出现索引定义就认定构建完成。 +- 如果集合已创建而索引创建失败,保留集合及原始异常,不自动删除集合补偿。 +- 初始化要求可确认的写关注;不使用 `w=0` 宣称结构准备成功。 +- 本期不处理分片规划;分片集合的全局唯一性继续遵守原生分片键规则。 + +MongoDB 自定义唯一索引可能因已有重复数据而建立失败,这属于应向调用方保留的真实错误。[MongoDB 唯一索引](https://www.mongodb.com/docs/manual/core/index-unique/) + +#### 共同管理规则 + +- `ensure_*` 不返回推测的 `created=True`,避免并发初始化时误报创建者。 +- 相同定义重复执行应成功;遇到并发创建,只在核实最终结构兼容后视为成功。 +- `drop_*` 只处理显式名称,不支持通配删除,不在上下文退出时自动调用。 +- “不存在”与“权限不足”分开处理;明确的权限异常继续抛出。 +- 列表和存在性检查反映当前账号的目录可见性,不承诺全局可见性。 -### 6.2 每条记录分别修改 +### 2.2 记录 CRUD + +保留已有入口: ```python -result = orders.bulk_update([ - {"key": {"id": 1001}, "set_values": {"status": "paid"}}, - {"key": {"id": 1002}, "set_values": {"status": "cancelled"}}, -], batch_size=500) +Records.mysql(connection, table=..., key_fields=...) +Records.mongodb(connection, database=..., collection=..., key_fields=...) +Records.redis_hash(connection, namespace=..., key_fields=..., codec=...) ``` -| 后端 | 第一版执行策略 | 结果限制 | -| --- | --- | --- | -| MySQL | 按连续相同更新字段形状分组,以参数化 executemany 执行 | 通常只能获得组级 rowcount,失败时驱动可能不公开确切执行位置 | -| MongoDB | 集合级 bulk_write,使用 UpdateOne,默认 ordered=True | 保存组级统计、错误索引和已报告的执行范围,普通汇总未必含逐项匹配数 | -| Redis Hash | 单键条件更新命令/脚本组成 pipeline,显式 transaction=False | 成功读取回复时可获得逐项结果,传输中断后有无法确认的项 | +`AsyncRecords` 提供相同绑定方式,操作方法使用 `await`。 -Connector/Python 的 executemany 通常逐组调用 execute,针对 INSERT 有多行优化,因此一次方法调用不等于只发送一条 SQL。[MySQL executemany](https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-executemany.html) -MongoDB 使用集合级 bulk_write,避免为本期功能额外要求 MongoClient 跨命名空间批量接口及其服务器版本。[PyMongo bulk_write](https://www.mongodb.com/docs/languages/python/pymongo-driver/current/crud/bulk-write/) +| 能力 | 方法 | MySQL | MongoDB | Redis Hash | +| --- | --- | --- | --- | --- | +| 按键读取 | `get`、`get_many` | 支持 | 支持 | 支持 | +| 新增 | `insert`、`insert_many` | 支持 | 支持 | 支持 | +| 指定键修改 | `update`、`bulk_update` | 支持 | 支持 | 支持 | +| 指定键删除 | `delete`、`delete_many` | 支持 | 支持 | 支持 | +| 条件查询、排序、分页 | `find` | 支持公共条件范围 | 支持公共条件范围 | 不支持 | +| 条件计数 | `count` | 支持 | 支持 | 不支持 | +| 相同修改作用于匹配记录 | `update_many` | 支持 | 支持 | 不支持 | +| 条件删除 | `delete_where` | 支持 | 支持 | 不支持 | -批量写入接收有限序列,在首次发送前完成可由 SDK 判断的结构、键、字段和编码校验。 -同一调用中重复的已知记录键默认报配置错误,避免分组、Pipeline 或不同后端执行策略改变同一记录上的依赖顺序。 -依赖前一次结果的操作由调用方明确顺序执行。 -分块按输入顺序提交,发生异常后停止发送后续分块;已经发送的分块可能继续执行,尤其是 Pipeline 中已经排队的命令。 -本期不提供跨后端统一的“发现第一条错误后其余项保证未执行”承诺。 -batch_size 控制传给原生批量接口的分块规模,后端仍可能按消息大小再次拆分。 +不支持的操作明确抛出 `UnsupportedOperationError`。Redis 不通过隐式扫描整个键空间来模拟条件查询和条件修改。 -### 6.3 插入与删除 +批量修改明确区分两种需求: + +```python +# MySQL / MongoDB:匹配的记录全部改为同一个值。 +records.update_many( + where={"status": "pending"}, + set_values={"status": "ready"}, +) -MySQL 批量插入可采用 executemany 的多行优化,MongoDB 使用 insert_many 或集合批量写入,Redis 使用单键仅创建操作组成 Pipeline。 -所有目标后端都将重复身份视为插入冲突,保留原始错误;Redis 需要单键原子存在性判断以避免覆盖已有键。 -批量删除按已知键执行,MongoDB/MySQL 可以使用集合条件组合,但不得把独立复合键展开成各字段的笛卡尔积。 -Redis Cluster 批量删除采用单键命令分发,避免将任意跨槽键列表拼进一个 DEL。 -删除不存在的记录是成功的零删除结果。 -空批次为成功的空结果且不发起网络请求。 -服务端约束和数据竞态仍可能在发送后失败,发送前校验不保证整个批次成功。 +# 三种记录后端:不同记录分别修改不同值。 +records.bulk_update([ + {"key": {"id": "u1"}, "set_values": {"age": 31}}, + {"key": {"id": "u2"}, "set_values": {"age": 42}}, +]) +``` -## 7. 写入返回结果、投影与失败 +MySQL 的 `update_many` 使用条件 UPDATE;`bulk_update` 按记录执行参数化修改并保留逐项结果。本期不引入 CASE SQL 生成器或批量执行优化框架。 -### 7.1 写入结果不隐含回读 +其他约定: -fields 只用于 get/get_many/find 等读取方法。 -insert/update/delete 及其批量方法第一版返回执行信息,不支持统一的 `returning=fields`。 -这是因为三种后端对“修改并返回前值/后值”的能力和原子性不同。 -调用方可以写入后显式 get/get_many 并投影,但两次操作之间可能有其他写入,回读结果不能冒充本次操作原子返回的后值。 -MongoDB find_one_and_update 等专有原子返回能力通过原生对象使用。 -如后续增加公共 returning,需要单独评审各后端保证。 +- `insert` 为仅新增;`update` 为仅修改已有记录。 +- `set_values` 只修改列出的字段,不修改身份字段。 +- 无条件批量修改或删除必须显式设置 `all_records=True`。 +- `find` 必须给出 `limit`,分页稳定性由明确排序及数据库实际并发语义决定。 +- 事务提交、回滚和重试仍由调用方决定。 -### 7.2 WriteResult +#### 字段投影 -| 字段 | 含义 | +| 参数 | 返回约定 | | --- | --- | -| `backend`、`operation` | 执行后端和操作名称 | -| `acknowledged` | 是否收到可解释的执行确认,无法确定为 None | -| `matched_count` | 后端可确定的匹配记录数,否则 None | -| `modified_count` | 后端可确定的实际修改记录数,否则 None | -| `inserted_count`、`deleted_count` | 后端可确定的插入/删除记录数,否则 None | -| `generated_keys` | 后端明确返回的新身份,缺少完整映射时为 None | -| `native_result` | 原生返回对象,或游标关闭前保存的明确原生统计快照 | - -各计数字段按操作和后端填写,None 表示不适用或无法确定,0 必须有明确依据。 -MySQL UPDATE 的 rowcount 保存在 native_result 中,并附带 changed/matched 模式说明;只有驱动配置和语句语义足以确定时才填入对应公共计数,不通过额外 SELECT 猜测匹配数。 -MongoDB 保留 matched_count 与 modified_count 的区别;w=0 等未确认写入的确认标志为 False,无法获得的计数保持 None。[MySQL rowcount](https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-rowcount.html)、[PyMongo 写入结果](https://pymongo.readthedocs.io/en/stable/api/pymongo/results.html) - -Redis HSET 的整数回复是新增字段数,不能直接赋给 modified_count 作为“修改记录数”。 -记录适配的单键脚本可以明确返回是否匹配已有记录或是否成功创建,未做实际值比较时 modified_count 保持 None。[Redis HSET](https://redis.io/docs/latest/commands/hset/) -MySQL 批量插入只有 lastrowid 时,不推算所有自增身份;触发器、配置和并发等因素都可能使连续推算失真。 -生成或分配的身份不代表记录已经成功写入,尤其是未确认写入和随后失败的批量操作。 -原生返回对象或快照可包含敏感数据,日志处理沿用 [安全说明](../SECURITY.md)。 - -### 7.3 BatchResult 与失败状态 - -BatchResult 保存输入项数量、按输入索引排列的项状态、原生分块结果和能够明确汇总的计数。 -拟议字段为 requested_count、items、counts 和 native_results;每个项包含 index、key、state、result 和驱动提供的 error 信息。 -写入成功项的 result 可为 WriteResult,读取成功项的 result 为 ReadItem,无法获得的明细为 None。 -批量读取异常也使用此结构保存 partial_result,只有状态已确认的 ReadItem.record 才能解释为缺失记录或实际字典。 -分块统计覆盖重叠或明细不足时,公共汇总保持 None,不能将“发送项数”当作“成功修改记录数”。 -成功项可能没有逐项匹配或修改数,这与组级执行已确认并不冲突。 - -| 项状态 | 确定含义 | -| --- | --- | -| `acknowledged` | 所属操作执行已获确认,结果可能是零匹配或零删除 | -| `unacknowledged` | 后端配置有意省略执行确认,不能判为已成功修改 | -| `failed` | 收到该项明确失败回复,保存输入索引和原始异常信息 | -| `not_sent` | SDK 或驱动可靠确认该项未发送 | -| `not_executed` | 已发送但后端明确报告因前项失败等原因未执行 | -| `unknown` | 已可能发送或执行,但证据不足以确定结果 | - -批量调用出现实际错误时抛出拟议 `BatchOperationError`,其 partial_result 保存已知结果。 -original 保留驱动交付的最外层异常对象,cause 保留异常链,驱动提供的错误明细对象也按原样保存,不重新拼接出不存在的逐项异常。 -MySQL executemany 失败而无法确定组内执行位置时,该组相应项可为 unknown,后续未发送分组为 not_sent。 -已确认的 MongoDB 错误索引映射回整个输入序列;有 ordered 停止信息时区分后续未执行项。 -Redis 收到完整 Pipeline 回复时保留每条命令错误及成功回复,连接中断时只填写可以确定的结果。 - -执行确认与事务最终提交分别判断。 -MySQL 操作确认后可能仍处于未提交事务,MongoDB session 内操作可能随后被回滚,记录适配不把 WriteResult/BatchResult 的确认当成提交凭证。 -单个命令也可能因原生表引擎、脚本或事务执行语义出现部分影响;失败状态不是“该项绝对没有产生任何效果”的承诺。 -SDK 不根据批量状态自动重试写入。 - -以三个按键更新为例,第一条记录的新值与旧值相同,第二条发生变化,第三条不存在;全部请求正常返回时,统计可以出现以下差异。 - -| 后端 | matched_count | modified_count | 解释 | -| --- | --- | --- | --- | -| MySQL,已明确确认 rowcount 为 changed 模式 | None | 1 | 保存原生 rowcount=1,不补查询以猜测匹配数 | -| MongoDB,已确认写入 | 2 | 1 | 原生结果分别提供匹配数和修改数 | -| Redis Hash 记录脚本 | 2 | None | 脚本确认两条记录存在,未逐值比较时不报告实际改变记录数 | +| `fields=None` | 返回全部业务字段 | +| `fields=["name", "age"]` | 仅返回指定字段 | +| `fields=[]` | 已有记录返回 `{}`;记录不存在返回 `None` | + +三种后端分别使用 SQL 列选择、MongoDB 投影和 Redis Hash 字段读取完成投影。 + +- MongoDB 未请求 `_id` 时不附带 `_id`。 +- MongoDB、Redis 缺失字段省略;明确空值保留为 `None`。 +- MySQL 不存在的列保留为查询错误。 +- 不自动把 `Decimal`、`ObjectId` 等原生值转成字符串。 +- Redis String 不支持字段投影,也不自动解析其中的 JSON。 + +#### 返回结果与批量失败 + +保留已有结果结构: -三项的执行状态均可为 acknowledged,但第三项没有匹配记录;批次执行成功和实际修改数量分别表达。 +- 单条读取:字典或 `None`。 +- 批量读取:按输入位置对应的 `ReadItem` 列表。 +- 单次写入:`WriteResult`。 +- 批量写入:`BatchResult`。 +- 批量失败:`BatchOperationError.partial_result`。 -## 8. Redis 的兼容、格式和功能边界 +返回值遵循证据: -### 8.1 Hash 记录格式 +- 无法确定的匹配数、修改数或生成键使用 `None`。 +- 保留原生结果或游标关闭前取得的统计快照。 +- 不把 MySQL `rowcount` 无条件解释为修改条数。 +- 不推算连续自增 ID,不把执行成功等同于事务已提交。 +- 批量结果区分已确认、失败、未发送、未执行和结果未知。 +- 网络异常后的未知项不能当成失败后直接安全重试。 +- 已派发的同一批次中,某项失败不意味着其他项尚未执行。 -调用方显式启用 redis_hash 适配后,一个记录对应一个 Redis Hash 键。 -namespace 定义独立命名空间;键身份采用确定性、带类型的组合编码,避免字符串拼接造成复合键碰撞。 -默认身份编码仅支持明确约定的字符串与整数,其他类型通过显式编码器支持,不自动将任意对象转成字符串。 +### 2.3 Redis String -Hash 保存业务字段和一个保留的格式版本标记,例如 `__database_record__=1`。 -该标记用于记录存在性、空记录和存储格式识别,业务字段名不能占用保留名称,读取结果隐藏内部字段。 -这一存储约定需要用户审查,它会影响使用同一命名空间的其他程序。 -遇到同名的 String/List/其他类型键,或缺少正确格式标记的 Hash,报告数据格式或原生 WRONGTYPE 错误,不自动覆盖或迁移已有数据。 -现有任意 Redis 数据仍通过原生客户端使用;若需要将已有 Hash 纳入记录接口,应先明确格式、codec 和迁移方案。 +在 `database.crud` 中新增: -字段 codec 负责逐字段编码与解码。 -拟提供 JSON 字段 codec 处理字符串、整数、布尔值、有限浮点数、null 及 JSON 容器,null 以实际编码值保存,以区分字段不存在。 -Decimal、bytes、datetime、ObjectId 等类型需要显式支持的 codec,遇到不支持的类型在发送前报错,不能隐式降精度或转换为随意字符串。 -字段 JSON 容器可以作为整体字段保存;其内部路径查询和局部修改不属于本期公共能力。 -codec 必须与原连接的编码和 decode_responses 设置兼容,二进制 codec 需要能够取得原始字节;不在绑定时偷偷修改原客户端配置。 +```python +from database.crud import AsyncRedisStrings +from database.crud import RedisStrings +from database.crud import StringSetItem +``` -### 8.2 原子存在性与 TTL +使用方式: -仅创建、仅更新、记录格式检查和投影读取使用单键 Lua 脚本或经验证等价的原生原子命令组合。 -存在性检查与写入之间不留客户端竞争窗口。 -脚本先进行参数和键类型校验,再执行写入;不把 Lua 自动理解为具有错误回滚能力的事务。 -部署必须允许所需脚本命令,权限不足时给出明确错误;不能静默降级成非原子的先查再写。 +```python +with RedisStrings(connection) as strings: + strings.set("app:message", "hello", ttl_s=60) + value = strings.get("app:message") +``` -| 动作 | 建议的 TTL 约定 | +| 方法 | 返回值 | | --- | --- | -| 新建记录未指定 ttl_s | 无自动过期 | -| 新建记录指定正整数 ttl_s | 创建和设置过期在同一次单键原子操作内完成 | -| 更新时省略 ttl_s | 保留现有键 TTL | -| 更新时 ttl_s 为正整数 | 从更新时重新设置 TTL | -| 更新时显式 ttl_s=None | 清除键 TTL,变为持久键 | -| 更新时键已过期或不存在 | 返回未匹配,不复活记录 | - -省略与显式 None 使用内部“未提供”标记区分。 -TTL 为零、负数或布尔值时报告配置错误,避免将非法值解释为删除或永久保存。 -批量 Redis 插入和修改项可分别携带 ttl_s。 -对 MySQL/MongoDB 传入 TTL 选项报告能力错误,不能翻译成临时任务或自动 TTL 索引。 -记录级 TTL 不代表业务字段级过期;本期格式不启用 Hash 字段独立过期。 - -### 8.3 单机、Sentinel 和 Cluster - -| 拓扑 | 数据操作与资源归属 | +| `get(key)` | `str`、`bytes` 或 `None` | +| `set(key, value, *, ttl_s=None)` | 成功确认后返回 `True` | +| `delete(key)` | `0` 或 `1` | +| `exists(key)` | `bool` | +| `expire(key, seconds)` / `persist(key)` | `bool` | +| `ttl(key)` | 原生 TTL 整数 | +| `get_many(keys, *, batch_size=500)` | 保留输入顺序、重复项和缺失位置的值列表 | +| `set_many(items, *, batch_size=500)` | `StringBatchResult` | +| `delete_many(keys, *, batch_size=500)` | `StringBatchResult` | + +`StringSetItem(key, value, ttl_s=None)` 支持每项独立设置过期时间。 + +具体语义: + +- 键和值只接受 `str` 或 `bytes`,不自动 JSON 编解码。 +- 使用调用方指定的完整键名,不隐式添加命名空间。 +- 返回类型遵循原生连接的 `decode_responses`。 +- 普通 `set` 为创建或覆写;未指定 TTL 时清除已有 TTL。 +- 指定 TTL 使用单条 `SET EX`,不拆成两次命令。 +- TTL 和 `expire` 接受正整数秒;移除过期使用 `persist`,立即删除使用 `delete`。 +- `ttl` 保留 `-2` 表示键不存在、`-1` 表示没有过期时间的语义。 +- `SET` 和 `DEL` 保留原生命令对同名其他类型键的作用;`GET` 遇到错误类型保留驱动错误。 + +这些行为与 Redis 原生语义一致。[Redis SET](https://redis.io/docs/latest/commands/set/) · [Redis TTL](https://redis.io/docs/latest/commands/ttl/) + +批量实现采用非事务流水线中的逐键命令: + +- 同步、异步均覆盖单机、Sentinel、Cluster。 +- Cluster 按驱动能力路由,不要求所有键处于同一槽。 +- 不承诺批量整体原子性、全局执行顺序或跨节点一致快照。 +- 写批次在发送前完成参数校验,按驱动实际编码后的键拒绝重复写入目标。 +- 新增轻量 `StringBatchItem`、`StringBatchResult`,保存逐项状态、实际返回值、错误及原生结果。 +- 复用 `BatchState`、`BatchOperationError`,不把 String 结果伪装成记录字典。 +- SET 成功不能推断是新增还是覆盖,不编造新增数、修改数。 +- 取消仍传播原取消对象,并附带已知部分结果。 + +Redis Hash 保持现有记录格式、显式 codec 和 TTL 约定。Hash 更新未传 TTL 时保留原 TTL,与 String 普通 SET 的默认行为分别说明。 + +### 2.4 原生能力、异常与资源 + +基础封装范围内的操作不要求调用方直接使用原生驱动。 + +以下能力继续通过 `connection.native` 使用: + +| 数据库 | 原生高级能力示例 | | --- | --- | -| 单机 | 复用 connection.native,CRUD 层不关闭它或外部连接池 | -| Sentinel | connection.native 仍为 Sentinel;记录上下文通过已配置 service_name 创建并复用自有主节点数据客户端,退出时关闭其自有资源 | -| Cluster | 使用原生集群客户端路由单键命令和脚本,批量跨键分发遵循驱动能力 | +| MySQL | 表结构变更、复杂索引、外键、视图、触发器、事件、存储过程、JOIN、复杂 SQL、事务 | +| MongoDB | 通用索引管理、聚合、嵌套查询、复杂更新操作符、验证器、时序集合、分片、事务、Change Stream | +| Redis | List、Set、Sorted Set、Stream、Pub/Sub、锁、专用原子命令、自定义 Lua、事务、扫描与整库操作 | -Redis 记录对象使用 with/async with 管理其自有临时资源;进入上下文可构造临时客户端,但不额外探测服务。 -所有后端的记录对象均支持同样的上下文用法,MySQL/MongoDB 退出记录上下文时仅清理其自有操作资源。 -第一版约定记录对象在有效上下文内执行操作,退出后拒绝继续使用,调用方可基于仍然打开的连接重新绑定对象。 -记录对象借用的主连接应覆盖整个记录上下文生命周期。 +新增结构管理层沿用现有异常上下文,记录操作名和目标;`SchemaConflictError` 用于 SDK 检测到的结构不兼容。驱动错误保留原对象、因果链和已有分类规则。 -跨槽 Pipeline 可以分发多个独立单键操作,但不能提供跨槽整体事务、统一回滚或多键原子性。 -多键事务和脚本受 Redis Cluster 同槽约束,namespace 不会自动强制所有记录落在同一槽。[Redis Cluster](https://redis.io/docs/latest/management/scaling/) -Sentinel 故障切换和 Cluster 重定向遵循驱动策略;对“命令是否已生效”的不确定性按批量结果契约报告。 -原生驱动配置的重试行为需纳入验证,SDK 不额外叠加重试循环。 +原生调用在 `connection.errors(...)` 外保持原生异常行为;调用方可以显式选择转换范围。 -### 8.4 查询、事务及精度 +资源管理继续遵循: -普通 Hash 记录接口提供按键能力;任意字段搜索、范围查询、排序、聚合和跨记录条件更新,需要额外索引与查询能力。 -本期不自动维护索引集合或接入 Search,也不通过 KEYS/SCAN 将整个库拉回本地筛选。 -以后若支持索引查询,应单独说明部署要求、索引一致性、写入成本和失败恢复。 +- 连接由连接句柄关闭。 +- 记录和 String 上下文不关闭借用的连接。 +- Sentinel 操作上下文创建的数据客户端及其自有资源由该上下文清理。 +- 外部连接池不被误关闭。 +- 清理错误不覆盖正在传播的业务异常。 +- 异步对象在创建它的事件循环内使用和关闭。 -Redis Pipeline 优化往返开销,Redis MULTI/EXEC 具有自己的执行语义,但执行期错误不提供 SQL 式回滚。[Redis Pipeline](https://redis.io/docs/latest/develop/using-commands/pipelining/)、[Redis 事务](https://redis.io/docs/latest/develop/using-commands/transactions/) -跨 MySQL/MongoDB/Redis 的原子提交需要应用层协议,本期 CRUD 不能保证三个数据库一起成功或一起撤销。 +本 SDK 不提供跨数据库事务、自动同步、双写补偿或资金账户模型。金额精度由字段类型和 codec 决定;Redis 默认 JSON codec 不承担 `Decimal` 的精确编码,不能把浮点操作或批量成功当作账务一致性保证。 -如涉及金额,业务可选择明确币种与单位的整数最小单位,或配置支持 Decimal 的精确 codec。 -SDK 不默认把 Decimal 转成 float,也不自动将金额更新映射为 Redis INCRBYFLOAT。 -Redis INCRBYFLOAT 使用浮点运算,数值存储精度、算术原子性与账务事务必须分别评估。[Redis INCRBYFLOAT](https://redis.io/docs/latest/commands/incrbyfloat/) -本期 set_values 表达字段赋值,不自动实现“读取余额后加减再写回”的原子账务操作。 -持久化、复制确认、故障切换期间的数据保证由实际数据库部署及配置决定,CRUD 执行确认不充当业务持久化或账务一致性保证。 +## 三、从初始化到 CRUD 的同步、异步示例 -## 9. 同步与异步契约 +以下是**拟实现 API 的验收示例**,不是对当前版本可直接运行的声明。先复用公共配置与表定义,再选择运行同步或异步示例。 -| 项目 | 同步 | 异步 | -| --- | --- | --- | -| 创建连接 | connections.create(name) | await connections.create_async(name) | -| 绑定记录对象 | Records.mysql/mongodb/redis_hash | AsyncRecords.mysql/mongodb/redis_hash,绑定本身为普通方法 | -| 记录上下文 | with | async with | -| 单条、查询、批量方法 | 普通调用 | await,对应参数与结果含义相同 | -| 多数据库同时使用 | ExitStack 管理多个命名连接与记录对象 | AsyncExitStack 管理多个异步连接与记录对象 | -| 原生异常转换范围 | 普通 with connection.errors(...) | 同样使用普通 with,里面可以 await | +示例假设三个数据库服务和账号已经准备好。MySQL 初始化连接具有建库建表权限;生产项目可以为初始化与业务操作配置不同账号。 + +### 3.1 公共配置和表定义 + +```python +from database import ConnectionConfig +from database import Connections +from database.schema import Column +from database.schema import TableSpec + + +USER_TABLE = TableSpec( + columns={ + "id": Column("varchar", length=32), + "name": Column("varchar", length=100), + "age": Column("int"), + }, + primary_key=("id",), +) -异步实现调用原生异步驱动,不使用同步数据库调用阻塞事件循环。 -异步记录对象在所属连接的事件循环内使用,所有高层 CRUD 操作都检查连接状态与循环归属。 -批量执行默认顺序提交分块,异步不意味着把同一 MySQL 会话上的操作自动并发执行。 -同步与异步接受相同的输入范围,具有相同的字段、计数、错误状态和 TTL 契约。 -同步线程安全和异步并发能力仍遵循各原生客户端约束。 -取消信号保持原始 CancelledError,不转换为普通数据库失败。 -若取消时批量操作可能已发送,给原取消对象附加可获取的 `database_partial_result`,保留已知结果与未知范围。 -清理错误沿用 `database_cleanup_errors`,不得覆盖主异常或取消。 -取消或断开不意味着数据库回滚,不通过自动重试推断本次写入是否成功。 +def make_connections( + mysql_password: str, + mongodb_uri: str, + redis_password: str | None = None, +) -> Connections: + return Connections({ + # 初始化时不能选择一个尚未创建的数据库。 + "mysql_setup": ConnectionConfig( + backend="mysql", + host="127.0.0.1", + username="sdk_demo", + password=mysql_password, + options={"autocommit": True}, + ), + "mysql_app": ConnectionConfig( + backend="mysql", + host="127.0.0.1", + username="sdk_demo", + password=mysql_password, + options={ + "database": "demo_app", + "autocommit": True, + }, + ), + "documents": ConnectionConfig( + backend="mongodb", + host=mongodb_uri, + ), + "cache": ConnectionConfig( + backend="redis", + host="127.0.0.1", + password=redis_password, + options={"db": 0, "decode_responses": True}, + ), + }) +``` -## 10. 拟议用法 +密码和 URI 由调用方传入。调用方可以自行读取环境变量或密钥服务,SDK 不参与配置来源选择。 -以下示例演示审查中的 API 形状,`connections` 是调用方用显式参数创建的现有 Connections 集合。 -示例中的连接名分别绑定业务用途,不形成按数据库类型划分的全局对象。 +增加第二个 MySQL 业务库或第二个 Redis 实例时,增加一个命名配置即可;不会覆盖现有连接或形成按后端共享的单例。 + +### 3.2 同步完整流程 ```python +from contextlib import ExitStack + +from database.crud import JsonFieldCodec from database.crud import Records +from database.crud import RedisStrings +from database.crud import StringSetItem +from database.schema import Schema + + +def run_sync(connections: Connections) -> None: + # 一、显式准备 MySQL 数据库与表。 + with connections.create("mysql_setup") as setup: + schema = Schema.mysql(setup) + schema.ensure_database("demo_app") + schema.ensure_table("demo_app", "users", USER_TABLE) + + # 二、三个数据库连接在同一业务作用域内同时使用。 + with ExitStack() as stack: + mysql = stack.enter_context(connections.create("mysql_app")) + mongo = stack.enter_context(connections.create("documents")) + redis = stack.enter_context(connections.create("cache")) + + for connection in (mysql, mongo, redis): + connection.check() + + # 三、显式准备 MongoDB 普通集合。 + Schema.mongodb(mongo).ensure_collection( + "demo_app", + "users", + ) + + mysql_users = stack.enter_context( + Records.mysql( + mysql, + table="users", + key_fields=USER_TABLE.primary_key, + ) + ) + mongo_users = stack.enter_context( + Records.mongodb( + mongo, + database="demo_app", + collection="users", + ) + ) -with connections.create("orders") as connection: - with Records.mysql(connection, table="orders", key_fields=("id",)) as orders: - rows = orders.find( - where={"status": "pending"}, - fields=["id", "status"], - limit=100, - order_by=[("id", "asc")], + # Redis 不建表;绑定记录命名空间及 codec 即可开始使用。 + redis_users = stack.enter_context( + Records.redis_hash( + redis, + namespace="demo_app:users", + key_fields=("id",), + codec=JsonFieldCodec(), + ) ) - result = orders.bulk_update([ - {"key": {"id": 1001}, "set_values": {"status": "paid"}}, - {"key": {"id": 1002}, "set_values": {"status": "cancelled"}}, + + # 四、分别对三个后端执行相同的基础记录流程。 + for users, key_field in ( + (mysql_users, "id"), + (mongo_users, "_id"), + (redis_users, "id"), + ): + key = {key_field: "u1"} + + users.insert({**key, "name": "Alice", "age": 30}) + print(users.get(key, fields=["name", "age"])) + users.update(key, set_values={"age": 31}) + + users.insert_many([ + {key_field: "u2", "name": "Bob", "age": 20}, + {key_field: "u3", "name": "Carol", "age": 40}, + ]) + users.bulk_update([ + { + "key": {key_field: "u2"}, + "set_values": {"age": 21}, + }, + { + "key": {key_field: "u3"}, + "set_values": {"age": 41}, + }, + ]) + print(users.get_many( + [{key_field: "u2"}, {key_field: "u3"}], + fields=["name", "age"], + )) + + users.delete(key) + users.delete_many([ + {key_field: "u2"}, + {key_field: "u3"}, + ]) + + # 五、Redis String 键值操作。 + strings = stack.enter_context(RedisStrings(redis)) + + strings.set("demo:message", "hello", ttl_s=60) + print(strings.get("demo:message")) + strings.set("demo:message", "updated", ttl_s=120) + print(strings.ttl("demo:message")) + strings.delete("demo:message") + + strings.set_many([ + StringSetItem("demo:a", "A", ttl_s=60), + StringSetItem("demo:b", "B", ttl_s=60), ]) - # 应用在自己的事务边界决定提交或回滚。 - with connection.errors(operation="commit_orders"): - connection.native.commit() + print(strings.get_many(["demo:a", "demo:b", "demo:missing"])) + strings.delete_many(["demo:a", "demo:b"]) + + # 上下文退出:先释放操作对象自有资源,再关闭连接。 + # 不自动删除数据库、表或集合。 ``` +此示例对三个后端分别写入演示数据,不表示 SDK 自动复制数据,也不形成一个跨数据库事务。示例中的固定键用于独立演示目标,实际业务应使用自己的身份值。 + +### 3.3 异步完整流程 + +异步流程复用上面的配置和表定义;所有数据库操作显式 `await`。 + ```python +from contextlib import AsyncExitStack + from database.crud import AsyncRecords +from database.crud import AsyncRedisStrings +from database.crud import JsonFieldCodec +from database.crud import StringSetItem +from database.schema import AsyncSchema + -async def read_and_update_orders(connections): - async with await connections.create_async("orders") as connection: - async with AsyncRecords.mysql( - connection, table="orders", key_fields=("id",) - ) as orders: - items = await orders.get_many( - [{"id": 1001}, {"id": 1002}], fields=["status"] +async def run_async(connections: Connections) -> None: + # 一、异步准备 MySQL 数据库与表。 + async with await connections.create_async("mysql_setup") as setup: + schema = AsyncSchema.mysql(setup) + await schema.ensure_database("demo_app") + await schema.ensure_table("demo_app", "users", USER_TABLE) + + # 二、三个异步连接同时存在,分别管理各自资源。 + async with AsyncExitStack() as stack: + mysql = await stack.enter_async_context( + await connections.create_async("mysql_app") + ) + mongo = await stack.enter_async_context( + await connections.create_async("documents") + ) + redis = await stack.enter_async_context( + await connections.create_async("cache") + ) + + for connection in (mysql, mongo, redis): + await connection.check() + + # 三、异步准备 MongoDB 集合。 + await AsyncSchema.mongodb(mongo).ensure_collection( + "demo_app", + "users", + ) + + mysql_users = await stack.enter_async_context( + AsyncRecords.mysql( + mysql, + table="users", + key_fields=USER_TABLE.primary_key, ) - result = await orders.bulk_update([ - {"key": {"id": 1001}, "set_values": {"status": "paid"}}, - {"key": {"id": 1002}, "set_values": {"status": "cancelled"}}, - ]) - with connection.errors(operation="commit_orders"): - await connection.native.commit() - return items, result -``` + ) + mongo_users = await stack.enter_async_context( + AsyncRecords.mongodb( + mongo, + database="demo_app", + collection="users", + ) + ) + redis_users = await stack.enter_async_context( + AsyncRecords.redis_hash( + redis, + namespace="demo_app:users", + key_fields=("id",), + codec=JsonFieldCodec(), + ) + ) -MongoDB 绑定后同样支持单条/批量/条件接口,原生 session 由调用方传入并管理。 + # 四、三个后端分别执行异步单条与批量 CRUD。 + for users, key_field in ( + (mysql_users, "id"), + (mongo_users, "_id"), + (redis_users, "id"), + ): + key = {key_field: "u1"} + + await users.insert({**key, "name": "Alice", "age": 30}) + print(await users.get(key, fields=["name", "age"])) + await users.update(key, set_values={"age": 31}) + + await users.insert_many([ + {key_field: "u2", "name": "Bob", "age": 20}, + {key_field: "u3", "name": "Carol", "age": 40}, + ]) + await users.bulk_update([ + { + "key": {key_field: "u2"}, + "set_values": {"age": 21}, + }, + { + "key": {key_field: "u3"}, + "set_values": {"age": 41}, + }, + ]) + print(await users.get_many( + [{key_field: "u2"}, {key_field: "u3"}], + fields=["name", "age"], + )) + + await users.delete(key) + await users.delete_many([ + {key_field: "u2"}, + {key_field: "u3"}, + ]) -```python -with connections.create("documents") as connection: - with Records.mongodb( - connection, database="app", collection="orders", key_fields=("_id",) - ) as documents: - result = documents.update_many( - where={"status": "pending"}, set_values={"status": "closed"} + # 五、Redis String 异步操作。 + strings = await stack.enter_async_context( + AsyncRedisStrings(redis) ) - with connection.errors(operation="aggregate_orders"): - # 示例展示原生能力仍可达,完整迭代和关闭位于异常转换范围内。 - with connection.native["app"]["orders"].aggregate([ - {"$group": {"_id": "$status", "count": {"$sum": 1}}} - ]) as cursor: - summary = list(cursor) -``` -```python -from database.crud import JsonFieldCodec + await strings.set("demo:message", "hello", ttl_s=60) + print(await strings.get("demo:message")) + await strings.set("demo:message", "updated", ttl_s=120) + print(await strings.ttl("demo:message")) + await strings.delete("demo:message") -with connections.create("cache") as connection: - with Records.redis_hash( - connection, namespace="order", key_fields=("id",), codec=JsonFieldCodec() - ) as cache: - created = cache.insert({"id": 1001, "status": "pending"}, ttl_s=60) - item = cache.get({"id": 1001}, fields=["status"]) - updated = cache.update({"id": 1001}, set_values={"status": "paid"}) - # 该次更新省略 ttl_s,保留已有过期时间。 + await strings.set_many([ + StringSetItem("demo:a", "A", ttl_s=60), + StringSetItem("demo:b", "B", ttl_s=60), + ]) + print(await strings.get_many([ + "demo:a", + "demo:b", + "demo:missing", + ])) + await strings.delete_many(["demo:a", "demo:b"]) ``` -MongoDB 和 Redis 异步用法分别采用 AsyncRecords.mongodb/redis_hash、async with 和 await,返回结构与同步一致。 -原生异步 MongoDB/Redis 工厂是否 await 的既有差异,继续由连接层按现有契约处理。 -高级 CRUD 的异步方法始终通过 await 调用。 +结构查看与删除也采用对应同步方法或异步 `await` 方法。删除库表属于单独的显式管理操作,不放进普通 CRUD 示例的自动退出逻辑。 + +## 四、开发组织与实施顺序 + +以下为后续实施流程。本轮仅生成本文档,不启动 worker 实现、不修改产品代码、不执行提交或合并。 + +### 4.1 集成策略 + +继续使用 `feature/database-crud` 作为本功能集成主线,保留已经完成的记录 CRUD 历史,不重新从 `develop` 开始实现。 + +后续开发遵循: + +- 不操作 `develop`,最终由用户审核后决定合入。 +- 只进行本地 Git 操作,不检查或同步远端。 +- 不改写既有提交,不混入 `temp/save` 或无关工作区文件。 +- 实现分支使用 `feature/`,缺陷修复使用 `fix/`,不使用 `docs/` 或 Codex 前缀承载代码实现。 +- 独立 worker 使用独立 worktree 与暂存区。 +- 集成采用串行 `--no-ff` 合并,中文 Conventional Commits。 +- 工作树和分支保留至用户审查,不擅自清理已有工作树或草稿。 + +### 4.2 实施阶段 + +| 阶段 | 工作内容 | 完成条件 | +| --- | --- | --- | +| A:设计与公共契约 | 审核本文;固定 Schema、表定义、String 结果类型和异常接口,建立公共契约提交检查点 | 公共接口无待定项,下游使用同一已提交检查点 | +| B:三个模块并行 | MySQL 结构管理、MongoDB 集合管理、Redis String 操作分别实现 | 各模块同步异步定向测试通过,提交模块交付记录 | +| C:串行集成 | 按 MySQL、MongoDB、Redis 顺序合入;协调者维护公共导出及共享结果类型 | 每次合并复验相关功能,公共接口一致 | +| D:完整验收 | 从空结构执行示例,验证多数据库、多实例及故障场景 | 实际结果与文档一致,未验证事项明确 | +| E:审查交付 | 更新使用文档、README、CHANGELOG 和验收记录 | 提供差异、提交关系、测试证据,交由用户审查 | + +并行实现分支: + +- `feature/database-crud-mysql-schema` +- `feature/database-crud-mongodb-schema` +- `feature/database-crud-redis-strings` + +三个 worker 从阶段 A 的同一检查点创建分支,分别拥有自己的实现和测试文件。公共文件由协调者负责,避免并行修改同一文件。最多同时运行三个实现 worker,协调者负责总集成,不让不同工作者交叉操作同一工作树。 + +工作者交付模块范围、提交 SHA、测试结果和 Git 操作记录。协调者在合并前核对分支仍指向已审查 SHA,检查差异范围,再串行合并并记录集成后的 HEAD。不使用 squash、强制改写历史或把整个功能分支压为一次 cherry-pick。 -## 11. 实施阶段与验收计划 +本轮不新增 ORM、自动迁移系统、后端插件注册框架、SQL 优化器、跨库事务协调器或新的运行时依赖。 -功能开发主线现为 `feature/database-crud`;前期使用的 `docs/database-crud-design` 已按用户的命名要求更名。各实现分支从主线的已提交检查点建立,代码任务不再使用 `docs/*` 前缀。 -按 [协作规范](../CONTRIBUTING.md) 安排公共契约、MySQL、MongoDB、Redis 和验收工作。 -公共结果与条件语义先形成检查点,后端模块再从同一检查点开展并行实现,最后串行合并。 -分支命名使用 feature/fix/refactor/docs 等项目规范前缀。 -是否提交、合并和推送遵循实施时用户指令;目前 Git 操作限定本地。 +## 五、测试、文档与交付标准 -| 阶段 | 交付与通过条件 | +### 5.1 自动化测试 + +沿用 `unittest`,异步使用 `IsolatedAsyncioTestCase`。 + +重点覆盖: + +| 范围 | 必须验证的场景 | | --- | --- | -| 契约 | 资源绑定、方法签名、fields/空值/键/结果状态均有明确测试场景 | -| MySQL | 单条与批量、参数绑定、字段引用、复合键、结果消费、原生事务和游标关闭 | -| MongoDB | 原生键类型、投影、空值与缺失、集合更新、批量错误索引、session、未确认写入 | -| Redis | 明确 Hash 格式与 codec、原子存在性、TTL、部分 Pipeline 失败、三拓扑资源管理 | -| 集成 | 同进程三种数据库及同类多实例并用,同步/异步契约一致,原生访问与原工厂回归 | - -必须覆盖以下验收场景。 - -- 相同条件更新多条记录,每条记录不同值的批量更新,批量插入和删除,复合键及重复输入。 -- fields=None、空列表、部分字段、身份字段、缺失字段、NULL/null、未知 SQL 列,原生类型保真。 -- get_many 输入顺序、重复键、缺失记录、后端乱序、跨分块部分读取失败。 -- MongoDB 相同值更新 matched_count 与 modified_count 不同,MySQL rowcount 模式差异,Redis 新增字段数不得误报为修改记录数。 -- 批量前置校验失败时无请求,服务端约束失败、未知执行位置、未发送项、发送后断线、取消后的不确定状态。 -- MySQL 回滚、显式提交和自动提交配置,MongoDB session 事务行为,Redis 原生事务及 Cluster 限制。 -- Redis create-only/update-only 的并发竞争、Hash 类型冲突、无标记旧数据、全字段缺失、codec 错误、TTL 保留/重设/清除和过期竞态。 -- Sentinel 主节点数据路径、临时客户端清理,Cluster 跨槽独立批量和不支持的多键原子要求。 -- 同步与异步逐项对照,事件循环归属,异常原对象、因果链、取消、主异常与清理失败。 -- 多个连接独立关闭,原生客户端对象身份、方法、事务和返回值继续可用。 - -离线测试沿用 unittest 和 IsolatedAsyncioTestCase;对应模块执行 Ruff lint 与格式检查,公共契约改动执行完整回归。 -真实实例验证使用隔离 MySQL、MongoDB、Redis 单机/Sentinel/Cluster,记录客户端与服务端版本,并由外部测试进程控制故障用例时限。 -覆盖项目最低驱动版本和选定当前版本,MongoDB 异步要求 PyMongo >=4.9。 -当前基线的默认安装已包含三种驱动;CRUD 提案不以旧版 extras 安装方案为前提。 -之前的连接管理测试结果不代表这些新 CRUD 行为已验证。 -实现后再更新 README、测试指南和 CHANGELOG 未发布部分,按实际完成范围描述支持能力。 - -## 12. 审查重点与本次文档交付 - -重点审查按键操作与条件操作的边界、Redis Hash 格式及保留标记、TTL 省略/None 语义、fields 的空值规则、批量确认和事务提交的区别。 -同样需要确认 MySQL/MongoDB 公共条件的支持范围,以及第一版写入不包含原子投影回读的范围选择。 -以上均是可调整的设计决策,审核通过后才进入实现。 - -本次由协调者负责两份文档,文档本身和 docs 索引。 -Git 操作在本地依次完成,主目录为 `D:\996-work\loongspears\database`,文档 worktree 为 `D:\996-work\loongspears\database-worktrees\crud-design`。 - -| 顺序 | 执行者与位置 | 命令或操作 | 结果 | -| --- | --- | --- | --- | -| 1 | 协调者,主目录 develop | git status --short --branch;git diff --cached --stat;git diff --stat | 开始时无待提交改动 | -| 2 | 协调者,主目录 | git worktree list --porcelain;git rev-parse HEAD develop | 核对既有 15 个 worktree,基线为 b6707125620d1cea28cabbc7a72e72c84ae73bbd | -| 3 | 协调者,主目录 | git worktree add -b docs/database-crud-design D:\996-work\loongspears\database-worktrees\crud-design b6707125620d1cea28cabbc7a72e72c84ae73bbd | 创建独立文档分支与第 16 个 worktree | -| 4 | 协调者,文档 worktree | 写入 docs/crud-design.md 并补充 docs/README.md 索引 | 交付工作区文档差异,待审查 | -| 5 | 协调者,文档 worktree | python -B -(内联文档检查);git diff --check;git status --short | UTF-8、相对链接、代码围栏、Python 示例语法及空白检查通过 | - -文档验证已检查 UTF-8、相对链接目标、代码块语法和 Git 空白差异;检查脚本对新增未跟踪文档也进行了逐行空白核对。 -Python 示例只做语法核对,因为拟议模块尚未实现;真实驱动行为及新接口测试均留在实施验收阶段。 -上述记录为 2026-11-15 文档编写阶段。 -2026-11-16 已将设计与索引提交为 `80fd877360787309bf4a679bd94b1baa76c733ea`,并将主目录切换到 `docs/database-crud-design`。 -随后将开发主线更名为 `feature/database-crud`;以上旧分支名称及命令保留为历史记录。 -实施记录及验收状态由后续交付补充。 +| 初始化 | 空目标创建成功;重复执行不破坏数据;并发相同初始化不虚报成功 | +| 结构查看与删除 | 同步异步 list、exists、describe、drop;仅作用于指定目标;重复及不存在删除;权限错误 | +| MySQL 结构 | 必要字段和主键匹配;允许可省略的额外字段;结构不兼容时不改表 | +| MySQL 事务 | 结构写操作拒绝活动事务;不暗中提交、回滚、切库或修改 autocommit | +| MongoDB 集合 | 首个集合完成初始化;同名非普通集合冲突;自定义唯一键生效 | +| MongoDB 失败 | 索引建立失败保留集合和原异常;不把构建中索引当成就绪 | +| 元数据权限 | 权限异常不被转换为“目标不存在” | +| Redis String | 空值与缺失区分、bytes/str、TTL、错误类型、编码后重复键 | +| 批量结果 | 逐项成功、部分失败、发送状态未知、取消及未派发项 | +| 字段投影 | 三种记录后端只返回指定字段;空投影、缺失字段、原生值类型 | +| 多连接 | 三种数据库同时使用;两个同类实例相互隔离;关闭一个不影响另一个 | +| 生命周期 | 游标、流水线和 Sentinel 自有客户端清理;外部池保护 | +| 兼容性 | 原生工厂、已有连接接口和记录 CRUD 测试继续通过 | + +对 Python 改动执行定向测试、Ruff lint 和格式检查;涉及公共接口与共享结果类型后执行完整离线回归,并运行 `git diff --check`。 + +```bash +python -B -m unittest discover -s tests -q +python -m ruff check <本次改动的Python文件> +python -m ruff format --check <本次改动的Python文件> +git diff --check +``` + +执行前核对实际导入的是当前工作树源码。文档中分别记录离线测试、驱动兼容性测试和真实服务验收,不混合表述。 + +### 5.2 真实服务验收 + +使用显式测试配置和隔离实例,不读取生产配置。 + +验收范围包括: + +- MySQL 8.4:同步、异步从建库建表到完整 CRUD。 +- MongoDB:同步、异步集合准备、必要唯一约束及完整 CRUD。 +- Redis 单机、Sentinel、Cluster:同步、异步 Hash 与 String 操作。 +- Redis Cluster 批量包含不同槽的键。 +- 同一进程同时打开三个后端,并创建两个同类连接。 +- 原生高级操作与基础封装可以在同一连接上按驱动规则使用。 +- 认证失败、权限不足、重复键、结构冲突、网络中断及清理失败保留诊断信息。 + +驱动范围保持现有约定:Python 3.10+、PyMongo 4.6+,MongoDB 异步要求 4.9+;MySQL Connector/Python 8.3+;redis-py 5.0+。验证最低支持版本与选定当前版本,记录实际环境。 + +现有真实 Redis 拓扑验证缺口必须单独补齐。若隔离服务暂不可用,明确标为未完成验收,不能用替身测试宣称真实拓扑已经验证。 + +### 5.3 文档与最终验收标准 + +设计审查通过、进入实施后: + +- 在本文中更新实际接口和验收状态,不把计划能力写成已实现。 +- 使用文档补齐初始化、结构管理、同步异步、多数据库示例。 +- 保留此前记录 CRUD 阶段的历史验收证据。 +- README 展示最短完整使用流程。 +- CHANGELOG 的“未发布”部分登记新增能力。 + +**功能完成标准:开发者面对已经部署好的三个数据库服务,可以仅通过本 SDK 完成基础存储准备、单条与批量读写、字段投影以及资源关闭;高级功能可以继续直接使用原生客户端。** + +最终交付提供集成分支、基线与最终提交、worker 合并关系、按执行顺序记录的 Git 操作、测试结果和未验证事项,停留在功能分支供用户审查。 + +Git 操作记录注明执行者、分支、工作树、命令和结果;合并记录写明被合入 SHA 与集成后的 HEAD。核对 `develop` 未改变,未混入无关文件,未访问或同步远端。本轮仅生成文档,不产生上述实现与验收交付记录。 \ No newline at end of file diff --git a/docs/crud-implementation.md b/docs/crud-implementation.md index 27cf582..e709353 100644 --- a/docs/crud-implementation.md +++ b/docs/crud-implementation.md @@ -9,18 +9,18 @@ ## 1. 模块和工作树 -协调者在 `D:\996-work\loongspears\database` 集成。 -各 worker 在 `D:\996-work\loongspears\database-worktrees\` 下独立开发,使用独立分支、目录和暂存区。 +协调者在仓库根目录 `.` 集成。 +各 worker 在相对于仓库根目录的 `../database-worktrees/` 下独立开发,使用独立分支、目录和暂存区。 旧工作树及用户其他分支全部保留。 -| 模块 | 分支 | 工作树目录 | 交付提交 | +| 模块 | 分支 | 工作树目录(相对仓库根目录) | 交付提交 | | --- | --- | --- | --- | -| 公共协议、记录层 | `feature/database-crud-core` | `crud-core` | `0ba757a925247f632d32b9d236457fad5e8fbe00` | -| MySQL | `feature/database-crud-mysql` | `crud-mysql` | `1e7f52ee5f4646498cca57632a99bb2ca2a3704b` | -| Redis Hash / codec | `feature/database-crud-redis` | `crud-redis` | `32c990422a3fdd972b5e8d089b13932dc7f33d29` | -| MongoDB | `feature/database-crud-mongodb` | `crud-mongodb` | `c0e4246b54274d3222ed75ab57141e492da6131f` | -| MongoDB 取消修复 | `fix/database-mongodb-cancellation` | `crud-mongodb-cancellation` | `a6d778f2a919695ae77012e2426e8c26ce57ea0d` | -| 分块派发与取消链 | `fix/database-crud-batch-evidence` | `crud-batch-evidence` | `a64d43861366726176a70cf242c54825d1712049` | +| 公共协议、记录层 | `feature/database-crud-core` | `../database-worktrees/crud-core` | `0ba757a925247f632d32b9d236457fad5e8fbe00` | +| MySQL | `feature/database-crud-mysql` | `../database-worktrees/crud-mysql` | `1e7f52ee5f4646498cca57632a99bb2ca2a3704b` | +| Redis Hash / codec | `feature/database-crud-redis` | `../database-worktrees/crud-redis` | `32c990422a3fdd972b5e8d089b13932dc7f33d29` | +| MongoDB | `feature/database-crud-mongodb` | `../database-worktrees/crud-mongodb` | `c0e4246b54274d3222ed75ab57141e492da6131f` | +| MongoDB 取消修复 | `fix/database-mongodb-cancellation` | `../database-worktrees/crud-mongodb-cancellation` | `a6d778f2a919695ae77012e2426e8c26ce57ea0d` | +| 分块派发与取消链 | `fix/database-crud-batch-evidence` | `../database-worktrees/crud-batch-evidence` | `a64d43861366726176a70cf242c54825d1712049` | 公共协议检查点先集成,再启动各数据库模块;公共层完成后,各 worker 可以合入明确的集成检查点进行公开 API 验证。 各 worker 不交叉合并彼此的分支,不操作总线暂存区。 @@ -119,11 +119,11 @@ Lua 模拟器验证不等于 Redis 服务器、Sentinel 故障切换或 Cluster 上述 MySQL、MongoDB 各版本及 Redis 模拟器验收均在最终代码验收基点复跑。 Redis 5.0 的临时验收脚本最初使用了仅新版存在的 `aclose()`;改为适配 `close()` 后完整通过,此处属于测试脚本兼容修正,不是 SDK 清理逻辑变更。 -临时 QA 根为 `C:\Users\MSR\AppData\Local\Temp\loongspears-db-connection-qa-9d596ec4bf4d`,以下脚本不纳入运行依赖或仓库产品代码。 +历史 QA 材料保存在仓库外的临时目录,实际位置由运行者指定,本文不保留本机绝对地址。以下仅记录验收脚本的文件名和用途,不表示它们位于仓库根目录,也不纳入运行依赖或仓库产品代码。 -- MySQL:`mysql-crud-runtime/verify_crud.py`、`verify_transactions.py`,参数依次为预期驱动版本、`current` 或隔离驱动目录、QA 根。 -- MongoDB:`mongodb-crud-runtime-4148fab7/verify_mongo_crud.py`,参数为同目录 `fixture.json` 与预期驱动版本;由 `PYTHONPATH` 选当前源码和驱动。 -- BSON 数值:同一目录 `verify_int64_main.py`,同样参数,包含 Int64 与 Decimal128。 +- MySQL:`verify_crud.py`、`verify_transactions.py`,参数依次为预期驱动版本、`current` 或隔离驱动目录、QA 根。 +- MongoDB:`verify_mongo_crud.py`,参数为同目录 `fixture.json` 与预期驱动版本;由 `PYTHONPATH` 选当前源码和驱动。 +- BSON 数值:`verify_int64_main.py`,与 MongoDB 验收脚本位于同一隔离目录,使用同样参数,包含 Int64 与 Decimal128。 - Redis:`crud-redis-smoke.py`,使用显式 `PYTHONPATH` 加载当前源码、选定 redis-py 及独立 `crud-redis-lua` 模拟器依赖。 测试配置仅指向上述隔离服务;凭据不写入本文,也不直接输出配置内容。 -- Gitee From bdc326d41fb43eb893ea9c85166be8ed474242a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:08:23 +0800 Subject: [PATCH 66/86] =?UTF-8?q?feat(crud):=20=E5=BB=BA=E7=AB=8B=E7=BB=93?= =?UTF-8?q?=E6=9E=84=E7=AE=A1=E7=90=86=E4=B8=8E=E9=94=AE=E5=80=BC=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E5=85=AC=E5=85=B1=E5=A5=91=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/__init__.py | 14 +++++ src/database/crud/_errors.py | 3 +- src/database/crud/_string_types.py | 39 +++++++++++++ src/database/schema/__init__.py | 67 ++++++++++++++++++++++ src/database/schema/_base.py | 55 ++++++++++++++++++ tests/test_schema_binding.py | 89 ++++++++++++++++++++++++++++++ 6 files changed, 266 insertions(+), 1 deletion(-) create mode 100644 src/database/crud/_string_types.py create mode 100644 src/database/schema/__init__.py create mode 100644 src/database/schema/_base.py create mode 100644 tests/test_schema_binding.py diff --git a/src/database/crud/__init__.py b/src/database/crud/__init__.py index eb9f14c..ecff037 100644 --- a/src/database/crud/__init__.py +++ b/src/database/crud/__init__.py @@ -20,6 +20,9 @@ from ._errors import RecordIdentityError from ._errors import UnsupportedOperationError from ._records import AsyncRecords from ._records import Records +from ._string_types import StringBatchItem +from ._string_types import StringBatchResult +from ._string_types import StringSetItem from ._types import BatchItem from ._types import BatchResult from ._types import BatchState @@ -31,10 +34,13 @@ from ._types import WriteResult if TYPE_CHECKING: from ._codecs import FieldCodec from ._codecs import JsonFieldCodec + from ._strings import AsyncRedisStrings + from ._strings import RedisStrings __all__ = [ 'And', 'AsyncRecords', + 'AsyncRedisStrings', 'BatchItem', 'BatchOperationError', 'BatchResult', @@ -56,6 +62,10 @@ __all__ = [ 'RecordFormatError', 'RecordIdentityError', 'Records', + 'RedisStrings', + 'StringBatchItem', + 'StringBatchResult', + 'StringSetItem', 'UnsupportedOperationError', 'WriteCounts', 'WriteResult', @@ -67,4 +77,8 @@ def __getattr__(name: str) -> Any: from . import _codecs return getattr(_codecs, name) + if name in ('RedisStrings', 'AsyncRedisStrings'): + from . import _strings + + return getattr(_strings, name) raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/src/database/crud/_errors.py b/src/database/crud/_errors.py index 65358bd..168e380 100644 --- a/src/database/crud/_errors.py +++ b/src/database/crud/_errors.py @@ -3,6 +3,7 @@ from typing import Any from ..errors import DatabaseError +from ._string_types import StringBatchResult from ._types import BatchResult @@ -25,6 +26,6 @@ class RecordIdentityError(DatabaseError): class BatchOperationError(DatabaseError): """A batch operation failed; the confirmed and uncertain results are retained.""" - def __init__(self, message: str, *, partial_result: BatchResult, **context: Any) -> None: + def __init__(self, message: str, *, partial_result: BatchResult | StringBatchResult, **context: Any) -> None: super().__init__(message, **context) self.partial_result = partial_result diff --git a/src/database/crud/_string_types.py b/src/database/crud/_string_types.py new file mode 100644 index 0000000..7975260 --- /dev/null +++ b/src/database/crud/_string_types.py @@ -0,0 +1,39 @@ +"""Redis String inputs and evidence-based batch results.""" + +from dataclasses import dataclass +from typing import Any + +from ._types import BatchState + +StringKey = str | bytes +StringValue = str | bytes + + +@dataclass(frozen=True) +class StringSetItem: + """One SET, with an optional positive expiration in seconds.""" + + key: StringKey + value: StringValue + ttl_s: int | None = None + + +@dataclass(frozen=True) +class StringBatchItem: + """A reply or uncertain execution state at one original input position.""" + + index: int + key: StringKey + state: BatchState + result: str | bytes | bool | int | None = None + error: Any = None + + +@dataclass(frozen=True) +class StringBatchResult: + """Per-key evidence without inferred inserted or modified record counts.""" + + requested_count: int + items: tuple[StringBatchItem, ...] + deleted_count: int | None = None + native_results: tuple[Any, ...] = () diff --git a/src/database/schema/__init__.py b/src/database/schema/__init__.py new file mode 100644 index 0000000..95d8d6d --- /dev/null +++ b/src/database/schema/__init__.py @@ -0,0 +1,67 @@ +"""Explicit basic schema preparation on borrowed managed connections.""" + +from typing import TYPE_CHECKING +from typing import Any + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import DependencyError +from ._base import SchemaConflictError + +if TYPE_CHECKING: + from ._models import Column + from ._models import TableSpec + from ._mongodb import AsyncMongoDBSchema + from ._mongodb import MongoDBSchema + from ._mysql import AsyncMySQLSchema + from ._mysql import MySQLSchema + +__all__ = ['AsyncSchema', 'Column', 'Schema', 'SchemaConflictError', 'TableSpec'] + + +def _factory(connection: Connection, backend: str, class_name: str) -> Any: + from importlib import import_module + + try: + module = import_module(f'{__name__}._{backend}') + except ImportError as original: + raise DependencyError( + 'The selected schema adapter or database driver is unavailable.', + connection_name=getattr(connection, 'name', None), + backend=backend, + phase='config', + original=original, + ) from original + return getattr(module, class_name)(connection) + + +class Schema: + """Bind synchronous schema operations; binding performs no network I/O.""" + + @staticmethod + def mysql(connection: Connection) -> 'MySQLSchema': + return _factory(connection, 'mysql', 'MySQLSchema') + + @staticmethod + def mongodb(connection: Connection) -> 'MongoDBSchema': + return _factory(connection, 'mongodb', 'MongoDBSchema') + + +class AsyncSchema: + """Bind asynchronous operations; await methods, not this factory.""" + + @staticmethod + def mysql(connection: AsyncConnection) -> 'AsyncMySQLSchema': + return _factory(connection, 'mysql', 'AsyncMySQLSchema') + + @staticmethod + def mongodb(connection: AsyncConnection) -> 'AsyncMongoDBSchema': + return _factory(connection, 'mongodb', 'AsyncMongoDBSchema') + + +def __getattr__(name: str) -> Any: + if name in ('Column', 'TableSpec'): + from . import _models + + return getattr(_models, name) + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/src/database/schema/_base.py b/src/database/schema/_base.py new file mode 100644 index 0000000..dbbbb8c --- /dev/null +++ b/src/database/schema/_base.py @@ -0,0 +1,55 @@ +"""Driver-independent binding and error context for explicit schema operations.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DatabaseError +from ..errors import error_context + + +class SchemaConflictError(DatabaseError): + """An existing object does not satisfy the explicitly requested structure.""" + + def __init__(self, message: str, *, differences: tuple[str, ...] = (), **context: Any) -> None: + super().__init__(message, **context) + self.differences = tuple(differences) + + +class BoundSchema: + """Borrow a connection without owning its lifecycle or starting network I/O.""" + + def __init__(self, connection: Connection, *, backend: str, is_async: bool) -> None: + if ( + not isinstance(connection, Connection) + or isinstance(connection, AsyncConnection) != is_async + or connection.backend != backend + ): + raise ConfigurationError('Schema binding requires the matching backend and sync mode.', phase='config') + connection._ensure_open('operation') + self._connection = connection + + def _context_fields(self, operation: str, target: str | None = None) -> dict[str, Any]: + return dict( + connection_name=self._connection.name, + backend=self._connection.backend, + phase='operation', + operation=operation, + target=target, + ) + + @contextmanager + def _operation(self, operation: str, target: str | None = None) -> Iterator[None]: + connection = self._connection + if isinstance(connection, AsyncConnection): + connection._ensure_loop('operation') + connection._ensure_open('operation') + with error_context( + **self._context_fields(operation, target), + classifier=connection._adapter.classify, + owned=True, + ): + yield diff --git a/tests/test_schema_binding.py b/tests/test_schema_binding.py new file mode 100644 index 0000000..1dfd1ce --- /dev/null +++ b/tests/test_schema_binding.py @@ -0,0 +1,89 @@ +"""Lifecycle and exception contracts independent of optional database drivers.""" + +import asyncio +import subprocess +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +from database.connections import AsyncConnection +from database.connections import Connection +from database.errors import ConfigurationError +from database.errors import ConnectionClosedError +from database.errors import DriverError +from database.schema import SchemaConflictError +from database.schema._base import BoundSchema + + +class NativeFailure(Exception): + pass + + +def adapter(): + return SimpleNamespace(classify=lambda exc: DriverError if isinstance(exc, NativeFailure) else None) + + +class SchemaBindingTests(unittest.TestCase): + def test_public_import_does_not_import_drivers(self): + source = str(Path(__file__).resolve().parents[1] / 'src') + code = ( + f'import sys; sys.path.insert(0, {source!r}); ' + 'import database.schema; from database.crud import StringSetItem; ' + 'assert not any(k == n or k.startswith(n + ".") ' + 'for n in ("pymongo", "mysql.connector", "redis") for k in sys.modules)' + ) + subprocess.run([sys.executable, '-B', '-S', '-c', code], check=True, capture_output=True) + + def test_bind_checks_backend_and_closed_handle(self): + conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') + with self.assertRaises(ConfigurationError): + BoundSchema(conn, backend='mongodb', is_async=False) + conn._closed = True + with self.assertRaises(ConnectionClosedError): + BoundSchema(conn, backend='mysql', is_async=False) + + def test_exception_preserves_original_and_target(self): + conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') + schema = BoundSchema(conn, backend='mysql', is_async=False) + original = NativeFailure('driver detail') + with self.assertRaises(DriverError) as raised: + with schema._operation('ensure_table', 'app.users'): + raise original + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.assertEqual(raised.exception.target, 'app.users') + self.assertEqual(raised.exception.connection_name, 'orders') + self.assertEqual(raised.exception.operation, 'ensure_table') + + def test_business_and_unified_errors_keep_identity(self): + conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') + schema = BoundSchema(conn, backend='mysql', is_async=False) + for original in (ValueError('business'), SchemaConflictError('mismatch', differences=('primary_key',))): + with self.assertRaises(type(original)) as raised: + with schema._operation('ensure_table'): + raise original + self.assertIs(raised.exception, original) + + +class AsyncSchemaBindingTests(unittest.IsolatedAsyncioTestCase): + async def test_sync_mode_and_loop_are_enforced(self): + conn = AsyncConnection(object(), adapter(), 'documents', 'mongodb', 'single', asyncio.get_running_loop()) + with self.assertRaises(ConfigurationError): + BoundSchema(conn, backend='mongodb', is_async=False) + schema = BoundSchema(conn, backend='mongodb', is_async=True) + with schema._operation('list_collections'): + pass + conn._loop = object() + with self.assertRaises(ConfigurationError): + with schema._operation('list_collections'): + pass + + async def test_cancellation_is_not_converted(self): + conn = AsyncConnection(object(), adapter(), 'documents', 'mongodb', 'single', asyncio.get_running_loop()) + schema = BoundSchema(conn, backend='mongodb', is_async=True) + original = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError) as raised: + with schema._operation('ensure_collection'): + raise original + self.assertIs(raised.exception, original) -- Gitee From ca247e3e80ea66ce633fcd516b3bec00df4a6491 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:12:47 +0800 Subject: [PATCH 67/86] =?UTF-8?q?fix(schema):=20=E4=BF=9D=E7=95=99?= =?UTF-8?q?=E9=A9=B1=E5=8A=A8=E5=BC=82=E5=B8=B8=E8=BD=AC=E6=8D=A2=E6=97=B6?= =?UTF-8?q?=E7=9A=84=E9=99=84=E5=8A=A0=E6=B8=85=E7=90=86=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/schema/_base.py | 28 ++++++++++++++++++++++------ tests/test_schema_binding.py | 14 ++++++++++++++ 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/src/database/schema/_base.py b/src/database/schema/_base.py index dbbbb8c..d91f746 100644 --- a/src/database/schema/_base.py +++ b/src/database/schema/_base.py @@ -8,6 +8,7 @@ from ..connections import AsyncConnection from ..connections import Connection from ..errors import ConfigurationError from ..errors import DatabaseError +from ..errors import attach_cleanup_errors from ..errors import error_context @@ -47,9 +48,24 @@ class BoundSchema: if isinstance(connection, AsyncConnection): connection._ensure_loop('operation') connection._ensure_open('operation') - with error_context( - **self._context_fields(operation, target), - classifier=connection._adapter.classify, - owned=True, - ): - yield + try: + with error_context( + **self._context_fields(operation, target), + classifier=connection._adapter.classify, + owned=True, + ): + yield + except DatabaseError as error: + if error.original is not None: + attributes = BaseException.__dict__['__dict__'].__get__(error.original) + cleanup_errors = attributes.get('database_cleanup_errors', ()) + if isinstance(cleanup_errors, tuple): + attach_cleanup_errors( + error, + tuple( + cleanup + for cleanup in cleanup_errors + if not any(cleanup is existing for existing in error.database_cleanup_errors) + ), + ) + raise diff --git a/tests/test_schema_binding.py b/tests/test_schema_binding.py index 1dfd1ce..6bda674 100644 --- a/tests/test_schema_binding.py +++ b/tests/test_schema_binding.py @@ -56,6 +56,20 @@ class SchemaBindingTests(unittest.TestCase): self.assertEqual(raised.exception.connection_name, 'orders') self.assertEqual(raised.exception.operation, 'ensure_table') + def test_cleanup_errors_survive_conversion_without_duplicates(self): + conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') + schema = BoundSchema(conn, backend='mysql', is_async=False) + original = NativeFailure('driver detail') + cleanup = OSError('cursor close failed') + original.database_cleanup_errors = (cleanup,) + with self.assertRaises(DriverError) as raised: + with schema._operation('ensure_table', 'app.users'): + with schema._operation('inspect_table', 'app.users'): + raise original + self.assertIs(raised.exception.original, original) + self.assertEqual(raised.exception.database_cleanup_errors, (cleanup,)) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + def test_business_and_unified_errors_keep_identity(self): conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') schema = BoundSchema(conn, backend='mysql', is_async=False) -- Gitee From 808bd908715061cd5f8f0baaaeef6310bd89c110 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:16:33 +0800 Subject: [PATCH 68/86] =?UTF-8?q?feat(redis):=20=E5=B0=81=E8=A3=85?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E4=B8=B2=E5=8D=95=E9=94=AE=E4=B8=8E=E6=89=B9?= =?UTF-8?q?=E9=87=8F=E6=93=8D=E4=BD=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/crud/_strings.py | 503 ++++++++++++++++++++++++++++++++++ tests/test_crud_strings.py | 439 +++++++++++++++++++++++++++++ 2 files changed, 942 insertions(+) create mode 100644 src/database/crud/_strings.py create mode 100644 tests/test_crud_strings.py diff --git a/src/database/crud/_strings.py b/src/database/crud/_strings.py new file mode 100644 index 0000000..0afcc7b --- /dev/null +++ b/src/database/crud/_strings.py @@ -0,0 +1,503 @@ +"""Redis String operations on a managed connection's original keys.""" + +from asyncio import CancelledError +from collections.abc import Iterator +from collections.abc import Sequence +from contextlib import AbstractContextManager +from dataclasses import dataclass +from dataclasses import replace +from typing import Any +from typing import ClassVar +from typing import NoReturn +from typing import cast + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DatabaseError +from ..errors import DriverError +from ..errors import attach_cleanup_errors +from ._conditions import is_sequence +from ._errors import BatchOperationError +from ._string_types import StringBatchItem +from ._string_types import StringBatchResult +from ._string_types import StringKey +from ._string_types import StringSetItem +from ._string_types import StringValue +from ._types import BatchState + +# Redis represents expiration in signed 64-bit milliseconds. +_MAX_TTL_SECONDS = (2**63 - 1) // 1000 +_Reply = StringValue | bool | int | None + + +@dataclass(frozen=True) +class _Request: + key: StringKey + command: tuple[str | bytes | int, ...] + + +class _Batch: + """Keep proof of each dispatched command in caller input order.""" + + def __init__(self, requests: Sequence[_Request]) -> None: + self.items = [ + StringBatchItem(index, request.key, BatchState.NOT_SENT) for index, request in enumerate(requests) + ] + self.native_results: list[Any] = [] + + def dispatch(self, start: int, size: int) -> None: + for index in range(start, start + size): + self.items[index] = replace(self.items[index], state=BatchState.UNKNOWN) + + def record( + self, index: int, state: BatchState, *, result: _Reply = None, error: BaseException | None = None + ) -> None: + self.items[index] = replace(self.items[index], state=state, result=result, error=error) + + def result(self, operation: str) -> StringBatchResult: + deleted_count = None + if operation == 'delete_many': + sent = [item for item in self.items if item.state != BatchState.NOT_SENT] + if all(item.state == BatchState.ACKNOWLEDGED for item in sent): + deleted_count = sum(cast(int, item.result) for item in sent) + return StringBatchResult(len(self.items), tuple(self.items), deleted_count, tuple(self.native_results)) + + +class _Strings: + _async: ClassVar[bool] = False + + def __init__(self, connection: Connection) -> None: + if not isinstance(connection, Connection) or isinstance(connection, AsyncConnection) != self._async: + raise ConfigurationError( + 'Redis String operations require a connection with the same sync mode.', phase='config' + ) + if connection.backend != 'redis': + raise ConfigurationError( + 'Redis String operations require a Redis connection.', + backend=connection.backend, + connection_name=connection.name, + phase='config', + ) + self._connection = connection + self._sentinel = connection.mode == 'sentinel' + self._client: Any = None + self._entered = False + + def _error( + self, + kind: type[DatabaseError], + message: str, + operation: str, + *, + phase: str = 'operation', + original: BaseException | None = None, + ) -> DatabaseError: + return kind( + message, + connection_name=self._connection.name, + backend='redis', + phase=phase, + operation=operation, + original=original, + ) + + def _context(self, operation: str, *, phase: str = 'operation') -> AbstractContextManager: + return self._connection._context(phase, operation=operation) + + def _check_loop(self, phase: str) -> None: + if isinstance(self._connection, AsyncConnection): + self._connection._ensure_loop(phase) + + def _check_client(self, operation: str) -> None: + self._check_loop('operation') + self._connection._ensure_open('operation') + if self._client is None: + raise self._error(ConfigurationError, 'Redis String operations require an active context.', operation) + + def _open(self) -> None: + operation = 'open_strings' + self._check_loop('operation') + native = self._connection.native + if self._entered: + raise self._error(ConfigurationError, 'Redis String contexts can only be entered once.', operation) + self._entered = True + with self._context(operation): + if self._sentinel: + config = self._connection._adapter._config + native = native.master_for(config.service_name) + self._client = native + + def _encode(self, value: StringKey | StringValue, operation: str) -> bytes: + if not isinstance(value, (str, bytes)): + raise self._error( + ConfigurationError, 'Redis String keys and values require str or bytes.', operation, phase='config' + ) + try: + return self._client.get_encoder().encode(value) + except (LookupError, UnicodeError) as original: + raise self._error( + ConfigurationError, + 'Could not encode a Redis String key or value.', + operation, + phase='config', + original=original, + ) from original + + def _seconds(self, value: int, operation: str) -> int: + if type(value) is not int or not 1 <= value <= _MAX_TTL_SECONDS: + raise self._error( + ConfigurationError, + 'Redis TTL requires representable positive integer seconds.', + operation, + phase='config', + ) + return value + + def _key_request(self, operation: str, command: str, key: StringKey, *args: int) -> _Request: + return _Request(key, (command, self._encode(key, operation), *args)) + + def _set_request(self, operation: str, item: StringSetItem) -> _Request: + key = self._encode(item.key, operation) + value = self._encode(item.value, operation) + ttl = () if item.ttl_s is None else ('EX', self._seconds(item.ttl_s, operation)) + return _Request(item.key, ('SET', key, value, *ttl)) + + def _parse(self, command: str, value: Any, operation: str) -> _Reply: + if command == 'GET' and (value is None or isinstance(value, (str, bytes))): + return value + if command == 'SET' and value is True: + return True + if command == 'DEL' and type(value) is int and value in (0, 1): + return value + if command == 'TTL' and type(value) is int and value >= -2: + return value + if command in ('EXISTS', 'EXPIRE', 'PERSIST') and type(value) in (int, bool) and value in (0, 1): + return bool(value) + raise self._error(DriverError, 'Redis returned an unexpected single-key reply.', operation) + + def _prepare_batch( + self, operation: str, inputs: Sequence[StringKey] | Sequence[StringSetItem], batch_size: int + ) -> list[_Request]: + self._check_client(operation) + if type(batch_size) is not int or batch_size < 1: + raise self._error(ConfigurationError, 'Batch size must be a positive integer.', operation, phase='config') + if not is_sequence(inputs): + raise self._error( + ConfigurationError, 'Redis String batches require a finite input sequence.', operation, phase='config' + ) + + requests = [] + seen: set[bytes] = set() + for item in inputs: + if operation == 'set_many': + if not isinstance(item, StringSetItem): + raise self._error( + ConfigurationError, 'set_many requires StringSetItem inputs.', operation, phase='config' + ) + request = self._set_request(operation, item) + else: + command = 'GET' if operation == 'get_many' else 'DEL' + request = self._key_request(operation, command, item) + if operation != 'get_many': + wire_key = cast(bytes, request.command[1]) + if wire_key in seen: + raise self._error( + ConfigurationError, + 'The same encoded Redis key appears more than once in this batch.', + operation, + phase='config', + ) + seen.add(wire_key) + requests.append(request) + return requests + + @staticmethod + def _chunks(requests: Sequence[_Request], batch_size: int) -> Iterator[tuple[int, Sequence[_Request]]]: + for start in range(0, len(requests), batch_size): + yield start, requests[start : start + batch_size] + + def _consume_replies( + self, batch: _Batch, start: int, requests: Sequence[_Request], replies: Any, operation: str + ) -> BaseException | None: + from redis import exceptions + + batch.native_results.append(replies) + if not isinstance(replies, (list, tuple)): + return self._error(DriverError, 'Redis returned an unexpected batch reply.', operation) + size_error = None + if len(replies) != len(requests): + size_error = self._error( + DriverError, 'Redis batch replies do not match the requested command count.', operation + ) + first_error = None + for offset, request in enumerate(requests[: len(replies)]): + reply = replies[offset] + index = start + offset + error = None + if isinstance(reply, BaseException): + error = reply + state = BatchState.UNKNOWN + if isinstance( + reply, (exceptions.ResponseError, exceptions.AuthenticationError, exceptions.AuthorizationError) + ): + state = BatchState.FAILED + batch.record(index, state, error=error) + else: + try: + output = self._parse(request.command[0], reply, operation) + except DriverError as unexpected: + error = unexpected + batch.record(index, BatchState.UNKNOWN, error=error) + else: + batch.record(index, BatchState.ACKNOWLEDGED, result=output) + if error is not None and (first_error is None or isinstance(error, CancelledError)): + first_error = error + return first_error if first_error is not None else size_error + + def _raise_batch(self, error: BaseException, partial: StringBatchResult, operation: str) -> NoReturn: + if not isinstance(error, Exception): + if isinstance(error, CancelledError): + BaseException.__dict__['__dict__'].__get__(error)['database_partial_result'] = partial + descriptor = BaseException.__dict__['__context__'] + original_context = descriptor.__get__(error) + try: + raise error + finally: + descriptor.__set__(error, original_context) + + original = error.original if isinstance(error, DatabaseError) and error.original is not None else error + wrapped = BatchOperationError( + 'A Redis String batch failed.', + partial_result=partial, + connection_name=self._connection.name, + backend='redis', + phase='operation', + operation=operation, + original=original, + ) + attach_cleanup_errors(wrapped, getattr(original, 'database_cleanup_errors', ())) + if error is not original: + attach_cleanup_errors(wrapped, getattr(error, 'database_cleanup_errors', ())) + raise wrapped from original + + +class RedisStrings(_Strings): + """Synchronous access to Redis String keys within a borrowed connection.""" + + def __enter__(self) -> 'RedisStrings': + self._open() + return self + + def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: Any) -> bool: + try: + self._check_loop('close') + client, self._client = self._client, None + if self._sentinel and client is not None: + adapter = self._connection._adapter + errors = adapter._close_single(client, True, target='sentinel_master') + adapter._finish_cleanup(errors, exc) + except BaseException as cleanup: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup,)) + return False + + def _invoke(self, operation: str, request: _Request) -> _Reply: + with self._context(operation): + reply = self._client.execute_command(*request.command) + return self._parse(request.command[0], reply, operation) + + def get(self, key: StringKey) -> StringValue | None: + """Read a String, returning None only when the key is missing.""" + self._check_client('get') + return cast(StringValue | None, self._invoke('get', self._key_request('get', 'GET', key))) + + def set(self, key: StringKey, value: StringValue, *, ttl_s: int | None = None) -> bool: + """Replace one value and set or clear the expiration in the same command.""" + self._check_client('set') + request = self._set_request('set', StringSetItem(key, value, ttl_s)) + return cast(bool, self._invoke('set', request)) + + def delete(self, key: StringKey) -> int: + """Delete the named key and return the number removed (0 or 1).""" + self._check_client('delete') + return cast(int, self._invoke('delete', self._key_request('delete', 'DEL', key))) + + def exists(self, key: StringKey) -> bool: + """Report whether the named key exists.""" + self._check_client('exists') + return cast(bool, self._invoke('exists', self._key_request('exists', 'EXISTS', key))) + + def expire(self, key: StringKey, seconds: int) -> bool: + """Set positive integer seconds of expiration for one key.""" + self._check_client('expire') + request = self._key_request('expire', 'EXPIRE', key, self._seconds(seconds, 'expire')) + return cast(bool, self._invoke('expire', request)) + + def persist(self, key: StringKey) -> bool: + """Remove an existing expiration, returning whether it was removed.""" + self._check_client('persist') + return cast(bool, self._invoke('persist', self._key_request('persist', 'PERSIST', key))) + + def ttl(self, key: StringKey) -> int: + """Return the remaining seconds, -1 for no expiration, or -2 for a missing key.""" + self._check_client('ttl') + return cast(int, self._invoke('ttl', self._key_request('ttl', 'TTL', key))) + + def get_many(self, keys: Sequence[StringKey], *, batch_size: int = 500) -> list[StringValue | None]: + """Read each input position with its own GET, keeping duplicates and missing values.""" + result = self._run_batch('get_many', keys, batch_size) + return [cast(StringValue | None, item.result) for item in result.items] + + def set_many(self, items: Sequence[StringSetItem], *, batch_size: int = 500) -> StringBatchResult: + """Issue independent SET commands in non-transactional batches.""" + return self._run_batch('set_many', items, batch_size) + + def delete_many(self, keys: Sequence[StringKey], *, batch_size: int = 500) -> StringBatchResult: + """Issue independent DEL commands in non-transactional batches.""" + return self._run_batch('delete_many', keys, batch_size) + + def _run_batch( + self, operation: str, inputs: Sequence[StringKey] | Sequence[StringSetItem], batch_size: int + ) -> StringBatchResult: + requests = self._prepare_batch(operation, inputs, batch_size) + batch = _Batch(requests) + for start, chunk in self._chunks(requests, batch_size): + pipe = None + failure = None + try: + with self._context(operation): + pipe = self._client.pipeline(transaction=False) + for request in chunk: + pipe.execute_command(*request.command) + batch.dispatch(start, len(chunk)) + replies = pipe.execute(raise_on_error=False) + failure = self._consume_replies(batch, start, chunk, replies, operation) + except BaseException as error: + failure = error + finally: + if pipe is not None: + try: + with self._context(operation, phase='close'): + pipe.__exit__(None, None, None) + except BaseException as cleanup: + if failure is None: + failure = cleanup + else: + attach_cleanup_errors(failure, (cleanup,)) + if failure is not None: + self._raise_batch(failure, batch.result(operation), operation) + return batch.result(operation) + + +class AsyncRedisStrings(_Strings): + """Asynchronous access to Redis String keys within a borrowed connection.""" + + _async = True + + async def __aenter__(self) -> 'AsyncRedisStrings': + self._open() + return self + + async def __aexit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: Any) -> bool: + try: + self._check_loop('close') + client, self._client = self._client, None + if self._sentinel and client is not None: + adapter = self._connection._adapter + errors = await adapter._close_single_async(client, True, target='sentinel_master') + adapter._finish_cleanup(errors, exc) + except BaseException as cleanup: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup,)) + return False + + async def _invoke(self, operation: str, request: _Request) -> _Reply: + with self._context(operation): + reply = await self._client.execute_command(*request.command) + return self._parse(request.command[0], reply, operation) + + async def get(self, key: StringKey) -> StringValue | None: + """Read a String, returning None only when the key is missing.""" + self._check_client('get') + return cast(StringValue | None, await self._invoke('get', self._key_request('get', 'GET', key))) + + async def set(self, key: StringKey, value: StringValue, *, ttl_s: int | None = None) -> bool: + """Replace one value and set or clear the expiration in the same command.""" + self._check_client('set') + request = self._set_request('set', StringSetItem(key, value, ttl_s)) + return cast(bool, await self._invoke('set', request)) + + async def delete(self, key: StringKey) -> int: + """Delete the named key and return the number removed (0 or 1).""" + self._check_client('delete') + return cast(int, await self._invoke('delete', self._key_request('delete', 'DEL', key))) + + async def exists(self, key: StringKey) -> bool: + """Report whether the named key exists.""" + self._check_client('exists') + return cast(bool, await self._invoke('exists', self._key_request('exists', 'EXISTS', key))) + + async def expire(self, key: StringKey, seconds: int) -> bool: + """Set positive integer seconds of expiration for one key.""" + self._check_client('expire') + request = self._key_request('expire', 'EXPIRE', key, self._seconds(seconds, 'expire')) + return cast(bool, await self._invoke('expire', request)) + + async def persist(self, key: StringKey) -> bool: + """Remove an existing expiration, returning whether it was removed.""" + self._check_client('persist') + return cast(bool, await self._invoke('persist', self._key_request('persist', 'PERSIST', key))) + + async def ttl(self, key: StringKey) -> int: + """Return the remaining seconds, -1 for no expiration, or -2 for a missing key.""" + self._check_client('ttl') + return cast(int, await self._invoke('ttl', self._key_request('ttl', 'TTL', key))) + + async def get_many(self, keys: Sequence[StringKey], *, batch_size: int = 500) -> list[StringValue | None]: + """Read each input position with its own GET, keeping duplicates and missing values.""" + result = await self._run_batch('get_many', keys, batch_size) + return [cast(StringValue | None, item.result) for item in result.items] + + async def set_many(self, items: Sequence[StringSetItem], *, batch_size: int = 500) -> StringBatchResult: + """Issue independent SET commands in non-transactional batches.""" + return await self._run_batch('set_many', items, batch_size) + + async def delete_many(self, keys: Sequence[StringKey], *, batch_size: int = 500) -> StringBatchResult: + """Issue independent DEL commands in non-transactional batches.""" + return await self._run_batch('delete_many', keys, batch_size) + + async def _run_batch( + self, operation: str, inputs: Sequence[StringKey] | Sequence[StringSetItem], batch_size: int + ) -> StringBatchResult: + requests = self._prepare_batch(operation, inputs, batch_size) + batch = _Batch(requests) + for start, chunk in self._chunks(requests, batch_size): + pipe = None + failure = None + try: + with self._context(operation): + pipe = self._client.pipeline(transaction=False) + for request in chunk: + pipe.execute_command(*request.command) + batch.dispatch(start, len(chunk)) + replies = await pipe.execute(raise_on_error=False) + failure = self._consume_replies(batch, start, chunk, replies, operation) + except BaseException as error: + failure = error + finally: + if pipe is not None: + try: + with self._context(operation, phase='close'): + # Async ClusterPipeline.reset() in redis-py 5.0 issues Redis RESET. + await pipe.__aexit__(None, None, None) + except BaseException as cleanup: + if failure is None: + failure = cleanup + else: + attach_cleanup_errors(failure, (cleanup,)) + if failure is not None: + self._raise_batch(failure, batch.result(operation), operation) + return batch.result(operation) diff --git a/tests/test_crud_strings.py b/tests/test_crud_strings.py new file mode 100644 index 0000000..bd57ef9 --- /dev/null +++ b/tests/test_crud_strings.py @@ -0,0 +1,439 @@ +"""Redis String commands, lifetimes, and evidence from mixed batch replies.""" + +import asyncio +import unittest +from contextlib import ExitStack +from unittest.mock import AsyncMock +from unittest.mock import Mock +from unittest.mock import call +from unittest.mock import patch + +import redis + +from database import ConnectionConfig +from database import Connections +from database.crud import AsyncRedisStrings +from database.crud import BatchOperationError +from database.crud import BatchState +from database.crud import RedisStrings +from database.crud import StringSetItem +from database.errors import ConfigurationError +from database.errors import ConnectionClosedError +from database.errors import DriverError + + +def registry(mode='single', **options): + endpoint = {'nodes': [('sentinel.test', 26379)], 'service_name': 'main'} + if mode != 'sentinel': + endpoint = {} if 'connection_pool' in options else {'host': 'cache.test'} + config = ConnectionConfig('redis', mode=mode, options=options, **endpoint) + return Connections({'cache': config}) + + +def pipeline(replies=None, *, asynchronous=False): + pipe = Mock() + if asynchronous: + pipe.execute = AsyncMock(return_value=replies) + pipe.__aexit__ = AsyncMock() + else: + pipe.execute.return_value = replies + pipe.__exit__ = Mock() + return pipe + + +class RedisStringTests(unittest.TestCase): + def setUp(self): + self.connection = registry().create('cache') + self.addCleanup(self.connection.close) + self.native = self.connection.native + + def test_single_operations_preserve_wire_semantics_and_empty_strings(self): + replies = [b'', None, True, True, 1, -1, -2, 0, True, False, 1, 0] + with patch.object(self.native, 'execute_command', side_effect=replies) as execute: + with RedisStrings(self.connection) as strings: + self.assertEqual(strings.get('empty'), b'') + self.assertIsNone(strings.get(b'missing')) + self.assertTrue(strings.set('msg', '你好', ttl_s=20)) + self.assertTrue(strings.set('msg', b'new')) + self.assertTrue(strings.exists('msg')) + self.assertEqual(strings.ttl('msg'), -1) + self.assertEqual(strings.ttl('missing'), -2) + self.assertEqual(strings.ttl('expiring'), 0) + self.assertTrue(strings.expire('msg', 15)) + self.assertFalse(strings.persist('msg')) + self.assertEqual(strings.delete('msg'), 1) + self.assertEqual(strings.delete('msg'), 0) + self.assertEqual( + execute.call_args_list, + [ + call('GET', b'empty'), + call('GET', b'missing'), + call('SET', b'msg', '你好'.encode(), 'EX', 20), + call('SET', b'msg', b'new'), + call('EXISTS', b'msg'), + call('TTL', b'msg'), + call('TTL', b'missing'), + call('TTL', b'expiring'), + call('EXPIRE', b'msg', 15), + call('PERSIST', b'msg'), + call('DEL', b'msg'), + call('DEL', b'msg'), + ], + ) + + def test_get_preserves_driver_decoded_value_and_original_wrong_type(self): + wrong_type = redis.exceptions.ResponseError('WRONGTYPE') + with registry(decode_responses=True).create('cache') as connection: + with ( + patch.object(connection.native, 'execute_command', side_effect=['', wrong_type]), + RedisStrings(connection) as strings, + ): + self.assertEqual(strings.get('empty'), '') + with self.assertRaises(DriverError) as caught: + strings.get('hash') + self.assertIs(caught.exception.original, wrong_type) + self.assertIs(caught.exception.__cause__, wrong_type) + self.assertEqual(caught.exception.operation, 'get') + + def test_invalid_single_arguments_make_no_native_calls(self): + with patch.object(self.native, 'execute_command') as execute, RedisStrings(self.connection) as strings: + invalid = (True, 0, -1, 1.5, 2**63) + for seconds in invalid: + with self.subTest(ttl=seconds), self.assertRaises(ConfigurationError): + strings.set('key', 'value', ttl_s=seconds) + with self.subTest(seconds=seconds), self.assertRaises(ConfigurationError): + strings.expire('key', seconds) + for invalid_value in (True, 1, ['list'], None): + with self.subTest(value=invalid_value), self.assertRaises(ConfigurationError): + strings.set('key', invalid_value) + with self.assertRaises(ConfigurationError): + strings.get(42) + execute.assert_not_called() + + def test_batch_validates_complete_wire_keys_and_values_before_pipeline(self): + with registry(encoding='latin-1').create('cache') as connection: + with patch.object(connection.native, 'pipeline') as factory, RedisStrings(connection) as strings: + with self.assertRaises(ConfigurationError): + strings.delete_many(['café', b'caf\xe9'], batch_size=1) + with self.assertRaises(ConfigurationError): + strings.set_many([StringSetItem('é', 'a'), StringSetItem(b'\xe9', 'b')], batch_size=1) + with self.assertRaises(ConfigurationError) as caught: + strings.set_many([StringSetItem('valid', 'ok'), StringSetItem('bad', '€')], batch_size=1) + self.assertIsInstance(caught.exception.original, UnicodeEncodeError) + self.assertIs(caught.exception.__cause__, caught.exception.original) + with self.assertRaises(ConfigurationError): + strings.get_many(['ok'], batch_size=False) + self.assertEqual(strings.get_many([]), []) + self.assertEqual(strings.set_many([]).items, ()) + self.assertEqual(strings.delete_many([]).deleted_count, 0) + factory.assert_not_called() + + def test_batched_get_uses_separate_commands_and_preserves_positions(self): + first = pipeline([b'', b'changed']) + last = pipeline([None]) + with ( + patch.object(self.native, 'pipeline', side_effect=[first, last]) as factory, + RedisStrings(self.connection) as strings, + ): + self.assertEqual(strings.get_many(['key', b'key', 'absent'], batch_size=2), [b'', b'changed', None]) + self.assertEqual(factory.call_args_list, [call(transaction=False), call(transaction=False)]) + self.assertEqual(first.execute_command.call_args_list, [call('GET', b'key'), call('GET', b'key')]) + self.assertEqual(last.execute_command.call_args_list, [call('GET', b'absent')]) + last.execute.assert_called_once_with(raise_on_error=False) + + def test_batches_keep_set_acknowledgment_and_delete_reply_counts(self): + first = pipeline([True, True]) + last = pipeline([1, 0]) + with patch.object(self.native, 'pipeline', side_effect=[first, last]), RedisStrings(self.connection) as strings: + batch = strings.set_many([StringSetItem('a', 'a', ttl_s=99), StringSetItem('b', b'b')]) + self.assertEqual([item.result for item in batch.items], [True, True]) + self.assertEqual(batch.requested_count, 2) + self.assertIsNone(batch.deleted_count) + deleted = strings.delete_many(['a', 'missing']) + self.assertEqual(deleted.deleted_count, 1) + self.assertEqual([item.result for item in deleted.items], [1, 0]) + self.assertEqual( + first.execute_command.call_args_list, [call('SET', b'a', b'a', 'EX', 99), call('SET', b'b', b'b')] + ) + self.assertEqual(last.execute_command.call_args_list, [call('DEL', b'a'), call('DEL', b'missing')]) + + def test_batch_exception_keeps_server_failures_and_unknowns_distinct(self): + wrong_type = redis.exceptions.ResponseError('WRONGTYPE') + disconnected = redis.exceptions.ConnectionError('connection dropped') + replies = [None, wrong_type, disconnected, b'valid'] + pipe = pipeline(replies) + with ( + patch.object(self.native, 'pipeline', return_value=pipe) as factory, + RedisStrings(self.connection) as strings, + ): + with self.assertRaises(BatchOperationError) as caught: + strings.get_many(['missing', 'hash', 'maybe', 'last', 'not_sent'], batch_size=4) + error = caught.exception + self.assertIs(error.original, wrong_type) + self.assertIs(error.__cause__, wrong_type) + self.assertIs(error.partial_result.native_results[0], replies) + self.assertEqual( + [item.state for item in error.partial_result.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.FAILED, + BatchState.UNKNOWN, + BatchState.ACKNOWLEDGED, + BatchState.NOT_SENT, + ], + ) + self.assertIsNone(error.partial_result.items[0].result) + self.assertIs(error.partial_result.items[2].error, disconnected) + self.assertEqual(error.partial_result.items[3].result, b'valid') + factory.assert_called_once() + + def test_queue_and_transport_failures_preserve_dispatch_evidence_and_cleanup(self): + for failure_point, state in (('execute_command', BatchState.NOT_SENT), ('execute', BatchState.UNKNOWN)): + pipe = pipeline() + driver_error = redis.exceptions.ConnectionError('transport failed') + cleanup_error = RuntimeError('pipeline cleanup failed') + getattr(pipe, failure_point).side_effect = driver_error + pipe.__exit__.side_effect = cleanup_error + with ( + self.subTest(step=failure_point), + patch.object(self.native, 'pipeline', return_value=pipe), + RedisStrings(self.connection) as strings, + self.assertRaises(BatchOperationError) as caught, + ): + strings.delete_many(['a', 'b'], batch_size=1) + error = caught.exception + self.assertIs(error.original, driver_error) + self.assertIn(cleanup_error, error.database_cleanup_errors) + self.assertEqual([item.state for item in error.partial_result.items], [state, BatchState.NOT_SENT]) + + def test_cleanup_only_failure_keeps_acknowledged_write_results(self): + first = pipeline([1]) + cleanup_error = OSError('close failed') + first.__exit__.side_effect = cleanup_error + with patch.object(self.native, 'pipeline', return_value=first), RedisStrings(self.connection) as strings: + with self.assertRaises(BatchOperationError) as caught: + strings.delete_many(['completed', 'not_queued'], batch_size=1) + self.assertIs(caught.exception.original, cleanup_error) + partial = caught.exception.partial_result + self.assertEqual(partial.deleted_count, 1) + self.assertEqual([item.state for item in partial.items], [BatchState.ACKNOWLEDGED, BatchState.NOT_SENT]) + + def test_single_scope_never_closes_borrowed_native_or_external_pool(self): + external_pool = redis.ConnectionPool(host='cache.test') + with registry(connection_pool=external_pool).create('cache') as connection: + native = connection.native + wrapper = RedisStrings(connection) + with patch.object(native, 'close') as close, patch.object(external_pool, 'disconnect') as disconnect: + with self.assertRaises(ConfigurationError): + wrapper.get('inactive') + with wrapper: + self.assertFalse(connection.closed) + with self.assertRaises(ConfigurationError): + wrapper.get('outside') + self.assertIs(connection.native, native) + close.assert_not_called() + disconnect.assert_not_called() + with self.assertRaises(ConnectionClosedError): + with RedisStrings(connection): + self.fail('a closed connection cannot enter a String context') + + def test_sentinel_scope_borrows_wrapper_but_owns_one_data_client(self): + with registry('sentinel').create('cache') as connection: + sentinel = connection.native + master = sentinel.master_for('main') + with ( + patch.object(sentinel, 'master_for', return_value=master) as factory, + patch.object(master, 'execute_command', return_value=b'value') as execute, + patch.object(master.connection_pool, 'disconnect') as disconnect, + RedisStrings(connection) as strings, + ): + self.assertEqual(strings.get('key'), b'value') + self.assertEqual(strings.get('key'), b'value') + self.assertIs(connection.native, sentinel) + factory.assert_called_once_with('main') + self.assertEqual(execute.call_count, 2) + disconnect.assert_called_once_with() + + +class AsyncRedisStringTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.connection = await registry().create_async('cache') + self.addAsyncCleanup(self.connection.close) + self.native = self.connection.native + + async def test_uses_native_async_commands_and_nontransactional_pipeline(self): + pipe = pipeline([True, True], asynchronous=True) + with ( + patch.object(self.native, 'execute_command', new_callable=AsyncMock) as execute, + patch.object(self.native, 'pipeline', return_value=pipe) as factory, + ): + execute.side_effect = ['value', True, 1, 1, 1, 0, -2] + async with AsyncRedisStrings(self.connection) as strings: + self.assertEqual(await strings.get('a'), 'value') + self.assertTrue(await strings.set('a', 'x', ttl_s=10)) + self.assertTrue(await strings.exists('a')) + self.assertEqual(await strings.delete('a'), 1) + self.assertTrue(await strings.expire('a', 5)) + self.assertFalse(await strings.persist('a')) + self.assertEqual(await strings.ttl('a'), -2) + result = await strings.set_many([StringSetItem('a', 'a'), StringSetItem('b', 'b')]) + self.assertEqual(execute.await_count, 7) + self.assertEqual([item.result for item in result.items], [True, True]) + factory.assert_called_once_with(transaction=False) + pipe.execute.assert_awaited_once_with(raise_on_error=False) + pipe.__aexit__.assert_awaited_once_with(None, None, None) + + async def test_cancelled_pipeline_keeps_identity_and_partial_delivery(self): + completed = pipeline([b'confirmed'], asynchronous=True) + cancelled = pipeline(asynchronous=True) + + class ReadOnlyCancellation(asyncio.CancelledError): + def __setattr__(self, name, value): + raise AttributeError('read only') + + original = ReadOnlyCancellation() + cancelled.execute.side_effect = original + cleanup = OSError('cancel cleanup failed') + cancelled.__aexit__.side_effect = cleanup + with patch.object(self.native, 'pipeline', side_effect=[completed, cancelled]): + async with AsyncRedisStrings(self.connection) as strings: + with self.assertRaises(ReadOnlyCancellation) as caught: + await strings.get_many(['first', 'second', 'third'], batch_size=1) + self.assertIs(caught.exception, original) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + partial = original.database_partial_result + self.assertEqual(partial.items[0].result, b'confirmed') + self.assertEqual( + [item.state for item in partial.items], + [BatchState.ACKNOWLEDGED, BatchState.UNKNOWN, BatchState.NOT_SENT], + ) + + async def test_pipeline_cancellation_reply_keeps_prior_exception_context(self): + original = asyncio.CancelledError() + initial_context = ValueError('initial context') + original.__context__ = initial_context + pipe = pipeline([original], asynchronous=True) + with patch.object(self.native, 'pipeline', return_value=pipe): + async with AsyncRedisStrings(self.connection) as strings: + try: + raise RuntimeError('caller exception') + except RuntimeError: + with self.assertRaises(asyncio.CancelledError) as caught: + await strings.delete_many(['a']) + self.assertIs(caught.exception, original) + self.assertIs(original.__context__, initial_context) + self.assertEqual(original.database_partial_result.items[0].state, BatchState.UNKNOWN) + + async def test_async_scope_rejects_wrong_loop_and_bind_mode(self): + with self.assertRaises(ConfigurationError): + RedisStrings(self.connection) + async with AsyncRedisStrings(self.connection) as strings: + + def run_in_other_loop(): + return asyncio.run(strings.get('illegal')) + + with patch.object(self.native, 'execute_command', new_callable=AsyncMock) as execute: + with self.assertRaises(ConfigurationError): + await asyncio.to_thread(run_in_other_loop) + execute.assert_not_awaited() + + async def test_async_sentinel_closes_owned_pool_without_hiding_primary_error(self): + connection = await registry('sentinel').create_async('cache') + self.addAsyncCleanup(connection.close) + sentinel = connection.native + master = sentinel.master_for('main') + original = ValueError('user body exception') + cleanup = redis.exceptions.ConnectionError('data pool cleanup failed') + with ( + patch.object(sentinel, 'master_for', return_value=master) as factory, + patch.object(master.connection_pool, 'disconnect', new_callable=AsyncMock, side_effect=cleanup), + ): + with self.assertRaises(ValueError) as caught: + async with AsyncRedisStrings(connection): + raise original + self.assertIs(caught.exception, original) + self.assertIs(original.database_cleanup_errors[0].original, cleanup) + self.assertIs(connection.native, sentinel) + factory.assert_called_once_with('main') + + +def cluster_topology(manager, nodes, keys=()): + manager.nodes_cache = {node.name: node for node in nodes} + manager.default_node = nodes[0] + manager.slots_cache = {redis.crc.key_slot(key): [node] for key, node in zip(keys, nodes)} + + +class RedisStringNativeClusterTests(unittest.IsolatedAsyncioTestCase): + def test_sync_driver_routes_each_set_to_its_own_slot(self): + keys = [b'{first}:string', b'{second}:string'] + clients = [redis.Redis(host=f'primary-{index}.test') for index in (1, 2)] + nodes = [ + redis.cluster.ClusterNode(client.connection_pool.connection_kwargs['host'], 7000, 'primary', client) + for client in clients + ] + with ExitStack() as stack: + stack.enter_context( + patch.object( + redis.cluster.NodesManager, 'initialize', lambda manager, *a, **k: cluster_topology(manager, nodes) + ) + ) + stack.enter_context(patch.object(redis.cluster.CommandsParser, 'initialize')) + connection = stack.enter_context(registry('cluster').create('cache')) + cluster_topology(connection.native.nodes_manager, nodes, keys) + stack.enter_context( + patch.object( + connection.native.commands_parser, 'get_keys', side_effect=lambda client, *args: (args[1],) + ) + ) + sockets = [] + for client in clients: + socket = Mock() + socket.pack_commands.side_effect = list + sockets.append(socket) + stack.enter_context(patch.object(client.connection_pool, 'get_connection', return_value=socket)) + stack.enter_context(patch.object(client.connection_pool, 'release')) + stack.enter_context(patch.object(client, 'parse_response', return_value=True)) + with RedisStrings(connection) as strings: + result = strings.set_many([StringSetItem(key, b'value') for key in keys]) + self.assertNotEqual(redis.crc.key_slot(keys[0]), redis.crc.key_slot(keys[1])) + self.assertEqual([item.result for item in result.items], [True, True]) + self.assertEqual( + [socket.send_packed_command.call_args.args[0][0][:2] for socket in sockets], + [('SET', keys[0]), ('SET', keys[1])], + ) + + async def test_async_driver_routes_get_to_multiple_slots_and_keeps_per_item_error(self): + async with await registry('cluster').create_async('cache') as connection: + keys = [b'{first}:string', b'{second}:string'] + nodes = [ + redis.asyncio.cluster.ClusterNode(f'primary-{index}.test', 7000, server_type='primary') + for index in (1, 2) + ] + cluster_topology(connection.native.nodes_manager, nodes, keys) + connection.native._initialize = False + wrong_type = redis.exceptions.ResponseError('WRONGTYPE') + routed = {} + + async def execute(node, commands): + for command in commands: + routed[node.host] = command.args[1] + command.result = b'first' if node is nodes[0] else wrong_type + return node is nodes[1] + + with ( + patch.object( + connection.native.commands_parser, + 'get_keys', + new_callable=AsyncMock, + side_effect=lambda *args: (args[1],), + ), + patch.object(redis.asyncio.cluster.ClusterNode, 'execute_pipeline', autospec=True, side_effect=execute), + ): + async with AsyncRedisStrings(connection) as strings: + with self.assertRaises(BatchOperationError) as caught: + await strings.get_many(keys) + self.assertIs(caught.exception.original, wrong_type) + self.assertEqual( + [item.state for item in caught.exception.partial_result.items], [BatchState.ACKNOWLEDGED, BatchState.FAILED] + ) + self.assertEqual(routed, {node.host: key for node, key in zip(nodes, keys)}) -- Gitee From 2e4c14752a65f061f1d022e8889142695fbbfd7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:23:36 +0800 Subject: [PATCH 69/86] =?UTF-8?q?feat(schema):=20=E5=AE=9E=E7=8E=B0=20Mong?= =?UTF-8?q?oDB=20=E9=9B=86=E5=90=88=E5=87=86=E5=A4=87=E4=B8=8E=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 同步和异步接口复用原生集合目录与建索引确认,保留并发冲突、重复数据和游标清理的原始错误。 完整离线回归通过。MongoDB 8.0.24 验证 PyMongo 4.6.0 同步及 4.9、4.13.0、4.18.3 同步和异步。 --- src/database/schema/_mongodb.py | 430 ++++++++++++++++++++++++++++++++ tests/test_schema_mongodb.py | 411 ++++++++++++++++++++++++++++++ 2 files changed, 841 insertions(+) create mode 100644 src/database/schema/_mongodb.py create mode 100644 tests/test_schema_mongodb.py diff --git a/src/database/schema/_mongodb.py b/src/database/schema/_mongodb.py new file mode 100644 index 0000000..26d7e37 --- /dev/null +++ b/src/database/schema/_mongodb.py @@ -0,0 +1,430 @@ +"""Explicit MongoDB collection and database operations.""" + +from collections.abc import Mapping +from collections.abc import Sequence +from hashlib import sha256 +from typing import Any + +from pymongo.errors import InvalidName +from pymongo.errors import OperationFailure + +from ..connections import AsyncConnection +from ..connections import Connection +from ..crud._records import _names +from ..errors import ConfigurationError +from ..errors import attach_cleanup_errors +from ._base import BoundSchema +from ._base import SchemaConflictError + +_Document = Mapping[str, Any] +_IndexRequest = tuple[list[tuple[str, int]], dict[str, Any]] +_ID_FIELDS = ('_id',) +_NAMESPACE_EXISTS = 48 +_INDEX_CONFLICTS = (85, 86) + + +def _collection_type(document: _Document) -> str: + if document['type'] != 'collection': + return document['type'] + options = document['options'] + if options.get('capped'): + return 'capped collection' + if 'timeseries' in options: + return 'timeseries' + if document['name'].startswith('system.'): + return 'system collection' + return 'collection' + + +def _collation(document: _Document) -> _Document: + collation = document.get('collation', {}) + return {} if collation.get('locale') == 'simple' else dict(collation) + + +def _has_id_index(document: _Document) -> bool: + options = document['options'] + index = document.get('idIndex', {}) + clustered = options.get('clusteredIndex', {}) + return (index.get('key') == {'_id': 1} and _collation(index) == _collation(options)) or ( + clustered.get('key') == {'_id': 1} and clustered.get('unique') is True + ) + + +def _equivalent_index(indexes: list[_Document], fields: tuple[str, ...], collation: _Document) -> _Document | None: + for index in indexes: + key = index['key'] + if ( + tuple(key) == fields + and all(direction in (1, -1) for direction in key.values()) + and index.get('unique') is True + and not index.get('sparse', False) + and 'partialFilterExpression' not in index + and _collation(index) == collation + ): + return index + return None + + +def _index_request(fields: tuple[str, ...], existing: _Document | None) -> _IndexRequest: + if existing is None: + # Use a separate stable name so an ordinary index with the driver's default name can coexist. + digest = sha256(repr(fields).encode('utf-8')).hexdigest()[:16] + return [(name, 1) for name in fields], {'name': f'database_unique_{digest}', 'unique': True} + + # Reissue the matching definition, preserving its name and options. The collection supplies + # the already-verified collation. A successful createIndexes confirms the index is ready. + options = {name: value for name, value in existing.items() if name not in ('key', 'ns', 'collation')} + return list(existing['key'].items()), options + + +def _target(database: str, collection: str) -> str | None: + if isinstance(database, str) and isinstance(collection, str): + return f'{database}.{collection}' + return None + + +def _rows(cursor: Any) -> list[_Document]: + try: + rows = list(cursor) + except BaseException as original: + try: + cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(original, (cleanup,)) + raise + cursor.close() + return rows + + +async def _async_rows(cursor: Any) -> list[_Document]: + try: + rows = [row async for row in cursor] + except BaseException as original: + try: + await cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(original, (cleanup,)) + raise + await cursor.close() + return rows + + +def _describe(database: Any, collection: str) -> _Document | None: + rows = _rows(database.list_collections(filter={'name': collection})) + return rows[0] if rows else None + + +async def _async_describe(database: Any, collection: str) -> _Document | None: + rows = await _async_rows(await database.list_collections(filter={'name': collection})) + return rows[0] if rows else None + + +class _MongoDBSchema(BoundSchema): + def _name(self, name: str, operation: str, target: str | None) -> None: + if not isinstance(name, str) or not name.strip() or '\x00' in name: + raise ConfigurationError( + 'Expected an explicit MongoDB database or collection name.', + **self._context_fields(operation, target), + ) + + def _database(self, name: str, operation: str, target: str | None) -> Any: + self._name(name, operation, target) + try: + return self._connection.native.get_database(name) + except InvalidName as original: + raise ConfigurationError( + 'Invalid MongoDB database name.', original=original, **self._context_fields(operation, target) + ) from original + + def _collection(self, database: Any, name: str, operation: str, target: str | None) -> Any: + self._name(name, operation, target) + try: + return database.get_collection(name) + except InvalidName as original: + raise ConfigurationError( + 'Invalid MongoDB collection name.', original=original, **self._context_fields(operation, target) + ) from original + + def _require_acknowledged(self, database: Any, operation: str, target: str | None) -> None: + if not database.write_concern.acknowledged: + raise ConfigurationError( + 'MongoDB schema writes require acknowledged write concern.', + **self._context_fields(operation, target), + ) + + def _key_fields(self, key_fields: Sequence[str], target: str | None) -> tuple[str, ...]: + try: + fields = _names(key_fields, empty=False) + for name in fields: + name.encode('utf-8') + except (ConfigurationError, UnicodeError) as original: + raise ConfigurationError( + 'Expected distinct, nonempty MongoDB record key fields.', + original=original, + **self._context_fields('ensure_collection', target), + ) from original + return fields + + def _conflict( + self, difference: str, target: str | None, original: OperationFailure | None = None + ) -> SchemaConflictError: + return SchemaConflictError( + 'MongoDB collection does not satisfy the requested structure.', + differences=(difference,), + original=original, + **self._context_fields('ensure_collection', target), + ) + + def _require_collection( + self, + metadata: _Document | None, + fields: tuple[str, ...], + target: str | None, + original: OperationFailure | None = None, + ) -> _Document: + if metadata is None: + raise self._conflict('Collection is absent from the visible catalog after creation.', target) + kind = _collection_type(metadata) + if kind != 'collection': + raise self._conflict(f'Expected an ordinary collection, found {kind}.', target, original) from original + if fields == _ID_FIELDS and not _has_id_index(metadata): + raise self._conflict('No compatible built-in _id identity exists.', target, original) from original + return metadata + + +class MongoDBSchema(_MongoDBSchema): + """Manage MongoDB schema using a borrowed synchronous connection.""" + + def __init__(self, connection: Connection) -> None: + super().__init__(connection, backend='mongodb', is_async=False) + + def list_databases(self) -> list[str]: + """Return database names visible to the current account.""" + with self._operation('list_databases'): + return self._connection.native.list_database_names() + + def database_exists(self, name: str) -> bool: + """Check whether a database is visible to the current account.""" + operation = 'database_exists' + with self._operation(operation, name): + self._database(name, operation, name) + return name in self._connection.native.list_database_names() + + def list_collections(self, database: str) -> list[str]: + """Return visible ordinary collections in the named database.""" + operation = 'list_collections' + with self._operation(operation, database): + native = self._database(database, operation, database) + rows = _rows(native.list_collections(filter={'type': 'collection'})) + return [row['name'] for row in rows if _collection_type(row) == 'collection'] + + def collection_exists(self, database: str, collection: str) -> bool: + """Check whether the named ordinary collection is visible.""" + operation = 'collection_exists' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + metadata = _describe(native, collection) + return metadata is not None and _collection_type(metadata) == 'collection' + + def describe_collection(self, database: str, collection: str) -> dict[str, Any] | None: + """Return native collection metadata when visible, including for views.""" + operation = 'describe_collection' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + metadata = _describe(native, collection) + return None if metadata is None else dict(metadata) + + def ensure_collection(self, database: str, collection: str, *, key_fields: Sequence[str] = _ID_FIELDS) -> None: + """Prepare an ordinary collection and confirm its record identity. + + Notes + ----- + The default identity uses the built-in _id index. Other key fields + require a complete unique index. An index failure leaves the collection + intact. Writes require acknowledged write concern. + """ + operation = 'ensure_collection' + target = _target(database, collection) + with self._operation(operation, target): + fields = self._key_fields(key_fields, target) + native = self._database(database, operation, target) + handle = self._collection(native, collection, operation, target) + self._require_acknowledged(native, operation, target) + metadata = _describe(native, collection) + creation_error = None + if metadata is None: + try: + native.create_collection(collection, check_exists=False) + except OperationFailure as original: + if original.code != _NAMESPACE_EXISTS: + raise + metadata = _describe(native, collection) + if metadata is None: + raise + creation_error = original + else: + metadata = _describe(native, collection) + metadata = self._require_collection(metadata, fields, target, creation_error) + if fields != _ID_FIELDS: + self._ensure_index(handle, fields, _collation(metadata['options']), target) + + def _ensure_index(self, collection: Any, fields: tuple[str, ...], collation: _Document, target: str | None) -> None: + existing = _equivalent_index(_rows(collection.list_indexes()), fields, collation) + keys, options = _index_request(fields, existing) + try: + collection.create_index(keys, **options) + except OperationFailure as original: + if original.code not in _INDEX_CONFLICTS: + raise + concurrent = _equivalent_index(_rows(collection.list_indexes()), fields, collation) + if concurrent is None: + raise self._conflict( + 'No compatible complete unique index could be created.', target, original + ) from original + confirmed_keys, confirmed_options = _index_request(fields, concurrent) + if (confirmed_keys, confirmed_options) == (keys, options): + raise + collection.create_index(confirmed_keys, **confirmed_options) + + def drop_collection(self, database: str, collection: str) -> None: + """Drop the exact collection namespace using the native driver.""" + operation = 'drop_collection' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + self._require_acknowledged(native, operation, target) + native.drop_collection(collection) + + def drop_database(self, name: str) -> None: + """Drop the exact named database using the native driver.""" + operation = 'drop_database' + with self._operation(operation, name): + native = self._database(name, operation, name) + self._require_acknowledged(native, operation, name) + self._connection.native.drop_database(name) + + +class AsyncMongoDBSchema(_MongoDBSchema): + """Manage MongoDB schema using a borrowed asynchronous connection.""" + + def __init__(self, connection: AsyncConnection) -> None: + super().__init__(connection, backend='mongodb', is_async=True) + + async def list_databases(self) -> list[str]: + """Return database names visible to the current account.""" + with self._operation('list_databases'): + return await self._connection.native.list_database_names() + + async def database_exists(self, name: str) -> bool: + """Check whether a database is visible to the current account.""" + operation = 'database_exists' + with self._operation(operation, name): + self._database(name, operation, name) + return name in await self._connection.native.list_database_names() + + async def list_collections(self, database: str) -> list[str]: + """Return visible ordinary collections in the named database.""" + operation = 'list_collections' + with self._operation(operation, database): + native = self._database(database, operation, database) + rows = await _async_rows(await native.list_collections(filter={'type': 'collection'})) + return [row['name'] for row in rows if _collection_type(row) == 'collection'] + + async def collection_exists(self, database: str, collection: str) -> bool: + """Check whether the named ordinary collection is visible.""" + operation = 'collection_exists' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + metadata = await _async_describe(native, collection) + return metadata is not None and _collection_type(metadata) == 'collection' + + async def describe_collection(self, database: str, collection: str) -> dict[str, Any] | None: + """Return native collection metadata when visible, including for views.""" + operation = 'describe_collection' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + metadata = await _async_describe(native, collection) + return None if metadata is None else dict(metadata) + + async def ensure_collection( + self, database: str, collection: str, *, key_fields: Sequence[str] = _ID_FIELDS + ) -> None: + """Prepare an ordinary collection and confirm its record identity. + + Notes + ----- + The default identity uses the built-in _id index. Other key fields + require a complete unique index. An index failure leaves the collection + intact. Writes require acknowledged write concern. + """ + operation = 'ensure_collection' + target = _target(database, collection) + with self._operation(operation, target): + fields = self._key_fields(key_fields, target) + native = self._database(database, operation, target) + handle = self._collection(native, collection, operation, target) + self._require_acknowledged(native, operation, target) + metadata = await _async_describe(native, collection) + creation_error = None + if metadata is None: + try: + await native.create_collection(collection, check_exists=False) + except OperationFailure as original: + if original.code != _NAMESPACE_EXISTS: + raise + metadata = await _async_describe(native, collection) + if metadata is None: + raise + creation_error = original + else: + metadata = await _async_describe(native, collection) + metadata = self._require_collection(metadata, fields, target, creation_error) + if fields != _ID_FIELDS: + await self._ensure_index(handle, fields, _collation(metadata['options']), target) + + async def _ensure_index( + self, collection: Any, fields: tuple[str, ...], collation: _Document, target: str | None + ) -> None: + existing = _equivalent_index(await _async_rows(await collection.list_indexes()), fields, collation) + keys, options = _index_request(fields, existing) + try: + await collection.create_index(keys, **options) + except OperationFailure as original: + if original.code not in _INDEX_CONFLICTS: + raise + concurrent = _equivalent_index(await _async_rows(await collection.list_indexes()), fields, collation) + if concurrent is None: + raise self._conflict( + 'No compatible complete unique index could be created.', target, original + ) from original + confirmed_keys, confirmed_options = _index_request(fields, concurrent) + if (confirmed_keys, confirmed_options) == (keys, options): + raise + await collection.create_index(confirmed_keys, **confirmed_options) + + async def drop_collection(self, database: str, collection: str) -> None: + """Drop the exact collection namespace using the native driver.""" + operation = 'drop_collection' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + self._require_acknowledged(native, operation, target) + await native.drop_collection(collection) + + async def drop_database(self, name: str) -> None: + """Drop the exact named database using the native driver.""" + operation = 'drop_database' + with self._operation(operation, name): + native = self._database(name, operation, name) + self._require_acknowledged(native, operation, name) + await self._connection.native.drop_database(name) diff --git a/tests/test_schema_mongodb.py b/tests/test_schema_mongodb.py new file mode 100644 index 0000000..2f496d3 --- /dev/null +++ b/tests/test_schema_mongodb.py @@ -0,0 +1,411 @@ +"""MongoDB schema behavior with synchronous and asynchronous driver doubles.""" + +import asyncio +from unittest import IsolatedAsyncioTestCase +from unittest import TestCase +from unittest.mock import AsyncMock +from unittest.mock import Mock + +from pymongo.errors import DuplicateKeyError +from pymongo.errors import OperationFailure +from pymongo.errors import WriteConcernError +from pymongo.write_concern import WriteConcern + +from database._adapters.mongodb import Adapter +from database.connections import AsyncConnection +from database.connections import Connection +from database.errors import ConfigurationError +from database.errors import DriverError +from database.errors import PermissionDeniedError +from database.schema import AsyncSchema +from database.schema import Schema +from database.schema import SchemaConflictError + + +def _metadata(name='orders', kind='collection', **options): + document = {'name': name, 'type': kind, 'options': options, 'info': {'readOnly': kind == 'view'}} + if kind == 'collection': + document['idIndex'] = {'key': {'_id': 1}, 'name': '_id_'} + if 'collation' in options: + document['idIndex']['collation'] = options['collation'] + return document + + +def _index(name='order_key', **options): + return {'name': name, 'key': {'tenant': 1, 'number': -1}, 'v': 2, 'unique': True, **options} + + +class _Cursor: + def __init__(self, rows=(), *, error=None, cleanup=None): + self.rows = rows + self.error = error + self.close = Mock(side_effect=cleanup) + + def __iter__(self): + yield from self.rows + if self.error is not None: + raise self.error + + +class _AsyncCursor: + def __init__(self, rows=(), *, error=None, cleanup=None): + self.rows = rows + self.error = error + self.close = AsyncMock(side_effect=cleanup) + + async def __aiter__(self): + for row in self.rows: + yield row + if self.error is not None: + raise self.error + + +class MongoDBSchemaTests(TestCase): + def setUp(self): + self.collection = Mock(list_indexes=Mock(return_value=_Cursor())) + self.database = Mock(write_concern=WriteConcern(), get_collection=Mock(return_value=self.collection)) + self.client = Mock(get_database=Mock(return_value=self.database)) + self.adapter = Mock(classify=Adapter.classify) + self.connection = Connection(self.client, self.adapter, 'app-mongo', 'mongodb', 'single') + self.schema = Schema.mongodb(self.connection) + self.client.get_database.assert_not_called() + + def test_visible_catalog_metadata_and_exact_drop(self): + ordinary = _metadata() + capped = _metadata('queue', capped=True) + view = _metadata('daily', 'view', viewOn='orders', pipeline=[]) + series = _metadata('events', 'timeseries', timeseries={'timeField': 'at'}) + buckets = _metadata('system.buckets.events', timeseries={'timeField': 'at'}) + system = _metadata('system.views') + list_cursor = _Cursor([ordinary, capped, view, series, buckets, system]) + view_cursor = _Cursor([view]) + absent_cursor = _Cursor() + self.client.list_database_names.return_value = ['sales'] + self.database.list_collections.side_effect = [ + list_cursor, + _Cursor([ordinary]), + _Cursor([capped]), + view_cursor, + absent_cursor, + ] + + self.assertEqual(self.schema.list_databases(), ['sales']) + self.assertTrue(self.schema.database_exists('sales')) + self.assertFalse(self.schema.database_exists('invisible')) + self.assertEqual(self.schema.list_collections('sales'), ['orders']) + self.assertTrue(self.schema.collection_exists('sales', 'orders')) + self.assertFalse(self.schema.collection_exists('sales', 'queue')) + self.assertEqual(self.schema.describe_collection('sales', 'daily'), view) + self.assertIsNone(self.schema.describe_collection('sales', 'missing')) + self.assertIsNone(self.schema.drop_collection('sales', 'daily')) + self.assertIsNone(self.schema.drop_database('missing')) + + for cursor in (list_cursor, view_cursor, absent_cursor): + cursor.close.assert_called_once_with() + self.assertEqual(self.database.list_collections.call_args_list[0].kwargs, {'filter': {'type': 'collection'}}) + self.assertEqual(self.database.list_collections.call_args_list[3].kwargs, {'filter': {'name': 'daily'}}) + self.database.drop_collection.assert_called_once_with('daily') + self.client.drop_database.assert_called_once_with('missing') + self.adapter.close.assert_not_called() + self.client.close.assert_not_called() + + def test_ensure_default_creates_once_and_uses_builtin_id(self): + metadata = _metadata() + self.database.list_collections.side_effect = [_Cursor(), _Cursor([metadata]), _Cursor([metadata])] + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders')) + self.assertIsNone(self.schema.ensure_collection('sales', 'orders')) + + self.database.create_collection.assert_called_once_with('orders', check_exists=False) + self.collection.list_indexes.assert_not_called() + self.collection.create_index.assert_not_called() + + def test_ensure_accepts_clustered_builtin_id_but_rejects_missing_id(self): + clustered = _metadata(clusteredIndex={'key': {'_id': 1}, 'unique': True}) + clustered.pop('idIndex') + unindexed = _metadata(autoIndexId=False) + unindexed.pop('idIndex') + self.database.list_collections.side_effect = [_Cursor([clustered]), _Cursor([unindexed])] + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders')) + with self.assertRaises(SchemaConflictError) as raised: + self.schema.ensure_collection('sales', 'orders') + self.assertIn('_id', raised.exception.differences[0]) + self.collection.create_index.assert_not_called() + + def test_nonordinary_collection_is_not_changed(self): + for metadata in ( + _metadata(kind='view'), + _metadata(kind='timeseries', timeseries={'timeField': 'at'}), + _metadata(capped=True), + ): + with self.subTest(metadata=metadata): + self.database.list_collections.return_value = _Cursor([metadata]) + with self.assertRaises(SchemaConflictError) as raised: + self.schema.ensure_collection('sales', 'orders') + self.assertEqual(raised.exception.operation, 'ensure_collection') + self.assertEqual(raised.exception.target, 'sales.orders') + self.assertTrue(raised.exception.differences) + self.database.create_collection.assert_not_called() + self.collection.create_index.assert_not_called() + + def test_concurrent_collection_creation_is_verified(self): + original = OperationFailure('namespace exists', 48) + self.database.create_collection.side_effect = original + self.database.list_collections.side_effect = [ + _Cursor(), + _Cursor([_metadata()]), + _Cursor(), + _Cursor([_metadata(kind='view')]), + _Cursor(), + _Cursor(), + ] + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders')) + with self.assertRaises(SchemaConflictError) as conflict: + self.schema.ensure_collection('sales', 'orders') + with self.assertRaises(DriverError) as disappeared: + self.schema.ensure_collection('sales', 'orders') + for raised in (conflict, disappeared): + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + + def test_acknowledged_create_without_visible_metadata_is_unconfirmed(self): + self.database.list_collections.side_effect = [_Cursor(), _Cursor()] + with self.assertRaises(SchemaConflictError) as raised: + self.schema.ensure_collection('sales', 'orders') + self.assertIn('visible catalog', raised.exception.differences[0]) + + def test_custom_index_reuses_and_confirms_equivalent_definition(self): + collation = {'locale': 'en', 'strength': 2, 'version': '57.1'} + matching = _index(collation=collation, hidden=True, ns='sales.orders') + wrong_collation = _index('binary') + cursor = _Cursor([wrong_collation, matching]) + self.database.list_collections.return_value = _Cursor([_metadata(collation=collation)]) + self.collection.list_indexes.return_value = cursor + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders', key_fields=['tenant', 'number'])) + + self.collection.create_index.assert_called_once_with( + [('tenant', 1), ('number', -1)], name='order_key', unique=True, v=2, hidden=True + ) + cursor.close.assert_called_once_with() + self.database.create_collection.assert_not_called() + + def test_nonmatching_index_does_not_replace_full_unique_index(self): + indexes = [ + _index('sparse', sparse=True), + _index('partial', partialFilterExpression={'number': {'$exists': True}}), + _index('nonunique', unique=False), + _index('reversed', key={'number': 1, 'tenant': 1}), + _index('nonbinary', collation={'locale': 'en'}), + _index('hashed', key={'tenant': 1, 'number': 'hashed'}), + _index('tenant_1_number_1', key={'tenant': 1, 'number': 1}, unique=False), + ] + self.database.list_collections.return_value = _Cursor([_metadata()]) + self.collection.list_indexes.return_value = _Cursor(indexes) + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number'))) + + keys = self.collection.create_index.call_args.args[0] + options = self.collection.create_index.call_args.kwargs + self.assertEqual(keys, [('tenant', 1), ('number', 1)]) + self.assertIs(options['unique'], True) + self.assertNotIn(options['name'], {index['name'] for index in indexes}) + self.collection.drop_index.assert_not_called() + + def test_index_conflict_requires_matching_definition_and_acknowledgment(self): + original = OperationFailure('conflicting index name', 85) + self.database.list_collections.return_value = _Cursor([_metadata()]) + self.collection.list_indexes.side_effect = [_Cursor(), _Cursor([_index()]), _Cursor(), _Cursor()] + self.collection.create_index.side_effect = [original, 'order_key', original] + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number'))) + self.assertEqual(self.collection.create_index.call_args.kwargs['name'], 'order_key') + with self.assertRaises(SchemaConflictError) as raised: + self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number')) + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.assertEqual(self.collection.create_index.call_count, 3) + + def test_known_index_rejection_does_not_reissue_unchanged_definition(self): + original = OperationFailure('conflicting index options', 85) + self.database.list_collections.return_value = _Cursor([_metadata()]) + self.collection.list_indexes.side_effect = [_Cursor([_index()]), _Cursor([_index()])] + self.collection.create_index.side_effect = original + + with self.assertRaises(DriverError) as raised: + self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number')) + + self.assertIs(raised.exception.original, original) + self.collection.create_index.assert_called_once() + + def test_duplicate_data_keeps_new_collection_and_native_failure(self): + original = DuplicateKeyError('duplicate value', 11000) + self.database.list_collections.side_effect = [_Cursor(), _Cursor([_metadata()])] + self.collection.create_index.side_effect = original + + with self.assertRaises(DriverError) as raised: + self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number')) + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.assertEqual(raised.exception.target, 'sales.orders') + self.database.create_collection.assert_called_once() + self.database.drop_collection.assert_not_called() + self.collection.drop_index.assert_not_called() + + def test_write_concern_error_cannot_become_success(self): + original = WriteConcernError('replication incomplete', 64) + self.database.list_collections.return_value = _Cursor() + self.database.create_collection.side_effect = original + with self.assertRaises(DriverError) as raised: + self.schema.ensure_collection('sales', 'orders') + self.assertIs(raised.exception.original, original) + + def test_w0_and_invalid_keys_fail_before_catalog_or_write(self): + for fields in ('tenant', (), ('tenant', 'tenant'), ('nested.key',), ('$expr',), ('\ud800',)): + with self.subTest(fields=fields): + with self.assertRaises(ConfigurationError) as raised: + self.schema.ensure_collection('sales', 'orders', key_fields=fields) + self.assertEqual(raised.exception.operation, 'ensure_collection') + self.database.write_concern = WriteConcern(w=0) + with self.assertRaises(ConfigurationError): + self.schema.ensure_collection('sales', 'orders') + with self.assertRaises(ConfigurationError): + self.schema.drop_collection('sales', 'orders') + self.database.list_collections.assert_not_called() + self.database.create_collection.assert_not_called() + self.database.drop_collection.assert_not_called() + + def test_catalog_permission_error_is_not_reported_as_missing(self): + original = OperationFailure('not authorized', 13) + self.database.list_collections.side_effect = original + with self.assertRaises(PermissionDeniedError) as raised: + self.schema.collection_exists('sales', 'orders') + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.assertEqual(raised.exception.operation, 'collection_exists') + + def test_cursor_cleanup_preserves_primary_and_cleanup(self): + original = OperationFailure('getMore failed', 43) + cleanup = OperationFailure('killCursors failed', 13) + cursor = _Cursor([_metadata()], error=original, cleanup=cleanup) + self.database.list_collections.return_value = cursor + + with self.assertRaises(DriverError) as raised: + self.schema.describe_collection('sales', 'orders') + + self.assertIs(raised.exception.original, original) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + self.assertEqual(raised.exception.database_cleanup_errors, (cleanup,)) + cursor.close.assert_called_once_with() + + +class AsyncMongoDBSchemaTests(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.collection = Mock(list_indexes=AsyncMock(return_value=_AsyncCursor()), create_index=AsyncMock()) + self.database = Mock( + write_concern=WriteConcern(), + get_collection=Mock(return_value=self.collection), + list_collections=AsyncMock(), + create_collection=AsyncMock(), + drop_collection=AsyncMock(), + ) + self.client = Mock( + get_database=Mock(return_value=self.database), + list_database_names=AsyncMock(return_value=['sales']), + drop_database=AsyncMock(), + ) + self.adapter = Mock(classify=Adapter.classify) + connection = AsyncConnection( + self.client, self.adapter, 'app-mongo', 'mongodb', 'single', asyncio.get_running_loop() + ) + self.schema = AsyncSchema.mongodb(connection) + self.client.get_database.assert_not_called() + + async def test_async_catalog_default_identity_and_exact_drop(self): + metadata = _metadata() + cursor = _AsyncCursor([metadata, _metadata('capped', capped=True)]) + self.database.list_collections.side_effect = [ + _AsyncCursor(), + _AsyncCursor([metadata]), + cursor, + _AsyncCursor([metadata]), + _AsyncCursor([metadata]), + _AsyncCursor(), + ] + + self.assertIsNone(await self.schema.ensure_collection('sales', 'orders')) + self.assertEqual(await self.schema.list_databases(), ['sales']) + self.assertTrue(await self.schema.database_exists('sales')) + self.assertFalse(await self.schema.database_exists('missing')) + self.assertEqual(await self.schema.list_collections('sales'), ['orders']) + self.assertTrue(await self.schema.collection_exists('sales', 'orders')) + self.assertEqual(await self.schema.describe_collection('sales', 'orders'), metadata) + self.assertIsNone(await self.schema.describe_collection('sales', 'missing')) + self.assertIsNone(await self.schema.drop_collection('sales', 'orders')) + self.assertIsNone(await self.schema.drop_database('sales')) + + self.database.create_collection.assert_awaited_once_with('orders', check_exists=False) + self.collection.create_index.assert_not_awaited() + self.database.drop_collection.assert_awaited_once_with('orders') + self.client.drop_database.assert_awaited_once_with('sales') + cursor.close.assert_awaited_once_with() + self.adapter.close_async.assert_not_called() + + async def test_concurrent_index_listing_does_not_bypass_acknowledgment(self): + self.database.list_collections.return_value = _AsyncCursor([_metadata()]) + self.collection.list_indexes.side_effect = [_AsyncCursor([_index()]), _AsyncCursor([_index()])] + both_started = asyncio.Event() + ready = asyncio.Event() + started = 0 + + async def create_index(*args, **kwargs): + nonlocal started + started += 1 + if started == 2: + both_started.set() + await ready.wait() + return 'order_key' + + self.collection.create_index.side_effect = create_index + tasks = [ + asyncio.create_task(self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number'))) + for _ in range(2) + ] + try: + await asyncio.wait_for(both_started.wait(), timeout=2) + self.assertTrue(all(not task.done() for task in tasks)) + finally: + ready.set() + results = await asyncio.gather(*tasks) + self.assertEqual(results, [None, None]) + self.assertEqual( + [call.kwargs['name'] for call in self.collection.create_index.await_args_list], + ['order_key', 'order_key'], + ) + + async def test_async_cleanup_cannot_replace_cancellation(self): + original = asyncio.CancelledError('read canceled') + cleanup = OperationFailure('failed to close cursor', 43) + cursor = _AsyncCursor([_metadata()], error=original, cleanup=cleanup) + self.database.list_collections.return_value = cursor + + with self.assertRaises(asyncio.CancelledError) as raised: + await self.schema.collection_exists('sales', 'orders') + + self.assertIs(raised.exception, original) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + cursor.close.assert_awaited_once_with() + + async def test_async_conflicting_concurrent_namespace_has_native_cause(self): + original = OperationFailure('namespace exists', 48) + self.database.create_collection.side_effect = original + self.database.list_collections.side_effect = [_AsyncCursor(), _AsyncCursor([_metadata(capped=True)])] + + with self.assertRaises(SchemaConflictError) as raised: + await self.schema.ensure_collection('sales', 'orders', key_fields=['tenant']) + + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.collection.create_index.assert_not_awaited() -- Gitee From dc963d2b777a0acda75b65a058bde14920eed576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:28:20 +0800 Subject: [PATCH 70/86] =?UTF-8?q?feat(schema):=20=E5=AE=9E=E7=8E=B0=20MySQ?= =?UTF-8?q?L=20=E5=BA=93=E8=A1=A8=E7=BB=93=E6=9E=84=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/schema/_models.py | 182 +++++++++++++ src/database/schema/_mysql.py | 464 ++++++++++++++++++++++++++++++++ tests/test_schema_mysql.py | 477 +++++++++++++++++++++++++++++++++ 3 files changed, 1123 insertions(+) create mode 100644 src/database/schema/_models.py create mode 100644 src/database/schema/_mysql.py create mode 100644 tests/test_schema_mysql.py diff --git a/src/database/schema/_models.py b/src/database/schema/_models.py new file mode 100644 index 0000000..795380b --- /dev/null +++ b/src/database/schema/_models.py @@ -0,0 +1,182 @@ +"""Driver-independent declarations for basic MySQL table creation.""" + +import re +from collections.abc import Mapping +from collections.abc import Sequence +from dataclasses import dataclass +from dataclasses import field +from datetime import date +from datetime import datetime +from decimal import Context +from decimal import Decimal +from enum import Enum +from math import isfinite +from types import MappingProxyType + +from ..errors import ConfigurationError + +_IDENTIFIERS = re.compile(r'\w{1,64}') +_TYPES = frozenset({'int', 'bigint', 'boolean', 'decimal', 'double', 'varchar', 'text', 'date', 'datetime', 'blob'}) +_INT_BITS = {'int': 32, 'bigint': 64} + + +class _Default(Enum): + UNSPECIFIED = 0 + + +_UNSET = _Default.UNSPECIFIED + + +def _invalid(message: str) -> ConfigurationError: + return ConfigurationError(message, backend='mysql', phase='config') + + +def _identifier(name: str) -> str: + if type(name) is not str or _IDENTIFIERS.fullmatch(name) is None: + raise _invalid('MySQL identifiers require 1–64 letters, digits or underscores.') + return f'`{name}`' + + +def _integer_in_range(value: object, minimum: int, maximum: int) -> bool: + return type(value) is int and minimum <= value <= maximum + + +@dataclass(frozen=True) +class Column: + """MySQL column type, dimensions, nullability, literal default and auto-increment.""" + + type: str + length: int | None = None + precision: int | None = None + scale: int | None = None + nullable: bool = False + default: str | int | float | Decimal | date | datetime | None | _Default = field(default=_UNSET, repr=False) + auto_increment: bool = False + + def __post_init__(self) -> None: + if type(self.type) is not str or self.type.lower() not in _TYPES: + raise _invalid('Unsupported MySQL column type.') + object.__setattr__(self, 'type', self.type.lower()) + if type(self.nullable) is not bool or type(self.auto_increment) is not bool: + raise _invalid('Column nullable and auto_increment must be boolean.') + self._validate_dimensions() + if self.auto_increment and (self.type not in _INT_BITS or self.nullable or self.default is not _UNSET): + raise _invalid('AUTO_INCREMENT requires a non-nullable integer column with no explicit default.') + if self.default is _UNSET: + return + if self.default is None: + if not self.nullable: + raise _invalid('A NULL default requires a nullable MySQL column.') + return + object.__setattr__(self, 'default', self._literal_default()) + + def _validate_dimensions(self) -> None: + if self.type == 'varchar': + if not _integer_in_range(self.length, 1, 65535): + raise _invalid('VARCHAR requires a length between 1 and 65535.') + elif self.length is not None: + raise _invalid('Column length is supported for VARCHAR only.') + + if self.precision is not None: + if self.type == 'decimal': + valid = _integer_in_range(self.precision, 1, 65) + else: + valid = self.type == 'datetime' and _integer_in_range(self.precision, 0, 6) + if not valid: + raise _invalid('Precision requires DECIMAL (1–65) or DATETIME (0–6).') + if self.scale is not None and ( + self.type != 'decimal' or not _integer_in_range(self.scale, 0, min(30, self.precision or 10)) + ): + raise _invalid('DECIMAL scale must be between 0 and 30 and cannot exceed precision.') + + def _literal_default(self) -> str | int | float | Decimal | date | datetime: + value = self.default + if self.type in _INT_BITS: + limit = 2 ** (_INT_BITS[self.type] - 1) + if _integer_in_range(value, -limit, limit - 1): + return value + elif self.type == 'boolean': + if type(value) is bool or _integer_in_range(value, 0, 1): + return bool(value) + elif self.type == 'decimal': + if type(value) in (Decimal, float, int): + number = Decimal(str(value)) if type(value) is float else Decimal(value) + scale = self.scale or 0 + quantized = number.quantize( + Decimal((0, (1,), -scale)), context=Context(prec=self.precision or 10, traps=[]) + ) + if number.is_finite() and quantized == number: + return quantized + elif self.type == 'double': + if type(value) in (Decimal, float, int): + try: + number = float(value) + if isfinite(number): + return number + except (OverflowError, ValueError): + pass + elif self.type == 'varchar': + if type(value) is str and len(value) <= self.length: + try: + value.encode('utf-8') + except UnicodeEncodeError: + pass + else: + return value + elif self.type == 'date': + if type(value) is date and value.year >= 1000: + return value + elif self.type == 'datetime': + if ( + type(value) is datetime + and value.year >= 1000 + and value.tzinfo is None + and value.microsecond % (10 ** (6 - (self.precision or 0))) == 0 + ): + return value + elif self.type in ('text', 'blob'): + raise _invalid('TEXT and BLOB support only a NULL explicit default.') + raise _invalid('Expected a literal default that fits the declared MySQL column.') + + +@dataclass(frozen=True) +class TableSpec: + """Ordered MySQL columns and an explicit simple or composite primary key.""" + + columns: Mapping[str, Column] + primary_key: Sequence[str] + + def __post_init__(self) -> None: + if not isinstance(self.columns, Mapping) or not self.columns: + raise _invalid('A MySQL table requires a nonempty mapping of columns.') + columns = dict(self.columns) + names = {} + for name, column in columns.items(): + _identifier(name) + if not isinstance(column, Column): + raise _invalid('MySQL table columns must be Column objects.') + folded = name.casefold() + if folded in names: + raise _invalid('MySQL column names must be distinct regardless of case.') + names[folded] = name + + if isinstance(self.primary_key, (str, bytes)) or not isinstance(self.primary_key, Sequence): + raise _invalid('A MySQL primary key requires a sequence of column names.') + primary_key = [] + for name in self.primary_key: + _identifier(name) + declared = names.get(name.casefold()) + if declared is None: + raise _invalid('Every MySQL primary key field must be a declared column.') + if declared in primary_key: + raise _invalid('MySQL primary key fields must be distinct.') + if columns[declared].nullable or columns[declared].type in ('text', 'blob'): + raise _invalid('MySQL primary key fields must be non-nullable and support an unprefixed index.') + primary_key.append(declared) + if not primary_key: + raise _invalid('A MySQL table requires an explicit primary key.') + if any(column.auto_increment and primary_key != [name] for name, column in columns.items()): + raise _invalid('AUTO_INCREMENT is supported on a single-column integer primary key.') + + object.__setattr__(self, 'columns', MappingProxyType(columns)) + object.__setattr__(self, 'primary_key', tuple(primary_key)) diff --git a/src/database/schema/_mysql.py b/src/database/schema/_mysql.py new file mode 100644 index 0000000..6cc0699 --- /dev/null +++ b/src/database/schema/_mysql.py @@ -0,0 +1,464 @@ +"""Basic MySQL schema preparation on a borrowed Connector/Python connection.""" + +import re +from collections.abc import AsyncIterator +from collections.abc import Iterator +from contextlib import asynccontextmanager +from contextlib import contextmanager +from datetime import date +from datetime import datetime +from decimal import Decimal +from decimal import InvalidOperation +from typing import Any +from typing import NoReturn + +from mysql.connector.errorcode import ER_BAD_TABLE_ERROR +from mysql.connector.errorcode import ER_DB_CREATE_EXISTS +from mysql.connector.errorcode import ER_DB_DROP_EXISTS +from mysql.connector.errorcode import ER_TABLE_EXISTS_ERROR +from mysql.connector.errors import Error as MySQLError + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import attach_cleanup_errors +from ._base import BoundSchema +from ._base import SchemaConflictError +from ._models import _UNSET +from ._models import Column +from ._models import TableSpec +from ._models import _identifier +from ._models import _invalid + +_CHARSET = re.compile(r'[A-Za-z][A-Za-z0-9_]{0,63}') +_LEGACY_INTEGER_WIDTH = re.compile(r'(int|bigint)\(\d+\)') +_DATABASES = 'SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA ORDER BY SCHEMA_NAME' +_DATABASE = 'SELECT DEFAULT_CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s' +_TABLES = """ +SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA = %s AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME +""" +_TABLE = """ +SELECT t.TABLE_SCHEMA, t.TABLE_NAME, t.TABLE_TYPE, t.ENGINE, c.CHARACTER_SET_NAME, t.TABLE_COLLATION +FROM INFORMATION_SCHEMA.TABLES AS t +LEFT JOIN INFORMATION_SCHEMA.COLLATION_CHARACTER_SET_APPLICABILITY AS c + ON t.TABLE_COLLATION = c.COLLATION_NAME +WHERE t.TABLE_SCHEMA = %s AND t.TABLE_NAME = %s +""" +_COLUMNS = """ +SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA, GENERATION_EXPRESSION +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s ORDER BY ORDINAL_POSITION +""" +_PRIMARY_KEY = """ +SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE +WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND CONSTRAINT_NAME = 'PRIMARY' +ORDER BY ORDINAL_POSITION +""" + +_Row = tuple[Any, ...] + + +def _charset_name(value: str) -> str: + if type(value) is not str or _CHARSET.fullmatch(value) is None: + raise _invalid('Expected a MySQL character set name.') + name = value.lower() + return 'utf8mb3' if name == 'utf8' else name + + +def _qualified(database: str, table: str) -> str: + return f'{_identifier(database)}.{_identifier(table)}' + + +def _column_type(column: Column) -> str: + if column.type == 'decimal': + return f'decimal({column.precision or 10},{column.scale or 0})' + if column.type == 'datetime' and column.precision: + return f'datetime({column.precision})' + if column.type == 'varchar': + return f'varchar({column.length})' + if column.type == 'boolean': + return 'tinyint(1)' + return column.type + + +def _table_definition(qualified: str, spec: TableSpec) -> tuple[str, tuple[Any, ...]]: + if not isinstance(spec, TableSpec): + raise _invalid('MySQL table creation requires a TableSpec.') + columns = [] + literals = [] + for name, column in spec.columns.items(): + sql_type = 'BOOLEAN' if column.type == 'boolean' else _column_type(column).upper() + definitions = [_identifier(name), sql_type, 'NULL' if column.nullable else 'NOT NULL'] + if column.default is None: + definitions.append('DEFAULT NULL') + elif column.default is not _UNSET: + definitions.append('DEFAULT %s') + literals.append(column.default) + if column.auto_increment: + definitions.append('AUTO_INCREMENT') + columns.append(' '.join(definitions)) + key = ', '.join(map(_identifier, spec.primary_key)) + columns.append(f'PRIMARY KEY ({key})') + return f'CREATE TABLE IF NOT EXISTS {qualified} ({", ".join(columns)}) ENGINE=InnoDB', tuple(literals) + + +def _describe(table: _Row, columns: list[_Row], key: list[_Row]) -> dict[str, Any]: + database, name, _, engine, charset, collation = table + fields = {} + for column, native_type, nullable, default, extra, generated in columns: + flags = (extra or '').lower() + fields[column] = dict( + type=native_type, + nullable=nullable == 'YES', + default=default, + auto_increment='auto_increment' in flags.split(), + generated=bool(generated) or 'virtual generated' in flags or 'stored generated' in flags, + generation_expression=generated, + extra=extra, + ) + return dict( + database=database, + table=name, + columns=fields, + primary_key=tuple(column for (column,) in key), + engine=engine, + charset=charset, + collation=collation, + ) + + +def _compare_default(column: Column, actual: dict[str, Any]) -> bool: + if column.default is _UNSET: + return True + if 'default_generated' in (actual['extra'] or '').lower().split(): + return False + value = actual['default'] + if value is None or column.default is None: + return value is column.default + + # INFORMATION_SCHEMA exposes even numeric and temporal literal defaults as text. + try: + if column.type in ('int', 'bigint', 'boolean'): + value = int(value) + elif column.type == 'decimal': + value = Decimal(value) + elif column.type == 'double': + value = float(value) + elif column.type == 'date' and isinstance(value, str): + value = date.fromisoformat(value) + elif column.type == 'datetime' and isinstance(value, str): + value = datetime.fromisoformat(value) + return value == column.default + except (InvalidOperation, TypeError, ValueError): + return False + + +def _compare_table(spec: TableSpec, record: dict[str, Any]) -> tuple[str, ...]: + differences = [] + native_columns = {name.casefold(): (name, field) for name, field in record['columns'].items()} + for name, column in spec.columns.items(): + native = native_columns.pop(name.casefold(), None) + if native is None: + differences.append(f'Missing column {name!r}.') + continue + _, actual = native + native_type = actual['type'].lower() + width = _LEGACY_INTEGER_WIDTH.fullmatch(native_type) + if width is not None: + native_type = width[1] + elif native_type == 'datetime(0)': + native_type = 'datetime' + expected_type = _column_type(column) + if expected_type != native_type: + differences.append(f'Column {name!r} has type {actual["type"]}, expected {expected_type}.') + if actual['nullable'] != column.nullable: + differences.append(f'Column {name!r} has different nullability.') + if actual['generated']: + differences.append(f'Column {name!r} is generated.') + if actual['auto_increment'] != column.auto_increment: + differences.append(f'Column {name!r} has different auto-increment behavior.') + if not _compare_default(column, actual): + differences.append(f'Column {name!r} has a different default.') + + expected_key = tuple(name.casefold() for name in spec.primary_key) + native_key = tuple(name.casefold() for name in record['primary_key']) + if expected_key != native_key: + differences.append(f'Primary key {record["primary_key"]!r} differs from {tuple(spec.primary_key)!r}.') + for name, column in native_columns.values(): + if not (column['nullable'] or column['default'] is not None or column['auto_increment'] or column['generated']): + differences.append(f'Extra column {name!r} cannot be omitted.') + return tuple(differences) + + +def _decode_rows(native: Any, rows: list[_Row]) -> list[_Row]: + charset = getattr(native, 'python_charset', 'utf8') + return [tuple(value.decode(charset) if isinstance(value, bytes) else value for value in row) for row in rows] + + +class _MySQLSchema(BoundSchema): + def _require_ddl(self, allowed: bool, operation: str, target: str) -> None: + if not allowed: + raise ConfigurationError( + 'MySQL schema writes require autocommit and no active transaction.', + **self._context_fields(operation, target), + ) + + def _check_database(self, name: str, charset: str, rows: list[_Row]) -> None: + if not rows: + differences = ('Database is not visible after creation.',) + elif rows[0][0].lower() != charset: + differences = (f'Database has character set {rows[0][0]}, expected {charset}.',) + else: + return + self._conflict('ensure_database', name, differences) + + def _check_table_kind(self, target: str, rows: list[_Row]) -> None: + if not rows: + self._conflict('ensure_table', target, ('Table is not visible after creation.',)) + if rows[0][2] != 'BASE TABLE': + self._conflict('ensure_table', target, ('Existing object is not a base table.',)) + + def _check_table(self, target: str, spec: TableSpec, actual: dict[str, Any]) -> None: + differences = _compare_table(spec, actual) + if differences: + self._conflict('ensure_table', target, differences) + + def _conflict(self, operation: str, target: str, differences: tuple[str, ...]) -> NoReturn: + raise SchemaConflictError( + 'MySQL schema does not satisfy the declared structure.', + differences=differences, + **self._context_fields(operation, target), + ) + + +class MySQLSchema(_MySQLSchema): + """Manage MySQL databases and base tables synchronously.""" + + def __init__(self, connection: Connection) -> None: + super().__init__(connection, backend='mysql', is_async=False) + + @contextmanager + def _cursor(self) -> Iterator[Any]: + cursor = self._connection.native.cursor(buffered=True, raw=False) + try: + yield cursor + except BaseException as error: + try: + cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(error, (cleanup,)) + raise + else: + cursor.close() + + def _read(self, sql: str, params: tuple[Any, ...] = ()) -> list[_Row]: + with self._cursor() as cursor: + cursor.execute(sql, params) + return _decode_rows(self._connection.native, cursor.fetchall()) + + def _write(self, sql: str, params: tuple[Any, ...], *, ignore_errno: int) -> MySQLError | None: + with self._cursor() as cursor: + try: + cursor.execute(sql, params) + except MySQLError as error: + if error.errno != ignore_errno: + raise + return error + return None + + def _check_writable(self, operation: str, target: str) -> None: + native = self._connection.native + autocommit = not native.in_transaction and native.autocommit + self._require_ddl(autocommit and not native.in_transaction, operation, target) + + def ensure_database(self, name: str, *, charset: str = 'utf8mb4') -> None: + quoted = _identifier(name) + charset = _charset_name(charset) + with self._operation('ensure_database', name): + self._check_writable('ensure_database', name) + rows = self._read(_DATABASE, (name,)) + if not rows: + error = self._write( + f'CREATE DATABASE IF NOT EXISTS {quoted} DEFAULT CHARACTER SET {charset}', + (), + ignore_errno=ER_DB_CREATE_EXISTS, + ) + rows = self._read(_DATABASE, (name,)) + if not rows and error is not None: + raise error + self._check_database(name, charset, rows) + + def list_databases(self) -> list[str]: + with self._operation('list_databases'): + return [name for (name,) in self._read(_DATABASES)] + + def database_exists(self, name: str) -> bool: + _identifier(name) + with self._operation('database_exists', name): + return bool(self._read(_DATABASE, (name,))) + + def ensure_table(self, database: str, table: str, spec: TableSpec) -> None: + statement, literals = _table_definition(_qualified(database, table), spec) + target = f'{database}.{table}' + params = (database, table) + with self._operation('ensure_table', target): + self._check_writable('ensure_table', target) + rows = self._read(_TABLE, params) + if not rows: + error = self._write(statement, literals, ignore_errno=ER_TABLE_EXISTS_ERROR) + rows = self._read(_TABLE, params) + if not rows and error is not None: + raise error + self._check_table_kind(target, rows) + actual = _describe(rows[0], self._read(_COLUMNS, params), self._read(_PRIMARY_KEY, params)) + self._check_table(target, spec, actual) + + def list_tables(self, database: str) -> list[str]: + _identifier(database) + with self._operation('list_tables', database): + return [table for (table,) in self._read(_TABLES, (database,))] + + def table_exists(self, database: str, table: str) -> bool: + _qualified(database, table) + with self._operation('table_exists', f'{database}.{table}'): + rows = self._read(_TABLE, (database, table)) + return bool(rows and rows[0][2] == 'BASE TABLE') + + def describe_table(self, database: str, table: str) -> dict[str, Any] | None: + _qualified(database, table) + params = (database, table) + with self._operation('describe_table', f'{database}.{table}'): + rows = self._read(_TABLE, params) + if not rows or rows[0][2] != 'BASE TABLE': + return None + return _describe(rows[0], self._read(_COLUMNS, params), self._read(_PRIMARY_KEY, params)) + + def drop_table(self, database: str, table: str) -> None: + qualified = _qualified(database, table) + target = f'{database}.{table}' + with self._operation('drop_table', target): + self._check_writable('drop_table', target) + self._write(f'DROP TABLE IF EXISTS {qualified}', (), ignore_errno=ER_BAD_TABLE_ERROR) + + def drop_database(self, name: str) -> None: + quoted = _identifier(name) + with self._operation('drop_database', name): + self._check_writable('drop_database', name) + self._write(f'DROP DATABASE IF EXISTS {quoted}', (), ignore_errno=ER_DB_DROP_EXISTS) + + +class AsyncMySQLSchema(_MySQLSchema): + """Manage MySQL databases and base tables using an async connection.""" + + def __init__(self, connection: AsyncConnection) -> None: + super().__init__(connection, backend='mysql', is_async=True) + + @asynccontextmanager + async def _cursor(self) -> AsyncIterator[Any]: + cursor = await self._connection.native.cursor(buffered=True, raw=False) + try: + yield cursor + except BaseException as error: + try: + await cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(error, (cleanup,)) + raise + else: + await cursor.close() + + async def _read(self, sql: str, params: tuple[Any, ...] = ()) -> list[_Row]: + async with self._cursor() as cursor: + await cursor.execute(sql, params) + return _decode_rows(self._connection.native, await cursor.fetchall()) + + async def _write(self, sql: str, params: tuple[Any, ...], *, ignore_errno: int) -> MySQLError | None: + async with self._cursor() as cursor: + try: + await cursor.execute(sql, params) + except MySQLError as error: + if error.errno != ignore_errno: + raise + return error + return None + + async def _check_writable(self, operation: str, target: str) -> None: + native = self._connection.native + autocommit = not native.in_transaction and await native.get_autocommit() + self._require_ddl(autocommit and not native.in_transaction, operation, target) + + async def ensure_database(self, name: str, *, charset: str = 'utf8mb4') -> None: + quoted = _identifier(name) + charset = _charset_name(charset) + with self._operation('ensure_database', name): + await self._check_writable('ensure_database', name) + rows = await self._read(_DATABASE, (name,)) + if not rows: + error = await self._write( + f'CREATE DATABASE IF NOT EXISTS {quoted} DEFAULT CHARACTER SET {charset}', + (), + ignore_errno=ER_DB_CREATE_EXISTS, + ) + rows = await self._read(_DATABASE, (name,)) + if not rows and error is not None: + raise error + self._check_database(name, charset, rows) + + async def list_databases(self) -> list[str]: + with self._operation('list_databases'): + return [name for (name,) in await self._read(_DATABASES)] + + async def database_exists(self, name: str) -> bool: + _identifier(name) + with self._operation('database_exists', name): + return bool(await self._read(_DATABASE, (name,))) + + async def ensure_table(self, database: str, table: str, spec: TableSpec) -> None: + statement, literals = _table_definition(_qualified(database, table), spec) + target = f'{database}.{table}' + params = (database, table) + with self._operation('ensure_table', target): + await self._check_writable('ensure_table', target) + rows = await self._read(_TABLE, params) + if not rows: + error = await self._write(statement, literals, ignore_errno=ER_TABLE_EXISTS_ERROR) + rows = await self._read(_TABLE, params) + if not rows and error is not None: + raise error + self._check_table_kind(target, rows) + actual = _describe(rows[0], await self._read(_COLUMNS, params), await self._read(_PRIMARY_KEY, params)) + self._check_table(target, spec, actual) + + async def list_tables(self, database: str) -> list[str]: + _identifier(database) + with self._operation('list_tables', database): + return [table for (table,) in await self._read(_TABLES, (database,))] + + async def table_exists(self, database: str, table: str) -> bool: + _qualified(database, table) + with self._operation('table_exists', f'{database}.{table}'): + rows = await self._read(_TABLE, (database, table)) + return bool(rows and rows[0][2] == 'BASE TABLE') + + async def describe_table(self, database: str, table: str) -> dict[str, Any] | None: + _qualified(database, table) + params = (database, table) + with self._operation('describe_table', f'{database}.{table}'): + rows = await self._read(_TABLE, params) + if not rows or rows[0][2] != 'BASE TABLE': + return None + return _describe(rows[0], await self._read(_COLUMNS, params), await self._read(_PRIMARY_KEY, params)) + + async def drop_table(self, database: str, table: str) -> None: + qualified = _qualified(database, table) + target = f'{database}.{table}' + with self._operation('drop_table', target): + await self._check_writable('drop_table', target) + await self._write(f'DROP TABLE IF EXISTS {qualified}', (), ignore_errno=ER_BAD_TABLE_ERROR) + + async def drop_database(self, name: str) -> None: + quoted = _identifier(name) + with self._operation('drop_database', name): + await self._check_writable('drop_database', name) + await self._write(f'DROP DATABASE IF EXISTS {quoted}', (), ignore_errno=ER_DB_DROP_EXISTS) diff --git a/tests/test_schema_mysql.py b/tests/test_schema_mysql.py new file mode 100644 index 0000000..ba12dab --- /dev/null +++ b/tests/test_schema_mysql.py @@ -0,0 +1,477 @@ +"""Offline checks for the public MySQL Schema bindings and declarations.""" + +import asyncio +import subprocess +import sys +from datetime import date +from datetime import datetime +from datetime import timezone +from decimal import Decimal +from pathlib import Path +from types import SimpleNamespace +from unittest import IsolatedAsyncioTestCase +from unittest import TestCase +from unittest.mock import AsyncMock +from unittest.mock import Mock +from unittest.mock import call + +from mysql.connector.errorcode import ER_BAD_TABLE_ERROR +from mysql.connector.errorcode import ER_DB_CREATE_EXISTS +from mysql.connector.errorcode import ER_DB_DROP_EXISTS +from mysql.connector.errorcode import ER_TABLE_EXISTS_ERROR +from mysql.connector.errorcode import ER_TABLEACCESS_DENIED_ERROR +from mysql.connector.errors import ProgrammingError + +from database import AsyncConnection +from database import Connection +from database._adapters.mysql import Adapter as ConnectionAdapter +from database.errors import ConfigurationError +from database.errors import DriverError +from database.errors import PermissionDeniedError +from database.schema import AsyncSchema +from database.schema import Column +from database.schema import Schema +from database.schema import SchemaConflictError +from database.schema import TableSpec + +_LITERAL = "quote'; DROP DATABASE victim; --" +_STAMP = datetime(2026, 1, 2, 12, 15, 0, 230000) +_DATE = date(2026, 1, 2) +_BASE = ('Catalog', 'Users', 'BASE TABLE', 'InnoDB', 'utf8mb4', 'utf8mb4_0900_ai_ci') +_VIEW = ('Catalog', 'Users', 'VIEW', None, None, None) +_SPEC = TableSpec( + { + 'tenant': Column('bigint'), + 'id': Column('int'), + 'name': Column('varchar', length=100, default=_LITERAL), + 'balance': Column('decimal', precision=12, scale=2, default=Decimal('32.50')), + 'subscribed': Column('boolean', default=True), + 'score': Column('double', default=1.25), + 'born': Column('date', default=_DATE), + 'created': Column('datetime', precision=3, default=_STAMP), + 'note': Column('text', nullable=True, default=None), + 'payload': Column('blob', nullable=True), + }, + primary_key=('tenant', 'id'), +) +_COLUMNS = [ + ('tenant', 'bigint(20)', 'NO', None, '', ''), + ('id', 'int', 'NO', None, '', ''), + ('Name', 'varchar(100)', 'NO', _LITERAL, '', ''), + ('balance', 'decimal(12,2)', 'NO', '32.50', '', ''), + ('subscribed', 'tinyint(1)', 'NO', '1', '', ''), + ('score', 'double', 'NO', '1.25', '', ''), + ('born', 'date', 'NO', '2026-01-02', '', ''), + ('created', 'datetime(3)', 'NO', '2026-01-02 12:15:00.230', '', ''), + ('note', 'text', 'YES', None, '', ''), + ('payload', 'blob', 'YES', None, '', ''), +] +_KEYS = [('tenant',), ('id',)] + + +def _handle(*, is_async=False): + cursor = Mock() + native = Mock(in_transaction=False, autocommit=True, python_charset='utf8') + adapter = SimpleNamespace(classify=ConnectionAdapter.classify) + if is_async: + cursor.execute = AsyncMock() + cursor.fetchall = AsyncMock() + cursor.close = AsyncMock() + native.cursor = AsyncMock(return_value=cursor) + native.get_autocommit = AsyncMock(return_value=True) + connection = AsyncConnection(native, adapter, 'setup', 'mysql', 'single', asyncio.get_running_loop()) + return AsyncSchema.mysql(connection), native, cursor + native.cursor.return_value = cursor + connection = Connection(native, adapter, 'setup', 'mysql', 'single') + return Schema.mysql(connection), native, cursor + + +def _executed_sql(cursor): + return [entry.args[0].strip() for entry in cursor.execute.call_args_list] + + +class MySQLTableSpecTests(TestCase): + def test_model_import_has_no_driver_dependency(self): + source = str(Path(__file__).resolve().parents[1] / 'src') + code = ( + f'import sys; sys.path.insert(0, {source!r}); ' + 'from database.schema import Column, TableSpec; ' + 'TableSpec({"id": Column("int")}, ("id",)); ' + 'assert "mysql.connector" not in sys.modules' + ) + subprocess.run([sys.executable, '-B', '-S', '-c', code], check=True, capture_output=True) + + def test_accepts_supported_types_and_normalizes_defaults(self): + self.assertEqual(Column('BIGINT', default=-(2**63)).type, 'bigint') + self.assertEqual(Column('decimal', default=42).default, Decimal('42')) + self.assertEqual(Column('decimal', precision=8, scale=2, default=1.25).default, Decimal('1.25')) + self.assertEqual(Column('boolean', default=1).default, True) + self.assertEqual(Column('varchar', length=32, default='你好').default, '你好') + self.assertIsNone(Column('blob', nullable=True, default=None).default) + self.assertNotEqual(Column('text', nullable=True), Column('text', nullable=True, default=None)) + self.assertEqual(Column('datetime', default=_STAMP.replace(microsecond=0)).precision, None) + + def test_rejects_invalid_types_dimensions_and_defaults(self): + cases = [ + ('int unsigned', {}), + ('double); DROP TABLE t', {}), + ('int', {'length': 4}), + ('int', {'nullable': 1}), + ('int', {'default': None}), + ('int', {'default': 2**31}), + ('int', {'default': True}), + ('boolean', {'default': 2}), + ('varchar', {}), + ('varchar', {'length': True}), + ('varchar', {'length': 3, 'default': 'four'}), + ('decimal', {'precision': 66}), + ('decimal', {'precision': 2, 'scale': 3}), + ('decimal', {'precision': 8, 'scale': 2, 'default': Decimal('1.003')}), + ('decimal', {'precision': 8, 'scale': 2, 'default': Decimal('NaN')}), + ('double', {'default': float('inf')}), + ('date', {'default': _STAMP}), + ('datetime', {'precision': 7}), + ('datetime', {'precision': 1, 'default': _STAMP}), + ('datetime', {'precision': 6, 'default': _STAMP.replace(tzinfo=timezone.utc)}), + ('datetime', {'default': 'NOW()'}), + ('text', {'default': 'ordinary string'}), + ('blob', {'default': b'binary'}), + ('bigint', {'auto_increment': True, 'default': 0}), + ('boolean', {'auto_increment': True}), + ] + for kind, options in cases: + with self.subTest(kind=kind, options=options), self.assertRaises(ConfigurationError): + Column(kind, **options) + + def test_table_snapshots_columns_and_ordered_case_insensitive_key(self): + columns = {'tenant': Column('int'), 'Id': Column('bigint')} + key = ['ID', 'tenant'] + spec = TableSpec(columns, key) + columns.clear() + key.reverse() + self.assertEqual(list(spec.columns), ['tenant', 'Id']) + self.assertEqual(spec.primary_key, ('Id', 'tenant')) + with self.assertRaises(TypeError): + spec.columns['id'] = Column('int') + + def test_rejects_invalid_keys_columns_and_auto_increment(self): + identity = Column('bigint', auto_increment=True) + value = Column('int') + cases = [ + ({}, ('id',)), + ({'id': object()}, ('id',)), + ({'id': value}, 'id'), + ({'id': value}, ()), + ({'id': value}, ('id', 'ID')), + ({'id': value}, ('missing',)), + ({'id;DROP': value}, ('id;DROP',)), + ({'id': value, 'ID': value}, ('id',)), + ({'id': Column('int', nullable=True)}, ('id',)), + ({'id': Column('text')}, ('id',)), + ({'id': identity, 'tenant': value}, ('id', 'tenant')), + ({'id': identity, 'tenant': value}, ('tenant',)), + ] + for columns, key in cases: + with self.subTest(columns=columns, key=key), self.assertRaises(ConfigurationError): + TableSpec(columns, key) + self.assertEqual(TableSpec({'id': identity}, ('id',)).columns['id'], identity) + + +class SyncMySQLSchemaTests(TestCase): + def setUp(self): + self.schema, self.native, self.cursor = _handle() + + def test_create_database_rechecks_visibility_and_charset(self): + self.native.cursor.assert_not_called() + self.cursor.fetchall.side_effect = [[], [('utf8mb4',)], [('utf8mb4',)]] + self.assertIsNone(self.schema.ensure_database('Catalog')) + self.assertIsNone(self.schema.ensure_database('Catalog', charset='UTF8MB4')) + + qualified = chr(96) + 'Catalog' + chr(96) + self.assertEqual( + self.cursor.execute.call_args_list[1], + call(f'CREATE DATABASE IF NOT EXISTS {qualified} DEFAULT CHARACTER SET utf8mb4', ()), + ) + self.assertEqual(sum(sql.startswith('CREATE DATABASE') for sql in _executed_sql(self.cursor)), 1) + self.assertEqual(self.cursor.close.call_count, 4) + self.native.close.assert_not_called() + + def test_database_charset_conflict_is_read_only(self): + self.cursor.fetchall.return_value = [('latin1',)] + with self.assertRaises(SchemaConflictError) as caught: + self.schema.ensure_database('Catalog') + self.assertEqual(caught.exception.differences, ('Database has character set latin1, expected utf8mb4.',)) + self.assertEqual((caught.exception.connection_name, caught.exception.target), ('setup', 'Catalog')) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_create_table_binds_literals_and_checks_compatible_shape(self): + self.cursor.fetchall.side_effect = [[], [_BASE], _COLUMNS, _KEYS] + self.schema.ensure_table('Catalog', 'Users', _SPEC) + create = self.cursor.execute.call_args_list[1].args + quote = chr(96) + self.assertTrue(create[0].startswith(f'CREATE TABLE IF NOT EXISTS {quote}Catalog{quote}.{quote}Users{quote}')) + self.assertIn(f'{quote}name{quote} VARCHAR(100) NOT NULL DEFAULT %s', create[0]) + self.assertIn(f'{quote}subscribed{quote} BOOLEAN NOT NULL DEFAULT %s', create[0]) + self.assertIn(f'{quote}note{quote} TEXT NULL DEFAULT NULL', create[0]) + self.assertIn(f'PRIMARY KEY ({quote}tenant{quote}, {quote}id{quote})', create[0]) + self.assertTrue(create[0].endswith('ENGINE=InnoDB')) + self.assertNotIn(_LITERAL, create[0]) + self.assertEqual(create[1], (_LITERAL, Decimal('32.50'), True, 1.25, _DATE, _STAMP)) + self.assertEqual(self.cursor.close.call_count, 5) + self.native.close.assert_not_called() + + def test_compatible_table_allows_omittable_extra_columns_and_preserves_engine(self): + columns = _COLUMNS + [ + ('optional', 'int', 'YES', None, '', ''), + ('defaulted', 'int', 'NO', '(2 + 2)', 'DEFAULT_GENERATED', ''), + ('derived', 'int', 'NO', None, 'STORED GENERATED', 'tenant + id'), + ] + other_engine = (*_BASE[:3], 'MyISAM', *_BASE[4:]) + self.cursor.fetchall.side_effect = [[other_engine], columns, _KEYS] + self.schema.ensure_table('Catalog', 'Users', _SPEC) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_conflicting_columns_default_expression_and_key_order_report_differences(self): + columns = [ + ('tenant', 'bigint unsigned', 'NO', None, '', ''), + ('id', 'int', 'NO', None, 'auto_increment', ''), + ('Name', 'varchar(100)', 'YES', _LITERAL, 'DEFAULT_GENERATED', ''), + ('balance', 'decimal(13,2)', 'NO', '32.60', '', ''), + ('required', 'int', 'NO', None, '', ''), + *_COLUMNS[4:9], + ] + self.cursor.fetchall.side_effect = [[_BASE], columns, list(reversed(_KEYS))] + with self.assertRaises(SchemaConflictError) as caught: + self.schema.ensure_table('Catalog', 'Users', _SPEC) + self.assertEqual(caught.exception.operation, 'ensure_table') + self.assertEqual(caught.exception.target, 'Catalog.Users') + self.assertEqual(len(caught.exception.differences), 9) + for word in ('auto-increment', 'default', 'Extra column', 'Missing column', 'nullability', 'Primary key'): + self.assertTrue(any(word in item for item in caught.exception.differences), caught.exception.differences) + self.assertNotIn(_LITERAL, ' '.join(caught.exception.differences)) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_database_and_table_catalog_read_only(self): + self.cursor.fetchall.side_effect = [ + [(b'Catalog',), ('system',)], + [('utf8mb4',)], + [], + [('Users',)], + [_BASE], + [_VIEW], + [_BASE], + _COLUMNS, + _KEYS, + [], + ] + self.assertEqual(self.schema.list_databases(), ['Catalog', 'system']) + self.assertTrue(self.schema.database_exists('Catalog')) + self.assertFalse(self.schema.database_exists('Missing')) + self.assertEqual(self.schema.list_tables('Catalog'), ['Users']) + self.assertTrue(self.schema.table_exists('Catalog', 'Users')) + self.assertFalse(self.schema.table_exists('Catalog', 'View')) + record = self.schema.describe_table('Catalog', 'Users') + self.assertEqual(record['table'], 'Users') + self.assertEqual(record['database'], 'Catalog') + self.assertEqual(record['primary_key'], ('tenant', 'id')) + self.assertEqual(record['charset'], 'utf8mb4') + self.assertEqual(record['collation'], 'utf8mb4_0900_ai_ci') + self.assertEqual(record['engine'], 'InnoDB') + self.assertEqual(record['columns']['Name']['default'], _LITERAL) + self.assertEqual(record['columns']['balance']['type'], 'decimal(12,2)') + self.assertIsNone(self.schema.describe_table('Catalog', 'Missing')) + self.assertIn("TABLE_TYPE = 'BASE TABLE'", _executed_sql(self.cursor)[3]) + self.assertEqual(self.cursor.execute.call_args_list[6].args[1], ('Catalog', 'Users')) + self.native.close.assert_not_called() + + def test_existing_view_is_a_conflict_and_not_described_as_table(self): + self.cursor.fetchall.return_value = [_VIEW] + with self.assertRaises(SchemaConflictError) as caught: + self.schema.ensure_table('Catalog', 'Users', _SPEC) + self.assertEqual(caught.exception.differences, ('Existing object is not a base table.',)) + self.assertIsNone(self.schema.describe_table('Catalog', 'Users')) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_concurrent_database_creation_rechecks_winning_charset(self): + self.cursor.execute.side_effect = [None, ProgrammingError(errno=ER_DB_CREATE_EXISTS), None] + self.cursor.fetchall.side_effect = [[], [('latin1',)]] + with self.assertRaises(SchemaConflictError): + self.schema.ensure_database('Catalog') + self.assertEqual(self.cursor.close.call_count, 3) + + def test_drop_is_explicit_idempotent_and_qualified(self): + self.cursor.execute.side_effect = [ + None, + ProgrammingError(errno=ER_BAD_TABLE_ERROR), + None, + ProgrammingError(errno=ER_DB_DROP_EXISTS), + ] + for _ in range(2): + self.schema.drop_table('Catalog', 'Users') + for _ in range(2): + self.schema.drop_database('Catalog') + + quote = chr(96) + drop_table = f'DROP TABLE IF EXISTS {quote}Catalog{quote}.{quote}Users{quote}' + drop_database = f'DROP DATABASE IF EXISTS {quote}Catalog{quote}' + self.assertEqual(_executed_sql(self.cursor), [drop_table, drop_table, drop_database, drop_database]) + self.cursor.fetchall.assert_not_called() + self.native.close.assert_not_called() + + def test_database_and_table_ddl_refuse_active_transactions_or_disabled_autocommit(self): + for in_transaction, autocommit in [(True, True), (False, False)]: + self.native.in_transaction = in_transaction + self.native.autocommit = autocommit + for operation, args in [ + ('ensure_database', ('Catalog',)), + ('ensure_table', ('Catalog', 'Users', _SPEC)), + ('drop_table', ('Catalog', 'Users')), + ('drop_database', ('Catalog',)), + ]: + with self.subTest(operation=operation), self.assertRaises(ConfigurationError) as caught: + getattr(self.schema, operation)(*args) + self.assertEqual(caught.exception.operation, operation) + self.assertTrue(caught.exception.target.startswith('Catalog')) + self.assertEqual(self.native.in_transaction, in_transaction) + self.assertEqual(self.native.autocommit, autocommit) + self.native.cursor.assert_not_called() + self.native.commit.assert_not_called() + self.native.rollback.assert_not_called() + + def test_invalid_names_and_non_table_spec_only_fail_locally(self): + for operation, args, kwargs in [ + ('ensure_database', ('evil;DROP',), {}), + ('ensure_database', ('app',), {'charset': 'utf8mb4 /* unsafe */'}), + ('database_exists', ('db dot',), {}), + ('ensure_table', ('app', 'safe', object()), {}), + ('drop_table', ('app', 'dangerous' + chr(96)), {}), + ]: + with self.subTest(operation=operation), self.assertRaises(ConfigurationError): + getattr(self.schema, operation)(*args, **kwargs) + self.native.cursor.assert_not_called() + + def test_metadata_permission_failure_and_cursor_close_failure_are_preserved(self): + original = ProgrammingError(errno=ER_TABLEACCESS_DENIED_ERROR) + failure = OSError('cursor close failed') + self.cursor.execute.side_effect = original + self.cursor.close.side_effect = failure + with self.assertRaises(PermissionDeniedError) as caught: + self.schema.table_exists('Catalog', 'Users') + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.target, 'Catalog.Users') + self.assertEqual(caught.exception.database_cleanup_errors, (failure,)) + self.assertEqual(original.database_cleanup_errors, (failure,)) + self.native.close.assert_not_called() + + +class AsyncMySQLSchemaTests(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.schema, self.native, self.cursor = _handle(is_async=True) + + async def test_async_create_database_and_table_and_recheck(self): + self.native.cursor.assert_not_awaited() + self.cursor.fetchall.side_effect = [[], [('utf8mb4',)], [], [_BASE], _COLUMNS, _KEYS] + self.assertIsNone(await self.schema.ensure_database('Catalog')) + self.assertIsNone(await self.schema.ensure_table('Catalog', 'Users', _SPEC)) + self.assertEqual(self.native.get_autocommit.await_count, 2) + self.assertEqual(self.cursor.close.await_count, 8) + self.assertEqual(sum(sql.startswith('CREATE') for sql in _executed_sql(self.cursor)), 2) + self.assertEqual( + self.cursor.execute.await_args_list[4].args[1], + (_LITERAL, Decimal('32.50'), True, 1.25, _DATE, _STAMP), + ) + self.native.close.assert_not_called() + + async def test_async_catalog_and_drop(self): + self.cursor.fetchall.side_effect = [ + [('Catalog',)], + [('utf8mb4',)], + [('Users',)], + [_BASE], + [_BASE], + _COLUMNS, + _KEYS, + [], + ] + self.assertEqual(await self.schema.list_databases(), ['Catalog']) + self.assertTrue(await self.schema.database_exists('Catalog')) + self.assertEqual(await self.schema.list_tables('Catalog'), ['Users']) + self.assertTrue(await self.schema.table_exists('Catalog', 'Users')) + record = await self.schema.describe_table('Catalog', 'Users') + self.assertEqual(record['primary_key'], ('tenant', 'id')) + self.assertIsNone(await self.schema.describe_table('Catalog', 'Missing')) + self.native.get_autocommit.assert_not_awaited() + + self.cursor.execute.side_effect = [ + ProgrammingError(errno=ER_BAD_TABLE_ERROR), + ProgrammingError(errno=ER_DB_DROP_EXISTS), + ] + await self.schema.drop_table('Catalog', 'Users') + await self.schema.drop_database('Catalog') + self.assertEqual(self.cursor.close.await_count, 10) + self.native.close.assert_not_called() + + async def test_async_transaction_guard_uses_async_method(self): + self.native.in_transaction = True + with self.assertRaises(ConfigurationError): + await self.schema.ensure_database('Catalog') + self.native.get_autocommit.assert_not_awaited() + self.native.in_transaction = False + self.native.get_autocommit.return_value = False + with self.assertRaises(ConfigurationError) as caught: + await self.schema.drop_table('Catalog', 'Users') + self.assertEqual(caught.exception.operation, 'drop_table') + self.native.get_autocommit.assert_awaited_once_with() + self.native.cursor.assert_not_awaited() + self.native.commit.assert_not_called() + self.native.rollback.assert_not_called() + + async def test_async_transaction_state_is_checked_after_autocommit_query(self): + def get_autocommit(): + self.native.in_transaction = True + return True + + self.native.get_autocommit.side_effect = get_autocommit + with self.assertRaises(ConfigurationError): + await self.schema.drop_database('Catalog') + + self.native.cursor.assert_not_awaited() + + async def test_async_concurrent_create_checks_final_matching_table(self): + self.cursor.execute.side_effect = [None, ProgrammingError(errno=ER_TABLE_EXISTS_ERROR), None, None, None] + self.cursor.fetchall.side_effect = [[], [_BASE], _COLUMNS, _KEYS] + self.assertIsNone(await self.schema.ensure_table('Catalog', 'Users', _SPEC)) + self.assertEqual(self.cursor.close.await_count, 5) + + async def test_async_hidden_concurrent_table_does_not_report_success(self): + original = ProgrammingError(errno=ER_TABLE_EXISTS_ERROR) + self.cursor.execute.side_effect = [None, original, None] + self.cursor.fetchall.side_effect = [[], []] + with self.assertRaises(DriverError) as caught: + await self.schema.ensure_table('Catalog', 'Users', _SPEC) + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(self.cursor.close.await_count, 3) + + async def test_async_drop_does_not_ignore_permission_error(self): + original = ProgrammingError(errno=ER_TABLEACCESS_DENIED_ERROR) + failure = RuntimeError('cursor close failed') + self.cursor.execute.side_effect = original + self.cursor.close.side_effect = failure + with self.assertRaises(PermissionDeniedError) as caught: + await self.schema.drop_table('Catalog', 'Users') + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.database_cleanup_errors, (failure,)) + self.cursor.close.assert_awaited_once_with() + self.native.close.assert_not_called() + + async def test_async_cancelled_metadata_closes_cursor_and_propagates_cancellation(self): + cancellation = asyncio.CancelledError('cancelled') + self.cursor.fetchall.side_effect = cancellation + with self.assertRaises(asyncio.CancelledError) as caught: + await self.schema.list_tables('Catalog') + self.assertIs(caught.exception, cancellation) + self.cursor.close.assert_awaited_once_with() + self.native.close.assert_not_called() -- Gitee From 13091647d28093d12f1557239d744ea80b71d6df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:40:53 +0800 Subject: [PATCH 71/86] =?UTF-8?q?test(crud):=20=E8=A1=A5=E5=85=85=E5=AE=8C?= =?UTF-8?q?=E6=95=B4=E4=BD=BF=E7=94=A8=E7=A4=BA=E4=BE=8B=E4=B8=8E=E9=9A=94?= =?UTF-8?q?=E7=A6=BB=E6=9C=8D=E5=8A=A1=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/basic_usage.py | 296 +++++++++++++++++++++++++++++ scripts/verify_basics.py | 353 +++++++++++++++++++++++++++++++++++ tests/test_basic_verifier.py | 81 ++++++++ tests/test_schema_binding.py | 3 +- 4 files changed, 732 insertions(+), 1 deletion(-) create mode 100644 examples/basic_usage.py create mode 100644 scripts/verify_basics.py create mode 100644 tests/test_basic_verifier.py diff --git a/examples/basic_usage.py b/examples/basic_usage.py new file mode 100644 index 0000000..ff431fe --- /dev/null +++ b/examples/basic_usage.py @@ -0,0 +1,296 @@ +"""Complete explicit sync/async usage; supply credentials from the calling application.""" + +from contextlib import AsyncExitStack +from contextlib import ExitStack + +from database import ConnectionConfig +from database import Connections +from database.crud import AsyncRecords +from database.crud import AsyncRedisStrings +from database.crud import JsonFieldCodec +from database.crud import Records +from database.crud import RedisStrings +from database.crud import StringSetItem +from database.schema import AsyncSchema +from database.schema import Column +from database.schema import Schema +from database.schema import TableSpec + +USER_TABLE = TableSpec( + columns={ + 'id': Column('varchar', length=32), + 'name': Column('varchar', length=100), + 'age': Column('int'), + }, + primary_key=('id',), +) + + +def make_connections( + mysql_password: str, + mongodb_uri: str, + redis_password: str | None = None, +) -> Connections: + return Connections( + { + # 初始化时不能选择一个尚未创建的数据库。 + 'mysql_setup': ConnectionConfig( + backend='mysql', + host='127.0.0.1', + username='sdk_demo', + password=mysql_password, + options={'autocommit': True}, + ), + 'mysql_app': ConnectionConfig( + backend='mysql', + host='127.0.0.1', + username='sdk_demo', + password=mysql_password, + options={ + 'database': 'demo_app', + 'autocommit': True, + }, + ), + 'documents': ConnectionConfig( + backend='mongodb', + host=mongodb_uri, + ), + 'cache': ConnectionConfig( + backend='redis', + host='127.0.0.1', + password=redis_password, + options={'db': 0, 'decode_responses': True}, + ), + } + ) + + +def run_sync(connections: Connections) -> None: + # 一、显式准备 MySQL 数据库与表。 + with connections.create('mysql_setup') as setup: + schema = Schema.mysql(setup) + schema.ensure_database('demo_app') + schema.ensure_table('demo_app', 'users', USER_TABLE) + + # 二、三个数据库连接在同一业务作用域内同时使用。 + with ExitStack() as stack: + mysql = stack.enter_context(connections.create('mysql_app')) + mongo = stack.enter_context(connections.create('documents')) + redis = stack.enter_context(connections.create('cache')) + + for connection in (mysql, mongo, redis): + connection.check() + + # 三、显式准备 MongoDB 普通集合。 + Schema.mongodb(mongo).ensure_collection( + 'demo_app', + 'users', + ) + + mysql_users = stack.enter_context( + Records.mysql( + mysql, + table='users', + key_fields=USER_TABLE.primary_key, + ) + ) + mongo_users = stack.enter_context( + Records.mongodb( + mongo, + database='demo_app', + collection='users', + ) + ) + + # Redis 不建表;绑定记录命名空间及 codec 即可开始使用。 + redis_users = stack.enter_context( + Records.redis_hash( + redis, + namespace='demo_app:users', + key_fields=('id',), + codec=JsonFieldCodec(), + ) + ) + + # 四、分别对三个后端执行相同的基础记录流程。 + for users, key_field in ( + (mysql_users, 'id'), + (mongo_users, '_id'), + (redis_users, 'id'), + ): + key = {key_field: 'u1'} + + users.insert({**key, 'name': 'Alice', 'age': 30}) + print(users.get(key, fields=['name', 'age'])) + users.update(key, set_values={'age': 31}) + + users.insert_many( + [ + {key_field: 'u2', 'name': 'Bob', 'age': 20}, + {key_field: 'u3', 'name': 'Carol', 'age': 40}, + ] + ) + users.bulk_update( + [ + { + 'key': {key_field: 'u2'}, + 'set_values': {'age': 21}, + }, + { + 'key': {key_field: 'u3'}, + 'set_values': {'age': 41}, + }, + ] + ) + print( + users.get_many( + [{key_field: 'u2'}, {key_field: 'u3'}], + fields=['name', 'age'], + ) + ) + + users.delete(key) + users.delete_many( + [ + {key_field: 'u2'}, + {key_field: 'u3'}, + ] + ) + + # 五、Redis String 键值操作。 + strings = stack.enter_context(RedisStrings(redis)) + + strings.set('demo:message', 'hello', ttl_s=60) + print(strings.get('demo:message')) + strings.set('demo:message', 'updated', ttl_s=120) + print(strings.ttl('demo:message')) + strings.delete('demo:message') + + strings.set_many( + [ + StringSetItem('demo:a', 'A', ttl_s=60), + StringSetItem('demo:b', 'B', ttl_s=60), + ] + ) + print(strings.get_many(['demo:a', 'demo:b', 'demo:missing'])) + strings.delete_many(['demo:a', 'demo:b']) + + # 上下文退出:先释放操作对象自有资源,再关闭连接。 + # 不自动删除数据库、表或集合。 + + +async def run_async(connections: Connections) -> None: + # 一、异步准备 MySQL 数据库与表。 + async with await connections.create_async('mysql_setup') as setup: + schema = AsyncSchema.mysql(setup) + await schema.ensure_database('demo_app') + await schema.ensure_table('demo_app', 'users', USER_TABLE) + + # 二、三个异步连接同时存在,分别管理各自资源。 + async with AsyncExitStack() as stack: + mysql = await stack.enter_async_context(await connections.create_async('mysql_app')) + mongo = await stack.enter_async_context(await connections.create_async('documents')) + redis = await stack.enter_async_context(await connections.create_async('cache')) + + for connection in (mysql, mongo, redis): + await connection.check() + + # 三、异步准备 MongoDB 集合。 + await AsyncSchema.mongodb(mongo).ensure_collection( + 'demo_app', + 'users', + ) + + mysql_users = await stack.enter_async_context( + AsyncRecords.mysql( + mysql, + table='users', + key_fields=USER_TABLE.primary_key, + ) + ) + mongo_users = await stack.enter_async_context( + AsyncRecords.mongodb( + mongo, + database='demo_app', + collection='users', + ) + ) + redis_users = await stack.enter_async_context( + AsyncRecords.redis_hash( + redis, + namespace='demo_app:users', + key_fields=('id',), + codec=JsonFieldCodec(), + ) + ) + + # 四、三个后端分别执行异步单条与批量 CRUD。 + for users, key_field in ( + (mysql_users, 'id'), + (mongo_users, '_id'), + (redis_users, 'id'), + ): + key = {key_field: 'u1'} + + await users.insert({**key, 'name': 'Alice', 'age': 30}) + print(await users.get(key, fields=['name', 'age'])) + await users.update(key, set_values={'age': 31}) + + await users.insert_many( + [ + {key_field: 'u2', 'name': 'Bob', 'age': 20}, + {key_field: 'u3', 'name': 'Carol', 'age': 40}, + ] + ) + await users.bulk_update( + [ + { + 'key': {key_field: 'u2'}, + 'set_values': {'age': 21}, + }, + { + 'key': {key_field: 'u3'}, + 'set_values': {'age': 41}, + }, + ] + ) + print( + await users.get_many( + [{key_field: 'u2'}, {key_field: 'u3'}], + fields=['name', 'age'], + ) + ) + + await users.delete(key) + await users.delete_many( + [ + {key_field: 'u2'}, + {key_field: 'u3'}, + ] + ) + + # 五、Redis String 异步操作。 + strings = await stack.enter_async_context(AsyncRedisStrings(redis)) + + await strings.set('demo:message', 'hello', ttl_s=60) + print(await strings.get('demo:message')) + await strings.set('demo:message', 'updated', ttl_s=120) + print(await strings.ttl('demo:message')) + await strings.delete('demo:message') + + await strings.set_many( + [ + StringSetItem('demo:a', 'A', ttl_s=60), + StringSetItem('demo:b', 'B', ttl_s=60), + ] + ) + print( + await strings.get_many( + [ + 'demo:a', + 'demo:b', + 'demo:missing', + ] + ) + ) + await strings.delete_many(['demo:a', 'demo:b']) diff --git a/scripts/verify_basics.py b/scripts/verify_basics.py new file mode 100644 index 0000000..83c6f93 --- /dev/null +++ b/scripts/verify_basics.py @@ -0,0 +1,353 @@ +"""Verify basic initialization and CRUD against explicit isolated fixtures. + +Run from the repository root with --config and optionally --mode sync/async. +The fixture is a mapping of names to ConnectionConfig keyword arguments. +MySQL fixtures must omit database/db and explicitly enable autocommit. +This script creates and drops randomly named test databases/keys. Use isolated +services only. It never loads environment variables or prints credentials. +""" + +import argparse +import asyncio +import json +from collections.abc import AsyncIterator +from collections.abc import Awaitable +from collections.abc import Callable +from collections.abc import Iterator +from collections.abc import Mapping +from contextlib import AsyncExitStack +from contextlib import ExitStack +from contextlib import asynccontextmanager +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path +from traceback import extract_tb +from typing import Any +from uuid import uuid4 + +from database import ConnectionConfig +from database import Connections +from database.crud import AsyncRecords +from database.crud import AsyncRedisStrings +from database.crud import JsonFieldCodec +from database.crud import Records +from database.crud import RedisStrings +from database.crud import StringSetItem +from database.errors import attach_cleanup_errors +from database.schema import AsyncSchema +from database.schema import Column +from database.schema import Schema +from database.schema import TableSpec + +TABLE = TableSpec( + columns={'id': Column('varchar', length=32), 'name': Column('varchar', length=100), 'age': Column('int')}, + primary_key=('id',), +) + + +@contextmanager +def cleaning(cleanup: Callable[[], Any]) -> Iterator[None]: + """Always attempt test-object cleanup without masking the initial failure.""" + try: + yield + except BaseException as primary: + try: + cleanup() + except BaseException as secondary: + attach_cleanup_errors(primary, (secondary,)) + raise + else: + cleanup() + + +@asynccontextmanager +async def cleaning_async(cleanup: Callable[[], Awaitable[Any]]) -> AsyncIterator[None]: + try: + yield + except BaseException as primary: + try: + await cleanup() + except BaseException as secondary: + attach_cleanup_errors(primary, (secondary,)) + raise + else: + await cleanup() + + +def record_cycle(records: Any) -> None: + keys = [{'id': key} for key in ('one', 'two', 'three')] + with cleaning(lambda: records.delete_many(keys)): + _record_cycle_steps(records) + + +async def record_cycle_async(records: Any) -> None: + keys = [{'id': key} for key in ('one', 'two', 'three')] + async with cleaning_async(lambda: records.delete_many(keys)): + await _record_cycle_steps_async(records) + + +def _record_cycle_steps(records: Any) -> None: + records.insert({'id': 'one', 'name': 'Alice', 'age': 30}) + assert records.get({'id': 'one'}, fields=['name']) == {'name': 'Alice'} + assert records.get({'id': 'one'}, fields=[]) == {} + assert records.get({'id': 'missing'}) is None + records.update({'id': 'one'}, set_values={'age': 31}) + records.insert_many([{'id': 'two', 'name': 'Bob', 'age': 20}, {'id': 'three', 'name': 'Carol', 'age': 40}]) + records.bulk_update( + [ + {'key': {'id': 'two'}, 'set_values': {'age': 21}}, + {'key': {'id': 'three'}, 'set_values': {'age': 41}}, + ] + ) + result = records.get_many([{'id': 'three'}, {'id': 'one'}, {'id': 'three'}, {'id': 'two'}], fields=['age']) + assert [item.record for item in result] == [{'age': 41}, {'age': 31}, {'age': 41}, {'age': 21}] + records.delete({'id': 'one'}) + records.delete_many([{'id': 'two'}, {'id': 'three'}]) + assert all(records.get({'id': key}) is None for key in ('one', 'two', 'three')) + + +async def _record_cycle_steps_async(records: Any) -> None: + await records.insert({'id': 'one', 'name': 'Alice', 'age': 30}) + assert await records.get({'id': 'one'}, fields=['name']) == {'name': 'Alice'} + assert await records.get({'id': 'one'}, fields=[]) == {} + assert await records.get({'id': 'missing'}) is None + await records.update({'id': 'one'}, set_values={'age': 31}) + await records.insert_many([{'id': 'two', 'name': 'Bob', 'age': 20}, {'id': 'three', 'name': 'Carol', 'age': 40}]) + await records.bulk_update( + [ + {'key': {'id': 'two'}, 'set_values': {'age': 21}}, + {'key': {'id': 'three'}, 'set_values': {'age': 41}}, + ] + ) + result = await records.get_many([{'id': 'three'}, {'id': 'one'}, {'id': 'three'}, {'id': 'two'}], fields=['age']) + assert [item.record for item in result] == [{'age': 41}, {'age': 31}, {'age': 41}, {'age': 21}] + await records.delete({'id': 'one'}) + await records.delete_many([{'id': 'two'}, {'id': 'three'}]) + for key in ('one', 'two', 'three'): + assert await records.get({'id': key}) is None + + +def strings_cycle(connection: Any, prefix: str) -> None: + keys = [prefix + ':a', prefix + ':b', prefix + ':empty'] + with RedisStrings(connection) as strings: + with cleaning(lambda: strings.delete_many(keys)): + strings.set(keys[0], 'hello', ttl_s=60) + assert strings.exists(keys[0]) and 0 <= strings.ttl(keys[0]) <= 60 + strings.set(keys[0], 'updated') + assert strings.ttl(keys[0]) == -1 + assert strings.expire(keys[0], 60) + assert strings.persist(keys[0]) + strings.set(keys[2], '') + assert strings.get(keys[2]) in ('', b'') + strings.set_many([StringSetItem(keys[0], 'A', 60), StringSetItem(keys[1], b'B', 60)]) + values = strings.get_many([keys[1], keys[0], keys[1], prefix + ':missing']) + assert values in ([b'B', b'A', b'B', None], ['B', 'A', 'B', None]) + assert all(strings.ttl(key) == -2 for key in keys) + + +async def strings_cycle_async(connection: Any, prefix: str) -> None: + keys = [prefix + ':a', prefix + ':b', prefix + ':empty'] + async with AsyncRedisStrings(connection) as strings: + async with cleaning_async(lambda: strings.delete_many(keys)): + await strings.set(keys[0], 'hello', ttl_s=60) + assert await strings.exists(keys[0]) + assert 0 <= await strings.ttl(keys[0]) <= 60 + await strings.set(keys[0], 'updated') + assert await strings.ttl(keys[0]) == -1 + assert await strings.expire(keys[0], 60) + assert await strings.persist(keys[0]) + await strings.set(keys[2], '') + assert await strings.get(keys[2]) in ('', b'') + await strings.set_many([StringSetItem(keys[0], 'A', 60), StringSetItem(keys[1], b'B', 60)]) + values = await strings.get_many([keys[1], keys[0], keys[1], prefix + ':missing']) + assert values in ([b'B', b'A', b'B', None], ['B', 'A', 'B', None]) + for key in keys: + assert await strings.ttl(key) == -2 + + +def verify_sync(fixtures: Mapping[str, ConnectionConfig]) -> None: + connections = Connections(fixtures) + with ExitStack() as stack: + handles = {} + for name in fixtures: + first = stack.enter_context(connections.create(name)) + second = stack.enter_context(connections.create(name)) + assert first.native is not second.native + first.check() + second.check() + first.close() + second.check() + handles[name] = second + for name, config in fixtures.items(): + token = 'sdk_basic_' + uuid4().hex + connection = handles[name] + backend = config.backend.value + if backend == 'mysql': + schema = Schema.mysql(connection) + with cleaning(lambda: schema.drop_database(token)): + schema.ensure_database(token) + schema.ensure_table(token, 'users', TABLE) + schema.ensure_table(token, 'users', TABLE) + assert schema.database_exists(token) and token in schema.list_databases() + assert schema.table_exists(token, 'users') and 'users' in schema.list_tables(token) + assert schema.describe_table(token, 'users') is not None + runtime = Connections({'data': replace(config, options={**config.options, 'database': token})}) + with ( + runtime.create('data') as data, + Records.mysql(data, table='users', key_fields=('id',)) as records, + ): + with data.native.cursor() as selected: + selected.execute('SELECT DATABASE()') + assert selected.fetchone()[0] in (token, token.encode('ascii')) + record_cycle(records) + with data.native.cursor() as cursor: + cursor.execute('SELECT 1') + assert cursor.fetchone()[0] == 1 + schema.drop_table(token, 'users') + schema.drop_table(token, 'users') + assert not schema.table_exists(token, 'users') + schema.drop_database(token) + elif backend == 'mongodb': + schema = Schema.mongodb(connection) + with cleaning(lambda: schema.drop_database(token)): + schema.ensure_collection(token, 'users', key_fields=('id',)) + schema.ensure_collection(token, 'users', key_fields=('id',)) + assert schema.database_exists(token) and token in schema.list_databases() + assert schema.collection_exists(token, 'users') and 'users' in schema.list_collections(token) + assert schema.describe_collection(token, 'users') is not None + with Records.mongodb(connection, database=token, collection='users', key_fields=('id',)) as records: + record_cycle(records) + assert connection.native[token].command('ping')['ok'] == 1 + schema.drop_collection(token, 'users') + schema.drop_collection(token, 'users') + assert not schema.collection_exists(token, 'users') + schema.drop_database(token) + else: + with Records.redis_hash( + connection, namespace=token, key_fields=('id',), codec=JsonFieldCodec() + ) as records: + record_cycle(records) + strings_cycle(connection, token) + print(json.dumps({'connection': name, 'backend': backend, 'mode': 'sync', 'result': 'passed'})) + for connection in handles.values(): + connection.check() + print(json.dumps({'mode': 'sync', 'simultaneous_connections': len(handles), 'result': 'passed'})) + + +async def verify_async(fixtures: Mapping[str, ConnectionConfig]) -> None: + connections = Connections(fixtures) + async with AsyncExitStack() as stack: + handles = {} + for name in fixtures: + first = await stack.enter_async_context(await connections.create_async(name)) + second = await stack.enter_async_context(await connections.create_async(name)) + assert first.native is not second.native + await first.check() + await second.check() + await first.close() + await second.check() + handles[name] = second + for name, config in fixtures.items(): + token = 'sdk_basic_' + uuid4().hex + connection = handles[name] + backend = config.backend.value + if backend == 'mysql': + schema = AsyncSchema.mysql(connection) + async with cleaning_async(lambda: schema.drop_database(token)): + await schema.ensure_database(token) + await schema.ensure_table(token, 'users', TABLE) + await schema.ensure_table(token, 'users', TABLE) + assert await schema.database_exists(token) and token in await schema.list_databases() + assert await schema.table_exists(token, 'users') and 'users' in await schema.list_tables(token) + assert await schema.describe_table(token, 'users') is not None + runtime = Connections({'data': replace(config, options={**config.options, 'database': token})}) + async with await runtime.create_async('data') as data: + async with await data.native.cursor() as selected: + await selected.execute('SELECT DATABASE()') + assert (await selected.fetchone())[0] in (token, token.encode('ascii')) + async with AsyncRecords.mysql(data, table='users', key_fields=('id',)) as records: + await record_cycle_async(records) + async with await data.native.cursor() as cursor: + await cursor.execute('SELECT 1') + assert (await cursor.fetchone())[0] == 1 + await schema.drop_table(token, 'users') + await schema.drop_table(token, 'users') + assert not await schema.table_exists(token, 'users') + await schema.drop_database(token) + elif backend == 'mongodb': + schema = AsyncSchema.mongodb(connection) + async with cleaning_async(lambda: schema.drop_database(token)): + await schema.ensure_collection(token, 'users', key_fields=('id',)) + await schema.ensure_collection(token, 'users', key_fields=('id',)) + assert await schema.database_exists(token) and token in await schema.list_databases() + assert await schema.collection_exists(token, 'users') and 'users' in await schema.list_collections( + token + ) + assert await schema.describe_collection(token, 'users') is not None + async with AsyncRecords.mongodb( + connection, database=token, collection='users', key_fields=('id',) + ) as records: + await record_cycle_async(records) + assert (await connection.native[token].command('ping'))['ok'] == 1 + await schema.drop_collection(token, 'users') + await schema.drop_collection(token, 'users') + assert not await schema.collection_exists(token, 'users') + await schema.drop_database(token) + else: + async with AsyncRecords.redis_hash( + connection, namespace=token, key_fields=('id',), codec=JsonFieldCodec() + ) as records: + await record_cycle_async(records) + await strings_cycle_async(connection, token) + print(json.dumps({'connection': name, 'backend': backend, 'mode': 'async', 'result': 'passed'})) + for connection in handles.values(): + await connection.check() + print(json.dumps({'mode': 'async', 'simultaneous_connections': len(handles), 'result': 'passed'})) + + +def load_fixtures(path: Path) -> dict[str, ConnectionConfig]: + fixtures = { + name: ConnectionConfig(**parameters) + for name, parameters in json.loads(path.read_text(encoding='utf-8-sig')).items() + } + if not fixtures: + raise ValueError('Provide at least one explicit isolated service configuration.') + for config in fixtures.values(): + if config.backend.value == 'mysql': + if 'database' in config.options or 'db' in config.options or config.options.get('autocommit') is not True: + raise ValueError('MySQL fixtures must omit database/db and explicitly set autocommit=True.') + return fixtures + + +def main() -> None: + if not __debug__: + raise RuntimeError('Run verification without Python optimization flags.') + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--config', required=True, type=Path) + parser.add_argument('--mode', choices=('sync', 'async', 'both'), default='both') + args = parser.parse_args() + fixtures = load_fixtures(args.config) + if args.mode in ('sync', 'both'): + verify_sync(fixtures) + if args.mode in ('async', 'both'): + asyncio.run(verify_async(fixtures)) + + +if __name__ == '__main__': + try: + main() + except Exception as error: + trace = extract_tb(error.__traceback__) + print( + json.dumps( + { + 'result': 'failed', + 'error_type': type(error).__name__, + 'operation': getattr(error, 'operation', None), + 'file': Path(trace[-1].filename).name if trace else None, + 'line': trace[-1].lineno if trace else None, + 'cleanup_errors': len(getattr(error, 'database_cleanup_errors', ())), + } + ) + ) + raise SystemExit(1) from None diff --git a/tests/test_basic_verifier.py b/tests/test_basic_verifier.py new file mode 100644 index 0000000..494a6fd --- /dev/null +++ b/tests/test_basic_verifier.py @@ -0,0 +1,81 @@ +"""Safety and failure handling of the explicitly configured integration runner.""" + +import asyncio +import json +import tempfile +import unittest +from pathlib import Path + +from scripts.verify_basics import cleaning +from scripts.verify_basics import cleaning_async +from scripts.verify_basics import load_fixtures + + +class VerificationTests(unittest.TestCase): + def test_cleanup_keeps_primary_object(self): + primary = ValueError('operation') + secondary = OSError('cleanup') + + def fail(): + raise secondary + + with self.assertRaises(ValueError) as raised: + with cleaning(fail): + raise primary + self.assertIs(raised.exception, primary) + self.assertEqual(primary.database_cleanup_errors, (secondary,)) + + def test_cleanup_without_primary_propagates(self): + secondary = OSError('cleanup') + + def fail(): + raise secondary + + with self.assertRaises(OSError) as raised: + with cleaning(fail): + pass + self.assertIs(raised.exception, secondary) + + def test_mysql_fixture_requires_explicit_initialization_connection(self): + with tempfile.TemporaryDirectory() as directory: + fixture = Path(directory) / 'config.json' + for options in ({}, {'autocommit': False}, {'autocommit': True, 'database': 'existing'}): + fixture.write_text(json.dumps({'mysql': {'backend': 'mysql', 'host': '127.0.0.1', 'options': options}})) + with self.assertRaises(ValueError): + load_fixtures(fixture) + fixture.write_text( + json.dumps({'mysql': {'backend': 'mysql', 'host': '127.0.0.1', 'options': {'autocommit': True}}}) + ) + self.assertEqual(load_fixtures(fixture)['mysql'].options, {'autocommit': True}) + + def test_empty_fixture_is_not_a_successful_verification(self): + with tempfile.TemporaryDirectory() as directory: + fixture = Path(directory) / 'config.json' + fixture.write_text('{}') + with self.assertRaises(ValueError): + load_fixtures(fixture) + + +class AsyncVerificationTests(unittest.IsolatedAsyncioTestCase): + async def test_cancellation_survives_cleanup_failure(self): + primary = asyncio.CancelledError() + secondary = OSError('cleanup') + + async def fail(): + raise secondary + + with self.assertRaises(asyncio.CancelledError) as raised: + async with cleaning_async(fail): + raise primary + self.assertIs(raised.exception, primary) + self.assertEqual(primary.database_cleanup_errors, (secondary,)) + + async def test_successful_body_still_attempts_cleanup(self): + calls = [] + + async def cleanup(): + calls.append('closed') + + async with cleaning_async(cleanup): + pass + self.assertEqual(calls, ['closed']) diff --git a/tests/test_schema_binding.py b/tests/test_schema_binding.py index 6bda674..d06ac17 100644 --- a/tests/test_schema_binding.py +++ b/tests/test_schema_binding.py @@ -29,7 +29,8 @@ class SchemaBindingTests(unittest.TestCase): source = str(Path(__file__).resolve().parents[1] / 'src') code = ( f'import sys; sys.path.insert(0, {source!r}); ' - 'import database.schema; from database.crud import StringSetItem; ' + 'from database.schema import Column, TableSpec; ' + 'from database.crud import RedisStrings, AsyncRedisStrings, StringSetItem; ' 'assert not any(k == n or k.startswith(n + ".") ' 'for n in ("pymongo", "mysql.connector", "redis") for k in sys.modules)' ) -- Gitee From c9d0ee79b4b94f1fe94c64db553cb813de76cefe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:43:05 +0800 Subject: [PATCH 72/86] =?UTF-8?q?fix(schema):=20=E6=8B=92=E7=BB=9D?= =?UTF-8?q?=E4=B8=8D=E5=8F=AF=E8=A7=81=E7=9A=84=E5=A3=B0=E6=98=8E=E5=88=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/schema/_mysql.py | 2 ++ tests/test_schema_mysql.py | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/database/schema/_mysql.py b/src/database/schema/_mysql.py index 6cc0699..2ddc653 100644 --- a/src/database/schema/_mysql.py +++ b/src/database/schema/_mysql.py @@ -176,6 +176,8 @@ def _compare_table(spec: TableSpec, record: dict[str, Any]) -> tuple[str, ...]: differences.append(f'Column {name!r} has different nullability.') if actual['generated']: differences.append(f'Column {name!r} is generated.') + if 'invisible' in (actual['extra'] or '').lower().split(): + differences.append(f'Column {name!r} is invisible in default projections.') if actual['auto_increment'] != column.auto_increment: differences.append(f'Column {name!r} has different auto-increment behavior.') if not _compare_default(column, actual): diff --git a/tests/test_schema_mysql.py b/tests/test_schema_mysql.py index ba12dab..71c658e 100644 --- a/tests/test_schema_mysql.py +++ b/tests/test_schema_mysql.py @@ -223,7 +223,7 @@ class SyncMySQLSchemaTests(TestCase): def test_compatible_table_allows_omittable_extra_columns_and_preserves_engine(self): columns = _COLUMNS + [ ('optional', 'int', 'YES', None, '', ''), - ('defaulted', 'int', 'NO', '(2 + 2)', 'DEFAULT_GENERATED', ''), + ('defaulted', 'int', 'NO', '(2 + 2)', 'DEFAULT_GENERATED INVISIBLE', ''), ('derived', 'int', 'NO', None, 'STORED GENERATED', 'tenant + id'), ] other_engine = (*_BASE[:3], 'MyISAM', *_BASE[4:]) @@ -231,13 +231,24 @@ class SyncMySQLSchemaTests(TestCase): self.schema.ensure_table('Catalog', 'Users', _SPEC) self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + def test_declared_invisible_column_conflicts_without_writing(self): + columns = _COLUMNS.copy() + columns[2] = ('Name', 'varchar(100)', 'NO', _LITERAL, 'INVISIBLE', '') + self.cursor.fetchall.side_effect = [[_BASE], columns, _KEYS] + + with self.assertRaises(SchemaConflictError) as caught: + self.schema.ensure_table('Catalog', 'Users', _SPEC) + + self.assertEqual(caught.exception.differences, ("Column 'name' is invisible in default projections.",)) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + def test_conflicting_columns_default_expression_and_key_order_report_differences(self): columns = [ ('tenant', 'bigint unsigned', 'NO', None, '', ''), ('id', 'int', 'NO', None, 'auto_increment', ''), ('Name', 'varchar(100)', 'YES', _LITERAL, 'DEFAULT_GENERATED', ''), ('balance', 'decimal(13,2)', 'NO', '32.60', '', ''), - ('required', 'int', 'NO', None, '', ''), + ('required', 'int', 'NO', None, 'INVISIBLE', ''), *_COLUMNS[4:9], ] self.cursor.fetchall.side_effect = [[_BASE], columns, list(reversed(_KEYS))] @@ -438,6 +449,23 @@ class AsyncMySQLSchemaTests(IsolatedAsyncioTestCase): self.native.cursor.assert_not_awaited() + async def test_async_declared_generated_invisible_column_conflicts_for_both_attributes(self): + spec = TableSpec({'id': Column('int'), 'derived': Column('int', nullable=True)}, ('id',)) + columns = [ + ('id', 'int', 'NO', None, '', ''), + ('derived', 'int', 'YES', None, 'STORED GENERATED INVISIBLE', '(id + 1)'), + ] + self.cursor.fetchall.side_effect = [[_BASE], columns, [('id',)]] + + with self.assertRaises(SchemaConflictError) as caught: + await self.schema.ensure_table('Catalog', 'Users', spec) + + self.assertEqual( + caught.exception.differences, + ("Column 'derived' is generated.", "Column 'derived' is invisible in default projections."), + ) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + async def test_async_concurrent_create_checks_final_matching_table(self): self.cursor.execute.side_effect = [None, ProgrammingError(errno=ER_TABLE_EXISTS_ERROR), None, None, None] self.cursor.fetchall.side_effect = [[], [_BASE], _COLUMNS, _KEYS] -- Gitee From de705d2d6f86c21a40290503a011d31b9167614d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 17:49:34 +0800 Subject: [PATCH 73/86] =?UTF-8?q?docs(crud):=20=E4=BA=A4=E4=BB=98=E5=9F=BA?= =?UTF-8?q?=E7=A1=80=E4=BD=BF=E7=94=A8=E8=AF=B4=E6=98=8E=E4=B8=8E=E5=88=86?= =?UTF-8?q?=E9=98=B6=E6=AE=B5=E9=AA=8C=E6=94=B6=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 3 + README.md | 31 ++++++++ docs/README.md | 10 ++- docs/crud-basic-implementation.md | 124 ++++++++++++++++++++++++++++++ docs/crud-design.md | 22 ++++-- docs/crud-usage.md | 91 +++++++++++++++++++++- tests/README.md | 8 ++ 7 files changed, 276 insertions(+), 13 deletions(-) create mode 100644 docs/crud-basic-implementation.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a74db3..25ea8c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,9 @@ ### 新增 +- 新增同步/异步 Schema 基础管理:MySQL 库表创建、兼容校验、目录查看和删除;MongoDB 集合准备、必要身份约束及目录管理。初始化显式执行,不自动迁移。 +- 新增 `RedisStrings` / `AsyncRedisStrings` 的单键、批量读写、删除和 TTL,保留逐项执行证据、原始错误与取消信号。 +- 新增三数据库完整同步/异步使用示例和显式隔离验收脚本;原生客户端及已有记录接口保持兼容。 - 新增可选的 `database.crud.Records` / `AsyncRecords`,在已有连接上绑定 MySQL 表、MongoDB 集合或 Redis Hash,支持复合键、字段投影、批量读取及分块写入;MySQL/MongoDB 另支持结构化条件查询和集合更新、删除。 - CRUD 写入结果区分已知计数与未知计数;批量失败保留逐项执行状态、部分结果和原始异常,异步取消保持原信号。记录层借用连接,不自动提交、回滚或重试。 - Redis Hash 记录使用显式 namespace、字段 codec、版本标记和单键脚本,支持创建、更新、投影及 TTL;保留原生其他数据结构和客户端能力。 diff --git a/README.md b/README.md index 8dbcc2b..719ad9d 100644 --- a/README.md +++ b/README.md @@ -237,6 +237,37 @@ MongoDB 的检查结果使用 `scope="selected_server"`,`targets` 为空:原 异步创建尚未返回句柄时的资源清理由原生驱动负责。当前受测 Connector/Python 的握手取消存在上游清理限制,详见设计文档。Redis 拓扑已完成驱动级验证,真实服务验证仍待完成。 +## 基础结构准备与键值操作 + +在使用记录 CRUD 前,可以通过 `database.schema.Schema` / `AsyncSchema` 显式准备 MySQL 数据库与基础表、MongoDB 普通集合及必要身份约束。已有结构只做兼容性校验,不自动迁移。Redis 直接绑定 Hash 命名空间,或通过 `RedisStrings` / `AsyncRedisStrings` 使用 String 单键、批量与 TTL 接口。 + +MySQL 的初始化连接应省略尚未创建的业务库并启用 autocommit;创建完毕后另建选定业务库的连接。MongoDB 的集合准备和 Redis 的键空间绑定保留各自概念。上述对象都借用既有连接,原生高级能力仍通过 `connection.native` 使用。 + +下面用 MongoDB 演示最短完整流程,包含配置、集合准备、绑定、增删改查和资源关闭。请传入演示实例地址并使用专用演示集合。 + +```python +from database import ConnectionConfig, Connections +from database.crud import Records +from database.schema import Schema + + +def demo(mongodb_uri): + connections = Connections({ + "documents": ConnectionConfig(backend="mongodb", host=mongodb_uri), + }) + with connections.create("documents") as connection: + Schema.mongodb(connection).ensure_collection("sdk_demo", "users") + with Records.mongodb( + connection, database="sdk_demo", collection="users" + ) as users: + users.insert({"_id": "u1", "name": "Alice"}) + print(users.get({"_id": "u1"}, fields=["name"])) + users.update({"_id": "u1"}, set_values={"name": "Alicia"}) + users.delete({"_id": "u1"}) +``` + +从显式配置到三种数据库同步、异步 CRUD 的完整代码见 [完整使用示例](examples/basic_usage.py),接口、结构管理和 String 语义见 [使用说明](docs/crud-usage.md)。实际验证范围及剩余事项见 [基础使用封装验收记录](docs/crud-basic-implementation.md)。 + ## 高级记录操作 diff --git a/docs/README.md b/docs/README.md index 90afc65..3966a65 100644 --- a/docs/README.md +++ b/docs/README.md @@ -16,10 +16,12 @@ ## CRUD 开发设计 -- [高级记录操作使用说明](crud-usage.md) 提供批量操作、字段投影、同步/异步和 Redis Hash 的使用边界。 +- [基础结构与记录操作使用说明](crud-usage.md) 提供初始化、结构管理、批量操作、字段投影、同步/异步及 Redis Hash、String 的使用边界。 - [适配器契约](crud-adapter-contract.md) 定义公共层与各后端的模块职责、调用和部分结果协议。 - [CRUD 实施与验收记录](crud-implementation.md) 记录本地分支、Git 集成过程、测试及待验证项。 -- [高级 CRUD 设计](crud-design.md) 说明已实现接口的设计依据,包括批量操作、字段投影、返回结果、Redis 边界、同步与异步及原生客户端访问。 +- [基础使用设计与开发计划](crud-design.md) 说明从初始化到 CRUD 的接口、原生能力边界和完整示例。 +- [基础使用实施与验收记录](crud-basic-implementation.md) 记录本轮三个模块的提交、合并、测试及真实 Redis 验收缺口。 +- [同步异步完整示例](../examples/basic_usage.py) 演示三个后端同时使用;由调用方提供显式配置。 ## 目录职责 @@ -27,7 +29,9 @@ | --- | --- | | [src/database/](../src/database/) | 公共导出、连接配置、统一句柄和异常转换 | | [src/database/_adapters/](../src/database/_adapters/) | 三种后端高层连接的创建、检查、关闭及驱动错误映射 | -| [src/database/crud/](../src/database/crud/) | 可选的同步/异步记录操作、条件模型、批量结果和后端适配 | +| [src/database/crud/](../src/database/crud/) | 同步/异步记录与 Redis String 操作、条件模型、批量结果和后端适配 | +| [src/database/schema/](../src/database/schema/) | MySQL 基础库表与 MongoDB 普通集合管理 | | [src/database/mongodb/](../src/database/mongodb/)、[src/database/mysql/](../src/database/mysql/)、[src/database/redis/](../src/database/redis/) | 后端原生客户端工厂及各自的辅助功能 | | [tests/](../tests/) | 离线自动化测试 | | [scripts/verify_connections.py](../scripts/verify_connections.py) | 使用显式测试配置验证真实连接 | +| [scripts/verify_basics.py](../scripts/verify_basics.py) | 在显式隔离服务上验证结构准备、完整读写及清理 | diff --git a/docs/crud-basic-implementation.md b/docs/crud-basic-implementation.md new file mode 100644 index 0000000..c178172 --- /dev/null +++ b/docs/crud-basic-implementation.md @@ -0,0 +1,124 @@ +# 基础使用封装实施与验收记录 + +日期:2026-11-17。 + +本轮落实 [基础使用设计](crud-design.md),在既有连接管理和记录 CRUD 之上增加基础结构管理与 Redis String 操作。代码已集成到本地 `feature/database-crud`,供用户审查。真实 Redis 验收未完成,不能据此宣称全部真实拓扑均已通过。 + +## 1. 交付范围与边界 + +- MySQL:`Schema.mysql()`、`AsyncSchema.mysql()`,数据库和基础表的创建、兼容检查、列表、存在性、描述和显式删除;`Column`、`TableSpec` 描述基础结构。DDL 要求 autocommit 且无活动事务,不执行 ALTER、USE、提交或回滚。 +- MongoDB:`Schema.mongodb()`、`AsyncSchema.mongodb()`,普通集合准备、目录读取和显式删除。默认复用内建 `_id`;自定义身份只准备必要的完整唯一索引,并通过原生命令完成应答确认就绪。保留集合创建成功、索引创建失败时的集合和原始异常。 +- Redis:`RedisStrings`、`AsyncRedisStrings` 提供 String 的单键和逐项批量读写、删除、存在性与 TTL 管理,覆盖三种拓扑的适配路径。批量使用非事务流水线,保留顺序位置、未知状态和原始错误,不承诺跨槽原子性。 +- 继续复用 `Records`、`AsyncRecords` 的单条与批量 CRUD、字段投影、条件能力及原始返回类型。Redis Hash 不模拟全库条件扫描,String 不进行 JSON 编解码或字段投影。 +- 不新增运行时依赖、ORM、结构迁移、重试、跨库事务或统一高级索引管理。原生客户端始终可访问;配置仍由调用方显式提供。 + +同步、异步的完整多数据库示例见 [examples/basic_usage.py](../examples/basic_usage.py)。该示例用独立 MySQL 初始化连接,再在同一作用域打开三个后端;一个连接可以再绑定多个记录对象。同类连接按名称分别创建,实例和关闭相互独立。 + +表定义的实际限制:标识符沿用基础记录层的规则;`varchar` 必须提供长度,`decimal` 默认精度为 `(10, 0)`,`datetime` 支持 0–6 位小数精度;`text`、`blob` 只支持省略默认值,或可空时显式声明 `None`。声明列不得为生成列或 `INVISIBLE` 列,额外可省略列可保留原生属性。 + +MongoDB 集合列表和 ensure 接口限定普通集合;describe 返回指定命名空间的原生元数据,drop 按原生语义删除明确名称。ensure 和 drop 都拒绝未确认写关注;自定义身份索引的复用仍需要目录查询和创建索引权限。分片唯一性遵循原生规则。 + +## 2. 基线、分支与工作树 + +- 本轮起点:`ce740b3aefa554c7e4f0babb7c09b9b53f520ea0`。 +- 集成分支:`feature/database-crud`,工作树为 `.`。 +- `develop` 始终为 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 +- 公共接口检查点:`bdc326d41fb43eb893ea9c85166be8ed474242a6`。 +- 公共清理修复:`ca247e3e80ea66ce633fcd516b3bec00df4a6491`。 +- 最终代码检查点:`938c2a4d5cfe3a09a8b7bc782ce6131b1e407dbd`;随后文档提交的完整 SHA 可用 `git rev-parse feature/database-crud` 查看。 + +任务开始时,工作区已有八份计划及相对路径文档修改。本轮按原意保留并提交为 `6d5abb030e239d0887966db714810a6578e0701f`,未恢复、覆盖或混入其他草稿。 + +| 工作者 | 分支 | 工作树(相对根目录) | 首次交付 SHA | 首次合并后的主线 SHA | +| --- | --- | --- | --- | --- | +| MySQL | `feature/database-crud-mysql-schema` | `../database-worktrees/crud-mysql-schema` | `dc963d2b777a0acda75b65a058bde14920eed576` | `87f1ef1b2d43df43c851edba8cae76912d704edf` | +| MongoDB | `feature/database-crud-mongodb-schema` | `../database-worktrees/crud-mongodb-schema` | `2e4c14752a65f061f1d022e8889142695fbbfd7a` | `f2d870f7ea793eb6ed73509f37857bab85c615c1` | +| Redis | `feature/database-crud-redis-strings` | `../database-worktrees/crud-redis-strings` | `808bd908715061cd5f8f0baaaeef6310bd89c110` | `055f8c8819d7749d7ed6cf6d1f56caf9dbcf8d43` | + +MySQL 最终复核增加声明列可见性检查:`c9d0ee79b4b94f1fe94c64db553cb813de76cefe`,再次合入后的主线为 `938c2a4d5cfe3a09a8b7bc782ce6131b1e407dbd`。原始模块提交及合并边界均保留。 + +## 3. Git 操作顺序 + +以下命令以仓库根目录为路径基准。并行 worker 内部各自串行执行;不同 worker 的完成先后不表示依赖关系。所有操作均为本地操作。 + +| 顺序 | 执行者与位置 | 操作与结果 | +| --- | --- | --- | +| 1 | 协调者,`feature/database-crud`,`.` | `git status --short --branch`、`git diff --cached`、`git worktree list --porcelain`、`git rev-parse HEAD`、`git rev-parse develop`;记录起点、八份已有文档改动及 22 个已有工作树。 | +| 2 | 协调者,同上 | 按路径暂存八份计划文档,核对暂存差异后 `git commit -m "docs(crud): 确定基础使用计划并统一相对路径"`,得到 `6d5abb0`。 | +| 3 | 协调者,同上 | 提交 Schema 绑定、异常上下文、String 结果类型及契约测试,得到 `bdc326d`;185 项离线测试通过。 | +| 4 | 三位 worker,各自工作树 | 从 `bdc326d` 分别执行 `git worktree add -b <表中分支> <表中工作树> bdc326d`,开始模块实现。新增三个工作树,不复用其他任务暂存区。 | +| 5 | 协调者,主线 | 提交清理异常保真修复 `ca247e3`。MySQL、MongoDB worker 分别在自己工作树执行 `git merge --ff-only ca247e3`,继承这一公共检查点;Redis 使用既有连接清理辅助逻辑。 | +| 6 | 三位 worker,各自分支 | 分别按所属实现和测试路径 `git add -- <路径>`、`git diff --cached --stat`、`git diff --cached --check`,创建中文提交。交付 SHA 如上表,不改写其他 worker 分支。 | +| 7 | 协调者,主线 | 核对 MySQL 分支 HEAD 与交付 SHA、`git diff HEAD...dc963d2`,执行 `git merge --no-ff -m "feat(schema): 合入 MySQL 基础库表管理" dc963d2`;记录 HEAD `87f1ef1`,25 项定向测试通过。 | +| 8 | 协调者,主线 | 同样核对 MongoDB 的 `2e4c147` 后,`git merge --no-ff -m "feat(schema): 合入 MongoDB 集合管理与身份约束" 2e4c147`;HEAD `f2d870f`,19 项定向测试通过。 | +| 9 | 协调者,主线 | 同样核对 Redis 的 `808bd90` 后,`git merge --no-ff -m "feat(redis): 合入同步异步 String 与批量操作" 808bd90`;HEAD `055f8c8`,18 项定向测试通过。 | +| 10 | 协调者,主线 | 提交示例、显式隔离验收脚本、脚本清理测试及公共无驱动导入测试,得到 `13091647d28093d12f1557239d744ea80b71d6df`,主题为 `test(crud): 补充完整使用示例与隔离服务验收`。 | +| 11 | MySQL worker;随后协调者 | worker 在原模块分支增量提交 `c9d0ee79b4b94f1fe94c64db553cb813de76cefe`;协调者核对后 `--no-ff` 合入,得到 `938c2a4d5cfe3a09a8b7bc782ce6131b1e407dbd`。没有 amend、rebase 或 squash。 | +| 12 | 协调者,主线 | 复验完整矩阵,按路径提交 README、设计、使用指南、测试指南、发布日志与本记录;最后检查 `git status`、`git diff --check`、`git log --first-parent` 和本地 `develop` SHA。 | + +没有执行 fetch、pull、远端查询或 push。没有切换、提交、合并或更新 `develop`。保留全部 25 个工作树,包括原有 detached 工作树;本轮三位 worker 的工作区干净。 + +## 4. 自动化验证 + +解释器为 Python 3.10.20,Ruff 0.17.5。显式设置 `PYTHONPATH` 指向本工作树 `src`,并实际核对 `database.__file__`,确保加载当前源码。矩阵目录只用于隔离依赖版本,未修改项目依赖声明。 + +| 环境 | PyMongo | Connector/Python | redis-py | 最终完整离线结果 | +| --- | --- | --- | --- | --- | +| 选定当前版本 | 4.18.3 | 26.10.0 | 8.1.0 | 256 项通过 | +| 最低同步版本 | 4.6.0 | 8.3.0 | 5.0.0 | 256 项通过 | +| 最低异步版本 | 4.9 | 8.3.0 | 5.0.0 | 256 项通过 | + +完整测试包含原生工厂、连接管理、原记录 CRUD、新 Schema/String、批量部分结果、清理错误及取消传播。无驱动导入测试以 `python -S` 子进程确认公共入口、表定义与结果类型不强制加载驱动。PyMongo 4.6 环境的异步功能用替身验证协议,真实驱动只支持同步;并未把测试数量解释为该版本具备原生异步能力。 + +各模块额外版本验证:MongoDB 在 PyMongo 4.6、4.9、4.13 各通过 19 项定向测试;MySQL 初次交付在 8.3.0、26.10.0 各通过 211 项该工作树全量测试,Redis 初次交付在 5.0.0、8.1.0 各通过 203 项该工作树全量测试。模块工作树的历史数量不同于最终集成数量。 + +从仓库根目录执行: + +```powershell +$env:PYTHONPATH = (Join-Path (Get-Location) "src") +python -B -m unittest discover -s tests -q +python -m ruff check --no-cache src/database/schema src/database/crud/_strings.py src/database/crud/_string_types.py src/database/crud/_types.py src/database/crud/__init__.py tests/test_schema_binding.py tests/test_schema_mysql.py tests/test_schema_mongodb.py tests/test_crud_strings.py tests/test_basic_verifier.py scripts/verify_basics.py examples/basic_usage.py +python -m ruff format --check --no-cache src/database/schema src/database/crud/_strings.py src/database/crud/_string_types.py src/database/crud/_types.py src/database/crud/__init__.py tests/test_schema_binding.py tests/test_schema_mysql.py tests/test_schema_mongodb.py tests/test_crud_strings.py tests/test_basic_verifier.py scripts/verify_basics.py examples/basic_usage.py +git diff --check +``` + +上述 lint、格式、差异检查通过。完整示例通过语法和导入检查;Markdown 本地链接及 Python 示例语法已核对。文档中的项目路径均使用相对路径。 + +一次受限只读沙箱测试因无法创建临时目录而失败;该次进程也未成功覆盖最低依赖路径,因此不计入版本验收。随后在获准的隔离测试环境重新执行,输出核对版本及源码路径,以上三组结果以该次成功运行及最终修复后复验为准。 + +## 5. 隔离真实服务验证 + +测试使用独立临时服务、显式 JSON 参数和随机目标名,不加载项目私密配置。验收材料保存在仓库外的隔离 QA 目录;此处仅记录文件名与用途,不记录本机绝对路径、密码或完整私密 URI。 + +- `basic-verification-config.json`:本轮 MySQL 与 MongoDB 两个隔离实例。 +- `basic-verification-pure-config.json`:纯 Python MySQL 同步路径。 +- `basic-verification-mongo-config.json`:MongoDB 最低同步版本。 +- `scripts/verify_basics.py`:仓库内可复用的验收脚本。使用 `--config <调用方明确指定的配置文件> --mode both`;最低同步版本使用 `--mode sync`。 + +MySQL 配置必须省略业务数据库并明确 `autocommit=True`。脚本创建完整随机 UUID 命名的测试库,业务连接先核对实际选中的库再进行 CRUD;使用完仅清理本次创建的对象。两个同类实例均先完成探测,再关闭第一个并检查第二个。全部被测后端在同一作用域保持连接,最终再次探测。测试断言不能用 Python `-O` 关闭,脚本对此显式拒绝。失败时仅输出异常类别、操作名和代码位置;清理错误附加到原错误。 + +| 服务与路径 | 实际结果 | +| --- | --- | +| MySQL 8.4.11,Connector/Python 26.10.0,C 扩展、纯 Python、异步 | 通过空库建表、重复初始化、结构查看、单条/批量 CRUD、投影、原生 SELECT 1 和清理。 | +| MySQL 8.4.11,Connector/Python 8.3.0,纯 Python与异步 | 同上通过。该隔离环境没有 8.3.0 的 C 扩展,未声称验证该扩展。 | +| MongoDB 8.0.24,PyMongo 4.18.3、4.9、4.13,同步与异步 | 通过普通集合、自定义身份索引、重复初始化、完整 CRUD 与清理;4.13 在 worker 定向真实场景验证。 | +| MongoDB 8.0.24,PyMongo 4.6.0,同步 | 通过集合与唯一约束准备、完整 CRUD、结构读取及清理。 | +| MySQL、MongoDB 同一进程同时使用,同类双连接 | 当前驱动和最低异步驱动组合的同步/异步均通过,关闭一个不会破坏另一个。 | +| Redis 单机、Sentinel、Cluster | 未完成真实服务验收;实现适配、离线驱动替身及 fakeredis 结果见下文。 | + +MySQL worker 真实场景还覆盖十种字段类型与默认值、额外可省略列、生成列、视图冲突、活动事务拒绝、重复删除、同定义并发创建及不兼容定义竞争。末轮在五条驱动路径复验不可见列:声明列冲突、额外可省略列兼容,27 项定向测试分别在两个驱动版本通过。真实 MySQL 8.4 的类型弃用警告促使建表使用 `BOOLEAN` 而非显式整数显示宽度,复验通过。 + +MongoDB worker 真实场景包括同名视图、capped/时序集合拒绝,索引排序规则和隐藏/降序选项复用,部分索引不充当完整唯一约束,重复数据导致的原始索引错误、集合保留、未确认写关注拒绝及并发初始化。等待索引完成的控制流程和权限异常传播由离线替身覆盖,未人为暂停真实索引构建或配置真实拒绝权限账号。 + +Redis 的额外语义验证使用 fakeredis 2.40.0,覆盖同步异步 String、TTL、str/bytes、WRONGTYPE 与批量结果。Cluster 使用原生 Pipeline 配合两个替身节点,验证跨槽命令路由及错误/清理路径;Sentinel 资源归属由替身验证。这些结果不能替代真实拓扑。 + +本轮随机测试库均已清理,协调者核对两个隔离实例中 `sdk_` 前缀测试库数量为零。两个自行启动的服务已正常关停,并复核原 PID 和测试端口均不再运行;隔离数据目录及历史验收材料保留,未删除其他工作树或用户文件。 + +## 6. 待验证事项与审查入口 + +1. 真实 Redis 单机、Sentinel、Cluster 的同步异步 Hash 和 String 操作,以及 Cluster 跨槽批量。当前没有 Redis 服务、Docker 或 Podman;WSL 因虚拟化条件不可用。未修改主机虚拟化配置,也未用替身冒充真实服务。 +2. 三个真实数据库同时使用的完整示例。当前真实端到端完成的是 MySQL 与 MongoDB 同时使用;三个后端的协作已有离线覆盖。 +3. 本轮新增结构接口在真实权限不足账号、真实网络中断和暂停索引构建条件下的故障验证。离线测试覆盖异常分类、原对象、清理失败、未知批量状态和取消传播,既有连接层历史结果另见连接验收记录。 +4. Connector/Python 8.3.0 C 扩展,以及 Python 3.10 以外解释器的本轮真实服务矩阵。 + +审查可先阅读 [使用说明](crud-usage.md) 和 [完整示例](../examples/basic_usage.py),再查看 `git log --first-parent ce740b3..feature/database-crud` 及 `git diff ce740b3..feature/database-crud`。代码已交付可审查,以上缺口保留为待完成验收;合入 `develop` 由用户决定。 diff --git a/docs/crud-design.md b/docs/crud-design.md index 85aaee1..fd55566 100644 --- a/docs/crud-design.md +++ b/docs/crud-design.md @@ -2,11 +2,13 @@ 日期:2026-11-16 -状态:计划文档已生成,供用户审查;本轮未修改产品代码、未实施新增功能。 +更新日期:2026-11-17 + +状态:基础封装已在 `feature/database-crud` 集成,待用户审查。MySQL、MongoDB 基础流程的真实服务验收已通过;Redis 真实单机、Sentinel、Cluster 及三种真实后端同时运行的验收尚未完成。 文档位置:`docs\crud-design.md`。 -本文替代上一版仅覆盖记录 CRUD 的设计。当前已有的连接管理与记录 CRUD 可以复用,新增结构管理和 Redis String 能力仍为计划内容。以下开发、提交、合并与验收步骤均为后续工作,不表示本轮已经执行。 +本文替代上一版仅面向记录 CRUD 的设计,保留已经实现的连接管理和记录 CRUD,并增加基础结构管理与 Redis String 操作。实际提交、验证环境和未验证事项见 [本轮实施与验收记录](crud-basic-implementation.md);文档中的目标范围不等同于全部真实拓扑均已验收。 ## 一、目标与实现范围 @@ -47,7 +49,7 @@ Redis 不增加虚构的表或集合。编号库由部署提供,Cluster 仅支 ### 1.3 当前实现与本轮增量 -当前工作分支为 `feature/database-crud`。生成本文之前,工作区没有待提交修改。 +集成和交付分支为 `feature/database-crud`。以下提交为本轮设计与开发的历史起点,完整演进见实施记录。 - 当前实现起点:`ce740b3aefa554c7e4f0babb7c09b9b53f520ea0`。 - 本地 `develop`:`b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 @@ -121,6 +123,7 @@ user_table = TableSpec( - `Column` 支持 `int`、`bigint`、`boolean`、`decimal`、`double`、`varchar`、`text`、`date`、`datetime`、`blob`。 - 支持适用的长度、精度、小数位、可空性、字面量默认值,以及单一整数主键的自增。 +- `varchar` 必须声明长度;`decimal` 未指定精度时为 `(10, 0)`,`datetime` 支持 0–6 位小数精度,默认 0。`text`、`blob` 本期只支持省略默认值,或在可空时显式指定 `None`;不生成默认表达式。 - `nullable` 默认 `False`;未提供默认值表示不声明默认值。 - 不接受任意 SQL 类型片段、默认值表达式或约束表达式。 - 新表使用 InnoDB,并继承目标数据库的字符集与排序规则。 @@ -130,6 +133,7 @@ user_table = TableSpec( - 声明的字段必须存在,规范化后的类型、长度或精度、可空性及明确声明的默认值、自增属性需要匹配。 - 主键字段及顺序需要匹配。 +- 声明字段不能是生成列或 `INVISIBLE` 列,避免基础写入失败或默认读取漏字段;额外可省略字段仍可保留这些原生属性。 - 允许额外字段,但额外字段必须允许省略,例如可空、有默认值或由数据库生成。 - 保留已有表的其他索引、引擎及高级约束,不擅自修改。 - 不兼容时抛出 `SchemaConflictError`,列出差异,不执行 `ALTER TABLE`。 @@ -157,7 +161,9 @@ MySQL 创建、删除结构可能隐式提交。因此,结构写操作要求 - 已有同名视图、时序集合或 capped 集合不作为普通记录集合接受。 - 集合创建和必要索引建立必须得到确认,不能仅凭目录中出现索引定义就认定构建完成。 - 如果集合已创建而索引创建失败,保留集合及原始异常,不自动删除集合补偿。 -- 初始化要求可确认的写关注;不使用 `w=0` 宣称结构准备成功。 +- 初始化和删除均要求可确认的写关注;不使用 `w=0` 宣称结构操作成功。 +- 自定义身份约束需要目录查询和创建索引权限;复用已有索引也需取得原生命令的完成应答。 +- `describe_collection()` 保留指定命名空间的原生元数据,包含视图等类型;`drop_collection()` 按原生语义删除显式指定的命名空间。普通记录类型限制由 `ensure_collection()` 与集合列表执行。 - 本期不处理分片规划;分片集合的全局唯一性继续遵守原生分片键规则。 MongoDB 自定义唯一索引可能因已有重复数据而建立失败,这属于应向调用方保留的真实错误。[MongoDB 唯一索引](https://www.mongodb.com/docs/manual/core/index-unique/) @@ -344,7 +350,7 @@ Redis Hash 保持现有记录格式、显式 codec 和 TTL 约定。Hash 更新 ## 三、从初始化到 CRUD 的同步、异步示例 -以下是**拟实现 API 的验收示例**,不是对当前版本可直接运行的声明。先复用公共配置与表定义,再选择运行同步或异步示例。 +以下示例使用本分支已实现的 API,完整源码见 [同步异步示例](../examples/basic_usage.py)。由调用方准备隔离服务和凭据,先复用公共配置与表定义,再选择同步或异步入口。MySQL、MongoDB 已完成真实服务端到端验收;Redis 真实拓扑验收仍待补齐。示例会写入并清理指定演示记录,应使用专用演示库和命名空间。 示例假设三个数据库服务和账号已经准备好。MySQL 初始化连接具有建库建表权限;生产项目可以为初始化与业务操作配置不同账号。 @@ -655,13 +661,13 @@ async def run_async(connections: Connections) -> None: ## 四、开发组织与实施顺序 -以下为后续实施流程。本轮仅生成本文档,不启动 worker 实现、不修改产品代码、不执行提交或合并。 +以下为本轮遵循的实施流程。三个模块已独立开发并串行合入功能分支;交付状态与验收缺口以实施记录为准。 ### 4.1 集成策略 继续使用 `feature/database-crud` 作为本功能集成主线,保留已经完成的记录 CRUD 历史,不重新从 `develop` 开始实现。 -后续开发遵循: +开发遵循: - 不操作 `develop`,最终由用户审核后决定合入。 - 只进行本地 Git 操作,不检查或同步远端。 @@ -760,4 +766,4 @@ git diff --check 最终交付提供集成分支、基线与最终提交、worker 合并关系、按执行顺序记录的 Git 操作、测试结果和未验证事项,停留在功能分支供用户审查。 -Git 操作记录注明执行者、分支、工作树、命令和结果;合并记录写明被合入 SHA 与集成后的 HEAD。核对 `develop` 未改变,未混入无关文件,未访问或同步远端。本轮仅生成文档,不产生上述实现与验收交付记录。 \ No newline at end of file +Git 操作记录注明执行者、分支、工作树、命令和结果;合并记录写明被合入 SHA 与集成后的 HEAD。核对 `develop` 未改变,未混入无关文件,未访问或同步远端。本轮实现与验收证据另存于 [基础使用实施记录](crud-basic-implementation.md)。 diff --git a/docs/crud-usage.md b/docs/crud-usage.md index 32c01e1..d73abd9 100644 --- a/docs/crud-usage.md +++ b/docs/crud-usage.md @@ -1,12 +1,55 @@ -# 高级记录操作使用说明 +# 基础结构与记录操作使用说明 `database.crud` 在已有连接之上提供可选的记录操作层。 MySQL 对应表,MongoDB 对应集合,Redis 对应采用本库格式的 Hash 记录。 它们共享调用方式、字段投影和结果结构;事务、类型系统和查询能力仍遵循各数据库的真实语义。 -本功能在 `feature/database-crud` 分支开发,具体测试范围与待验收事项见 [实施记录](crud-implementation.md)。 +本功能在 `feature/database-crud` 分支开发。原记录 CRUD 测试见 [历史实施记录](crud-implementation.md),新增结构与 String 的验证范围及缺口见 [本轮实施记录](crud-basic-implementation.md)。 完整设计背景见 [CRUD 设计](crud-design.md),内部模块协作见 [适配器契约](crud-adapter-contract.md)。 + +## 0. 从初始化开始使用 + +先显式传入连接参数,然后准备数据库或集合,最后绑定记录对象。MySQL 初始化连接应不选择尚未存在的业务库,并显式设置 `options={"autocommit": True}`;业务连接在初始化完成后通过 `options={"database": "app", "autocommit": True}` 选择目标库。MongoDB 使用显式集合准备;Redis 使用配置中的编号库以及 Hash 命名空间或 String 键,不创建虚构的表。 + +```python +from database.schema import Column +from database.schema import Schema +from database.schema import TableSpec + +users = TableSpec( + columns={ + "id": Column("varchar", length=32), + "name": Column("varchar", length=100), + "age": Column("int"), + }, + primary_key=("id",), +) + +with connections.create("mysql_setup") as setup: + schema = Schema.mysql(setup) + schema.ensure_database("app") + schema.ensure_table("app", "users", users) + +with connections.create("documents") as documents: + Schema.mongodb(documents).ensure_collection("app", "users") +``` + +`connections` 由调用方通过 `ConnectionConfig` 与 `Connections` 显式建立。上面的结构准备不提交业务事务,不改变连接默认数据库,不缓存客户端。已有表只校验声明字段和主键;额外列必须能够省略。不兼容时抛出 `SchemaConflictError`,其 `differences` 保存差异。声明字段不能是生成列或 `INVISIBLE` 列;额外的可省略字段可保留这些原生属性。 + +异步使用 `AsyncSchema.mysql(connection)`、`AsyncSchema.mongodb(connection)`,绑定不需要 await,管理方法需要 await;连接创建使用 `await connections.create_async(name)`。三种数据库同时使用的完整同步、异步流程见 [完整示例](../examples/basic_usage.py) 和 [设计中的使用示例](crud-design.md#三从初始化到-crud-的同步异步示例)。 + +### 基础结构管理 + +MySQL 提供 `ensure_database`、`ensure_table`、`list_databases`、`database_exists`、`list_tables`、`table_exists`、`describe_table`、`drop_table`、`drop_database`。查看方法反映当前账号可见目录,权限错误不会作为“不存在”处理;结构写操作要求 autocommit 且没有活动事务。显式删除按指定名称执行,不在上下文退出时自动删库表。 + +`Column` 支持 int、bigint、boolean、decimal、double、varchar、text、date、datetime、blob;VARCHAR 必须指定 length,DECIMAL 默认精度 10、小数位 0,DATETIME 默认精度 0。TEXT/BLOB 的显式默认值仅支持可空列的 None;其他复杂默认表达式使用原生 API。自增仅支持单列整数主键,通用索引、外键和结构变更仍由原生客户端处理。 + +MongoDB 提供 `ensure_collection`、`list_databases`、`database_exists`、`list_collections`、`collection_exists`、`describe_collection`、`drop_collection`、`drop_database`。准备和普通集合列表排除视图、时序、capped 及系统集合。describe 保留原生目录信息;drop 按原生命名空间语义精确删除,也可以删除明确指定的视图,不先查询是否为普通集合。 + +默认 `key_fields=("_id",)` 复用内建约束。声明自定义键时,准备完整且排序规则兼容的唯一索引;需要目录读取及索引创建权限,已存在的索引也会通过原生命令确认就绪。不能把部分、稀疏或构建中的索引当作完整身份保证。结构写入和删除要求 acknowledged 写关注;失败保留已有集合及原异常,不补偿删库或重建其他索引。分片唯一性仍受原生分片键约束。 + + ## 1. 绑定已有连接 安装方式和连接配置见 [README](../README.md)。 @@ -272,3 +315,47 @@ MySQL autocommit 和连接关闭行为按原生驱动规则执行;需要明确 MongoDB 可绑定原生 session,事务及 session 生命周期由调用方管理。 Redis 其他结构、MongoDB 聚合、SQL JOIN、数据库专有原子操作都通过 `connection.native` 使用。 上下文外原生异常行为不变;需要转换原生操作错误时显式使用 `connection.errors(operation=...)`。 + +## 9. Redis String 键值与批量操作 + +`RedisStrings` / `AsyncRedisStrings` 借用既有 Redis 连接,支持单机、Sentinel 和 Cluster。String 使用调用方提供的完整键,不自动加前缀,也不解析 JSON。 + +```python +from database.crud import RedisStrings +from database.crud import StringSetItem + +with connections.create("cache") as connection: + with RedisStrings(connection) as strings: + strings.set("app:message", "hello", ttl_s=60) + print(strings.get("app:message")) + strings.set_many([ + StringSetItem("app:a", "A", ttl_s=60), + StringSetItem("app:b", b"B"), + ]) + print(strings.get_many(["app:b", "app:a", "app:missing"])) + strings.delete_many(["app:a", "app:b", "app:message"]) +``` + +```python +from database.crud import AsyncRedisStrings +from database.crud import StringSetItem + +async with await connections.create_async("cache") as connection: + async with AsyncRedisStrings(connection) as strings: + await strings.set("app:message", "hello", ttl_s=60) + print(await strings.get("app:message")) + await strings.set_many([ + StringSetItem("app:a", "A", ttl_s=60), + StringSetItem("app:b", b"B"), + ]) + print(await strings.get_many(["app:b", "app:a", "app:missing"])) + await strings.delete_many(["app:a", "app:b", "app:message"]) +``` + +键和值仅接受 str/bytes;get 返回类型遵循连接的 decode_responses,缺失返回 None,空字符串保留。set 成功返回 True,delete 返回 0/1;exists、expire、persist 返回布尔值。TTL 正整数秒由单条 SET EX 设置;普通 set 不指定 TTL 会清除原 TTL,expire 修改正整数秒过期,persist 清除过期。ttl 保留 -2(缺失)、-1(不过期)及剩余秒数。SET/DEL 保留对同名其他类型键的原生命令作用,GET 遇到类型不符抛出保留原始信息的异常。String 不提供字段投影。 + +批量输入为有限序列,默认每批 500。get_many 使用逐键 GET 保留顺序和重复位置,不保证重复读取来自同一时刻,也不会将 WRONGTYPE 当成缺失。set_many 接受 StringSetItem 序列,delete_many 接受键列表;写批次按实际编码后的 bytes 拒绝重复目标,所有输入先校验再发送。 + +批量写入返回 StringBatchResult,包含 requested_count、items、deleted_count、native_results;StringBatchItem 包含 index、key、state、result、error。失败通过 BatchOperationError.partial_result 保留上述结构,已派发但未收到明确回复的项标为 unknown,后续未派发项标为 not_sent。SET 不推测新增数或修改数;Cluster 跨槽批量不提供整体原子性。异步取消保留原对象及 database_partial_result,清理错误不覆盖主错误。 + +String 与 Hash 上下文只清理自身资源。Sentinel 使用配置的 service_name 创建临时数据客户端;外部池和借用的连接由原所有者管理。 diff --git a/tests/README.md b/tests/README.md index 49fd6c2..bed44f6 100644 --- a/tests/README.md +++ b/tests/README.md @@ -42,3 +42,11 @@ python -B scripts/verify_connections.py --config "<测试配置文件路径>" ``` 脚本探测同步与异步连接,验证实例独立关闭,并对 MySQL 执行只读的 `SELECT 1`。分别记录离线回归与真实服务验证的结果,具体环境和实际结果见 [连接管理验收记录](../docs/connection-management-validation.md)。 + +## 基础结构与键值封装验收 + +新增测试覆盖 `tests/test_schema_binding.py`、`tests/test_schema_mysql.py`、`tests/test_schema_mongodb.py`、`tests/test_crud_strings.py` 和 `tests/test_basic_verifier.py`。依次验证公共异常与生命周期、基础库表与集合、Redis String 及验收清理规则。 + +`scripts/verify_basics.py --config <隔离配置文件> --mode both` 使用显式 ConnectionConfig 参数映射,创建并清理随机命名的验收数据库与键,覆盖初始化、投影、单条/批量 CRUD、原生访问、两个同类连接和多个后端同时在线。该脚本会写入数据,配置必须指向隔离服务;MySQL 配置不选择业务数据库,且 options.autocommit 必须为 true。可用 --mode sync 验证仅支持同步的 PyMongo 4.6。 + +脚本不读取环境变量、不输出配置或原始异常详情;失败摘要仅包含错误类型、操作名、文件名、行号和清理错误数量。不要使用 Python -O 运行验收。版本矩阵与真实服务边界见 [基础使用封装验收记录](../docs/crud-basic-implementation.md)。 -- Gitee From fb05e168fc3e76042ce672394fcf6a04caebf9f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Tue, 17 Nov 2026 19:40:07 +0800 Subject: [PATCH 74/86] =?UTF-8?q?feat(mongodb):=20=E5=AE=8C=E5=96=84?= =?UTF-8?q?=E5=88=86=E6=89=B9=E5=86=99=E5=85=A5=E5=92=8C=E6=B8=B8=E6=A0=87?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 按配置条数流式写入,降低大批量插入与更新的缓冲压力。提供同步和异步混合写入、待写数据丢弃及自动关闭游标的分批读取工具。 通过 63 项 MongoDB 定向测试、269 项完整离线测试、Ruff 和暂存差异检查。 --- CHANGELOG.md | 2 + README.md | 41 ++++ src/database/crud/_mongodb.py | 33 +-- src/database/mongodb/_async_utils.py | 177 ++++++++------- src/database/mongodb/_batch.py | 90 ++++++++ src/database/mongodb/_cursor_batches.py | 68 ++++++ src/database/mongodb/_resources.py | 37 ++++ src/database/mongodb/_sync_utils.py | 176 ++++++++------- src/database/mongodb/utils.py | 10 + tests/test_mongodb_utils.py | 275 ++++++++++++++++++++++++ 10 files changed, 698 insertions(+), 211 deletions(-) create mode 100644 src/database/mongodb/_batch.py create mode 100644 src/database/mongodb/_cursor_batches.py create mode 100644 src/database/mongodb/_resources.py create mode 100644 tests/test_mongodb_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 25ea8c3..2b470fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### 新增 +- 增加 MongoDB 同步与异步混合批写、待写缓冲丢弃和自动关闭游标的分批读取工具。异步批处理支持消费异步可迭代对象。 - 新增同步/异步 Schema 基础管理:MySQL 库表创建、兼容校验、目录查看和删除;MongoDB 集合准备、必要身份约束及目录管理。初始化显式执行,不自动迁移。 - 新增 `RedisStrings` / `AsyncRedisStrings` 的单键、批量读写、删除和 TTL,保留逐项执行证据、原始错误与取消信号。 - 新增三数据库完整同步/异步使用示例和显式隔离验收脚本;原生客户端及已有记录接口保持兼容。 @@ -20,6 +21,7 @@ ### 修复 +- 修复 MongoDB 批处理工具的导入与插入批量预缓冲问题,按配置条数派发生成器和列表输入。失败时释放批次,清理失败时保留业务异常,错误计数使用驱动报告的逐条写入错误。 - MySQL 目标配置提前拒绝 Windows 上无法使用的纯 Unix socket 配置,并把缺少驱动能力的错误转换为 `DependencyError`。 - 识别 MySQL 的 `password1` 密码别名,在公共密码参数与该别名同时传入时报告冲突。 - Redis 异步 Cluster 关闭失败后保留待清理节点,以便后续关闭重试。关闭期间的错误附加到已有业务异常。 diff --git a/README.md b/README.md index 719ad9d..02cd6c8 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,47 @@ async def ping_mongodb(): asyncio.run(ping_mongodb()) ``` +#### MongoDB 分批写入与读取 + +`database.mongodb.utils` 接收原生 PyMongo 集合和游标。批处理器借用集合,MongoDB 客户端和连接池由调用方关闭。 + +```python +from database.mongodb.utils import BatchInsert, BatchUpdate, cursor_batches +from pymongo import UpdateOne + + +def migrate(source, target): + with cursor_batches(source.find({'ready': True}).batch_size(500), step=500) as pages: + with BatchInsert(target, step=500, ordered=False) as writer: + for page in pages: + writer.extend(page) + + +def mark_processed(target, ids): + with BatchUpdate(target, step=500) as writer: + writer.extend(UpdateOne({'_id': key}, {'$set': {'processed': True}}) for key in ids) +``` + +- `BatchInsert` 写入文档,`BatchUpdate` 支持 `add(filter, update)` 和原生 `UpdateOne`。`BatchWrite` 接收原生 `InsertOne`、`UpdateOne`、`UpdateMany`、`ReplaceOne`、`DeleteOne` 和 `DeleteMany`,可以混合插入、更新、替换和删除。 +- `extend()` 逐项消费输入,达到 `step` 立即写入。`step` 调整每批条数,驱动的网络读取批量可用游标的 `batch_size()` 单独调整。大输入可使用生成器或游标,调用方应及时释放消费完的批次。 +- `execute()` 和 `close()` 都会写出尾批。上下文退出时也会写出尾批,包括因业务异常退出。需要放弃尾批时,在退出前调用 `discard()`。每次尝试写入后都会释放该批缓冲,失败的批次由应用结合原生错误判断如何处理。 +- `on_error` 默认抛出原生 `BulkWriteError`,也可设为 `'log'` 或 `'ignore'`。`success_count` 分别累计已确认的插入数、修改数或混合写入影响的文档数。`error_count` 统计驱动报告的逐条写入错误,`unacknowledged_count` 统计使用非确认写入提交的操作数。 +- 有序批次可能在首个逐条写入错误处停止。保留默认错误策略可从原生异常的 `details` 查看部分写入结果和写关注错误。 +- `cursor_batches()` 接管传入游标,在离开 `with` 时关闭它。提前 `break` 后退出作用域,或消费过程中抛出异常,都会执行关闭。 + +异步使用 `AsyncBatchInsert`、`AsyncBatchUpdate`、`AsyncBatchWrite` 和 `async_cursor_batches()`。`extend()` 也接收异步可迭代对象。 + +```python +from database.mongodb.utils import AsyncBatchInsert, async_cursor_batches + + +async def migrate_async(source, target): + async with async_cursor_batches(source.find({'ready': True}), step=500) as pages: + async with AsyncBatchInsert(target, step=500) as writer: + async for page in pages: + await writer.extend(page) +``` + ### MySQL 使用 MySQL Connector/Python。`create_async()` 需要 `await`,驱动完成建连后返回原生异步连接。示例从 `MYSQL_USER` 和 `MYSQL_PASSWORD` 环境变量读取凭据。 diff --git a/src/database/crud/_mongodb.py b/src/database/crud/_mongodb.py index 5a499fe..700f163 100644 --- a/src/database/crud/_mongodb.py +++ b/src/database/crud/_mongodb.py @@ -1,11 +1,9 @@ """MongoDB record operations on a borrowed collection and optional session.""" import asyncio -from collections.abc import AsyncIterator from collections.abc import Iterator from collections.abc import Mapping from collections.abc import Sequence -from contextlib import asynccontextmanager from contextlib import contextmanager from dataclasses import dataclass from dataclasses import field @@ -23,7 +21,8 @@ from pymongo.errors import InvalidName from ..connections import Connection from ..errors import ConfigurationError -from ..errors import attach_cleanup_errors +from ..mongodb._resources import _async_closing +from ..mongodb._resources import _closing from ._batch import Record from ._conditions import And from ._conditions import Condition @@ -97,34 +96,6 @@ def _record(row: Mapping[str, Any], fields: _Fields) -> Record: return {field: row[field] for field in fields if field in row} -@contextmanager -def _closing(cursor: Any) -> Iterator[Any]: - try: - yield cursor - except BaseException as error: - try: - cursor.close() - except BaseException as cleanup_error: - attach_cleanup_errors(error, (cleanup_error,)) - raise - else: - cursor.close() - - -@asynccontextmanager -async def _async_closing(cursor: Any) -> AsyncIterator[Any]: - try: - yield cursor - except BaseException as error: - try: - await cursor.close() - except BaseException as cleanup_error: - attach_cleanup_errors(error, (cleanup_error,)) - raise - else: - await cursor.close() - - @dataclass class _ReadProgress: rows: list[Record | None] = field(default_factory=list) diff --git a/src/database/mongodb/_async_utils.py b/src/database/mongodb/_async_utils.py index 6a55e06..e968170 100644 --- a/src/database/mongodb/_async_utils.py +++ b/src/database/mongodb/_async_utils.py @@ -1,106 +1,105 @@ -import pymongo.errors -from pymongo import UpdateOne - -from ..utils.logger import get_logger - -logger = get_logger(__name__) - +from __future__ import annotations -class AsyncBatch: +from asyncio import Lock +from collections.abc import AsyncIterable +from collections.abc import Iterable +from types import TracebackType +from typing import TypeVar - def __init__(self, collection, step=1000, ordered=True, on_error='raise'): - """初始化异步批量操作. +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError + +from ..errors import attach_cleanup_errors +from ._batch import _BatchBuffer +from ._batch import _Document +from ._batch import _ErrorPolicy +from ._batch import _Item +from ._batch import _Update +from ._batch import _update_operation +from ._batch import _WriteOperation + +_AsyncBatchT = TypeVar('_AsyncBatchT', bound='AsyncBatch') + + +class AsyncBatch(_BatchBuffer[_Item]): + def __init__( + self, collection: object, step: int = 1000, ordered: bool = True, on_error: _ErrorPolicy = 'raise' + ) -> None: + super().__init__(collection, step, ordered, on_error) + self._lock = Lock() + + async def add(self, doc_or_op: _Item) -> None: + """Queue one item, writing the batch when it reaches step.""" + async with self._lock: + self.ops.append(doc_or_op) + if self.count >= self.step: + await self._flush() + + async def extend(self, docs_or_ops: Iterable[_Item] | AsyncIterable[_Item]) -> None: + """Consume items incrementally, leaving any final partial batch pending.""" + if isinstance(docs_or_ops, AsyncIterable): + async for item in docs_or_ops: + await self.add(item) + else: + for item in docs_or_ops: + await self.add(item) - Args: - collection: pymongo AsyncCollection 对象 - step: 批次大小,默认 1000 - ordered: 是否有序操作,默认 True - on_error: 错误处理策略,默认 'raise' - - 'raise': 抛出异常(默认) - - 'log': 记录日志但继续执行 - - 'ignore': 忽略错误 - """ - self.ops = [] - self.count = 0 - self.collection = collection - self.step = step - self.ordered = ordered - self.on_error = on_error + async def execute(self) -> None: + """Write the pending batch, if any.""" + async with self._lock: + await self._flush() - self.success_count = 0 - self.error_count = 0 + async def _flush(self) -> None: + if not self.ops: + return + try: + write = getattr(self.collection, self._write_method) + self._record_result(await write(self.ops, ordered=self.ordered)) + except BulkWriteError as error: + self._record_error(error) + if self.on_error == 'raise': + raise + finally: + self.ops.clear() - async def add(self, doc_or_op): - self.ops.append(doc_or_op) - self.count += 1 - if self.count >= self.step: - await self.execute() + async def discard(self) -> None: + """Release pending items without sending them to MongoDB.""" + async with self._lock: + self.ops.clear() - async def extend(self, docs_or_ops): - self.ops.extend(docs_or_ops) - self.count += len(docs_or_ops) - while self.count >= self.step: - await self.execute() + async def close(self) -> None: + """Flush pending items. The batch and its collection remain usable.""" + await self.execute() - async def execute(self): - pass + async def __aenter__(self: _AsyncBatchT) -> _AsyncBatchT: + return self - async def close(self): - if self.count: - await self.execute() + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> None: + try: + await self.close() + except BaseException as error: + if exc is None: + raise + attach_cleanup_errors(exc, (error,)) - async def __aenter__(self): - return self - async def __aexit__(self, exc_type, exc_val, exc_tb): - await self.close() +class AsyncBatchInsert(AsyncBatch[_Document]): + """Buffer documents for batched calls to AsyncCollection.insert_many().""" + _write_method = 'insert_many' + _success_keys = ('nInserted',) -class AsyncBatchInsert(AsyncBatch): - async def execute(self): - try: - result = await self.collection.insert_many(self.ops, ordered=self.ordered) - self.success_count += len(result.inserted_ids) - except pymongo.errors.BulkWriteError as error: - self.success_count += error.details.get('nInserted', 0) - self.error_count += len(self.ops) - error.details.get('nInserted', 0) - if self.on_error == 'raise': - raise - elif self.on_error == 'log': - logger.warning(f'AsyncBatchInsert error: {error.details}') - finally: - self.ops.clear() - self.count = 0 +class AsyncBatchUpdate(AsyncBatch[UpdateOne]): + """Buffer PyMongo UpdateOne operations or build them from filter/update pairs.""" + _success_keys = ('nModified',) -class AsyncBatchUpdate(AsyncBatch): - async def add(self, query_or_op, update=None): - if isinstance(query_or_op, UpdateOne): - await AsyncBatch.add(self, query_or_op) - elif update: - await AsyncBatch.add(self, UpdateOne(query_or_op, update)) - else: - raise pymongo.errors.InvalidOperation('Missing update parameter.') + async def add(self, query_or_op: _Document | UpdateOne, update: _Update | None = None) -> None: + await super().add(_update_operation(query_or_op, update)) - async def extend(self, ops): - for op in ops: - if isinstance(op, UpdateOne): - await AsyncBatch.add(self, op) - else: - raise pymongo.errors.InvalidOperation('Operation type Error.') - async def execute(self): - try: - result = await self.collection.bulk_write(self.ops, ordered=self.ordered) - self.success_count += result.modified_count - except pymongo.errors.BulkWriteError as error: - self.success_count += error.details.get('nModified', 0) - self.error_count += len(self.ops) - error.details.get('nModified', 0) - if self.on_error == 'raise': - raise - elif self.on_error == 'log': - logger.warning(f"AsyncBatchUpdate error: {error.details}") - finally: - self.ops.clear() - self.count = 0 +class AsyncBatchWrite(AsyncBatch[_WriteOperation]): + """Buffer mixed native write models for AsyncCollection.bulk_write().""" diff --git a/src/database/mongodb/_batch.py b/src/database/mongodb/_batch.py new file mode 100644 index 0000000..a3e715c --- /dev/null +++ b/src/database/mongodb/_batch.py @@ -0,0 +1,90 @@ +"""Shared buffering and write accounting for native MongoDB batch utilities.""" + +from collections.abc import Mapping +from logging import getLogger +from typing import Any +from typing import ClassVar +from typing import Generic +from typing import Literal +from typing import TypeVar + +from pymongo import DeleteMany +from pymongo import DeleteOne +from pymongo import InsertOne +from pymongo import ReplaceOne +from pymongo import UpdateMany +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError +from pymongo.errors import InvalidOperation +from pymongo.results import BulkWriteResult +from pymongo.results import InsertManyResult + +logger = getLogger(__name__) + +_Item = TypeVar('_Item') +_Document = Mapping[str, Any] +_Update = _Document | list[_Document] +_WriteOperation = InsertOne | UpdateOne | UpdateMany | ReplaceOne | DeleteOne | DeleteMany +_ErrorPolicy = Literal['raise', 'log', 'ignore'] + + +def _validate_step(step: int) -> None: + if not isinstance(step, int) or isinstance(step, bool) or step < 1: + raise ValueError('step must be a positive integer.') + + +def _update_operation(query_or_op: _Document | UpdateOne, update: _Update | None) -> UpdateOne: + if isinstance(query_or_op, UpdateOne): + return query_or_op + if not update: + raise InvalidOperation('Missing update parameter.') + return UpdateOne(query_or_op, update) + + +class _BatchBuffer(Generic[_Item]): + _write_method: ClassVar[str] = 'bulk_write' + _success_keys: ClassVar[tuple[str, ...]] = ('nInserted', 'nModified', 'nRemoved', 'nUpserted') + + def __init__( + self, collection: object, step: int = 1000, ordered: bool = True, on_error: _ErrorPolicy = 'raise' + ) -> None: + _validate_step(step) + if on_error not in ('raise', 'log', 'ignore'): + raise ValueError("on_error must be 'raise', 'log', or 'ignore'.") + self.ops: list[_Item] = [] + self.collection = collection + self.step = step + self.ordered = ordered + self.on_error = on_error + self.success_count = 0 + self.error_count = 0 + self.unacknowledged_count = 0 + + @property + def count(self) -> int: + """Number of operations currently in the buffer.""" + return len(self.ops) + + def _record_result(self, result: InsertManyResult | BulkWriteResult) -> None: + if not result.acknowledged: + self.unacknowledged_count += self.count + elif isinstance(result, InsertManyResult): + self.success_count += len(result.inserted_ids) + else: + self._record_successes(result.bulk_api_result) + + def _record_successes(self, details: Mapping[str, Any]) -> None: + self.success_count += sum(details.get(key, 0) for key in self._success_keys) + + def _record_error(self, error: BulkWriteError) -> None: + details = error.details or {} + self._record_successes(details) + write_errors = len(details.get('writeErrors') or ()) + self.error_count += write_errors + if self.on_error == 'log': + logger.warning( + '%s: %d write errors, %d write concern errors', + type(self).__name__, + write_errors, + len(details.get('writeConcernErrors') or ()), + ) diff --git a/src/database/mongodb/_cursor_batches.py b/src/database/mongodb/_cursor_batches.py new file mode 100644 index 0000000..2219d92 --- /dev/null +++ b/src/database/mongodb/_cursor_batches.py @@ -0,0 +1,68 @@ +"""Read a native MongoDB cursor in a bounded, explicitly owned scope.""" + +from collections.abc import AsyncGenerator +from collections.abc import AsyncIterator +from collections.abc import Generator +from collections.abc import Iterator +from contextlib import aclosing +from contextlib import asynccontextmanager +from contextlib import closing +from contextlib import contextmanager +from itertools import islice +from typing import Protocol +from typing import TypeVar + +from ._batch import _validate_step +from ._resources import _async_closing +from ._resources import _closing + +_Row = TypeVar('_Row') +_Row_co = TypeVar('_Row_co', covariant=True) + + +class _Cursor(Protocol[_Row_co]): + def __iter__(self) -> Iterator[_Row_co]: ... + + def close(self) -> None: ... + + +class _AsyncCursor(Protocol[_Row_co]): + def __aiter__(self) -> AsyncIterator[_Row_co]: ... + + async def close(self) -> None: ... + + +def _chunks(cursor: _Cursor[_Row], step: int) -> Generator[list[_Row], None, None]: + rows = iter(cursor) + while batch := list(islice(rows, step)): + yield batch + del batch + + +async def _async_chunks(cursor: _AsyncCursor[_Row], step: int) -> AsyncGenerator[list[_Row], None]: + batch: list[_Row] = [] + async for row in cursor: + batch.append(row) + if len(batch) == step: + yield batch + batch = [] + if batch: + yield batch + + +@contextmanager +def cursor_batches(cursor: _Cursor[_Row], *, step: int = 1000) -> Iterator[Iterator[list[_Row]]]: + """Yield a batch iterator and close its native cursor when the scope exits.""" + _validate_step(step) + with _closing(cursor), closing(_chunks(cursor, step)) as batches: + yield batches + + +@asynccontextmanager +async def async_cursor_batches( + cursor: _AsyncCursor[_Row], *, step: int = 1000 +) -> AsyncIterator[AsyncIterator[list[_Row]]]: + """Yield an async batch iterator and await native cursor cleanup on exit.""" + _validate_step(step) + async with _async_closing(cursor), aclosing(_async_chunks(cursor, step)) as batches: + yield batches diff --git a/src/database/mongodb/_resources.py b/src/database/mongodb/_resources.py new file mode 100644 index 0000000..39fd9b0 --- /dev/null +++ b/src/database/mongodb/_resources.py @@ -0,0 +1,37 @@ +"""Close MongoDB cursors while preserving an in-flight error.""" + +from collections.abc import AsyncIterator +from collections.abc import Iterator +from contextlib import asynccontextmanager +from contextlib import contextmanager +from typing import Any + +from ..errors import attach_cleanup_errors + + +@contextmanager +def _closing(cursor: Any) -> Iterator[Any]: + try: + yield cursor + except BaseException as error: + try: + cursor.close() + except BaseException as cleanup_error: + attach_cleanup_errors(error, (cleanup_error,)) + raise + else: + cursor.close() + + +@asynccontextmanager +async def _async_closing(cursor: Any) -> AsyncIterator[Any]: + try: + yield cursor + except BaseException as error: + try: + await cursor.close() + except BaseException as cleanup_error: + attach_cleanup_errors(error, (cleanup_error,)) + raise + else: + await cursor.close() diff --git a/src/database/mongodb/_sync_utils.py b/src/database/mongodb/_sync_utils.py index 5b9394b..5cfcc00 100644 --- a/src/database/mongodb/_sync_utils.py +++ b/src/database/mongodb/_sync_utils.py @@ -1,106 +1,100 @@ -import pymongo.errors -from pymongo import UpdateOne +from __future__ import annotations -from ..utils.logger import get_logger +from collections.abc import Iterable +from threading import Lock +from types import TracebackType +from typing import TypeVar -logger = get_logger(__name__) +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError + +from ..errors import attach_cleanup_errors +from ._batch import _BatchBuffer +from ._batch import _Document +from ._batch import _ErrorPolicy +from ._batch import _Item +from ._batch import _Update +from ._batch import _update_operation +from ._batch import _WriteOperation + +_BatchT = TypeVar('_BatchT', bound='Batch') + + +class Batch(_BatchBuffer[_Item]): + def __init__( + self, collection: object, step: int = 1000, ordered: bool = True, on_error: _ErrorPolicy = 'raise' + ) -> None: + super().__init__(collection, step, ordered, on_error) + self._lock = Lock() + + def add(self, doc_or_op: _Item) -> None: + """Queue one item, writing the batch when it reaches step.""" + with self._lock: + self.ops.append(doc_or_op) + if self.count >= self.step: + self._flush() + + def extend(self, docs_or_ops: Iterable[_Item]) -> None: + """Consume items incrementally, leaving any final partial batch pending.""" + for item in docs_or_ops: + self.add(item) + + def execute(self) -> None: + """Write the pending batch, if any.""" + with self._lock: + self._flush() + + def _flush(self) -> None: + if not self.ops: + return + try: + write = getattr(self.collection, self._write_method) + self._record_result(write(self.ops, ordered=self.ordered)) + except BulkWriteError as error: + self._record_error(error) + if self.on_error == 'raise': + raise + finally: + self.ops.clear() + def discard(self) -> None: + """Release pending items without sending them to MongoDB.""" + with self._lock: + self.ops.clear() -class Batch: + def close(self) -> None: + """Flush pending items. The batch and its collection remain usable.""" + self.execute() - def __init__(self, collection, step=1000, ordered=True, on_error='raise'): - """初始化批量操作. + def __enter__(self: _BatchT) -> _BatchT: + return self - Args: - collection: pymongo Collection 对象 - step: 批次大小,默认 1000 - ordered: 是否有序操作,默认 True - on_error: 错误处理策略,默认 'raise' - - 'raise': 抛出异常(默认) - - 'log': 记录日志但继续执行 - - 'ignore': 忽略错误 - """ - self.ops = [] - self.count = 0 - self.collection = collection - self.step = step - self.ordered = ordered - self.on_error = on_error + def __exit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> None: + try: + self.close() + except BaseException as error: + if exc is None: + raise + attach_cleanup_errors(exc, (error,)) - self.success_count = 0 - self.error_count = 0 - def add(self, doc_or_op): - self.ops.append(doc_or_op) - self.count += 1 - if self.count >= self.step: - self.execute() +class BatchInsert(Batch[_Document]): + """Buffer documents for batched calls to Collection.insert_many().""" - def extend(self, docs_or_ops): - self.ops.extend(docs_or_ops) - self.count += len(docs_or_ops) - while self.count >= self.step: - self.execute() + _write_method = 'insert_many' + _success_keys = ('nInserted',) - def execute(self): - pass - def close(self): - if self.count: - self.execute() +class BatchUpdate(Batch[UpdateOne]): + """Buffer PyMongo UpdateOne operations or build them from filter/update pairs.""" - def __enter__(self): - return self + _success_keys = ('nModified',) - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() + def add(self, query_or_op: _Document | UpdateOne, update: _Update | None = None) -> None: + super().add(_update_operation(query_or_op, update)) -class BatchInsert(Batch): - - def execute(self): - try: - result = self.collection.insert_many(self.ops, ordered=self.ordered) - self.success_count += len(result.inserted_ids) - except pymongo.errors.BulkWriteError as error: - self.success_count += error.details.get('nInserted', 0) - self.error_count += len(self.ops) - error.details.get('nInserted', 0) - if self.on_error == 'raise': - raise - elif self.on_error == 'log': - logger.warning(f'BatchInsert error: {error.details}') - finally: - self.ops.clear() - self.count = 0 - - -class BatchUpdate(Batch): - def add(self, query_or_op, update=None): - if isinstance(query_or_op, UpdateOne): - super().add(query_or_op) - elif update: - super().add(UpdateOne(query_or_op, update)) - else: - raise pymongo.errors.InvalidOperation('Missing update parameter.') - - def extend(self, ops): - for op in ops: - if isinstance(op, UpdateOne): - super().add(op) - else: - raise pymongo.errors.InvalidOperation('Operation type Error.') - - def execute(self): - try: - result = self.collection.bulk_write(self.ops, ordered=self.ordered) - self.success_count += result.modified_count - except pymongo.errors.BulkWriteError as error: - self.success_count += error.details.get('nModified', 0) - self.error_count += len(self.ops) - error.details.get('nModified', 0) - if self.on_error == 'raise': - raise - elif self.on_error == 'log': - logger.warning(f"BatchUpdate error: {error.details}") - finally: - self.ops.clear() - self.count = 0 +class BatchWrite(Batch[_WriteOperation]): + """Buffer mixed native write models for Collection.bulk_write().""" diff --git a/src/database/mongodb/utils.py b/src/database/mongodb/utils.py index 2890e74..ae40097 100644 --- a/src/database/mongodb/utils.py +++ b/src/database/mongodb/utils.py @@ -1,11 +1,21 @@ +"""Batch writes and cursor utilities for native PyMongo collections.""" + from ._async_utils import AsyncBatchInsert from ._async_utils import AsyncBatchUpdate +from ._async_utils import AsyncBatchWrite +from ._cursor_batches import async_cursor_batches +from ._cursor_batches import cursor_batches from ._sync_utils import BatchInsert from ._sync_utils import BatchUpdate +from ._sync_utils import BatchWrite __all__ = [ 'AsyncBatchInsert', 'AsyncBatchUpdate', + 'AsyncBatchWrite', 'BatchInsert', 'BatchUpdate', + 'BatchWrite', + 'async_cursor_batches', + 'cursor_batches', ] diff --git a/tests/test_mongodb_utils.py b/tests/test_mongodb_utils.py new file mode 100644 index 0000000..e1f6c94 --- /dev/null +++ b/tests/test_mongodb_utils.py @@ -0,0 +1,275 @@ +import asyncio +import unittest + +from pymongo import DeleteOne +from pymongo import InsertOne +from pymongo import ReplaceOne +from pymongo import UpdateMany +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError +from pymongo.errors import InvalidOperation +from pymongo.results import BulkWriteResult +from pymongo.results import InsertManyResult + +from database.mongodb.utils import AsyncBatchInsert +from database.mongodb.utils import AsyncBatchUpdate +from database.mongodb.utils import AsyncBatchWrite +from database.mongodb.utils import BatchInsert +from database.mongodb.utils import BatchUpdate +from database.mongodb.utils import BatchWrite +from database.mongodb.utils import async_cursor_batches +from database.mongodb.utils import cursor_batches + + +class Collection: + def __init__(self, *, error=None, acknowledged=True): + self.calls = [] + self.error = error + self.acknowledged = acknowledged + + def insert_many(self, documents, *, ordered): + self.calls.append((list(documents), ordered)) + if self.error is not None: + raise self.error + return InsertManyResult([doc['_id'] for doc in documents], self.acknowledged) + + def bulk_write(self, operations, *, ordered): + self.calls.append((list(operations), ordered)) + if self.error is not None: + raise self.error + return BulkWriteResult({'nInserted': 1, 'nModified': 2, 'nRemoved': 1, 'nUpserted': 1}, self.acknowledged) + + +class AsyncCollection(Collection): + async def insert_many(self, documents, *, ordered): + return super().insert_many(documents, ordered=ordered) + + async def bulk_write(self, operations, *, ordered): + return super().bulk_write(operations, ordered=ordered) + + +class Cursor: + def __init__(self, rows=range(7), *, close_error=None): + self.rows = iter(rows) + self.read = 0 + self.closed = 0 + self.close_error = close_error + + def __iter__(self): + for row in self.rows: + self.read += 1 + yield row + + def close(self): + self.closed += 1 + if self.close_error is not None: + raise self.close_error + + +class AsyncCursor(Cursor): + async def __aiter__(self): + for row in self: + yield row + + async def close(self): + super().close() + + +class TestBatch(unittest.TestCase): + def test_incremental_extend_flushes_at_the_threshold(self): + collection = Collection() + + def documents(): + for index in range(7): + self.assertEqual(len(collection.calls), index // 3) + yield {'_id': index} + + batch = BatchInsert(collection, step=3, ordered=False) + with batch: + batch.extend(documents()) + self.assertEqual(batch.count, 1) + batch.execute() + batch.close() + self.assertEqual([len(docs) for docs, _ in collection.calls], [3, 3, 1]) + self.assertTrue(all(not ordered for _, ordered in collection.calls)) + self.assertEqual(batch.success_count, 7) + self.assertFalse(batch.ops) + + def test_discard_releases_tail_and_leaves_batch_usable(self): + collection = Collection() + batch = BatchInsert(collection, step=2) + batch.extend({'_id': index} for index in range(3)) + batch.discard() + batch.close() + self.assertEqual(collection.calls, [([{'_id': 0}, {'_id': 1}], True)]) + batch.add({'_id': 3}) + batch.close() + self.assertEqual(collection.calls[-1], ([{'_id': 3}], True)) + + def test_update_shortcut_and_mixed_operations(self): + collection = Collection() + native = UpdateOne({'_id': 1}, {'$set': {'ready': True}}, upsert=True) + with BatchUpdate(collection, step=2) as batch: + batch.add({'_id': 2}, [{'$set': {'ready': True}}]) + batch.extend(iter([native])) + with self.assertRaises(InvalidOperation): + batch.add({'_id': 3}) + self.assertEqual(batch.success_count, 2) + self.assertIs(collection.calls[0][0][1], native) + models = [InsertOne({'_id': 1}), UpdateMany({}, {'$inc': {'n': 1}}), ReplaceOne({}, {}), DeleteOne({})] + with BatchWrite(collection, step=3, ordered=False) as mixed: + mixed.extend(models) + self.assertEqual(collection.calls[1:], [(models[:3], False), (models[3:], False)]) + + def test_bulk_failure_has_raw_evidence_and_releases_buffer(self): + error = BulkWriteError({'nInserted': 1, 'writeErrors': [{'op': {'value': 'private'}}]}) + batch = BatchInsert(Collection(error=error), step=2) + with self.assertRaises(BulkWriteError) as caught: + batch.extend([{'_id': 1}, {'_id': 2}]) + self.assertIs(caught.exception, error) + self.assertEqual((batch.success_count, batch.error_count, batch.count), (1, 1, 0)) + batch.close() + + batch = BatchInsert(Collection(error=error), step=1, on_error='log') + with self.assertLogs('database.mongodb._batch', level='WARNING') as logs: + batch.add({'_id': 1}) + self.assertNotIn('private', '\n'.join(logs.output)) + + def test_context_preserves_primary_error_when_tail_write_fails(self): + error = RuntimeError('driver') + primary = ValueError('producer') + collection = Collection(error=error) + with self.assertRaises(ValueError) as caught: + with BatchInsert(collection) as batch: + batch.add({'_id': 1}) + raise primary + self.assertIs(caught.exception, primary) + self.assertEqual(primary.database_cleanup_errors, (error,)) + self.assertEqual(collection.calls, [([{'_id': 1}], True)]) + self.assertFalse(batch.ops) + + def test_validation_and_unacknowledged_counts(self): + with self.assertRaises(ValueError): + BatchInsert(Collection(), step=0) + with self.assertRaises(ValueError): + BatchUpdate(Collection(), on_error='invalid') + collection = Collection(acknowledged=False) + with BatchWrite(collection) as batch: + batch.add(DeleteOne({})) + self.assertEqual((batch.success_count, batch.error_count, batch.unacknowledged_count), (0, 0, 1)) + + +class TestAsyncBatch(unittest.IsolatedAsyncioTestCase): + async def test_extend_consumes_both_kinds_of_iterable(self): + collection = AsyncCollection() + + async def documents(): + for index in range(5): + self.assertEqual(len(collection.calls), index // 2) + yield {'_id': index} + + async with AsyncBatchInsert(collection, step=2) as batch: + await batch.extend(documents()) + await batch.extend(iter([{'_id': 5}, {'_id': 6}])) + await batch.discard() + self.assertEqual([len(docs) for docs, _ in collection.calls], [2, 2, 2]) + self.assertEqual(batch.success_count, 6) + + async def test_update_and_mixed_chunk_tail(self): + collection = AsyncCollection() + update = UpdateOne({'_id': 1}, {'$inc': {'n': 1}}) + async with AsyncBatchUpdate(collection, step=2) as batch: + await batch.extend([update]) + await batch.add({'_id': 2}, {'$unset': {'n': True}}) + self.assertIs(collection.calls[0][0][0], update) + self.assertEqual(batch.success_count, 2) + delete = DeleteOne({'_id': 3}) + async with AsyncBatchWrite(collection, step=2) as mixed: + await mixed.add(delete) + self.assertEqual(collection.calls[-1], ([delete], True)) + + async def test_concurrent_add_and_cancelled_flush_do_not_replay_or_drop_other_items(self): + first_started = asyncio.Event() + + class PausedCollection(AsyncCollection): + async def insert_many(self, documents, *, ordered): + if documents[0]['_id'] == 0: + first_started.set() + await asyncio.Future() + return await super().insert_many(documents, ordered=ordered) + + collection = PausedCollection() + batch = AsyncBatchInsert(collection, step=1) + first = asyncio.create_task(batch.add({'_id': 0})) + await first_started.wait() + second = asyncio.create_task(batch.add({'_id': 1})) + await asyncio.sleep(0) + self.assertFalse(second.done()) + first.cancel() + with self.assertRaises(asyncio.CancelledError): + await first + await second + await batch.close() + self.assertEqual(collection.calls, [([{'_id': 1}], True)]) + self.assertFalse(batch.ops) + + +class TestCursorBatches(unittest.TestCase): + def test_normal_consumption_and_early_exit(self): + cursor = Cursor() + with cursor_batches(cursor, step=3) as batches: + self.assertEqual(list(batches), [[0, 1, 2], [3, 4, 5], [6]]) + self.assertEqual(cursor.closed, 1) + cursor = Cursor() + with cursor_batches(cursor, step=3) as batches: + self.assertEqual(next(batches), [0, 1, 2]) + self.assertEqual(cursor.read, 3) + self.assertEqual(list(batches), []) + self.assertEqual(cursor.closed, 1) + + def test_cursor_iteration_and_close_failures_preserve_primary_error(self): + primary = ValueError('read') + cleanup = RuntimeError('close') + + def rows(): + yield 1 + raise primary + + cursor = Cursor(rows(), close_error=cleanup) + with self.assertRaises(ValueError) as caught: + with cursor_batches(cursor, step=2) as batches: + next(batches) + self.assertEqual(cursor.closed, 1) + self.assertIs(caught.exception, primary) + self.assertEqual(primary.database_cleanup_errors, (cleanup,)) + + +class TestAsyncCursorBatches(unittest.IsolatedAsyncioTestCase): + async def test_normal_consumption_and_early_exit(self): + cursor = AsyncCursor() + async with async_cursor_batches(cursor, step=3) as batches: + self.assertEqual([batch async for batch in batches], [[0, 1, 2], [3, 4, 5], [6]]) + self.assertEqual(cursor.closed, 1) + cursor = AsyncCursor() + async with async_cursor_batches(cursor, step=3) as batches: + self.assertEqual(await anext(batches), [0, 1, 2]) + self.assertEqual(cursor.read, 3) + self.assertEqual([batch async for batch in batches], []) + self.assertEqual(cursor.closed, 1) + + async def test_consumer_cancellation_closes_cursor(self): + cursor = AsyncCursor() + entered = asyncio.Event() + + async def consume(): + async with async_cursor_batches(cursor, step=3) as batches: + await anext(batches) + entered.set() + await asyncio.Future() + + consumer = asyncio.create_task(consume()) + await entered.wait() + consumer.cancel() + with self.assertRaises(asyncio.CancelledError): + await consumer + self.assertEqual(cursor.closed, 1) -- Gitee From c288b0133bee00e7b66b03dcee0ce35528d4e3e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 14:38:02 +0800 Subject: [PATCH 75/86] =?UTF-8?q?feat(mongodb):=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E5=88=86=E9=A1=B5=E8=AF=BB=E5=8F=96=E5=92=8C=E6=B5=81=E5=BC=8F?= =?UTF-8?q?=E7=BB=AD=E8=AF=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 提供同步和异步集合扫描工具。分页读完即关闭游标,流式读取遇到可恢复中断时按最后交付的 _id 继续。 提交前通过 290 项完整离线测试、Ruff 检查和暂存差异检查。 --- CHANGELOG.md | 1 + README.md | 24 +++ src/database/mongodb/_batch.py | 4 +- src/database/mongodb/_scan.py | 266 +++++++++++++++++++++++++++ src/database/mongodb/utils.py | 8 + tests/test_mongodb_scan.py | 318 +++++++++++++++++++++++++++++++++ 6 files changed, 619 insertions(+), 2 deletions(-) create mode 100644 src/database/mongodb/_scan.py create mode 100644 tests/test_mongodb_scan.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b470fc..31247e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### 新增 +- 增加 MongoDB 同步与异步分页读取和流式续读工具,按 `_id` 索引定位,可在游标失效或连接中断后有界恢复。 - 增加 MongoDB 同步与异步混合批写、待写缓冲丢弃和自动关闭游标的分批读取工具。异步批处理支持消费异步可迭代对象。 - 新增同步/异步 Schema 基础管理:MySQL 库表创建、兼容校验、目录查看和删除;MongoDB 集合准备、必要身份约束及目录管理。初始化显式执行,不自动迁移。 - 新增 `RedisStrings` / `AsyncRedisStrings` 的单键、批量读写、删除和 TTL,保留逐项执行证据、原始错误与取消信号。 diff --git a/README.md b/README.md index 02cd6c8..edb119a 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,30 @@ async def migrate_async(source, target): await writer.extend(page) ``` +#### MongoDB 分页读取与流式续读 + +`scan_collection()` 和 `stream_collection()` 接收集合与筛选条件,按 `_id` 升序逐条交付。适用于具有常规 `_id_` 索引、简单排序规则和全局唯一 `_id` 的普通集合。续读用索引下界定位,保留原筛选条件。每次查询读取当时可见的数据,恢复位置保存在本次遍历中。 + +```python +from database.mongodb.utils import scan_collection, stream_collection + + +def process_in_pages(articles, process): + for article in scan_collection(articles, {'approved': True}, step=1000): + process(article) + + +def process_in_stream(articles, process): + with stream_collection(articles, {'approved': True}, batch_size=1) as rows: + for article in rows: + process(article) +``` + +- 分页在交付前读完并关闭本页游标,适合批间有长时间等待的任务。默认每页最多 1000 条,内存开销随页大小和文档大小增长。默认 `max_retries=0`,显式开启后从上一页末尾的断点重读失败页。 +- 流式在游标有效时按需取数,遇到可恢复的读取中断时从最后交付的 `_id` 续读。默认 `batch_size=1`,同一位置最多额外尝试 3 次。离开 `with` 时关闭当前游标。较大的驱动批次可减少网络往返,也会增加缓冲。 +- 可恢复的读取中断包括游标失效、查询计划被终止和 `AutoReconnect` 类连接中断。权限错误、查询执行超时、业务异常和取消信号直接传播。重试次数耗尽时抛出最后一次原生异常。`max_retries` 控制 SDK 的恢复次数,驱动的读重试由 PyMongo 客户端选项单独控制。 +- 异步使用 `async_scan_collection()` 配合 `async for`,或使用 `async_stream_collection()` 配合 `async with` 和 `async for`。 + ### MySQL 使用 MySQL Connector/Python。`create_async()` 需要 `await`,驱动完成建连后返回原生异步连接。示例从 `MYSQL_USER` 和 `MYSQL_PASSWORD` 环境变量读取凭据。 diff --git a/src/database/mongodb/_batch.py b/src/database/mongodb/_batch.py index a3e715c..a85a5e7 100644 --- a/src/database/mongodb/_batch.py +++ b/src/database/mongodb/_batch.py @@ -28,9 +28,9 @@ _WriteOperation = InsertOne | UpdateOne | UpdateMany | ReplaceOne | DeleteOne | _ErrorPolicy = Literal['raise', 'log', 'ignore'] -def _validate_step(step: int) -> None: +def _validate_step(step: int, *, name: str = 'step') -> None: if not isinstance(step, int) or isinstance(step, bool) or step < 1: - raise ValueError('step must be a positive integer.') + raise ValueError(f'{name} must be a positive integer.') def _update_operation(query_or_op: _Document | UpdateOne, update: _Update | None) -> UpdateOne: diff --git a/src/database/mongodb/_scan.py b/src/database/mongodb/_scan.py new file mode 100644 index 0000000..fd02093 --- /dev/null +++ b/src/database/mongodb/_scan.py @@ -0,0 +1,266 @@ +"""Ordered collection traversal with bounded cursor recovery.""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from collections.abc import AsyncIterator +from collections.abc import Generator +from collections.abc import Iterator +from collections.abc import Mapping +from contextlib import AsyncExitStack +from contextlib import ExitStack +from contextlib import aclosing +from contextlib import asynccontextmanager +from contextlib import closing +from contextlib import contextmanager +from copy import deepcopy +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar + +from bson import MaxKey +from pymongo.errors import AutoReconnect +from pymongo.errors import CursorNotFound +from pymongo.errors import OperationFailure +from pymongo.errors import PyMongoError + +from ._batch import _validate_step +from ._resources import _async_closing +from ._resources import _closing + +if TYPE_CHECKING: + from pymongo.asynchronous.collection import AsyncCollection + from pymongo.collection import Collection + +_Filter = Mapping[str, Any] +_Document = TypeVar('_Document', bound=Mapping[str, Any]) +_START = object() +_CURSOR_LOST_CODES = frozenset({43, 175, 237}) # CursorNotFound, QueryPlanKilled, CursorKilled + + +def _validate_retries(max_retries: int) -> None: + if not isinstance(max_retries, int) or isinstance(max_retries, bool) or max_retries < 0: + raise ValueError('max_retries must be a non-negative integer.') + + +def _with_id_filter(filter: _Filter | None, id_filter: _Filter) -> _Filter: + return {'$and': [filter, id_filter]} if filter else id_filter + + +def _find_options(filter: _Filter | None, last_id: Any, *, batch_size: int, limit: int = 0) -> dict[str, Any]: + options: dict[str, Any] = { + 'filter': filter, + 'sort': [('_id', 1)], + 'hint': '_id_', + 'batch_size': batch_size, + 'limit': limit, + } + if last_id is not _START: + # The index lower bound is inclusive and follows BSON sort order across types. + options['min'] = {'_id': last_id} + options['filter'] = _with_id_filter(filter, {'_id': {'$ne': last_id}}) + return options + + +def _max_id_options(filter: _Filter | None) -> dict[str, Any]: + # A min-bounded scan has an implicit exclusive upper bound at MaxKey. + return _find_options(_with_id_filter(filter, {'_id': {'$eq': MaxKey()}}), _START, batch_size=1, limit=1) + + +def _can_retry(error: Exception, retries_left: int) -> bool: + if retries_left == 0 or getattr(error, 'database_cleanup_errors', ()): + return False + return isinstance(error, (AutoReconnect, CursorNotFound)) or ( + isinstance(error, OperationFailure) and error.code in _CURSOR_LOST_CODES + ) + + +def _read_page(collection: Collection[_Document], options: dict[str, Any], max_retries: int) -> list[_Document]: + while True: + read_complete = False + try: + with _closing(collection.find(**options)) as cursor: + page = list(cursor) + read_complete = True + return page + except PyMongoError as error: + if read_complete or not _can_retry(error, max_retries): + raise + max_retries -= 1 + + +async def _async_read_page( + collection: AsyncCollection[_Document], options: dict[str, Any], max_retries: int +) -> list[_Document]: + while True: + read_complete = False + try: + async with _async_closing(collection.find(**options)) as cursor: + page = [document async for document in cursor] + read_complete = True + return page + except PyMongoError as error: + if read_complete or not _can_retry(error, max_retries): + raise + max_retries -= 1 + + +def scan_collection( + collection: Collection[_Document], filter: _Filter | None = None, *, step: int = 1000, max_retries: int = 0 +) -> Iterator[_Document]: + """Yield documents in _id order, closing each page cursor before delivery. + + Use a regular collection with a simple collation and globally unique _id. + max_retries counts additional read attempts per page. + """ + _validate_step(step) + _validate_retries(max_retries) + last_id = _START + while True: + page = _read_page(collection, _find_options(filter, last_id, batch_size=step, limit=step), max_retries) + end_id = deepcopy(page[-1]['_id']) if page else last_id + full_page = len(page) == step + yield from page + del page + if isinstance(end_id, MaxKey): + return + if not full_page: + if last_id is not _START: + yield from _read_page(collection, _max_id_options(filter), max_retries) + return + last_id = end_id + + +async def async_scan_collection( + collection: AsyncCollection[_Document], filter: _Filter | None = None, *, step: int = 1000, max_retries: int = 0 +) -> AsyncIterator[_Document]: + """Yield async documents in _id order, closing each page cursor before delivery.""" + _validate_step(step) + _validate_retries(max_retries) + last_id = _START + while True: + page = await _async_read_page( + collection, _find_options(filter, last_id, batch_size=step, limit=step), max_retries + ) + end_id = deepcopy(page[-1]['_id']) if page else last_id + full_page = len(page) == step + for document in page: + yield document + del document + del page + if isinstance(end_id, MaxKey): + return + if not full_page: + if last_id is not _START: + for document in await _async_read_page(collection, _max_id_options(filter), max_retries): + yield document + return + last_id = end_id + + +def _stream( + collection: Collection[_Document], filter: _Filter | None, batch_size: int, max_retries: int, resources: ExitStack +) -> Generator[_Document, None, None]: + cursor = None + bounded = False + last_id = _START + retries_left = max_retries + while True: + try: + if cursor is None: + bounded = last_id is not _START + cursor = resources.enter_context( + _closing(collection.find(**_find_options(filter, last_id, batch_size=batch_size))) + ) + document = next(cursor) + except StopIteration: + resources.close() + if bounded and not isinstance(last_id, MaxKey): + yield from _read_page(collection, _max_id_options(filter), retries_left) + return + except PyMongoError as error: + resources.__exit__(type(error), error, error.__traceback__) + if not _can_retry(error, retries_left): + raise + retries_left -= 1 + cursor = None + continue + last_id = deepcopy(document['_id']) + retries_left = max_retries + yield document + del document + if isinstance(last_id, MaxKey): + resources.close() + return + + +async def _async_stream( + collection: AsyncCollection[_Document], + filter: _Filter | None, + batch_size: int, + max_retries: int, + resources: AsyncExitStack, +) -> AsyncGenerator[_Document, None]: + cursor = None + bounded = False + last_id = _START + retries_left = max_retries + while True: + try: + if cursor is None: + bounded = last_id is not _START + cursor = await resources.enter_async_context( + _async_closing(collection.find(**_find_options(filter, last_id, batch_size=batch_size))) + ) + document = await anext(cursor) + except StopAsyncIteration: + await resources.aclose() + if bounded and not isinstance(last_id, MaxKey): + for document in await _async_read_page(collection, _max_id_options(filter), retries_left): + yield document + return + except PyMongoError as error: + await resources.__aexit__(type(error), error, error.__traceback__) + if not _can_retry(error, retries_left): + raise + retries_left -= 1 + cursor = None + continue + last_id = deepcopy(document['_id']) + retries_left = max_retries + yield document + del document + if isinstance(last_id, MaxKey): + await resources.aclose() + return + + +@contextmanager +def stream_collection( + collection: Collection[_Document], filter: _Filter | None = None, *, batch_size: int = 1, max_retries: int = 3 +) -> Iterator[Iterator[_Document]]: + """Yield a streaming iterator that can reopen at the last delivered _id. + + The context owns the active cursor. max_retries counts additional read + attempts at the same position. + """ + _validate_step(batch_size, name='batch_size') + _validate_retries(max_retries) + # Keep cursor ownership outside the generator so closing it on break + # reports any cursor cleanup failure to the surrounding context. + with ExitStack() as resources, closing(_stream(collection, filter, batch_size, max_retries, resources)) as rows: + yield rows + + +@asynccontextmanager +async def async_stream_collection( + collection: AsyncCollection[_Document], filter: _Filter | None = None, *, batch_size: int = 1, max_retries: int = 3 +) -> AsyncIterator[AsyncIterator[_Document]]: + """Yield an async streaming iterator and close its active cursor on exit.""" + _validate_step(batch_size, name='batch_size') + _validate_retries(max_retries) + async with ( + AsyncExitStack() as resources, + aclosing(_async_stream(collection, filter, batch_size, max_retries, resources)) as rows, + ): + yield rows diff --git a/src/database/mongodb/utils.py b/src/database/mongodb/utils.py index ae40097..221ec77 100644 --- a/src/database/mongodb/utils.py +++ b/src/database/mongodb/utils.py @@ -5,6 +5,10 @@ from ._async_utils import AsyncBatchUpdate from ._async_utils import AsyncBatchWrite from ._cursor_batches import async_cursor_batches from ._cursor_batches import cursor_batches +from ._scan import async_scan_collection +from ._scan import async_stream_collection +from ._scan import scan_collection +from ._scan import stream_collection from ._sync_utils import BatchInsert from ._sync_utils import BatchUpdate from ._sync_utils import BatchWrite @@ -17,5 +21,9 @@ __all__ = [ 'BatchUpdate', 'BatchWrite', 'async_cursor_batches', + 'async_scan_collection', + 'async_stream_collection', 'cursor_batches', + 'scan_collection', + 'stream_collection', ] diff --git a/tests/test_mongodb_scan.py b/tests/test_mongodb_scan.py new file mode 100644 index 0000000..fd10edb --- /dev/null +++ b/tests/test_mongodb_scan.py @@ -0,0 +1,318 @@ +import asyncio +import unittest + +from bson import MaxKey +from bson import ObjectId +from pymongo.errors import AutoReconnect +from pymongo.errors import CursorNotFound +from pymongo.errors import ExecutionTimeout +from pymongo.errors import NetworkTimeout +from pymongo.errors import OperationFailure +from pymongo.errors import ServerSelectionTimeoutError + +from database.mongodb.utils import async_scan_collection +from database.mongodb.utils import async_stream_collection +from database.mongodb.utils import scan_collection +from database.mongodb.utils import stream_collection + + +class Cursor: + def __init__(self, *results, close_error=None): + self.results = iter(results) + self.read = 0 + self.closed = 0 + self.close_error = close_error + + def __iter__(self): + return self + + def __next__(self): + result = next(self.results) + if isinstance(result, BaseException): + raise result + self.read += 1 + return {'_id': result} + + def close(self): + self.closed += 1 + if self.close_error is not None: + raise self.close_error + + +class AsyncCursor(Cursor): + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self) + except StopIteration: + raise StopAsyncIteration from None + + async def close(self): + super().close() + + +class Collection: + def __init__(self, *results): + self.results = iter(results) + self.calls = [] + self.cursors = [] + + def find(self, **options): + if self.cursors and self.cursors[-1].closed != 1: + raise AssertionError('The previous cursor must be closed before opening another.') + self.calls.append(options) + try: + result = next(self.results) + except StopIteration: + raise AssertionError('Unexpected find call.') from None + if isinstance(result, BaseException): + raise result + self.cursors.append(result) + return result + + +class TestScanCollection(unittest.TestCase): + def test_default_page_is_closed_before_first_delivery(self): + cursor = Cursor(1, 2) + collection = Collection(cursor) + rows = scan_collection(collection) + self.assertEqual(next(rows), {'_id': 1}) + self.assertEqual((cursor.read, cursor.closed), (2, 1)) + self.assertEqual(list(rows), [{'_id': 2}]) + self.assertEqual(collection.calls[0]['limit'], 1000) + self.assertEqual(collection.calls[0]['sort'], [('_id', 1)]) + self.assertEqual(collection.calls[0]['hint'], '_id_') + + def test_each_page_preserves_filter_and_bson_boundary(self): + oid = ObjectId() + query = {'_id': {'$ne': 'hidden'}, '$or': [{'ready': True}, {'urgent': True}]} + collection = Collection(Cursor(None), Cursor('abc'), Cursor(oid), Cursor(), Cursor(MaxKey())) + self.assertEqual( + [doc['_id'] for doc in scan_collection(collection, query, step=1)], + [None, 'abc', oid, MaxKey()], + ) + self.assertNotIn('min', collection.calls[0]) + for call, previous in zip(collection.calls[1:], [None, 'abc', oid]): + self.assertEqual(call['min'], {'_id': previous}) + self.assertEqual(call['filter'], {'$and': [query, {'_id': {'$ne': previous}}]}) + self.assertEqual(collection.calls[-1]['filter'], {'$and': [query, {'_id': {'$eq': MaxKey()}}]}) + self.assertEqual(query, {'_id': {'$ne': 'hidden'}, '$or': [{'ready': True}, {'urgent': True}]}) + self.assertTrue(all(cursor.closed == 1 for cursor in collection.cursors)) + + def test_retry_is_off_by_default_even_after_partial_read(self): + error = CursorNotFound('expired', code=43) + cursor = Cursor(1, error) + collection = Collection(cursor) + with self.assertRaises(CursorNotFound) as raised: + next(scan_collection(collection)) + self.assertIs(raised.exception, error) + self.assertEqual((len(collection.calls), cursor.read, cursor.closed), (1, 1, 1)) + + def test_opt_in_rereads_entire_failed_page_at_same_boundary(self): + collection = Collection( + Cursor(1, 2), Cursor(3, NetworkTimeout('socket timeout')), Cursor(3, 4), Cursor(5), Cursor() + ) + self.assertEqual( + [doc['_id'] for doc in scan_collection(collection, step=2, max_retries=1)], + [1, 2, 3, 4, 5], + ) + self.assertEqual(collection.calls[1], collection.calls[2]) + self.assertEqual(collection.calls[2]['min'], {'_id': 2}) + self.assertTrue(all(cursor.closed == 1 for cursor in collection.cursors)) + + def test_close_error_is_preserved_without_retrying_a_successfully_read_page(self): + error = AutoReconnect('close') + collection = Collection(Cursor(1, close_error=error)) + with self.assertRaises(AutoReconnect) as raised: + next(scan_collection(collection, max_retries=1)) + self.assertIs(raised.exception, error) + self.assertEqual(len(collection.calls), 1) + + def test_invalid_arguments_start_no_query(self): + collection = Collection() + with self.assertRaisesRegex(ValueError, 'step'): + next(scan_collection(collection, step=0)) + with self.assertRaisesRegex(ValueError, 'max_retries'): + next(scan_collection(collection, max_retries=True)) + self.assertFalse(collection.calls) + + +class TestStreamCollection(unittest.TestCase): + def test_evaluates_only_on_demand_and_closes_on_early_exit(self): + cursor = Cursor(1, 2, 3) + collection = Collection(cursor) + with stream_collection(collection) as rows: + self.assertFalse(collection.calls) + self.assertEqual(next(rows), {'_id': 1}) + self.assertEqual((cursor.read, cursor.closed), (1, 0)) + self.assertEqual(collection.calls[0]['batch_size'], 1) + self.assertEqual(collection.calls[0]['limit'], 0) + self.assertEqual(cursor.closed, 1) + self.assertEqual(list(rows), []) + + def test_reopens_after_last_delivery_and_resets_retry_budget_on_progress(self): + oid = ObjectId() + query = {'ready': True} + collection = Collection( + Cursor(None, CursorNotFound('expired')), + Cursor('abc', OperationFailure('cursor killed', code=237)), + Cursor(oid, NetworkTimeout('connection timed out')), + Cursor(4), + Cursor(MaxKey()), + ) + with stream_collection(collection, query, batch_size=5, max_retries=1) as rows: + self.assertEqual([doc['_id'] for doc in rows], [None, 'abc', oid, 4, MaxKey()]) + for call, previous in zip(collection.calls[1:], [None, 'abc', oid]): + self.assertEqual(call['min'], {'_id': previous}) + self.assertEqual(call['filter'], {'$and': [query, {'_id': {'$ne': previous}}]}) + self.assertEqual(call['batch_size'], 5) + self.assertEqual(collection.calls[-1]['filter'], {'$and': [query, {'_id': {'$eq': MaxKey()}}]}) + self.assertTrue(all(cursor.closed == 1 for cursor in collection.cursors)) + + def test_embedded_id_checkpoint_is_captured_before_document_delivery(self): + collection = Collection(Cursor({'part': 1}, OperationFailure('plan killed', code=175)), Cursor(), Cursor()) + with stream_collection(collection) as rows: + next(rows)['_id']['part'] = 2 + self.assertEqual(list(rows), []) + self.assertEqual(collection.calls[1]['min'], {'_id': {'part': 1}}) + + def test_exhausted_retries_raise_last_native_error(self): + error = ServerSelectionTimeoutError('selection') + collection = Collection(Cursor(1, AutoReconnect('unavailable')), error) + with self.assertRaises(ServerSelectionTimeoutError) as raised: + with stream_collection(collection, max_retries=1) as rows: + self.assertEqual(next(rows), {'_id': 1}) + next(rows) + self.assertIs(raised.exception, error) + self.assertEqual(collection.calls[1]['min'], {'_id': 1}) + self.assertEqual(collection.cursors[0].closed, 1) + + def test_initial_network_failure_retries_without_a_checkpoint(self): + collection = Collection(AutoReconnect('initial'), Cursor(1)) + with stream_collection(collection) as rows: + self.assertEqual(list(rows), [{'_id': 1}]) + self.assertEqual(collection.calls[0], collection.calls[1]) + + def test_permission_and_query_timeout_errors_are_propagated(self): + for error in (OperationFailure('unauthorized', code=13), ExecutionTimeout('server time limit', code=50)): + with self.subTest(error=type(error)): + cursor = Cursor(1, error) + collection = Collection(cursor) + with self.assertRaises(type(error)) as raised: + with stream_collection(collection) as rows: + list(rows) + self.assertIs(raised.exception, error) + self.assertEqual((len(collection.calls), cursor.closed), (1, 1)) + + def test_read_error_retains_failed_cleanup(self): + error = CursorNotFound('read') + cleanup = RuntimeError('close') + collection = Collection(Cursor(error, close_error=cleanup)) + with self.assertRaises(CursorNotFound) as raised: + with stream_collection(collection) as rows: + next(rows) + self.assertIs(raised.exception, error) + self.assertEqual(error.database_cleanup_errors, (cleanup,)) + self.assertEqual(len(collection.calls), 1) + + def test_body_error_retains_failed_cleanup_and_never_retries(self): + error = CursorNotFound('from consumer') + cleanup = AutoReconnect('close') + collection = Collection(Cursor(1, close_error=cleanup)) + with self.assertRaises(CursorNotFound) as raised: + with stream_collection(collection) as rows: + next(rows) + raise error + self.assertIs(raised.exception, error) + self.assertEqual(error.database_cleanup_errors, (cleanup,)) + self.assertEqual(len(collection.calls), 1) + + def test_early_close_error_is_visible_without_a_body_error(self): + error = RuntimeError('close') + collection = Collection(Cursor(1, close_error=error)) + with self.assertRaises(RuntimeError) as raised: + with stream_collection(collection) as rows: + next(rows) + self.assertIs(raised.exception, error) + + def test_invalid_batch_size_starts_no_query(self): + collection = Collection() + with self.assertRaisesRegex(ValueError, 'batch_size'): + with stream_collection(collection, batch_size=0): + pass + self.assertFalse(collection.calls) + + +class TestAsyncCollectionReads(unittest.IsolatedAsyncioTestCase): + async def test_scan_delivers_only_after_close_and_rereads_partial_page(self): + collection = Collection( + AsyncCursor(1, 2), + AsyncCursor(3, NetworkTimeout('timeout')), + AsyncCursor(3, 4), + AsyncCursor(), + AsyncCursor(), + ) + rows = async_scan_collection(collection, step=2, max_retries=1) + self.assertEqual(await anext(rows), {'_id': 1}) + self.assertEqual(collection.cursors[0].closed, 1) + self.assertEqual([doc['_id'] async for doc in rows], [2, 3, 4]) + self.assertEqual(collection.calls[1], collection.calls[2]) + self.assertTrue(all(cursor.closed == 1 for cursor in collection.cursors)) + + async def test_async_scan_does_not_retry_by_default(self): + error = CursorNotFound('expired') + collection = Collection(AsyncCursor(1, error)) + with self.assertRaises(CursorNotFound) as raised: + await anext(async_scan_collection(collection)) + self.assertIs(raised.exception, error) + self.assertEqual((len(collection.calls), collection.cursors[0].closed), (1, 1)) + + async def test_stream_reuses_cursor_until_interruption_then_recovers(self): + first = AsyncCursor(1, 2, OperationFailure('plan killed', code=175)) + second = AsyncCursor(3, 4) + collection = Collection(first, second) + async with async_stream_collection(collection) as rows: + self.assertEqual(await anext(rows), {'_id': 1}) + self.assertEqual((first.read, first.closed, len(collection.calls)), (1, 0, 1)) + self.assertEqual(await anext(rows), {'_id': 2}) + self.assertEqual(await anext(rows), {'_id': 3}) + self.assertEqual(collection.calls[1]['min'], {'_id': 2}) + self.assertEqual((first.closed, second.closed), (1, 1)) + self.assertEqual([row async for row in rows], []) + + async def test_async_stream_cleanup_error_is_reported(self): + error = RuntimeError('close') + collection = Collection(AsyncCursor(1, close_error=error)) + with self.assertRaises(RuntimeError) as raised: + async with async_stream_collection(collection) as rows: + await anext(rows) + self.assertIs(raised.exception, error) + + async def test_cancellation_during_read_closes_cursor_without_retrying(self): + for read_type in ('scan', 'stream'): + with self.subTest(read_type=read_type): + reading = asyncio.Event() + + class WaitingCursor(AsyncCursor): + async def __anext__(self): + reading.set() + await asyncio.Future() + + cursor = WaitingCursor() + collection = Collection(cursor) + + async def consume(): + if read_type == 'scan': + await anext(async_scan_collection(collection, max_retries=1)) + else: + async with async_stream_collection(collection) as rows: + await anext(rows) + + task = asyncio.create_task(consume()) + await asyncio.wait_for(reading.wait(), timeout=2) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertEqual((len(collection.calls), cursor.closed), (1, 1)) -- Gitee From 5208a2e34f5625c522bfcf82560d87b8ec2e9ccc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 15:55:07 +0800 Subject: [PATCH 76/86] =?UTF-8?q?docs(accounts):=20=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E4=B8=89=E7=A7=8D=E6=95=B0=E6=8D=AE=E5=BA=93=E7=9A=84=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=E4=B8=8E=E6=9D=83=E9=99=90=E7=AE=A1=E7=90=86=E9=9C=80?= =?UTF-8?q?=E6=B1=82=E5=92=8C=E5=AE=9E=E7=8E=B0=E8=A7=84=E5=88=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 明确 MongoDB、MySQL 和 Redis 的账号身份、授权范围与验收条件,规划同步和异步 Python 工具的实现顺序。已检查暂存差异及文档格式。 --- docs/account-management-design.md | 201 ++++++++++++++++++++++++ docs/account-management-requirements.md | 86 ++++++++++ 2 files changed, 287 insertions(+) create mode 100644 docs/account-management-design.md create mode 100644 docs/account-management-requirements.md diff --git a/docs/account-management-design.md b/docs/account-management-design.md new file mode 100644 index 0000000..46b9358 --- /dev/null +++ b/docs/account-management-design.md @@ -0,0 +1,201 @@ +# MongoDB、MySQL、Redis 账号与权限管理工具实现设计 + +本文是 `database.accounts` 首版的实施规划,供实现、审查与验收使用。下列名称、签名和示例均为拟新增的 Python 能力。需求编号 R1–R9、使用场景和操作权限表见[需求文档](account-management-requirements.md)。既有连接与异常实现见[连接生命周期](../src/database/connections.py)和[异常转换](../src/database/errors.py)。 + +## 1. 组件与执行流程 + +1. 调用方把已有的本库连接句柄或原生驱动对象交给 `Accounts` 或 `AsyncAccounts` 的后端工厂。绑定阶段检查对象类型、后端、同步模式和适用拓扑,不发起网络请求。 +2. 返回后端专用的管理对象。模型与公共入口不导入数据库驱动,选定后端时再加载适配代码。管理对象只借用连接和连接池,操作方法只释放自己创建的资源。 +3. 调用方法时校验结构化参数和服务端所需的能力,构造一条命令或一个有明确输出的查询。权限裁决交给服务端。创建、授予角色、设置默认角色、持久化等可单独提交的步骤分别调用和确认。 +4. 对确认回复只投影公开字段。驱动错误复用现有分类并保留原因,已发送写入的确认失败走专门的结果未知路径。 + +| 入口 | 接受的输入 | 绑定后检查 | +| --- | --- | --- | +| `Accounts.mongodb(conn)` | 本库同步 MongoDB 句柄或 PyMongo `MongoClient` | 用户命令选择认证库,角色命令选择定义库 | +| `Accounts.mysql(conn)` | 本库同步 MySQL 句柄或 Connector/Python 的纯 Python、C 扩展及受支持的池化连接 | 执行写入前检查事务与自动提交状态 | +| `Accounts.redis(conn)` | 本库单节点 Redis 句柄或 redis-py 固定数据节点 `Redis` | 检查句柄模式及原生连接池类型,拒绝集群路由和可识别的 Sentinel 自动发现池 | +| `AsyncAccounts` 的同名工厂 | 本库 `AsyncConnection` 或对应后端的原生异步对象 | 工厂同步返回管理对象,执行方法使用 `await`。本库句柄还要检查所属事件循环 | + +绑定管理对象后,调用方若已关闭本库句柄,操作开始时报告 `ConnectionClosedError`。原生对象按驱动自己的关闭、重连和事件循环契约处理。绑定阶段只访问已知的类型或明确的属性,避免对 PyMongo 客户端使用会隐式生成数据库对象的属性探测。可沿用已有 schema 模块的绑定和异常上下文设计,扩展原生连接这条路径。 [已有绑定实现](../src/database/schema/_base.py) · [已有工厂](../src/database/schema/__init__.py) + +## 2. 公共输入、返回与错误 + +| 模型 | 拟提供的字段或行为 | +| --- | --- | +| `MongoUser(auth_db, name)`、`MongoRole(db, name)` | 两个字段共同确定操作目标,命令中明确指定角色定义库 | +| `MongoPrivilege(resource, actions)` | BSON 资源描述和动作集合。保留空库名、空集合名等有原生含义的作用域;构造时复制调用方容器 | +| `MySQLAccount(user, host)` | 两部分独立传递,用户名和来源主机分别按服务端规则处理。调用方显式选择通配主机 | +| `MySQLScope` | 用不同构造器表示 `global_()`、`database(name)`、`table(db, name)`、`columns(db, table, names)`、`procedure(db, name)`、`function(db, name)` | +| `AclRule` | 有序规则。构造器表达启停、命令和类别增减、键或频道模式、清除旧规则、选择器、添加密码、删除单个密码、替换全部密码。密码参数从 `repr` 中排除 | +| 查询结果 | 列表返回类型化摘要,精确查询返回后端详情或 `None`。MongoDB 记录展开状态,MySQL 保留逐条授权原文和指定角色集合,Redis 投影可展示字段 | +| 写入结果 | 已确认的 MongoDB/MySQL 单项变更及 Redis 规则应用、保存正常返回 `None`。Redis 删除操作按服务器确认的删除计数返回 `bool`,权限模拟返回含 `allowed` 和 `reason` 的结果 | + +**错误与执行证据** + +| 情况 | 调用方收到的结果 | 后续核对 | +| --- | --- | --- | +| 本地参数错误、连接模式错误、写入前发现活动事务 | 已有配置或输入校验错误。管理命令尚未发送 | 修正输入或使用调用方准备的管理连接 | +| 认证失败、授权不足、明确的服务端拒绝 | 分类异常、操作名、目标和原始原因。权限拒绝不会转成空列表 | 检查操作者的授权,不修改未涉及的其他账号 | +| 精确读取成功且服务端确认缺失 | `get_user`、`get_role`、`get_account` 返回 `None` | 盘点范围、原生重复创建与删除错误分别处理 | +| 服务端缺少请求的能力 | 明确报告所需版本或特性及原始错误 | 改用该版本可用的操作。管理员缺权和服务端命令不可用分别分类 | +| 写入已交给驱动,随后丢失确认、超时且无法排除已执行,或 MongoDB 回复写关注错误 | `AccountWriteOutcomeUnknown(DatabaseError)`,包含后端、目标、操作和原始异常或服务端确认信息 | 新连接回读身份、角色或 ACL。改密使用获准的独立认证验证 | +| 异步任务取消 | 传播 `CancelledError`。取消可能发生在写入发送之后 | 异步调用者可按同样流程核对状态 | +| 多步操作只完成前几步 | 已返回成功的步骤保持各自的确认状态,失败步骤单独报告 | 例如 MySQL 开户成功后只补授权,Redis 规则已应用而保存失败时只核对运行时状态与写盘条件 | + +SDK 不为账号写入增加自动重放。借入原生客户端时,驱动自身的重试配置和行为要写入使用说明,测试按驱动版本记录实际发送次数。读请求失败继续按读取错误处理。只有明确处于写入阶段并存在执行歧义时才转换为结果未知。异常摘要使用操作名和安全目标字段,避免将 SQL、BSON、ACL 规则或原始响应插入日志。保留的异常链仅供受控诊断。 [MongoDB 写关注](https://www.mongodb.com/docs/manual/reference/write-concern/) · [仓库安全约定](../SECURITY.md) + +## 3. 后端方法与原生命令 + +### 3.1 MongoDB + +| Python 方法 | 原生调用和契约 | +| --- | --- | +| `list_users(auth_db=...)` | 在目标认证库执行 `usersInfo: 1`,投影库名、用户名和直接角色 | +| `list_users_all_databases()` | 在 `admin` 明确执行 `usersInfo: {forAllDBs: true}`。按全实例权限裁决,原生命令一次返回数组 | +| `get_user(user, expand_privileges=False)` | 对指定用户使用 `usersInfo`。只有详情查询请求 `showPrivileges`。关闭凭据输出并对返回值做字段白名单投影 | +| `create_user(user, password=..., roles=...)`、`drop_user(user)`、`change_password(user, password=...)` | 分别用 `createUser`、`dropUser`、只设置 `pwd` 的 `updateUser`。密码操作限本地密码账号 | +| `grant_roles(user, roles=...)`、`revoke_roles(user, roles=...)` | `grantRolesToUser`、`revokeRolesFromUser`。每项角色使用显式 `role` 和 `db` | +| `list_roles(db=..., include_builtin=False)`、`get_role(role, expand_privileges=False)` | 使用 `rolesInfo`,按需请求 `showBuiltinRoles` 或 `showPrivileges`。详情保留直接和继承关系 | +| `create_role(role, privileges=..., roles=...)`、`drop_role(role)` | `createRole`、`dropRole`。创建时显式给出权限及继承角色,可为空 | +| `grant_privileges(role, privileges=...)`、`revoke_privileges(role, privileges=...)` | `grantPrivilegesToRole`、`revokePrivilegesFromRole`。传递结构化资源和动作 | +| `grant_roles_to_role(role, roles=...)`、`revoke_roles_from_role(role, roles=...)` | `grantRolesToRole`、`revokeRolesFromRole`。先明确目标角色,再列出要增减的父角色 | + +同步和异步共用命令构造与响应解析,分别调用 `client[db].command(...)` 和 `await client[db].command(...)`。列表查询不会附加只适用于指定用户的权限展开参数。读取用户详情和角色详情时区分未展开、展开后为空、权限不足三种情况。创建和角色增减等写入要检查原生命令成功标志以及 `writeConcernError`。服务端已作出变更而写关注未满足时,不得向调用方报告已回滚。对指定单库资源增减权限,要在该资源所在库具备相应动作。单条权限涉及多个数据库或集群资源时,检查 `admin` 上的 `grantRole` 或 `revokeRole`。删除角色会清理引用,分片集群还需考虑授权缓存刷新后再核对。 [用户信息](https://www.mongodb.com/docs/manual/reference/command/usersinfo/) · [角色信息](https://www.mongodb.com/docs/manual/reference/command/rolesinfo/) · [删除角色](https://www.mongodb.com/docs/manual/reference/command/droprole/) + +### 3.2 MySQL + +| Python 方法 | SQL 或读取来源 | +| --- | --- | +| `list_accounts()`、`get_account(account)` | 查询 `mysql.user` 的 `User`、`Host`、`account_locked` 等必要列。精确查询同时指定用户名和主机 | +| `show_grants(account, using_roles=())` | `SHOW GRANTS FOR`,按需附加 `USING`。返回主体、角色上下文和原文行,保留部分撤权输出 | +| `list_role_bindings(account)`、`get_default_roles(account)` | 查询 `mysql.role_edges` 的授予方向和转授标志,及 `mysql.default_roles`。角色绑定以目标成员为查询条件,直接父角色与默认角色分开返回 | +| `create_user(account, password=...)`、`drop_user(account)`、`change_password(account, password=...)` | `CREATE USER`、`DROP USER`、`ALTER USER ... IDENTIFIED BY`。单项严格执行,保留重复创建等原生拒绝原因 | +| `lock_user(account)`、`unlock_user(account)` | `ALTER USER ... ACCOUNT LOCK/UNLOCK` | +| `create_role(role)`、`drop_role(role)` | `CREATE ROLE`、`DROP ROLE`。使用完整主体,说明相同主体也可作为登录账号使用 | +| `grant_privileges(account, scope, privileges, grant_option=False)`、`revoke_privileges(account, scope, privileges)` | 对目标范围生成 `GRANT ... ON ... TO` 或 `REVOKE ... ON ... FROM`。列权限按列集合构造,过程和函数使用相应对象类型 | +| `grant_roles(account, roles=..., admin_option=False)`、`revoke_roles(account, roles=...)` | `GRANT role ... TO`、`REVOKE role ... FROM`。同一入口支持角色授予另一个角色。使用角色所带 `ADMIN OPTION` 转授时需按服务器要求激活该角色 | +| `set_default_roles(account, roles=...)` | 明确列表对应 `SET DEFAULT ROLE ... TO`,空列表对应 `NONE`。默认角色修改后在新会话核对 | + +**SQL 构造与作用域。** 用驱动支持的参数绑定分别处理用户名、主机和密码。使用结构化输入选择 SQL 关键字,每一段对象标识符独立引用。静态权限按服务器版本和对象类型匹配,动态权限只在全局范围使用并通过 `SHOW PRIVILEGES` 核对注册状态。读取账号信息时避免 `SELECT *` 与认证摘要字段。测试覆盖服务器 SQL 模式、特殊字符和 Connector/Python 纯 Python、C 扩展、异步路径。 [游标执行参数](https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-execute.html) · [服务器登记的权限](https://dev.mysql.com/doc/refman/8.4/en/show-privileges.html) + +**精确库名。** 目标为 `database("team_a")` 时,根据服务器版本和 `partial_revokes` 对数据库级授权模式的解释,编码能逐字匹配的库名,再构造 `ON ... .*`。对不支持该变量的早期 8.0 按对应版本规则处理。若无法确定精确语义,在发送写入前报告原因。授权与撤权必须采用相同的匹配规则。真实服务验收同时创建 `team_a` 和 `teamXa`,用受限账号分别验证;另测含 `%` 的库名和相关服务器设置。 [数据库级授权规则](https://dev.mysql.com/doc/refman/8.4/en/grant.html) + +**转授范围。** `grant_option=False` 仅省略本次语句的转授子句。静态权限在同一授权层级共享相应转授能力,例如对某库的 `SELECT` 增加转授时,同库同层级已有的 `INSERT` 也可能被转授。全局动态权限逐项计算。验收时记录目标账号操作前后完整的 `SHOW GRANTS`,同时用第二个临时主体验证实际转授范围。 [GRANT OPTION](https://dev.mysql.com/doc/refman/8.4/en/grant.html) + +**事务与结果。** 在同步、异步写入路径复用现有 DDL 守卫的原则,要求 `autocommit=True` 且当前没有活动事务;检查失败时把连接交还调用方处理。每个游标由创建方关闭,清理失败不得覆盖已捕获的主要异常。管理连接上依次调用创建账号、分配角色、设置默认角色,分别保存确认结果。 [隐式提交](https://dev.mysql.com/doc/refman/8.4/en/implicit-commit.html) · [现有 MySQL 结构管理实现](../src/database/schema/_mysql.py) + +### 3.3 Redis + +| Python 方法 | 原生调用和契约 | +| --- | --- | +| `list_users()` | `ACL USERS`,返回当前所连数据节点的用户名 | +| `get_user(name)` | `ACL GETUSER`,缺失时返回 `None`。解析 `bytes`、字符串、RESP/驱动版本相应字段,对选择器也执行公开字段过滤 | +| `apply_user(name, rules=...)` | 将按序输入的规则编译为参数,一次发送 `ACL SETUSER`。用户名可被创建或更新。成批规则在同一节点、同一命令内确定先后顺序 | +| `delete_user(name)` | `ACL DELUSER`,按删除计数返回布尔值。保护 `default` 用户的服务端错误原样保留为原因 | +| `dry_run(name, command, *args)` | `ACL DRYRUN`,对 `OK` 返回允许;目标被拒时记录模拟结果及可展示原因。操作者本身缺权、目标不存在、命令错误按调用错误处理 | +| `save_acl()` | `ACL SAVE` 单独确认整节点的 ACL 文件写入。缺少外部文件配置或写入失败可独立诊断 | + +**规则构造。** `AclRule.enable()/disable()` 控制状态;`allow_command/deny_command` 和 `allow_category/deny_category` 控制命令;`key_pattern`、`channel_pattern` 及相应的清除规则控制资源。Redis 7.0 的读写键模式和选择器保留其分组和顺序。`add_password`、`remove_password`、`replace_passwords` 分开表达;替换全部密码在一次请求中先清空旧密码再添加新密码。构造规则时保留口令原值以供提交,但任何面向日志的展示都只给动作名。 [ACL SETUSER](https://redis.io/docs/latest/commands/acl-setuser/) + +**驱动适配。** redis-py 5.0 的高层 `acl_setuser` 默认会附加 `off`,所以按调用方明确给出的规则通过 `execute_command("ACL SETUSER", name, *rules)` 提交。某些驱动把命令和类别解析为两个列表,读到的显示顺序无法还原跨组授权顺序。展示模型只用于读取。DRYRUN 的拒绝原因可能是正常的非空文本回复,禁止用 `bool(reply)` 作为授权判定。 [驱动的 SETUSER 实现](https://github.com/redis/redis-py/blob/v5.0.0/redis/commands/core.py) · [驱动的响应解析](https://github.com/redis/redis-py/blob/v5.0.0/redis/_parsers/helpers.py) · [ACL DRYRUN](https://redis.io/docs/latest/commands/acl-dryrun/) + +**版本与拓扑。** 基础 ACL 对应 6.0,频道对应 6.2,选择器、读写键模式和 DRYRUN 对应 7.0。使用新特性时才检查相关能力,避免为了基础盘点要求额外的管理权限。首版结果只代表该固定数据节点。后续若支持 Sentinel 或 Cluster,应显式列出数据节点与候选副本,按节点报告应用、保存、拒绝和未知状态,并处理核对期间拓扑变化。 [Redis ACL](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/) · [ACL SAVE](https://redis.io/docs/latest/commands/acl-save/) + +## 4. 开发者调用示例 + +以下示例展示拟议签名。`connections` 为调用方建立的 `Connections`,原生客户端和密码参数由调用方显式提供。管理对象在持有连接的作用域内使用。 + +```python +from database.accounts import Accounts, AclRule, MongoRole, MongoUser, MySQLAccount + + +def configure_mongo(connections, password): + user = MongoUser(auth_db="staff", name="analyst") + role = MongoRole(db="reports", name="read") + with connections.create("mongo_admin") as conn: + admin = Accounts.mongodb(conn) + admin.create_user(user, password=password, roles=[]) + admin.grant_roles(user, roles=[role]) + return admin.get_user(user, expand_privileges=True) + + +def inspect_mysql(native_mysql): + admin = Accounts.mysql(native_mysql) + user = MySQLAccount(user="analyst", host="10.0.0.8") + return admin.show_grants(user), admin.get_default_roles(user) + + +def rotate_redis_password(native_redis, new_password): + admin = Accounts.redis(native_redis) + admin.apply_user("analyst", rules=[AclRule.add_password(new_password)]) + return admin.get_user("analyst") +``` + +Redis 示例只添加一个密码。用于已有用户的轮换时,再以新密码独立验证、迁移调用方,最后单独移除旧密码。若用于不存在的用户,该操作遵守原生创建语义,启用与授权规则需要显式传入。 + +```python +from database.accounts import AsyncAccounts, MySQLAccount + + +async def inspect_async(connections, native_mysql, native_redis): + async with await connections.create_async("mongo_admin") as conn: + users = await AsyncAccounts.mongodb(conn).list_users(auth_db="staff") + + mysql = AsyncAccounts.mysql(native_mysql) + account = MySQLAccount(user="analyst", host="10.0.0.8") + grants = await mysql.show_grants(account) + + redis = AsyncAccounts.redis(native_redis) + acl = await redis.get_user("analyst") + return users, grants, acl +``` + +## 5. 实施顺序与交付物 + +| 阶段 | 依赖与工作 | 交付和验收门槛 | +| --- | --- | --- | +| 1. 公共契约 | 从经核对的 `develop` 基线确定模型、工厂、原生类型识别和错误上下文;在功能实施时更新原有 SDK 范围说明 | 公共入口无驱动环境可导入;借用绑定不联网;错误模式、关闭和循环检查有定向用例 | +| 2. 只读盘点 | 基于公共契约独立实现三个后端列表、详情及各自的权限读取 | 使用只读管理员验证可见范围,缺失、权限不足、未展开三种结果清楚可区分 | +| 3. 账号生命周期 | 实现 MongoDB/MySQL 新建、删除、改密,MySQL 启停,Redis 有序规则应用、删除和多密码轮换 | 每个后端在隔离服务上完成开户、凭据验证及回收 | +| 4. 授权与角色 | 实现 MongoDB 资源动作与继承,MySQL 对象范围、角色转授和默认角色,Redis 版本相关规则与授权模拟 | 用独立受限账号完成允许和拒绝的实际请求,核对撤权和无关授权 | +| 5. 异常与兼容 | 测试回包丢失、取消、写关注错误、事务保护、清理失败、驱动解析变化及 Redis 写盘失败 | 每一种故障能归到明确状态,日志和公开结果通过敏感信息检查 | +| 6. 集成与发布准备 | 完成定向测试、全量回归、隔离服务验证,补 README、示例、最低权限表及「未发布」日志 | 验收记录按版本给出证据,文档示例和公开签名一致,真实服务缺口列明 | + +实施时公共文件由协调者负责。公共契约经验证后,各后端在独立分支和 worktree 实现自己的适配、测试及隔离验收。协调者按提交逐个合入集成工作树,每次核对交付 SHA 和差异并重跑相关验证。Git 记录保存基线、交付和集成 SHA,按实际执行情况登记最终合并和推送。 [仓库协作规范](../AGENTS.md) · [贡献指南](../CONTRIBUTING.md) + +## 6. 测试矩阵与发布证据 + +下表是候选验收矩阵。服务器版本和 Python 驱动版本分别记录。最低驱动和一个当期驱动各跑代表组合,精确记录小版本,对不同服务器才支持的特性另设测试。仓库已有的连接或 CRUD 测试为环境准备提供参考;账号管理需要自己的验收结果。 [依赖声明](../pyproject.toml) · [已有验收记录](crud-implementation.md) + +| 后端 | 计划覆盖 | 高价值用例 | +| --- | --- | --- | +| 公共 | Python 3.10 及项目当前开发版本。本库与原生连接,同步与异步 | 延迟导入、绑定零 I/O、关闭归属、错误模式、异步取消、游标清理、保留原始异常 | +| MongoDB | 自管 7.0、8.0;PyMongo 4.6 起同步,4.9 起异步,并测当期驱动 | 管理员认证库、目标账号库、角色库不同;同名异库;全库盘点授权;继承与展开;独立追加;删除后的引用;写关注错误 | +| MySQL | 服务器 8.0、8.4;Connector/Python 8.3 起及当期版本,同步覆盖纯 Python 和 C 扩展 | 同名不同主机;已锁普通账号;部分撤权与默认角色;指定激活组合;全局至列/例程的授权;精确库名;绑定含引号的值;活动事务可回滚 | +| Redis | 服务器 6.0、6.2、7.2、一个 8.x;redis-py 5.0 及当期版本,固定数据节点 | 多密码轮换;单改权限保留启停;规则顺序;频道与选择器分版本;DRYRUN 的允许、拒绝与管理凭据缺权;删除断开会话;有无可写 ACL 文件 | +| 故障 | 驱动替身注入确定性故障,在隔离服务上核对可观察状态 | 写已执行但回包丢失,MongoDB 确认超时,Redis 已应用规则但保存失败;确保 SDK 不重复写入 | + +### 离线检查 + +使用 `unittest` 和驱动替身检查实际发出的数据库、命令、参数、顺序、响应投影和错误分类。分别用“存在但无权”“已确认不存在”“请求结果未知”的样本防止状态混淆。Redis 样本覆盖密码哈希字段和选择器嵌套字段;MongoDB 覆盖凭据与业务自定义数据;MySQL 覆盖含引号和 SQL 模式差异的值绑定。 + +功能实施后,从当前工作树源码运行定向账号测试,再运行完整离线回归。Ruff 对实际变更的 Python 文件做 lint 和格式检查。 [测试指南](../tests/README.md) + +```shell +python -B -m unittest discover -s tests -p "test_accounts*.py" -v +python -B -m unittest discover -s tests -q +python -m ruff check src/database/accounts +python -m ruff format --check src/database/accounts +git diff --check +``` + +### 真实服务验收与清理 + +1. 接受显式隔离配置,登记服务类型、精确版本、驱动版本、拓扑和连接模式。为本轮生成唯一的账号、角色及测试资源前缀,并记录实际成功创建的对象。 +2. 管理连接为临时账号授予范围明确的权限。受限账号用独立连接对同类对象执行一次应允许和一次应拒绝的操作。查询角色或 ACL 的结果与实际请求分别保存。 +3. 改密、锁定或撤权后,使用新连接重新认证、重做访问验证。Redis 另保留既有会话验证 `off` 和删除的差异,MySQL 指定激活角色,MongoDB 区分直接和继承的授权来源。 +4. 对连接中断、写关注和持久化失败,记录最后确认步骤,按对象回读。MySQL 额外验证管理写入被事务守卫阻止后,原连接可以回滚此前的业务写入。 +5. 仅清理本轮拥有的对象。发生创建结果未知时先核对唯一标识及隔离上下文,再决定清理。分别记录业务失败和清理失败。 +6. 验收记录登记提交 SHA、配置代号、同步/异步、本库/原生、服务器与驱动版本、测试命令、期望及实际结果、未验证组合。真实 Redis 节点列为首版 ACL 的发布验收项。历史项目曾记录真实 Redis 覆盖缺口,不能用旧替身结果代替本功能的真机验收。 [已有验证范围](crud-implementation.md) + +发布判定对应需求 R1–R9。三个后端须各完成“盘点 → 查看权限 → 开户或配置 → 授权 → 以受限账号验证 → 撤权或回收”,并能在任一步失败后说明已确认的结果、未知目标和核对方法。 diff --git a/docs/account-management-requirements.md b/docs/account-management-requirements.md new file mode 100644 index 0000000..c3fc77b --- /dev/null +++ b/docs/account-management-requirements.md @@ -0,0 +1,86 @@ +# MongoDB、MySQL、Redis 账号与权限管理需求 + +本文定义拟新增的 Python 工具包 `database.accounts` 的首版需求。读者是使用本库的开发者、负责数据库账号的管理员,以及负责验收的工程师。方法、类型和版本矩阵均属设计目标。开发任务、命令映射、示例和验证步骤见[实现设计与交付规划](account-management-design.md)。 + +## 1. 使用目标与 SDK 边界 + +开发者已经有一条能访问数据库的连接,并持有相应管理权限。工具应让他能在 Python 中盘点账号、解释权限来源、授权、回收权限和轮换凭据。数据库若只允许服务器本地访问,脚本可以运行在服务器上,或复用调用方已建立的网络通道。SDK 接受本库命名连接和对应驱动的原生连接,凭据与连接均由调用方提供。 + +现行 CRUD 设计把账号创建和权限授予归入运行环境准备。本提案建议由调用方准备数据库服务、网络通道和初始管理凭据,SDK 借用已授权连接管理后续账号。`Connections` 负责注册、创建、检查和关闭连接。首版交付 Python 类、方法和类型化结果。业务系统若需要员工身份关联、审批或操作审计,可以在调用这些方法时记录自己的操作者和业务原因。 [现行 CRUD 设计](crud-design.md) · [连接管理设计](connection-management-design.md) · [凭据处理约定](../SECURITY.md) + +## 2. 功能清单与业务验收 + +| 编号 | 场景 | 用户可完成的操作 | 验收证据 | +| --- | --- | --- | --- | +| R1 | 盘点 | 按明确的实例、认证库或节点列出账号 | 同名不同身份可分别查询。报告给出范围,查询权限不足时返回错误 | +| R2 | 权限核查 | 查看直接授权及可展开的权限,说明继承来源和展开条件 | MongoDB 显示继承角色,MySQL 标明使用的角色集合,Redis 显示规则和选择器 | +| R3 | 开户 | 创建账号或配置 Redis ACL 用户,设置初始凭据和指定权限 | 用新账号建立独立连接,应获准和应拒绝的真实请求各验证一次 | +| R4 | 授权与撤权 | 增减指定角色、对象权限或 ACL 规则 | 除目标授权外,独立授予的其他权限仍可回读并验证 | +| R5 | 角色管理 | 在 MongoDB、MySQL 创建角色,维护权限和角色继承,分配或删除角色 | 建立两级角色,分配给用户,回读父子关系,验证撤权或删除后的访问 | +| R6 | 凭据与停用 | 修改密码,按后端能力锁定、停用或轮换密码 | 独立新连接验证新旧凭据。已有会话的表现单独记录 | +| R7 | 账号回收 | 精确删除指定账号,重新盘点 | 目标账号无法重新登录,同名但身份不同的账号仍可独立使用 | +| R8 | 故障核对 | 从拒绝、部分步骤已完成、单次写入结果未知等状态继续处理 | 能定位目标和步骤,有可执行的回读或独立认证方式 | +| R9 | 直接集成 | 用同步或异步 Python 程序复用现有连接 | 本库句柄和原生连接均可调用。调用完成后借入的连接仍归调用方使用 | + +## 3. 三个后端的能力对照 + +| 能力 | 自管 MongoDB | MySQL 8.0、8.4 | Redis 固定数据节点 | +| --- | --- | --- | --- | +| 账号身份 | 认证库、用户名 | 用户名、来源主机 | 节点、ACL 用户名 | +| 账号详情 | 直接角色,可展开继承角色和资源动作 | 主体及锁定状态、直接授权、默认角色、指定角色组合 | 启停标志、命令与类别、键模式、频道、选择器 | +| 建立与删除 | 创建用户时可指定初始角色,按认证库删除 | 创建、删除完整的 `user@host` 主体,再分别授权 | 应用规则可创建或更新,删除返回是否实际删除 | +| 凭据与停用 | 修改本地密码。撤销角色影响授权,删除账号影响后续认证 | 支持改密、锁定、解锁 | 启停、多密码并存、增删密码和替换全部密码 | +| 分配权限 | 为账号分配角色,为自定义角色赋予资源和动作 | 对账号或角色直接授权,也可授予角色、指定默认角色 | 对目标用户设置有序规则,按命令、键及频道授权 | +| 公用角色 | 内置角色可查询和分配,自定义角色可维护 | 支持角色创建、继承、分配、撤回和删除 | 以用户 ACL 规则表达访问条件 | +| 确认范围 | 指定部署和认证库。分片集群用户由 `mongos` 管理 | 当前服务器返回的授权,以新登录会话核对生效状态 | 当前数据节点的运行时状态;写入 ACL 文件单独确认 | + +### 3.1 MongoDB 用户和角色 + +- **身份和盘点。** 管理员自己的 `authSource`、目标用户的认证库和角色的定义库是三个独立字段。例如,管理员在 `admin` 认证,可以给在 `staff` 创建的 `analyst` 分配 `reports.read`。按 `reports` 列用户只能说明账号创建在哪个库;要回答谁能访问报表,须盘点有权查看的账号,再检查其直接和继承权限。提供显式跨库盘点。`usersInfo` 返回用户数组;在首版需要处理大量账号时,可按已知认证库分批发起查询。 [用户身份](https://www.mongodb.com/docs/manual/core/security-users/) · [usersInfo](https://www.mongodb.com/docs/manual/reference/command/usersinfo/) +- **查看权限。** 列表返回身份和直接角色。详情可按指定用户展开 `inheritedRoles` 和 `inheritedPrivileges`。全库枚举与展开权限使用独立查询。返回值区分“未要求展开”和“展开后为空”。默认只公开允许展示的字段,排除凭据和业务自定义资料。 [usersInfo](https://www.mongodb.com/docs/manual/reference/command/usersinfo/) +- **管理账号。** 支持创建、删除、改密、追加和撤销角色。管理员给用户追加一个角色时,应保留原先的角色分配,包括其他管理员并发完成的独立追加。MongoDB 对 `updateUser.roles` 采用整组替换语义,增量变更应走专门的角色命令。 [updateUser](https://www.mongodb.com/docs/manual/reference/command/updateuser/) · [增量授权](https://www.mongodb.com/docs/manual/reference/command/grantrolestouser/) +- **管理角色。** 按定义库列出角色,调用方可选择内置角色。对自定义角色支持创建、删除、增减资源动作、增减继承角色。保留数据库、集合、集群资源和动作的原生表示。管理非 `admin` 库角色时,由数据库执行资源范围及角色继承限制。删除角色后需回读引用方,再验证受影响账号的权限。 [角色查询](https://www.mongodb.com/docs/manual/reference/command/rolesinfo/) · [创建角色](https://www.mongodb.com/docs/manual/reference/command/createrole/) · [删除角色](https://www.mongodb.com/docs/manual/reference/command/droprole/) +- **部署范围。** 首版覆盖自管 MongoDB 的数据库本地用户,密码操作面向密码认证账户。Atlas 数据库用户由 Atlas 管理平面维护。首版的自管命令和验收结果适用于自管部署。 [Atlas 数据库用户管理](https://www.mongodb.com/docs/atlas/security-add-mongodb-users/) + +### 3.2 MySQL 账号、角色和对象授权 + +- **主体。** 账号和角色共享 `user@host` 身份空间。用户名和来源主机分别传入,使用 `%` 时由调用方显式选择。账号列表保留锁定状态。角色绑定列表说明哪些主体已授给哪些成员,尚未分配的角色可能没有绑定记录。工具不得仅凭锁定状态把普通登录账号认定为可删除的角色。 [账号命名](https://dev.mysql.com/doc/refman/8.4/en/account-names.html) · [角色管理](https://dev.mysql.com/doc/refman/8.4/en/roles.html) +- **生命周期。** 创建、删除、改密、锁定、解锁针对完整主体。创建账号和授予权限分别确认,以便处理“已开户但授权被拒”的结果。删除账号或锁定后使用新连接验证登录,已有会话按服务器原生规则处理。 +- **权限解释。** 保存 `SHOW GRANTS FOR` 的完整逐行结果,包括部分撤权。支持按调用方指定的已授角色展开授权,列出默认角色。默认角色、强制角色、登录激活设置和会话中的 `SET ROLE` 都会影响解释。报告记录主体和展开条件;指定用户名的授权输出对强制角色的可见性遵循服务器规则。 [SHOW GRANTS](https://dev.mysql.com/doc/refman/8.4/en/show-grants.html) +- **授权范围。** 覆盖全局、库、表、列、存储过程和函数。静态权限须与对象类型匹配,动态权限按当前服务器登记情况在合法的全局范围使用。数据库级授权默认精确匹配库名,包括名称内的下划线和百分号。对象权限的 `GRANT OPTION` 和角色转授的 `ADMIN OPTION` 分别控制,默认关闭。省略转授选项会保留原有转授能力。静态权限的转授可能同时影响同一授权层级上的其他权限,全局动态权限的转授按单项生效。 [GRANT](https://dev.mysql.com/doc/refman/8.4/en/grant.html) · [SHOW PRIVILEGES](https://dev.mysql.com/doc/refman/8.4/en/show-privileges.html) +- **角色。** 支持创建、删除、配置角色权限、给用户或其他角色授予和撤销角色,以及设置默认角色。用户获授某角色后,可以通过指定该角色的展开查询解释权限,并在独立会话中实际激活验证。 [角色管理](https://dev.mysql.com/doc/refman/8.4/en/roles.html) + +### 3.3 Redis 命名用户与 ACL + +- **身份与查询。** ACL 作用于连接的数据节点。同一节点的不同编号逻辑库共享用户。列表使用 `ACL USERS`,单一用户详情使用 `ACL GETUSER`,只返回允许展示的标志和权限字段。服务端返回的密码摘要按凭据材料处理。 [ACL GETUSER](https://redis.io/docs/latest/commands/acl-getuser/) · [Redis ACL](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/) +- **创建与更新。** `apply_user` 采用 `ACL SETUSER` 的原生语义。不存在的用户会被创建,已存在的用户按从左到右的顺序应用规则。调用者可以在同一请求中指定账号启用、密码及允许访问的资源。读取再写入之间存在并发窗口,因此工具只确认规则已应用。 [ACL SETUSER](https://redis.io/docs/latest/commands/acl-setuser/) +- **权限表达。** 基础能力覆盖允许或拒绝命令和类别、键模式。Redis 6.2 起支持频道规则,7.0 起支持选择器和区分读写的键模式。对既有账号仅追加某命令时,应保留原有启停和凭据状态。对资源收紧权限时,应能显式清除原有键或频道模式后重设,避免宽泛的旧模式仍然有效。 [ACL 规则](https://redis.io/docs/latest/commands/acl-setuser/) +- **轮换与回收。** 支持添加密码、移除单个密码和替换全部密码,可先添加新密码,迁移客户端后再撤销旧密码。`off` 阻止后续认证,现有已认证连接可继续工作。删除用户会断开该用户连接,服务端保护 `default` 用户。 [用户规则](https://redis.io/docs/latest/commands/acl-setuser/) · [ACL DELUSER](https://redis.io/docs/latest/commands/acl-deluser/) +- **验证和持久化。** Redis 7.0 起可模拟某用户对具体命令及参数的权限。运行时授权与持久化是两个操作。显式 `ACL SAVE` 要求可写的外部 `aclfile`,保存当前节点的整套 ACL。保存失败时应能核对已在内存生效的状态。首版只接受固定数据节点,Sentinel、Cluster 的多节点管理另行规划。 [ACL DRYRUN](https://redis.io/docs/latest/commands/acl-dryrun/) · [ACL SAVE](https://redis.io/docs/latest/commands/acl-save/) + +## 4. 操作者权限与可见范围 + +调用方按需要授予管理账号相应能力。下表列出普通本地账号的常见授权路径,受保护主体、只读服务器或部署策略可能有附加要求。每项操作直接进行服务端授权检查,拥有单项写权限的管理员可直接使用该方法。 + +| 后端及操作 | 管理凭据所需能力或资源范围 | +| --- | --- | +| MongoDB 查询他人账号或角色 | 查看用户通常需其认证库的 `viewUser`,查看非自身获授的角色通常需角色定义库的 `viewRole`。查询自身资料适用服务端自身权限规则 | +| MongoDB 枚举全实例用户 | MongoDB 8.0 在集群资源上检查 `viewUser`,拥有个别认证库的读取权限只适合相应库的盘点。[服务端授权检查](https://github.com/mongodb/mongo/blob/r8.0.0/src/mongo/db/commands/user_management_commands_common.cpp#L321-L350) | +| MongoDB 用户创建、删除、改密 | 用户认证库上的 `createUser`、`dropUser` 或 `changePassword`。创建时带角色还需各角色定义库的 `grantRole` | +| MongoDB 授予和撤销角色 | 对所授或所撤角色的定义库具有 `grantRole` 或 `revokeRole`。跨库授予时检查角色所在库 | +| MongoDB 创建、删除、修改角色 | 角色定义库对应的 `createRole` 或 `dropRole`,赋予、撤回权限与继承关系还需目标范围的 `grantRole` 或 `revokeRole`,依具体命令检查 | +| MySQL 列主体和绑定 | 可读取 `mysql.user` 的必要列、`mysql.role_edges` 或 `mysql.default_roles` 的对应列。各查询独立调用 | +| MySQL 查询他人授权 | `SHOW GRANTS FOR` 他人通常需要对 `mysql` 系统库的 `SELECT`。查询当前账号有单独规则 | +| MySQL 管理账号或角色 | 常用管理路径为全局 `CREATE USER`。单独创建或删除角色也可由 `CREATE ROLE` 或 `DROP ROLE` 授权,并遵循服务器对目标的限制 | +| MySQL 授权和设置默认角色 | 授对象权限需要相应权限及转授权资格。分配角色使用 `ROLE_ADMIN` 或对该角色的 `ADMIN OPTION` 等服务端认可的资格。设置他人默认角色需要 `CREATE USER` 或对 `mysql.default_roles` 的 `UPDATE` | +| Redis ACL | 按目标版本授予所需 ACL 子命令,例如 `USERS`、`GETUSER`、`SETUSER`、`DELUSER`、`DRYRUN`、`SAVE`。写盘另需服务进程能写入配置的文件 | + +权限依据见 [MongoDB 账号创建](https://www.mongodb.com/docs/manual/reference/command/createuser/)、[MongoDB 角色管理](https://www.mongodb.com/docs/manual/tutorial/manage-users-and-roles/)、[MySQL 授权](https://dev.mysql.com/doc/refman/8.4/en/grant.html)、[MySQL 默认角色](https://dev.mysql.com/doc/refman/8.4/en/set-default-role.html)和 [Redis ACL](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/)。 + +## 5. 首版通过条件 + +1. 每个后端用显式配置的隔离服务走完 R1–R9 中适用场景。每项变更分别由管理连接回读和受限账号新连接验证。账号有多个权限来源时,撤销一项后继续追溯其他授权来源。 +2. 只读管理员可以盘点自己的可见范围。对无权查询的用户、角色或规则,返回拒绝信息。只有查询成功且能确认目标缺失时,详情才返回 `None`。 +3. 两种绑定和两种执行方式遵守连接归属。本库句柄关闭、异步取消、权限不足、网络中断及 Redis 版本差异均有可核对的结果。 +4. 普通日志、可展示详情和异常摘要排除密码、密码摘要、连接 URI 和命令中承载的凭据。受控诊断保留原始异常和因果链。 +5. 验收记录给出实际服务器、驱动、拓扑、调用模式、运行命令和结果。计划版本、替身测试和已通过的真机组合分别标记。具体矩阵和各后端用例见[实现设计与交付规划](account-management-design.md)。 -- Gitee From ef0cb1a10b8f2944c96423c24d41ac49d7bf3012 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 16:02:34 +0800 Subject: [PATCH 77/86] =?UTF-8?q?feat(accounts):=20=E5=BB=BA=E7=AB=8B?= =?UTF-8?q?=E8=B4=A6=E5=8F=B7=E7=AE=A1=E7=90=86=E5=85=A5=E5=8F=A3=E4=B8=8E?= =?UTF-8?q?=E5=80=9F=E7=94=A8=E8=BF=9E=E6=8E=A5=E5=A5=91=E7=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/accounts/__init__.py | 94 ++++++++++++++++++++++++ src/database/accounts/_base.py | 118 ++++++++++++++++++++++++++++++ tests/test_accounts_binding.py | 75 +++++++++++++++++++ 3 files changed, 287 insertions(+) create mode 100644 src/database/accounts/__init__.py create mode 100644 src/database/accounts/_base.py create mode 100644 tests/test_accounts_binding.py diff --git a/src/database/accounts/__init__.py b/src/database/accounts/__init__.py new file mode 100644 index 0000000..c79a32d --- /dev/null +++ b/src/database/accounts/__init__.py @@ -0,0 +1,94 @@ +"""Database user and privilege management on borrowed connections.""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING +from typing import Any + +from ._base import AccountWriteOutcomeUnknown +from ._base import Binding + +if TYPE_CHECKING: + from ._mongodb import AsyncMongoDBAccounts + from ._mongodb import MongoDBAccounts + from ._mongodb import MongoPrivilege + from ._mongodb import MongoRole + from ._mongodb import MongoUser + from ._mysql import AsyncMySQLAccounts + from ._mysql import MySQLAccount + from ._mysql import MySQLAccounts + from ._mysql import MySQLScope + from ._redis import AclRule + from ._redis import AsyncRedisAccounts + from ._redis import RedisAccounts + +__all__ = [ + 'AccountWriteOutcomeUnknown', + 'Accounts', + 'AclRule', + 'AsyncAccounts', + 'MongoPrivilege', + 'MongoRole', + 'MongoUser', + 'MySQLAccount', + 'MySQLScope', +] + + +class Accounts: + """Bind synchronous account operations without network I/O.""" + + @staticmethod + def mongodb(connection: object) -> MongoDBAccounts: + from ._mongodb import MongoDBAccounts + + return MongoDBAccounts(Binding(connection, 'mongodb', is_async=False)) + + @staticmethod + def mysql(connection: object) -> MySQLAccounts: + from ._mysql import MySQLAccounts + + return MySQLAccounts(Binding(connection, 'mysql', is_async=False)) + + @staticmethod + def redis(connection: object) -> RedisAccounts: + from ._redis import RedisAccounts + + return RedisAccounts(Binding(connection, 'redis', is_async=False)) + + +class AsyncAccounts: + """Bind asynchronous account operations without network I/O.""" + + @staticmethod + def mongodb(connection: object) -> AsyncMongoDBAccounts: + from ._mongodb import AsyncMongoDBAccounts + + return AsyncMongoDBAccounts(Binding(connection, 'mongodb', is_async=True)) + + @staticmethod + def mysql(connection: object) -> AsyncMySQLAccounts: + from ._mysql import AsyncMySQLAccounts + + return AsyncMySQLAccounts(Binding(connection, 'mysql', is_async=True)) + + @staticmethod + def redis(connection: object) -> AsyncRedisAccounts: + from ._redis import AsyncRedisAccounts + + return AsyncRedisAccounts(Binding(connection, 'redis', is_async=True)) + + +def __getattr__(name: str) -> Any: + modules = { + 'MongoUser': 'mongodb', + 'MongoRole': 'mongodb', + 'MongoPrivilege': 'mongodb', + 'MySQLAccount': 'mysql', + 'MySQLScope': 'mysql', + 'AclRule': 'redis', + } + if name in modules: + return getattr(import_module(f'{__name__}._{modules[name]}'), name) + raise AttributeError(name) diff --git a/src/database/accounts/_base.py b/src/database/accounts/_base.py new file mode 100644 index 0000000..a51ce6c --- /dev/null +++ b/src/database/accounts/_base.py @@ -0,0 +1,118 @@ +"""Binding and operation boundaries for borrowed account-management clients.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from importlib import import_module +from typing import Any + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DatabaseConnectionError +from ..errors import DatabaseError +from ..errors import DatabaseTimeoutError +from ..errors import DependencyError +from ..errors import attach_cleanup_errors +from ..errors import convert_error + + +class AccountWriteOutcomeUnknown(DatabaseError): + """A mutation was submitted but its result could not be confirmed.""" + + +def _matches_native(client: object, backend: str, is_async: bool) -> bool: + if backend == 'mongodb': + import pymongo + + if is_async and not hasattr(pymongo, 'AsyncMongoClient'): + raise DependencyError('Async MongoDB requires PyMongo >= 4.9.', backend=backend, phase='config') + return isinstance(client, pymongo.AsyncMongoClient if is_async else pymongo.MongoClient) + if backend == 'mysql': + prefix = 'mysql.connector.aio' if is_async else 'mysql.connector' + abstract = import_module(f'{prefix}.abstracts').MySQLConnectionAbstract + if isinstance(client, abstract): + return True + try: + pooled = import_module(f'{prefix}.pooling').PooledMySQLConnection + except (ModuleNotFoundError, AttributeError): + return False + return isinstance(client, pooled) + + redis = import_module('redis.asyncio' if is_async else 'redis') + return isinstance(client, redis.Redis) + + +def _fixed_redis_node(client: Any, is_async: bool) -> bool: + module = import_module('redis.asyncio.sentinel' if is_async else 'redis.sentinel') + return not isinstance(client.connection_pool, module.SentinelConnectionPool) + + +class Binding: + """Validate a managed handle or native client without creating a connection.""" + + def __init__(self, connection: object, backend: str, *, is_async: bool) -> None: + self.backend = backend + self._managed = connection if isinstance(connection, Connection) else None + self.connection_name = self._managed.name if self._managed is not None else None + invalid = ConfigurationError( + 'Expected a matching account-management client and connection mode.', + backend=backend, + connection_name=self.connection_name, + phase='config', + ) + try: + if self._managed is not None: + if self._managed.backend != backend or isinstance(self._managed, AsyncConnection) != is_async: + raise invalid + if backend == 'redis' and self._managed.mode != 'single': + raise invalid + self._client = self._managed.native + self._classify = self._managed._adapter.classify + else: + if not _matches_native(connection, backend, is_async): + raise invalid + self._client = connection + self._classify = import_module(f'database._adapters.{backend}').Adapter.classify + if backend == 'redis' and not _fixed_redis_node(self._client, is_async): + raise invalid + except ImportError as original: + raise DependencyError( + 'The selected database driver is unavailable.', backend=backend, phase='config', original=original + ) from original + + @property + def native(self) -> Any: + if self._managed is not None: + if isinstance(self._managed, AsyncConnection): + self._managed._ensure_loop('operation') + return self._managed.native + return self._client + + @contextmanager + def operation(self, operation: str, target: str | None = None, *, write: bool = False) -> Iterator[None]: + """Mark only the actual write dispatch as possibly having changed server state.""" + if self._managed is not None: + if isinstance(self._managed, AsyncConnection): + self._managed._ensure_loop('operation') + self._managed._ensure_open('operation') + context = dict( + connection_name=self.connection_name, + backend=self.backend, + phase='operation', + operation=operation, + target=target, + ) + try: + yield + except Exception as original: + converted = convert_error(original, **context, classifier=self._classify, owned=True) + if converted is original: + raise + if write and isinstance(converted, (DatabaseConnectionError, DatabaseTimeoutError)): + converted = AccountWriteOutcomeUnknown( + 'The database did not confirm the account change.', **context, original=original + ) + attach_cleanup_errors(converted, getattr(original, 'database_cleanup_errors', ())) + raise converted from original diff --git a/tests/test_accounts_binding.py b/tests/test_accounts_binding.py new file mode 100644 index 0000000..d16669b --- /dev/null +++ b/tests/test_accounts_binding.py @@ -0,0 +1,75 @@ +"""Account factories must not own connections or perform implicit probes.""" + +import asyncio +import os +import subprocess +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +from database.accounts import AccountWriteOutcomeUnknown +from database.accounts._base import Binding +from database.connections import AsyncConnection +from database.connections import Connection +from database.errors import ConfigurationError +from database.errors import ConnectionClosedError +from database.errors import DatabaseConnectionError + + +class BindingTests(unittest.TestCase): + def test_import_without_drivers(self): + source = str(Path(__file__).resolve().parents[1] / 'src') + code = ( + 'import sys; from database.accounts import Accounts, AsyncAccounts; ' + "assert not {'pymongo', 'redis', 'mysql.connector'}.intersection(sys.modules)" + ) + subprocess.run([sys.executable, '-B', '-S', '-c', code], env={**os.environ, 'PYTHONPATH': source}, check=True) + + def test_borrow_and_close(self): + client = object() + handle = Connection( + client, SimpleNamespace(classify=lambda exc: None, close=lambda native: None), 'test', 'mongodb', 'single' + ) + bound = Binding(handle, 'mongodb', is_async=False) + self.assertIs(bound.native, client) + with self.assertRaises(ConfigurationError): + Binding(handle, 'mysql', is_async=False) + handle.close() + with self.assertRaises(ConnectionClosedError): + with bound.operation('read'): + self.fail('closed handle entered the operation') + + def test_write_uncertainty_and_safe_error(self): + handle = Connection(object(), SimpleNamespace(classify=lambda exc: None), 'test', 'mysql', 'single') + bound = Binding(handle, 'mysql', is_async=False) + for write, error_type in [(False, DatabaseConnectionError), (True, AccountWriteOutcomeUnknown)]: + with self.subTest(write=write): + original = ConnectionResetError('private payload') + with self.assertRaises(error_type) as caught: + with bound.operation('grant_roles', 'target', write=write): + raise original + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.target, 'target') + self.assertNotIn('private payload', str(caught.exception)) + + def test_foreign_loop(self): + async def make_binding(): + handle = AsyncConnection( + object(), + SimpleNamespace(classify=lambda exc: None), + 'async', + 'mongodb', + 'single', + asyncio.get_running_loop(), + ) + return Binding(handle, 'mongodb', is_async=True) + + async def use(binding): + with binding.operation('read'): + self.fail('entered using a different loop') + + binding = asyncio.run(make_binding()) + with self.assertRaises(ConfigurationError): + asyncio.run(use(binding)) -- Gitee From 00fe5ac51b20a353065d1170251ad5fd18f3c845 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 16:31:40 +0800 Subject: [PATCH 78/86] =?UTF-8?q?feat(redis):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E5=8D=95=E8=8A=82=E7=82=B9=E8=B4=A6=E5=8F=B7=E4=B8=8E=E6=9D=83?= =?UTF-8?q?=E9=99=90=E7=AE=A1=E7=90=86=E5=B9=B6=E9=AA=8C=E8=AF=81=E5=9B=9B?= =?UTF-8?q?=E4=B8=AA=E6=9C=8D=E5=8A=A1=E5=99=A8=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/verify_accounts_redis.py | 220 ++++++++++++++++ src/database/accounts/_redis.py | 434 +++++++++++++++++++++++++++++++ tests/test_accounts_redis.py | 185 +++++++++++++ 3 files changed, 839 insertions(+) create mode 100644 scripts/verify_accounts_redis.py create mode 100644 src/database/accounts/_redis.py create mode 100644 tests/test_accounts_redis.py diff --git a/scripts/verify_accounts_redis.py b/scripts/verify_accounts_redis.py new file mode 100644 index 0000000..8df78c4 --- /dev/null +++ b/scripts/verify_accounts_redis.py @@ -0,0 +1,220 @@ +"""Run Redis account acceptance on a fresh private loopback instance of a supplied server. + +Example: python -B scripts/verify_accounts_redis.py --server /path/to/redis-server +The optional --protocol 3 exercises RESP3. No existing Redis connection is accepted. +""" + +from __future__ import annotations + +import argparse +import asyncio +import gc +import secrets +import socket +import subprocess +import sys +import tempfile +import time +from contextlib import contextmanager +from pathlib import Path + +import redis +import redis.asyncio + +from database import ConnectionConfig +from database import Connections +from database.accounts import Accounts +from database.accounts import AclRule +from database.accounts import AsyncAccounts +from database.errors import DependencyError +from database.errors import DriverError +from database.errors import PermissionDeniedError + + +def require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def expect(error_type, action) -> None: + try: + action() + except error_type: + return + raise AssertionError(f'Expected {error_type.__name__}') + + +def client(port, password, username='default', protocol=2, *, decode=False): + return redis.Redis( + host='127.0.0.1', + port=port, + username=username, + password=password, + protocol=protocol, + decode_responses=decode, + socket_connect_timeout=2, + socket_timeout=3, + ) + + +@contextmanager +def server(binary: Path, folder: Path, port: int, password: str, *, persistent: bool): + acl_file = folder / 'users.acl' + if persistent and not acl_file.exists(): + acl_file.write_text(f'user default on >{password} ~* +@all\n', encoding='ascii') + acl_config = 'aclfile users.acl' if persistent else f'requirepass {password}' + config = folder / 'redis.conf' + config.write_text(f'bind 127.0.0.1\nport {port}\nsave ""\nappendonly no\n{acl_config}\n', encoding='ascii') + flags = getattr(subprocess, 'CREATE_NO_WINDOW', 0) + with (folder / 'server.log').open('ab') as log: + process = subprocess.Popen([str(binary), config.name], cwd=folder, stdout=log, stderr=log, creationflags=flags) + try: + deadline = time.monotonic() + 20 + with client(port, password) as admin: + while True: + if process.poll() is not None: + raise RuntimeError(f'Disposable Redis exited with code {process.returncode}.') + try: + admin.ping() + break + except redis.ConnectionError: + if time.monotonic() >= deadline: + raise RuntimeError('Disposable Redis did not become ready.') from None + time.sleep(0.1) + yield admin + finally: + # This test owns this child and explicitly persists ACLs with ACL SAVE. + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def fresh_port(): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + return sock.getsockname()[1] + + +async def check_async(port, password, protocol, name, config, major): + raw = redis.asyncio.Redis(host='127.0.0.1', port=port, password=password, protocol=protocol) + try: + acl = AsyncAccounts.redis(raw) + require(name in await acl.list_users(), 'Async native inventory missing target.') + require((await acl.get_user(name)).enabled, 'Async native target missing.') + async with await Connections({'node': config}).create_async('node') as handle: + acl = AsyncAccounts.redis(handle) + await acl.apply_user(name, rules=[AclRule.allow_command('exists')]) + require('+exists' in (await acl.get_user(name)).commands, 'Async grant not applied.') + if major >= 7: + require((await acl.dry_run(name, 'exists', 'qa:read')).allowed, 'Async simulation failed.') + await acl.save_acl() + require(not await acl.delete_user(name + '-missing'), 'Async missing delete count.') + require(await raw.ping(), 'Borrowed async connection was closed.') + finally: + closer = getattr(raw, 'aclose', None) or raw.close + await closer() + + +def check_runtime(admin, name, password, port, protocol): + version = admin.info('server')['redis_version'] + major, minor = [int(i) for i in version.split('.')[:2]] + old, new = secrets.token_hex(16), secrets.token_hex(16) + config = ConnectionConfig( + backend='redis', host='127.0.0.1', port=port, password=password, options={'protocol': protocol} + ) + with Connections({'node': config}).create('node') as handle: + acl = Accounts.redis(handle) + require(acl.get_user(name) is None, 'Unexpected existing account.') + rules = [ + AclRule.enable(), + AclRule.add_password(old), + AclRule.allow_command('get'), + AclRule.key_pattern('qa:*'), + AclRule.allow_command('ping'), + ] + if (major, minor) >= (6, 2): + rules.extend([AclRule.clear_channels(), AclRule.channel_pattern('events:*')]) + acl.apply_user(name, rules=rules) + data = acl.get_user(name) + require(data.enabled and name in acl.list_users(), 'User missing or disabled.') + if (major, minor) >= (6, 2): + require(any('events:*' in p for p in data.channels), 'Missing channel rule.') + else: + expect(DependencyError, lambda: acl.apply_user(name, rules=[AclRule.clear_channels()])) + if major >= 7: + acl.apply_user( + name, + rules=[AclRule.selector(AclRule.allow_command('set'), AclRule.key_pattern('out:*', access='write'))], + ) + require(acl.get_user(name).selectors, 'Missing selector.') + require(acl.dry_run(name, 'set', 'out:result', 'sample').allowed, 'Selector simulation failed.') + require(not acl.dry_run(name, 'set', 'qa:read', 'sample').allowed, 'Unexpected permission.') + expect(DriverError, lambda: acl.dry_run(name + '-missing', 'ping')) + expect(DriverError, lambda: acl.dry_run(name, 'invalid-command-for-qa')) + else: + expect(DependencyError, lambda: acl.dry_run(name, 'ping')) + expect(DependencyError, lambda: acl.apply_user(name, rules=[AclRule.clear_selectors()])) + with client(port, old, name, protocol) as limited: + require(limited.get('qa:read') is None, 'Authorized read failed.') + expect(redis.exceptions.NoPermissionError, lambda: limited.get('forbidden:read')) + expect(PermissionDeniedError, lambda: Accounts.redis(limited).get_user(name)) + if major >= 7: + expect(PermissionDeniedError, lambda: Accounts.redis(limited).dry_run(name, 'ping')) + acl.apply_user(name, rules=[AclRule.add_password(new), AclRule.deny_command('get')]) + expect(redis.exceptions.NoPermissionError, lambda: limited.get('qa:read')) + with client(port, new, name, protocol) as migrated: + require(migrated.ping(), 'Rotated password rejected.') + acl.apply_user(name, rules=[AclRule.remove_password(old), AclRule.allow_command('get')]) + with client(port, old, name, protocol) as previous: + expect(redis.AuthenticationError, previous.ping) + acl.apply_user(name, rules=[AclRule.disable()]) + require(limited.ping(), 'Disabling unexpectedly revoked an existing session.') + with client(port, new, name, protocol) as disabled: + expect(redis.AuthenticationError, disabled.ping) + acl.apply_user(name, rules=[AclRule.enable(), AclRule.replace_passwords([new])]) + asyncio.run(check_async(port, password, protocol, name, config, major)) + require(acl.delete_user(name), 'Existing user was not deleted.') + expect(redis.ConnectionError, limited.ping) + require(acl.get_user(name) is None and not acl.delete_user(name), 'Deletion did not converge.') + expect(DriverError, lambda: acl.delete_user('default')) + require(admin.ping(), 'Independent admin connection lost.') + return version + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--server', type=Path, required=True) + parser.add_argument('--protocol', type=int, choices=(2, 3), default=2) + args = parser.parse_args() + binary = args.server.resolve(strict=True) + password, name = secrets.token_hex(24), 'qa-' + secrets.token_hex(8) + port = fresh_port() + with tempfile.TemporaryDirectory(prefix='redis-accounts-live-') as tmp: + folder = Path(tmp) + with server(binary, folder, port, password, persistent=True) as admin: + version = check_runtime(admin, name, password, port, args.protocol) + with client(port, password, protocol=args.protocol, decode=True) as strings: + a = Accounts.redis(strings) + require(a.get_user('default').enabled, 'decode_responses=True failed.') + a.apply_user(name, rules=[AclRule.disable()]) + a.save_acl() + with server(binary, folder, port, password, persistent=True) as admin: + require(Accounts.redis(admin).get_user(name) is not None, 'ACL file did not survive restart.') + with server(binary, folder, port, password, persistent=False) as admin: + a = Accounts.redis(admin) + a.apply_user(name, rules=[AclRule.disable()]) + expect(DriverError, a.save_acl) + require(a.get_user(name) is not None, 'Failed ACL SAVE discarded runtime changes.') + gc.collect() + print( + f'PASS Redis {version} / redis-py {redis.__version__} / RESP{args.protocol} / Python ' + f'{sys.version_info.major}.{sys.version_info.minor}: lifecycle, ACL, sync/async, persistence, cleanup' + ) + + +if __name__ == '__main__': + main() diff --git a/src/database/accounts/_redis.py b/src/database/accounts/_redis.py new file mode 100644 index 0000000..4e03910 --- /dev/null +++ b/src/database/accounts/_redis.py @@ -0,0 +1,434 @@ +"""Ordered ACL changes and read-only permission views for a single Redis data node.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from collections.abc import Sequence +from dataclasses import dataclass +from dataclasses import field +from typing import Any +from typing import Literal + +from ..errors import ConfigurationError +from ..errors import DatabaseError +from ..errors import DependencyError +from ..errors import DriverError +from ._base import AccountWriteOutcomeUnknown +from ._base import Binding + + +def _argument(value: str, what: str) -> str: + if not isinstance(value, str) or not value or any(c.isspace() or c == '\x00' for c in value): + raise ConfigurationError(f'Expected a nonempty Redis {what} without whitespace or NUL.') + return value + + +def _command(value: str) -> str: + if not re.fullmatch(r'[a-zA-Z0-9_.:-]+(?:\|[a-zA-Z0-9_.:-]+)?', _argument(value, 'command')): + raise ConfigurationError('Invalid Redis command name.') + return value + + +def _category(value: str) -> str: + if not re.fullmatch(r'[a-zA-Z0-9_-]+', _argument(value, 'category')): + raise ConfigurationError('Expected a Redis category name without its ACL prefix.') + return value + + +def _password(value: str | bytes, prefix: str) -> str | bytes: + if not isinstance(value, (bytes, str)): + raise ConfigurationError('Expected a Redis password string or bytes.') + return prefix.encode('ascii') + value if isinstance(value, bytes) else prefix + value + + +@dataclass(frozen=True, init=False, slots=True) +class AclRule: + """One ordered ACL change. Use the named constructors to build rules.""" + + _kind: str + _tokens: tuple[str | bytes, ...] = field(repr=False) + _version: tuple[int, int] = field(repr=False) + + def __init__(self) -> None: + raise TypeError('Use an AclRule constructor.') + + @classmethod + def _new(cls, kind: str, *tokens: str | bytes, version: tuple[int, int] = (6, 0)) -> AclRule: + rule = object.__new__(cls) + object.__setattr__(rule, '_kind', kind) + object.__setattr__(rule, '_tokens', tokens) + object.__setattr__(rule, '_version', version) + return rule + + @classmethod + def enable(cls) -> AclRule: + return cls._new('enable', 'on') + + @classmethod + def disable(cls) -> AclRule: + return cls._new('disable', 'off') + + @classmethod + def allow_command(cls, command: str) -> AclRule: + return cls._new('command', '+' + _command(command)) + + @classmethod + def deny_command(cls, command: str) -> AclRule: + return cls._new('command', '-' + _command(command)) + + @classmethod + def allow_category(cls, category: str) -> AclRule: + return cls._new('category', '+@' + _category(category)) + + @classmethod + def deny_category(cls, category: str) -> AclRule: + return cls._new('category', '-@' + _category(category)) + + @classmethod + def key_pattern(cls, pattern: str, *, access: Literal['read', 'write', 'both'] = 'both') -> AclRule: + prefix = {'read': '%R~', 'write': '%W~', 'both': '~'} + if access not in prefix: + raise ConfigurationError('Expected read, write or both for ACL key access.') + token = prefix[access] + _argument(pattern, 'key pattern') + return cls._new('key', token, version=(6, 0) if access == 'both' else (7, 0)) + + @classmethod + def clear_keys(cls) -> AclRule: + return cls._new('key', 'resetkeys') + + @classmethod + def channel_pattern(cls, pattern: str) -> AclRule: + return cls._new('channel', '&' + _argument(pattern, 'channel pattern'), version=(6, 2)) + + @classmethod + def clear_channels(cls) -> AclRule: + return cls._new('channel', 'resetchannels', version=(6, 2)) + + @classmethod + def selector(cls, *rules: AclRule) -> AclRule: + """Group command, key and channel rules under Redis 7 selector semantics.""" + tokens = [] + for rule in rules: + if not isinstance(rule, AclRule) or rule._kind not in ('command', 'category', 'key', 'channel'): + raise ConfigurationError('Selectors accept only access rules.') + token = rule._tokens[0] + # Redis reparses a selector as a quoted/escaped argument string. + # Preserve patterns literally and make each rule a single argument. + escaped = token.replace('\\', '\\\\').replace('"', '\\"') + tokens.append('"' + escaped + '"') + return cls._new('selector', '(' + ' '.join(tokens) + ')', version=(7, 0)) + + @classmethod + def clear_selectors(cls) -> AclRule: + return cls._new('clear_selectors', 'clearselectors', version=(7, 0)) + + @classmethod + def add_password(cls, password: str | bytes) -> AclRule: + return cls._new('add_password', _password(password, '>')) + + @classmethod + def remove_password(cls, password: str | bytes) -> AclRule: + return cls._new('remove_password', _password(password, '<')) + + @classmethod + def replace_passwords(cls, passwords: Sequence[str | bytes]) -> AclRule: + if not isinstance(passwords, Sequence) or isinstance(passwords, (str, bytes)): + raise ConfigurationError('Expected an ordered sequence of Redis passwords.') + return cls._new('replace_passwords', 'resetpass', *(_password(p, '>') for p in passwords)) + + +@dataclass(frozen=True) +class RedisSelector: + commands: tuple[str, ...] + categories: tuple[str, ...] + keys: tuple[str, ...] + channels: tuple[str, ...] + + +@dataclass(frozen=True) +class RedisUser: + name: str + flags: tuple[str, ...] + commands: tuple[str, ...] + categories: tuple[str, ...] + keys: tuple[str, ...] + channels: tuple[str, ...] | None + selectors: tuple[RedisSelector, ...] | None + + @property + def enabled(self) -> bool: + return 'on' in self.flags + + +@dataclass(frozen=True) +class RedisDryRun: + allowed: bool + reason: str | None + + +def _text(value: str | bytes, encoding: str) -> str: + return value.decode(encoding, errors='surrogateescape') if isinstance(value, bytes) else value + + +def _fields(raw: Any, encoding: str) -> dict[str, Any]: + pairs = raw.items() if isinstance(raw, Mapping) else zip(raw[::2], raw[1::2], strict=True) + return {_text(key, encoding): value for key, value in pairs} + + +def _words(raw: Any, encoding: str) -> tuple[str, ...]: + if isinstance(raw, (bytes, str)): + return tuple(_text(raw, encoding).split()) + return tuple(_text(v, encoding) for v in raw if v) + + +def _access(data: Mapping[str, Any], encoding: str) -> RedisSelector: + words = _words(data['commands'], encoding) + return RedisSelector( + tuple(x for x in words if x[1:2] != '@'), + tuple(x for x in words if x[1:2] == '@') + _words(data.get('categories', ()), encoding), + _words(data['keys'], encoding), + _words(data.get('channels', ()), encoding), + ) + + +def _user(name: str, raw: Any, encoding: str) -> RedisUser | None: + if raw is None: + return None + data = _fields(raw, encoding) + base = _access(data, encoding) + selectors = data.get('selectors') + return RedisUser( + name, + _words(data['flags'], encoding), + base.commands, + base.categories, + base.keys, + base.channels if 'channels' in data else None, + tuple(_access(_fields(x, encoding), encoding) for x in selectors) if selectors is not None else None, + ) + + +def _rules(rules: Sequence[AclRule]) -> tuple[tuple[str | bytes, ...], tuple[int, int]]: + if ( + not isinstance(rules, Sequence) + or isinstance(rules, (bytes, str)) + or not all(isinstance(r, AclRule) for r in rules) + ): + raise ConfigurationError('Expected an ordered sequence of AclRule instances.') + return tuple(t for r in rules for t in r._tokens), max((r._version for r in rules), default=(6, 0)) + + +def _dry_result(raw: Any) -> RedisDryRun: + if raw in ('OK', b'OK', True): + return RedisDryRun(True, None) + if not isinstance(raw, (str, bytes)): + raise DriverError('Unexpected ACL DRYRUN response.') + # The server can interpolate a key/channel (including credentials) in the reply. + return RedisDryRun(False, 'The target user lacks permission for this command or resource.') + + +def _capability( + binding: Binding, + operation: str, + target: str | None, + error: DriverError, + required: tuple[int, int], + version: str | None, +) -> None: + original = error.original + if version is not None: + match = re.match(r'(\d+)\.(\d+)', version) + if match and tuple(map(int, match.groups())) < required: + raise DependencyError( + f'This ACL operation requires Redis {required[0]}.{required[1]} or later.', + backend='redis', + connection_name=binding.connection_name, + phase='operation', + operation=operation, + target=target, + original=original, + ) from original + + +class _Accounts: + def __init__(self, binding: Binding) -> None: + self._binding = binding + + @property + def _encoding(self) -> str: + return self._binding.native.connection_pool.get_encoder().encoding + + +class RedisAccounts(_Accounts): + """Manage ACLs of the connected data node with a borrowed synchronous client.""" + + def _run( + self, + operation: str, + target: str | None, + subcommand: str, + *args: Any, + write: bool = False, + required: tuple[int, int] = (6, 0), + ) -> Any: + try: + with self._binding.operation(operation, target, write=write): + # Two command tokens request the native RESP result. A single + # 'ACL GETUSER' would apply redis-py's version-dependent parser. + return self._binding.native.execute_command('ACL', subcommand, *args) + except DriverError as error: + # Ask for the version only after a server refusal. A limited ACL + # administrator may legitimately lack INFO permission. + from redis.exceptions import ResponseError + + if isinstance(error.original, ResponseError): + try: + with self._binding.operation('acl_version', target): + info = self._binding.native.info('server') + except DatabaseError: + info = {} + _capability(self._binding, operation, target, error, required, info.get('redis_version')) + raise + + def list_users(self) -> tuple[str, ...]: + raw = self._run('list_users', None, 'USERS') + return tuple(_text(name, self._encoding) for name in raw) + + def get_user(self, name: str) -> RedisUser | None: + name = _argument(name, 'username') + raw = self._run('get_user', name, 'GETUSER', name) + return _user(name, raw, self._encoding) + + def apply_user(self, name: str, *, rules: Sequence[AclRule]) -> None: + """Create or update a user by applying these rules in their given order.""" + name = _argument(name, 'username') + tokens, required = _rules(rules) + raw = self._run('apply_user', name, 'SETUSER', name, *tokens, write=True, required=required) + if raw not in ('OK', b'OK', True): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL SETUSER acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='apply_user', + target=name, + ) + + def delete_user(self, name: str) -> bool: + name = _argument(name, 'username') + raw = self._run('delete_user', name, 'DELUSER', name, write=True) + if raw not in (0, 1): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL DELUSER acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='delete_user', + target=name, + ) + return bool(raw) + + def dry_run(self, name: str, command: str, *args: str | bytes | int | float) -> RedisDryRun: + name = _argument(name, 'username') + raw = self._run('dry_run', name, 'DRYRUN', name, _command(command), *args, required=(7, 0)) + return _dry_result(raw) + + def save_acl(self) -> None: + """Persist the entire node's ACL to its configured external ACL file.""" + raw = self._run('save_acl', None, 'SAVE', write=True) + if raw not in ('OK', b'OK', True): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL SAVE acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='save_acl', + ) + + +class AsyncRedisAccounts(_Accounts): + """Manage ACLs of the connected data node with a borrowed asynchronous client.""" + + async def _run( + self, + operation: str, + target: str | None, + subcommand: str, + *args: Any, + write: bool = False, + required: tuple[int, int] = (6, 0), + ) -> Any: + try: + with self._binding.operation(operation, target, write=write): + # Two command tokens request the native RESP result. A single + # 'ACL GETUSER' would apply redis-py's version-dependent parser. + return await self._binding.native.execute_command('ACL', subcommand, *args) + except DriverError as error: + # Ask for the version only after a server refusal. A limited ACL + # administrator may legitimately lack INFO permission. + from redis.exceptions import ResponseError + + if isinstance(error.original, ResponseError): + try: + with self._binding.operation('acl_version', target): + info = await self._binding.native.info('server') + except DatabaseError: + info = {} + _capability(self._binding, operation, target, error, required, info.get('redis_version')) + raise + + async def list_users(self) -> tuple[str, ...]: + raw = await self._run('list_users', None, 'USERS') + return tuple(_text(name, self._encoding) for name in raw) + + async def get_user(self, name: str) -> RedisUser | None: + name = _argument(name, 'username') + raw = await self._run('get_user', name, 'GETUSER', name) + return _user(name, raw, self._encoding) + + async def apply_user(self, name: str, *, rules: Sequence[AclRule]) -> None: + """Create or update a user by applying these rules in their given order.""" + name = _argument(name, 'username') + tokens, required = _rules(rules) + raw = await self._run('apply_user', name, 'SETUSER', name, *tokens, write=True, required=required) + if raw not in ('OK', b'OK', True): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL SETUSER acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='apply_user', + target=name, + ) + + async def delete_user(self, name: str) -> bool: + name = _argument(name, 'username') + raw = await self._run('delete_user', name, 'DELUSER', name, write=True) + if raw not in (0, 1): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL DELUSER acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='delete_user', + target=name, + ) + return bool(raw) + + async def dry_run(self, name: str, command: str, *args: str | bytes | int | float) -> RedisDryRun: + name = _argument(name, 'username') + raw = await self._run('dry_run', name, 'DRYRUN', name, _command(command), *args, required=(7, 0)) + return _dry_result(raw) + + async def save_acl(self) -> None: + """Persist the entire node's ACL to its configured external ACL file.""" + raw = await self._run('save_acl', None, 'SAVE', write=True) + if raw not in ('OK', b'OK', True): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL SAVE acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='save_acl', + ) diff --git a/tests/test_accounts_redis.py b/tests/test_accounts_redis.py new file mode 100644 index 0000000..eb50248 --- /dev/null +++ b/tests/test_accounts_redis.py @@ -0,0 +1,185 @@ +"""Redis account boundaries, rule compilation, and RESP permission views.""" + +import asyncio +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import Mock + +from database.accounts import Accounts +from database.accounts import AccountWriteOutcomeUnknown +from database.accounts import AclRule +from database.accounts._base import Binding +from database.accounts._redis import AsyncRedisAccounts +from database.accounts._redis import RedisAccounts +from database.connections import Connection +from database.errors import ConfigurationError +from database.errors import DatabaseConnectionError +from database.errors import DriverError + + +def fake_binding(native): + native.connection_pool.get_encoder.return_value.encoding = 'utf-8' + adapter = SimpleNamespace(classify=lambda error: None) + handle = Connection(native, adapter, 'cache', 'redis', 'single') + return Binding(handle, 'redis', is_async=False) + + +class RedisRulesTests(unittest.TestCase): + def test_order_and_secret_hygiene(self): + native = Mock() + native.execute_command.return_value = b'OK' + acl = RedisAccounts(fake_binding(native)) + rules = [ + AclRule.clear_keys(), + AclRule.key_pattern('team:*'), + AclRule.allow_category('read'), + AclRule.deny_command('get'), + AclRule.replace_passwords([b'private-a', 'private-b']), + AclRule.remove_password('private-old'), + ] + acl.apply_user('reader', rules=rules) + native.execute_command.assert_called_once_with( + 'ACL', + 'SETUSER', + 'reader', + 'resetkeys', + '~team:*', + '+@read', + '-get', + 'resetpass', + b'>private-a', + '>private-b', + ' Date: Wed, 18 Nov 2026 16:32:40 +0800 Subject: [PATCH 79/86] =?UTF-8?q?feat(accounts):=20=E5=AE=9E=E7=8E=B0=20Mo?= =?UTF-8?q?ngoDB=20=E7=94=A8=E6=88=B7=E8=A7=92=E8=89=B2=E4=B8=8E=E6=9D=83?= =?UTF-8?q?=E9=99=90=E7=AE=A1=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 同步和异步复用结构化命令、权限投影及写关注处理。新增离线故障测试和使用显式配置的真实服务验收脚本。 --- scripts/verify_accounts_mongodb.py | 235 +++++++++++++++++ src/database/accounts/_mongodb.py | 405 +++++++++++++++++++++++++++++ tests/test_accounts_mongodb.py | 192 ++++++++++++++ 3 files changed, 832 insertions(+) create mode 100644 scripts/verify_accounts_mongodb.py create mode 100644 src/database/accounts/_mongodb.py create mode 100644 tests/test_accounts_mongodb.py diff --git a/scripts/verify_accounts_mongodb.py b/scripts/verify_accounts_mongodb.py new file mode 100644 index 0000000..a563c46 --- /dev/null +++ b/scripts/verify_accounts_mongodb.py @@ -0,0 +1,235 @@ +"""Verify MongoDB account management against an explicitly configured isolated deployment.""" + +from __future__ import annotations + +import argparse +import asyncio +import inspect +import json +from contextlib import asynccontextmanager +from pathlib import Path +from secrets import token_urlsafe +from uuid import uuid4 + +import pymongo +from pymongo.errors import OperationFailure + +from database import ConnectionConfig +from database import Connections +from database.accounts import Accounts +from database.accounts import AsyncAccounts +from database.accounts import MongoPrivilege +from database.accounts import MongoRole +from database.accounts import MongoUser +from database.errors import DriverError +from database.errors import PermissionDeniedError +from database.errors import attach_cleanup_errors + + +def settings(path: Path) -> tuple[dict, ConnectionConfig]: + config = json.loads(path.read_text(encoding='utf-8-sig')) + raw = dict( + host=config['host'], + port=config['port'], + username=config['username'], + password=config['password'], + authSource=config['auth_db'], + serverSelectionTimeoutMS=6000, + ) + sdk = ConnectionConfig( + backend='mongodb', + host=raw['host'], + port=raw['port'], + username=raw['username'], + password=raw['password'], + options={'authSource': raw['authSource'], 'serverSelectionTimeoutMS': 6000}, + ) + return raw, sdk + + +def check(condition: bool, label: str) -> None: + if not condition: + raise AssertionError(label) + + +async def result(value): + """Use identical assertions for both native driver execution modes.""" + return await value if inspect.isawaitable(value) else value + + +@asynccontextmanager +async def owned(client): + try: + yield client + finally: + await result(client.close()) + + +async def run_suite(raw: dict, sdk: ConnectionConfig, *, managed: bool, asynchronous: bool) -> None: + reg = Connections({'admin': sdk}) + factory = AsyncAccounts if asynchronous else Accounts + client_type = pymongo.AsyncMongoClient if asynchronous else pymongo.MongoClient + if managed: + owner = await reg.create_async('admin') if asynchronous else reg.create('admin') + else: + owner = client_type(**raw) + client = owner.native if managed else owner + admin = factory.mongodb(owner) + stem = 'qa_' + uuid4().hex + staff = stem + 'staff' + data = stem + 'data' + user = MongoUser(staff, 'analyst') + twin = MongoUser(stem + 'other', 'analyst') + reader = MongoUser(staff, 'viewer') + parent = MongoRole(data, 'parent') + child = MongoRole(data, 'child') + audit = MongoRole(data, 'viewer') + read = MongoPrivilege({'db': data, 'collection': 'reports'}, ['find']) + update = MongoPrivilege({'db': data, 'collection': 'reports'}, ['update']) + password, replacement, viewer_secret = token_urlsafe(24), token_urlsafe(24), token_urlsafe(24) + cleanup = [] + primary = None + + def login(account, pwd): + return client_type(**{**raw, 'username': account.name, 'password': pwd, 'authSource': account.auth_db}) + + try: + check(await result(admin.get_user(user)) is None, 'fresh identity') + await result(admin.create_role(parent, privileges=[read])) + cleanup.append((admin.drop_role, parent)) + await result(admin.create_role(child)) + cleanup.append((admin.drop_role, child)) + await result(admin.grant_roles_to_role(child, roles=[parent])) + await result(admin.grant_privileges(parent, privileges=[update])) + await result(admin.revoke_privileges(parent, privileges=[update])) + check( + tuple((await result(admin.get_role(child, expand_privileges=True))).inherited_privileges) == (read,), + 'inherited privilege and incremental removal', + ) + check(child in [item.role for item in await result(admin.list_roles(db=data))], 'role list') + check( + any(item.builtin for item in await result(admin.list_roles(db=data, include_builtin=True))), + 'built-in roles', + ) + await result(admin.create_user(user, password=password)) + cleanup.append((admin.drop_user, user)) + await result(admin.create_user(twin, password=token_urlsafe(24))) + cleanup.append((admin.drop_user, twin)) + try: + await result(admin.create_user(user, password=replacement)) + except DriverError: + pass + else: + raise AssertionError('duplicate identity unexpectedly created') + await result(admin.grant_roles(user, roles=[child])) + view = await result(admin.get_user(user, expand_privileges=True)) + check(view.roles == (child,) and read in view.inherited_privileges, 'user details') + check((await result(admin.get_user(user))).inherited_privileges is None, 'explicit expansion') + check(user in [v.user for v in await result(admin.list_users(auth_db=staff))], 'per-db list') + check({user, twin}.issubset({v.user for v in await result(admin.list_users_all_databases())}), 'cross-db list') + async with owned(login(user, password)) as restricted: + await result(restricted[data].reports.find_one({})) + try: + await result(restricted[data].private.find_one({})) + except OperationFailure as exc: + check(exc.code == 13, 'unexpected read rejection') + else: + raise AssertionError('unrelated collection readable') + await result(admin.change_password(user, password=replacement)) + async with owned(login(user, password)) as old: + try: + await result(old.admin.command('ping')) + except OperationFailure as exc: + check(exc.code == 18, 'expected authentication failure') + else: + raise AssertionError('old password accepted') + await result(admin.revoke_roles_from_role(child, roles=[parent])) + async with owned(login(user, replacement)) as restricted: + try: + await result(restricted[data].reports.find_one({})) + except OperationFailure as exc: + check(exc.code == 13, 'expected revoked access') + else: + raise AssertionError('read survived revocation') + await result(admin.grant_roles_to_role(child, roles=[parent])) + await result(admin.revoke_roles(user, roles=[child])) + check(not (await result(admin.get_user(user))).roles, 'user role revocation') + await result(admin.grant_roles(user, roles=[parent])) + await result(admin.drop_role(parent)) + cleanup.remove((admin.drop_role, parent)) + check( + not (await result(admin.get_role(child))).roles and not (await result(admin.get_user(user))).roles, + 'dropped role references', + ) + await result( + admin.create_role(audit, privileges=[MongoPrivilege({'db': data, 'collection': ''}, ['viewUser'])]) + ) + cleanup.append((admin.drop_role, audit)) + await result(admin.create_user(reader, password=viewer_secret, roles=[audit])) + cleanup.append((admin.drop_user, reader)) + async with owned(login(reader, viewer_secret)) as viewer_client: + viewer = factory.mongodb(viewer_client) + await result(viewer.list_users(auth_db=data)) + try: + await result(viewer.drop_user(user)) + except PermissionDeniedError: + pass + else: + raise AssertionError('read-only manager could delete a user') + await result(admin.drop_user(user)) + cleanup.remove((admin.drop_user, user)) + check( + await result(admin.get_user(user)) is None and await result(admin.get_user(twin)) is not None, + 'exact deletion', + ) + async with owned(login(user, replacement)) as removed: + try: + await result(removed.admin.command('ping')) + except OperationFailure as exc: + check(exc.code == 18, 'deleted user authentication') + else: + raise AssertionError('deleted account can authenticate') + await result(client.admin.command('ping')) + except BaseException as exc: + primary = exc + raise + finally: + failures = [] + for remove, item in reversed(cleanup): + try: + await result(remove(item)) + except Exception as exc: + failures.append(exc) + try: + await result(owner.close()) + except Exception as exc: + failures.append(exc) + if failures: + if primary is None: + raise failures[0] + attach_cleanup_errors(primary, failures) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--config', + required=True, + type=Path, + help='Dedicated test service JSON: host, port, username, password, auth_db', + ) + parser.add_argument('--mode', choices=['sync', 'async', 'both'], default='both') + args = parser.parse_args() + raw, sdk = settings(args.config) + with pymongo.MongoClient(**raw) as client: + version = client.server_info()['version'] + for mode in ('sync', 'async'): + if args.mode not in (mode, 'both'): + continue + for managed in (False, True): + asyncio.run(run_suite(raw, sdk, managed=managed, asynchronous=mode == 'async')) + print(f'PASS MongoDB={version} PyMongo={pymongo.version} mode={mode} managed={managed}') + + +if __name__ == '__main__': + main() diff --git a/src/database/accounts/_mongodb.py b/src/database/accounts/_mongodb.py new file mode 100644 index 0000000..9f99aab --- /dev/null +++ b/src/database/accounts/_mongodb.py @@ -0,0 +1,405 @@ +"""Account and role operations for a self-managed MongoDB deployment.""" + +from __future__ import annotations + +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import field +from types import MappingProxyType +from typing import Any +from typing import Generic +from typing import TypeVar + +from ..errors import ConfigurationError +from ._base import AccountWriteOutcomeUnknown +from ._base import Binding + + +def _name(value: str) -> str: + if not isinstance(value, str) or not value or '\0' in value: + raise ValueError('Expected a nonempty MongoDB name without a null character.') + return value + + +@dataclass(frozen=True) +class MongoUser: + auth_db: str + name: str + + def __post_init__(self) -> None: + _name(self.auth_db) + _name(self.name) + + +@dataclass(frozen=True) +class MongoRole: + db: str + name: str + + def __post_init__(self) -> None: + _name(self.db) + _name(self.name) + + +@dataclass(frozen=True, init=False) +class MongoPrivilege: + resource: Mapping[str, str | bool] + actions: tuple[str, ...] + + def __init__(self, resource: Mapping[str, str | bool], actions: Iterable[str]) -> None: + if not isinstance(resource, Mapping) or not resource: + raise ValueError('Expected a MongoDB resource document.') + snapshot = dict(resource) + if not all(isinstance(k, str) and isinstance(v, (str, bool)) for k, v in snapshot.items()): + raise ValueError('Invalid MongoDB resource field.') + if isinstance(actions, (str, bytes, Mapping)): + raise TypeError('Expected an iterable of action names.') + names = tuple(_name(action) for action in actions) + if not names: + raise ValueError('Expected at least one MongoDB privilege action.') + object.__setattr__(self, 'resource', MappingProxyType(snapshot)) + object.__setattr__(self, 'actions', names) + + +@dataclass(frozen=True) +class MongoUserInfo: + user: MongoUser + roles: tuple[MongoRole, ...] + inherited_roles: tuple[MongoRole, ...] | None + inherited_privileges: tuple[MongoPrivilege, ...] | None + + +@dataclass(frozen=True) +class MongoRoleInfo: + role: MongoRole + builtin: bool + roles: tuple[MongoRole, ...] + privileges: tuple[MongoPrivilege, ...] | None + inherited_roles: tuple[MongoRole, ...] + inherited_privileges: tuple[MongoPrivilege, ...] | None + + +def _roles(items: Iterable[MongoRole]) -> list[dict[str, str]]: + if isinstance(items, (str, bytes, Mapping)): + raise TypeError('Expected a sequence of MongoRole objects.') + result = [] + for role in items: + if not isinstance(role, MongoRole): + raise TypeError('Expected a MongoRole.') + result.append({'role': role.name, 'db': role.db}) + return result + + +def _privileges(items: Iterable[MongoPrivilege]) -> list[dict[str, Any]]: + if isinstance(items, (str, bytes, Mapping)): + raise TypeError('Expected a sequence of MongoPrivilege objects.') + result = [] + for privilege in items: + if not isinstance(privilege, MongoPrivilege): + raise TypeError('Expected a MongoPrivilege.') + result.append({'resource': dict(privilege.resource), 'actions': list(privilege.actions)}) + return result + + +def _password(password: str) -> str: + if not isinstance(password, str) or not password: + raise ValueError('Expected a nonempty password.') + return password + + +def _read_roles(document: dict[str, Any], field: str) -> tuple[MongoRole, ...]: + return tuple(MongoRole(r['db'], r['role']) for r in document[field]) + + +def _read_privileges(document: dict[str, Any], field: str) -> tuple[MongoPrivilege, ...]: + return tuple(MongoPrivilege(p['resource'], p['actions']) for p in document[field]) + + +def _user_info(row: dict[str, Any], expanded: bool = False) -> MongoUserInfo: + return MongoUserInfo( + MongoUser(row['db'], row['user']), + _read_roles(row, 'roles'), + _read_roles(row, 'inheritedRoles') if expanded else None, + _read_privileges(row, 'inheritedPrivileges') if expanded else None, + ) + + +def _role_info(row: dict[str, Any], expanded: bool = False) -> MongoRoleInfo: + return MongoRoleInfo( + MongoRole(row['db'], row['role']), + row['isBuiltin'], + _read_roles(row, 'roles'), + _read_privileges(row, 'privileges') if expanded else None, + _read_roles(row, 'inheritedRoles'), + _read_privileges(row, 'inheritedPrivileges') if expanded else None, + ) + + +_USERS = {'showCredentials': False, 'showCustomData': False} + + +class _MongoAccounts: + def __init__(self, binding: Binding) -> None: + self._binding = binding + + @contextmanager + def _dispatch(self, operation: str, target: str, write: bool) -> Iterator[None]: + from pymongo.errors import ServerSelectionTimeoutError + from pymongo.errors import WaitQueueTimeoutError + from pymongo.errors import WriteConcernError + + with self._binding.operation(operation, target, write=write): + try: + yield + except (ServerSelectionTimeoutError, WaitQueueTimeoutError): + # No connection was selected or checked out for this command. + with self._binding.operation(operation, target): + raise + except WriteConcernError as error: + if not write: + raise + raise AccountWriteOutcomeUnknown( + 'MongoDB did not confirm the requested write concern.', + connection_name=self._binding.connection_name, + backend='mongodb', + phase='operation', + operation=operation, + target=target, + original=error, + ) from error + + def _prepare(self, db: str, command: dict[str, Any], write: bool) -> Any: + database = self._binding.native[_name(db)] + if write: + concern = database.write_concern + if not concern.acknowledged: + raise ConfigurationError('MongoDB account changes require acknowledged writes.', backend='mongodb') + if concern.document: + command['writeConcern'] = concern.document + return database + + @staticmethod + def _confirm(reply: dict[str, Any]) -> None: + from pymongo.errors import WriteConcernError + + if 'writeConcernError' in reply: + detail = reply['writeConcernError'] + raise WriteConcernError('MongoDB write concern failed.', detail.get('code'), detail) + + +_T = TypeVar('_T') + + +@dataclass(frozen=True) +class _Request(Generic[_T]): + operation: str + db: str + spec: dict[str, Any] = field(repr=False) + decode: Callable[[dict[str, Any]], _T] = field(repr=False) + target: str + write: bool = False + + +def _users(db: str, all_databases: bool = False) -> _Request[tuple[MongoUserInfo, ...]]: + spec = {'usersInfo': {'forAllDBs': True} if all_databases else 1, **_USERS} + op = 'list_users_all_databases' if all_databases else 'list_users' + return _Request(op, db, spec, lambda reply: tuple(_user_info(u) for u in reply['users']), db) + + +def _user(user: MongoUser, expanded: bool) -> _Request[MongoUserInfo | None]: + if not isinstance(user, MongoUser): + raise TypeError('Expected a MongoUser.') + spec = {'usersInfo': user.name, 'showPrivileges': expanded, **_USERS} + return _Request( + 'get_user', + user.auth_db, + spec, + lambda reply: _user_info(reply['users'][0], expanded) if reply['users'] else None, + f'{user.auth_db}.{user.name}', + ) + + +def _list_roles(db: str, builtin: bool) -> _Request[tuple[MongoRoleInfo, ...]]: + return _Request( + 'list_roles', + db, + {'rolesInfo': 1, 'showBuiltinRoles': builtin}, + lambda reply: tuple(_role_info(r) for r in reply['roles']), + db, + ) + + +def _role(role: MongoRole, expanded: bool) -> _Request[MongoRoleInfo | None]: + if not isinstance(role, MongoRole): + raise TypeError('Expected a MongoRole.') + return _Request( + 'get_role', + role.db, + {'rolesInfo': role.name, 'showPrivileges': expanded}, + lambda reply: _role_info(reply['roles'][0], expanded) if reply['roles'] else None, + f'{role.db}.{role.name}', + ) + + +def _user_write(operation: str, command: str, user: MongoUser, **fields: Any) -> _Request[None]: + if not isinstance(user, MongoUser): + raise TypeError('Expected a MongoUser.') + return _Request( + operation, + user.auth_db, + {command: user.name, **fields}, + _MongoAccounts._confirm, + f'{user.auth_db}.{user.name}', + write=True, + ) + + +def _role_write(operation: str, command: str, role: MongoRole, **fields: Any) -> _Request[None]: + if not isinstance(role, MongoRole): + raise TypeError('Expected a MongoRole.') + return _Request( + operation, + role.db, + {command: role.name, **fields}, + _MongoAccounts._confirm, + f'{role.db}.{role.name}', + write=True, + ) + + +class MongoDBAccounts(_MongoAccounts): + """Manage MongoDB users and roles on a borrowed synchronous client.""" + + def _run(self, request: _Request[_T]) -> _T: + database = self._prepare(request.db, request.spec, request.write) + with self._dispatch(request.operation, request.target, request.write): + reply = database.command(request.spec) + return request.decode(reply) + + def list_users(self, *, auth_db: str) -> tuple[MongoUserInfo, ...]: + return self._run(_users(auth_db)) + + def list_users_all_databases(self) -> tuple[MongoUserInfo, ...]: + return self._run(_users('admin', all_databases=True)) + + def get_user(self, user: MongoUser, *, expand_privileges: bool = False) -> MongoUserInfo | None: + return self._run(_user(user, expand_privileges)) + + def list_roles(self, *, db: str, include_builtin: bool = False) -> tuple[MongoRoleInfo, ...]: + return self._run(_list_roles(db, include_builtin)) + + def get_role(self, role: MongoRole, *, expand_privileges: bool = False) -> MongoRoleInfo | None: + return self._run(_role(role, expand_privileges)) + + def create_user(self, user: MongoUser, *, password: str, roles: Iterable[MongoRole] = ()) -> None: + return self._run(_user_write('create_user', 'createUser', user, pwd=_password(password), roles=_roles(roles))) + + def drop_user(self, user: MongoUser) -> None: + return self._run(_user_write('drop_user', 'dropUser', user)) + + def change_password(self, user: MongoUser, *, password: str) -> None: + return self._run(_user_write('change_password', 'updateUser', user, pwd=_password(password))) + + def grant_roles(self, user: MongoUser, *, roles: Iterable[MongoRole]) -> None: + return self._run(_user_write('grant_roles', 'grantRolesToUser', user, roles=_roles(roles))) + + def revoke_roles(self, user: MongoUser, *, roles: Iterable[MongoRole]) -> None: + return self._run(_user_write('revoke_roles', 'revokeRolesFromUser', user, roles=_roles(roles))) + + def create_role( + self, role: MongoRole, *, privileges: Iterable[MongoPrivilege] = (), roles: Iterable[MongoRole] = () + ) -> None: + return self._run( + _role_write('create_role', 'createRole', role, privileges=_privileges(privileges), roles=_roles(roles)) + ) + + def drop_role(self, role: MongoRole) -> None: + return self._run(_role_write('drop_role', 'dropRole', role)) + + def grant_privileges(self, role: MongoRole, *, privileges: Iterable[MongoPrivilege]) -> None: + return self._run( + _role_write('grant_privileges', 'grantPrivilegesToRole', role, privileges=_privileges(privileges)) + ) + + def revoke_privileges(self, role: MongoRole, *, privileges: Iterable[MongoPrivilege]) -> None: + return self._run( + _role_write('revoke_privileges', 'revokePrivilegesFromRole', role, privileges=_privileges(privileges)) + ) + + def grant_roles_to_role(self, role: MongoRole, *, roles: Iterable[MongoRole]) -> None: + return self._run(_role_write('grant_roles_to_role', 'grantRolesToRole', role, roles=_roles(roles))) + + def revoke_roles_from_role(self, role: MongoRole, *, roles: Iterable[MongoRole]) -> None: + return self._run(_role_write('revoke_roles_from_role', 'revokeRolesFromRole', role, roles=_roles(roles))) + + +class AsyncMongoDBAccounts(_MongoAccounts): + """Manage MongoDB users and roles on a borrowed asynchronous client.""" + + async def _run(self, request: _Request[_T]) -> _T: + database = self._prepare(request.db, request.spec, request.write) + with self._dispatch(request.operation, request.target, request.write): + reply = await database.command(request.spec) + return request.decode(reply) + + async def list_users(self, *, auth_db: str) -> tuple[MongoUserInfo, ...]: + return await self._run(_users(auth_db)) + + async def list_users_all_databases(self) -> tuple[MongoUserInfo, ...]: + return await self._run(_users('admin', all_databases=True)) + + async def get_user(self, user: MongoUser, *, expand_privileges: bool = False) -> MongoUserInfo | None: + return await self._run(_user(user, expand_privileges)) + + async def list_roles(self, *, db: str, include_builtin: bool = False) -> tuple[MongoRoleInfo, ...]: + return await self._run(_list_roles(db, include_builtin)) + + async def get_role(self, role: MongoRole, *, expand_privileges: bool = False) -> MongoRoleInfo | None: + return await self._run(_role(role, expand_privileges)) + + async def create_user(self, user: MongoUser, *, password: str, roles: Iterable[MongoRole] = ()) -> None: + return await self._run( + _user_write('create_user', 'createUser', user, pwd=_password(password), roles=_roles(roles)) + ) + + async def drop_user(self, user: MongoUser) -> None: + return await self._run(_user_write('drop_user', 'dropUser', user)) + + async def change_password(self, user: MongoUser, *, password: str) -> None: + return await self._run(_user_write('change_password', 'updateUser', user, pwd=_password(password))) + + async def grant_roles(self, user: MongoUser, *, roles: Iterable[MongoRole]) -> None: + return await self._run(_user_write('grant_roles', 'grantRolesToUser', user, roles=_roles(roles))) + + async def revoke_roles(self, user: MongoUser, *, roles: Iterable[MongoRole]) -> None: + return await self._run(_user_write('revoke_roles', 'revokeRolesFromUser', user, roles=_roles(roles))) + + async def create_role( + self, role: MongoRole, *, privileges: Iterable[MongoPrivilege] = (), roles: Iterable[MongoRole] = () + ) -> None: + return await self._run( + _role_write('create_role', 'createRole', role, privileges=_privileges(privileges), roles=_roles(roles)) + ) + + async def drop_role(self, role: MongoRole) -> None: + return await self._run(_role_write('drop_role', 'dropRole', role)) + + async def grant_privileges(self, role: MongoRole, *, privileges: Iterable[MongoPrivilege]) -> None: + return await self._run( + _role_write('grant_privileges', 'grantPrivilegesToRole', role, privileges=_privileges(privileges)) + ) + + async def revoke_privileges(self, role: MongoRole, *, privileges: Iterable[MongoPrivilege]) -> None: + return await self._run( + _role_write('revoke_privileges', 'revokePrivilegesFromRole', role, privileges=_privileges(privileges)) + ) + + async def grant_roles_to_role(self, role: MongoRole, *, roles: Iterable[MongoRole]) -> None: + return await self._run(_role_write('grant_roles_to_role', 'grantRolesToRole', role, roles=_roles(roles))) + + async def revoke_roles_from_role(self, role: MongoRole, *, roles: Iterable[MongoRole]) -> None: + return await self._run(_role_write('revoke_roles_from_role', 'revokeRolesFromRole', role, roles=_roles(roles))) diff --git a/tests/test_accounts_mongodb.py b/tests/test_accounts_mongodb.py new file mode 100644 index 0000000..98f0d08 --- /dev/null +++ b/tests/test_accounts_mongodb.py @@ -0,0 +1,192 @@ +"""MongoDB account commands, secret projection, and dispatch boundaries.""" + +import asyncio +import importlib.util +import unittest +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +from database.accounts import Accounts +from database.accounts import AccountWriteOutcomeUnknown +from database.accounts import AsyncAccounts +from database.accounts import MongoPrivilege +from database.accounts import MongoRole +from database.accounts import MongoUser +from database.connections import AsyncConnection +from database.connections import Connection +from database.errors import ConfigurationError +from database.errors import DatabaseTimeoutError +from database.errors import PermissionDeniedError + +HAS_DRIVER = importlib.util.find_spec('pymongo') is not None + + +def fixture(asynchronous=False): + from pymongo.write_concern import WriteConcern + + from database._adapters.mongodb import Adapter + + client = MagicMock() + database = client.__getitem__.return_value + database.write_concern = WriteConcern() + if asynchronous: + database.command = AsyncMock() + handle = AsyncConnection(client, Adapter, 'mg', 'mongodb', 'single', asyncio.get_running_loop()) + admin = AsyncAccounts.mongodb(handle) + else: + handle = Connection(client, Adapter, 'mg', 'mongodb', 'single') + admin = Accounts.mongodb(handle) + return client, database, admin + + +class ModelTests(unittest.TestCase): + def test_owns_permissions_and_validates_names(self): + resource = {'db': '', 'collection': ''} + actions = ['find'] + value = MongoPrivilege(resource, actions) + resource['db'] = 'changed' + actions.clear() + self.assertEqual(dict(value.resource), {'db': '', 'collection': ''}) + self.assertEqual(value.actions, ('find',)) + with self.assertRaises(TypeError): + value.resource['db'] = 'mutated' + for args in [({}, ['find']), ({'cluster': 1}, ['find']), ({'cluster': True}, [])]: + with self.subTest(args=args), self.assertRaises(ValueError): + MongoPrivilege(*args) + with self.assertRaises(TypeError): + MongoPrivilege({'cluster': True}, 'find') + for value in [None, '', '\0']: + with self.subTest(value=value), self.assertRaises(ValueError): + MongoRole('admin', value) + + +@unittest.skipUnless(HAS_DRIVER, 'requires PyMongo') +class MongoTests(unittest.TestCase): + def setUp(self): + self.client, self.db, self.admin = fixture() + self.user = MongoUser('staff', 'reader') + self.role = MongoRole('reports', 'read') + + def test_selection_and_projection(self): + self.db.command.return_value = { + 'ok': 1, + 'users': [ + { + 'db': 'staff', + 'user': 'reader', + 'roles': [{'db': 'reports', 'role': 'read'}], + 'credentials': {'password': 'private'}, + 'customData': {'code': 'secret'}, + } + ], + } + info = self.admin.list_users(auth_db='staff')[0] + self.client.__getitem__.assert_called_once_with('staff') + self.db.command.assert_called_once_with({'usersInfo': 1, 'showCustomData': False, 'showCredentials': False}) + self.assertEqual(info.user, self.user) + self.assertEqual(info.roles, (self.role,)) + self.assertIsNone(info.inherited_privileges) + self.assertNotIn('private', repr(info)) + self.assertNotIn('secret', repr(info)) + self.admin.list_users_all_databases() + self.client.__getitem__.assert_called_with('admin') + self.assertEqual(self.db.command.call_args.args[0]['usersInfo'], {'forAllDBs': True}) + self.db.command.return_value = {'users': []} + self.assertIsNone(self.admin.get_user(self.user)) + self.db.command.return_value = {'roles': []} + self.assertIsNone(self.admin.get_role(self.role, expand_privileges=True)) + + def test_invalid_inputs_do_not_dispatch(self): + with self.assertRaises(TypeError): + self.admin.drop_role(self.user) + with self.assertRaises(TypeError): + self.admin.grant_roles(self.user, roles=['read']) + with self.assertRaises(ValueError): + self.admin.create_user(self.user, password='') + self.db.command.assert_not_called() + + def test_explicit_mutations_and_concern(self): + from pymongo.write_concern import WriteConcern + + self.db.command.return_value = {'ok': 1} + self.db.write_concern = WriteConcern(w='majority') + self.admin.grant_roles(self.user, roles=[self.role]) + self.db.command.assert_called_once_with( + { + 'grantRolesToUser': 'reader', + 'roles': [{'db': 'reports', 'role': 'read'}], + 'writeConcern': {'w': 'majority'}, + } + ) + self.admin.change_password(self.user, password='replacement') + self.db.command.assert_called_with( + {'updateUser': 'reader', 'pwd': 'replacement', 'writeConcern': {'w': 'majority'}} + ) + self.db.write_concern = WriteConcern(w=0) + with self.assertRaises(ConfigurationError): + self.admin.drop_user(self.user) + self.assertEqual(self.db.command.call_count, 2) + + def test_failures_distinguish_point_of_dispatch(self): + from pymongo.errors import AutoReconnect + from pymongo.errors import OperationFailure + from pymongo.errors import ServerSelectionTimeoutError + from pymongo.errors import WriteConcernError + + for exc, expected in [ + (ServerSelectionTimeoutError('no target'), DatabaseTimeoutError), + (AutoReconnect('lost response'), AccountWriteOutcomeUnknown), + (WriteConcernError('concern', 64), AccountWriteOutcomeUnknown), + (OperationFailure('secret detail', 13), PermissionDeniedError), + ]: + with self.subTest(error=exc): + self.db.command.side_effect = exc + with self.assertRaises(expected) as caught: + self.admin.drop_user(self.user) + self.assertIs(caught.exception.__cause__, exc) + self.assertEqual(caught.exception.operation, 'drop_user') + self.assertNotIn('secret detail', str(caught.exception)) + self.assertEqual(self.db.command.call_count, 4) + self.db.command.side_effect = None + self.db.command.return_value = {'ok': 1, 'writeConcernError': {'code': 64, 'errmsg': 'not acknowledged'}} + with self.assertRaises(AccountWriteOutcomeUnknown) as caught: + self.admin.drop_user(self.user) + self.assertEqual(caught.exception.original.code, 64) + self.assertEqual(caught.exception.target, 'staff.reader') + + +@unittest.skipUnless(HAS_DRIVER, 'requires PyMongo') +class AsyncMongoTests(unittest.IsolatedAsyncioTestCase): + async def test_async_roles_read_write_and_cancel(self): + _, db, admin = fixture(True) + role = MongoRole('db', 'reader') + privilege = MongoPrivilege({'db': 'db', 'collection': 'c'}, ['find']) + db.command.return_value = { + 'roles': [ + { + 'db': 'db', + 'role': 'reader', + 'isBuiltin': False, + 'roles': [], + 'privileges': [], + 'inheritedRoles': [], + 'inheritedPrivileges': [], + 'credentials': 'secret', + } + ] + } + detail = await admin.get_role(role, expand_privileges=True) + self.assertEqual(detail.privileges, ()) + self.assertEqual(detail.inherited_privileges, ()) + self.assertNotIn('secret', repr(detail)) + await admin.grant_privileges(role, privileges=[privilege]) + db.command.assert_awaited_with( + { + 'grantPrivilegesToRole': 'reader', + 'privileges': [{'resource': {'db': 'db', 'collection': 'c'}, 'actions': ['find']}], + } + ) + db.command.side_effect = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError): + await admin.drop_role(role) + self.assertEqual(db.command.await_count, 3) -- Gitee From c656848b60df094afad219808db1f208706e1d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 16:40:00 +0800 Subject: [PATCH 80/86] =?UTF-8?q?fix(accounts):=20=E6=A0=B8=E5=AF=B9=20Mon?= =?UTF-8?q?goDB=20=E9=AA=8C=E6=94=B6=E8=B5=84=E6=BA=90=E5=88=9B=E5=BB=BA?= =?UTF-8?q?=E7=BB=93=E6=9E=9C=E5=90=8E=E6=B8=85=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/verify_accounts_mongodb.py | 45 +++++++++++++++++-------- tests/test_accounts_mongodb.py | 53 ++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 14 deletions(-) diff --git a/scripts/verify_accounts_mongodb.py b/scripts/verify_accounts_mongodb.py index a563c46..960eaba 100644 --- a/scripts/verify_accounts_mongodb.py +++ b/scripts/verify_accounts_mongodb.py @@ -17,6 +17,7 @@ from pymongo.errors import OperationFailure from database import ConnectionConfig from database import Connections from database.accounts import Accounts +from database.accounts import AccountWriteOutcomeUnknown from database.accounts import AsyncAccounts from database.accounts import MongoPrivilege from database.accounts import MongoRole @@ -65,6 +66,28 @@ async def owned(client): await result(client.close()) +async def create_fixture(admin, identity, cleanup, **options) -> None: + """Record a unique fixture after success or after reconciling an ambiguous creation.""" + if isinstance(identity, MongoUser): + get, create, remove = admin.get_user, admin.create_user, admin.drop_user + else: + get, create, remove = admin.get_role, admin.create_role, admin.drop_role + if await result(get(identity)) is not None: + raise RuntimeError(f'Isolated fixture already exists: {identity!r}') + try: + await result(create(identity, **options)) + except (AccountWriteOutcomeUnknown, asyncio.CancelledError) as primary: + try: + observed = await result(get(identity)) + except Exception as lookup_error: + attach_cleanup_errors(primary, (lookup_error,)) + else: + if observed is not None: + cleanup.append((remove, identity)) + raise + cleanup.append((remove, identity)) + + async def run_suite(raw: dict, sdk: ConnectionConfig, *, managed: bool, asynchronous: bool) -> None: reg = Connections({'admin': sdk}) factory = AsyncAccounts if asynchronous else Accounts @@ -94,11 +117,8 @@ async def run_suite(raw: dict, sdk: ConnectionConfig, *, managed: bool, asynchro return client_type(**{**raw, 'username': account.name, 'password': pwd, 'authSource': account.auth_db}) try: - check(await result(admin.get_user(user)) is None, 'fresh identity') - await result(admin.create_role(parent, privileges=[read])) - cleanup.append((admin.drop_role, parent)) - await result(admin.create_role(child)) - cleanup.append((admin.drop_role, child)) + await create_fixture(admin, parent, cleanup, privileges=[read]) + await create_fixture(admin, child, cleanup) await result(admin.grant_roles_to_role(child, roles=[parent])) await result(admin.grant_privileges(parent, privileges=[update])) await result(admin.revoke_privileges(parent, privileges=[update])) @@ -111,10 +131,8 @@ async def run_suite(raw: dict, sdk: ConnectionConfig, *, managed: bool, asynchro any(item.builtin for item in await result(admin.list_roles(db=data, include_builtin=True))), 'built-in roles', ) - await result(admin.create_user(user, password=password)) - cleanup.append((admin.drop_user, user)) - await result(admin.create_user(twin, password=token_urlsafe(24))) - cleanup.append((admin.drop_user, twin)) + await create_fixture(admin, user, cleanup, password=password) + await create_fixture(admin, twin, cleanup, password=token_urlsafe(24)) try: await result(admin.create_user(user, password=replacement)) except DriverError: @@ -161,12 +179,10 @@ async def run_suite(raw: dict, sdk: ConnectionConfig, *, managed: bool, asynchro not (await result(admin.get_role(child))).roles and not (await result(admin.get_user(user))).roles, 'dropped role references', ) - await result( - admin.create_role(audit, privileges=[MongoPrivilege({'db': data, 'collection': ''}, ['viewUser'])]) + await create_fixture( + admin, audit, cleanup, privileges=[MongoPrivilege({'db': data, 'collection': ''}, ['viewUser'])] ) - cleanup.append((admin.drop_role, audit)) - await result(admin.create_user(reader, password=viewer_secret, roles=[audit])) - cleanup.append((admin.drop_user, reader)) + await create_fixture(admin, reader, cleanup, password=viewer_secret, roles=[audit]) async with owned(login(reader, viewer_secret)) as viewer_client: viewer = factory.mongodb(viewer_client) await result(viewer.list_users(auth_db=data)) @@ -206,6 +222,7 @@ async def run_suite(raw: dict, sdk: ConnectionConfig, *, managed: bool, asynchro failures.append(exc) if failures: if primary is None: + attach_cleanup_errors(failures[0], failures[1:]) raise failures[0] attach_cleanup_errors(primary, failures) diff --git a/tests/test_accounts_mongodb.py b/tests/test_accounts_mongodb.py index 98f0d08..6bf4482 100644 --- a/tests/test_accounts_mongodb.py +++ b/tests/test_accounts_mongodb.py @@ -190,3 +190,56 @@ class AsyncMongoTests(unittest.IsolatedAsyncioTestCase): with self.assertRaises(asyncio.CancelledError): await admin.drop_role(role) self.assertEqual(db.command.await_count, 3) + + +@unittest.skipUnless(HAS_DRIVER, 'requires PyMongo') +class MongoVerifierTests(unittest.IsolatedAsyncioTestCase): + async def test_uncertain_creation_is_reconciled_before_cleanup(self): + from scripts.verify_accounts_mongodb import create_fixture + + for applied in (True, False): + for identity, what in [(MongoUser('unique', 'analyst'), 'user'), (MongoRole('unique', 'reader'), 'role')]: + with self.subTest(applied=applied, what=what): + admin = AsyncMock() + create = getattr(admin, f'create_{what}') + get = getattr(admin, f'get_{what}') + remove = getattr(admin, f'drop_{what}') + error = AccountWriteOutcomeUnknown('lost response') + create.side_effect = error + get.side_effect = [None, object() if applied else None] + cleanup = [] + with self.assertRaises(AccountWriteOutcomeUnknown) as caught: + await create_fixture(admin, identity, cleanup) + self.assertIs(caught.exception, error) + self.assertEqual(cleanup, [(remove, identity)] if applied else []) + + async def test_preexisting_or_conflicting_target_never_enters_cleanup(self): + from scripts.verify_accounts_mongodb import create_fixture + + admin = AsyncMock() + identity = MongoUser('unique', 'analyst') + cleanup = [] + admin.get_user.return_value = object() + with self.assertRaises(RuntimeError): + await create_fixture(admin, identity, cleanup) + admin.create_user.assert_not_awaited() + admin.get_user.return_value = None + admin.create_user.side_effect = PermissionDeniedError('definite rejection') + with self.assertRaises(PermissionDeniedError): + await create_fixture(admin, identity, cleanup) + self.assertEqual(cleanup, []) + + async def test_readback_failure_retains_uncertain_creation(self): + from scripts.verify_accounts_mongodb import create_fixture + + admin = AsyncMock() + error = AccountWriteOutcomeUnknown('lost response') + read_error = DatabaseTimeoutError('disconnected') + admin.create_role.side_effect = error + admin.get_role.side_effect = [None, read_error] + cleanup = [] + with self.assertRaises(AccountWriteOutcomeUnknown) as caught: + await create_fixture(admin, MongoRole('unique', 'reader'), cleanup) + self.assertIs(caught.exception, error) + self.assertEqual(caught.exception.database_cleanup_errors, (read_error,)) + self.assertEqual(cleanup, []) -- Gitee From 19dca930216e6953994e1dc8fc419d8bd6aabd85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 16:44:53 +0800 Subject: [PATCH 81/86] =?UTF-8?q?feat(mysql):=20=E5=AE=9E=E7=8E=B0?= =?UTF-8?q?=E8=B4=A6=E5=8F=B7=E6=9D=83=E9=99=90=E7=AE=A1=E7=90=86=E5=92=8C?= =?UTF-8?q?=E9=9A=94=E7=A6=BB=E6=9C=8D=E5=8A=A1=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/verify_accounts_mysql.py | 263 ++++++++++++++++ src/database/accounts/_mysql.py | 414 +++++++++++++++++++++++++ src/database/accounts/_mysql_models.py | 197 ++++++++++++ tests/test_accounts_mysql.py | 159 ++++++++++ 4 files changed, 1033 insertions(+) create mode 100644 scripts/verify_accounts_mysql.py create mode 100644 src/database/accounts/_mysql.py create mode 100644 src/database/accounts/_mysql_models.py create mode 100644 tests/test_accounts_mysql.py diff --git a/scripts/verify_accounts_mysql.py b/scripts/verify_accounts_mysql.py new file mode 100644 index 0000000..43ba432 --- /dev/null +++ b/scripts/verify_accounts_mysql.py @@ -0,0 +1,263 @@ +"""Run MySQL account acceptance on an explicitly supplied disposable instance.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import secrets +import uuid +from contextlib import ExitStack +from contextlib import closing +from contextlib import contextmanager +from pathlib import Path + +import mysql.connector +from mysql.connector import aio +from mysql.connector import errors + +from database import ConnectionConfig +from database import Connections +from database.accounts import Accounts +from database.accounts import AccountWriteOutcomeUnknown +from database.accounts import AsyncAccounts +from database.accounts import MySQLAccount +from database.accounts import MySQLScope +from database.errors import ConfigurationError +from database.errors import DependencyError +from database.errors import PermissionDeniedError +from database.errors import attach_cleanup_errors + + +def statement(conn, sql: str, args: tuple = ()) -> list[tuple]: + with closing(conn.cursor()) as cur: + cur.execute(sql, args or None) + return cur.fetchall() if cur.with_rows else [] + + +def require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def denied(action, codes=(1044, 1045, 1142, 3118)): + try: + action() + except errors.Error as err: + if err.errno in codes: + return + raise + raise AssertionError('Expected server denial') + + +def credentials(cfg, user, password): + return {**cfg, 'user': user.user, 'password': password} + + +def managed(cfg): + values = dict(cfg) + host, port = values.pop('host'), values.pop('port') + user, password = values.pop('user'), values.pop('password') + return Connections( + { + 'admin': ConnectionConfig( + backend='mysql', host=host, port=port, username=user, password=password, options=values + ) + } + ) + + +@contextmanager +def track_create(stack, cfg, target, *, kind): + """Reconcile only a prechecked, unique identity in the caller's isolated instance.""" + + def exists(conn): + if kind == 'database': + return bool( + statement(conn, 'SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME=%s', (target,)) + ) + return Accounts.mysql(conn).get_account(target) is not None + + def release(exc_type, primary, traceback): + try: + with closing(mysql.connector.connect(**cfg)) as conn: + if exists(conn): + if kind == 'database': + statement(conn, f'DROP DATABASE `{target}`') + elif kind == 'role': + Accounts.mysql(conn).drop_role(target) + else: + Accounts.mysql(conn).drop_user(target) + except Exception as cleanup_error: + if primary is None: + raise + attach_cleanup_errors(primary, (cleanup_error,)) + + with closing(mysql.connector.connect(**cfg)) as check: + require(not exists(check), 'collision in isolated resource namespace') + try: + yield + except (AccountWriteOutcomeUnknown, asyncio.CancelledError, errors.OperationalError, errors.InterfaceError): + stack.push(release) + raise + else: + stack.push(release) + + +def verify_sync(cfg: dict) -> None: + token = 'qa' + uuid.uuid4().hex[:12] + a, b = token + '_a', token + 'Xa' + with closing(mysql.connector.connect(**cfg)) as root, ExitStack() as cleanup: + tool = Accounts.mysql(root) + user = MySQLAccount(token + "'u", '%') + sibling = MySQLAccount(user.user, '192.0.2.77') + role = MySQLAccount(token + 'r', '%') + child = MySQLAccount(token + 'c', '%') + password = secrets.token_urlsafe(20) + replacement = secrets.token_urlsafe(20) + with managed(cfg).create('admin') as handle: + require(Accounts.mysql(handle).get_account(user) is None, 'unexpected account collision') + percent, percent_other = token + '%schema', token + 'Zschema' + for db in (a, b, percent, percent_other): + with track_create(cleanup, cfg, db, kind='database'): + statement(root, f'CREATE DATABASE `{db}`') + statement(root, f'CREATE TABLE `{db}`.t (id INT)') + statement(root, f'CREATE PROCEDURE `{a}`.testproc() SELECT 1') + statement(root, f'CREATE FUNCTION `{a}`.testfn() RETURNS INTEGER DETERMINISTIC RETURN 1') + for account in (user, sibling): + with track_create(cleanup, cfg, account, kind='user'): + tool.create_user(account, password=password) + require(tool.get_account(user).account == user and tool.get_account(sibling).account == sibling, 'get identity') + require({user, sibling} <= {r.account for r in tool.list_accounts()}, 'list identity') + for r in (role, child): + with track_create(cleanup, cfg, r, kind='role'): + tool.create_role(r) + tool.grant_privileges(child, MySQLScope.database(a), ['SELECT']) + tool.grant_roles(role, roles=[child]) + tool.grant_roles(user, roles=[role]) + tool.set_default_roles(user, roles=[role]) + require(tool.get_default_roles(user) == (role,), 'default role') + require(any(b.role == role for b in tool.list_role_bindings(user)), 'role binding') + require(any('SELECT' in g for g in tool.show_grants(user, using_roles=[role]).statements), 'expanded grants') + require(not any('SELECT' in g for g in tool.show_grants(user).statements), 'direct grants') + for scope, priv in [ + (MySQLScope.columns(a, 't', ['id']), 'UPDATE'), + (MySQLScope.table(a, 't'), 'INSERT'), + (MySQLScope.procedure(a, 'testproc'), 'EXECUTE'), + (MySQLScope.function(a, 'testfn'), 'EXECUTE'), + (MySQLScope.global_(), 'CONNECTION_ADMIN'), + ]: + tool.grant_privileges(role, scope, [priv]) + tool.revoke_privileges(role, scope, [priv]) + with closing(mysql.connector.connect(**credentials(cfg, user, password))) as limited: + statement(limited, f'SELECT * FROM `{a}`.t') + denied(lambda: statement(limited, f'SELECT * FROM `{b}`.t')) + try: + Accounts.mysql(limited).list_accounts() + except PermissionDeniedError: + pass + else: + raise AssertionError('limited user accessed account catalog') + tool.grant_privileges(child, MySQLScope.database(percent), ['SELECT']) + with closing(mysql.connector.connect(**credentials(cfg, user, password))) as limited: + statement(limited, f'SELECT * FROM `{percent}`.t') + denied(lambda: statement(limited, f'SELECT * FROM `{percent_other}`.t')) + tool.revoke_privileges(child, MySQLScope.database(percent), ['SELECT']) + if statement(root, 'SELECT @@GLOBAL.partial_revokes')[0][0]: + tool.grant_privileges(sibling, MySQLScope.global_(), ['SELECT']) + tool.revoke_privileges(sibling, MySQLScope.database(a), ['SELECT']) + require(any(line.startswith('REVOKE') for line in tool.show_grants(sibling).statements), 'partial revoke') + tool.revoke_privileges(child, MySQLScope.database(a), ['SELECT']) + with closing(mysql.connector.connect(**credentials(cfg, user, password))) as limited: + denied(lambda: statement(limited, f'SELECT * FROM `{a}`.t')) + tool.revoke_roles(user, roles=[role]) + tool.grant_privileges(user, MySQLScope.database(a), ['INSERT']) + tool.grant_privileges(user, MySQLScope.database(a), ['SELECT'], grant_option=True) + with closing(mysql.connector.connect(**credentials(cfg, user, password))) as delegated: + Accounts.mysql(delegated).grant_privileges(sibling, MySQLScope.database(a), ['INSERT']) + require(any('INSERT' in line for line in tool.show_grants(sibling).statements), 'static grant option scope') + tool.lock_user(user) + require(tool.get_account(user).locked, 'lock state') + denied(lambda: mysql.connector.connect(**credentials(cfg, user, password)).close()) + tool.unlock_user(user) + tool.change_password(user, password=replacement) + denied(lambda: mysql.connector.connect(**credentials(cfg, user, password)).close()) + with closing(mysql.connector.connect(**credentials(cfg, user, replacement))) as limited: + statement(limited, 'SELECT 1') + # An account write must never commit this row implicitly. + root.start_transaction() + statement(root, f'INSERT INTO `{a}`.t VALUES (1)') + try: + tool.drop_user(sibling) + except ConfigurationError: + pass + else: + raise AssertionError('account statement accepted active transaction') + root.rollback() + require(not statement(root, f'SELECT * FROM `{a}`.t'), 'transaction committed') + require(tool.get_account(sibling) is not None, 'sibling changed') + # Check the more demanding SQL mode. Some older connector versions cannot encode it. + statement(root, "SET SESSION sql_mode='STRICT_ALL_TABLES,NO_BACKSLASH_ESCAPES'") + try: + tool.change_password(user, password=replacement + "'\\x") + except DependencyError: + with closing(mysql.connector.connect(**credentials(cfg, user, replacement))) as limited: + statement(limited, 'SELECT 1') + else: + with closing(mysql.connector.connect(**credentials(cfg, user, replacement + "'\\x"))) as limited: + statement(limited, 'SELECT 1') + finally: + statement(root, 'SET SESSION sql_mode=DEFAULT') + with closing(mysql.connector.connect(**cfg)) as root: + require(Accounts.mysql(root).get_account(user) is None, 'account survived cleanup') + + +async def verify_async(sync_cfg: dict) -> None: + cfg = {k: v for k, v in sync_cfg.items() if k != 'use_pure'} + conn = await aio.connect(**cfg) + user = MySQLAccount('qa' + uuid.uuid4().hex[:14], '%') + role = MySQLAccount(user.user + 'r', '%') + pw = secrets.token_urlsafe(20) + try: + admin = AsyncAccounts.mysql(conn) + with ExitStack() as cleanup: + with track_create(cleanup, sync_cfg, user, kind='user'): + await admin.create_user(user, password=pw) + with track_create(cleanup, sync_cfg, role, kind='role'): + await admin.create_role(role) + async with await managed(cfg).create_async('admin') as handle: + bound = AsyncAccounts.mysql(handle) + await bound.grant_roles(user, roles=[role]) + require(any(b.role == role for b in await bound.list_role_bindings(user)), 'async grant') + await bound.revoke_roles(user, roles=[role]) + async with await aio.connect(**credentials(cfg, user, pw)) as limited: + async with await limited.cursor() as cur: + await cur.execute('SELECT 1') + require((await cur.fetchone())[0] == 1, 'async login') + await admin.drop_role(role) + await admin.drop_user(user) + require(await admin.get_account(user) is None, 'async deletion') + finally: + await conn.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--config', required=True, type=Path, help='JSON Connector/Python parameters for an isolated test server' + ) + args = parser.parse_args() + cfg = json.loads(args.config.read_text(encoding='utf-8-sig')) + for field in ('host', 'port', 'user', 'password'): + if field not in cfg: + parser.error(f'Missing required configuration key {field}') + cfg['autocommit'] = True + with closing(mysql.connector.connect(**cfg)) as conn: + print('MySQL', statement(conn, 'SELECT VERSION()')[0][0], 'Connector/Python', mysql.connector.__version__) + verify_sync(cfg) + asyncio.run(verify_async(cfg)) + print('MySQL account acceptance passed (sync, async, managed, native).') + + +if __name__ == '__main__': + main() diff --git a/src/database/accounts/_mysql.py b/src/database/accounts/_mysql.py new file mode 100644 index 0000000..fb960d2 --- /dev/null +++ b/src/database/accounts/_mysql.py @@ -0,0 +1,414 @@ +"""Account administration through borrowed MySQL Connector/Python connections.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from ..errors import ConfigurationError +from ..errors import DependencyError +from ..errors import attach_cleanup_errors +from ._base import Binding +from ._mysql_models import AccountInfo +from ._mysql_models import Grants +from ._mysql_models import MySQLAccount +from ._mysql_models import MySQLScope +from ._mysql_models import PrivilegeChange +from ._mysql_models import RoleBinding +from ._mysql_models import _tuple + +_ACCOUNTS = 'SELECT User, Host, account_locked FROM mysql.user' +_BINDINGS = ( + 'SELECT FROM_USER, FROM_HOST, TO_USER, TO_HOST, WITH_ADMIN_OPTION FROM mysql.role_edges ' + 'WHERE TO_USER = %s AND TO_HOST = %s ORDER BY FROM_USER, FROM_HOST' +) +_DEFAULTS = ( + 'SELECT DEFAULT_ROLE_USER, DEFAULT_ROLE_HOST FROM mysql.default_roles ' + 'WHERE USER = %s AND HOST = %s ORDER BY DEFAULT_ROLE_USER, DEFAULT_ROLE_HOST' +) +_ESCAPE_PROBE = "account'probe\\end" + + +def _account(value: MySQLAccount) -> MySQLAccount: + if not isinstance(value, MySQLAccount): + raise ConfigurationError('Expected MySQLAccount with explicit user and host.', phase='config') + return value + + +def _roles(values: Iterable[MySQLAccount], *, empty: bool = False) -> tuple[MySQLAccount, ...]: + result = tuple(_account(value) for value in _tuple(values, 'roles')) + if not result and not empty: + raise ConfigurationError('At least one role is required.', phase='config') + return result + + +def _role_params(roles: tuple[MySQLAccount, ...]) -> tuple[str, ...]: + return tuple(value for role in roles for value in role.params()) + + +def _markers(roles: tuple[MySQLAccount, ...]) -> str: + return ', '.join('%s@%s' for _ in roles) + + +def _password(value: str) -> str: + if not isinstance(value, str): + raise ConfigurationError('Expected password text.', phase='config') + return value + + +def _str(value: Any) -> str: + if isinstance(value, (bytes, bytearray, memoryview)): + return bytes(value).decode('utf-8') + return str(value) + + +def _decode_rows(rows: list[tuple], encoding: str) -> list[tuple]: + # Raw cursors and use_unicode=False expose result bytes in the negotiated charset. + return [ + tuple( + bytes(value).decode(encoding) if isinstance(value, (bytes, memoryview, bytearray)) else value + for value in row + ) + for row in rows + ] + + +def _info(row: tuple) -> AccountInfo: + user, host, locked = map(_str, row) + return AccountInfo(MySQLAccount(user, host), locked == 'Y') + + +def _needs_probe(params: tuple[str, ...]) -> bool: + return any(any(char in s for char in '\'\\\x00\r\n\x1a"') for s in params) + + +def _transaction_error() -> ConfigurationError: + return ConfigurationError( + 'Account changes require autocommit and no active transaction.', backend='mysql', phase='operation' + ) + + +def _escape_error() -> DependencyError: + return DependencyError( + 'The connector cannot safely bind values in this session SQL mode.', backend='mysql', phase='operation' + ) + + +def _check_registered(change: PrivilegeChange, rows: list[tuple]) -> None: + names = {_str(row[0]).upper() for row in rows} + if not change.dynamic <= names: + raise ConfigurationError('Unregistered MySQL global privilege.', backend='mysql', phase='config') + + +def _target(account: MySQLAccount | None) -> str | None: + return f'{account.user}@{account.host}' if account is not None else None + + +class MySQLAccounts: + """Manage users and roles on a borrowed synchronous MySQL connection.""" + + def __init__(self, binding: Binding) -> None: + self._binding = binding + + def _execute( + self, + operation: str, + account: MySQLAccount | None, + sql: str, + params: tuple[str, ...] = (), + *, + write: bool = False, + change: PrivilegeChange | None = None, + ) -> list[tuple]: + target = _target(account) + with self._binding.operation(operation, target): + native = self._binding.native + if write and (native.in_transaction or not native.autocommit or native.in_transaction): + raise _transaction_error() + cursor = native.cursor() + primary = None + try: + if _needs_probe(params): + try: + cursor.execute('SELECT %s', (_ESCAPE_PROBE,)) + except Exception as error: + if getattr(error, 'errno', None) == 1064: + raise _escape_error() from error + raise + probe = cursor.fetchone() + if probe is None or _str(probe[0]) != _ESCAPE_PROBE: + raise _escape_error() + if change is not None: + literal_wildcards = False + if change.scope.kind == 'database' and any(c in change.scope.db for c in '_%\\\\'): + try: + cursor.execute('SELECT @@GLOBAL.partial_revokes') + except Exception as error: + if getattr(error, 'errno', None) != 1193: + raise + else: + row = cursor.fetchone() + literal_wildcards = bool(int(row[0])) + if change.dynamic: + cursor.execute('SHOW PRIVILEGES') + _check_registered(change, cursor.fetchall()) + sql, params = change.statement(literal_wildcards, params) + with self._binding.operation(operation, target, write=write): + cursor.execute(sql, params or None) + return [] if write else _decode_rows(cursor.fetchall(), native.python_charset) + except BaseException as error: + primary = error + raise + finally: + try: + cursor.close() + except BaseException as cleanup: + if primary is None: + raise + attach_cleanup_errors(primary, (cleanup,)) + + def list_accounts(self) -> tuple[AccountInfo, ...]: + rows = self._execute('list_accounts', None, _ACCOUNTS + ' ORDER BY User, Host') + return tuple(_info(row) for row in rows) + + def get_account(self, account: MySQLAccount) -> AccountInfo | None: + rows = self._execute( + 'get_account', account, _ACCOUNTS + ' WHERE User = %s AND Host = %s', _account(account).params() + ) + return _info(rows[0]) if rows else None + + def show_grants(self, account: MySQLAccount, *, using_roles: Iterable[MySQLAccount] = ()) -> Grants: + roles = _roles(using_roles, empty=True) + sql = 'SHOW GRANTS FOR %s@%s' + (' USING ' + _markers(roles) if roles else '') + rows = self._execute('show_grants', account, sql, _account(account).params() + _role_params(roles)) + return Grants(account, roles, tuple(_str(row[0]) for row in rows)) + + def list_role_bindings(self, account: MySQLAccount) -> tuple[RoleBinding, ...]: + rows = self._execute('list_role_bindings', account, _BINDINGS, _account(account).params()) + return tuple( + RoleBinding(MySQLAccount(_str(u), _str(h)), MySQLAccount(_str(m), _str(s)), _str(flag) == 'Y') + for u, h, m, s, flag in rows + ) + + def get_default_roles(self, account: MySQLAccount) -> tuple[MySQLAccount, ...]: + rows = self._execute('get_default_roles', account, _DEFAULTS, _account(account).params()) + return tuple(MySQLAccount(_str(u), _str(h)) for u, h in rows) + + def create_user(self, account: MySQLAccount, *, password: str) -> None: + self._execute( + 'create_user', + account, + 'CREATE USER %s@%s IDENTIFIED BY %s', + _account(account).params() + (_password(password),), + write=True, + ) + + def drop_user(self, account: MySQLAccount) -> None: + self._execute('drop_user', account, 'DROP USER %s@%s', _account(account).params(), write=True) + + def change_password(self, account: MySQLAccount, *, password: str) -> None: + self._execute( + 'change_password', + account, + 'ALTER USER %s@%s IDENTIFIED BY %s', + _account(account).params() + (_password(password),), + write=True, + ) + + def lock_user(self, account: MySQLAccount) -> None: + self._execute('lock_user', account, 'ALTER USER %s@%s ACCOUNT LOCK', _account(account).params(), write=True) + + def unlock_user(self, account: MySQLAccount) -> None: + self._execute('unlock_user', account, 'ALTER USER %s@%s ACCOUNT UNLOCK', _account(account).params(), write=True) + + def create_role(self, role: MySQLAccount) -> None: + self._execute('create_role', role, 'CREATE ROLE %s@%s', _account(role).params(), write=True) + + def drop_role(self, role: MySQLAccount) -> None: + self._execute('drop_role', role, 'DROP ROLE %s@%s', _account(role).params(), write=True) + + def grant_privileges( + self, account: MySQLAccount, scope: MySQLScope, privileges: Iterable[str], *, grant_option: bool = False + ) -> None: + change = PrivilegeChange.build(scope, privileges, True, grant_option) + self._execute('grant_privileges', account, '', _account(account).params(), write=True, change=change) + + def revoke_privileges(self, account: MySQLAccount, scope: MySQLScope, privileges: Iterable[str]) -> None: + change = PrivilegeChange.build(scope, privileges, False, False) + self._execute('revoke_privileges', account, '', _account(account).params(), write=True, change=change) + + def grant_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount], admin_option: bool = False) -> None: + if type(admin_option) is not bool: + raise ConfigurationError('Expected a Boolean admin option.', phase='config') + items = _roles(roles) + sql = 'GRANT ' + _markers(items) + ' TO %s@%s' + (' WITH ADMIN OPTION' if admin_option else '') + self._execute('grant_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + def revoke_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount]) -> None: + items = _roles(roles) + sql = 'REVOKE ' + _markers(items) + ' FROM %s@%s' + self._execute('revoke_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + def set_default_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount]) -> None: + items = _roles(roles, empty=True) + sql = 'SET DEFAULT ROLE ' + (_markers(items) if items else 'NONE') + ' TO %s@%s' + self._execute('set_default_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + +class AsyncMySQLAccounts: + """Manage users and roles on a borrowed asynchronous MySQL connection.""" + + def __init__(self, binding: Binding) -> None: + self._binding = binding + + async def _execute( + self, + operation: str, + account: MySQLAccount | None, + sql: str, + params: tuple[str, ...] = (), + *, + write: bool = False, + change: PrivilegeChange | None = None, + ) -> list[tuple]: + target = _target(account) + with self._binding.operation(operation, target): + native = self._binding.native + if write and (native.in_transaction or not (await native.get_autocommit()) or native.in_transaction): + raise _transaction_error() + cursor = await native.cursor() + primary = None + try: + if _needs_probe(params): + try: + await cursor.execute('SELECT %s', (_ESCAPE_PROBE,)) + except Exception as error: + if getattr(error, 'errno', None) == 1064: + raise _escape_error() from error + raise + probe = await cursor.fetchone() + if probe is None or _str(probe[0]) != _ESCAPE_PROBE: + raise _escape_error() + if change is not None: + literal_wildcards = False + if change.scope.kind == 'database' and any(c in change.scope.db for c in '_%\\\\'): + try: + await cursor.execute('SELECT @@GLOBAL.partial_revokes') + except Exception as error: + if getattr(error, 'errno', None) != 1193: + raise + else: + row = await cursor.fetchone() + literal_wildcards = bool(int(row[0])) + if change.dynamic: + await cursor.execute('SHOW PRIVILEGES') + _check_registered(change, await cursor.fetchall()) + sql, params = change.statement(literal_wildcards, params) + with self._binding.operation(operation, target, write=write): + await cursor.execute(sql, params or None) + return [] if write else _decode_rows(await cursor.fetchall(), native.python_charset) + except BaseException as error: + primary = error + raise + finally: + try: + await cursor.close() + except BaseException as cleanup: + if primary is None: + raise + attach_cleanup_errors(primary, (cleanup,)) + + async def list_accounts(self) -> tuple[AccountInfo, ...]: + rows = await self._execute('list_accounts', None, _ACCOUNTS + ' ORDER BY User, Host') + return tuple(_info(row) for row in rows) + + async def get_account(self, account: MySQLAccount) -> AccountInfo | None: + rows = await self._execute( + 'get_account', account, _ACCOUNTS + ' WHERE User = %s AND Host = %s', _account(account).params() + ) + return _info(rows[0]) if rows else None + + async def show_grants(self, account: MySQLAccount, *, using_roles: Iterable[MySQLAccount] = ()) -> Grants: + roles = _roles(using_roles, empty=True) + sql = 'SHOW GRANTS FOR %s@%s' + (' USING ' + _markers(roles) if roles else '') + rows = await self._execute('show_grants', account, sql, _account(account).params() + _role_params(roles)) + return Grants(account, roles, tuple(_str(row[0]) for row in rows)) + + async def list_role_bindings(self, account: MySQLAccount) -> tuple[RoleBinding, ...]: + rows = await self._execute('list_role_bindings', account, _BINDINGS, _account(account).params()) + return tuple( + RoleBinding(MySQLAccount(_str(u), _str(h)), MySQLAccount(_str(m), _str(s)), _str(flag) == 'Y') + for u, h, m, s, flag in rows + ) + + async def get_default_roles(self, account: MySQLAccount) -> tuple[MySQLAccount, ...]: + rows = await self._execute('get_default_roles', account, _DEFAULTS, _account(account).params()) + return tuple(MySQLAccount(_str(u), _str(h)) for u, h in rows) + + async def create_user(self, account: MySQLAccount, *, password: str) -> None: + await self._execute( + 'create_user', + account, + 'CREATE USER %s@%s IDENTIFIED BY %s', + _account(account).params() + (_password(password),), + write=True, + ) + + async def drop_user(self, account: MySQLAccount) -> None: + await self._execute('drop_user', account, 'DROP USER %s@%s', _account(account).params(), write=True) + + async def change_password(self, account: MySQLAccount, *, password: str) -> None: + await self._execute( + 'change_password', + account, + 'ALTER USER %s@%s IDENTIFIED BY %s', + _account(account).params() + (_password(password),), + write=True, + ) + + async def lock_user(self, account: MySQLAccount) -> None: + await self._execute( + 'lock_user', account, 'ALTER USER %s@%s ACCOUNT LOCK', _account(account).params(), write=True + ) + + async def unlock_user(self, account: MySQLAccount) -> None: + await self._execute( + 'unlock_user', account, 'ALTER USER %s@%s ACCOUNT UNLOCK', _account(account).params(), write=True + ) + + async def create_role(self, role: MySQLAccount) -> None: + await self._execute('create_role', role, 'CREATE ROLE %s@%s', _account(role).params(), write=True) + + async def drop_role(self, role: MySQLAccount) -> None: + await self._execute('drop_role', role, 'DROP ROLE %s@%s', _account(role).params(), write=True) + + async def grant_privileges( + self, account: MySQLAccount, scope: MySQLScope, privileges: Iterable[str], *, grant_option: bool = False + ) -> None: + change = PrivilegeChange.build(scope, privileges, True, grant_option) + await self._execute('grant_privileges', account, '', _account(account).params(), write=True, change=change) + + async def revoke_privileges(self, account: MySQLAccount, scope: MySQLScope, privileges: Iterable[str]) -> None: + change = PrivilegeChange.build(scope, privileges, False, False) + await self._execute('revoke_privileges', account, '', _account(account).params(), write=True, change=change) + + async def grant_roles( + self, account: MySQLAccount, *, roles: Iterable[MySQLAccount], admin_option: bool = False + ) -> None: + if type(admin_option) is not bool: + raise ConfigurationError('Expected a Boolean admin option.', phase='config') + items = _roles(roles) + sql = 'GRANT ' + _markers(items) + ' TO %s@%s' + (' WITH ADMIN OPTION' if admin_option else '') + await self._execute('grant_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + async def revoke_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount]) -> None: + items = _roles(roles) + sql = 'REVOKE ' + _markers(items) + ' FROM %s@%s' + await self._execute('revoke_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + async def set_default_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount]) -> None: + items = _roles(roles, empty=True) + sql = 'SET DEFAULT ROLE ' + (_markers(items) if items else 'NONE') + ' TO %s@%s' + await self._execute( + 'set_default_roles', account, sql, _role_params(items) + _account(account).params(), write=True + ) diff --git a/src/database/accounts/_mysql_models.py b/src/database/accounts/_mysql_models.py new file mode 100644 index 0000000..83c19fe --- /dev/null +++ b/src/database/accounts/_mysql_models.py @@ -0,0 +1,197 @@ +"""MySQL account identities, privilege scopes and server-independent SQL composition.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass + +from ..errors import ConfigurationError + + +def _text(value: str, name: str, *, empty: bool = False) -> str: + if not isinstance(value, str) or (not empty and not value) or '\x00' in value: + raise ConfigurationError(f'Invalid {name}.', backend='mysql', phase='config') + return value + + +@dataclass(frozen=True) +class MySQLAccount: + user: str + host: str + + def __post_init__(self) -> None: + _text(self.user, 'user name', empty=True) + _text(self.host, 'account host', empty=True) + + def params(self) -> tuple[str, str]: + return self.user, self.host + + +def _tuple(items: Iterable, name: str) -> tuple: + if isinstance(items, (bytes, str)) or not isinstance(items, Iterable): + raise ConfigurationError(f'Expected an iterable of {name}.', phase='config') + return tuple(items) + + +@dataclass(frozen=True) +class MySQLScope: + kind: str + db: str = '' + name: str = '' + column_names: tuple[str, ...] = () + + def __post_init__(self) -> None: + columns = _tuple(self.column_names, 'column names') + object.__setattr__(self, 'column_names', columns) + if self.kind not in {'global', 'database', 'table', 'columns', 'procedure', 'function'}: + raise ConfigurationError('Unknown MySQL privilege scope.', phase='config') + if self.kind != 'global': + _text(self.db, 'database name') + if self.kind not in ('global', 'database'): + _text(self.name, 'object name') + if ( + (self.kind == 'global' and self.db) + or (self.kind in ('global', 'database') and self.name) + or bool(columns) != (self.kind == 'columns') + ): + raise ConfigurationError('Invalid fields for this MySQL privilege scope.', phase='config') + for column in columns: + _text(column, 'column name') + + @classmethod + def global_(cls) -> MySQLScope: + return cls('global') + + @classmethod + def database(cls, name: str) -> MySQLScope: + return cls('database', name) + + @classmethod + def table(cls, db: str, name: str) -> MySQLScope: + return cls('table', db, name) + + @classmethod + def columns(cls, db: str, table: str, names: Iterable[str]) -> MySQLScope: + return cls('columns', db, table, _tuple(names, 'column names')) + + @classmethod + def procedure(cls, db: str, name: str) -> MySQLScope: + return cls('procedure', db, name) + + @classmethod + def function(cls, db: str, name: str) -> MySQLScope: + return cls('function', db, name) + + +@dataclass(frozen=True) +class AccountInfo: + account: MySQLAccount + locked: bool + + +@dataclass(frozen=True) +class Grants: + account: MySQLAccount + using_roles: tuple[MySQLAccount, ...] + statements: tuple[str, ...] + + +@dataclass(frozen=True) +class RoleBinding: + role: MySQLAccount + member: MySQLAccount + admin_option: bool + + +_COLUMN = {'SELECT', 'INSERT', 'UPDATE', 'REFERENCES'} +_TABLE = _COLUMN | {'ALTER', 'CREATE', 'CREATE VIEW', 'DELETE', 'DROP', 'INDEX', 'SHOW VIEW', 'TRIGGER'} +_ROUTINE = {'ALTER ROUTINE', 'EXECUTE'} +_DATABASE = _TABLE | _ROUTINE | {'CREATE ROUTINE', 'CREATE TEMPORARY TABLES', 'EVENT', 'LOCK TABLES'} +_GLOBAL = _DATABASE | { + 'CREATE ROLE', + 'CREATE TABLESPACE', + 'CREATE USER', + 'DROP ROLE', + 'FILE', + 'PROCESS', + 'RELOAD', + 'REPLICATION CLIENT', + 'REPLICATION SLAVE', + 'SHOW DATABASES', + 'SHUTDOWN', + 'SUPER', + 'USAGE', +} +_SCOPES = { + 'global': _GLOBAL, + 'database': _DATABASE, + 'table': _TABLE, + 'columns': _COLUMN, + 'procedure': _ROUTINE, + 'function': _ROUTINE, +} + + +def identifiers(name: str) -> str: + return '`' + name.replace('`', '``') + '`' + + +def scope_sql(scope: MySQLScope, literal_wildcards: bool) -> str: + if scope.kind == 'global': + return '*.*' + db = scope.db + if scope.kind == 'database': + if not literal_wildcards: + db = db.replace('\\', '\\\\').replace('_', '\\_').replace('%', '\\%') + return identifiers(db) + '.*' + prefix = scope.kind.upper() + ' ' if scope.kind in ('procedure', 'function') else '' + return prefix + identifiers(db) + '.' + identifiers(scope.name) + + +@dataclass(frozen=True) +class PrivilegeChange: + scope: MySQLScope + privileges: tuple[str, ...] + grant: bool + grant_option: bool = False + + @classmethod + def build(cls, scope: MySQLScope, privileges: Iterable[str], grant: bool, grant_option: bool) -> PrivilegeChange: + if not isinstance(scope, MySQLScope) or type(grant_option) is not bool: + raise ConfigurationError('Expected a typed scope and Boolean grant option.', phase='config') + values = tuple(_text(p, 'privilege name').upper() for p in _tuple(privileges, 'privileges')) + if not values: + raise ConfigurationError('At least one privilege is required.', phase='config') + for name in values: + if name in _SCOPES[scope.kind]: + continue + if scope.kind != 'global' or not re.fullmatch('[A-Z][A-Z0-9_]*', name) or name in {'ALL', 'PROXY', 'GRANT'}: + raise ConfigurationError('Invalid privilege for the selected scope.', phase='config') + return cls(scope, values, grant, grant_option) + + @property + def dynamic(self) -> set[str]: + return set(self.privileges) - _GLOBAL + + def statement( + self, literal_wildcards: bool, params: tuple[str, ...] + ) -> tuple[str, tuple[str, ...] | dict[str, str]]: + names = self.privileges + if self.scope.kind == 'columns': + cols = ', '.join(identifiers(c) for c in self.scope.column_names) + names = tuple(f'{name} ({cols})' for name in names) + target = scope_sql(self.scope, literal_wildcards) + verb, preposition = ('GRANT', 'TO') if self.grant else ('REVOKE', 'FROM') + suffix = ' WITH GRANT OPTION' if self.grant_option else '' + prefix = f'{verb} {", ".join(names)} ON {target} {preposition} ' + if '%s' not in prefix: + return prefix + '%s@%s' + suffix, params + # Connector/Python scans quoted identifiers for parameter markers too. + # Switch placeholder styles when the object name contains the positional marker. + if '%(' in prefix: + raise ConfigurationError('Identifier combines MySQL placeholder syntaxes.', phase='config') + user_key, host_key = 'account_user', 'account_host' + while any(f'%({host_key})s' in value for value in params): + host_key += '_' + return prefix + f'%({user_key})s@%({host_key})s' + suffix, dict(zip((user_key, host_key), params)) diff --git a/tests/test_accounts_mysql.py b/tests/test_accounts_mysql.py new file mode 100644 index 0000000..f1cd79f --- /dev/null +++ b/tests/test_accounts_mysql.py @@ -0,0 +1,159 @@ +"""Offline checks for MySQL account SQL, resource boundaries and fault classification.""" + +import asyncio +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +from database.accounts import AccountWriteOutcomeUnknown +from database.accounts import MySQLAccount +from database.accounts import MySQLScope +from database.accounts._base import Binding +from database.accounts._mysql import AsyncMySQLAccounts +from database.accounts._mysql import MySQLAccounts +from database.accounts._mysql_models import PrivilegeChange +from database.connections import AsyncConnection +from database.connections import Connection +from database.errors import ConfigurationError +from database.errors import DatabaseConnectionError +from database.errors import DependencyError + + +class MySQLAccountTests(unittest.TestCase): + def setUp(self): + self.user = MySQLAccount('user', 'localhost') + self.role = MySQLAccount('reader', '%') + self.native = MagicMock(in_transaction=False, autocommit=True, python_charset='utf8') + self.cur = self.native.cursor.return_value + bound = Binding( + Connection(self.native, SimpleNamespace(classify=lambda _: None), 'test', 'mysql', 'single'), + 'mysql', + is_async=False, + ) + self.admin = MySQLAccounts(bound) + + def test_exact_database_scopes(self): + change = PrivilegeChange.build(MySQLScope.database('team_a%'), ['select'], True, False) + stmt, params = change.statement(False, self.user.params()) + self.assertIn(r'`team\_a\%`.*', stmt) + self.assertIn('`team_a%`.*', change.statement(True, self.user.params())[0]) + self.assertEqual(params, ('user', 'localhost')) + change = PrivilegeChange.build(MySQLScope.table('db', 'report%s'), ['select'], True, False) + stmt, params = change.statement(False, self.user.params()) + self.assertIn('`report%s`', stmt) + self.assertEqual(params, {'account_user': 'user', 'account_host': 'localhost'}) + + def test_scope_validation(self): + for scope, privilege in [ + (MySQLScope.columns('db', 't', ['a']), 'DELETE'), + (MySQLScope.function('db', 'f'), 'DROP'), + (MySQLScope.table('db', 't'), 'CREATE USER'), + (MySQLScope.database('db'), 'FILE'), + ]: + with self.subTest(scope=scope), self.assertRaises(ConfigurationError): + self.admin.grant_privileges(self.user, scope, [privilege]) + self.native.cursor.assert_not_called() + with self.assertRaises(ConfigurationError): + MySQLScope.columns('db', 't', 'abc') + with self.assertRaises(ConfigurationError): + self.admin.grant_roles(self.user, roles=[]) + with self.assertRaises(ConfigurationError): + self.admin.grant_roles(self.user, roles=[self.role], admin_option='false') + + def test_punctuation_and_columns(self): + self.admin.grant_privileges(self.user, MySQLScope.columns('d', 'a`b', ['id']), ['select', 'update']) + sql, params = self.cur.execute.call_args.args + self.assertEqual(sql, 'GRANT SELECT (`id`), UPDATE (`id`) ON `d`.`a``b` TO %s@%s') + self.assertEqual(params, self.user.params()) + + def test_check_registration_of_dynamic_global_only(self): + self.cur.fetchall.return_value = [('CONNECTION_ADMIN', 'Server Admin', '')] + self.admin.grant_privileges(self.user, MySQLScope.global_(), ['connection_admin']) + self.assertEqual(self.cur.execute.call_args_list[0].args, ('SHOW PRIVILEGES',)) + self.assertIn('CONNECTION_ADMIN ON *.*', self.cur.execute.call_args.args[0]) + self.cur.execute.reset_mock() + with self.assertRaises(ConfigurationError): + self.admin.grant_privileges(self.user, MySQLScope.global_(), ['FAKE_ADMIN']) + self.assertEqual(self.cur.execute.call_count, 1) + + def test_explicit_empty_host_survives_catalog_projection(self): + self.cur.fetchall.return_value = [('analyst', '', 'N'), ('analyst', '%', 'N')] + identities = tuple(item.account for item in self.admin.list_accounts()) + self.assertEqual(identities, (MySQLAccount('analyst', ''), MySQLAccount('analyst', '%'))) + self.admin.get_account(identities[0]) + self.assertEqual(self.cur.execute.call_args.args[1], ('analyst', '')) + + def test_grant_context_and_role_membership(self): + self.cur.fetchall.return_value = [('GRANT ...',), ('REVOKE ...',)] + grants = self.admin.show_grants(self.user, using_roles=[self.role]) + self.assertEqual(grants.using_roles, (self.role,)) + self.assertEqual(grants.statements, ('GRANT ...', 'REVOKE ...')) + self.assertEqual( + self.cur.execute.call_args.args, + ('SHOW GRANTS FOR %s@%s USING %s@%s', self.user.params() + self.role.params()), + ) + self.admin.grant_roles(self.user, roles=[self.role], admin_option=True) + self.assertIn('WITH ADMIN OPTION', self.cur.execute.call_args.args[0]) + self.admin.set_default_roles(self.user, roles=[]) + self.assertIn('SET DEFAULT ROLE NONE', self.cur.execute.call_args.args[0]) + + def test_guard_leaves_transaction_to_caller(self): + for active, auto in [(True, True), (False, False)]: + self.native.in_transaction, self.native.autocommit = active, auto + with self.assertRaises(ConfigurationError): + self.admin.create_user(self.user, password='secret') + self.native.cursor.assert_not_called() + self.native.rollback.assert_not_called() + + def test_probe_failure_blocks_account_write(self): + self.cur.fetchone.return_value = ('wrong',) + with self.assertRaises(DependencyError): + self.admin.create_user(self.user, password="val'ue") + self.assertEqual(self.cur.execute.call_count, 1) + self.assertNotIn("val'ue", repr(self.cur.execute.call_args)) + self.cur.close.assert_called_once() + + def test_decodes_raw_non_utf8_metadata(self): + self.native.python_charset = 'latin1' + self.cur.fetchall.return_value = [(b'caf\xe9', b'%', b'N')] + self.assertEqual(self.admin.list_accounts()[0].account, MySQLAccount('caf\u00e9', '%')) + + def test_results_and_missing_account(self): + self.cur.fetchall.return_value = [(b'user', b'localhost', b'Y')] + info = self.admin.get_account(self.user) + self.assertEqual((info.account, info.locked), (self.user, True)) + self.cur.fetchall.return_value = [] + self.assertIsNone(self.admin.get_account(self.user)) + + def test_write_and_read_failures_preserve_stage_and_cause(self): + err = ConnectionResetError('sensitive') + self.cur.execute.side_effect = err + with self.assertRaises(AccountWriteOutcomeUnknown) as raised: + self.admin.change_password(self.user, password='private') + self.assertIs(raised.exception.original, err) + self.assertNotIn('sensitive', str(raised.exception)) + self.cur.close.assert_called_once() + with self.assertRaises(DatabaseConnectionError): + self.admin.show_grants(self.user) + self.cur.close.side_effect = RuntimeError('cursor') + with self.assertRaises(AccountWriteOutcomeUnknown) as dual: + self.admin.drop_user(self.user) + self.assertEqual(len(dual.exception.database_cleanup_errors), 1) + + +class AsyncMySQLTests(unittest.IsolatedAsyncioTestCase): + async def test_cancellation_and_cursor_cleanup(self): + cursor = AsyncMock() + cursor.execute.side_effect = asyncio.CancelledError() + native = SimpleNamespace( + cursor=AsyncMock(return_value=cursor), in_transaction=False, get_autocommit=AsyncMock(return_value=True) + ) + conn = AsyncConnection( + native, SimpleNamespace(classify=lambda _: None), 'qa', 'mysql', 'single', asyncio.get_running_loop() + ) + admin = AsyncMySQLAccounts(Binding(conn, 'mysql', is_async=True)) + with self.assertRaises(asyncio.CancelledError): + await admin.create_role(MySQLAccount('qa', 'localhost')) + cursor.close.assert_awaited_once() + self.assertFalse(conn.closed) -- Gitee From d87d973cfd42e346ac8d6f0abcf3e919d41282cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 16:47:18 +0800 Subject: [PATCH 82/86] =?UTF-8?q?test(mysql):=20=E9=9A=94=E7=A6=BB?= =?UTF-8?q?=E9=83=A8=E5=88=86=E6=92=A4=E6=9D=83=E4=B8=8E=E8=BD=AC=E6=8E=88?= =?UTF-8?q?=E6=9D=83=E7=9A=84=E9=AA=8C=E6=94=B6=E4=B8=BB=E4=BD=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/verify_accounts_mysql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/verify_accounts_mysql.py b/scripts/verify_accounts_mysql.py index 43ba432..efc7842 100644 --- a/scripts/verify_accounts_mysql.py +++ b/scripts/verify_accounts_mysql.py @@ -174,8 +174,8 @@ def verify_sync(cfg: dict) -> None: tool.grant_privileges(user, MySQLScope.database(a), ['INSERT']) tool.grant_privileges(user, MySQLScope.database(a), ['SELECT'], grant_option=True) with closing(mysql.connector.connect(**credentials(cfg, user, password))) as delegated: - Accounts.mysql(delegated).grant_privileges(sibling, MySQLScope.database(a), ['INSERT']) - require(any('INSERT' in line for line in tool.show_grants(sibling).statements), 'static grant option scope') + Accounts.mysql(delegated).grant_privileges(child, MySQLScope.database(a), ['INSERT']) + require(any('INSERT' in line for line in tool.show_grants(child).statements), 'static grant option scope') tool.lock_user(user) require(tool.get_account(user).locked, 'lock state') denied(lambda: mysql.connector.connect(**credentials(cfg, user, password)).close()) -- Gitee From 0c0040fc1c7578eba717b2a3df42ab58068eb778 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 16:49:10 +0800 Subject: [PATCH 83/86] =?UTF-8?q?fix(accounts):=20=E6=A0=A1=E9=AA=8C=20Red?= =?UTF-8?q?is=20=E7=AE=A1=E7=BA=BF=E5=B9=B6=E4=BF=9D=E7=95=99=E6=B8=85?= =?UTF-8?q?=E7=90=86=E5=BC=82=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/database/accounts/_base.py | 12 +++++++++--- tests/test_accounts_binding.py | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/src/database/accounts/_base.py b/src/database/accounts/_base.py index a51ce6c..a7665c0 100644 --- a/src/database/accounts/_base.py +++ b/src/database/accounts/_base.py @@ -45,8 +45,11 @@ def _matches_native(client: object, backend: str, is_async: bool) -> bool: def _fixed_redis_node(client: Any, is_async: bool) -> bool: - module = import_module('redis.asyncio.sentinel' if is_async else 'redis.sentinel') - return not isinstance(client.connection_pool, module.SentinelConnectionPool) + prefix = 'redis.asyncio' if is_async else 'redis' + pipeline = import_module(f'{prefix}.client').Pipeline + sentinel_pool = import_module(f'{prefix}.sentinel').SentinelConnectionPool + # Pipelines subclass Redis but queue commands instead of returning an acknowledgement. + return not isinstance(client, pipeline) and not isinstance(client.connection_pool, sentinel_pool) class Binding: @@ -114,5 +117,8 @@ class Binding: converted = AccountWriteOutcomeUnknown( 'The database did not confirm the account change.', **context, original=original ) - attach_cleanup_errors(converted, getattr(original, 'database_cleanup_errors', ())) + attributes = BaseException.__dict__['__dict__'].__get__(original) + cleanup_errors = attributes.get('database_cleanup_errors', ()) + if isinstance(cleanup_errors, tuple): + attach_cleanup_errors(converted, cleanup_errors) raise converted from original diff --git a/tests/test_accounts_binding.py b/tests/test_accounts_binding.py index d16669b..334728e 100644 --- a/tests/test_accounts_binding.py +++ b/tests/test_accounts_binding.py @@ -21,7 +21,7 @@ class BindingTests(unittest.TestCase): def test_import_without_drivers(self): source = str(Path(__file__).resolve().parents[1] / 'src') code = ( - 'import sys; from database.accounts import Accounts, AsyncAccounts; ' + 'import sys; from database.accounts import *; ' "assert not {'pymongo', 'redis', 'mysql.connector'}.intersection(sys.modules)" ) subprocess.run([sys.executable, '-B', '-S', '-c', code], env={**os.environ, 'PYTHONPATH': source}, check=True) @@ -46,14 +46,45 @@ class BindingTests(unittest.TestCase): for write, error_type in [(False, DatabaseConnectionError), (True, AccountWriteOutcomeUnknown)]: with self.subTest(write=write): original = ConnectionResetError('private payload') + cleanup_error = RuntimeError('cursor close failed') + original.database_cleanup_errors = (cleanup_error,) with self.assertRaises(error_type) as caught: with bound.operation('grant_roles', 'target', write=write): raise original self.assertIs(caught.exception.original, original) self.assertIs(caught.exception.__cause__, original) self.assertEqual(caught.exception.target, 'target') + self.assertEqual(caught.exception.database_cleanup_errors, (cleanup_error,)) self.assertNotIn('private payload', str(caught.exception)) + def test_native_binding_checks_mode_and_sentinel_pools(self): + import redis + from redis.asyncio.sentinel import Sentinel as AsyncSentinel + from redis.sentinel import Sentinel + + for client, is_async in ((redis.Redis(), False), (redis.asyncio.Redis(), True)): + with self.subTest(is_async=is_async): + self.assertIs(Binding(client, 'redis', is_async=is_async).native, client) + with self.assertRaises(ConfigurationError): + Binding(client, 'redis', is_async=not is_async) + with self.assertRaises(ConfigurationError): + Binding(client.pipeline(), 'redis', is_async=is_async) + for cls, is_async in ((Sentinel, False), (AsyncSentinel, True)): + primary = cls([('127.0.0.1', 26379)]).master_for('test') + handle = Connection(primary, SimpleNamespace(classify=lambda exc: None), 'r', 'redis', 'single') + with self.assertRaises(ConfigurationError): + Binding(primary, 'redis', is_async=is_async) + if not is_async: + with self.assertRaises(ConfigurationError): + Binding(handle, 'redis', is_async=False) + + def test_cancelled_write_is_propagated(self): + handle = Connection(object(), SimpleNamespace(classify=lambda exc: None), 'test', 'mysql', 'single') + bound = Binding(handle, 'mysql', is_async=False) + with self.assertRaises(asyncio.CancelledError): + with bound.operation('write', write=True): + raise asyncio.CancelledError() + def test_foreign_loop(self): async def make_binding(): handle = AsyncConnection( -- Gitee From 043cfe306e79219625c3583f06aa847fd5d38755 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 16:51:40 +0800 Subject: [PATCH 84/86] =?UTF-8?q?docs(accounts):=20=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E4=BD=BF=E7=94=A8=E7=A4=BA=E4=BE=8B=E4=B8=8E=E4=B8=89=E7=A7=8D?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=BA=93=E9=AA=8C=E6=94=B6=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 1 + README.md | 22 ++++++++++ docs/README.md | 5 +++ docs/account-management-design.md | 43 ++++++++++++++++++-- docs/account-management-requirements.md | 4 +- docs/account-management-validation.md | 53 +++++++++++++++++++++++++ docs/crud-design.md | 2 +- tests/README.md | 4 ++ 8 files changed, 127 insertions(+), 7 deletions(-) create mode 100644 docs/account-management-validation.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 31247e2..cc0d45e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ ### 新增 +- 新增 `database.accounts` 同步与异步 Python 工具,支持自管 MongoDB 用户及角色、MySQL 账号及对象授权和固定 Redis 数据节点的 ACL 管理。 - 增加 MongoDB 同步与异步分页读取和流式续读工具,按 `_id` 索引定位,可在游标失效或连接中断后有界恢复。 - 增加 MongoDB 同步与异步混合批写、待写缓冲丢弃和自动关闭游标的分批读取工具。异步批处理支持消费异步可迭代对象。 - 新增同步/异步 Schema 基础管理:MySQL 库表创建、兼容校验、目录查看和删除;MongoDB 集合准备、必要身份约束及目录管理。初始化显式执行,不自动迁移。 diff --git a/README.md b/README.md index edb119a..bb88b76 100644 --- a/README.md +++ b/README.md @@ -360,6 +360,28 @@ with connections.create('orders') as connection: 完整接口、同步/异步示例、Redis 格式与 TTL、原始错误及事务边界见 [高级记录操作使用说明](docs/crud-usage.md)。本功能的实际验证范围见 [CRUD 实施与验收记录](docs/crud-implementation.md)。 +## 数据库账号与权限管理 + +`database.accounts` 借用现有管理连接,支持本库句柄或原生 PyMongo、Connector/Python、redis-py 客户端。MongoDB 用认证库和用户名定位账号,MySQL 用用户名和来源主机定位账号,Redis ACL 作用于固定的数据节点。下例中的连接由调用方提供。 + +```python +from database.accounts import Accounts, AsyncAccounts, MongoRole, MongoUser, MySQLAccount + +with connections.create("mongo_admin") as conn: + mongo = Accounts.mongodb(conn) + user = MongoUser(auth_db="staff", name="analyst") + mongo.grant_roles(user, roles=[MongoRole(db="reports", name="read")]) + details = mongo.get_user(user, expand_privileges=True) + +mysql = Accounts.mysql(native_mysql) +grants = mysql.show_grants(MySQLAccount(user="analyst", host="10.0.0.8")) + +async def inspect_redis(native_async_redis): + return await AsyncAccounts.redis(native_async_redis).get_user("analyst") +``` + +MySQL 账号写操作使用启用自动提交且没有活动事务的连接。Redis 的 `apply_user` 对同名用户按顺序应用规则,`save_acl` 单独确认当前节点的 ACL 文件写入。账号写入遇到 `AccountWriteOutcomeUnknown` 时,请先回读目标状态,改密可通过独立认证核对。各方法所需权限、使用示例与已验证的环境见[需求清单](docs/account-management-requirements.md)、[实现设计](docs/account-management-design.md)和[验收记录](docs/account-management-validation.md)。 + ## 文档与贡献 - [文档索引](docs/README.md) 汇总连接管理设计、验收记录和源码目录说明。 diff --git a/docs/README.md b/docs/README.md index 3966a65..7a7b3eb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,11 @@ - [测试指南](../tests/README.md) 提供离线回归和隔离服务探测的运行方式。 - [连接管理验收记录](connection-management-validation.md) 记录交付时实际验证的驱动版本、真实拓扑及待验证项。 +## 账号与权限管理 + +- [需求与权限清单](account-management-requirements.md) 说明三个后端的身份、授权范围和操作者所需权限。 +- [实现设计与调用示例](account-management-design.md) 给出方法映射、异常契约和验证步骤。实际覆盖见[验收记录](account-management-validation.md)。 + ## CRUD 开发设计 - [基础结构与记录操作使用说明](crud-usage.md) 提供初始化、结构管理、批量操作、字段投影、同步/异步及 Redis Hash、String 的使用边界。 diff --git a/docs/account-management-design.md b/docs/account-management-design.md index 46b9358..a1e5f03 100644 --- a/docs/account-management-design.md +++ b/docs/account-management-design.md @@ -1,6 +1,6 @@ # MongoDB、MySQL、Redis 账号与权限管理工具实现设计 -本文是 `database.accounts` 首版的实施规划,供实现、审查与验收使用。下列名称、签名和示例均为拟新增的 Python 能力。需求编号 R1–R9、使用场景和操作权限表见[需求文档](account-management-requirements.md)。既有连接与异常实现见[连接生命周期](../src/database/connections.py)和[异常转换](../src/database/errors.py)。 +本文是 `database.accounts` 首版的实施规划,供实现、审查与验收使用。已执行的版本组合及测试结果见[验收记录](account-management-validation.md)。需求编号 R1–R9、使用场景和操作权限表见[需求文档](account-management-requirements.md)。既有连接与异常实现见[连接生命周期](../src/database/connections.py)和[异常转换](../src/database/errors.py)。 ## 1. 组件与执行流程 @@ -20,7 +20,7 @@ ## 2. 公共输入、返回与错误 -| 模型 | 拟提供的字段或行为 | +| 模型 | 字段或行为 | | --- | --- | | `MongoUser(auth_db, name)`、`MongoRole(db, name)` | 两个字段共同确定操作目标,命令中明确指定角色定义库 | | `MongoPrivilege(resource, actions)` | BSON 资源描述和动作集合。保留空库名、空集合名等有原生含义的作用域;构造时复制调用方容器 | @@ -42,7 +42,7 @@ | 异步任务取消 | 传播 `CancelledError`。取消可能发生在写入发送之后 | 异步调用者可按同样流程核对状态 | | 多步操作只完成前几步 | 已返回成功的步骤保持各自的确认状态,失败步骤单独报告 | 例如 MySQL 开户成功后只补授权,Redis 规则已应用而保存失败时只核对运行时状态与写盘条件 | -SDK 不为账号写入增加自动重放。借入原生客户端时,驱动自身的重试配置和行为要写入使用说明,测试按驱动版本记录实际发送次数。读请求失败继续按读取错误处理。只有明确处于写入阶段并存在执行歧义时才转换为结果未知。异常摘要使用操作名和安全目标字段,避免将 SQL、BSON、ACL 规则或原始响应插入日志。保留的异常链仅供受控诊断。 [MongoDB 写关注](https://www.mongodb.com/docs/manual/reference/write-concern/) · [仓库安全约定](../SECURITY.md) +SDK 不为账号写入增加自动重放。原生客户端沿用调用方的驱动重试设置,例如 redis-py 的 `retry` 决定哪些连接故障可由驱动自行重试。故障注入测试核对 SDK 对一次操作调用驱动的次数。读请求失败继续按读取错误处理。只有明确处于写入阶段并存在执行歧义时才转换为结果未知。异常摘要使用操作名和安全目标字段,避免将 SQL、BSON、ACL 规则或原始响应插入日志。保留的异常链仅供受控诊断。 [MongoDB 写关注](https://www.mongodb.com/docs/manual/reference/write-concern/) · [仓库安全约定](../SECURITY.md) ## 3. 后端方法与原生命令 @@ -80,6 +80,8 @@ SDK 不为账号写入增加自动重放。借入原生客户端时,驱动自 **精确库名。** 目标为 `database("team_a")` 时,根据服务器版本和 `partial_revokes` 对数据库级授权模式的解释,编码能逐字匹配的库名,再构造 `ON ... .*`。对不支持该变量的早期 8.0 按对应版本规则处理。若无法确定精确语义,在发送写入前报告原因。授权与撤权必须采用相同的匹配规则。真实服务验收同时创建 `team_a` 和 `teamXa`,用受限账号分别验证;另测含 `%` 的库名和相关服务器设置。 [数据库级授权规则](https://dev.mysql.com/doc/refman/8.4/en/grant.html) +同一条授权 SQL 的对象范围若同时含有 `%s` 和 `%(` 两种驱动参数标记语法,工具在发送前报告 `ConfigurationError`。若驱动在当前 SQL 模式下无法准确绑定含引号等特殊字符的值,工具会先执行无业务数据的探测并报告 `DependencyError`。 + **转授范围。** `grant_option=False` 仅省略本次语句的转授子句。静态权限在同一授权层级共享相应转授能力,例如对某库的 `SELECT` 增加转授时,同库同层级已有的 `INSERT` 也可能被转授。全局动态权限逐项计算。验收时记录目标账号操作前后完整的 `SHOW GRANTS`,同时用第二个临时主体验证实际转授范围。 [GRANT OPTION](https://dev.mysql.com/doc/refman/8.4/en/grant.html) **事务与结果。** 在同步、异步写入路径复用现有 DDL 守卫的原则,要求 `autocommit=True` 且当前没有活动事务;检查失败时把连接交还调用方处理。每个游标由创建方关闭,清理失败不得覆盖已捕获的主要异常。管理连接上依次调用创建账号、分配角色、设置默认角色,分别保存确认结果。 [隐式提交](https://dev.mysql.com/doc/refman/8.4/en/implicit-commit.html) · [现有 MySQL 结构管理实现](../src/database/schema/_mysql.py) @@ -103,7 +105,7 @@ SDK 不为账号写入增加自动重放。借入原生客户端时,驱动自 ## 4. 开发者调用示例 -以下示例展示拟议签名。`connections` 为调用方建立的 `Connections`,原生客户端和密码参数由调用方显式提供。管理对象在持有连接的作用域内使用。 +以下示例通过 Python 直接调用管理方法。`connections` 为调用方建立的 `Connections`,原生客户端和密码参数由调用方显式提供。管理对象在持有连接的作用域内使用。 ```python from database.accounts import Accounts, AclRule, MongoRole, MongoUser, MySQLAccount @@ -150,6 +152,39 @@ async def inspect_async(connections, native_mysql, native_redis): return users, grants, acl ``` +### 从开户到回收 + +下例使用三个调用方提供的管理连接,分别代表三种独立部署。密码由调用方在运行时传入。示例中的身份、角色名和键前缀应替换为当前隔离环境的专用目标。 + +```python +from database.accounts import Accounts, AclRule, MongoPrivilege, MongoRole, MongoUser, MySQLAccount, MySQLScope + +def grant_report_access(mongo_client, mysql_conn, redis_client, mongo_pwd, mysql_pwd, redis_pwd): + mongo = Accounts.mongodb(mongo_client) + report_role = MongoRole("reports", "analyst_read") + mongo.create_role(report_role, privileges=[ + MongoPrivilege({"db": "reports", "collection": "daily"}, ["find"]) + ]) + mongo.create_user(MongoUser("staff", "analyst"), password=mongo_pwd, roles=[report_role]) + + mysql = Accounts.mysql(mysql_conn) + user = MySQLAccount("analyst", "10.0.0.8") + role = MySQLAccount("report_reader", "%") + mysql.create_role(role) + mysql.grant_privileges(role, MySQLScope.table("reports", "daily"), ["SELECT"]) + mysql.create_user(user, password=mysql_pwd) + mysql.grant_roles(user, roles=[role]) + mysql.set_default_roles(user, roles=[role]) + + redis = Accounts.redis(redis_client) + redis.apply_user("report_reader", rules=[ + AclRule.enable(), AclRule.add_password(redis_pwd), + AclRule.allow_command("get"), AclRule.key_pattern("reports:*"), + ]) +``` + +分别用新账号建立独立连接,验证报表集合读取、报表表的 SELECT 和匹配键的 GET 确实获准,再对未授权的集合、表或键进行核对。MongoDB 可用 `revoke_roles` 撤销角色,MySQL 可以 `revoke_privileges` 调整角色的表权限,Redis 用 `AclRule.deny_command("get")` 撤销对应命令。多条授权路径并存时要核对其余路径。上例的 Redis 规则会追加到同名用户已有策略;初始化示例应使用新用户名,调整既有用户应检查其全部规则(含选择器)。各后端可复跑的验收脚本、显式配置字段和测试范围见[账号管理验收记录](account-management-validation.md)。 + ## 5. 实施顺序与交付物 | 阶段 | 依赖与工作 | 交付和验收门槛 | diff --git a/docs/account-management-requirements.md b/docs/account-management-requirements.md index c3fc77b..97855d6 100644 --- a/docs/account-management-requirements.md +++ b/docs/account-management-requirements.md @@ -1,12 +1,12 @@ # MongoDB、MySQL、Redis 账号与权限管理需求 -本文定义拟新增的 Python 工具包 `database.accounts` 的首版需求。读者是使用本库的开发者、负责数据库账号的管理员,以及负责验收的工程师。方法、类型和版本矩阵均属设计目标。开发任务、命令映射、示例和验证步骤见[实现设计与交付规划](account-management-design.md)。 +本文定义 Python 工具包 `database.accounts` 的首版需求。读者是使用本库的开发者、负责数据库账号的管理员,以及负责验收的工程师。方法、类型和版本矩阵均属设计目标。开发任务、命令映射、示例和验证步骤见[实现设计与交付规划](account-management-design.md)。 ## 1. 使用目标与 SDK 边界 开发者已经有一条能访问数据库的连接,并持有相应管理权限。工具应让他能在 Python 中盘点账号、解释权限来源、授权、回收权限和轮换凭据。数据库若只允许服务器本地访问,脚本可以运行在服务器上,或复用调用方已建立的网络通道。SDK 接受本库命名连接和对应驱动的原生连接,凭据与连接均由调用方提供。 -现行 CRUD 设计把账号创建和权限授予归入运行环境准备。本提案建议由调用方准备数据库服务、网络通道和初始管理凭据,SDK 借用已授权连接管理后续账号。`Connections` 负责注册、创建、检查和关闭连接。首版交付 Python 类、方法和类型化结果。业务系统若需要员工身份关联、审批或操作审计,可以在调用这些方法时记录自己的操作者和业务原因。 [现行 CRUD 设计](crud-design.md) · [连接管理设计](connection-management-design.md) · [凭据处理约定](../SECURITY.md) +调用方准备数据库服务、网络通道和初始管理凭据,SDK 借用已授权连接管理后续账号。`Connections` 负责注册、创建、检查和关闭连接。首版交付 Python 类、方法和类型化结果。业务系统若需要员工身份关联、审批或操作审计,可以在调用这些方法时记录自己的操作者和业务原因。 [现行 CRUD 设计](crud-design.md) · [连接管理设计](connection-management-design.md) · [凭据处理约定](../SECURITY.md) ## 2. 功能清单与业务验收 diff --git a/docs/account-management-validation.md b/docs/account-management-validation.md new file mode 100644 index 0000000..b4e8a38 --- /dev/null +++ b/docs/account-management-validation.md @@ -0,0 +1,53 @@ +# 数据库账号与权限管理验收记录 + +## 验收对象与口径 + +本次验收针对 `feature/database-account-management` 上的 `database.accounts`,开发基线为 `5208a2e`。验证使用当前工作树的 `src`、Python 3.10.20、临时回环服务、随机命名的主体以及独立的受限客户端。盘点、授予、撤销或删除后的状态由管理连接回读,访问效果由受限账号的实际请求验证。各后端的“组”按下表中不同的维度计数。 + +| 后端 | 服务器与驱动 | 组合定义与核对内容 | +| --- | --- | --- | +| MongoDB | 7.0.15、8.0.24;PyMongo 4.6.0、4.9、4.18.3 | 20 组。每个服务端覆盖 4.6 同步、4.9 和 4.18.3 同步及异步,各走本库句柄和原生连接。涵盖同名异库、角色继承、权限展开、追加与撤销、改密与回收、只读管理凭据被拒 | +| MySQL | 8.0.46、8.4.11;Connector/Python 8.3.0、26.10.0 | 16 组。每个服务器分别组合两个驱动、纯 Python/C 扩展及两种 `partial_revokes` 设置。检查账号来源主机、角色激活、对象范围、精确库名及隐式提交保护 | +| Redis | 6.0.20、6.2.18、7.2.8、8.0.4;redis-py 5.0.0、8.1.0 | 16 组。四个服务器版本分别组合两个驱动与 RESP2/RESP3。检查规则顺序与权限效果、多密码轮换、会话启停与删除、按版本模拟、ACL 文件写入并重启读回、保存失败后回读 | + +Redis 采用 Windows 社区 MSYS2 移植构建。每次验收使用新建的固定数据节点,目标编号逻辑库共享其 ACL。MongoDB 使用自管认证实例,MySQL 使用新建数据目录。矩阵对应这些实际版本与平台,未验证 Atlas 管理平面、Redis Sentinel/Cluster 的跨节点账号一致性或托管商专用权限模型。 + +## 集成结果 + +三种后端共实现 39 项操作,MongoDB 16 项、MySQL 17 项、Redis 6 项,同步和异步公开签名逐项一致。上表 52 组真实服务验收在集成工作树复跑通过,回环测试实例在验收后关闭。MySQL 的转授权验证与部分撤权验证使用不同受体,避免其他授权限制影响测试结论。MongoDB 与 MySQL 的测试对象创建遇到回包丢失时,脚本按本轮唯一身份回读并核对清理。 + +| 检查 | 实际结果 | +| --- | --- | +| 完整离线回归 | `python -B -m unittest discover -s tests -q`。当前驱动组合(4.18.3 / 26.10.0 / 8.1.0)324 项通过。最低驱动组合(4.6.0 / 8.3.0 / 5.0.0)324 项通过。驱动顺序为 PyMongo、Connector/Python、redis-py | +| 账号定向回归 | `python -B -m unittest discover -s tests -p "test_accounts*.py" -q`。最低驱动组合 34 项通过;完整回归亦包含这 34 项 | +| 静态检查 | 对账号包、定向测试和三个验收脚本执行 Ruff lint 与格式检查,13 个 Python 文件通过。Git 空白检查通过 | +| 构建与延迟导入 | 在临时源码副本中对产品提交 `0c0040f` 离线构建 wheel,确认其中包含所有账号模块。在关闭 site 初始化的解释器中直接从 wheel 导入 9 个公开入口、模型与异常,三个驱动均未被导入 | +| 文档 | 检查 README、需求、设计、测试与验收记录中的本地文件链接及账号示例 Python 语法 | + +## 如何复跑 + +设置 `PYTHONPATH` 指向当前工作树的 `src`,或在隔离环境中安装当前源码。Python 和原生驱动版本由运行者按目标组合选择。两个接收配置文件的脚本会在指定的隔离数据库**创建和删除测试对象**,所需的管理员凭据由运行者提供。配置保存在仓库之外。Redis 脚本接收服务器可执行文件,自己选择回环端口并启动临时服务。 + +| 验收入口 | 调用方式及配置 | +| --- | --- | +| [MongoDB 验收脚本](../scripts/verify_accounts_mongodb.py) | `python -B scripts/verify_accounts_mongodb.py --config <隔离配置.json> --mode both`。JSON 包含 `host`、`port`、`username`、`password`、`auth_db`。PyMongo 4.6 使用 `--mode sync` | +| [MySQL 验收脚本](../scripts/verify_accounts_mysql.py) | `python -B scripts/verify_accounts_mysql.py --config <隔离配置.json>`。使用 Connector/Python 连接参数,含 `host`、`port`、`user` 和 `password`。可用 `use_pure` 选择驱动实现,服务端配置决定 `partial_revokes` | +| [Redis 验收脚本](../scripts/verify_accounts_redis.py) | `python -B scripts/verify_accounts_redis.py --server --protocol 2`,将协议改为 `3` 复验 RESP3。脚本检查临时节点启动、重启、退出和文件清理 | + +实测时管理凭据未进入仓库。离线故障测试使用驱动替身确定性地模拟响应丢失、取消和写关注错误;这些结果与上表的真实登录、授权和撤权验证分别计入。配置说明与所需权限见[账号管理需求](account-management-requirements.md)。 + +## Git 集成记录 + +主智能体在仓库根工作树使用 `git status --short --branch`、`git diff --cached`、`git rev-parse HEAD` 和 `git worktree list --porcelain` 核对基线。公共契约通过四项定向测试后,按路径 `git add`、检查 `git diff --cached --check` 并提交。随后用 `git worktree add -b` 从公共提交建立下列三个工作树。各后端工作者在自己的工作树按路径暂存、检查、中文提交,协调者核对 `git diff HEAD...<交付 SHA>` 后在当前功能分支串行执行 `git merge --no-ff`。本表中的工作树路径相对主仓库目录。 + +| 执行者与分支 | worktree | 阶段或交付 SHA | 集成后的 HEAD | +| --- | --- | --- | --- | +| 主智能体,`feature/database-account-management` | `.` | 公共绑定 `ef0cb1a` | `ef0cb1a` | +| Redis,`feature/accounts-redis` | `../database-worktrees/accounts-redis` | `00fe5ac` | `cd95481` | +| MongoDB,`feature/accounts-mongodb` | `../database-worktrees/accounts-mongodb` | `8f3d03d` | `a2a5e7a` | +| MongoDB 验收清理修复,同一分支 | `../database-worktrees/accounts-mongodb` | `c656848` | `7f0a6ec` | +| MySQL,`feature/accounts-mysql` | `../database-worktrees/accounts-mysql` | `19dca93` | `39d334f` | +| MySQL 验收用例隔离,同一分支 | `../database-worktrees/accounts-mysql` | `d87d973` | `a2211e2` | +| 主智能体,`feature/database-account-management` | `.` | Redis Pipeline 绑定和清理异常处理 `0c0040f` | `0c0040f` | + +发布说明和验收记录在产品提交后单独暂存,执行 `git diff --cached --check`,通过后以中文文档提交收尾。 diff --git a/docs/crud-design.md b/docs/crud-design.md index fd55566..2570b32 100644 --- a/docs/crud-design.md +++ b/docs/crud-design.md @@ -27,7 +27,7 @@ - 初始化由调用方显式执行,CRUD 不自动建表、改表、建立索引或迁移结构。 - SDK 不读取环境变量、`.env` 或其他隐式配置来源。 -数据库服务的安装、启动、账号创建、权限授予和集群部署属于运行环境准备,不纳入 SDK。 +数据库服务、集群部署和初始管理凭据由调用方准备。后续账号与授权管理由独立的 [database.accounts 工具](account-management-design.md) 提供。 ### 1.2 三种数据库的基础封装边界 diff --git a/tests/README.md b/tests/README.md index bed44f6..74f40ee 100644 --- a/tests/README.md +++ b/tests/README.md @@ -33,6 +33,10 @@ python -B -m unittest discover -s tests -p "test_managed_redis.py" -q 新增回归沿用 `unittest.TestCase` 和 `unittest.IsolatedAsyncioTestCase`。涉及后端适配时,按改动覆盖可用的同步和异步路径,并核对失败条件、清理和兼容性。驱动兼容测试在独立环境中运行,记录实际导入的版本。 +## 账号与权限管理验证 + +`python -B -m unittest discover -s tests -p "test_accounts*.py" -v` 运行账号管理离线测试。真实授权验证使用三个后端各自的验收脚本,配置格式、服务端和驱动组合以及实测结果见[账号管理验收记录](../docs/account-management-validation.md)。 + ## 隔离服务验证 [scripts/verify_connections.py](../scripts/verify_connections.py) 使用 `--config` 读取测试环境的 JSON 配置。JSON 的键为连接名称,值为 `ConnectionConfig` 的关键字参数。配置文件保存在仓库外,并指向隔离测试实例。 -- Gitee From 76abbbe7bc9c4126213f58eca552ee96008a8d8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 18:14:50 +0800 Subject: [PATCH 85/86] =?UTF-8?q?refactor!:=20=E5=B0=86=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=20SDK=20=E6=9B=B4=E5=90=8D=E4=B8=BA=20DATVIA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 发行包和 Python 导入名统一为 datvia,同步更新运行时模块加载、文档、示例、脚本与测试。源码包包含 README 的两份 SVG。 验证通过 324 项离线测试、sdist 与 wheel 构建、wheel 与可编辑安装及 pip check。 BREAKING CHANGE: 依赖原 database SDK 的应用需将依赖声明和导入路径改为 datvia。 --- .gitignore | 2 +- AGENTS.md | 18 +++---- CHANGELOG.md | 5 +- CONTRIBUTING.md | 4 +- MANIFEST.in | 1 + README.md | 49 +++++++++--------- docs/README.md | 12 ++--- docs/account-management-design.md | 16 +++--- docs/account-management-requirements.md | 2 +- ...{database-badges.svg => datvia-badges.svg} | 0 ...{database-banner.svg => datvia-banner.svg} | 6 +-- docs/connection-management-design.md | 14 +++--- docs/crud-adapter-contract.md | 4 +- docs/crud-design.md | 50 +++++++++---------- docs/crud-usage.md | 32 ++++++------ examples/basic_usage.py | 24 ++++----- pyproject.toml | 6 +-- scripts/verify_accounts_mongodb.py | 22 ++++---- scripts/verify_accounts_mysql.py | 22 ++++---- scripts/verify_accounts_redis.py | 16 +++--- scripts/verify_basics.py | 26 +++++----- scripts/verify_connections.py | 2 +- src/{database => datvia}/__init__.py | 2 +- .../_adapters/__init__.py | 0 src/{database => datvia}/_adapters/mongodb.py | 0 src/{database => datvia}/_adapters/mysql.py | 0 src/{database => datvia}/_adapters/redis.py | 0 src/{database => datvia}/accounts/__init__.py | 0 src/{database => datvia}/accounts/_base.py | 2 +- src/{database => datvia}/accounts/_mongodb.py | 0 src/{database => datvia}/accounts/_mysql.py | 0 .../accounts/_mysql_models.py | 0 src/{database => datvia}/accounts/_redis.py | 0 src/{database => datvia}/config.py | 0 src/{database => datvia}/connections.py | 2 +- src/{database => datvia}/crud/__init__.py | 0 src/{database => datvia}/crud/_batch.py | 0 src/{database => datvia}/crud/_codecs.py | 0 src/{database => datvia}/crud/_conditions.py | 0 src/{database => datvia}/crud/_errors.py | 0 src/{database => datvia}/crud/_mongodb.py | 0 src/{database => datvia}/crud/_mysql.py | 0 src/{database => datvia}/crud/_records.py | 2 +- src/{database => datvia}/crud/_redis_hash.py | 0 .../crud/_string_types.py | 0 src/{database => datvia}/crud/_strings.py | 0 src/{database => datvia}/crud/_types.py | 0 src/{database => datvia}/errors.py | 0 src/{database => datvia}/mongodb/__init__.py | 0 .../mongodb/_async_utils.py | 0 src/{database => datvia}/mongodb/_batch.py | 0 .../mongodb/_cursor_batches.py | 0 .../mongodb/_resources.py | 0 src/{database => datvia}/mongodb/_scan.py | 0 .../mongodb/_sync_utils.py | 0 src/{database => datvia}/mongodb/client.py | 0 src/{database => datvia}/mongodb/types.py | 0 src/{database => datvia}/mongodb/utils.py | 0 src/{database => datvia}/mysql/__init__.py | 0 src/{database => datvia}/mysql/client.py | 0 src/{database => datvia}/redis/__init__.py | 0 src/{database => datvia}/redis/client.py | 0 src/{database => datvia}/schema/__init__.py | 0 src/{database => datvia}/schema/_base.py | 0 src/{database => datvia}/schema/_models.py | 0 src/{database => datvia}/schema/_mongodb.py | 0 src/{database => datvia}/schema/_mysql.py | 0 tests/README.md | 4 +- tests/test_accounts_binding.py | 16 +++--- tests/test_accounts_mongodb.py | 24 ++++----- tests/test_accounts_mysql.py | 24 ++++----- tests/test_accounts_redis.py | 24 ++++----- tests/test_connection_config.py | 8 +-- tests/test_connection_errors.py | 2 +- tests/test_connections.py | 12 ++--- tests/test_crud_conditions.py | 24 ++++----- tests/test_crud_core.py | 50 +++++++++---------- tests/test_crud_integration.py | 15 +++--- tests/test_crud_mongodb.py | 34 ++++++------- tests/test_crud_mysql.py | 38 +++++++------- tests/test_crud_redis.py | 34 ++++++------- tests/test_crud_strings.py | 20 ++++---- tests/test_managed_mongodb.py | 18 +++---- tests/test_managed_mysql.py | 22 ++++---- tests/test_managed_redis.py | 6 +-- tests/test_mongodb_client.py | 8 +-- tests/test_mongodb_scan.py | 8 +-- tests/test_mongodb_utils.py | 18 +++---- tests/test_mysql_client.py | 10 ++-- tests/test_schema_binding.py | 18 +++---- tests/test_schema_mongodb.py | 18 +++---- tests/test_schema_mysql.py | 24 ++++----- 92 files changed, 412 insertions(+), 408 deletions(-) create mode 100644 MANIFEST.in rename docs/assets/{database-badges.svg => datvia-badges.svg} (100%) rename docs/assets/{database-banner.svg => datvia-banner.svg} (95%) rename src/{database => datvia}/__init__.py (89%) rename src/{database => datvia}/_adapters/__init__.py (100%) rename src/{database => datvia}/_adapters/mongodb.py (100%) rename src/{database => datvia}/_adapters/mysql.py (100%) rename src/{database => datvia}/_adapters/redis.py (100%) rename src/{database => datvia}/accounts/__init__.py (100%) rename src/{database => datvia}/accounts/_base.py (98%) rename src/{database => datvia}/accounts/_mongodb.py (100%) rename src/{database => datvia}/accounts/_mysql.py (100%) rename src/{database => datvia}/accounts/_mysql_models.py (100%) rename src/{database => datvia}/accounts/_redis.py (100%) rename src/{database => datvia}/config.py (100%) rename src/{database => datvia}/connections.py (98%) rename src/{database => datvia}/crud/__init__.py (100%) rename src/{database => datvia}/crud/_batch.py (100%) rename src/{database => datvia}/crud/_codecs.py (100%) rename src/{database => datvia}/crud/_conditions.py (100%) rename src/{database => datvia}/crud/_errors.py (100%) rename src/{database => datvia}/crud/_mongodb.py (100%) rename src/{database => datvia}/crud/_mysql.py (100%) rename src/{database => datvia}/crud/_records.py (99%) rename src/{database => datvia}/crud/_redis_hash.py (100%) rename src/{database => datvia}/crud/_string_types.py (100%) rename src/{database => datvia}/crud/_strings.py (100%) rename src/{database => datvia}/crud/_types.py (100%) rename src/{database => datvia}/errors.py (100%) rename src/{database => datvia}/mongodb/__init__.py (100%) rename src/{database => datvia}/mongodb/_async_utils.py (100%) rename src/{database => datvia}/mongodb/_batch.py (100%) rename src/{database => datvia}/mongodb/_cursor_batches.py (100%) rename src/{database => datvia}/mongodb/_resources.py (100%) rename src/{database => datvia}/mongodb/_scan.py (100%) rename src/{database => datvia}/mongodb/_sync_utils.py (100%) rename src/{database => datvia}/mongodb/client.py (100%) rename src/{database => datvia}/mongodb/types.py (100%) rename src/{database => datvia}/mongodb/utils.py (100%) rename src/{database => datvia}/mysql/__init__.py (100%) rename src/{database => datvia}/mysql/client.py (100%) rename src/{database => datvia}/redis/__init__.py (100%) rename src/{database => datvia}/redis/client.py (100%) rename src/{database => datvia}/schema/__init__.py (100%) rename src/{database => datvia}/schema/_base.py (100%) rename src/{database => datvia}/schema/_models.py (100%) rename src/{database => datvia}/schema/_mongodb.py (100%) rename src/{database => datvia}/schema/_mysql.py (100%) diff --git a/.gitignore b/.gitignore index c185862..910f78d 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,7 @@ __pycache__/ # test scripts !/tests /test*.py -src/database/_version.py +src/datvia/_version.py # logs /logs/ diff --git a/AGENTS.md b/AGENTS.md index 12e753b..b5d43d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,14 +2,14 @@ ## 项目概览 -`database` 是支持 Python 3.10 及以上版本的 MongoDB、MySQL 和 Redis 连接 SDK。本文件适用于整个仓库。进入子目录时先检查是否有该目录的 `AGENTS.md` 或 `AGENTS.override.md`。 +DATVIA(包名 `datvia`)是支持 Python 3.10 及以上版本的 MongoDB、MySQL 和 Redis 连接 SDK。本文件适用于整个仓库。进入子目录时先检查是否有该目录的 `AGENTS.md` 或 `AGENTS.override.md`。 ## 从哪里开始 - 先看 `git status --short --branch`、任务相关差异和暂存区,保留工作区已有改动。 - 使用示例见 [README](README.md),开发和提交规范见 [贡献指南](CONTRIBUTING.md)。 -- `src/database/config.py` 负责配置和参数校验,`src/database/connections.py` 负责命名连接和生命周期,`src/database/errors.py` 负责异常转换。 -- `src/database/_adapters/` 负责高层连接与驱动之间的适配。原生工厂维护在 `src/database/{mongodb,mysql,redis}/`,离线测试放在 `tests/`。 +- `src/datvia/config.py` 负责配置和参数校验,`src/datvia/connections.py` 负责命名连接和生命周期,`src/datvia/errors.py` 负责异常转换。 +- `src/datvia/_adapters/` 负责高层连接与驱动之间的适配。原生工厂维护在 `src/datvia/{mongodb,mysql,redis}/`,离线测试放在 `tests/`。 ## 修改连接代码 @@ -42,9 +42,9 @@ git diff --check ```powershell $base = (git rev-parse develop).Trim() -git worktree add -b feature/task-integration ../database-worktrees/task-integration $base -git worktree add -b feature/task-config ../database-worktrees/task-config $base -git worktree add -b feature/task-errors ../database-worktrees/task-errors $base +git worktree add -b feature/task-integration ../datvia-worktrees/task-integration $base +git worktree add -b feature/task-config ../datvia-worktrees/task-config $base +git worktree add -b feature/task-errors ../datvia-worktrees/task-errors $base ``` ### 开发和集成 @@ -55,9 +55,9 @@ git worktree add -b feature/task-errors ../database-worktrees/task-errors $base ```powershell $workerHead = (git rev-parse feature/task-config).Trim() -git -C ../database-worktrees/task-integration diff "HEAD...$workerHead" -git -C ../database-worktrees/task-integration merge --no-ff -m "feat(config): 合入连接配置" $workerHead -git -C ../database-worktrees/task-integration rev-parse HEAD +git -C ../datvia-worktrees/task-integration diff "HEAD...$workerHead" +git -C ../datvia-worktrees/task-integration merge --no-ff -m "feat(config): 合入连接配置" $workerHead +git -C ../datvia-worktrees/task-integration rev-parse HEAD ``` ### Git 操作记录 diff --git a/CHANGELOG.md b/CHANGELOG.md index cc0d45e..22f6aaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,13 +6,13 @@ ### 新增 -- 新增 `database.accounts` 同步与异步 Python 工具,支持自管 MongoDB 用户及角色、MySQL 账号及对象授权和固定 Redis 数据节点的 ACL 管理。 +- 新增 `datvia.accounts` 同步与异步 Python 工具,支持自管 MongoDB 用户及角色、MySQL 账号及对象授权和固定 Redis 数据节点的 ACL 管理。 - 增加 MongoDB 同步与异步分页读取和流式续读工具,按 `_id` 索引定位,可在游标失效或连接中断后有界恢复。 - 增加 MongoDB 同步与异步混合批写、待写缓冲丢弃和自动关闭游标的分批读取工具。异步批处理支持消费异步可迭代对象。 - 新增同步/异步 Schema 基础管理:MySQL 库表创建、兼容校验、目录查看和删除;MongoDB 集合准备、必要身份约束及目录管理。初始化显式执行,不自动迁移。 - 新增 `RedisStrings` / `AsyncRedisStrings` 的单键、批量读写、删除和 TTL,保留逐项执行证据、原始错误与取消信号。 - 新增三数据库完整同步/异步使用示例和显式隔离验收脚本;原生客户端及已有记录接口保持兼容。 -- 新增可选的 `database.crud.Records` / `AsyncRecords`,在已有连接上绑定 MySQL 表、MongoDB 集合或 Redis Hash,支持复合键、字段投影、批量读取及分块写入;MySQL/MongoDB 另支持结构化条件查询和集合更新、删除。 +- 新增可选的 `datvia.crud.Records` / `AsyncRecords`,在已有连接上绑定 MySQL 表、MongoDB 集合或 Redis Hash,支持复合键、字段投影、批量读取及分块写入;MySQL/MongoDB 另支持结构化条件查询和集合更新、删除。 - CRUD 写入结果区分已知计数与未知计数;批量失败保留逐项执行状态、部分结果和原始异常,异步取消保持原信号。记录层借用连接,不自动提交、回滚或重试。 - Redis Hash 记录使用显式 namespace、字段 codec、版本标记和单键脚本,支持创建、更新、投影及 TTL;保留原生其他数据结构和客户端能力。 - 新增 `ConnectionConfig` 和 `Connections`,支持按名称登记 MongoDB、MySQL 和 Redis 配置,按需创建独立的同步或异步句柄。 @@ -30,6 +30,7 @@ ### 兼容性与依赖 +- 项目更名为 DATVIA,发行包名和 Python 导入名从 `database` 改为 `datvia`。升级时需更新应用的依赖声明和导入语句。已有 Redis Hash v1 记录可直接读取。 - 源码安装统一使用 `python -m pip install .`,默认包含 MongoDB、MySQL 和 Redis 驱动。原有安装 extras 已移除,运行依赖及版本约束与 `requirements.txt` 一致。 - PyMongo 的默认依赖下限为 `4.6.0`,同步连接支持该版本。异步连接需要 PyMongo 4.9 或以上版本,缺少异步 API 时给出升级提示。 - 原生工厂继续返回驱动对象。统一入口的 `create_async()` 需要 `await`,原生工厂沿用各自的异步调用方式。 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fe22b56..2532b4d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,14 +11,14 @@ python -m pip install -e . python -m pip install ruff ``` -开发安装包含三个后端的驱动,测试使用标准库 `unittest`。运行依赖及版本约束在 [pyproject.toml](pyproject.toml) 的 `project.dependencies` 和 [requirements.txt](requirements.txt) 中同步维护。验证驱动最低版本时,在独立环境使用对应的版本约束。源码构建时由 `setuptools-scm` 生成 `src/database/_version.py`。 +开发安装包含三个后端的驱动,测试使用标准库 `unittest`。运行依赖及版本约束在 [pyproject.toml](pyproject.toml) 的 `project.dependencies` 和 [requirements.txt](requirements.txt) 中同步维护。验证驱动最低版本时,在独立环境使用对应的版本约束。源码构建时由 `setuptools-scm` 生成 `src/datvia/_version.py`。 ## 代码与文档风格 - [.ruff.toml](.ruff.toml) 统一 Python 3.10 语法目标、120 字符行宽、单引号格式,以及 `E`、`F`、`I` 规则。一行导入一个对象。 - 新增公开接口写明参数和返回值类型。涉及默认值、异常或资源生命周期时,用 NumPy 风格的 docstring 描述契约,完整用法示例维护在 README 或相关文档。 - Markdown 普通段落及引用正文采用一段一行、段间空行。列表、表格、代码块和显式换行保留各自结构。 -- 文档正文、表格和命令中的项目路径以仓库根目录为基准,使用相对路径,不写盘符、用户主目录等本机绝对地址;仓库根目录记为 `.`,相邻工作树可写为 `../database-worktrees/<任务目录>`。 +- 文档正文、表格和命令中的项目路径以仓库根目录为基准,使用相对路径,不写盘符、用户主目录等本机绝对地址;仓库根目录记为 `.`,相邻工作树可写为 `../datvia-worktrees/<任务目录>`。 - Markdown 本地链接使用相对目标,按文档所在目录保留必要的 `../` 跳转;链接文字展示路径时使用仓库根目录相对路径。仓库外的历史验收材料只记录文件名与用途,实际存储位置由运行者指定,不将其写成仓库内文件。 - 根据 [文档索引](docs/README.md) 更新对应文档。文档中的连接地址、凭据和测试数据使用本地占位值。 diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..b5dd996 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include docs/assets/datvia-banner.svg docs/assets/datvia-badges.svg diff --git a/README.md b/README.md index bb88b76..1cefef9 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,9 @@
-

database

-

用一套 Python API 管理 MongoDB、MySQL 和 Redis 连接

+

DATVIA

+

Data via one Python API.

+

用一套 Python API 接入 MongoDB、MySQL 和 Redis,按需使用原生驱动。

- Python 3.10+ · 同步与异步 API · MIT 许可证 + Python 3.10+ · 同步与异步 API · MIT 许可证

快速开始 · @@ -28,6 +29,8 @@ python -m pip install . ``` +DATVIA 的发行包名和导入名均为 `datvia`。从原 `database` 项目升级时,请将依赖声明、Python 导入和按包名配置的日志过滤规则改为 `datvia`。 + 驱动版本约束与 [requirements.txt](requirements.txt) 一致。 | 后端 | 默认驱动依赖 | @@ -41,8 +44,8 @@ python -m pip install . 下面连接到本机 `127.0.0.1:6379` 上的 Redis,检查一次连接并在退出上下文时关闭它。 ```python -from database import ConnectionConfig -from database import Connections +from datvia import ConnectionConfig +from datvia import Connections connections = Connections({ 'cache': ConnectionConfig(backend='redis', host='127.0.0.1', port=6379), @@ -71,7 +74,7 @@ PyMongo 4.6–4.8 支持同步的 `create()`。异步的 `create_async()` 需要 ```python import asyncio -from database.mongodb import Client +from datvia.mongodb import Client with Client.create("mongodb://127.0.0.1:27017/") as client: client.admin.command("ping") @@ -87,10 +90,10 @@ asyncio.run(ping_mongodb()) #### MongoDB 分批写入与读取 -`database.mongodb.utils` 接收原生 PyMongo 集合和游标。批处理器借用集合,MongoDB 客户端和连接池由调用方关闭。 +`datvia.mongodb.utils` 接收原生 PyMongo 集合和游标。批处理器借用集合,MongoDB 客户端和连接池由调用方关闭。 ```python -from database.mongodb.utils import BatchInsert, BatchUpdate, cursor_batches +from datvia.mongodb.utils import BatchInsert, BatchUpdate, cursor_batches from pymongo import UpdateOne @@ -116,7 +119,7 @@ def mark_processed(target, ids): 异步使用 `AsyncBatchInsert`、`AsyncBatchUpdate`、`AsyncBatchWrite` 和 `async_cursor_batches()`。`extend()` 也接收异步可迭代对象。 ```python -from database.mongodb.utils import AsyncBatchInsert, async_cursor_batches +from datvia.mongodb.utils import AsyncBatchInsert, async_cursor_batches async def migrate_async(source, target): @@ -131,7 +134,7 @@ async def migrate_async(source, target): `scan_collection()` 和 `stream_collection()` 接收集合与筛选条件,按 `_id` 升序逐条交付。适用于具有常规 `_id_` 索引、简单排序规则和全局唯一 `_id` 的普通集合。续读用索引下界定位,保留原筛选条件。每次查询读取当时可见的数据,恢复位置保存在本次遍历中。 ```python -from database.mongodb.utils import scan_collection, stream_collection +from datvia.mongodb.utils import scan_collection, stream_collection def process_in_pages(articles, process): @@ -158,7 +161,7 @@ def process_in_stream(articles, process): import asyncio import os -from database.mysql import Client +from datvia.mysql import Client options = { "host": "127.0.0.1", @@ -185,7 +188,7 @@ asyncio.run(ping_mysql()) ```python import asyncio -from database.redis import Client +from datvia.redis import Client with Client.create(host="127.0.0.1", port=6379) as client: client.ping() @@ -207,7 +210,7 @@ asyncio.run(ping_redis()) ```python from contextlib import ExitStack -from database import ConnectionConfig, Connections +from datvia import ConnectionConfig, Connections def build_connections(mysql_password): return Connections({ @@ -253,7 +256,7 @@ async def read_document(connections, document_id): ) ``` -新高层 `create_async()` 均需 `await`。旧 `database..Client` 的调用方式保持不变。异步句柄应在创建它的事件循环内检查和关闭;同步与异步实例相互独立。 +高层 `Connections.create_async()` 统一使用 `await`。原生工厂 `datvia..Client` 的调用方式见各后端示例。异步句柄应在创建它的事件循环内检查和关闭;同步与异步实例相互独立。 ### Redis 拓扑 @@ -277,7 +280,7 @@ Cluster 检查客户端当前发现的全部主节点快照,逐个 PING;首 ### 异常和原生访问 ```python -from database.errors import DatabaseError +from datvia.errors import DatabaseError try: with connections.create("orders") as connection: @@ -304,16 +307,16 @@ MongoDB 的检查结果使用 `scope="selected_server"`,`targets` 为空:原 ## 基础结构准备与键值操作 -在使用记录 CRUD 前,可以通过 `database.schema.Schema` / `AsyncSchema` 显式准备 MySQL 数据库与基础表、MongoDB 普通集合及必要身份约束。已有结构只做兼容性校验,不自动迁移。Redis 直接绑定 Hash 命名空间,或通过 `RedisStrings` / `AsyncRedisStrings` 使用 String 单键、批量与 TTL 接口。 +在使用记录 CRUD 前,可以通过 `datvia.schema.Schema` / `AsyncSchema` 显式准备 MySQL 数据库与基础表、MongoDB 普通集合及必要身份约束。已有结构只做兼容性校验,不自动迁移。Redis 直接绑定 Hash 命名空间,或通过 `RedisStrings` / `AsyncRedisStrings` 使用 String 单键、批量与 TTL 接口。 MySQL 的初始化连接应省略尚未创建的业务库并启用 autocommit;创建完毕后另建选定业务库的连接。MongoDB 的集合准备和 Redis 的键空间绑定保留各自概念。上述对象都借用既有连接,原生高级能力仍通过 `connection.native` 使用。 下面用 MongoDB 演示最短完整流程,包含配置、集合准备、绑定、增删改查和资源关闭。请传入演示实例地址并使用专用演示集合。 ```python -from database import ConnectionConfig, Connections -from database.crud import Records -from database.schema import Schema +from datvia import ConnectionConfig, Connections +from datvia.crud import Records +from datvia.schema import Schema def demo(mongodb_uri): @@ -337,10 +340,10 @@ def demo(mongodb_uri): ## 高级记录操作 -`database.crud` 提供可选的记录层:MySQL 表、MongoDB 集合和显式选择的 Redis Hash 格式共用按键读取、字段投影及批量写入接口。它借用已有连接,保留 `connection.native` 的全部能力。 +`datvia.crud` 提供可选的记录层:MySQL 表、MongoDB 集合和显式选择的 Redis Hash 格式共用按键读取、字段投影及批量写入接口。它借用已有连接,保留 `connection.native` 的全部能力。 ```python -from database.crud import Records +from datvia.crud import Records with connections.create('orders') as connection: with Records.mysql(connection, table='orders', key_fields=('id',)) as orders: @@ -362,10 +365,10 @@ with connections.create('orders') as connection: ## 数据库账号与权限管理 -`database.accounts` 借用现有管理连接,支持本库句柄或原生 PyMongo、Connector/Python、redis-py 客户端。MongoDB 用认证库和用户名定位账号,MySQL 用用户名和来源主机定位账号,Redis ACL 作用于固定的数据节点。下例中的连接由调用方提供。 +`datvia.accounts` 借用现有管理连接,支持本库句柄或原生 PyMongo、Connector/Python、redis-py 客户端。MongoDB 用认证库和用户名定位账号,MySQL 用用户名和来源主机定位账号,Redis ACL 作用于固定的数据节点。下例中的连接由调用方提供。 ```python -from database.accounts import Accounts, AsyncAccounts, MongoRole, MongoUser, MySQLAccount +from datvia.accounts import Accounts, AsyncAccounts, MongoRole, MongoUser, MySQLAccount with connections.create("mongo_admin") as conn: mongo = Accounts.mongodb(conn) diff --git a/docs/README.md b/docs/README.md index 7a7b3eb..e35ae38 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,4 @@ -# database 文档索引 +# DATVIA 文档索引 ## 使用与开发 @@ -32,11 +32,11 @@ | 路径 | 职责 | | --- | --- | -| [src/database/](../src/database/) | 公共导出、连接配置、统一句柄和异常转换 | -| [src/database/_adapters/](../src/database/_adapters/) | 三种后端高层连接的创建、检查、关闭及驱动错误映射 | -| [src/database/crud/](../src/database/crud/) | 同步/异步记录与 Redis String 操作、条件模型、批量结果和后端适配 | -| [src/database/schema/](../src/database/schema/) | MySQL 基础库表与 MongoDB 普通集合管理 | -| [src/database/mongodb/](../src/database/mongodb/)、[src/database/mysql/](../src/database/mysql/)、[src/database/redis/](../src/database/redis/) | 后端原生客户端工厂及各自的辅助功能 | +| [src/datvia/](../src/datvia/) | 公共导出、连接配置、统一句柄和异常转换 | +| [src/datvia/_adapters/](../src/datvia/_adapters/) | 三种后端高层连接的创建、检查、关闭及驱动错误映射 | +| [src/datvia/crud/](../src/datvia/crud/) | 同步/异步记录与 Redis String 操作、条件模型、批量结果和后端适配 | +| [src/datvia/schema/](../src/datvia/schema/) | MySQL 基础库表与 MongoDB 普通集合管理 | +| [src/datvia/mongodb/](../src/datvia/mongodb/)、[src/datvia/mysql/](../src/datvia/mysql/)、[src/datvia/redis/](../src/datvia/redis/) | 后端原生客户端工厂及各自的辅助功能 | | [tests/](../tests/) | 离线自动化测试 | | [scripts/verify_connections.py](../scripts/verify_connections.py) | 使用显式测试配置验证真实连接 | | [scripts/verify_basics.py](../scripts/verify_basics.py) | 在显式隔离服务上验证结构准备、完整读写及清理 | diff --git a/docs/account-management-design.md b/docs/account-management-design.md index a1e5f03..4a34d4e 100644 --- a/docs/account-management-design.md +++ b/docs/account-management-design.md @@ -1,6 +1,6 @@ # MongoDB、MySQL、Redis 账号与权限管理工具实现设计 -本文是 `database.accounts` 首版的实施规划,供实现、审查与验收使用。已执行的版本组合及测试结果见[验收记录](account-management-validation.md)。需求编号 R1–R9、使用场景和操作权限表见[需求文档](account-management-requirements.md)。既有连接与异常实现见[连接生命周期](../src/database/connections.py)和[异常转换](../src/database/errors.py)。 +本文是 `datvia.accounts` 首版的实施规划,供实现、审查与验收使用。已执行的版本组合及测试结果见[验收记录](account-management-validation.md)。需求编号 R1–R9、使用场景和操作权限表见[需求文档](account-management-requirements.md)。既有连接与异常实现见[连接生命周期](../src/datvia/connections.py)和[异常转换](../src/datvia/errors.py)。 ## 1. 组件与执行流程 @@ -16,7 +16,7 @@ | `Accounts.redis(conn)` | 本库单节点 Redis 句柄或 redis-py 固定数据节点 `Redis` | 检查句柄模式及原生连接池类型,拒绝集群路由和可识别的 Sentinel 自动发现池 | | `AsyncAccounts` 的同名工厂 | 本库 `AsyncConnection` 或对应后端的原生异步对象 | 工厂同步返回管理对象,执行方法使用 `await`。本库句柄还要检查所属事件循环 | -绑定管理对象后,调用方若已关闭本库句柄,操作开始时报告 `ConnectionClosedError`。原生对象按驱动自己的关闭、重连和事件循环契约处理。绑定阶段只访问已知的类型或明确的属性,避免对 PyMongo 客户端使用会隐式生成数据库对象的属性探测。可沿用已有 schema 模块的绑定和异常上下文设计,扩展原生连接这条路径。 [已有绑定实现](../src/database/schema/_base.py) · [已有工厂](../src/database/schema/__init__.py) +绑定管理对象后,调用方若已关闭本库句柄,操作开始时报告 `ConnectionClosedError`。原生对象按驱动自己的关闭、重连和事件循环契约处理。绑定阶段只访问已知的类型或明确的属性,避免对 PyMongo 客户端使用会隐式生成数据库对象的属性探测。可沿用已有 schema 模块的绑定和异常上下文设计,扩展原生连接这条路径。 [已有绑定实现](../src/datvia/schema/_base.py) · [已有工厂](../src/datvia/schema/__init__.py) ## 2. 公共输入、返回与错误 @@ -84,7 +84,7 @@ SDK 不为账号写入增加自动重放。原生客户端沿用调用方的驱 **转授范围。** `grant_option=False` 仅省略本次语句的转授子句。静态权限在同一授权层级共享相应转授能力,例如对某库的 `SELECT` 增加转授时,同库同层级已有的 `INSERT` 也可能被转授。全局动态权限逐项计算。验收时记录目标账号操作前后完整的 `SHOW GRANTS`,同时用第二个临时主体验证实际转授范围。 [GRANT OPTION](https://dev.mysql.com/doc/refman/8.4/en/grant.html) -**事务与结果。** 在同步、异步写入路径复用现有 DDL 守卫的原则,要求 `autocommit=True` 且当前没有活动事务;检查失败时把连接交还调用方处理。每个游标由创建方关闭,清理失败不得覆盖已捕获的主要异常。管理连接上依次调用创建账号、分配角色、设置默认角色,分别保存确认结果。 [隐式提交](https://dev.mysql.com/doc/refman/8.4/en/implicit-commit.html) · [现有 MySQL 结构管理实现](../src/database/schema/_mysql.py) +**事务与结果。** 在同步、异步写入路径复用现有 DDL 守卫的原则,要求 `autocommit=True` 且当前没有活动事务;检查失败时把连接交还调用方处理。每个游标由创建方关闭,清理失败不得覆盖已捕获的主要异常。管理连接上依次调用创建账号、分配角色、设置默认角色,分别保存确认结果。 [隐式提交](https://dev.mysql.com/doc/refman/8.4/en/implicit-commit.html) · [现有 MySQL 结构管理实现](../src/datvia/schema/_mysql.py) ### 3.3 Redis @@ -108,7 +108,7 @@ SDK 不为账号写入增加自动重放。原生客户端沿用调用方的驱 以下示例通过 Python 直接调用管理方法。`connections` 为调用方建立的 `Connections`,原生客户端和密码参数由调用方显式提供。管理对象在持有连接的作用域内使用。 ```python -from database.accounts import Accounts, AclRule, MongoRole, MongoUser, MySQLAccount +from datvia.accounts import Accounts, AclRule, MongoRole, MongoUser, MySQLAccount def configure_mongo(connections, password): @@ -136,7 +136,7 @@ def rotate_redis_password(native_redis, new_password): Redis 示例只添加一个密码。用于已有用户的轮换时,再以新密码独立验证、迁移调用方,最后单独移除旧密码。若用于不存在的用户,该操作遵守原生创建语义,启用与授权规则需要显式传入。 ```python -from database.accounts import AsyncAccounts, MySQLAccount +from datvia.accounts import AsyncAccounts, MySQLAccount async def inspect_async(connections, native_mysql, native_redis): @@ -157,7 +157,7 @@ async def inspect_async(connections, native_mysql, native_redis): 下例使用三个调用方提供的管理连接,分别代表三种独立部署。密码由调用方在运行时传入。示例中的身份、角色名和键前缀应替换为当前隔离环境的专用目标。 ```python -from database.accounts import Accounts, AclRule, MongoPrivilege, MongoRole, MongoUser, MySQLAccount, MySQLScope +from datvia.accounts import Accounts, AclRule, MongoPrivilege, MongoRole, MongoUser, MySQLAccount, MySQLScope def grant_report_access(mongo_client, mysql_conn, redis_client, mongo_pwd, mysql_pwd, redis_pwd): mongo = Accounts.mongodb(mongo_client) @@ -219,8 +219,8 @@ def grant_report_access(mongo_client, mysql_conn, redis_client, mongo_pwd, mysql ```shell python -B -m unittest discover -s tests -p "test_accounts*.py" -v python -B -m unittest discover -s tests -q -python -m ruff check src/database/accounts -python -m ruff format --check src/database/accounts +python -m ruff check src/datvia/accounts +python -m ruff format --check src/datvia/accounts git diff --check ``` diff --git a/docs/account-management-requirements.md b/docs/account-management-requirements.md index 97855d6..22d699c 100644 --- a/docs/account-management-requirements.md +++ b/docs/account-management-requirements.md @@ -1,6 +1,6 @@ # MongoDB、MySQL、Redis 账号与权限管理需求 -本文定义 Python 工具包 `database.accounts` 的首版需求。读者是使用本库的开发者、负责数据库账号的管理员,以及负责验收的工程师。方法、类型和版本矩阵均属设计目标。开发任务、命令映射、示例和验证步骤见[实现设计与交付规划](account-management-design.md)。 +本文定义 Python 工具包 `datvia.accounts` 的首版需求。读者是使用本库的开发者、负责数据库账号的管理员,以及负责验收的工程师。方法、类型和版本矩阵均属设计目标。开发任务、命令映射、示例和验证步骤见[实现设计与交付规划](account-management-design.md)。 ## 1. 使用目标与 SDK 边界 diff --git a/docs/assets/database-badges.svg b/docs/assets/datvia-badges.svg similarity index 100% rename from docs/assets/database-badges.svg rename to docs/assets/datvia-badges.svg diff --git a/docs/assets/database-banner.svg b/docs/assets/datvia-banner.svg similarity index 95% rename from docs/assets/database-banner.svg rename to docs/assets/datvia-banner.svg index 429da3a..f729ac7 100644 --- a/docs/assets/database-banner.svg +++ b/docs/assets/datvia-banner.svg @@ -1,6 +1,6 @@ - database - The database Python SDK connects to MongoDB, MySQL, and Redis through the Connections API. + DATVIA + DATVIA connects Python applications to MongoDB, MySQL, and Redis through one API. @@ -38,7 +38,7 @@ PYTHON SDK - database + DATVIA Configure · Connect · Check · Close diff --git a/docs/connection-management-design.md b/docs/connection-management-design.md index 8d89b93..f4c865d 100644 --- a/docs/connection-management-design.md +++ b/docs/connection-management-design.md @@ -1,11 +1,11 @@ -# database 统一连接管理设计与开发流程 +# DATVIA 统一连接管理设计与开发流程 日期:2026-11-15。状态:实现已集成,实际验证范围和剩余事项见 [验收记录](connection-management-validation.md)。 基线 develop:7d38b1ffe13a17b37d0bffc15e175085dcf0bb19。 参考 temp/save:89497823698812e1c85e6c4c12af86d4281543cf。 ## 目标和边界 -公共 SDK 支持同一项目同时使用 MongoDB、MySQL、Redis,以及多个同类型连接。提供统一配置、显式检查、保留原始信息的异常转换。保留现有 database..Client 的参数、返回类型、异步方式与原生异常,不封装 CRUD、事务、数据模型,不增加自动重试、后台检查、故障转移或自建连接池。 +公共 SDK 支持同一项目同时使用 MongoDB、MySQL、Redis,以及多个同类型连接。提供统一配置、显式检查、保留原始信息的异常转换。保留现有 datvia..Client 的参数、返回类型、异步方式与原生异常,不封装 CRUD、事务、数据模型,不增加自动重试、后台检查、故障转移或自建连接池。 SDK 只接收调用方显式参数;不读取环境变量、不加载 .env、不搜索配置文件、不提供环境读取助手。调用方可自行读取私密配置并传入。原生驱动自身已有行为按显式配置及驱动规则执行。 ## 草稿复用 @@ -24,7 +24,7 @@ SDK 只接收调用方显式参数;不读取环境变量、不加载 .env、 ## 公共接口 ```python -from database import ConnectionConfig, Connections +from datvia import ConnectionConfig, Connections ConnectionConfig(backend, host=None, port=None, username=None, password=None, connect_timeout_s=None, options={}, mode="single", nodes=(), service_name=None) ``` @@ -72,10 +72,10 @@ errors 上下文只转换所属驱动的异常;业务 ValueError/TypeError、 SDK 直接调用驱动的边界可以识别明确来源的底层超时/传输异常。 ## 内部协作契约 -- database.config: DatabaseType, ConnectionConfig;snapshot() 返回配置的隔离副本;driver_kwargs() 返回共同字段映射后与 options 合并的全新参数字典。Redis nodes 转换由适配器负责,service_name 不传给驱动构造器。 -- database.errors: 上述异常;convert_error(error, *, connection_name, backend, phase, operation=None, target=None, classifier=None, owned=False) 返回统一异常或原异常;error_context 使用相同关键字并抛 from original;attach_cleanup_errors(primary, errors) 附加元组。 -- database.connections: Connections, Connection, AsyncConnection, CheckResult,运行时才 import database._adapters.。 -- database._adapters..Adapter(config, name, is_async=False):create()、async create_async() 返回原生对象;check(native)、async check_async(native) 返回(scope, targets);close(native)、async close_async(native);classify(error) 返回统一异常类或 None(非所属驱动错误)。 +- datvia.config: DatabaseType, ConnectionConfig;snapshot() 返回配置的隔离副本;driver_kwargs() 返回共同字段映射后与 options 合并的全新参数字典。Redis nodes 转换由适配器负责,service_name 不传给驱动构造器。 +- datvia.errors: 上述异常;convert_error(error, *, connection_name, backend, phase, operation=None, target=None, classifier=None, owned=False) 返回统一异常或原异常;error_context 使用相同关键字并抛 from original;attach_cleanup_errors(primary, errors) 附加元组。 +- datvia.connections: Connections, Connection, AsyncConnection, CheckResult,运行时才 import datvia._adapters.。 +- datvia._adapters..Adapter(config, name, is_async=False):create()、async create_async() 返回原生对象;check(native)、async check_async(native) 返回(scope, targets);close(native)、async close_async(native);classify(error) 返回统一异常类或 None(非所属驱动错误)。 - 适配器应在需要标注具体目标的检查异常处调用 convert_error(... owned=True, classifier=self.classify) 并 raise from;核心同样处理创建/检查/关闭外围异常。已统一错误不重复包装。 - `errors` 和配置层独立于驱动。创建连接时由对应后端加载驱动。 - 公共 __init__ 导出由协调者统一修改。worker 不改现有原生工厂文件。 diff --git a/docs/crud-adapter-contract.md b/docs/crud-adapter-contract.md index 559efcd..42f4471 100644 --- a/docs/crud-adapter-contract.md +++ b/docs/crud-adapter-contract.md @@ -4,7 +4,7 @@ ## 工厂和生命周期 -- 模块 `database.crud._mysql`、`_mongodb`、`_redis_hash` 导出同步 `Adapter` 和异步 `AsyncAdapter`。 +- 模块 `datvia.crud._mysql`、`_mongodb`、`_redis_hash` 导出同步 `Adapter` 和异步 `AsyncAdapter`。 - 构造函数为关键字参数 `connection`、`key_fields`(有序字段元组)及后端参数。MySQL 接收 `table`。MongoDB 接收 `database`、`collection`、`session`。Redis 接收 `namespace`、`codec`。构造阶段只校验配置。 - 同步类提供 `__enter__()`、`__exit__(exc_type, exc, traceback)`,异步类提供对应异步上下文方法。可返回自身,不能抑制主异常。释放适配器自行持有的资源,借用连接、客户端和 session 归调用方。 - 入口上下文可构造 Sentinel 主节点数据客户端,不额外探测。Redis 适配模块暂用内部连接适配器配置 `connection._adapter._config.service_name`,该桥接不构成公共接口。 @@ -52,4 +52,4 @@ Redis 不需要实现 `find/count/update_many/delete_where`,公共层会拒绝 已发送或可能发送的块无法判断的项留为 `unknown`。SQL 分组在块内停止时,明确未发送的组可标记 `not_sent`。读分块有已确认明细时,`BatchItem.result` 为 `ReadItem`,其 `index` 同样是分块内索引。 核心将已发送块的未知项与后续未发送项区分,保留原始异常及因果链。捕获取消时按同样方式附加部分结果后重新抛出同一个对象。适配器有已知明细时通过 `ChunkError.partial_result` 提供;保留原始取消时,可在原对象上附加 `database_partial_result`(值为 `ChunkResult`)。 -`UnsupportedOperationError`、`RecordConflictError`、`RecordFormatError`、`RecordIdentityError` 位于 `src/database/crud/_errors.py`。适配器已明确分类的异常保留原生对象和 `raise ... from ...` 关系。其他驱动异常由公共层使用连接句柄现有转换。 +`UnsupportedOperationError`、`RecordConflictError`、`RecordFormatError`、`RecordIdentityError` 位于 `src/datvia/crud/_errors.py`。适配器已明确分类的异常保留原生对象和 `raise ... from ...` 关系。其他驱动异常由公共层使用连接句柄现有转换。 diff --git a/docs/crud-design.md b/docs/crud-design.md index 2570b32..af1f629 100644 --- a/docs/crud-design.md +++ b/docs/crud-design.md @@ -1,4 +1,4 @@ -# database 基础使用封装设计与开发计划 +# DATVIA 基础使用封装设计与开发计划 日期:2026-11-16 @@ -27,7 +27,7 @@ - 初始化由调用方显式执行,CRUD 不自动建表、改表、建立索引或迁移结构。 - SDK 不读取环境变量、`.env` 或其他隐式配置来源。 -数据库服务、集群部署和初始管理凭据由调用方准备。后续账号与授权管理由独立的 [database.accounts 工具](account-management-design.md) 提供。 +数据库服务、集群部署和初始管理凭据由调用方准备。后续账号与授权管理由独立的 [datvia.accounts 工具](account-management-design.md) 提供。 ### 1.2 三种数据库的基础封装边界 @@ -67,13 +67,13 @@ Redis 不增加虚构的表或集合。编号库由部署提供,Cluster 仅支 ### 2.1 初始化与基础结构管理 -新增 `database.schema`,提供: +新增 `datvia.schema`,提供: ```python -from database.schema import AsyncSchema -from database.schema import Column -from database.schema import Schema -from database.schema import TableSpec +from datvia.schema import AsyncSchema +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import TableSpec ``` 绑定入口为: @@ -265,12 +265,12 @@ MySQL 的 `update_many` 使用条件 UPDATE;`bulk_update` 按记录执行参 ### 2.3 Redis String -在 `database.crud` 中新增: +在 `datvia.crud` 中新增: ```python -from database.crud import AsyncRedisStrings -from database.crud import RedisStrings -from database.crud import StringSetItem +from datvia.crud import AsyncRedisStrings +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem ``` 使用方式: @@ -357,10 +357,10 @@ Redis Hash 保持现有记录格式、显式 codec 和 TTL 约定。Hash 更新 ### 3.1 公共配置和表定义 ```python -from database import ConnectionConfig -from database import Connections -from database.schema import Column -from database.schema import TableSpec +from datvia import ConnectionConfig +from datvia import Connections +from datvia.schema import Column +from datvia.schema import TableSpec USER_TABLE = TableSpec( @@ -419,11 +419,11 @@ def make_connections( ```python from contextlib import ExitStack -from database.crud import JsonFieldCodec -from database.crud import Records -from database.crud import RedisStrings -from database.crud import StringSetItem -from database.schema import Schema +from datvia.crud import JsonFieldCodec +from datvia.crud import Records +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +from datvia.schema import Schema def run_sync(connections: Connections) -> None: @@ -539,11 +539,11 @@ def run_sync(connections: Connections) -> None: ```python from contextlib import AsyncExitStack -from database.crud import AsyncRecords -from database.crud import AsyncRedisStrings -from database.crud import JsonFieldCodec -from database.crud import StringSetItem -from database.schema import AsyncSchema +from datvia.crud import AsyncRecords +from datvia.crud import AsyncRedisStrings +from datvia.crud import JsonFieldCodec +from datvia.crud import StringSetItem +from datvia.schema import AsyncSchema async def run_async(connections: Connections) -> None: diff --git a/docs/crud-usage.md b/docs/crud-usage.md index d73abd9..75e164a 100644 --- a/docs/crud-usage.md +++ b/docs/crud-usage.md @@ -1,6 +1,6 @@ # 基础结构与记录操作使用说明 -`database.crud` 在已有连接之上提供可选的记录操作层。 +`datvia.crud` 在已有连接之上提供可选的记录操作层。 MySQL 对应表,MongoDB 对应集合,Redis 对应采用本库格式的 Hash 记录。 它们共享调用方式、字段投影和结果结构;事务、类型系统和查询能力仍遵循各数据库的真实语义。 @@ -13,9 +13,9 @@ MySQL 对应表,MongoDB 对应集合,Redis 对应采用本库格式的 Hash 先显式传入连接参数,然后准备数据库或集合,最后绑定记录对象。MySQL 初始化连接应不选择尚未存在的业务库,并显式设置 `options={"autocommit": True}`;业务连接在初始化完成后通过 `options={"database": "app", "autocommit": True}` 选择目标库。MongoDB 使用显式集合准备;Redis 使用配置中的编号库以及 Hash 命名空间或 String 键,不创建虚构的表。 ```python -from database.schema import Column -from database.schema import Schema -from database.schema import TableSpec +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import TableSpec users = TableSpec( columns={ @@ -56,7 +56,7 @@ MongoDB 提供 `ensure_collection`、`list_databases`、`database_exists`、`lis 配置始终由调用方显式传入;记录层也不会读取环境变量或 `.env`。 ```python -from database.crud import Records +from datvia.crud import Records def read_orders(connections): with connections.create('orders') as connection: @@ -92,7 +92,7 @@ MongoDB 插入必须提供除可由驱动生成的 `_id` 之外的所有声明 异步方法调用原生异步驱动,参数与返回结构和同步方法一致。 ```python -from database.crud import AsyncRecords +from datvia.crud import AsyncRecords async def read_documents(connections, document_id): async with await connections.create_async('documents') as connection: @@ -166,9 +166,9 @@ MongoDB 当前对每个去重后的键执行一次 `limit=2` 的有界查询, 只支持普通标量条件;原生数组、嵌套路径、SQL 表达式和聚合继续使用原生客户端。 ```python -from database.crud import And -from database.crud import Gte -from database.crud import In +from datvia.crud import And +from datvia.crud import Gte +from datvia.crud import In pending = orders.find( where=And(In('status', ['pending', 'paid']), Gte('total', 100)), @@ -230,7 +230,7 @@ MySQL 不根据 `lastrowid` 推算一批连续主键。 | `unknown` | 可能已经发送,现有证据无法确定结果 | ```python -from database.crud import BatchOperationError +from datvia.crud import BatchOperationError try: result = orders.bulk_update(changes, batch_size=500) @@ -249,8 +249,8 @@ except BatchOperationError as error: ## 7. Redis Hash 格式与边界 ```python -from database.crud import InsertItem -from database.crud import JsonFieldCodec +from datvia.crud import InsertItem +from datvia.crud import JsonFieldCodec with connections.create('cache') as connection: with Records.redis_hash( @@ -321,8 +321,8 @@ Redis 其他结构、MongoDB 聚合、SQL JOIN、数据库专有原子操作都 `RedisStrings` / `AsyncRedisStrings` 借用既有 Redis 连接,支持单机、Sentinel 和 Cluster。String 使用调用方提供的完整键,不自动加前缀,也不解析 JSON。 ```python -from database.crud import RedisStrings -from database.crud import StringSetItem +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem with connections.create("cache") as connection: with RedisStrings(connection) as strings: @@ -337,8 +337,8 @@ with connections.create("cache") as connection: ``` ```python -from database.crud import AsyncRedisStrings -from database.crud import StringSetItem +from datvia.crud import AsyncRedisStrings +from datvia.crud import StringSetItem async with await connections.create_async("cache") as connection: async with AsyncRedisStrings(connection) as strings: diff --git a/examples/basic_usage.py b/examples/basic_usage.py index ff431fe..b6fdae0 100644 --- a/examples/basic_usage.py +++ b/examples/basic_usage.py @@ -3,18 +3,18 @@ from contextlib import AsyncExitStack from contextlib import ExitStack -from database import ConnectionConfig -from database import Connections -from database.crud import AsyncRecords -from database.crud import AsyncRedisStrings -from database.crud import JsonFieldCodec -from database.crud import Records -from database.crud import RedisStrings -from database.crud import StringSetItem -from database.schema import AsyncSchema -from database.schema import Column -from database.schema import Schema -from database.schema import TableSpec +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRecords +from datvia.crud import AsyncRedisStrings +from datvia.crud import JsonFieldCodec +from datvia.crud import Records +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +from datvia.schema import AsyncSchema +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import TableSpec USER_TABLE = TableSpec( columns={ diff --git a/pyproject.toml b/pyproject.toml index 8079248..4914787 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,9 @@ requires = [ build-backend = "setuptools.build_meta" [project] -name = "database" +name = "datvia" dynamic = ["version"] -description = "database SDK" +description = "Data via one Python API." readme = "README.md" license = { text = "MIT" } authors = [ @@ -38,4 +38,4 @@ Homepage = "https://gitee.com/loongspears/database" Repository = "https://gitee.com/loongspears/database" [tool.setuptools_scm] -write_to = "src/database/_version.py" +write_to = "src/datvia/_version.py" diff --git a/scripts/verify_accounts_mongodb.py b/scripts/verify_accounts_mongodb.py index 960eaba..72ab80f 100644 --- a/scripts/verify_accounts_mongodb.py +++ b/scripts/verify_accounts_mongodb.py @@ -14,17 +14,17 @@ from uuid import uuid4 import pymongo from pymongo.errors import OperationFailure -from database import ConnectionConfig -from database import Connections -from database.accounts import Accounts -from database.accounts import AccountWriteOutcomeUnknown -from database.accounts import AsyncAccounts -from database.accounts import MongoPrivilege -from database.accounts import MongoRole -from database.accounts import MongoUser -from database.errors import DriverError -from database.errors import PermissionDeniedError -from database.errors import attach_cleanup_errors +from datvia import ConnectionConfig +from datvia import Connections +from datvia.accounts import Accounts +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import AsyncAccounts +from datvia.accounts import MongoPrivilege +from datvia.accounts import MongoRole +from datvia.accounts import MongoUser +from datvia.errors import DriverError +from datvia.errors import PermissionDeniedError +from datvia.errors import attach_cleanup_errors def settings(path: Path) -> tuple[dict, ConnectionConfig]: diff --git a/scripts/verify_accounts_mysql.py b/scripts/verify_accounts_mysql.py index efc7842..b55dba0 100644 --- a/scripts/verify_accounts_mysql.py +++ b/scripts/verify_accounts_mysql.py @@ -16,17 +16,17 @@ import mysql.connector from mysql.connector import aio from mysql.connector import errors -from database import ConnectionConfig -from database import Connections -from database.accounts import Accounts -from database.accounts import AccountWriteOutcomeUnknown -from database.accounts import AsyncAccounts -from database.accounts import MySQLAccount -from database.accounts import MySQLScope -from database.errors import ConfigurationError -from database.errors import DependencyError -from database.errors import PermissionDeniedError -from database.errors import attach_cleanup_errors +from datvia import ConnectionConfig +from datvia import Connections +from datvia.accounts import Accounts +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import AsyncAccounts +from datvia.accounts import MySQLAccount +from datvia.accounts import MySQLScope +from datvia.errors import ConfigurationError +from datvia.errors import DependencyError +from datvia.errors import PermissionDeniedError +from datvia.errors import attach_cleanup_errors def statement(conn, sql: str, args: tuple = ()) -> list[tuple]: diff --git a/scripts/verify_accounts_redis.py b/scripts/verify_accounts_redis.py index 8df78c4..5d3011e 100644 --- a/scripts/verify_accounts_redis.py +++ b/scripts/verify_accounts_redis.py @@ -21,14 +21,14 @@ from pathlib import Path import redis import redis.asyncio -from database import ConnectionConfig -from database import Connections -from database.accounts import Accounts -from database.accounts import AclRule -from database.accounts import AsyncAccounts -from database.errors import DependencyError -from database.errors import DriverError -from database.errors import PermissionDeniedError +from datvia import ConnectionConfig +from datvia import Connections +from datvia.accounts import Accounts +from datvia.accounts import AclRule +from datvia.accounts import AsyncAccounts +from datvia.errors import DependencyError +from datvia.errors import DriverError +from datvia.errors import PermissionDeniedError def require(condition: bool, message: str) -> None: diff --git a/scripts/verify_basics.py b/scripts/verify_basics.py index 83c6f93..4ac879b 100644 --- a/scripts/verify_basics.py +++ b/scripts/verify_basics.py @@ -25,19 +25,19 @@ from traceback import extract_tb from typing import Any from uuid import uuid4 -from database import ConnectionConfig -from database import Connections -from database.crud import AsyncRecords -from database.crud import AsyncRedisStrings -from database.crud import JsonFieldCodec -from database.crud import Records -from database.crud import RedisStrings -from database.crud import StringSetItem -from database.errors import attach_cleanup_errors -from database.schema import AsyncSchema -from database.schema import Column -from database.schema import Schema -from database.schema import TableSpec +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRecords +from datvia.crud import AsyncRedisStrings +from datvia.crud import JsonFieldCodec +from datvia.crud import Records +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +from datvia.errors import attach_cleanup_errors +from datvia.schema import AsyncSchema +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import TableSpec TABLE = TableSpec( columns={'id': Column('varchar', length=32), 'name': Column('varchar', length=100), 'age': Column('int')}, diff --git a/scripts/verify_connections.py b/scripts/verify_connections.py index 06a15f5..e82400b 100644 --- a/scripts/verify_connections.py +++ b/scripts/verify_connections.py @@ -10,7 +10,7 @@ import json from contextlib import AsyncExitStack, ExitStack from pathlib import Path -from database import ConnectionConfig, Connections +from datvia import ConnectionConfig, Connections def verify_sync(connections, name, backend): diff --git a/src/database/__init__.py b/src/datvia/__init__.py similarity index 89% rename from src/database/__init__.py rename to src/datvia/__init__.py index 01038c3..0a7783a 100644 --- a/src/database/__init__.py +++ b/src/datvia/__init__.py @@ -3,7 +3,7 @@ try: except ImportError: __version__ = '0.0.0+unknown' -__description__ = 'database SDK' +__description__ = 'Data via one Python API.' from .config import ConnectionConfig, DatabaseType from .connections import AsyncConnection, CheckResult, Connection, Connections diff --git a/src/database/_adapters/__init__.py b/src/datvia/_adapters/__init__.py similarity index 100% rename from src/database/_adapters/__init__.py rename to src/datvia/_adapters/__init__.py diff --git a/src/database/_adapters/mongodb.py b/src/datvia/_adapters/mongodb.py similarity index 100% rename from src/database/_adapters/mongodb.py rename to src/datvia/_adapters/mongodb.py diff --git a/src/database/_adapters/mysql.py b/src/datvia/_adapters/mysql.py similarity index 100% rename from src/database/_adapters/mysql.py rename to src/datvia/_adapters/mysql.py diff --git a/src/database/_adapters/redis.py b/src/datvia/_adapters/redis.py similarity index 100% rename from src/database/_adapters/redis.py rename to src/datvia/_adapters/redis.py diff --git a/src/database/accounts/__init__.py b/src/datvia/accounts/__init__.py similarity index 100% rename from src/database/accounts/__init__.py rename to src/datvia/accounts/__init__.py diff --git a/src/database/accounts/_base.py b/src/datvia/accounts/_base.py similarity index 98% rename from src/database/accounts/_base.py rename to src/datvia/accounts/_base.py index a7665c0..80e5677 100644 --- a/src/database/accounts/_base.py +++ b/src/datvia/accounts/_base.py @@ -77,7 +77,7 @@ class Binding: if not _matches_native(connection, backend, is_async): raise invalid self._client = connection - self._classify = import_module(f'database._adapters.{backend}').Adapter.classify + self._classify = import_module(f'datvia._adapters.{backend}').Adapter.classify if backend == 'redis' and not _fixed_redis_node(self._client, is_async): raise invalid except ImportError as original: diff --git a/src/database/accounts/_mongodb.py b/src/datvia/accounts/_mongodb.py similarity index 100% rename from src/database/accounts/_mongodb.py rename to src/datvia/accounts/_mongodb.py diff --git a/src/database/accounts/_mysql.py b/src/datvia/accounts/_mysql.py similarity index 100% rename from src/database/accounts/_mysql.py rename to src/datvia/accounts/_mysql.py diff --git a/src/database/accounts/_mysql_models.py b/src/datvia/accounts/_mysql_models.py similarity index 100% rename from src/database/accounts/_mysql_models.py rename to src/datvia/accounts/_mysql_models.py diff --git a/src/database/accounts/_redis.py b/src/datvia/accounts/_redis.py similarity index 100% rename from src/database/accounts/_redis.py rename to src/datvia/accounts/_redis.py diff --git a/src/database/config.py b/src/datvia/config.py similarity index 100% rename from src/database/config.py rename to src/datvia/config.py diff --git a/src/database/connections.py b/src/datvia/connections.py similarity index 98% rename from src/database/connections.py rename to src/datvia/connections.py index bbf2ffa..a5f05f2 100644 --- a/src/database/connections.py +++ b/src/datvia/connections.py @@ -172,7 +172,7 @@ class Connections: def _adapter(self, config, name, is_async): backend = config.backend.value try: - module = importlib.import_module("database._adapters." + backend) + module = importlib.import_module("datvia._adapters." + backend) adapter = module.Adapter(config, name, is_async=is_async) except ImportError as original: # Imports happen only after a backend is explicitly selected. diff --git a/src/database/crud/__init__.py b/src/datvia/crud/__init__.py similarity index 100% rename from src/database/crud/__init__.py rename to src/datvia/crud/__init__.py diff --git a/src/database/crud/_batch.py b/src/datvia/crud/_batch.py similarity index 100% rename from src/database/crud/_batch.py rename to src/datvia/crud/_batch.py diff --git a/src/database/crud/_codecs.py b/src/datvia/crud/_codecs.py similarity index 100% rename from src/database/crud/_codecs.py rename to src/datvia/crud/_codecs.py diff --git a/src/database/crud/_conditions.py b/src/datvia/crud/_conditions.py similarity index 100% rename from src/database/crud/_conditions.py rename to src/datvia/crud/_conditions.py diff --git a/src/database/crud/_errors.py b/src/datvia/crud/_errors.py similarity index 100% rename from src/database/crud/_errors.py rename to src/datvia/crud/_errors.py diff --git a/src/database/crud/_mongodb.py b/src/datvia/crud/_mongodb.py similarity index 100% rename from src/database/crud/_mongodb.py rename to src/datvia/crud/_mongodb.py diff --git a/src/database/crud/_mysql.py b/src/datvia/crud/_mysql.py similarity index 100% rename from src/database/crud/_mysql.py rename to src/datvia/crud/_mysql.py diff --git a/src/database/crud/_records.py b/src/datvia/crud/_records.py similarity index 99% rename from src/database/crud/_records.py rename to src/datvia/crud/_records.py index 4a36936..18f6ed7 100644 --- a/src/database/crud/_records.py +++ b/src/datvia/crud/_records.py @@ -127,7 +127,7 @@ class _Records: connection._ensure_open('operation') names = _names(key_fields, empty=False) try: - module = importlib.import_module(f'database.crud._{adapter_name}') + module = importlib.import_module(f'datvia.crud._{adapter_name}') except ImportError as error: raise DependencyError( 'The CRUD adapter or its database driver is unavailable.', diff --git a/src/database/crud/_redis_hash.py b/src/datvia/crud/_redis_hash.py similarity index 100% rename from src/database/crud/_redis_hash.py rename to src/datvia/crud/_redis_hash.py diff --git a/src/database/crud/_string_types.py b/src/datvia/crud/_string_types.py similarity index 100% rename from src/database/crud/_string_types.py rename to src/datvia/crud/_string_types.py diff --git a/src/database/crud/_strings.py b/src/datvia/crud/_strings.py similarity index 100% rename from src/database/crud/_strings.py rename to src/datvia/crud/_strings.py diff --git a/src/database/crud/_types.py b/src/datvia/crud/_types.py similarity index 100% rename from src/database/crud/_types.py rename to src/datvia/crud/_types.py diff --git a/src/database/errors.py b/src/datvia/errors.py similarity index 100% rename from src/database/errors.py rename to src/datvia/errors.py diff --git a/src/database/mongodb/__init__.py b/src/datvia/mongodb/__init__.py similarity index 100% rename from src/database/mongodb/__init__.py rename to src/datvia/mongodb/__init__.py diff --git a/src/database/mongodb/_async_utils.py b/src/datvia/mongodb/_async_utils.py similarity index 100% rename from src/database/mongodb/_async_utils.py rename to src/datvia/mongodb/_async_utils.py diff --git a/src/database/mongodb/_batch.py b/src/datvia/mongodb/_batch.py similarity index 100% rename from src/database/mongodb/_batch.py rename to src/datvia/mongodb/_batch.py diff --git a/src/database/mongodb/_cursor_batches.py b/src/datvia/mongodb/_cursor_batches.py similarity index 100% rename from src/database/mongodb/_cursor_batches.py rename to src/datvia/mongodb/_cursor_batches.py diff --git a/src/database/mongodb/_resources.py b/src/datvia/mongodb/_resources.py similarity index 100% rename from src/database/mongodb/_resources.py rename to src/datvia/mongodb/_resources.py diff --git a/src/database/mongodb/_scan.py b/src/datvia/mongodb/_scan.py similarity index 100% rename from src/database/mongodb/_scan.py rename to src/datvia/mongodb/_scan.py diff --git a/src/database/mongodb/_sync_utils.py b/src/datvia/mongodb/_sync_utils.py similarity index 100% rename from src/database/mongodb/_sync_utils.py rename to src/datvia/mongodb/_sync_utils.py diff --git a/src/database/mongodb/client.py b/src/datvia/mongodb/client.py similarity index 100% rename from src/database/mongodb/client.py rename to src/datvia/mongodb/client.py diff --git a/src/database/mongodb/types.py b/src/datvia/mongodb/types.py similarity index 100% rename from src/database/mongodb/types.py rename to src/datvia/mongodb/types.py diff --git a/src/database/mongodb/utils.py b/src/datvia/mongodb/utils.py similarity index 100% rename from src/database/mongodb/utils.py rename to src/datvia/mongodb/utils.py diff --git a/src/database/mysql/__init__.py b/src/datvia/mysql/__init__.py similarity index 100% rename from src/database/mysql/__init__.py rename to src/datvia/mysql/__init__.py diff --git a/src/database/mysql/client.py b/src/datvia/mysql/client.py similarity index 100% rename from src/database/mysql/client.py rename to src/datvia/mysql/client.py diff --git a/src/database/redis/__init__.py b/src/datvia/redis/__init__.py similarity index 100% rename from src/database/redis/__init__.py rename to src/datvia/redis/__init__.py diff --git a/src/database/redis/client.py b/src/datvia/redis/client.py similarity index 100% rename from src/database/redis/client.py rename to src/datvia/redis/client.py diff --git a/src/database/schema/__init__.py b/src/datvia/schema/__init__.py similarity index 100% rename from src/database/schema/__init__.py rename to src/datvia/schema/__init__.py diff --git a/src/database/schema/_base.py b/src/datvia/schema/_base.py similarity index 100% rename from src/database/schema/_base.py rename to src/datvia/schema/_base.py diff --git a/src/database/schema/_models.py b/src/datvia/schema/_models.py similarity index 100% rename from src/database/schema/_models.py rename to src/datvia/schema/_models.py diff --git a/src/database/schema/_mongodb.py b/src/datvia/schema/_mongodb.py similarity index 100% rename from src/database/schema/_mongodb.py rename to src/datvia/schema/_mongodb.py diff --git a/src/database/schema/_mysql.py b/src/datvia/schema/_mysql.py similarity index 100% rename from src/database/schema/_mysql.py rename to src/datvia/schema/_mysql.py diff --git a/tests/README.md b/tests/README.md index 74f40ee..3eff32a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -9,11 +9,11 @@ 按 [贡献指南](../CONTRIBUTING.md) 在独立环境运行 `python -m pip install -e .`,以 editable 方式安装当前工作区及三个后端的驱动。从仓库根目录运行以下命令。 ```bash -python -B -c "import database; print(database.__file__)" +python -B -c "import datvia; print(datvia.__file__)" python -B -m unittest discover -s tests -q ``` -源码回归时,第一条命令的路径应指向当前工作区的 `src/database/__init__.py`。自动测试使用固定数据、驱动替身或打桩的原生驱动,保持离线。 +源码回归时,第一条命令的路径应指向当前工作区的 `src/datvia/__init__.py`。自动测试使用固定数据、驱动替身或打桩的原生驱动,保持离线。 ## 按改动范围定位测试 diff --git a/tests/test_accounts_binding.py b/tests/test_accounts_binding.py index 334728e..c0ba90e 100644 --- a/tests/test_accounts_binding.py +++ b/tests/test_accounts_binding.py @@ -8,20 +8,20 @@ import unittest from pathlib import Path from types import SimpleNamespace -from database.accounts import AccountWriteOutcomeUnknown -from database.accounts._base import Binding -from database.connections import AsyncConnection -from database.connections import Connection -from database.errors import ConfigurationError -from database.errors import ConnectionClosedError -from database.errors import DatabaseConnectionError +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts._base import Binding +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import ConnectionClosedError +from datvia.errors import DatabaseConnectionError class BindingTests(unittest.TestCase): def test_import_without_drivers(self): source = str(Path(__file__).resolve().parents[1] / 'src') code = ( - 'import sys; from database.accounts import *; ' + 'import sys; from datvia.accounts import *; ' "assert not {'pymongo', 'redis', 'mysql.connector'}.intersection(sys.modules)" ) subprocess.run([sys.executable, '-B', '-S', '-c', code], env={**os.environ, 'PYTHONPATH': source}, check=True) diff --git a/tests/test_accounts_mongodb.py b/tests/test_accounts_mongodb.py index 6bf4482..e68ea9c 100644 --- a/tests/test_accounts_mongodb.py +++ b/tests/test_accounts_mongodb.py @@ -6,17 +6,17 @@ import unittest from unittest.mock import AsyncMock from unittest.mock import MagicMock -from database.accounts import Accounts -from database.accounts import AccountWriteOutcomeUnknown -from database.accounts import AsyncAccounts -from database.accounts import MongoPrivilege -from database.accounts import MongoRole -from database.accounts import MongoUser -from database.connections import AsyncConnection -from database.connections import Connection -from database.errors import ConfigurationError -from database.errors import DatabaseTimeoutError -from database.errors import PermissionDeniedError +from datvia.accounts import Accounts +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import AsyncAccounts +from datvia.accounts import MongoPrivilege +from datvia.accounts import MongoRole +from datvia.accounts import MongoUser +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import DatabaseTimeoutError +from datvia.errors import PermissionDeniedError HAS_DRIVER = importlib.util.find_spec('pymongo') is not None @@ -24,7 +24,7 @@ HAS_DRIVER = importlib.util.find_spec('pymongo') is not None def fixture(asynchronous=False): from pymongo.write_concern import WriteConcern - from database._adapters.mongodb import Adapter + from datvia._adapters.mongodb import Adapter client = MagicMock() database = client.__getitem__.return_value diff --git a/tests/test_accounts_mysql.py b/tests/test_accounts_mysql.py index f1cd79f..c055d5c 100644 --- a/tests/test_accounts_mysql.py +++ b/tests/test_accounts_mysql.py @@ -6,18 +6,18 @@ from types import SimpleNamespace from unittest.mock import AsyncMock from unittest.mock import MagicMock -from database.accounts import AccountWriteOutcomeUnknown -from database.accounts import MySQLAccount -from database.accounts import MySQLScope -from database.accounts._base import Binding -from database.accounts._mysql import AsyncMySQLAccounts -from database.accounts._mysql import MySQLAccounts -from database.accounts._mysql_models import PrivilegeChange -from database.connections import AsyncConnection -from database.connections import Connection -from database.errors import ConfigurationError -from database.errors import DatabaseConnectionError -from database.errors import DependencyError +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import MySQLAccount +from datvia.accounts import MySQLScope +from datvia.accounts._base import Binding +from datvia.accounts._mysql import AsyncMySQLAccounts +from datvia.accounts._mysql import MySQLAccounts +from datvia.accounts._mysql_models import PrivilegeChange +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import DatabaseConnectionError +from datvia.errors import DependencyError class MySQLAccountTests(unittest.TestCase): diff --git a/tests/test_accounts_redis.py b/tests/test_accounts_redis.py index eb50248..174ccc1 100644 --- a/tests/test_accounts_redis.py +++ b/tests/test_accounts_redis.py @@ -6,16 +6,16 @@ from types import SimpleNamespace from unittest.mock import AsyncMock from unittest.mock import Mock -from database.accounts import Accounts -from database.accounts import AccountWriteOutcomeUnknown -from database.accounts import AclRule -from database.accounts._base import Binding -from database.accounts._redis import AsyncRedisAccounts -from database.accounts._redis import RedisAccounts -from database.connections import Connection -from database.errors import ConfigurationError -from database.errors import DatabaseConnectionError -from database.errors import DriverError +from datvia.accounts import Accounts +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import AclRule +from datvia.accounts._base import Binding +from datvia.accounts._redis import AsyncRedisAccounts +from datvia.accounts._redis import RedisAccounts +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import DatabaseConnectionError +from datvia.errors import DriverError def fake_binding(native): @@ -124,7 +124,7 @@ class RedisRulesTests(unittest.TestCase): acl = Accounts.redis(native) native.execute_command.side_effect = redis.ResponseError('Syntax error') native.info.return_value = {'redis_version': '6.0.20'} - from database.errors import DependencyError + from datvia.errors import DependencyError with self.assertRaises(DependencyError) as caught: acl.apply_user('x', rules=[AclRule.channel_pattern('c:*')]) @@ -134,7 +134,7 @@ class RedisRulesTests(unittest.TestCase): with self.assertRaises(DriverError): acl.apply_user('x', rules=[AclRule.channel_pattern('c:*')]) native.info.reset_mock() - from database.errors import PermissionDeniedError + from datvia.errors import PermissionDeniedError native.execute_command.side_effect = redis.exceptions.NoPermissionError('no SETUSER') with self.assertRaises(PermissionDeniedError): diff --git a/tests/test_connection_config.py b/tests/test_connection_config.py index a82530a..4ecdad3 100644 --- a/tests/test_connection_config.py +++ b/tests/test_connection_config.py @@ -3,8 +3,8 @@ from types import MappingProxyType import unittest from unittest.mock import patch -from database.config import ConnectionConfig, DatabaseType -from database.errors import ConfigurationError +from datvia.config import ConnectionConfig, DatabaseType +from datvia.errors import ConfigurationError class ConnectionConfigTests(unittest.TestCase): @@ -27,12 +27,12 @@ class ConnectionConfigTests(unittest.TestCase): def test_mysql_accepts_a_socket_target_on_posix(self): options = {"unix_socket": "/tmp/mysql.sock", "db": "app"} - with patch("database.config.os.name", "posix"): + with patch("datvia.config.os.name", "posix"): self.assertEqual(ConnectionConfig("mysql", options=options).driver_kwargs(), options) def test_mysql_socket_without_a_host_does_not_fall_back_on_windows(self): options = {"unix_socket": "/tmp/mysql.sock", "db": "app"} - with patch("database.config.os.name", "nt"): + with patch("datvia.config.os.name", "nt"): with self.assertRaisesRegex(ConfigurationError, "explicit connection target"): ConnectionConfig("mysql", options=options) diff --git a/tests/test_connection_errors.py b/tests/test_connection_errors.py index a7a3cfb..b28f10d 100644 --- a/tests/test_connection_errors.py +++ b/tests/test_connection_errors.py @@ -4,7 +4,7 @@ import socket from unittest import IsolatedAsyncioTestCase, TestCase from unittest.mock import Mock -from database.errors import ( +from datvia.errors import ( AuthenticationError, ConfigurationError, ConnectionClosedError, diff --git a/tests/test_connections.py b/tests/test_connections.py index aa9bf60..62226f2 100644 --- a/tests/test_connections.py +++ b/tests/test_connections.py @@ -5,9 +5,9 @@ from datetime import timezone from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch -from database.config import ConnectionConfig -from database.connections import Connections -from database.errors import ( +from datvia.config import ConnectionConfig +from datvia.connections import Connections +from datvia.errors import ( ConfigurationError, ConnectionClosedError, DependencyError, DriverError, ) @@ -35,7 +35,7 @@ class ConnectionTests(unittest.TestCase): "cache": ConnectionConfig(backend="redis", host="localhost"), }) self.loader = patch( - "database.connections.importlib.import_module", + "datvia.connections.importlib.import_module", return_value=SimpleNamespace(Adapter=adapter_factory), ) self.loader.start() @@ -134,7 +134,7 @@ class ConnectionTests(unittest.TestCase): def test_import_failure_has_context_and_original(self): original = ModuleNotFoundError("driver") - with patch("database.connections.importlib.import_module", side_effect=original): + with patch("datvia.connections.importlib.import_module", side_effect=original): with self.assertRaises(DependencyError) as caught: self.registry.create("main") self.assertIs(caught.exception.original, original) @@ -145,7 +145,7 @@ class AsyncConnectionTests(unittest.IsolatedAsyncioTestCase): async def asyncSetUp(self): self.registry = Connections({"main": ConnectionConfig("mysql", host="localhost")}) self.loader = patch( - "database.connections.importlib.import_module", + "datvia.connections.importlib.import_module", return_value=SimpleNamespace(Adapter=adapter_factory), ) self.loader.start() diff --git a/tests/test_crud_conditions.py b/tests/test_crud_conditions.py index 11d5bdd..8a33a0f 100644 --- a/tests/test_crud_conditions.py +++ b/tests/test_crud_conditions.py @@ -2,16 +2,16 @@ import unittest -from database.crud._conditions import And -from database.crud._conditions import Eq -from database.crud._conditions import Exists -from database.crud._conditions import Gt -from database.crud._conditions import In -from database.crud._conditions import IsNull -from database.crud._conditions import Or -from database.crud._conditions import normalize_where -from database.crud._errors import UnsupportedOperationError -from database.errors import ConfigurationError +from datvia.crud._conditions import And +from datvia.crud._conditions import Eq +from datvia.crud._conditions import Exists +from datvia.crud._conditions import Gt +from datvia.crud._conditions import In +from datvia.crud._conditions import IsNull +from datvia.crud._conditions import Or +from datvia.crud._conditions import normalize_where +from datvia.crud._errors import UnsupportedOperationError +from datvia.errors import ConfigurationError class ConditionTests(unittest.TestCase): @@ -20,7 +20,7 @@ class ConditionTests(unittest.TestCase): from bson.decimal128 import Decimal128 except ImportError: self.skipTest('PyMongo is needed to test its native Decimal128 type.') - from database.crud._batch import unique_keys + from datvia.crud._batch import unique_keys identity = Decimal128('1234567890.123456789012345678') condition = normalize_where({'amount': identity}, 'mongodb', 'find') @@ -37,7 +37,7 @@ class ConditionTests(unittest.TestCase): from bson.int64 import Int64 except ImportError: self.skipTest('PyMongo is needed to test its native Int64 type.') - from database.crud._batch import unique_keys + from datvia.crud._batch import unique_keys identity = Int64(2**40) condition = normalize_where({'_id': identity}, 'mongodb', 'find') diff --git a/tests/test_crud_core.py b/tests/test_crud_core.py index df781b5..95c6097 100644 --- a/tests/test_crud_core.py +++ b/tests/test_crud_core.py @@ -10,26 +10,26 @@ from unittest.mock import AsyncMock from unittest.mock import Mock from unittest.mock import patch -from database import AsyncConnection -from database import Connection -from database.crud import AsyncRecords -from database.crud import BatchItem -from database.crud import BatchOperationError -from database.crud import BatchState -from database.crud import Eq -from database.crud import InsertItem -from database.crud import ReadItem -from database.crud import Records -from database.crud import UnsupportedOperationError -from database.crud import WriteCounts -from database.crud import WriteResult -from database.crud._types import UNSET -from database.crud._types import ChunkError -from database.crud._types import ChunkResult -from database.errors import ConfigurationError -from database.errors import ConnectionClosedError -from database.errors import DependencyError -from database.errors import DriverError +from datvia import AsyncConnection +from datvia import Connection +from datvia.crud import AsyncRecords +from datvia.crud import BatchItem +from datvia.crud import BatchOperationError +from datvia.crud import BatchState +from datvia.crud import Eq +from datvia.crud import InsertItem +from datvia.crud import ReadItem +from datvia.crud import Records +from datvia.crud import UnsupportedOperationError +from datvia.crud import WriteCounts +from datvia.crud import WriteResult +from datvia.crud._types import UNSET +from datvia.crud._types import ChunkError +from datvia.crud._types import ChunkResult +from datvia.errors import ConfigurationError +from datvia.errors import ConnectionClosedError +from datvia.errors import DependencyError +from datvia.errors import DriverError class FakeFailure(Exception): @@ -63,7 +63,7 @@ def raise_cancelled_chunk(error, cause, result, cleanup): def bind(connection, adapter, *, asynchronous=False): factory = Mock(return_value=adapter) with patch( - 'database.crud._records.importlib.import_module', + 'datvia.crud._records.importlib.import_module', return_value=SimpleNamespace(Adapter=factory, AsyncAdapter=factory), ) as load: cls = AsyncRecords if asynchronous else Records @@ -72,7 +72,7 @@ def bind(connection, adapter, *, asynchronous=False): else: resource = cls.mysql(connection, table='orders', key_fields=['id']) expected_module = '_redis_hash' if connection.backend == 'redis' else '_mysql' - load.assert_called_once_with(f'database.crud.{expected_module}') + load.assert_called_once_with(f'datvia.crud.{expected_module}') assert factory.call_args.kwargs['connection'] is connection assert factory.call_args.kwargs['key_fields'] == ('id',) if asynchronous: @@ -95,7 +95,7 @@ class RejectDriverImports(importlib.abc.MetaPathFinder): sys.meta_path.insert(0, RejectDriverImports()) sys.path.insert(0, sys.argv[1]) -from database.crud import AsyncRecords, Records, WriteResult +from datvia.crud import AsyncRecords, Records, WriteResult """ source = Path(__file__).resolve().parents[1] / 'src' result = subprocess.run( @@ -110,7 +110,7 @@ from database.crud import AsyncRecords, Records, WriteResult def test_binding_reports_unavailable_driver_with_original_import_error(self): connection = Connection(object(), connection_adapter(), 'orders', 'mysql', 'single') original = ModuleNotFoundError('The requested driver is unavailable.') - with patch('database.crud._records.importlib.import_module', side_effect=original): + with patch('datvia.crud._records.importlib.import_module', side_effect=original): with self.assertRaises(DependencyError) as caught: Records.mysql(connection, table='orders', key_fields=['id']) failure = caught.exception @@ -382,7 +382,7 @@ class RecordsTests(unittest.TestCase): self.assertIs(self.adapter.insert.call_args.args[0].ttl_s, UNSET) def test_rejects_async_binding_mismatch_before_importing_backend(self): - with patch('database.crud._records.importlib.import_module') as load: + with patch('datvia.crud._records.importlib.import_module') as load: with self.assertRaises(ConfigurationError): AsyncRecords.mysql(self.connection, table='users', key_fields=['id']) load.assert_not_called() diff --git a/tests/test_crud_integration.py b/tests/test_crud_integration.py index 4afd51e..de99783 100644 --- a/tests/test_crud_integration.py +++ b/tests/test_crud_integration.py @@ -7,11 +7,11 @@ from contextlib import ExitStack from types import SimpleNamespace from unittest.mock import patch -from database import ConnectionConfig -from database import Connections -from database.crud import AsyncRecords -from database.crud import Records -from database.errors import ConnectionClosedError +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRecords +from datvia.crud import Records +from datvia.errors import ConnectionClosedError class Native: @@ -78,12 +78,11 @@ class AsyncRecordAdapter(RecordAdapter): def modules(): result = { - f'database._adapters.{name}': SimpleNamespace(Adapter=ConnectionAdapter) - for name in ('mysql', 'mongodb', 'redis') + f'datvia._adapters.{name}': SimpleNamespace(Adapter=ConnectionAdapter) for name in ('mysql', 'mongodb', 'redis') } result.update( { - f'database.crud._{name}': SimpleNamespace(Adapter=RecordAdapter, AsyncAdapter=AsyncRecordAdapter) + f'datvia.crud._{name}': SimpleNamespace(Adapter=RecordAdapter, AsyncAdapter=AsyncRecordAdapter) for name in ('mysql', 'mongodb', 'redis_hash') } ) diff --git a/tests/test_crud_mongodb.py b/tests/test_crud_mongodb.py index 04ea7c0..51adf1c 100644 --- a/tests/test_crud_mongodb.py +++ b/tests/test_crud_mongodb.py @@ -17,23 +17,23 @@ from pymongo.results import InsertManyResult from pymongo.results import InsertOneResult from pymongo.results import UpdateResult -from database._adapters.mongodb import Adapter as ManagedMongoDB -from database.connections import AsyncConnection -from database.connections import Connection -from database.crud import And -from database.crud import AsyncRecords -from database.crud import BatchOperationError -from database.crud import BatchState -from database.crud import Eq -from database.crud import Exists -from database.crud import Gt -from database.crud import In -from database.crud import IsNull -from database.crud import RecordConflictError -from database.crud import RecordIdentityError -from database.crud import Records -from database.errors import ConfigurationError -from database.errors import DatabaseConnectionError +from datvia._adapters.mongodb import Adapter as ManagedMongoDB +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.crud import And +from datvia.crud import AsyncRecords +from datvia.crud import BatchOperationError +from datvia.crud import BatchState +from datvia.crud import Eq +from datvia.crud import Exists +from datvia.crud import Gt +from datvia.crud import In +from datvia.crud import IsNull +from datvia.crud import RecordConflictError +from datvia.crud import RecordIdentityError +from datvia.crud import Records +from datvia.errors import ConfigurationError +from datvia.errors import DatabaseConnectionError class _Cursor: diff --git a/tests/test_crud_mysql.py b/tests/test_crud_mysql.py index db19f7c..f61bddc 100644 --- a/tests/test_crud_mysql.py +++ b/tests/test_crud_mysql.py @@ -14,25 +14,25 @@ from mysql.connector.errorcode import ER_DUP_ENTRY from mysql.connector.errors import IntegrityError from mysql.connector.errors import ProgrammingError -from database import AsyncConnection -from database import Connection -from database._adapters.mysql import Adapter as ConnectionAdapter -from database.crud import And -from database.crud import AsyncRecords -from database.crud import BatchOperationError -from database.crud import BatchState -from database.crud import Eq -from database.crud import Exists -from database.crud import Gt -from database.crud import In -from database.crud import IsNull -from database.crud import Or -from database.crud import RecordConflictError -from database.crud import RecordIdentityError -from database.crud import Records -from database.crud import UnsupportedOperationError -from database.errors import ConfigurationError -from database.errors import DriverError +from datvia import AsyncConnection +from datvia import Connection +from datvia._adapters.mysql import Adapter as ConnectionAdapter +from datvia.crud import And +from datvia.crud import AsyncRecords +from datvia.crud import BatchOperationError +from datvia.crud import BatchState +from datvia.crud import Eq +from datvia.crud import Exists +from datvia.crud import Gt +from datvia.crud import In +from datvia.crud import IsNull +from datvia.crud import Or +from datvia.crud import RecordConflictError +from datvia.crud import RecordIdentityError +from datvia.crud import Records +from datvia.crud import UnsupportedOperationError +from datvia.errors import ConfigurationError +from datvia.errors import DriverError def key(index, tenant=1): diff --git a/tests/test_crud_redis.py b/tests/test_crud_redis.py index 18e09f4..daafc5b 100644 --- a/tests/test_crud_redis.py +++ b/tests/test_crud_redis.py @@ -14,23 +14,23 @@ from unittest.mock import patch import redis -from database import ConnectionConfig -from database import Connections -from database.crud import AsyncRecords -from database.crud import BatchOperationError -from database.crud import Records -from database.crud._codecs import JsonFieldCodec -from database.crud._errors import RecordConflictError -from database.crud._errors import RecordFormatError -from database.crud._errors import UnsupportedOperationError -from database.crud._redis_hash import _SCRIPT -from database.crud._redis_hash import Adapter -from database.crud._redis_hash import AsyncAdapter -from database.crud._types import UNSET -from database.crud._types import BatchState -from database.crud._types import ChunkError -from database.crud._types import WriteInput -from database.errors import ConfigurationError +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRecords +from datvia.crud import BatchOperationError +from datvia.crud import Records +from datvia.crud._codecs import JsonFieldCodec +from datvia.crud._errors import RecordConflictError +from datvia.crud._errors import RecordFormatError +from datvia.crud._errors import UnsupportedOperationError +from datvia.crud._redis_hash import _SCRIPT +from datvia.crud._redis_hash import Adapter +from datvia.crud._redis_hash import AsyncAdapter +from datvia.crud._types import UNSET +from datvia.crud._types import BatchState +from datvia.crud._types import ChunkError +from datvia.crud._types import WriteInput +from datvia.errors import ConfigurationError def _open(backend='single', **kwargs): diff --git a/tests/test_crud_strings.py b/tests/test_crud_strings.py index bd57ef9..99bc036 100644 --- a/tests/test_crud_strings.py +++ b/tests/test_crud_strings.py @@ -10,16 +10,16 @@ from unittest.mock import patch import redis -from database import ConnectionConfig -from database import Connections -from database.crud import AsyncRedisStrings -from database.crud import BatchOperationError -from database.crud import BatchState -from database.crud import RedisStrings -from database.crud import StringSetItem -from database.errors import ConfigurationError -from database.errors import ConnectionClosedError -from database.errors import DriverError +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRedisStrings +from datvia.crud import BatchOperationError +from datvia.crud import BatchState +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +from datvia.errors import ConfigurationError +from datvia.errors import ConnectionClosedError +from datvia.errors import DriverError def registry(mode='single', **options): diff --git a/tests/test_managed_mongodb.py b/tests/test_managed_mongodb.py index e7e77d3..948500b 100644 --- a/tests/test_managed_mongodb.py +++ b/tests/test_managed_mongodb.py @@ -6,8 +6,8 @@ from unittest.mock import AsyncMock, Mock, patch from pymongo import ReadPreference from pymongo import errors as mongo_errors -from database import ConnectionConfig, Connections -from database.errors import ( +from datvia import ConnectionConfig, Connections +from datvia.errors import ( AuthenticationError, ConfigurationError, DatabaseConnectionError, @@ -37,7 +37,7 @@ class MongoDBConnectionTests(TestCase): close=Mock(), ) - @patch("database.mongodb.Client.create") + @patch("datvia.mongodb.Client.create") def test_create_preserves_driver_options_and_check_uses_read_preference(self, create): create.return_value = self.native with self.connections.create("reports") as connection: @@ -62,7 +62,7 @@ class MongoDBConnectionTests(TestCase): ) self.native.close.assert_called_once_with() - @patch("database.mongodb.Client.create") + @patch("datvia.mongodb.Client.create") def test_check_translates_native_authentication_failure(self, create): create.return_value = self.native original = mongo_errors.OperationFailure( @@ -83,7 +83,7 @@ class MongoDBConnectionTests(TestCase): self.assertEqual(error.target, "selected_server") self.assertNotIn("credentials in original error", str(error)) - @patch("database.mongodb.Client.create") + @patch("datvia.mongodb.Client.create") def test_native_operations_classify_supported_driver_errors(self, create): create.return_value = self.native errors = ( @@ -104,7 +104,7 @@ class MongoDBConnectionTests(TestCase): self.assertEqual(caught.exception.phase, "operation") self.assertEqual(caught.exception.operation, "aggregate") - @patch("database.mongodb.Client.create") + @patch("datvia.mongodb.Client.create") def test_business_wrappers_are_preserved_in_operation_context(self, create): create.return_value = self.native original = RuntimeError("business rule") @@ -121,7 +121,7 @@ class AsyncMongoDBConnectionTests(IsolatedAsyncioTestCase): "reports": ConnectionConfig("mongodb", host="server.test") }) - @patch("database.mongodb.Client.create_async") + @patch("datvia.mongodb.Client.create_async") async def test_native_object_is_returned_directly_and_probe_is_awaited(self, create): class NativeClient(SimpleNamespace): def __await__(self): @@ -147,7 +147,7 @@ class AsyncMongoDBConnectionTests(IsolatedAsyncioTestCase): create.assert_called_once_with(host="server.test") native.close.assert_awaited_once_with() - @patch("database.mongodb.Client.create_async") + @patch("datvia.mongodb.Client.create_async") async def test_missing_async_driver_capability_preserves_import_error(self, create): original = ImportError("missing AsyncMongoClient") create.side_effect = original @@ -161,7 +161,7 @@ class AsyncMongoDBConnectionTests(IsolatedAsyncioTestCase): self.assertEqual(error.connection_name, "reports") self.assertIn("PyMongo >= 4.9", str(error)) - @patch("database.mongodb.Client.create_async") + @patch("datvia.mongodb.Client.create_async") async def test_ping_cancellation_is_preserved(self, create): cancellation = asyncio.CancelledError() create.return_value = SimpleNamespace( diff --git a/tests/test_managed_mysql.py b/tests/test_managed_mysql.py index fe9bae6..64f5ddc 100644 --- a/tests/test_managed_mysql.py +++ b/tests/test_managed_mysql.py @@ -4,8 +4,8 @@ from unittest.mock import AsyncMock, Mock, patch from mysql.connector import errorcode from mysql.connector import errors as driver_errors -from database import ConnectionConfig, Connections -from database.errors import ( +from datvia import ConnectionConfig, Connections +from datvia.errors import ( AuthenticationError, DatabaseConnectionError, DatabaseTimeoutError, @@ -28,7 +28,7 @@ def registry(): class ManagedMySQLTests(TestCase): - @patch("database.mysql.client.connector.connect") + @patch("datvia.mysql.client.connector.connect") def test_create_wraps_missing_driver_capability(self, connect): original = ImportError("MySQL C extension is unavailable") connect.side_effect = original @@ -41,7 +41,7 @@ class ManagedMySQLTests(TestCase): self.assertIs(error.original, original) self.assertIs(error.__cause__, original) - @patch("database.mysql.client.connector.connect") + @patch("datvia.mysql.client.connector.connect") def test_operation_preserves_unrelated_import_error(self, connect): original = ImportError("unrelated import") @@ -52,7 +52,7 @@ class ManagedMySQLTests(TestCase): self.assertIs(result.exception, original) - @patch("database.mysql.client.connector.connect") + @patch("datvia.mysql.client.connector.connect") def test_create_checks_current_endpoint_and_closes_native_connection(self, connect): native = Mock(server_host="2001:db8::1", server_port=3308, unix_socket=None) connect.return_value = native @@ -70,7 +70,7 @@ class ManagedMySQLTests(TestCase): self.assertEqual(result.scope, "current_connection") self.assertEqual(result.targets, ("[2001:db8::1]:3308",)) - @patch("database.mysql.client.connector.connect") + @patch("datvia.mysql.client.connector.connect") def test_check_preserves_wrapped_driver_error(self, connect): native = Mock(server_host="example.test", server_port=3307, unix_socket=None) connect.return_value = native @@ -90,7 +90,7 @@ class ManagedMySQLTests(TestCase): self.assertIs(error.original.__cause__, inner) native.ping.assert_called_once_with(reconnect=False) - @patch("database.mysql.client.connector.connect") + @patch("datvia.mysql.client.connector.connect") def test_classifies_only_structured_driver_failures(self, connect): cases = ( (errorcode.ER_ACCESS_DENIED_ERROR, AuthenticationError), @@ -118,7 +118,7 @@ class ManagedMySQLTests(TestCase): with handle.errors(): raise original - @patch("database.mysql.client.connector.connect") + @patch("datvia.mysql.client.connector.connect") def test_classifies_typed_transport_timeout_wrapped_by_driver(self, connect): original = driver_errors.InterfaceError(errno=errorcode.CR_CONN_HOST_ERROR) original.__cause__ = TimeoutError() @@ -131,7 +131,7 @@ class ManagedMySQLTests(TestCase): class AsyncManagedMySQLTests(IsolatedAsyncioTestCase): - @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) async def test_create_async_wraps_missing_driver_capability(self, connect): original = ImportError("async driver capability is unavailable") connect.side_effect = original @@ -144,7 +144,7 @@ class AsyncManagedMySQLTests(IsolatedAsyncioTestCase): self.assertIs(error.original, original) self.assertIs(error.__cause__, original) - @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) async def test_awaits_create_check_and_close(self, connect): native = Mock( server_host="example.test", server_port=3307, unix_socket=None, @@ -165,7 +165,7 @@ class AsyncManagedMySQLTests(IsolatedAsyncioTestCase): self.assertEqual(result.scope, "current_connection") self.assertEqual(result.targets, ("example.test:3307",)) - @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) async def test_check_failure_omits_a_non_host_target(self, connect): original = driver_errors.InterfaceError() native = Mock( diff --git a/tests/test_managed_redis.py b/tests/test_managed_redis.py index 9640c7e..25d06a0 100644 --- a/tests/test_managed_redis.py +++ b/tests/test_managed_redis.py @@ -4,8 +4,8 @@ from importlib.util import find_spec import unittest from unittest.mock import AsyncMock, Mock, call, patch -from database import ConnectionConfig, Connections -from database.errors import ( +from datvia import ConnectionConfig, Connections +from datvia.errors import ( AuthenticationError, ConfigurationError, DatabaseConnectionError, @@ -16,7 +16,7 @@ from database.errors import ( if find_spec("redis") is not None: import redis - from database._adapters.redis import Adapter + from datvia._adapters.redis import Adapter else: redis = None diff --git a/tests/test_mongodb_client.py b/tests/test_mongodb_client.py index f2b0b53..1ab3a29 100644 --- a/tests/test_mongodb_client.py +++ b/tests/test_mongodb_client.py @@ -2,7 +2,7 @@ import unittest from types import SimpleNamespace from unittest.mock import Mock, patch -from database.mongodb import Client +from datvia.mongodb import Client class MongoDBClientTests(unittest.TestCase): @@ -11,7 +11,7 @@ class MongoDBClientTests(unittest.TestCase): native_client = object() driver = SimpleNamespace(MongoClient=Mock(return_value=native_client)) - with patch("database.mongodb.client.pymongo", driver): + with patch("datvia.mongodb.client.pymongo", driver): self.assertIs(Client.create("mongodb://localhost", connect=False), native_client) with self.assertRaisesRegex(ImportError, r"PyMongo >= 4\.9"): Client.create_async("mongodb://localhost") @@ -22,7 +22,7 @@ class MongoDBClientTests(unittest.TestCase): native_client = object() driver = SimpleNamespace(AsyncMongoClient=Mock(return_value=native_client)) - with patch("database.mongodb.client.pymongo", driver): + with patch("datvia.mongodb.client.pymongo", driver): result = Client.create_async("mongodb://localhost", serverSelectionTimeoutMS=500) self.assertIs(result, native_client) @@ -32,7 +32,7 @@ class MongoDBClientTests(unittest.TestCase): error = AttributeError("driver constructor failed") driver = SimpleNamespace(AsyncMongoClient=Mock(side_effect=error)) - with patch("database.mongodb.client.pymongo", driver): + with patch("datvia.mongodb.client.pymongo", driver): with self.assertRaises(AttributeError) as caught: Client.create_async("mongodb://localhost") diff --git a/tests/test_mongodb_scan.py b/tests/test_mongodb_scan.py index fd10edb..b0df01b 100644 --- a/tests/test_mongodb_scan.py +++ b/tests/test_mongodb_scan.py @@ -10,10 +10,10 @@ from pymongo.errors import NetworkTimeout from pymongo.errors import OperationFailure from pymongo.errors import ServerSelectionTimeoutError -from database.mongodb.utils import async_scan_collection -from database.mongodb.utils import async_stream_collection -from database.mongodb.utils import scan_collection -from database.mongodb.utils import stream_collection +from datvia.mongodb.utils import async_scan_collection +from datvia.mongodb.utils import async_stream_collection +from datvia.mongodb.utils import scan_collection +from datvia.mongodb.utils import stream_collection class Cursor: diff --git a/tests/test_mongodb_utils.py b/tests/test_mongodb_utils.py index e1f6c94..428386a 100644 --- a/tests/test_mongodb_utils.py +++ b/tests/test_mongodb_utils.py @@ -11,14 +11,14 @@ from pymongo.errors import InvalidOperation from pymongo.results import BulkWriteResult from pymongo.results import InsertManyResult -from database.mongodb.utils import AsyncBatchInsert -from database.mongodb.utils import AsyncBatchUpdate -from database.mongodb.utils import AsyncBatchWrite -from database.mongodb.utils import BatchInsert -from database.mongodb.utils import BatchUpdate -from database.mongodb.utils import BatchWrite -from database.mongodb.utils import async_cursor_batches -from database.mongodb.utils import cursor_batches +from datvia.mongodb.utils import AsyncBatchInsert +from datvia.mongodb.utils import AsyncBatchUpdate +from datvia.mongodb.utils import AsyncBatchWrite +from datvia.mongodb.utils import BatchInsert +from datvia.mongodb.utils import BatchUpdate +from datvia.mongodb.utils import BatchWrite +from datvia.mongodb.utils import async_cursor_batches +from datvia.mongodb.utils import cursor_batches class Collection: @@ -131,7 +131,7 @@ class TestBatch(unittest.TestCase): batch.close() batch = BatchInsert(Collection(error=error), step=1, on_error='log') - with self.assertLogs('database.mongodb._batch', level='WARNING') as logs: + with self.assertLogs('datvia.mongodb._batch', level='WARNING') as logs: batch.add({'_id': 1}) self.assertNotIn('private', '\n'.join(logs.output)) diff --git a/tests/test_mysql_client.py b/tests/test_mysql_client.py index fcdb437..377590b 100644 --- a/tests/test_mysql_client.py +++ b/tests/test_mysql_client.py @@ -3,11 +3,11 @@ from unittest.mock import AsyncMock, patch from mysql.connector import Error -from database.mysql import Client +from datvia.mysql import Client class MySQLClientTests(TestCase): - @patch("database.mysql.client.connector.connect") + @patch("datvia.mysql.client.connector.connect") def test_create_returns_the_driver_connection(self, connect): connection = object() connect.return_value = connection @@ -17,7 +17,7 @@ class MySQLClientTests(TestCase): self.assertIs(result, connection) connect.assert_called_once_with("option", host="127.0.0.1", port=3306) - @patch("database.mysql.client.connector.connect") + @patch("datvia.mysql.client.connector.connect") def test_create_propagates_the_driver_error(self, connect): error = Error("connection failed") connect.side_effect = error @@ -29,7 +29,7 @@ class MySQLClientTests(TestCase): class AsyncMySQLClientTests(IsolatedAsyncioTestCase): - @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) async def test_create_async_returns_the_driver_connection(self, connect): connection = object() connect.return_value = connection @@ -39,7 +39,7 @@ class AsyncMySQLClientTests(IsolatedAsyncioTestCase): self.assertIs(result, connection) connect.assert_awaited_once_with("option", host="127.0.0.1", port=3306) - @patch("database.mysql.client.aio.connect", new_callable=AsyncMock) + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) async def test_create_async_propagates_the_driver_error(self, connect): error = Error("connection failed") connect.side_effect = error diff --git a/tests/test_schema_binding.py b/tests/test_schema_binding.py index d06ac17..e442e53 100644 --- a/tests/test_schema_binding.py +++ b/tests/test_schema_binding.py @@ -7,13 +7,13 @@ import unittest from pathlib import Path from types import SimpleNamespace -from database.connections import AsyncConnection -from database.connections import Connection -from database.errors import ConfigurationError -from database.errors import ConnectionClosedError -from database.errors import DriverError -from database.schema import SchemaConflictError -from database.schema._base import BoundSchema +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import ConnectionClosedError +from datvia.errors import DriverError +from datvia.schema import SchemaConflictError +from datvia.schema._base import BoundSchema class NativeFailure(Exception): @@ -29,8 +29,8 @@ class SchemaBindingTests(unittest.TestCase): source = str(Path(__file__).resolve().parents[1] / 'src') code = ( f'import sys; sys.path.insert(0, {source!r}); ' - 'from database.schema import Column, TableSpec; ' - 'from database.crud import RedisStrings, AsyncRedisStrings, StringSetItem; ' + 'from datvia.schema import Column, TableSpec; ' + 'from datvia.crud import RedisStrings, AsyncRedisStrings, StringSetItem; ' 'assert not any(k == n or k.startswith(n + ".") ' 'for n in ("pymongo", "mysql.connector", "redis") for k in sys.modules)' ) diff --git a/tests/test_schema_mongodb.py b/tests/test_schema_mongodb.py index 2f496d3..316b6c9 100644 --- a/tests/test_schema_mongodb.py +++ b/tests/test_schema_mongodb.py @@ -11,15 +11,15 @@ from pymongo.errors import OperationFailure from pymongo.errors import WriteConcernError from pymongo.write_concern import WriteConcern -from database._adapters.mongodb import Adapter -from database.connections import AsyncConnection -from database.connections import Connection -from database.errors import ConfigurationError -from database.errors import DriverError -from database.errors import PermissionDeniedError -from database.schema import AsyncSchema -from database.schema import Schema -from database.schema import SchemaConflictError +from datvia._adapters.mongodb import Adapter +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import DriverError +from datvia.errors import PermissionDeniedError +from datvia.schema import AsyncSchema +from datvia.schema import Schema +from datvia.schema import SchemaConflictError def _metadata(name='orders', kind='collection', **options): diff --git a/tests/test_schema_mysql.py b/tests/test_schema_mysql.py index 71c658e..8bbbad5 100644 --- a/tests/test_schema_mysql.py +++ b/tests/test_schema_mysql.py @@ -22,17 +22,17 @@ from mysql.connector.errorcode import ER_TABLE_EXISTS_ERROR from mysql.connector.errorcode import ER_TABLEACCESS_DENIED_ERROR from mysql.connector.errors import ProgrammingError -from database import AsyncConnection -from database import Connection -from database._adapters.mysql import Adapter as ConnectionAdapter -from database.errors import ConfigurationError -from database.errors import DriverError -from database.errors import PermissionDeniedError -from database.schema import AsyncSchema -from database.schema import Column -from database.schema import Schema -from database.schema import SchemaConflictError -from database.schema import TableSpec +from datvia import AsyncConnection +from datvia import Connection +from datvia._adapters.mysql import Adapter as ConnectionAdapter +from datvia.errors import ConfigurationError +from datvia.errors import DriverError +from datvia.errors import PermissionDeniedError +from datvia.schema import AsyncSchema +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import SchemaConflictError +from datvia.schema import TableSpec _LITERAL = "quote'; DROP DATABASE victim; --" _STAMP = datetime(2026, 1, 2, 12, 15, 0, 230000) @@ -95,7 +95,7 @@ class MySQLTableSpecTests(TestCase): source = str(Path(__file__).resolve().parents[1] / 'src') code = ( f'import sys; sys.path.insert(0, {source!r}); ' - 'from database.schema import Column, TableSpec; ' + 'from datvia.schema import Column, TableSpec; ' 'TableSpec({"id": Column("int")}, ("id",)); ' 'assert "mysql.connector" not in sys.modules' ) -- Gitee From 53085d0ecedea5c61350a753f311853c1e6316bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E5=8F=AF=E7=88=B1?= Date: Wed, 18 Nov 2026 21:18:04 +0800 Subject: [PATCH 86/86] =?UTF-8?q?chore(release):=20=E5=87=86=E5=A4=87=20DA?= =?UTF-8?q?TVIA=20v0.1.0=20=E9=A6=96=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将 v0.1.0 作为 DATVIA 的首个发行版本,整理发布日志、使用说明及验收文档。将源码克隆和发行元数据更新至 datvia 仓库地址。 已核对文档链接、Python 示例及 sdist 与 wheel 的仓库元数据,wheel 安装和 pip check 均通过。 --- CHANGELOG.md | 61 ++++++++---------------- CONTRIBUTING.md | 6 +++ README.md | 12 ++--- docs/README.md | 4 +- docs/account-management-design.md | 2 +- docs/connection-management-design.md | 4 +- docs/connection-management-validation.md | 2 +- docs/crud-design.md | 4 +- docs/crud-usage.md | 2 +- pyproject.toml | 4 +- tests/README.md | 2 +- 11 files changed, 45 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22f6aaf..067bd23 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,50 +2,31 @@ 记录面向使用者的功能、修复与兼容性变化。发布日期以仓库的 Git 标签为准。 -## 未发布 +## v0.1.0 -### 新增 +DATVIA 的首个版本。提供 MongoDB、MySQL 和 Redis 的同步与异步 Python API,覆盖连接管理、数据操作和账号权限管理。 -- 新增 `datvia.accounts` 同步与异步 Python 工具,支持自管 MongoDB 用户及角色、MySQL 账号及对象授权和固定 Redis 数据节点的 ACL 管理。 -- 增加 MongoDB 同步与异步分页读取和流式续读工具,按 `_id` 索引定位,可在游标失效或连接中断后有界恢复。 -- 增加 MongoDB 同步与异步混合批写、待写缓冲丢弃和自动关闭游标的分批读取工具。异步批处理支持消费异步可迭代对象。 -- 新增同步/异步 Schema 基础管理:MySQL 库表创建、兼容校验、目录查看和删除;MongoDB 集合准备、必要身份约束及目录管理。初始化显式执行,不自动迁移。 -- 新增 `RedisStrings` / `AsyncRedisStrings` 的单键、批量读写、删除和 TTL,保留逐项执行证据、原始错误与取消信号。 -- 新增三数据库完整同步/异步使用示例和显式隔离验收脚本;原生客户端及已有记录接口保持兼容。 -- 新增可选的 `datvia.crud.Records` / `AsyncRecords`,在已有连接上绑定 MySQL 表、MongoDB 集合或 Redis Hash,支持复合键、字段投影、批量读取及分块写入;MySQL/MongoDB 另支持结构化条件查询和集合更新、删除。 -- CRUD 写入结果区分已知计数与未知计数;批量失败保留逐项执行状态、部分结果和原始异常,异步取消保持原信号。记录层借用连接,不自动提交、回滚或重试。 -- Redis Hash 记录使用显式 namespace、字段 codec、版本标记和单键脚本,支持创建、更新、投影及 TTL;保留原生其他数据结构和客户端能力。 -- 新增 `ConnectionConfig` 和 `Connections`,支持按名称登记 MongoDB、MySQL 和 Redis 配置,按需创建独立的同步或异步句柄。 -- 新增显式连接检查、上下文管理和统一异常。可通过 `.native` 访问原生驱动对象,通过 `DatabaseError.original` 读取原始异常。 -- 统一连接管理覆盖 Redis 单机、Sentinel 与 Cluster。Cluster 的检查范围为当前发现的主节点。 -- 新增基于 MySQL Connector/Python 的同步和异步工厂。 -- 新增隔离服务验证脚本,以及按后端覆盖配置、检查、异常和关闭的离线测试。 +### 功能 -### 修复 +- `ConnectionConfig` 和 `Connections` 按名称登记和创建独立连接,提供显式检查、上下文管理及保留原始错误的异常转换。Redis 支持单机、Sentinel 和 Cluster。Cluster 检查覆盖当前发现的主节点。 +- 三个后端都提供原生客户端工厂。通过统一连接的 `.native` 也可以访问驱动对象,使用原生查询、事务和命令。 +- `Schema` 和 `AsyncSchema` 支持 MySQL 库表及 MongoDB 普通集合的显式准备、校验、目录查看和删除。 +- `Records` 和 `AsyncRecords` 支持 MySQL 表、MongoDB 集合及 Redis Hash 的复合键、字段投影、批量读取与分块写入。MySQL 与 MongoDB 也支持结构化条件查询及批量更新和删除。 +- 记录层借用调用方连接。批量操作返回逐项状态和已知计数,并在失败时保留部分结果、原始异常和取消信号。 +- Redis Hash 记录使用显式命名空间、字段编解码器和单键脚本,支持更新、投影及 TTL。`RedisStrings` 和 `AsyncRedisStrings` 提供 String 的单键和批量读写、删除及 TTL。 +- MongoDB 提供同步与异步混合批写、分批游标读取、分页扫描及流式续读工具。续读按 `_id` 索引定位,可在游标失效或连接中断后有界恢复。 +- `datvia.accounts` 管理自管 MongoDB 用户与角色、MySQL 账号及对象授权、固定 Redis 数据节点的 ACL,提供同步与异步接口。 +- 提供三种数据库的[完整使用示例](examples/basic_usage.py)和[测试指南](tests/README.md)。 -- 修复 MongoDB 批处理工具的导入与插入批量预缓冲问题,按配置条数派发生成器和列表输入。失败时释放批次,清理失败时保留业务异常,错误计数使用驱动报告的逐条写入错误。 -- MySQL 目标配置提前拒绝 Windows 上无法使用的纯 Unix socket 配置,并把缺少驱动能力的错误转换为 `DependencyError`。 -- 识别 MySQL 的 `password1` 密码别名,在公共密码参数与该别名同时传入时报告冲突。 -- Redis 异步 Cluster 关闭失败后保留待清理节点,以便后续关闭重试。关闭期间的错误附加到已有业务异常。 +### 兼容性 -### 兼容性与依赖 +- MongoDB 的同步接口要求 PyMongo 4.6.0 或以上版本,异步接口要求 PyMongo 4.9 或以上版本。统一连接的 `Connections.create_async()` 使用 `await`,原生工厂按各驱动的调用方式使用。 +- Windows 上的 MySQL 高层连接需要显式主机地址。公共密码与 Connector/Python 的 `password1` 同时配置时会报告冲突。缺失的驱动能力通过 `DependencyError` 报告。 +- Redis 异步 Cluster 会保留关闭失败的节点供后续重试。已有业务异常时,清理失败附加到该异常。 -- 项目更名为 DATVIA,发行包名和 Python 导入名从 `database` 改为 `datvia`。升级时需更新应用的依赖声明和导入语句。已有 Redis Hash v1 记录可直接读取。 -- 源码安装统一使用 `python -m pip install .`,默认包含 MongoDB、MySQL 和 Redis 驱动。原有安装 extras 已移除,运行依赖及版本约束与 `requirements.txt` 一致。 -- PyMongo 的默认依赖下限为 `4.6.0`,同步连接支持该版本。异步连接需要 PyMongo 4.9 或以上版本,缺少异步 API 时给出升级提示。 -- 原生工厂继续返回驱动对象。统一入口的 `create_async()` 需要 `await`,原生工厂沿用各自的异步调用方式。 +### 验证范围 -### 文档与验证 - -- 补充快速开始、架构示意、贡献和测试指南。 -- 已有隔离验证覆盖 MongoDB 和 MySQL。Redis 三种拓扑已进行驱动级离线测试,真实服务验收待补齐。[连接管理验收记录](docs/connection-management-validation.md) - -## v0.0.1-beta - 2026-07-17 - -初始 Beta 版本,对应 Git 标签 `v0.0.1-beta`。 - -### 新增 - -- 提供 MongoDB 同步和异步原生客户端工厂,以及批量插入和更新辅助工具。 -- 提供 Redis 同步和异步客户端,支持单机、Sentinel 与 Cluster 的统一创建入口。 -- 提供 `mongodb`、`redis` 和包含两者的 `all` 安装选项。 +- 在 Python 3.10 上通过 324 项离线回归。 +- MongoDB 和 MySQL 的同步与异步连接、基础结构管理和 CRUD 已通过隔离真实服务验收。[连接管理验收记录](docs/connection-management-validation.md) [基础使用验收记录](docs/crud-basic-implementation.md) +- 三种后端的账号与权限管理已通过 52 组真实服务专项验证,其中 Redis 覆盖固定数据节点的 ACL。[账号管理验收记录](docs/account-management-validation.md) +- Redis 单机、Sentinel 和 Cluster 的连接及记录操作有离线驱动测试覆盖。真实 Redis 拓扑的连接、Hash 和 String 操作,以及三个真实后端同时运行的端到端场景仍待验收。[基础使用验收记录](docs/crud-basic-implementation.md#6-待验证事项与审查入口) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2532b4d..bd33d47 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,3 +50,9 @@ docs(readme): 补充异步连接示例 ``` 提交合并请求时说明变更目的、影响的数据库或驱动版本、兼容性、验证结果及待验证事项。 + +## 准备发布 + +正式发行使用 `master` 上的 `vX.Y.Z` 标签。将 [发布日志](CHANGELOG.md) 的未发布条目归入目标版本。发布日期以实际 Git 标签为准。 + +用标签对应的源码构建 sdist 和 wheel,并在独立环境安装验证。核对发行元数据和 `datvia.__version__` 与标签对应,运行 `python -m pip check`,并按 [测试指南](tests/README.md) 核对离线测试和已有实测范围。 diff --git a/README.md b/README.md index 1cefef9..78062d0 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,15 @@ ### 安装 -准备 Python 3.10 或更高版本和 Git。在源码仓库根目录执行以下命令,安装包及三个后端的驱动。 +准备 Python 3.10 或更高版本和 Git。克隆 DATVIA 源码并安装。 ```bash +git clone https://gitee.com/loongspears/datvia.git +cd datvia python -m pip install . ``` -DATVIA 的发行包名和导入名均为 `datvia`。从原 `database` 项目升级时,请将依赖声明、Python 导入和按包名配置的日志过滤规则改为 `datvia`。 - -驱动版本约束与 [requirements.txt](requirements.txt) 一致。 +已检出源码时,在仓库根目录运行 `python -m pip install .`。安装包含三个后端的驱动,依赖约束见 [requirements.txt](requirements.txt)。 | 后端 | 默认驱动依赖 | | --- | --- | @@ -303,7 +303,7 @@ SDK 自动转换其自身配置、创建、检查及关闭范围内的已识别 MongoDB 的检查结果使用 `scope="selected_server"`,`targets` 为空:原生 `ping` 不公开实际执行节点,因此 SDK 不猜测地址。Windows 上 MySQL 需要显式主机地址,不能仅用驱动不支持的 Unix socket 参数指定目标。同步 Redis Cluster 的 `startup_nodes` 不接受已经绑定客户端的节点,以防意外关闭调用方资源。 -异步创建尚未返回句柄时的资源清理由原生驱动负责。当前受测 Connector/Python 的握手取消存在上游清理限制,详见设计文档。Redis 拓扑已完成驱动级验证,真实服务验证仍待完成。 +原生驱动负责异步创建返回句柄前的资源清理。Connector/Python 握手取消的限制和 Redis 拓扑的连接验证见 [连接管理验收记录](docs/connection-management-validation.md)。Redis 记录操作的验证见 [基础使用验收记录](docs/crud-basic-implementation.md#5-隔离真实服务验证)。 ## 基础结构准备与键值操作 @@ -388,7 +388,7 @@ MySQL 账号写操作使用启用自动提交且没有活动事务的连接。Re ## 文档与贡献 - [文档索引](docs/README.md) 汇总连接管理设计、验收记录和源码目录说明。 -- [发布日志](CHANGELOG.md) 记录已发布版本和未发布改动。 +- [发布日志](CHANGELOG.md) 记录功能、兼容性变化和验证范围。 - [贡献指南](CONTRIBUTING.md) 说明开发环境、代码检查和提交约定。 - [测试指南](tests/README.md) 提供离线测试及隔离数据库的验证命令。 - [安全说明](SECURITY.md) 说明凭据、日志和安全问题反馈方式。 diff --git a/docs/README.md b/docs/README.md index e35ae38..cd43341 100644 --- a/docs/README.md +++ b/docs/README.md @@ -3,10 +3,10 @@ ## 使用与开发 - [项目 README](../README.md) 提供安装、原生客户端、统一连接管理和同步、异步用法。 +- [发布日志](../CHANGELOG.md) 记录各次发行的功能、兼容性变化和验收范围。 - [贡献指南](../CONTRIBUTING.md) 提供本地环境、代码与文档风格、检查命令和分支提交约定。 - [智能体编码工作规范](../AGENTS.md) 提供仓库协作规则和任务阅读入口。 - [安全说明](../SECURITY.md) 说明连接凭据、异常日志和安全问题反馈方式。 -- [发布日志](../CHANGELOG.md) 记录已发布版本和未发布改动。 ## 连接管理 @@ -25,7 +25,7 @@ - [适配器契约](crud-adapter-contract.md) 定义公共层与各后端的模块职责、调用和部分结果协议。 - [CRUD 实施与验收记录](crud-implementation.md) 记录本地分支、Git 集成过程、测试及待验证项。 - [基础使用设计与开发计划](crud-design.md) 说明从初始化到 CRUD 的接口、原生能力边界和完整示例。 -- [基础使用实施与验收记录](crud-basic-implementation.md) 记录本轮三个模块的提交、合并、测试及真实 Redis 验收缺口。 +- [基础使用实施与验收记录](crud-basic-implementation.md) 记录基础结构管理与 Redis String 的实施和专项验收。 - [同步异步完整示例](../examples/basic_usage.py) 演示三个后端同时使用;由调用方提供显式配置。 ## 目录职责 diff --git a/docs/account-management-design.md b/docs/account-management-design.md index 4a34d4e..d7c8662 100644 --- a/docs/account-management-design.md +++ b/docs/account-management-design.md @@ -1,6 +1,6 @@ # MongoDB、MySQL、Redis 账号与权限管理工具实现设计 -本文是 `datvia.accounts` 首版的实施规划,供实现、审查与验收使用。已执行的版本组合及测试结果见[验收记录](account-management-validation.md)。需求编号 R1–R9、使用场景和操作权限表见[需求文档](account-management-requirements.md)。既有连接与异常实现见[连接生命周期](../src/datvia/connections.py)和[异常转换](../src/datvia/errors.py)。 +本文记录 `datvia.accounts` 的公共接口、原生命令和首版验收方案。已执行的版本组合及测试结果见[验收记录](account-management-validation.md)。需求编号 R1–R9、使用场景和操作权限表见[需求文档](account-management-requirements.md)。既有连接与异常实现见[连接生命周期](../src/datvia/connections.py)和[异常转换](../src/datvia/errors.py)。 ## 1. 组件与执行流程 diff --git a/docs/connection-management-design.md b/docs/connection-management-design.md index f4c865d..ba01402 100644 --- a/docs/connection-management-design.md +++ b/docs/connection-management-design.md @@ -1,6 +1,6 @@ # DATVIA 统一连接管理设计与开发流程 -日期:2026-11-15。状态:实现已集成,实际验证范围和剩余事项见 [验收记录](connection-management-validation.md)。 +本设计整理于 2026-11-15。统一连接管理随 [DATVIA v0.1.0](../CHANGELOG.md#v010) 提供,实际验证范围见 [验收记录](connection-management-validation.md)。 基线 develop:7d38b1ffe13a17b37d0bffc15e175085dcf0bb19。 参考 temp/save:89497823698812e1c85e6c4c12af86d4281543cf。 @@ -91,7 +91,7 @@ feature/database-mongodb-connections、feature/database-mysql-connections、feat 必要依赖只从总线明确检查点合入,不交叉复制或合并其他worker草稿。 中文 Conventional Commits,按功能自然拆分并附必要测试;不把全部工作压成单提交;已推送历史不rebase/amend。 总线使用 --no-ff 合并保留模块边界,不squash、不用单次cherry-pick替代分支。核对交付SHA和测试后合并并复验。 -允许向 origin https://gitee.com/loongspears/database 推送本任务明确分支;禁止推送develop、无关分支、--all、--mirror、force,禁止删除远端分支。worker推自己的分支,协调者推总线;推后核对SHA。鉴权失败仍完成本地可做工作并记录。 +当时允许向远端 `origin` 推送本任务明确分支;禁止推送develop、无关分支、--all、--mirror、force,禁止删除远端分支。worker推自己的分支,协调者推总线;推后核对SHA。鉴权失败仍完成本地可做工作并记录。 不改历史worktree、不删除temp/save、不处理无关文件;保留任务分支和worktree供审核。 ## 验收 diff --git a/docs/connection-management-validation.md b/docs/connection-management-validation.md index 52f9230..3d7e8af 100644 --- a/docs/connection-management-validation.md +++ b/docs/connection-management-validation.md @@ -4,7 +4,7 @@ ## 分支与工作区 -- 远端:`https://gitee.com/loongspears/database`,名称 `origin`。 +- 交付远端名称为 `origin`。 - 唯一交付分支:`feature/database-connection-management`。 - 基线 `develop`:`7d38b1ffe13a17b37d0bffc15e175085dcf0bb19`。 - 完整源码及安装验收检查点:`b2c23a8454590c705789d571c120f0fb684d3a70`;其后仅补充本验收记录和清理文件末尾空行,不改变受测代码行为。 diff --git a/docs/crud-design.md b/docs/crud-design.md index af1f629..e46d76d 100644 --- a/docs/crud-design.md +++ b/docs/crud-design.md @@ -4,7 +4,7 @@ 更新日期:2026-11-17 -状态:基础封装已在 `feature/database-crud` 集成,待用户审查。MySQL、MongoDB 基础流程的真实服务验收已通过;Redis 真实单机、Sentinel、Cluster 及三种真实后端同时运行的验收尚未完成。 +基础结构管理、记录操作和 Redis String 接口随 [DATVIA v0.1.0](../CHANGELOG.md#v010) 提供。接口用法见 [基础结构与记录操作使用说明](crud-usage.md)。 文档位置:`docs\crud-design.md`。 @@ -350,7 +350,7 @@ Redis Hash 保持现有记录格式、显式 codec 和 TTL 约定。Hash 更新 ## 三、从初始化到 CRUD 的同步、异步示例 -以下示例使用本分支已实现的 API,完整源码见 [同步异步示例](../examples/basic_usage.py)。由调用方准备隔离服务和凭据,先复用公共配置与表定义,再选择同步或异步入口。MySQL、MongoDB 已完成真实服务端到端验收;Redis 真实拓扑验收仍待补齐。示例会写入并清理指定演示记录,应使用专用演示库和命名空间。 +以下示例使用 DATVIA 的公开接口,完整源码见 [同步异步示例](../examples/basic_usage.py)。由调用方准备隔离服务和凭据,先复用公共配置与表定义,再选择同步或异步入口。MySQL、MongoDB 已完成真实服务端到端验收;Redis 真实拓扑验收仍待补齐。示例会写入并清理指定演示记录,应使用专用演示库和命名空间。 示例假设三个数据库服务和账号已经准备好。MySQL 初始化连接具有建库建表权限;生产项目可以为初始化与业务操作配置不同账号。 diff --git a/docs/crud-usage.md b/docs/crud-usage.md index 75e164a..43d4ab0 100644 --- a/docs/crud-usage.md +++ b/docs/crud-usage.md @@ -4,7 +4,7 @@ MySQL 对应表,MongoDB 对应集合,Redis 对应采用本库格式的 Hash 记录。 它们共享调用方式、字段投影和结果结构;事务、类型系统和查询能力仍遵循各数据库的真实语义。 -本功能在 `feature/database-crud` 分支开发。原记录 CRUD 测试见 [历史实施记录](crud-implementation.md),新增结构与 String 的验证范围及缺口见 [本轮实施记录](crud-basic-implementation.md)。 +记录操作的受测版本和场景见 [CRUD 验收记录](crud-implementation.md),结构管理与 Redis String 的验证见 [基础使用验收记录](crud-basic-implementation.md)。 完整设计背景见 [CRUD 设计](crud-design.md),内部模块协作见 [适配器契约](crud-adapter-contract.md)。 diff --git a/pyproject.toml b/pyproject.toml index 4914787..140e8db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,8 +34,8 @@ dependencies = [ ] [project.urls] -Homepage = "https://gitee.com/loongspears/database" -Repository = "https://gitee.com/loongspears/database" +Homepage = "https://gitee.com/loongspears/datvia" +Repository = "https://gitee.com/loongspears/datvia.git" [tool.setuptools_scm] write_to = "src/datvia/_version.py" diff --git a/tests/README.md b/tests/README.md index 3eff32a..dc5a03a 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,7 +2,7 @@ ## 离线回归 -新增记录层使用 `tests/test_crud_*.py`:公共条件与生命周期、三个后端适配、以及多数据库/同类多实例资源隔离分别维护。 +记录操作由 `tests/test_crud_*.py` 覆盖,包含公共条件与生命周期、三个后端适配、多后端及同类多实例的资源隔离。 批量状态、取消信号、字段投影、原始类型和资源归属均覆盖同步与异步。 该功能的版本矩阵、Lua 模拟器及隔离服务结果见 [CRUD 实施与验收记录](../docs/crud-implementation.md)。 -- Gitee