diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..34359e9fdd80c0b7d8802174491b949feba0f894 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,43 @@ +# 自动识别文本,入库和检出均使用 LF +* text=auto eol=lf + +# 图片 +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.ico binary +*.webp binary +*.avif binary + +# 字体 +*.woff binary +*.woff2 binary +*.ttf binary +*.otf binary + +# Word +*.[dD][oO][cC] binary +*.[dD][oO][cC][mMxX] binary +*.[dD][oO][tT][mMxX] binary + +# Excel +*.[xX][lL][sS] binary +*.[xX][lL][sS][bBmMxX] binary +*.[xX][lL][tT] binary +*.[xX][lL][tT][mMxX] binary + +# PowerPoint +*.[pP][pP][tT] binary +*.[pP][pP][tT][mMxX] binary +*.[pP][pP][sS] binary +*.[pP][pP][sS][mMxX] binary +*.[pP][oO][tT][mMxX] binary + +# PDF +*.[pP][dD][fF] binary + +# 数据库 +*.db binary +*.sqlite binary +*.sqlite3 binary diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..910f78d50a61167a731a82976c9bd4453dcde5a0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ +# Secrets +# Environments +.env +.env.dev +.env.develop +.env.test +.env.production +# database files +*.db +*.sql + +# macOS +.DS_Store + +# editor or IDE +.idea/ +.run/ +.vscode/ + +# AI agent +.claude/ +.codex/ + +# GIT +.gitee/ +.github/ + +# python venv +.venv/ +.uv-cache/ +uv.lock + +# Python cache +__pycache__/ +*.py[cod] +.pytest_cache/ +.pytest-tmp/ +.ruff_cache/ +.mypy_cache/ + +# Python build +/build/ +*.egg-info/ +/src/*.egg-info/ + +# test scripts +!/tests +/test*.py +src/datvia/_version.py + +# logs +/logs/ +/outputs/ +*.log diff --git a/.python-version b/.python-version new file mode 100644 index 0000000000000000000000000000000000000000..44677e5ccdecc89b96c9256d3f7deacec119e390 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.10.20 diff --git a/.ruff.toml b/.ruff.toml new file mode 100644 index 0000000000000000000000000000000000000000..b93a424ffad7ee1fe76f9052fe80b4e57ce18299 --- /dev/null +++ b/.ruff.toml @@ -0,0 +1,11 @@ +line-length = 120 +target-version = "py310" + +[lint] +select = ["E", "F", "I"] + +[lint.isort] +force-single-line = true + +[format] +quote-style = "single" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000000000000000000000000000000000000..b5d43d3e3c8ad51f33d739c8f839e2829139f521 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,70 @@ +# AGENTS.md + +## 项目概览 + +DATVIA(包名 `datvia`)是支持 Python 3.10 及以上版本的 MongoDB、MySQL 和 Redis 连接 SDK。本文件适用于整个仓库。进入子目录时先检查是否有该目录的 `AGENTS.md` 或 `AGENTS.override.md`。 + +## 从哪里开始 + +- 先看 `git status --short --branch`、任务相关差异和暂存区,保留工作区已有改动。 +- 使用示例见 [README](README.md),开发和提交规范见 [贡献指南](CONTRIBUTING.md)。 +- `src/datvia/config.py` 负责配置和参数校验,`src/datvia/connections.py` 负责命名连接和生命周期,`src/datvia/errors.py` 负责异常转换。 +- `src/datvia/_adapters/` 负责高层连接与驱动之间的适配。原生工厂维护在 `src/datvia/{mongodb,mysql,redis}/`,离线测试放在 `tests/`。 + +## 修改连接代码 + +- 核对 [连接管理设计](docs/connection-management-design.md)。保持原生 `Client` 的签名、参数透传和返回对象兼容。高层 `Connections.create_async()` 一律通过 `await` 调用,原生工厂的异步调用方式按各后端现有契约处理。 +- 高层配置由调用方显式传入。默认安装包含三个后端的驱动。公共接口、配置构造和连接登记保持独立于驱动导入,创建连接时再加载对应驱动。 +- 按改动覆盖同步和异步路径。检查只报告当次探测结果,关闭遵守自有资源与外部资源的归属,异常转换保留原始驱动错误和因果链。 +- 使用 [.ruff.toml](.ruff.toml) 的风格,为新增公开接口补充类型注解。修改运行依赖时,同步维护 [pyproject.toml](pyproject.toml) 的 `project.dependencies` 和 [requirements.txt](requirements.txt) 中的包名与版本约束。 + +## 运行检查 + +在仓库根目录使用当前工作区的源码。开发环境与工具安装见 [贡献指南](CONTRIBUTING.md)。 + +```bash +python -m pip install -e . +python -B -m unittest discover -s tests -q +git diff --check +``` + +- Python 改动先按 [测试指南](tests/README.md) 运行定向 `unittest`,对改动文件执行 Ruff lint 和格式检查。涉及公共入口、生命周期或共享逻辑时运行完整离线回归。 +- 自动测试使用本地样本和驱动替身。隔离服务验收使用 `scripts/verify_connections.py --config` 和显式测试配置,凭据和日志处理见 [安全说明](SECURITY.md)。 +- 文档改动核对链接、示例和差异。工具配置改动使用对应工具检查。 + +## Git 与 worktree 协作 + +### 分工和基线 + +主智能体负责协调。先用 `git status --short --branch`、`git diff --cached` 和 `git worktree list --porcelain` 核对现状,记录任务基线提交、集成目标和已有工作树。分配阶段目标、文件范围、依赖及验收命令。同一文件由一名工作者负责,公共导出和其他共享文件由协调者安排。 + +需要并行修改时,每个工作者使用独立分支、worktree 和暂存区。协调者在集成工作树负责合并。下面以已提交的 `develop` 为基线,示例中的分支名和路径按任务替换。 + +```powershell +$base = (git rev-parse develop).Trim() +git worktree add -b feature/task-integration ../datvia-worktrees/task-integration $base +git worktree add -b feature/task-config ../datvia-worktrees/task-config $base +git worktree add -b feature/task-errors ../datvia-worktrees/task-errors $base +``` + +### 开发和集成 + +1. 工作者在自己的 worktree 修改和验证,使用 `git -C <路径>` 或进入该目录执行 Git 命令。需要提交时,按分配的路径暂存、核对 `git diff --cached`,使用中文 Conventional Commits。 +2. 有前置依赖时,先由协调者集成公共变更,再从已验证的集成提交建立或同步后续任务分支。工作者在交付时提供改动范围、验证结果和提交 SHA。仅要求工作区改动的任务交付对应差异和验证结果。 +3. 需要合并时,协调者核对交付 SHA 和差异,在集成 worktree 串行用 `--no-ff` 合入。合并后记录新的 HEAD 并重跑相关验证。冲突由协调者在集成 worktree 处理,核对公共接口并保留各分支的原始提交。最终合入目标分支和推送按当次任务要求执行。 + +```powershell +$workerHead = (git rev-parse feature/task-config).Trim() +git -C ../datvia-worktrees/task-integration diff "HEAD...$workerHead" +git -C ../datvia-worktrees/task-integration merge --no-ff -m "feat(config): 合入连接配置" $workerHead +git -C ../datvia-worktrees/task-integration rev-parse HEAD +``` + +### Git 操作记录 + +在任务交付中按执行顺序记录 Git 命令,标明执行者、分支、worktree 与结果。汇总基线 SHA、各任务的交付 SHA 或差异路径,以及验证命令与结果。合并记录写明合入的 SHA 和集成后的 HEAD。推送记录写明远端、分支及核对过的远端 SHA。清理工作树前,用 `git worktree list` 和 `git status` 核对任务已验收且工作区干净。 + +## 文档与交付 + +- 公开用法变更时同步 README。对外可见的功能、修复和兼容性变化登记在 [发布日志](CHANGELOG.md) 的「未发布」部分,版本记录依据实际 Git 标签。 +- 分支、暂存、提交、合并和推送按当次任务要求执行。交付时注明检查结果和待验证事项。 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000000000000000000000000000000000..067bd23705c3724d4899d6e7a3d6d9abfbd2b6bd --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,32 @@ +# 发布日志 + +记录面向使用者的功能、修复与兼容性变化。发布日期以仓库的 Git 标签为准。 + +## v0.1.0 + +DATVIA 的首个版本。提供 MongoDB、MySQL 和 Redis 的同步与异步 Python API,覆盖连接管理、数据操作和账号权限管理。 + +### 功能 + +- `ConnectionConfig` 和 `Connections` 按名称登记和创建独立连接,提供显式检查、上下文管理及保留原始错误的异常转换。Redis 支持单机、Sentinel 和 Cluster。Cluster 检查覆盖当前发现的主节点。 +- 三个后端都提供原生客户端工厂。通过统一连接的 `.native` 也可以访问驱动对象,使用原生查询、事务和命令。 +- `Schema` 和 `AsyncSchema` 支持 MySQL 库表及 MongoDB 普通集合的显式准备、校验、目录查看和删除。 +- `Records` 和 `AsyncRecords` 支持 MySQL 表、MongoDB 集合及 Redis Hash 的复合键、字段投影、批量读取与分块写入。MySQL 与 MongoDB 也支持结构化条件查询及批量更新和删除。 +- 记录层借用调用方连接。批量操作返回逐项状态和已知计数,并在失败时保留部分结果、原始异常和取消信号。 +- Redis Hash 记录使用显式命名空间、字段编解码器和单键脚本,支持更新、投影及 TTL。`RedisStrings` 和 `AsyncRedisStrings` 提供 String 的单键和批量读写、删除及 TTL。 +- MongoDB 提供同步与异步混合批写、分批游标读取、分页扫描及流式续读工具。续读按 `_id` 索引定位,可在游标失效或连接中断后有界恢复。 +- `datvia.accounts` 管理自管 MongoDB 用户与角色、MySQL 账号及对象授权、固定 Redis 数据节点的 ACL,提供同步与异步接口。 +- 提供三种数据库的[完整使用示例](examples/basic_usage.py)和[测试指南](tests/README.md)。 + +### 兼容性 + +- MongoDB 的同步接口要求 PyMongo 4.6.0 或以上版本,异步接口要求 PyMongo 4.9 或以上版本。统一连接的 `Connections.create_async()` 使用 `await`,原生工厂按各驱动的调用方式使用。 +- Windows 上的 MySQL 高层连接需要显式主机地址。公共密码与 Connector/Python 的 `password1` 同时配置时会报告冲突。缺失的驱动能力通过 `DependencyError` 报告。 +- Redis 异步 Cluster 会保留关闭失败的节点供后续重试。已有业务异常时,清理失败附加到该异常。 + +### 验证范围 + +- 在 Python 3.10 上通过 324 项离线回归。 +- MongoDB 和 MySQL 的同步与异步连接、基础结构管理和 CRUD 已通过隔离真实服务验收。[连接管理验收记录](docs/connection-management-validation.md) [基础使用验收记录](docs/crud-basic-implementation.md) +- 三种后端的账号与权限管理已通过 52 组真实服务专项验证,其中 Redis 覆盖固定数据节点的 ACL。[账号管理验收记录](docs/account-management-validation.md) +- Redis 单机、Sentinel 和 Cluster 的连接及记录操作有离线驱动测试覆盖。真实 Redis 拓扑的连接、Hash 和 String 操作,以及三个真实后端同时运行的端到端场景仍待验收。[基础使用验收记录](docs/crud-basic-implementation.md#6-待验证事项与审查入口) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000000000000000000000000000000000..bd33d47c255228ed52e0577c807813bddf751712 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,58 @@ +# 贡献指南 + +## 本地开发 + +项目支持 Python 3.10 及以上版本。本地参考版本见 [.python-version](.python-version)。安装通过 Git 获取共享依赖,请先准备 Python 与 Git。 + +在仓库根目录执行 `python -m venv .venv`。PowerShell 使用 `.\.venv\Scripts\Activate.ps1` 激活环境,Bash 使用 `source .venv/bin/activate`。在虚拟环境中执行以下命令。 + +```bash +python -m pip install -e . +python -m pip install ruff +``` + +开发安装包含三个后端的驱动,测试使用标准库 `unittest`。运行依赖及版本约束在 [pyproject.toml](pyproject.toml) 的 `project.dependencies` 和 [requirements.txt](requirements.txt) 中同步维护。验证驱动最低版本时,在独立环境使用对应的版本约束。源码构建时由 `setuptools-scm` 生成 `src/datvia/_version.py`。 + +## 代码与文档风格 + +- [.ruff.toml](.ruff.toml) 统一 Python 3.10 语法目标、120 字符行宽、单引号格式,以及 `E`、`F`、`I` 规则。一行导入一个对象。 +- 新增公开接口写明参数和返回值类型。涉及默认值、异常或资源生命周期时,用 NumPy 风格的 docstring 描述契约,完整用法示例维护在 README 或相关文档。 +- Markdown 普通段落及引用正文采用一段一行、段间空行。列表、表格、代码块和显式换行保留各自结构。 +- 文档正文、表格和命令中的项目路径以仓库根目录为基准,使用相对路径,不写盘符、用户主目录等本机绝对地址;仓库根目录记为 `.`,相邻工作树可写为 `../datvia-worktrees/<任务目录>`。 +- Markdown 本地链接使用相对目标,按文档所在目录保留必要的 `../` 跳转;链接文字展示路径时使用仓库根目录相对路径。仓库外的历史验收材料只记录文件名与用途,实际存储位置由运行者指定,不将其写成仓库内文件。 +- 根据 [文档索引](docs/README.md) 更新对应文档。文档中的连接地址、凭据和测试数据使用本地占位值。 + +## 提交前检查 + +涉及 Python 改动时,从仓库根目录运行离线回归。用 `git diff --check` 核对本次差异。定向测试及真实服务探测见 [测试指南](tests/README.md)。 + +```bash +python -B -m unittest discover -s tests -q +git diff --check +``` + +修改 Python 后,对实际改动的文件运行 `python -m ruff check <文件路径>` 和 `python -m ruff format --check <文件路径>`。全库扫描可以作为现有代码风格的基线,检查本次改动时以涉及的诊断为准。 + +修改依赖或构建配置时,在独立环境构建 wheel 和 sdist,验证产物安装、`python -m pip check` 和导入来源。 + +## 分支与提交 + +日常改动从 `develop` 建立任务分支,审查后合入 `develop`。`master` 用于发布。按工作内容使用 `feature/*`、`fix/*`、`refactor/*`、`docs/*` 或 `chore/*`。保留任务分支的提交关系时使用 `--no-ff` 合并。智能体的并行分工、集成和交付记录见 [Git 与 worktree 协作](AGENTS.md#git-与-worktree-协作)。 + +`docs/*` 仅用于纯文档任务。包含代码实现的任务按实际类型使用 `feature/*`、`fix/*`、`refactor/*` 等前缀,不以 `docs/*` 分支推进功能开发。 + +采用中文 Conventional Commits,保留英文类型和可选的 scope,正文使用中文说明背景和验证情况。每次提交包含一组可独立审查的改动,按路径暂存后核对 `git diff --cached`。 + +```text +feat(connections): 支持按名称创建数据库连接 +fix(redis): 保留集群连接关闭失败的原始异常 +docs(readme): 补充异步连接示例 +``` + +提交合并请求时说明变更目的、影响的数据库或驱动版本、兼容性、验证结果及待验证事项。 + +## 准备发布 + +正式发行使用 `master` 上的 `vX.Y.Z` 标签。将 [发布日志](CHANGELOG.md) 的未发布条目归入目标版本。发布日期以实际 Git 标签为准。 + +用标签对应的源码构建 sdist 和 wheel,并在独立环境安装验证。核对发行元数据和 `datvia.__version__` 与标签对应,运行 `python -m pip check`,并按 [测试指南](tests/README.md) 核对离线测试和已有实测范围。 diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000000000000000000000000000000000000..b5dd9960df2d01e518021cea08e70263f3f5ebe6 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1 @@ +include docs/assets/datvia-banner.svg docs/assets/datvia-badges.svg diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..78062d01450da7ea8df09c5b83a70793fdf90ca1 --- /dev/null +++ b/README.md @@ -0,0 +1,396 @@ +
+

DATVIA

+

Data via one Python API.

+

用一套 Python API 接入 MongoDB、MySQL 和 Redis,按需使用原生驱动。

+

+ Python 3.10+ · 同步与异步 API · MIT 许可证 +

+

+ 快速开始 · + 原生客户端 · + 连接管理 · + 记录操作 · + 文档 · + 发布日志 +

+
+ +--- + + + +## 🎯 快速开始 + +### 安装 + +准备 Python 3.10 或更高版本和 Git。克隆 DATVIA 源码并安装。 + +```bash +git clone https://gitee.com/loongspears/datvia.git +cd datvia +python -m pip install . +``` + +已检出源码时,在仓库根目录运行 `python -m pip install .`。安装包含三个后端的驱动,依赖约束见 [requirements.txt](requirements.txt)。 + +| 后端 | 默认驱动依赖 | +| --- | --- | +| MongoDB | `pymongo>=4.6.0` | +| MySQL | `mysql-connector-python>=8.3.0` | +| Redis | `redis>=5.0.0` | + +### 第一个连接 + +下面连接到本机 `127.0.0.1:6379` 上的 Redis,检查一次连接并在退出上下文时关闭它。 + +```python +from datvia import ConnectionConfig +from datvia import Connections + +connections = Connections({ + 'cache': ConnectionConfig(backend='redis', host='127.0.0.1', port=6379), +}) + +with connections.create('cache') as connection: + result = connection.check() + print(result.backend, result.elapsed_ms) +``` + +## ✨ 核心特性 + +- 🗂️ **按名称管理连接**。在同一个 `Connections` 中登记多个后端或同一后端的多个实例。 +- 🔌 **直接使用原生驱动**。用 `Client` 工厂或句柄的 `.native` 访问驱动对象,继续使用原生查询、事务和命令。 +- ⚡ **同步与异步**。统一入口管理创建和关闭,支持 Redis 单机、Sentinel 与 Cluster。 +- 🩺 **显式检查与统一异常**。用 `check()` 查看一次探测结果,从 `DatabaseError.original` 获取原始异常。 + + + +## 原生客户端 + +### MongoDB + +PyMongo 4.6–4.8 支持同步的 `create()`。异步的 `create_async()` 需要 PyMongo 4.9 或更高版本。使用 MongoDB 异步连接的应用应在自己的依赖中声明 `pymongo>=4.9`。 + +```python +import asyncio + +from datvia.mongodb import Client + +with Client.create("mongodb://127.0.0.1:27017/") as client: + client.admin.command("ping") + + +async def ping_mongodb(): + async with Client.create_async("mongodb://127.0.0.1:27017/") as client: + await client.admin.command("ping") + + +asyncio.run(ping_mongodb()) +``` + +#### MongoDB 分批写入与读取 + +`datvia.mongodb.utils` 接收原生 PyMongo 集合和游标。批处理器借用集合,MongoDB 客户端和连接池由调用方关闭。 + +```python +from datvia.mongodb.utils import BatchInsert, BatchUpdate, cursor_batches +from pymongo import UpdateOne + + +def migrate(source, target): + with cursor_batches(source.find({'ready': True}).batch_size(500), step=500) as pages: + with BatchInsert(target, step=500, ordered=False) as writer: + for page in pages: + writer.extend(page) + + +def mark_processed(target, ids): + with BatchUpdate(target, step=500) as writer: + writer.extend(UpdateOne({'_id': key}, {'$set': {'processed': True}}) for key in ids) +``` + +- `BatchInsert` 写入文档,`BatchUpdate` 支持 `add(filter, update)` 和原生 `UpdateOne`。`BatchWrite` 接收原生 `InsertOne`、`UpdateOne`、`UpdateMany`、`ReplaceOne`、`DeleteOne` 和 `DeleteMany`,可以混合插入、更新、替换和删除。 +- `extend()` 逐项消费输入,达到 `step` 立即写入。`step` 调整每批条数,驱动的网络读取批量可用游标的 `batch_size()` 单独调整。大输入可使用生成器或游标,调用方应及时释放消费完的批次。 +- `execute()` 和 `close()` 都会写出尾批。上下文退出时也会写出尾批,包括因业务异常退出。需要放弃尾批时,在退出前调用 `discard()`。每次尝试写入后都会释放该批缓冲,失败的批次由应用结合原生错误判断如何处理。 +- `on_error` 默认抛出原生 `BulkWriteError`,也可设为 `'log'` 或 `'ignore'`。`success_count` 分别累计已确认的插入数、修改数或混合写入影响的文档数。`error_count` 统计驱动报告的逐条写入错误,`unacknowledged_count` 统计使用非确认写入提交的操作数。 +- 有序批次可能在首个逐条写入错误处停止。保留默认错误策略可从原生异常的 `details` 查看部分写入结果和写关注错误。 +- `cursor_batches()` 接管传入游标,在离开 `with` 时关闭它。提前 `break` 后退出作用域,或消费过程中抛出异常,都会执行关闭。 + +异步使用 `AsyncBatchInsert`、`AsyncBatchUpdate`、`AsyncBatchWrite` 和 `async_cursor_batches()`。`extend()` 也接收异步可迭代对象。 + +```python +from datvia.mongodb.utils import AsyncBatchInsert, async_cursor_batches + + +async def migrate_async(source, target): + async with async_cursor_batches(source.find({'ready': True}), step=500) as pages: + async with AsyncBatchInsert(target, step=500) as writer: + async for page in pages: + await writer.extend(page) +``` + +#### MongoDB 分页读取与流式续读 + +`scan_collection()` 和 `stream_collection()` 接收集合与筛选条件,按 `_id` 升序逐条交付。适用于具有常规 `_id_` 索引、简单排序规则和全局唯一 `_id` 的普通集合。续读用索引下界定位,保留原筛选条件。每次查询读取当时可见的数据,恢复位置保存在本次遍历中。 + +```python +from datvia.mongodb.utils import scan_collection, stream_collection + + +def process_in_pages(articles, process): + for article in scan_collection(articles, {'approved': True}, step=1000): + process(article) + + +def process_in_stream(articles, process): + with stream_collection(articles, {'approved': True}, batch_size=1) as rows: + for article in rows: + process(article) +``` + +- 分页在交付前读完并关闭本页游标,适合批间有长时间等待的任务。默认每页最多 1000 条,内存开销随页大小和文档大小增长。默认 `max_retries=0`,显式开启后从上一页末尾的断点重读失败页。 +- 流式在游标有效时按需取数,遇到可恢复的读取中断时从最后交付的 `_id` 续读。默认 `batch_size=1`,同一位置最多额外尝试 3 次。离开 `with` 时关闭当前游标。较大的驱动批次可减少网络往返,也会增加缓冲。 +- 可恢复的读取中断包括游标失效、查询计划被终止和 `AutoReconnect` 类连接中断。权限错误、查询执行超时、业务异常和取消信号直接传播。重试次数耗尽时抛出最后一次原生异常。`max_retries` 控制 SDK 的恢复次数,驱动的读重试由 PyMongo 客户端选项单独控制。 +- 异步使用 `async_scan_collection()` 配合 `async for`,或使用 `async_stream_collection()` 配合 `async with` 和 `async for`。 + +### MySQL + +使用 MySQL Connector/Python。`create_async()` 需要 `await`,驱动完成建连后返回原生异步连接。示例从 `MYSQL_USER` 和 `MYSQL_PASSWORD` 环境变量读取凭据。 + +```python +import asyncio +import os + +from datvia.mysql import Client + +options = { + "host": "127.0.0.1", + "user": os.environ["MYSQL_USER"], + "password": os.environ["MYSQL_PASSWORD"], +} + +with Client.create(**options) as connection: + connection.ping() + + +async def ping_mysql(): + async with await Client.create_async(**options) as connection: + await connection.ping() + + +asyncio.run(ping_mysql()) +``` + +### Redis + +`create()` 和 `create_async()` 默认获取单机客户端。`connect()` 还支持 `mode="sentinel"` 和 `mode="cluster"`,可传 `is_async=True` 获取异步对象。Sentinel 模式返回 `Sentinel`,可用 `master_for()` 获取主节点的数据客户端。 + +```python +import asyncio + +from datvia.redis import Client + +with Client.create(host="127.0.0.1", port=6379) as client: + client.ping() + + +async def ping_redis(): + async with Client.create_async(host="127.0.0.1", port=6379) as client: + await client.ping() + + +asyncio.run(ping_redis()) +``` + + + +## 统一配置和显式连接管理 + +新高层接口只接受调用方传入的参数,不读取环境变量、不加载 `.env`。上文 MySQL 示例中的环境读取发生在调用方代码中。可以自行读取密钥服务或配置文件,再传入 SDK。 + +```python +from contextlib import ExitStack +from datvia import ConnectionConfig, Connections + +def build_connections(mysql_password): + return Connections({ + "orders": ConnectionConfig( + backend="mysql", host="127.0.0.1", username="app", + password=mysql_password, options={"database": "orders"}, + ), + "documents": ConnectionConfig( + backend="mongodb", host="mongodb://127.0.0.1:27017/", + ), + "cache": ConnectionConfig( + backend="redis", host="127.0.0.1", options={"db": 0}, + ), + }) + +connections = build_connections("由调用方提供的密码") +with ExitStack() as stack: + orders = stack.enter_context(connections.create("orders")) + documents = stack.enter_context(connections.create("documents")) + cache = stack.enter_context(connections.create("cache")) + for connection in (orders, documents, cache): + result = connection.check() + print(result.scope, result.elapsed_ms) + with orders.errors(operation="read_order"): + with orders.native.cursor() as cursor: + cursor.execute("SELECT 1") + print(cursor.fetchone()) +``` + +同名配置每次创建独立对象,由调用方决定何时复用与关闭。可以增加另一个名字来连接第二个 MySQL。新配置必须给出显式目标,重复的共同参数和原生参数会报错;`options` 中使用驱动原生名称,例如 MySQL 的 `database`、Redis 的 `db`、MongoDB 的 `authSource`。 + +创建保持驱动原生初始化时机,不额外探测。显式 `check()` 成功也不代表持续在线或拥有全部 CRUD 权限;SDK 不提供持续有效的 `is_connected` 状态。 + +### 异步 + +```python +async def read_document(connections, document_id): + async with await connections.create_async("documents") as connection: + await connection.check() + with connection.errors(operation="read_document"): + return await connection.native["app"]["items"].find_one( + {"_id": document_id} + ) +``` + +高层 `Connections.create_async()` 统一使用 `await`。原生工厂 `datvia..Client` 的调用方式见各后端示例。异步句柄应在创建它的事件循环内检查和关闭;同步与异步实例相互独立。 + +### Redis 拓扑 + +```python +sentinel_config = ConnectionConfig( + backend="redis", mode="sentinel", + nodes=[("127.0.0.1", 26379)], service_name="cache-primary", + options={"socket_timeout": 2, "sentinel_kwargs": {"socket_timeout": 2}}, +) +cluster_config = ConnectionConfig( + backend="redis", mode="cluster", + nodes=[("127.0.0.1", 7000), ("127.0.0.1", 7001)], + options={"socket_timeout": 2}, +) +``` + +Sentinel 的 `native` 仍是原生 Sentinel 对象,可以调用 `master_for()` 和 `slave_for()`。检查会通过指定服务建立临时主节点数据客户端,执行 PING 后清理。Sentinel 节点凭据通过 `sentinel_kwargs` 显式设置,不自动复制数据节点密码。 + +Cluster 检查客户端当前发现的全部主节点快照,逐个 PING;首次失败即抛出异常,未检查节点不会标记为正常。不会检查全部副本或保证集群持续健康。 + +### 异常和原生访问 + +```python +from datvia.errors import DatabaseError + +try: + with connections.create("orders") as connection: + connection.check() +except DatabaseError as error: + print(error.connection_name, error.phase) + # 原始对象、错误码、详情及已有异常链保存在 error.original。 + # 输出原始异常前,由调用方决定如何处理其中可能包含的私密信息。 +``` + +SDK 自动转换其自身配置、创建、检查及关闭范围内的已识别错误。原生业务操作只有位于 `connection.errors(...)` 中时才转换所属驱动的异常;上下文外完全遵循原生行为。惰性游标迭代和异步等待也必须位于上下文中。业务异常和取消信号原样传播。 + +统一类别不意味着写入必然失败或可以安全重试。没有可靠分类的驱动错误保留为 `DriverError`,不按错误文本猜测原因。原始对象在 `original` 中,因果链通过 `__cause__` 保留。已有主异常时发生的关闭错误附加在 `database_cleanup_errors` 中,不覆盖主异常。 + +### 资源和超时 + +高层上下文管理器清理 SDK 创建的客户端和自有资源,不关闭调用方传入的外部池;MySQL 池化连接按驱动语义归还池。自行创建的游标、Pipeline 或额外客户端由调用方管理。不要依赖句柄为原生对象增加线程安全或并发安全保证。 + +`connect_timeout_s` 是正整数秒,仅控制建立连接阶段。读取响应、节点选择和等待连接池等限制通过 `options` 使用驱动支持的参数配置。MySQL Connector/Python 8.3 不支持后来 9.2 加入的 `read_timeout`,本 SDK 不承诺统一的整体检查硬超时,也不增加自动重试。 + +MongoDB 的检查结果使用 `scope="selected_server"`,`targets` 为空:原生 `ping` 不公开实际执行节点,因此 SDK 不猜测地址。Windows 上 MySQL 需要显式主机地址,不能仅用驱动不支持的 Unix socket 参数指定目标。同步 Redis Cluster 的 `startup_nodes` 不接受已经绑定客户端的节点,以防意外关闭调用方资源。 + +原生驱动负责异步创建返回句柄前的资源清理。Connector/Python 握手取消的限制和 Redis 拓扑的连接验证见 [连接管理验收记录](docs/connection-management-validation.md)。Redis 记录操作的验证见 [基础使用验收记录](docs/crud-basic-implementation.md#5-隔离真实服务验证)。 + +## 基础结构准备与键值操作 + +在使用记录 CRUD 前,可以通过 `datvia.schema.Schema` / `AsyncSchema` 显式准备 MySQL 数据库与基础表、MongoDB 普通集合及必要身份约束。已有结构只做兼容性校验,不自动迁移。Redis 直接绑定 Hash 命名空间,或通过 `RedisStrings` / `AsyncRedisStrings` 使用 String 单键、批量与 TTL 接口。 + +MySQL 的初始化连接应省略尚未创建的业务库并启用 autocommit;创建完毕后另建选定业务库的连接。MongoDB 的集合准备和 Redis 的键空间绑定保留各自概念。上述对象都借用既有连接,原生高级能力仍通过 `connection.native` 使用。 + +下面用 MongoDB 演示最短完整流程,包含配置、集合准备、绑定、增删改查和资源关闭。请传入演示实例地址并使用专用演示集合。 + +```python +from datvia import ConnectionConfig, Connections +from datvia.crud import Records +from datvia.schema import Schema + + +def demo(mongodb_uri): + connections = Connections({ + "documents": ConnectionConfig(backend="mongodb", host=mongodb_uri), + }) + with connections.create("documents") as connection: + Schema.mongodb(connection).ensure_collection("sdk_demo", "users") + with Records.mongodb( + connection, database="sdk_demo", collection="users" + ) as users: + users.insert({"_id": "u1", "name": "Alice"}) + print(users.get({"_id": "u1"}, fields=["name"])) + users.update({"_id": "u1"}, set_values={"name": "Alicia"}) + users.delete({"_id": "u1"}) +``` + +从显式配置到三种数据库同步、异步 CRUD 的完整代码见 [完整使用示例](examples/basic_usage.py),接口、结构管理和 String 语义见 [使用说明](docs/crud-usage.md)。实际验证范围及剩余事项见 [基础使用封装验收记录](docs/crud-basic-implementation.md)。 + + + +## 高级记录操作 + +`datvia.crud` 提供可选的记录层:MySQL 表、MongoDB 集合和显式选择的 Redis Hash 格式共用按键读取、字段投影及批量写入接口。它借用已有连接,保留 `connection.native` 的全部能力。 + +```python +from datvia.crud import Records + +with connections.create('orders') as connection: + with Records.mysql(connection, table='orders', key_fields=('id',)) as orders: + items = orders.get_many([{'id': 1001}, {'id': 1002}], fields=['status']) + result = orders.bulk_update([ + {'key': {'id': 1001}, 'set_values': {'status': 'paid'}}, + {'key': {'id': 1002}, 'set_values': {'status': 'cancelled'}}, + ]) + # 是否提交及如何回滚由调用方决定。 + with connection.errors(operation='commit_orders'): + connection.native.commit() +``` + +异步使用 `AsyncRecords`、`async with` 和 `await`,方法及返回结构一致。MongoDB/MySQL 支持有界条件查询和集合更新;Redis 只提供指定键的 Hash 操作,不扫描整个数据库模拟条件查询。 + +`fields=[]` 对存在记录返回 `{}`,未命中返回 `None`;MongoDB/Redis 的缺失字段与明确空值分开保留。批量失败提供逐项状态,已发送但无法确认的项标为 `unknown`,不会伪装成未执行。写入确认不等于事务提交,SDK 不自动重试、提交或回滚。 + +完整接口、同步/异步示例、Redis 格式与 TTL、原始错误及事务边界见 [高级记录操作使用说明](docs/crud-usage.md)。本功能的实际验证范围见 [CRUD 实施与验收记录](docs/crud-implementation.md)。 + +## 数据库账号与权限管理 + +`datvia.accounts` 借用现有管理连接,支持本库句柄或原生 PyMongo、Connector/Python、redis-py 客户端。MongoDB 用认证库和用户名定位账号,MySQL 用用户名和来源主机定位账号,Redis ACL 作用于固定的数据节点。下例中的连接由调用方提供。 + +```python +from datvia.accounts import Accounts, AsyncAccounts, MongoRole, MongoUser, MySQLAccount + +with connections.create("mongo_admin") as conn: + mongo = Accounts.mongodb(conn) + user = MongoUser(auth_db="staff", name="analyst") + mongo.grant_roles(user, roles=[MongoRole(db="reports", name="read")]) + details = mongo.get_user(user, expand_privileges=True) + +mysql = Accounts.mysql(native_mysql) +grants = mysql.show_grants(MySQLAccount(user="analyst", host="10.0.0.8")) + +async def inspect_redis(native_async_redis): + return await AsyncAccounts.redis(native_async_redis).get_user("analyst") +``` + +MySQL 账号写操作使用启用自动提交且没有活动事务的连接。Redis 的 `apply_user` 对同名用户按顺序应用规则,`save_acl` 单独确认当前节点的 ACL 文件写入。账号写入遇到 `AccountWriteOutcomeUnknown` 时,请先回读目标状态,改密可通过独立认证核对。各方法所需权限、使用示例与已验证的环境见[需求清单](docs/account-management-requirements.md)、[实现设计](docs/account-management-design.md)和[验收记录](docs/account-management-validation.md)。 + +## 文档与贡献 + +- [文档索引](docs/README.md) 汇总连接管理设计、验收记录和源码目录说明。 +- [发布日志](CHANGELOG.md) 记录功能、兼容性变化和验证范围。 +- [贡献指南](CONTRIBUTING.md) 说明开发环境、代码检查和提交约定。 +- [测试指南](tests/README.md) 提供离线测试及隔离数据库的验证命令。 +- [安全说明](SECURITY.md) 说明凭据、日志和安全问题反馈方式。 + +本项目采用 [MIT 许可证](LICENSE)。 diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000000000000000000000000000000000000..960bab978396a46439539311ca774443baaa5a52 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,11 @@ +# 安全说明 + +## 配置与日志 + +应用从自己的配置来源取得连接参数和凭据,再显式传给 SDK。保管好数据库 URI、密码和真实服务的测试配置,文档、测试和共享日志使用脱敏值。 + +`DatabaseError.original`、异常链和 `database_cleanup_errors` 会保留原始异常对象,其中可能包含驱动返回的连接信息。共享诊断信息前请检查连接名称、地址和原始异常详情,按需脱敏。 + +## 问题反馈 + +涉及凭据泄露或其他安全问题时,请通过项目维护邮箱联系维护者。报告附受影响的提交或版本、数据库及驱动版本、影响范围和使用占位值的复现信息。[联系维护者](mailto:msr20666@foxmail.com) diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000000000000000000000000000000000000..cd4334120fe3d4a582035f6eb1ce9633eb5ec659 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,42 @@ +# DATVIA 文档索引 + +## 使用与开发 + +- [项目 README](../README.md) 提供安装、原生客户端、统一连接管理和同步、异步用法。 +- [发布日志](../CHANGELOG.md) 记录各次发行的功能、兼容性变化和验收范围。 +- [贡献指南](../CONTRIBUTING.md) 提供本地环境、代码与文档风格、检查命令和分支提交约定。 +- [智能体编码工作规范](../AGENTS.md) 提供仓库协作规则和任务阅读入口。 +- [安全说明](../SECURITY.md) 说明连接凭据、异常日志和安全问题反馈方式。 + +## 连接管理 + +- [连接管理设计](connection-management-design.md) 记录配置、创建、检查、异常和关闭的接口契约。 +- [测试指南](../tests/README.md) 提供离线回归和隔离服务探测的运行方式。 +- [连接管理验收记录](connection-management-validation.md) 记录交付时实际验证的驱动版本、真实拓扑及待验证项。 + +## 账号与权限管理 + +- [需求与权限清单](account-management-requirements.md) 说明三个后端的身份、授权范围和操作者所需权限。 +- [实现设计与调用示例](account-management-design.md) 给出方法映射、异常契约和验证步骤。实际覆盖见[验收记录](account-management-validation.md)。 + +## CRUD 开发设计 + +- [基础结构与记录操作使用说明](crud-usage.md) 提供初始化、结构管理、批量操作、字段投影、同步/异步及 Redis Hash、String 的使用边界。 +- [适配器契约](crud-adapter-contract.md) 定义公共层与各后端的模块职责、调用和部分结果协议。 +- [CRUD 实施与验收记录](crud-implementation.md) 记录本地分支、Git 集成过程、测试及待验证项。 +- [基础使用设计与开发计划](crud-design.md) 说明从初始化到 CRUD 的接口、原生能力边界和完整示例。 +- [基础使用实施与验收记录](crud-basic-implementation.md) 记录基础结构管理与 Redis String 的实施和专项验收。 +- [同步异步完整示例](../examples/basic_usage.py) 演示三个后端同时使用;由调用方提供显式配置。 + +## 目录职责 + +| 路径 | 职责 | +| --- | --- | +| [src/datvia/](../src/datvia/) | 公共导出、连接配置、统一句柄和异常转换 | +| [src/datvia/_adapters/](../src/datvia/_adapters/) | 三种后端高层连接的创建、检查、关闭及驱动错误映射 | +| [src/datvia/crud/](../src/datvia/crud/) | 同步/异步记录与 Redis String 操作、条件模型、批量结果和后端适配 | +| [src/datvia/schema/](../src/datvia/schema/) | MySQL 基础库表与 MongoDB 普通集合管理 | +| [src/datvia/mongodb/](../src/datvia/mongodb/)、[src/datvia/mysql/](../src/datvia/mysql/)、[src/datvia/redis/](../src/datvia/redis/) | 后端原生客户端工厂及各自的辅助功能 | +| [tests/](../tests/) | 离线自动化测试 | +| [scripts/verify_connections.py](../scripts/verify_connections.py) | 使用显式测试配置验证真实连接 | +| [scripts/verify_basics.py](../scripts/verify_basics.py) | 在显式隔离服务上验证结构准备、完整读写及清理 | diff --git a/docs/account-management-design.md b/docs/account-management-design.md new file mode 100644 index 0000000000000000000000000000000000000000..d7c866272b0d897affc97b389eafd7a1a40e6c09 --- /dev/null +++ b/docs/account-management-design.md @@ -0,0 +1,236 @@ +# MongoDB、MySQL、Redis 账号与权限管理工具实现设计 + +本文记录 `datvia.accounts` 的公共接口、原生命令和首版验收方案。已执行的版本组合及测试结果见[验收记录](account-management-validation.md)。需求编号 R1–R9、使用场景和操作权限表见[需求文档](account-management-requirements.md)。既有连接与异常实现见[连接生命周期](../src/datvia/connections.py)和[异常转换](../src/datvia/errors.py)。 + +## 1. 组件与执行流程 + +1. 调用方把已有的本库连接句柄或原生驱动对象交给 `Accounts` 或 `AsyncAccounts` 的后端工厂。绑定阶段检查对象类型、后端、同步模式和适用拓扑,不发起网络请求。 +2. 返回后端专用的管理对象。模型与公共入口不导入数据库驱动,选定后端时再加载适配代码。管理对象只借用连接和连接池,操作方法只释放自己创建的资源。 +3. 调用方法时校验结构化参数和服务端所需的能力,构造一条命令或一个有明确输出的查询。权限裁决交给服务端。创建、授予角色、设置默认角色、持久化等可单独提交的步骤分别调用和确认。 +4. 对确认回复只投影公开字段。驱动错误复用现有分类并保留原因,已发送写入的确认失败走专门的结果未知路径。 + +| 入口 | 接受的输入 | 绑定后检查 | +| --- | --- | --- | +| `Accounts.mongodb(conn)` | 本库同步 MongoDB 句柄或 PyMongo `MongoClient` | 用户命令选择认证库,角色命令选择定义库 | +| `Accounts.mysql(conn)` | 本库同步 MySQL 句柄或 Connector/Python 的纯 Python、C 扩展及受支持的池化连接 | 执行写入前检查事务与自动提交状态 | +| `Accounts.redis(conn)` | 本库单节点 Redis 句柄或 redis-py 固定数据节点 `Redis` | 检查句柄模式及原生连接池类型,拒绝集群路由和可识别的 Sentinel 自动发现池 | +| `AsyncAccounts` 的同名工厂 | 本库 `AsyncConnection` 或对应后端的原生异步对象 | 工厂同步返回管理对象,执行方法使用 `await`。本库句柄还要检查所属事件循环 | + +绑定管理对象后,调用方若已关闭本库句柄,操作开始时报告 `ConnectionClosedError`。原生对象按驱动自己的关闭、重连和事件循环契约处理。绑定阶段只访问已知的类型或明确的属性,避免对 PyMongo 客户端使用会隐式生成数据库对象的属性探测。可沿用已有 schema 模块的绑定和异常上下文设计,扩展原生连接这条路径。 [已有绑定实现](../src/datvia/schema/_base.py) · [已有工厂](../src/datvia/schema/__init__.py) + +## 2. 公共输入、返回与错误 + +| 模型 | 字段或行为 | +| --- | --- | +| `MongoUser(auth_db, name)`、`MongoRole(db, name)` | 两个字段共同确定操作目标,命令中明确指定角色定义库 | +| `MongoPrivilege(resource, actions)` | BSON 资源描述和动作集合。保留空库名、空集合名等有原生含义的作用域;构造时复制调用方容器 | +| `MySQLAccount(user, host)` | 两部分独立传递,用户名和来源主机分别按服务端规则处理。调用方显式选择通配主机 | +| `MySQLScope` | 用不同构造器表示 `global_()`、`database(name)`、`table(db, name)`、`columns(db, table, names)`、`procedure(db, name)`、`function(db, name)` | +| `AclRule` | 有序规则。构造器表达启停、命令和类别增减、键或频道模式、清除旧规则、选择器、添加密码、删除单个密码、替换全部密码。密码参数从 `repr` 中排除 | +| 查询结果 | 列表返回类型化摘要,精确查询返回后端详情或 `None`。MongoDB 记录展开状态,MySQL 保留逐条授权原文和指定角色集合,Redis 投影可展示字段 | +| 写入结果 | 已确认的 MongoDB/MySQL 单项变更及 Redis 规则应用、保存正常返回 `None`。Redis 删除操作按服务器确认的删除计数返回 `bool`,权限模拟返回含 `allowed` 和 `reason` 的结果 | + +**错误与执行证据** + +| 情况 | 调用方收到的结果 | 后续核对 | +| --- | --- | --- | +| 本地参数错误、连接模式错误、写入前发现活动事务 | 已有配置或输入校验错误。管理命令尚未发送 | 修正输入或使用调用方准备的管理连接 | +| 认证失败、授权不足、明确的服务端拒绝 | 分类异常、操作名、目标和原始原因。权限拒绝不会转成空列表 | 检查操作者的授权,不修改未涉及的其他账号 | +| 精确读取成功且服务端确认缺失 | `get_user`、`get_role`、`get_account` 返回 `None` | 盘点范围、原生重复创建与删除错误分别处理 | +| 服务端缺少请求的能力 | 明确报告所需版本或特性及原始错误 | 改用该版本可用的操作。管理员缺权和服务端命令不可用分别分类 | +| 写入已交给驱动,随后丢失确认、超时且无法排除已执行,或 MongoDB 回复写关注错误 | `AccountWriteOutcomeUnknown(DatabaseError)`,包含后端、目标、操作和原始异常或服务端确认信息 | 新连接回读身份、角色或 ACL。改密使用获准的独立认证验证 | +| 异步任务取消 | 传播 `CancelledError`。取消可能发生在写入发送之后 | 异步调用者可按同样流程核对状态 | +| 多步操作只完成前几步 | 已返回成功的步骤保持各自的确认状态,失败步骤单独报告 | 例如 MySQL 开户成功后只补授权,Redis 规则已应用而保存失败时只核对运行时状态与写盘条件 | + +SDK 不为账号写入增加自动重放。原生客户端沿用调用方的驱动重试设置,例如 redis-py 的 `retry` 决定哪些连接故障可由驱动自行重试。故障注入测试核对 SDK 对一次操作调用驱动的次数。读请求失败继续按读取错误处理。只有明确处于写入阶段并存在执行歧义时才转换为结果未知。异常摘要使用操作名和安全目标字段,避免将 SQL、BSON、ACL 规则或原始响应插入日志。保留的异常链仅供受控诊断。 [MongoDB 写关注](https://www.mongodb.com/docs/manual/reference/write-concern/) · [仓库安全约定](../SECURITY.md) + +## 3. 后端方法与原生命令 + +### 3.1 MongoDB + +| Python 方法 | 原生调用和契约 | +| --- | --- | +| `list_users(auth_db=...)` | 在目标认证库执行 `usersInfo: 1`,投影库名、用户名和直接角色 | +| `list_users_all_databases()` | 在 `admin` 明确执行 `usersInfo: {forAllDBs: true}`。按全实例权限裁决,原生命令一次返回数组 | +| `get_user(user, expand_privileges=False)` | 对指定用户使用 `usersInfo`。只有详情查询请求 `showPrivileges`。关闭凭据输出并对返回值做字段白名单投影 | +| `create_user(user, password=..., roles=...)`、`drop_user(user)`、`change_password(user, password=...)` | 分别用 `createUser`、`dropUser`、只设置 `pwd` 的 `updateUser`。密码操作限本地密码账号 | +| `grant_roles(user, roles=...)`、`revoke_roles(user, roles=...)` | `grantRolesToUser`、`revokeRolesFromUser`。每项角色使用显式 `role` 和 `db` | +| `list_roles(db=..., include_builtin=False)`、`get_role(role, expand_privileges=False)` | 使用 `rolesInfo`,按需请求 `showBuiltinRoles` 或 `showPrivileges`。详情保留直接和继承关系 | +| `create_role(role, privileges=..., roles=...)`、`drop_role(role)` | `createRole`、`dropRole`。创建时显式给出权限及继承角色,可为空 | +| `grant_privileges(role, privileges=...)`、`revoke_privileges(role, privileges=...)` | `grantPrivilegesToRole`、`revokePrivilegesFromRole`。传递结构化资源和动作 | +| `grant_roles_to_role(role, roles=...)`、`revoke_roles_from_role(role, roles=...)` | `grantRolesToRole`、`revokeRolesFromRole`。先明确目标角色,再列出要增减的父角色 | + +同步和异步共用命令构造与响应解析,分别调用 `client[db].command(...)` 和 `await client[db].command(...)`。列表查询不会附加只适用于指定用户的权限展开参数。读取用户详情和角色详情时区分未展开、展开后为空、权限不足三种情况。创建和角色增减等写入要检查原生命令成功标志以及 `writeConcernError`。服务端已作出变更而写关注未满足时,不得向调用方报告已回滚。对指定单库资源增减权限,要在该资源所在库具备相应动作。单条权限涉及多个数据库或集群资源时,检查 `admin` 上的 `grantRole` 或 `revokeRole`。删除角色会清理引用,分片集群还需考虑授权缓存刷新后再核对。 [用户信息](https://www.mongodb.com/docs/manual/reference/command/usersinfo/) · [角色信息](https://www.mongodb.com/docs/manual/reference/command/rolesinfo/) · [删除角色](https://www.mongodb.com/docs/manual/reference/command/droprole/) + +### 3.2 MySQL + +| Python 方法 | SQL 或读取来源 | +| --- | --- | +| `list_accounts()`、`get_account(account)` | 查询 `mysql.user` 的 `User`、`Host`、`account_locked` 等必要列。精确查询同时指定用户名和主机 | +| `show_grants(account, using_roles=())` | `SHOW GRANTS FOR`,按需附加 `USING`。返回主体、角色上下文和原文行,保留部分撤权输出 | +| `list_role_bindings(account)`、`get_default_roles(account)` | 查询 `mysql.role_edges` 的授予方向和转授标志,及 `mysql.default_roles`。角色绑定以目标成员为查询条件,直接父角色与默认角色分开返回 | +| `create_user(account, password=...)`、`drop_user(account)`、`change_password(account, password=...)` | `CREATE USER`、`DROP USER`、`ALTER USER ... IDENTIFIED BY`。单项严格执行,保留重复创建等原生拒绝原因 | +| `lock_user(account)`、`unlock_user(account)` | `ALTER USER ... ACCOUNT LOCK/UNLOCK` | +| `create_role(role)`、`drop_role(role)` | `CREATE ROLE`、`DROP ROLE`。使用完整主体,说明相同主体也可作为登录账号使用 | +| `grant_privileges(account, scope, privileges, grant_option=False)`、`revoke_privileges(account, scope, privileges)` | 对目标范围生成 `GRANT ... ON ... TO` 或 `REVOKE ... ON ... FROM`。列权限按列集合构造,过程和函数使用相应对象类型 | +| `grant_roles(account, roles=..., admin_option=False)`、`revoke_roles(account, roles=...)` | `GRANT role ... TO`、`REVOKE role ... FROM`。同一入口支持角色授予另一个角色。使用角色所带 `ADMIN OPTION` 转授时需按服务器要求激活该角色 | +| `set_default_roles(account, roles=...)` | 明确列表对应 `SET DEFAULT ROLE ... TO`,空列表对应 `NONE`。默认角色修改后在新会话核对 | + +**SQL 构造与作用域。** 用驱动支持的参数绑定分别处理用户名、主机和密码。使用结构化输入选择 SQL 关键字,每一段对象标识符独立引用。静态权限按服务器版本和对象类型匹配,动态权限只在全局范围使用并通过 `SHOW PRIVILEGES` 核对注册状态。读取账号信息时避免 `SELECT *` 与认证摘要字段。测试覆盖服务器 SQL 模式、特殊字符和 Connector/Python 纯 Python、C 扩展、异步路径。 [游标执行参数](https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlcursor-execute.html) · [服务器登记的权限](https://dev.mysql.com/doc/refman/8.4/en/show-privileges.html) + +**精确库名。** 目标为 `database("team_a")` 时,根据服务器版本和 `partial_revokes` 对数据库级授权模式的解释,编码能逐字匹配的库名,再构造 `ON ... .*`。对不支持该变量的早期 8.0 按对应版本规则处理。若无法确定精确语义,在发送写入前报告原因。授权与撤权必须采用相同的匹配规则。真实服务验收同时创建 `team_a` 和 `teamXa`,用受限账号分别验证;另测含 `%` 的库名和相关服务器设置。 [数据库级授权规则](https://dev.mysql.com/doc/refman/8.4/en/grant.html) + +同一条授权 SQL 的对象范围若同时含有 `%s` 和 `%(` 两种驱动参数标记语法,工具在发送前报告 `ConfigurationError`。若驱动在当前 SQL 模式下无法准确绑定含引号等特殊字符的值,工具会先执行无业务数据的探测并报告 `DependencyError`。 + +**转授范围。** `grant_option=False` 仅省略本次语句的转授子句。静态权限在同一授权层级共享相应转授能力,例如对某库的 `SELECT` 增加转授时,同库同层级已有的 `INSERT` 也可能被转授。全局动态权限逐项计算。验收时记录目标账号操作前后完整的 `SHOW GRANTS`,同时用第二个临时主体验证实际转授范围。 [GRANT OPTION](https://dev.mysql.com/doc/refman/8.4/en/grant.html) + +**事务与结果。** 在同步、异步写入路径复用现有 DDL 守卫的原则,要求 `autocommit=True` 且当前没有活动事务;检查失败时把连接交还调用方处理。每个游标由创建方关闭,清理失败不得覆盖已捕获的主要异常。管理连接上依次调用创建账号、分配角色、设置默认角色,分别保存确认结果。 [隐式提交](https://dev.mysql.com/doc/refman/8.4/en/implicit-commit.html) · [现有 MySQL 结构管理实现](../src/datvia/schema/_mysql.py) + +### 3.3 Redis + +| Python 方法 | 原生调用和契约 | +| --- | --- | +| `list_users()` | `ACL USERS`,返回当前所连数据节点的用户名 | +| `get_user(name)` | `ACL GETUSER`,缺失时返回 `None`。解析 `bytes`、字符串、RESP/驱动版本相应字段,对选择器也执行公开字段过滤 | +| `apply_user(name, rules=...)` | 将按序输入的规则编译为参数,一次发送 `ACL SETUSER`。用户名可被创建或更新。成批规则在同一节点、同一命令内确定先后顺序 | +| `delete_user(name)` | `ACL DELUSER`,按删除计数返回布尔值。保护 `default` 用户的服务端错误原样保留为原因 | +| `dry_run(name, command, *args)` | `ACL DRYRUN`,对 `OK` 返回允许;目标被拒时记录模拟结果及可展示原因。操作者本身缺权、目标不存在、命令错误按调用错误处理 | +| `save_acl()` | `ACL SAVE` 单独确认整节点的 ACL 文件写入。缺少外部文件配置或写入失败可独立诊断 | + +**规则构造。** `AclRule.enable()/disable()` 控制状态;`allow_command/deny_command` 和 `allow_category/deny_category` 控制命令;`key_pattern`、`channel_pattern` 及相应的清除规则控制资源。Redis 7.0 的读写键模式和选择器保留其分组和顺序。`add_password`、`remove_password`、`replace_passwords` 分开表达;替换全部密码在一次请求中先清空旧密码再添加新密码。构造规则时保留口令原值以供提交,但任何面向日志的展示都只给动作名。 [ACL SETUSER](https://redis.io/docs/latest/commands/acl-setuser/) + +**驱动适配。** redis-py 5.0 的高层 `acl_setuser` 默认会附加 `off`,所以按调用方明确给出的规则通过 `execute_command("ACL SETUSER", name, *rules)` 提交。某些驱动把命令和类别解析为两个列表,读到的显示顺序无法还原跨组授权顺序。展示模型只用于读取。DRYRUN 的拒绝原因可能是正常的非空文本回复,禁止用 `bool(reply)` 作为授权判定。 [驱动的 SETUSER 实现](https://github.com/redis/redis-py/blob/v5.0.0/redis/commands/core.py) · [驱动的响应解析](https://github.com/redis/redis-py/blob/v5.0.0/redis/_parsers/helpers.py) · [ACL DRYRUN](https://redis.io/docs/latest/commands/acl-dryrun/) + +**版本与拓扑。** 基础 ACL 对应 6.0,频道对应 6.2,选择器、读写键模式和 DRYRUN 对应 7.0。使用新特性时才检查相关能力,避免为了基础盘点要求额外的管理权限。首版结果只代表该固定数据节点。后续若支持 Sentinel 或 Cluster,应显式列出数据节点与候选副本,按节点报告应用、保存、拒绝和未知状态,并处理核对期间拓扑变化。 [Redis ACL](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/) · [ACL SAVE](https://redis.io/docs/latest/commands/acl-save/) + +## 4. 开发者调用示例 + +以下示例通过 Python 直接调用管理方法。`connections` 为调用方建立的 `Connections`,原生客户端和密码参数由调用方显式提供。管理对象在持有连接的作用域内使用。 + +```python +from datvia.accounts import Accounts, AclRule, MongoRole, MongoUser, MySQLAccount + + +def configure_mongo(connections, password): + user = MongoUser(auth_db="staff", name="analyst") + role = MongoRole(db="reports", name="read") + with connections.create("mongo_admin") as conn: + admin = Accounts.mongodb(conn) + admin.create_user(user, password=password, roles=[]) + admin.grant_roles(user, roles=[role]) + return admin.get_user(user, expand_privileges=True) + + +def inspect_mysql(native_mysql): + admin = Accounts.mysql(native_mysql) + user = MySQLAccount(user="analyst", host="10.0.0.8") + return admin.show_grants(user), admin.get_default_roles(user) + + +def rotate_redis_password(native_redis, new_password): + admin = Accounts.redis(native_redis) + admin.apply_user("analyst", rules=[AclRule.add_password(new_password)]) + return admin.get_user("analyst") +``` + +Redis 示例只添加一个密码。用于已有用户的轮换时,再以新密码独立验证、迁移调用方,最后单独移除旧密码。若用于不存在的用户,该操作遵守原生创建语义,启用与授权规则需要显式传入。 + +```python +from datvia.accounts import AsyncAccounts, MySQLAccount + + +async def inspect_async(connections, native_mysql, native_redis): + async with await connections.create_async("mongo_admin") as conn: + users = await AsyncAccounts.mongodb(conn).list_users(auth_db="staff") + + mysql = AsyncAccounts.mysql(native_mysql) + account = MySQLAccount(user="analyst", host="10.0.0.8") + grants = await mysql.show_grants(account) + + redis = AsyncAccounts.redis(native_redis) + acl = await redis.get_user("analyst") + return users, grants, acl +``` + +### 从开户到回收 + +下例使用三个调用方提供的管理连接,分别代表三种独立部署。密码由调用方在运行时传入。示例中的身份、角色名和键前缀应替换为当前隔离环境的专用目标。 + +```python +from datvia.accounts import Accounts, AclRule, MongoPrivilege, MongoRole, MongoUser, MySQLAccount, MySQLScope + +def grant_report_access(mongo_client, mysql_conn, redis_client, mongo_pwd, mysql_pwd, redis_pwd): + mongo = Accounts.mongodb(mongo_client) + report_role = MongoRole("reports", "analyst_read") + mongo.create_role(report_role, privileges=[ + MongoPrivilege({"db": "reports", "collection": "daily"}, ["find"]) + ]) + mongo.create_user(MongoUser("staff", "analyst"), password=mongo_pwd, roles=[report_role]) + + mysql = Accounts.mysql(mysql_conn) + user = MySQLAccount("analyst", "10.0.0.8") + role = MySQLAccount("report_reader", "%") + mysql.create_role(role) + mysql.grant_privileges(role, MySQLScope.table("reports", "daily"), ["SELECT"]) + mysql.create_user(user, password=mysql_pwd) + mysql.grant_roles(user, roles=[role]) + mysql.set_default_roles(user, roles=[role]) + + redis = Accounts.redis(redis_client) + redis.apply_user("report_reader", rules=[ + AclRule.enable(), AclRule.add_password(redis_pwd), + AclRule.allow_command("get"), AclRule.key_pattern("reports:*"), + ]) +``` + +分别用新账号建立独立连接,验证报表集合读取、报表表的 SELECT 和匹配键的 GET 确实获准,再对未授权的集合、表或键进行核对。MongoDB 可用 `revoke_roles` 撤销角色,MySQL 可以 `revoke_privileges` 调整角色的表权限,Redis 用 `AclRule.deny_command("get")` 撤销对应命令。多条授权路径并存时要核对其余路径。上例的 Redis 规则会追加到同名用户已有策略;初始化示例应使用新用户名,调整既有用户应检查其全部规则(含选择器)。各后端可复跑的验收脚本、显式配置字段和测试范围见[账号管理验收记录](account-management-validation.md)。 + +## 5. 实施顺序与交付物 + +| 阶段 | 依赖与工作 | 交付和验收门槛 | +| --- | --- | --- | +| 1. 公共契约 | 从经核对的 `develop` 基线确定模型、工厂、原生类型识别和错误上下文;在功能实施时更新原有 SDK 范围说明 | 公共入口无驱动环境可导入;借用绑定不联网;错误模式、关闭和循环检查有定向用例 | +| 2. 只读盘点 | 基于公共契约独立实现三个后端列表、详情及各自的权限读取 | 使用只读管理员验证可见范围,缺失、权限不足、未展开三种结果清楚可区分 | +| 3. 账号生命周期 | 实现 MongoDB/MySQL 新建、删除、改密,MySQL 启停,Redis 有序规则应用、删除和多密码轮换 | 每个后端在隔离服务上完成开户、凭据验证及回收 | +| 4. 授权与角色 | 实现 MongoDB 资源动作与继承,MySQL 对象范围、角色转授和默认角色,Redis 版本相关规则与授权模拟 | 用独立受限账号完成允许和拒绝的实际请求,核对撤权和无关授权 | +| 5. 异常与兼容 | 测试回包丢失、取消、写关注错误、事务保护、清理失败、驱动解析变化及 Redis 写盘失败 | 每一种故障能归到明确状态,日志和公开结果通过敏感信息检查 | +| 6. 集成与发布准备 | 完成定向测试、全量回归、隔离服务验证,补 README、示例、最低权限表及「未发布」日志 | 验收记录按版本给出证据,文档示例和公开签名一致,真实服务缺口列明 | + +实施时公共文件由协调者负责。公共契约经验证后,各后端在独立分支和 worktree 实现自己的适配、测试及隔离验收。协调者按提交逐个合入集成工作树,每次核对交付 SHA 和差异并重跑相关验证。Git 记录保存基线、交付和集成 SHA,按实际执行情况登记最终合并和推送。 [仓库协作规范](../AGENTS.md) · [贡献指南](../CONTRIBUTING.md) + +## 6. 测试矩阵与发布证据 + +下表是候选验收矩阵。服务器版本和 Python 驱动版本分别记录。最低驱动和一个当期驱动各跑代表组合,精确记录小版本,对不同服务器才支持的特性另设测试。仓库已有的连接或 CRUD 测试为环境准备提供参考;账号管理需要自己的验收结果。 [依赖声明](../pyproject.toml) · [已有验收记录](crud-implementation.md) + +| 后端 | 计划覆盖 | 高价值用例 | +| --- | --- | --- | +| 公共 | Python 3.10 及项目当前开发版本。本库与原生连接,同步与异步 | 延迟导入、绑定零 I/O、关闭归属、错误模式、异步取消、游标清理、保留原始异常 | +| MongoDB | 自管 7.0、8.0;PyMongo 4.6 起同步,4.9 起异步,并测当期驱动 | 管理员认证库、目标账号库、角色库不同;同名异库;全库盘点授权;继承与展开;独立追加;删除后的引用;写关注错误 | +| MySQL | 服务器 8.0、8.4;Connector/Python 8.3 起及当期版本,同步覆盖纯 Python 和 C 扩展 | 同名不同主机;已锁普通账号;部分撤权与默认角色;指定激活组合;全局至列/例程的授权;精确库名;绑定含引号的值;活动事务可回滚 | +| Redis | 服务器 6.0、6.2、7.2、一个 8.x;redis-py 5.0 及当期版本,固定数据节点 | 多密码轮换;单改权限保留启停;规则顺序;频道与选择器分版本;DRYRUN 的允许、拒绝与管理凭据缺权;删除断开会话;有无可写 ACL 文件 | +| 故障 | 驱动替身注入确定性故障,在隔离服务上核对可观察状态 | 写已执行但回包丢失,MongoDB 确认超时,Redis 已应用规则但保存失败;确保 SDK 不重复写入 | + +### 离线检查 + +使用 `unittest` 和驱动替身检查实际发出的数据库、命令、参数、顺序、响应投影和错误分类。分别用“存在但无权”“已确认不存在”“请求结果未知”的样本防止状态混淆。Redis 样本覆盖密码哈希字段和选择器嵌套字段;MongoDB 覆盖凭据与业务自定义数据;MySQL 覆盖含引号和 SQL 模式差异的值绑定。 + +功能实施后,从当前工作树源码运行定向账号测试,再运行完整离线回归。Ruff 对实际变更的 Python 文件做 lint 和格式检查。 [测试指南](../tests/README.md) + +```shell +python -B -m unittest discover -s tests -p "test_accounts*.py" -v +python -B -m unittest discover -s tests -q +python -m ruff check src/datvia/accounts +python -m ruff format --check src/datvia/accounts +git diff --check +``` + +### 真实服务验收与清理 + +1. 接受显式隔离配置,登记服务类型、精确版本、驱动版本、拓扑和连接模式。为本轮生成唯一的账号、角色及测试资源前缀,并记录实际成功创建的对象。 +2. 管理连接为临时账号授予范围明确的权限。受限账号用独立连接对同类对象执行一次应允许和一次应拒绝的操作。查询角色或 ACL 的结果与实际请求分别保存。 +3. 改密、锁定或撤权后,使用新连接重新认证、重做访问验证。Redis 另保留既有会话验证 `off` 和删除的差异,MySQL 指定激活角色,MongoDB 区分直接和继承的授权来源。 +4. 对连接中断、写关注和持久化失败,记录最后确认步骤,按对象回读。MySQL 额外验证管理写入被事务守卫阻止后,原连接可以回滚此前的业务写入。 +5. 仅清理本轮拥有的对象。发生创建结果未知时先核对唯一标识及隔离上下文,再决定清理。分别记录业务失败和清理失败。 +6. 验收记录登记提交 SHA、配置代号、同步/异步、本库/原生、服务器与驱动版本、测试命令、期望及实际结果、未验证组合。真实 Redis 节点列为首版 ACL 的发布验收项。历史项目曾记录真实 Redis 覆盖缺口,不能用旧替身结果代替本功能的真机验收。 [已有验证范围](crud-implementation.md) + +发布判定对应需求 R1–R9。三个后端须各完成“盘点 → 查看权限 → 开户或配置 → 授权 → 以受限账号验证 → 撤权或回收”,并能在任一步失败后说明已确认的结果、未知目标和核对方法。 diff --git a/docs/account-management-requirements.md b/docs/account-management-requirements.md new file mode 100644 index 0000000000000000000000000000000000000000..22d699c39bc35f196cf3aaf50c33d5680f25e241 --- /dev/null +++ b/docs/account-management-requirements.md @@ -0,0 +1,86 @@ +# MongoDB、MySQL、Redis 账号与权限管理需求 + +本文定义 Python 工具包 `datvia.accounts` 的首版需求。读者是使用本库的开发者、负责数据库账号的管理员,以及负责验收的工程师。方法、类型和版本矩阵均属设计目标。开发任务、命令映射、示例和验证步骤见[实现设计与交付规划](account-management-design.md)。 + +## 1. 使用目标与 SDK 边界 + +开发者已经有一条能访问数据库的连接,并持有相应管理权限。工具应让他能在 Python 中盘点账号、解释权限来源、授权、回收权限和轮换凭据。数据库若只允许服务器本地访问,脚本可以运行在服务器上,或复用调用方已建立的网络通道。SDK 接受本库命名连接和对应驱动的原生连接,凭据与连接均由调用方提供。 + +调用方准备数据库服务、网络通道和初始管理凭据,SDK 借用已授权连接管理后续账号。`Connections` 负责注册、创建、检查和关闭连接。首版交付 Python 类、方法和类型化结果。业务系统若需要员工身份关联、审批或操作审计,可以在调用这些方法时记录自己的操作者和业务原因。 [现行 CRUD 设计](crud-design.md) · [连接管理设计](connection-management-design.md) · [凭据处理约定](../SECURITY.md) + +## 2. 功能清单与业务验收 + +| 编号 | 场景 | 用户可完成的操作 | 验收证据 | +| --- | --- | --- | --- | +| R1 | 盘点 | 按明确的实例、认证库或节点列出账号 | 同名不同身份可分别查询。报告给出范围,查询权限不足时返回错误 | +| R2 | 权限核查 | 查看直接授权及可展开的权限,说明继承来源和展开条件 | MongoDB 显示继承角色,MySQL 标明使用的角色集合,Redis 显示规则和选择器 | +| R3 | 开户 | 创建账号或配置 Redis ACL 用户,设置初始凭据和指定权限 | 用新账号建立独立连接,应获准和应拒绝的真实请求各验证一次 | +| R4 | 授权与撤权 | 增减指定角色、对象权限或 ACL 规则 | 除目标授权外,独立授予的其他权限仍可回读并验证 | +| R5 | 角色管理 | 在 MongoDB、MySQL 创建角色,维护权限和角色继承,分配或删除角色 | 建立两级角色,分配给用户,回读父子关系,验证撤权或删除后的访问 | +| R6 | 凭据与停用 | 修改密码,按后端能力锁定、停用或轮换密码 | 独立新连接验证新旧凭据。已有会话的表现单独记录 | +| R7 | 账号回收 | 精确删除指定账号,重新盘点 | 目标账号无法重新登录,同名但身份不同的账号仍可独立使用 | +| R8 | 故障核对 | 从拒绝、部分步骤已完成、单次写入结果未知等状态继续处理 | 能定位目标和步骤,有可执行的回读或独立认证方式 | +| R9 | 直接集成 | 用同步或异步 Python 程序复用现有连接 | 本库句柄和原生连接均可调用。调用完成后借入的连接仍归调用方使用 | + +## 3. 三个后端的能力对照 + +| 能力 | 自管 MongoDB | MySQL 8.0、8.4 | Redis 固定数据节点 | +| --- | --- | --- | --- | +| 账号身份 | 认证库、用户名 | 用户名、来源主机 | 节点、ACL 用户名 | +| 账号详情 | 直接角色,可展开继承角色和资源动作 | 主体及锁定状态、直接授权、默认角色、指定角色组合 | 启停标志、命令与类别、键模式、频道、选择器 | +| 建立与删除 | 创建用户时可指定初始角色,按认证库删除 | 创建、删除完整的 `user@host` 主体,再分别授权 | 应用规则可创建或更新,删除返回是否实际删除 | +| 凭据与停用 | 修改本地密码。撤销角色影响授权,删除账号影响后续认证 | 支持改密、锁定、解锁 | 启停、多密码并存、增删密码和替换全部密码 | +| 分配权限 | 为账号分配角色,为自定义角色赋予资源和动作 | 对账号或角色直接授权,也可授予角色、指定默认角色 | 对目标用户设置有序规则,按命令、键及频道授权 | +| 公用角色 | 内置角色可查询和分配,自定义角色可维护 | 支持角色创建、继承、分配、撤回和删除 | 以用户 ACL 规则表达访问条件 | +| 确认范围 | 指定部署和认证库。分片集群用户由 `mongos` 管理 | 当前服务器返回的授权,以新登录会话核对生效状态 | 当前数据节点的运行时状态;写入 ACL 文件单独确认 | + +### 3.1 MongoDB 用户和角色 + +- **身份和盘点。** 管理员自己的 `authSource`、目标用户的认证库和角色的定义库是三个独立字段。例如,管理员在 `admin` 认证,可以给在 `staff` 创建的 `analyst` 分配 `reports.read`。按 `reports` 列用户只能说明账号创建在哪个库;要回答谁能访问报表,须盘点有权查看的账号,再检查其直接和继承权限。提供显式跨库盘点。`usersInfo` 返回用户数组;在首版需要处理大量账号时,可按已知认证库分批发起查询。 [用户身份](https://www.mongodb.com/docs/manual/core/security-users/) · [usersInfo](https://www.mongodb.com/docs/manual/reference/command/usersinfo/) +- **查看权限。** 列表返回身份和直接角色。详情可按指定用户展开 `inheritedRoles` 和 `inheritedPrivileges`。全库枚举与展开权限使用独立查询。返回值区分“未要求展开”和“展开后为空”。默认只公开允许展示的字段,排除凭据和业务自定义资料。 [usersInfo](https://www.mongodb.com/docs/manual/reference/command/usersinfo/) +- **管理账号。** 支持创建、删除、改密、追加和撤销角色。管理员给用户追加一个角色时,应保留原先的角色分配,包括其他管理员并发完成的独立追加。MongoDB 对 `updateUser.roles` 采用整组替换语义,增量变更应走专门的角色命令。 [updateUser](https://www.mongodb.com/docs/manual/reference/command/updateuser/) · [增量授权](https://www.mongodb.com/docs/manual/reference/command/grantrolestouser/) +- **管理角色。** 按定义库列出角色,调用方可选择内置角色。对自定义角色支持创建、删除、增减资源动作、增减继承角色。保留数据库、集合、集群资源和动作的原生表示。管理非 `admin` 库角色时,由数据库执行资源范围及角色继承限制。删除角色后需回读引用方,再验证受影响账号的权限。 [角色查询](https://www.mongodb.com/docs/manual/reference/command/rolesinfo/) · [创建角色](https://www.mongodb.com/docs/manual/reference/command/createrole/) · [删除角色](https://www.mongodb.com/docs/manual/reference/command/droprole/) +- **部署范围。** 首版覆盖自管 MongoDB 的数据库本地用户,密码操作面向密码认证账户。Atlas 数据库用户由 Atlas 管理平面维护。首版的自管命令和验收结果适用于自管部署。 [Atlas 数据库用户管理](https://www.mongodb.com/docs/atlas/security-add-mongodb-users/) + +### 3.2 MySQL 账号、角色和对象授权 + +- **主体。** 账号和角色共享 `user@host` 身份空间。用户名和来源主机分别传入,使用 `%` 时由调用方显式选择。账号列表保留锁定状态。角色绑定列表说明哪些主体已授给哪些成员,尚未分配的角色可能没有绑定记录。工具不得仅凭锁定状态把普通登录账号认定为可删除的角色。 [账号命名](https://dev.mysql.com/doc/refman/8.4/en/account-names.html) · [角色管理](https://dev.mysql.com/doc/refman/8.4/en/roles.html) +- **生命周期。** 创建、删除、改密、锁定、解锁针对完整主体。创建账号和授予权限分别确认,以便处理“已开户但授权被拒”的结果。删除账号或锁定后使用新连接验证登录,已有会话按服务器原生规则处理。 +- **权限解释。** 保存 `SHOW GRANTS FOR` 的完整逐行结果,包括部分撤权。支持按调用方指定的已授角色展开授权,列出默认角色。默认角色、强制角色、登录激活设置和会话中的 `SET ROLE` 都会影响解释。报告记录主体和展开条件;指定用户名的授权输出对强制角色的可见性遵循服务器规则。 [SHOW GRANTS](https://dev.mysql.com/doc/refman/8.4/en/show-grants.html) +- **授权范围。** 覆盖全局、库、表、列、存储过程和函数。静态权限须与对象类型匹配,动态权限按当前服务器登记情况在合法的全局范围使用。数据库级授权默认精确匹配库名,包括名称内的下划线和百分号。对象权限的 `GRANT OPTION` 和角色转授的 `ADMIN OPTION` 分别控制,默认关闭。省略转授选项会保留原有转授能力。静态权限的转授可能同时影响同一授权层级上的其他权限,全局动态权限的转授按单项生效。 [GRANT](https://dev.mysql.com/doc/refman/8.4/en/grant.html) · [SHOW PRIVILEGES](https://dev.mysql.com/doc/refman/8.4/en/show-privileges.html) +- **角色。** 支持创建、删除、配置角色权限、给用户或其他角色授予和撤销角色,以及设置默认角色。用户获授某角色后,可以通过指定该角色的展开查询解释权限,并在独立会话中实际激活验证。 [角色管理](https://dev.mysql.com/doc/refman/8.4/en/roles.html) + +### 3.3 Redis 命名用户与 ACL + +- **身份与查询。** ACL 作用于连接的数据节点。同一节点的不同编号逻辑库共享用户。列表使用 `ACL USERS`,单一用户详情使用 `ACL GETUSER`,只返回允许展示的标志和权限字段。服务端返回的密码摘要按凭据材料处理。 [ACL GETUSER](https://redis.io/docs/latest/commands/acl-getuser/) · [Redis ACL](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/) +- **创建与更新。** `apply_user` 采用 `ACL SETUSER` 的原生语义。不存在的用户会被创建,已存在的用户按从左到右的顺序应用规则。调用者可以在同一请求中指定账号启用、密码及允许访问的资源。读取再写入之间存在并发窗口,因此工具只确认规则已应用。 [ACL SETUSER](https://redis.io/docs/latest/commands/acl-setuser/) +- **权限表达。** 基础能力覆盖允许或拒绝命令和类别、键模式。Redis 6.2 起支持频道规则,7.0 起支持选择器和区分读写的键模式。对既有账号仅追加某命令时,应保留原有启停和凭据状态。对资源收紧权限时,应能显式清除原有键或频道模式后重设,避免宽泛的旧模式仍然有效。 [ACL 规则](https://redis.io/docs/latest/commands/acl-setuser/) +- **轮换与回收。** 支持添加密码、移除单个密码和替换全部密码,可先添加新密码,迁移客户端后再撤销旧密码。`off` 阻止后续认证,现有已认证连接可继续工作。删除用户会断开该用户连接,服务端保护 `default` 用户。 [用户规则](https://redis.io/docs/latest/commands/acl-setuser/) · [ACL DELUSER](https://redis.io/docs/latest/commands/acl-deluser/) +- **验证和持久化。** Redis 7.0 起可模拟某用户对具体命令及参数的权限。运行时授权与持久化是两个操作。显式 `ACL SAVE` 要求可写的外部 `aclfile`,保存当前节点的整套 ACL。保存失败时应能核对已在内存生效的状态。首版只接受固定数据节点,Sentinel、Cluster 的多节点管理另行规划。 [ACL DRYRUN](https://redis.io/docs/latest/commands/acl-dryrun/) · [ACL SAVE](https://redis.io/docs/latest/commands/acl-save/) + +## 4. 操作者权限与可见范围 + +调用方按需要授予管理账号相应能力。下表列出普通本地账号的常见授权路径,受保护主体、只读服务器或部署策略可能有附加要求。每项操作直接进行服务端授权检查,拥有单项写权限的管理员可直接使用该方法。 + +| 后端及操作 | 管理凭据所需能力或资源范围 | +| --- | --- | +| MongoDB 查询他人账号或角色 | 查看用户通常需其认证库的 `viewUser`,查看非自身获授的角色通常需角色定义库的 `viewRole`。查询自身资料适用服务端自身权限规则 | +| MongoDB 枚举全实例用户 | MongoDB 8.0 在集群资源上检查 `viewUser`,拥有个别认证库的读取权限只适合相应库的盘点。[服务端授权检查](https://github.com/mongodb/mongo/blob/r8.0.0/src/mongo/db/commands/user_management_commands_common.cpp#L321-L350) | +| MongoDB 用户创建、删除、改密 | 用户认证库上的 `createUser`、`dropUser` 或 `changePassword`。创建时带角色还需各角色定义库的 `grantRole` | +| MongoDB 授予和撤销角色 | 对所授或所撤角色的定义库具有 `grantRole` 或 `revokeRole`。跨库授予时检查角色所在库 | +| MongoDB 创建、删除、修改角色 | 角色定义库对应的 `createRole` 或 `dropRole`,赋予、撤回权限与继承关系还需目标范围的 `grantRole` 或 `revokeRole`,依具体命令检查 | +| MySQL 列主体和绑定 | 可读取 `mysql.user` 的必要列、`mysql.role_edges` 或 `mysql.default_roles` 的对应列。各查询独立调用 | +| MySQL 查询他人授权 | `SHOW GRANTS FOR` 他人通常需要对 `mysql` 系统库的 `SELECT`。查询当前账号有单独规则 | +| MySQL 管理账号或角色 | 常用管理路径为全局 `CREATE USER`。单独创建或删除角色也可由 `CREATE ROLE` 或 `DROP ROLE` 授权,并遵循服务器对目标的限制 | +| MySQL 授权和设置默认角色 | 授对象权限需要相应权限及转授权资格。分配角色使用 `ROLE_ADMIN` 或对该角色的 `ADMIN OPTION` 等服务端认可的资格。设置他人默认角色需要 `CREATE USER` 或对 `mysql.default_roles` 的 `UPDATE` | +| Redis ACL | 按目标版本授予所需 ACL 子命令,例如 `USERS`、`GETUSER`、`SETUSER`、`DELUSER`、`DRYRUN`、`SAVE`。写盘另需服务进程能写入配置的文件 | + +权限依据见 [MongoDB 账号创建](https://www.mongodb.com/docs/manual/reference/command/createuser/)、[MongoDB 角色管理](https://www.mongodb.com/docs/manual/tutorial/manage-users-and-roles/)、[MySQL 授权](https://dev.mysql.com/doc/refman/8.4/en/grant.html)、[MySQL 默认角色](https://dev.mysql.com/doc/refman/8.4/en/set-default-role.html)和 [Redis ACL](https://redis.io/docs/latest/operate/oss_and_stack/management/security/acl/)。 + +## 5. 首版通过条件 + +1. 每个后端用显式配置的隔离服务走完 R1–R9 中适用场景。每项变更分别由管理连接回读和受限账号新连接验证。账号有多个权限来源时,撤销一项后继续追溯其他授权来源。 +2. 只读管理员可以盘点自己的可见范围。对无权查询的用户、角色或规则,返回拒绝信息。只有查询成功且能确认目标缺失时,详情才返回 `None`。 +3. 两种绑定和两种执行方式遵守连接归属。本库句柄关闭、异步取消、权限不足、网络中断及 Redis 版本差异均有可核对的结果。 +4. 普通日志、可展示详情和异常摘要排除密码、密码摘要、连接 URI 和命令中承载的凭据。受控诊断保留原始异常和因果链。 +5. 验收记录给出实际服务器、驱动、拓扑、调用模式、运行命令和结果。计划版本、替身测试和已通过的真机组合分别标记。具体矩阵和各后端用例见[实现设计与交付规划](account-management-design.md)。 diff --git a/docs/account-management-validation.md b/docs/account-management-validation.md new file mode 100644 index 0000000000000000000000000000000000000000..b4e8a3839564f02ea3d2ddf06406012fef7e550d --- /dev/null +++ b/docs/account-management-validation.md @@ -0,0 +1,53 @@ +# 数据库账号与权限管理验收记录 + +## 验收对象与口径 + +本次验收针对 `feature/database-account-management` 上的 `database.accounts`,开发基线为 `5208a2e`。验证使用当前工作树的 `src`、Python 3.10.20、临时回环服务、随机命名的主体以及独立的受限客户端。盘点、授予、撤销或删除后的状态由管理连接回读,访问效果由受限账号的实际请求验证。各后端的“组”按下表中不同的维度计数。 + +| 后端 | 服务器与驱动 | 组合定义与核对内容 | +| --- | --- | --- | +| MongoDB | 7.0.15、8.0.24;PyMongo 4.6.0、4.9、4.18.3 | 20 组。每个服务端覆盖 4.6 同步、4.9 和 4.18.3 同步及异步,各走本库句柄和原生连接。涵盖同名异库、角色继承、权限展开、追加与撤销、改密与回收、只读管理凭据被拒 | +| MySQL | 8.0.46、8.4.11;Connector/Python 8.3.0、26.10.0 | 16 组。每个服务器分别组合两个驱动、纯 Python/C 扩展及两种 `partial_revokes` 设置。检查账号来源主机、角色激活、对象范围、精确库名及隐式提交保护 | +| Redis | 6.0.20、6.2.18、7.2.8、8.0.4;redis-py 5.0.0、8.1.0 | 16 组。四个服务器版本分别组合两个驱动与 RESP2/RESP3。检查规则顺序与权限效果、多密码轮换、会话启停与删除、按版本模拟、ACL 文件写入并重启读回、保存失败后回读 | + +Redis 采用 Windows 社区 MSYS2 移植构建。每次验收使用新建的固定数据节点,目标编号逻辑库共享其 ACL。MongoDB 使用自管认证实例,MySQL 使用新建数据目录。矩阵对应这些实际版本与平台,未验证 Atlas 管理平面、Redis Sentinel/Cluster 的跨节点账号一致性或托管商专用权限模型。 + +## 集成结果 + +三种后端共实现 39 项操作,MongoDB 16 项、MySQL 17 项、Redis 6 项,同步和异步公开签名逐项一致。上表 52 组真实服务验收在集成工作树复跑通过,回环测试实例在验收后关闭。MySQL 的转授权验证与部分撤权验证使用不同受体,避免其他授权限制影响测试结论。MongoDB 与 MySQL 的测试对象创建遇到回包丢失时,脚本按本轮唯一身份回读并核对清理。 + +| 检查 | 实际结果 | +| --- | --- | +| 完整离线回归 | `python -B -m unittest discover -s tests -q`。当前驱动组合(4.18.3 / 26.10.0 / 8.1.0)324 项通过。最低驱动组合(4.6.0 / 8.3.0 / 5.0.0)324 项通过。驱动顺序为 PyMongo、Connector/Python、redis-py | +| 账号定向回归 | `python -B -m unittest discover -s tests -p "test_accounts*.py" -q`。最低驱动组合 34 项通过;完整回归亦包含这 34 项 | +| 静态检查 | 对账号包、定向测试和三个验收脚本执行 Ruff lint 与格式检查,13 个 Python 文件通过。Git 空白检查通过 | +| 构建与延迟导入 | 在临时源码副本中对产品提交 `0c0040f` 离线构建 wheel,确认其中包含所有账号模块。在关闭 site 初始化的解释器中直接从 wheel 导入 9 个公开入口、模型与异常,三个驱动均未被导入 | +| 文档 | 检查 README、需求、设计、测试与验收记录中的本地文件链接及账号示例 Python 语法 | + +## 如何复跑 + +设置 `PYTHONPATH` 指向当前工作树的 `src`,或在隔离环境中安装当前源码。Python 和原生驱动版本由运行者按目标组合选择。两个接收配置文件的脚本会在指定的隔离数据库**创建和删除测试对象**,所需的管理员凭据由运行者提供。配置保存在仓库之外。Redis 脚本接收服务器可执行文件,自己选择回环端口并启动临时服务。 + +| 验收入口 | 调用方式及配置 | +| --- | --- | +| [MongoDB 验收脚本](../scripts/verify_accounts_mongodb.py) | `python -B scripts/verify_accounts_mongodb.py --config <隔离配置.json> --mode both`。JSON 包含 `host`、`port`、`username`、`password`、`auth_db`。PyMongo 4.6 使用 `--mode sync` | +| [MySQL 验收脚本](../scripts/verify_accounts_mysql.py) | `python -B scripts/verify_accounts_mysql.py --config <隔离配置.json>`。使用 Connector/Python 连接参数,含 `host`、`port`、`user` 和 `password`。可用 `use_pure` 选择驱动实现,服务端配置决定 `partial_revokes` | +| [Redis 验收脚本](../scripts/verify_accounts_redis.py) | `python -B scripts/verify_accounts_redis.py --server --protocol 2`,将协议改为 `3` 复验 RESP3。脚本检查临时节点启动、重启、退出和文件清理 | + +实测时管理凭据未进入仓库。离线故障测试使用驱动替身确定性地模拟响应丢失、取消和写关注错误;这些结果与上表的真实登录、授权和撤权验证分别计入。配置说明与所需权限见[账号管理需求](account-management-requirements.md)。 + +## Git 集成记录 + +主智能体在仓库根工作树使用 `git status --short --branch`、`git diff --cached`、`git rev-parse HEAD` 和 `git worktree list --porcelain` 核对基线。公共契约通过四项定向测试后,按路径 `git add`、检查 `git diff --cached --check` 并提交。随后用 `git worktree add -b` 从公共提交建立下列三个工作树。各后端工作者在自己的工作树按路径暂存、检查、中文提交,协调者核对 `git diff HEAD...<交付 SHA>` 后在当前功能分支串行执行 `git merge --no-ff`。本表中的工作树路径相对主仓库目录。 + +| 执行者与分支 | worktree | 阶段或交付 SHA | 集成后的 HEAD | +| --- | --- | --- | --- | +| 主智能体,`feature/database-account-management` | `.` | 公共绑定 `ef0cb1a` | `ef0cb1a` | +| Redis,`feature/accounts-redis` | `../database-worktrees/accounts-redis` | `00fe5ac` | `cd95481` | +| MongoDB,`feature/accounts-mongodb` | `../database-worktrees/accounts-mongodb` | `8f3d03d` | `a2a5e7a` | +| MongoDB 验收清理修复,同一分支 | `../database-worktrees/accounts-mongodb` | `c656848` | `7f0a6ec` | +| MySQL,`feature/accounts-mysql` | `../database-worktrees/accounts-mysql` | `19dca93` | `39d334f` | +| MySQL 验收用例隔离,同一分支 | `../database-worktrees/accounts-mysql` | `d87d973` | `a2211e2` | +| 主智能体,`feature/database-account-management` | `.` | Redis Pipeline 绑定和清理异常处理 `0c0040f` | `0c0040f` | + +发布说明和验收记录在产品提交后单独暂存,执行 `git diff --cached --check`,通过后以中文文档提交收尾。 diff --git a/docs/assets/datvia-badges.svg b/docs/assets/datvia-badges.svg new file mode 100644 index 0000000000000000000000000000000000000000..41909c6371ef973b3e73575e2da4da434831ce06 --- /dev/null +++ b/docs/assets/datvia-badges.svg @@ -0,0 +1,19 @@ + + Python 3.10+, Sync and async API, MIT license + + + + + + + + + + Python + 3.10+ + API + sync + async + License + MIT + + diff --git a/docs/assets/datvia-banner.svg b/docs/assets/datvia-banner.svg new file mode 100644 index 0000000000000000000000000000000000000000..f729ac79b20f9ee7e610cba3de68a6a79796ab38 --- /dev/null +++ b/docs/assets/datvia-banner.svg @@ -0,0 +1,86 @@ + + DATVIA + DATVIA connects Python applications to MongoDB, MySQL, and Redis through one API. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PYTHON SDK + DATVIA + Configure · Connect · Check · Close + + + + + + + + + + Connections + sync / async + + + + + + + + + MongoDB + + + + + + + + + + + MySQL + + + + + + + + + + Redis + + + diff --git a/docs/connection-management-design.md b/docs/connection-management-design.md new file mode 100644 index 0000000000000000000000000000000000000000..ba014022a4976a995cf9607f6a296a180eecc0a3 --- /dev/null +++ b/docs/connection-management-design.md @@ -0,0 +1,143 @@ +# DATVIA 统一连接管理设计与开发流程 + +本设计整理于 2026-11-15。统一连接管理随 [DATVIA v0.1.0](../CHANGELOG.md#v010) 提供,实际验证范围见 [验收记录](connection-management-validation.md)。 +基线 develop:7d38b1ffe13a17b37d0bffc15e175085dcf0bb19。 +参考 temp/save:89497823698812e1c85e6c4c12af86d4281543cf。 + +## 目标和边界 +公共 SDK 支持同一项目同时使用 MongoDB、MySQL、Redis,以及多个同类型连接。提供统一配置、显式检查、保留原始信息的异常转换。保留现有 datvia..Client 的参数、返回类型、异步方式与原生异常,不封装 CRUD、事务、数据模型,不增加自动重试、后台检查、故障转移或自建连接池。 +SDK 只接收调用方显式参数;不读取环境变量、不加载 .env、不搜索配置文件、不提供环境读取助手。调用方可自行读取私密配置并传入。原生驱动自身已有行为按显式配置及驱动规则执行。 + +## 草稿复用 +| temp/save 内容 | 处理 | +| --- | --- | +| DatabaseType、配置校验 | 复用三种类型与有效规则,以标准库数据类实现,不新增 Pydantic | +| ConfigManager.load | 纯参数校验转换 | +| _process_env_vars、来源合并 | 不迁移,配置来源由调用方决定 | +| PoolConfig、HAConfig、重试、故障转移 | 不迁移,不公开未实现能力 | +| 异常基类 | 复用分类理念,保存原始对象、因果链、上下文 | +| BaseClient、BaseConnection | 提取生命周期理念,实现轻量同步异步句柄 | +| ConnectionState | 不提供持续 is_connected 状态 | +| CRUD、事务、Pipeline 草稿 | 不迁移 | +| 配置和异常测试 | 迁移有效行为场景,采用 unittest | +不整体合并或删除 temp/save,不改写用户原工作区。 + +## 公共接口 +```python +from datvia import ConnectionConfig, Connections +ConnectionConfig(backend, host=None, port=None, username=None, password=None, + connect_timeout_s=None, options={}, mode="single", nodes=(), service_name=None) +``` +Connections({name: config}) 保存配置,不连接、不缓存客户端。create(name) 每次返回独立 Connection;await create_async(name) 返回独立 AsyncConnection。name 必须是非空字符串;未知名称为 ConfigurationError。 +connection.native 为真实原生对象;check()/await check() 返回 CheckResult;close()/await close() 清理自有资源。支持 with/async with,上下文进入不增加探测。connection.errors(operation=...) 为普通 with,可在其中 await 原生方法。 + +### 参数契约 +backend 支持 mongodb/mysql/redis;port 为非 bool 整数 1..65535;connect_timeout_s 为非 bool 正整数秒。新高层配置必须提供显式目标,不能仅凭凭据或空配置连接 localhost。默认端口仍由驱动决定。 +host 对 MongoDB 可为 URI 或非空地址列表,其他后端为非空字符串。 +username 映射 MongoDB/Redis username、MySQL user;password 同名;connect_timeout_s 映射 MongoDB connectTimeoutMS(乘1000)、MySQL connection_timeout、Redis socket_connect_timeout。 +options 保留原生专有参数。MySQL database、Redis db、MongoDB authSource 等保持各自含义;MongoDB 业务数据库由 native 选择。 +共同字段与 options 中原生同义字段重复时报 ConfigurationError,不静默覆盖。同一个字段的原生别名重复也拒绝。节点与原生节点参数重复也拒绝。只校验 SDK 自身字段,不丢弃原生专有选项。 +复制标准配置容器,外部池/回调等对象保持引用。默认 repr 不打印密码、完整 URI、options。 +Redis mode 为 single/sentinel/cluster;nodes 为(host,port)列表。Sentinel 必须有 nodes 或显式 sentinels 原生参数,并提供 service_name;Cluster 可用 nodes 或 host 或原生 startup_nodes。 +Sentinel 节点凭据只通过 sentinel_kwargs 显式配置,不自动复制数据节点密码。 +外部 connection_pool 与共同连接参数混用拒绝,避免参数被池静默忽略。 +无 URI 环境变量替换;无隐式环境配置。原生工厂原来的默认行为不变。 + +### 创建和生命周期 +配置注册和校验不得有网络请求。创建保留各驱动原生初始化时机(MySQL 及同步 Redis Cluster 本身可能连网),不额外 ping。 +新高层 create_async 都需要 await;内部只等待原生 MySQL 工厂,不因 MongoDB/Redis 原生对象可等待而自动等待它。 +每次创建独立实例;没有全局单例、跨连接事务或额外线程安全保证。用户显式指定驱动池时允许原生资源复用。 +异步检查和关闭必须使用创建句柄的事件循环。成功关闭后幂等;关闭后 native/check 拒绝访问。已取出的原生引用无法收回。 +关闭仅释放自有资源,不额外提交、回滚或重连。外部池不关闭;MySQL 池化连接按原生语义归还池。原生游标/额外客户端由调用方管理。 +Redis 5.0 同步 close 不一定断开池,需显式处理自有池;异步优先 aclose,旧版本兼容 close。 +清理尝试所有资源;有主异常时保留主异常对象并附加 database_cleanup_errors;无主异常时抛首个转换后错误,附加其他清理错误。 + +### 检查 +CheckResult 冻结数据类字段:connection_name, backend, mode, checked_at(带时区UTC datetime), elapsed_ms(float), scope(str), targets(tuple[str,...])。 +MongoDB:native.admin.command("ping", read_preference=native.read_preference),异步等待;scope=selected_server。 +MySQL:native.ping(reconnect=False),异步等待;scope=current_connection。 +Redis single:PING;scope=single。 +Sentinel:native 仍为 Sentinel,通过 master_for(service_name) 创建临时客户端,PING 数据主节点并清理临时客户端/自有池;scope=sentinel_master。 +Cluster:异步必要时 initialize,再取 get_primaries() 快照逐个定向 PING;空列表失败;首个失败即停止,未检查节点不标正常;scope=cluster_primaries。 +失败抛带阶段/目标的统一异常,不返回 False。目标字符串不含凭据;不可无依据声称已检查所有节点。 +检查不写业务数据,不证明 CRUD 权限、持续健康、全部副本健康。无统一整体硬超时:MySQL 8.3 不支持后来9.2新增的 read_timeout;按实际原生参数约束各阶段。 + +### 异常 +DatabaseError 基类及 ConfigurationError, DependencyError, AuthenticationError, +PermissionDeniedError, DatabaseTimeoutError, DatabaseConnectionError, DriverError, ConnectionClosedError。 +字段 message, connection_name, backend, phase(config/create/check/operation/close), operation, target, original, database_cleanup_errors。 +original 保存原对象,raise unified from original 保留原因链、回溯、错误码、标签和详情。已有 DatabaseError 不重复包装。无法恢复驱动内部丢弃的信息。 +优先认证/权限/超时,再通用连接错误。MongoDB code18认证/code13权限;MySQL1045认证,1044/1142/1143/1227权限;Redis 专门子类优先。未知驱动错误为 DriverError。不按自然语言错误消息猜根因,不声明安全重试或确定写入结果。 +errors 上下文只转换所属驱动的异常;业务 ValueError/TypeError、取消、KeyboardInterrupt、SystemExit 原样传播。惰性游标迭代和 await 必须在上下文内。 +SDK 直接调用驱动的边界可以识别明确来源的底层超时/传输异常。 + +## 内部协作契约 +- datvia.config: DatabaseType, ConnectionConfig;snapshot() 返回配置的隔离副本;driver_kwargs() 返回共同字段映射后与 options 合并的全新参数字典。Redis nodes 转换由适配器负责,service_name 不传给驱动构造器。 +- datvia.errors: 上述异常;convert_error(error, *, connection_name, backend, phase, operation=None, target=None, classifier=None, owned=False) 返回统一异常或原异常;error_context 使用相同关键字并抛 from original;attach_cleanup_errors(primary, errors) 附加元组。 +- datvia.connections: Connections, Connection, AsyncConnection, CheckResult,运行时才 import datvia._adapters.。 +- datvia._adapters..Adapter(config, name, is_async=False):create()、async create_async() 返回原生对象;check(native)、async check_async(native) 返回(scope, targets);close(native)、async close_async(native);classify(error) 返回统一异常类或 None(非所属驱动错误)。 +- 适配器应在需要标注具体目标的检查异常处调用 convert_error(... owned=True, classifier=self.classify) 并 raise from;核心同样处理创建/检查/关闭外围异常。已统一错误不重复包装。 +- `errors` 和配置层独立于驱动。创建连接时由对应后端加载驱动。 +- 公共 __init__ 导出由协调者统一修改。worker 不改现有原生工厂文件。 +- 新测试独立 tests/test_connection_config.py、tests/test_connection_errors.py、tests/test_connections.py、tests/test_managed_.py。 + +## 开发、分支和历史 +总线 feature/database-connection-management 从 develop 已提交HEAD创建,独立 worktree;develop 只读,由用户最终审核合入。 +禁止 codex/Codex 前缀。功能 feature/,修复 fix/,重构 refactor/。 +配置 feature/database-connection-config 与异常 feature/database-connection-errors 并行;异常先集成,配置如需依赖从总线检查点合入。 +公共层 feature/database-connection-core 基于已集成配置异常;随后三个数据库分支从同一公共层检查点并行: +feature/database-mongodb-connections、feature/database-mysql-connections、feature/database-redis-connections。 +每worker独立worktree、分支、暂存区,负责自己的提交、测试、推送。协调者串行合入总线,维护导出/README/本文/验收。 +必要依赖只从总线明确检查点合入,不交叉复制或合并其他worker草稿。 +中文 Conventional Commits,按功能自然拆分并附必要测试;不把全部工作压成单提交;已推送历史不rebase/amend。 +总线使用 --no-ff 合并保留模块边界,不squash、不用单次cherry-pick替代分支。核对交付SHA和测试后合并并复验。 +当时允许向远端 `origin` 推送本任务明确分支;禁止推送develop、无关分支、--all、--mirror、force,禁止删除远端分支。worker推自己的分支,协调者推总线;推后核对SHA。鉴权失败仍完成本地可做工作并记录。 +不改历史worktree、不删除temp/save、不处理无关文件;保留任务分支和worktree供审核。 + +## 验收 +沿用 unittest 与 IsolatedAsyncioTestCase。覆盖:无环境读取、显式目标、配置隔离/冲突/映射、多实例及同名同步异步独立、原生对象身份、检查语义、异常对象和链保真、保守分类、取消透传、外部池保护、清理失败、驱动缺失诊断、旧工厂回归。 +保持 Python 3.10 及以上版本。默认安装 `pymongo>=4.6.0`、`mysql-connector-python>=8.3.0` 和 `redis>=5.0.0`,版本约束与 [requirements.txt](../requirements.txt) 一致。MongoDB 异步连接需要 PyMongo 4.9 或更高版本。 +验证最低及选定当前版本;PyMongo4.6/4.8同步及异步提示、4.9/4.13异步。 +隔离服务验证 MongoDB/MySQL8.4/Redis单机、Sentinel、Cluster 共10条同步异步路径,以及认证/权限/连接拒绝、未知服务、不可达主节点,多数据库并用和独立关闭。MySQL 原生 SELECT 1。外部测试进程限制故障测试时限,不冒充SDK硬超时。 +README补全显式配置、多实例、检查、原生访问、异常、资源归属和版本限制。 +交付总线/远端/基线及最终SHA/worker合并表/实际命令版本结果/未验证项/推送状态;用户最终合入develop。 + +## 实现后的边界说明 + +### 显式目标与配置隔离 + +- Windows 上仅提供 MySQL `unix_socket` 会被原生驱动忽略并回落到默认 TCP 地址,因此高层配置拒绝这种输入;POSIX 允许仅提供 socket。Windows 若显式提供 `host`,其余参数仍按驱动语义处理。 +- Redis 外部池必须是有效的真值对象。驱动会将假值池当成未提供池,可能转而使用默认地址,所以高层提前拒绝。 +- 同步 Redis Cluster 的原生 `startup_nodes` 会被驱动原地填入客户端。适配器为普通 `ClusterNode` 复制地址信息,保证复用配置时每个客户端独立;节点已经带有 `redis_connection` 时拒绝创建,防止关闭调用方资源。异步驱动自行从节点地址重建,两个受测版本已验证不共享节点连接。 +- `options` 中其他外部对象继续保留引用。SDK 只隔离标准配置容器,不承诺隔离调用方主动共享的原生资源。 + +### 检查结果的准确含义 + +| 后端或模式 | `scope` | `targets` | +| --- | --- | --- | +| MongoDB | `selected_server` | 空元组。公开 `ping` 返回值不提供实际执行节点,不另读 `address` 触发不同的节点选择 | +| MySQL | `current_connection` | 当前连接公开的主机和端口;socket 使用标记,不暴露完整路径 | +| Redis 单机 | `single` | 客户端池中的目标 | +| Sentinel | `sentinel_master` | 临时数据客户端实际发现的主节点 | +| Cluster | `cluster_primaries` | 本次快照中依次成功检查的全部主节点 | + +无法可靠取得地址时不编造目标。检查失败直接抛错,不返回部分成功的 `CheckResult`。`checked_at` 为本次检查开始时的 UTC 时间,`elapsed_ms` 为单调时钟测得的总耗时。 + +### 取消与关闭 + +高层句柄退出时如已有业务异常或取消信号,清理发生的新异常(包括中断或取消)附加至该原对象的 `database_cleanup_errors`。没有主异常时,清理中的取消或中断原样传播。成功关闭之后再次关闭无额外操作;关闭失败不标记为成功,调用方可再次尝试。 + +句柄尚未产生前的构造失败仍受原生驱动实现约束。对 Connector/Python 26.10.0 的事件控制复现实验发现:异步握手期间取消可能在驱动尚未返回连接时留下未关闭 socket。SDK 无法获取这个未返回对象;本期不通过屏蔽取消、后台任务或无限等待改变驱动行为。已经返回的句柄按本节清理规则管理。此限制独立记录,不宣称覆盖驱动内部无法访问的资源。 + +### 验证状态与交付 + +实现、测试及工作流程均在独立 worktree 完成。原工作目录不切换分支、不写入文档副本;本文在交付分支中的仓库相对位置为 `docs/connection-management-design.md`,审查合并后自然进入主工作目录。 + +MongoDB、MySQL 已完成隔离真实实例验证;Redis 三种拓扑已完成两版真实驱动对象和替身拓扑的测试,真实 Redis 服务及三数据库同时在线的验收仍待可用的隔离 Linux 环境。完整命令、版本、模块提交和合并记录见 [验收记录](connection-management-validation.md)。 + +## 调研依据 +- https://github.com/redis/redis-py/blob/v5.0.0/redis/client.py +- https://dev.mysql.com/doc/connector-python/en/connector-python-api-mysqlconnection-ping.html +- https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html +- https://www.mongodb.com/docs/manual/reference/error-codes/ +- https://pymongo.readthedocs.io/en/stable/api/pymongo/errors.html diff --git a/docs/connection-management-validation.md b/docs/connection-management-validation.md new file mode 100644 index 0000000000000000000000000000000000000000..3d7e8af44bbe4daef7d9c591a55066a24b6e2c2c --- /dev/null +++ b/docs/connection-management-validation.md @@ -0,0 +1,132 @@ +# 统一连接管理验收与交付记录 + +日期:2026-11-15。实现及可用环境内的验证已完成;Redis 真实拓扑验证仍待补齐,不能将驱动替身测试视为真实服务验收。 + +## 分支与工作区 + +- 交付远端名称为 `origin`。 +- 唯一交付分支:`feature/database-connection-management`。 +- 基线 `develop`:`7d38b1ffe13a17b37d0bffc15e175085dcf0bb19`。 +- 完整源码及安装验收检查点:`b2c23a8454590c705789d571c120f0fb684d3a70`;其后仅补充本验收记录和清理文件末尾空行,不改变受测代码行为。 +- 集成 worktree:`../database-worktrees/connection-management`。 +- 原工作区保留在 `chore/update-git-config`,提交 `8f1352295217973a1b8624a8a01cd39cf88439a2`。 +- `temp/save` 保持 `89497823698812e1c85e6c4c12af86d4281543cf`,仅参考有效设计,没有整体合并。 + +最终交付提交可通过 `git rev-parse feature/database-connection-management` 获取。交付分支由用户审核并合入 `develop`,本次未对 `develop` 提交、合并、重写或推送。 + +## 实际模块交付与合并 + +每个模块在独立 worktree 开发,提交说明使用中文。公共配置与异常先并行,公共连接层建立后,MongoDB/MySQL/Redis 从同一 `d0706e7` 检查点并行。协调者审查后串行使用 `--no-ff` 合并,修复通过新提交记录,未压缩或改写已推送历史。 + +| 工作分支 | 模块完成提交 | 总线合并提交 | 内容 | +| --- | --- | --- | --- | +| `feature/database-connection-errors` | `4f1c918` | `e041e34` | 分类、原始错误及异常链、显式转换上下文、清理错误附加 | +| `feature/database-connection-config` | `636f7ba` | `f43b59c`、`b091724` | 显式配置、参数转换、容器隔离、冲突校验、假值外部池保护 | +| `feature/database-connection-core` | `03af1ce` | `7ccc660` | 命名配置集合、同步/异步句柄、生命周期、CheckResult | +| `feature/database-mongodb-connections` | `ce745ea` | `1c38d78` | MongoDB 创建、读取偏好检查、关闭、版本及错误分类 | +| `feature/database-mysql-connections` | `034443b` | `a41c775` | MySQL 同步/异步、非重连检查、原生错误分类及池化关闭 | +| `feature/database-redis-connections` | `1064c2a` | `0c6a1e8` | 单机、Sentinel、Cluster 的分发、逐主节点检查和资源清理 | +| `fix/database-cleanup-primary-error` | `4aae92a` | `6a99124` | 清理中的取消/中断不覆盖正在传播的业务异常 | +| `fix/database-mysql-explicit-target` | `a94add5` | `a9e8ec5` | Windows socket-only 配置不回落到默认 TCP 目标 | +| `fix/database-mysql-driver-capability` | `5c83b61` | `97a99a6` | 高层创建时缺少 C 扩展等能力转换为 DependencyError | + +公共导出、文档和验收脚本由协调者独立提交。所有任务分支已明确命名推送;没有 `--all`、`--mirror` 或 force push。远端原有 `feature` 分支导致的命名冲突由用户删除该旧分支解决,之后 `fetch --prune` 同步并继续使用原定 `feature/` 前缀。 + +## 功能范围 + +新增 `ConnectionConfig`、`Connections`、`Connection`、`AsyncConnection`、`CheckResult` 和公共异常层。不同数据库及同类数据库均以名称区分;注册只保存配置,每次创建独立句柄。原生对象从 `.native` 获取,CRUD、事务和原生返回值不包装。 + +SDK 不读取环境变量、`.env` 或隐式配置文件。调用方需要自行读取私密配置后传参。配置缺少目标时提前失败;共同参数与原生选项重复时不静默覆盖。标准容器复制,调用方显式提供的外部资源保留引用。 + +原生三个 `Client` 工厂目录、`pyproject.toml` 和 `requirements.txt` 相对基线无修改。既有参数透传、返回类型、异步调用方式和原生异常行为保留。新高层 `create_async()` 统一使用 `await`。 + +## 自动化与驱动矩阵 + +主验收环境为 Windows、CPython 3.10.20。使用隔离虚拟环境和独立 `--target` 驱动目录,运行前断言实际导入的版本,避免误用系统已安装版本。 + +| PyMongo | MySQL Connector/Python | redis-py | 完整 unittest 结果 | +| --- | --- | --- | --- | +| 4.6.0 | 8.3.0 | 5.0.0 | 84 通过 | +| 4.8.0 | 8.3.0 | 5.0.0 | 84 通过 | +| 4.9 | 8.3.0 | 5.0.0 | 84 通过 | +| 4.13.0 | 8.3.0 | 5.0.0 | 84 通过 | +| 4.18.3 | 26.10.0 | 8.1.0 | 84 通过 | +| 干净 `.[all]`:4.18.3 | 26.10.0 | 8.1.0 | 安装后的包 84 通过 | + +命令(从交付 worktree 运行): + +```powershell +$env:PYTHONPATH = "$PWD\src" +python -B -m unittest discover -s tests -q +``` + +最低版本目录分别使用 `python -m pip install --target <独立目录> pymongo==4.6.0` 等安装。将对应 PyMongo、MySQL、Redis 目录置于源码目录之前的 `PYTHONPATH`,在新进程中打印并断言三者版本,再执行相同 unittest 命令。不要在一个解释器中切换已经导入的驱动。 + +覆盖配置来源、参数映射与冲突、独立实例、原生对象身份、显式检查、错误类别/原对象/标签/详情/异常链、业务异常及取消透传、同步异步关闭、清理部分失败、外部池保护、Redis 三拓扑和原工厂回归。Redis 测试使用真实驱动类,替换网络执行或拓扑发现;同步和异步 Cluster 定向 PING 使用真实分发逻辑验证。 + +在未安装三种驱动的基础 Python 3.10.9 上额外验证:导入配置层、构造三种配置和注册集合均无驱动导入;六个创建入口只加载对应驱动,缺失时统一 DependencyError,保留原始 ModuleNotFoundError 及 cause。未修改基础环境。 + +## 干净安装 + +以下命令记录当时交付提交的安装过程。当前分支通过 `python -m pip install .` 安装全部三个后端,版本约束见 [安装说明](../README.md#安装)。 + +在独立 Python 3.10.20 虚拟环境执行: + +```powershell +python -m pip install ".[all]" +python -I -m pip check +python -I -B -m unittest discover -s tests -q +python -I -B scripts/verify_connections.py --config <调用方准备的隔离服务JSON> +``` + +安装成功,`pip check` 输出 `No broken requirements found.`。显式核对 `database.__file__` 位于该虚拟环境的 site-packages,验收使用安装后的包。 + +生成包版本 `database 0.1.dev55+gb2c23a845`。原有 utils Git 依赖实际解析为 `3f1820ff19ad8d51e83a0133609493897fa9dc2c`,未改写其仓库。安装网络使用 Git 的进程级 OpenSSL 后端配置,没有修改用户全局 Git 配置。 + +## 隔离真实服务 + +使用官网下载并校验哈希的 MongoDB 8.0.24 与 MySQL 8.4.11 Windows 二进制,使用专属临时数据目录、随机凭据和回环监听端口。未使用用户现有数据库;凭据不进入仓库或日志。 + +| 路径 | 真实服务结果 | +| --- | --- | +| MongoDB 同步 | PyMongo 4.6.0、4.8.0、4.9、4.13.0、4.18.3 检查与关闭通过 | +| MongoDB 异步 | 4.9、4.13.0、4.18.3 检查与关闭通过;4.6/4.8 给出升级提示并保留原始 ImportError | +| MySQL 同步 | Connector/Python 8.3.0 纯 Python 实现、26.10.0 纯 Python/C 扩展实现通过 | +| MySQL 异步 | 两版驱动均通过检查、SELECT 1 和关闭 | +| Redis 三模式同步/异步 | 未进行真实服务连接;驱动级测试通过 | + +MySQL 同步连接池验证关闭归还后重新获取相同 connection ID;两个独立连接关闭互不影响。认证错误 1045、库权限 1044、查询权限 1142 和拒绝连接均验证统一分类及原始对象保留。MongoDB 认证错误 18、权限错误 13、不可达目标的 ServerSelectionTimeout 均验证原始信息与分类。 + +集成检查点及安装后的包均运行 `scripts/verify_connections.py`:MongoDB/MySQL 同步和异步共四条真实路径通过,每条创建两个独立实例、关闭一个后检查另一个;另外在同一进程内同时持有 MongoDB 和 MySQL 连接,关闭其中一个后继续检查另一个。MySQL 使用原生游标执行 `SELECT 1`。 + +验收 JSON 格式如下,由脚本调用方读取,SDK 本身不读取文件: + +```json +{ + "orders": { + "backend": "mysql", + "host": "127.0.0.1", + "port": 3306, + "username": "isolated_test_user", + "password": "由调用方提供", + "connect_timeout_s": 3, + "options": {"database": "isolated_test"} + } +} +``` + +可向该 JSON 增加 MongoDB、Redis 三种拓扑和同类多连接名称;脚本逐项检查后再次同时持有全部命名连接。服务故障测试须由外部测试进程限制运行时间,不代表 SDK 提供整体硬超时。 + +## 未验证事项和已知限制 + +1. 本机没有 Docker/Podman;已有 WSL Ubuntu 无法启动,报告需要 WSL2 内核及虚拟机平台支持。没有为本任务修改系统虚拟化设置。Redis 单机、Sentinel、Cluster 的六条真实路径,以及未知 Sentinel 服务、实际不可达主节点、真实 Redis 认证/权限故障仍待隔离 Linux 环境复验。三种数据库同时在线的真实验收也待补齐,当前完成的是 MongoDB/MySQL 同时在线。 +2. Connector/Python 8.3.0 的 C 扩展在本机不可用,最低版本真实同步验收使用纯 Python 实现。高层显式要求缺失 C 扩展时现给出 DependencyError。26.10.0 的异步池在本机 Windows/Python 3.10 组合由驱动报告 NotSupportedError;不宣称支持驱动禁用的能力。 +3. Connector/Python 26.10.0 在异步握手中取消、原生工厂尚未返回对象时,事件控制复现显示已打开 socket 未关闭。SDK 无法获取该内部对象;没有引入屏蔽取消或后台等待改变行为。已返回句柄的清理和取消行为通过回归测试。 +4. 未验证所有 Python 3.10 以上版本、全部操作系统、TLS/复制集/failover 等原生选项组合。原生选项继续透传,其能力以所选驱动为准。 +5. MongoDB ping 不公开实际执行节点,成功结果 targets 为空;不通过额外选择服务器伪造目标。检查成功不证明所有 CRUD 权限或持续可用。统一异常不代表操作可安全重试。 + +## 历史与保护核对 + +使用 `git log --first-parent develop..feature/database-connection-management` 查看集成顺序,进入各 merge 查看模块迭代。使用 `git diff --check` 检查空白问题;核对原工作区状态、develop/temp/save 引用、原工厂和依赖文件差异。 + +任务分支及 worktree 保留供用户审查。验收结束仅关闭本任务启动的临时 MySQL/MongoDB 进程,临时数据和原有用户服务不删除。最终推送只指定总集成分支,并以 `git ls-remote origin <分支完整引用>` 核对远端 SHA。 diff --git a/docs/crud-adapter-contract.md b/docs/crud-adapter-contract.md new file mode 100644 index 0000000000000000000000000000000000000000..42f4471869089bba9a6573fcc7d475fdc69a1c6c --- /dev/null +++ b/docs/crud-adapter-contract.md @@ -0,0 +1,55 @@ +# CRUD 适配器实现约定 + +设计语义见 [高级 CRUD 设计](crud-design.md)。公共模块负责校验、资源上下文、分块、错误转换和结果汇总。适配器负责原生驱动操作、后端特有校验和可证实的结果映射。 + +## 工厂和生命周期 + +- 模块 `datvia.crud._mysql`、`_mongodb`、`_redis_hash` 导出同步 `Adapter` 和异步 `AsyncAdapter`。 +- 构造函数为关键字参数 `connection`、`key_fields`(有序字段元组)及后端参数。MySQL 接收 `table`。MongoDB 接收 `database`、`collection`、`session`。Redis 接收 `namespace`、`codec`。构造阶段只校验配置。 +- 同步类提供 `__enter__()`、`__exit__(exc_type, exc, traceback)`,异步类提供对应异步上下文方法。可返回自身,不能抑制主异常。释放适配器自行持有的资源,借用连接、客户端和 session 归调用方。 +- 入口上下文可构造 Sentinel 主节点数据客户端,不额外探测。Redis 适配模块暂用内部连接适配器配置 `connection._adapter._config.service_name`,该桥接不构成公共接口。 +- 公共层负责连接状态和异步事件循环归属,在每次操作以及进入上下文前检查。适配器不新增自动重试、提交或回滚。 +- 导入公共 CRUD 模块不加载数据库驱动。绑定所需的适配模块或驱动缺失时,公共工厂抛出 `DependencyError` 并保留原始导入异常及因果链。 + +## 方法签名 + +| 方法 | 参数及返回值 | +| --- | --- | +| `get(key, fields)` | `dict` 或 `None` | +| `get_many(keys, fields)` | 按传入键位置严格对齐的 `dict\|None` 序列 | +| `find(where, fields, limit, offset, order_by)` | 受 limit 限制的字典列表 | +| `count(where)` | 整数 | +| `insert(item)`、`update(item)`、`delete(item)` | `WriteInput -> WriteResult` | +| `insert_many(items)`、`bulk_update(items)`、`delete_many(items)` | 每次一个分块,返回 `ChunkResult` | +| `update_many(where, set_values)`、`delete_where(where)` | `WriteResult` | + +同步和异步类的操作同名,异步操作使用原生驱动协程。输入键为按声明字段顺序整理的字典。复合键必须整组匹配。 +公共层会消除 `get_many` 中完全相同的键,保持其首次出现顺序。适配器按块返回一项对应一个传入键,并区分未命中与空投影。适配器负责数据库排序规则造成的键比较差异和多匹配检测,公共层不依赖查询结果的自然顺序。 +Redis 不需要实现 `find/count/update_many/delete_where`,公共层会拒绝这些调用。 + +`fields` 为字段元组或 `None`。空元组要求已有记录返回空字典。查询内部携带的键和版本字段不外泄。投影中 BSON/Hash 缺失字段从结果字典省略。 +`where` 为 `_conditions.Condition | None`,普通字典已规范化成等值条件。非条件写入仅当公共层已审核 `all_records=True` 时收到 `None`。排序为 `(field, direction)` 元组序列,方向 `asc` 或 `desc`。 +`Eq(field, None)` 规范化为 `IsNull(field)`。`In` 使用 `values` 元组,空元组明确零匹配,列表中的 null 改用 `Or(In(...), IsNull(...))` 表达。`And/Or` 使用 `conditions` 元组,其他比较节点使用 `field` 和 `value`。MongoDB `Exists` 与 `IsNull` 只有字段参数。 +公共层只校验普通字段名,如无点号、NUL 或开头的美元符。各适配器负责各自的标识符、安全引用、参数绑定和 BSON 兼容性。 + +## 写入准备和结果 + +`_types.WriteInput` 包含 `key`、`values`、`ttl_s` 和可选的不透明 `prepared`。插入的 `values` 是完整输入记录,未提供完整身份时 `key=None`。更新的 `values` 是字段赋值,删操作的 `values={}`。 +`ttl_s is UNSET` 表示省略,`None` 表示显式移除过期时间。TTL 只支持 Redis。 +公共 `insert_many` 接收普通记录或 `InsertItem(record, ttl_s=...)`,普通记录中的同名业务字段原样保留。 +需要在发送前完成编码或类型校验时,实现同步的 `prepare_write(operation, item) -> WriteInput`(`operation` 固定为 `insert/update/delete`),可利用 `dataclasses.replace` 保存准备好的参数。可实现同步 `prepare_key(key) -> None` 验证读取键的编码。两个准备钩子都不访问网络。 +公共层在批量发送之前对整批执行结构、键、字段及适配器准备。若后端的静态校验(如字段/值编码)可判断是否能执行,放在准备期间。 +读取投影可实现同步的 `prepare_fields(fields) -> None`,公共层在 `get/get_many/find` 分发前调用,包括整个读取批次的首次发送之前。 +该钩子不访问网络,用于 SQL 标识符、Redis 保留字段、字段编码等后端静态校验;配置错误直接传播,不伪装为已发送批次的未知结果。 + +`_types` 定义以下共享类型,写结果尽可能保留原生对象。 + +- `WriteResult(backend, operation, acknowledged=None, matched_count=None, modified_count=None, inserted_count=None, deleted_count=None, generated_keys=None, native_result=None)`。完整生成身份使用字典元组,缺少证据时使用 `None`。 +- `WriteCounts(matched_count=None, modified_count=None, inserted_count=None, deleted_count=None)`。分块只有能准确汇总的计数才填写。 +- `BatchItem(index, key, state, result=None, error=None)`,`index` 是**分块内相对索引**,`state` 使用 `BatchState`。分块返回尽可能完整的项,公共层依据相对索引复原全局顺序。 +- `ChunkResult(items, counts=WriteCounts(), native_results=())` 提供当前块结果,后续块仍由公共层发送。 +- `ChunkError(original, partial_result)` 用于原生错误有已知部分结果的情况。原始错误为驱动交付的对象,`partial_result` 为 `ChunkResult`,包含已知状态和原生错误明细。没有逐项证据时可直接抛驱动异常。 + +已发送或可能发送的块无法判断的项留为 `unknown`。SQL 分组在块内停止时,明确未发送的组可标记 `not_sent`。读分块有已确认明细时,`BatchItem.result` 为 `ReadItem`,其 `index` 同样是分块内索引。 +核心将已发送块的未知项与后续未发送项区分,保留原始异常及因果链。捕获取消时按同样方式附加部分结果后重新抛出同一个对象。适配器有已知明细时通过 `ChunkError.partial_result` 提供;保留原始取消时,可在原对象上附加 `database_partial_result`(值为 `ChunkResult`)。 +`UnsupportedOperationError`、`RecordConflictError`、`RecordFormatError`、`RecordIdentityError` 位于 `src/datvia/crud/_errors.py`。适配器已明确分类的异常保留原生对象和 `raise ... from ...` 关系。其他驱动异常由公共层使用连接句柄现有转换。 diff --git a/docs/crud-basic-implementation.md b/docs/crud-basic-implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..c17817290795064ffd0522f46d65296bfbd8caab --- /dev/null +++ b/docs/crud-basic-implementation.md @@ -0,0 +1,124 @@ +# 基础使用封装实施与验收记录 + +日期:2026-11-17。 + +本轮落实 [基础使用设计](crud-design.md),在既有连接管理和记录 CRUD 之上增加基础结构管理与 Redis String 操作。代码已集成到本地 `feature/database-crud`,供用户审查。真实 Redis 验收未完成,不能据此宣称全部真实拓扑均已通过。 + +## 1. 交付范围与边界 + +- MySQL:`Schema.mysql()`、`AsyncSchema.mysql()`,数据库和基础表的创建、兼容检查、列表、存在性、描述和显式删除;`Column`、`TableSpec` 描述基础结构。DDL 要求 autocommit 且无活动事务,不执行 ALTER、USE、提交或回滚。 +- MongoDB:`Schema.mongodb()`、`AsyncSchema.mongodb()`,普通集合准备、目录读取和显式删除。默认复用内建 `_id`;自定义身份只准备必要的完整唯一索引,并通过原生命令完成应答确认就绪。保留集合创建成功、索引创建失败时的集合和原始异常。 +- Redis:`RedisStrings`、`AsyncRedisStrings` 提供 String 的单键和逐项批量读写、删除、存在性与 TTL 管理,覆盖三种拓扑的适配路径。批量使用非事务流水线,保留顺序位置、未知状态和原始错误,不承诺跨槽原子性。 +- 继续复用 `Records`、`AsyncRecords` 的单条与批量 CRUD、字段投影、条件能力及原始返回类型。Redis Hash 不模拟全库条件扫描,String 不进行 JSON 编解码或字段投影。 +- 不新增运行时依赖、ORM、结构迁移、重试、跨库事务或统一高级索引管理。原生客户端始终可访问;配置仍由调用方显式提供。 + +同步、异步的完整多数据库示例见 [examples/basic_usage.py](../examples/basic_usage.py)。该示例用独立 MySQL 初始化连接,再在同一作用域打开三个后端;一个连接可以再绑定多个记录对象。同类连接按名称分别创建,实例和关闭相互独立。 + +表定义的实际限制:标识符沿用基础记录层的规则;`varchar` 必须提供长度,`decimal` 默认精度为 `(10, 0)`,`datetime` 支持 0–6 位小数精度;`text`、`blob` 只支持省略默认值,或可空时显式声明 `None`。声明列不得为生成列或 `INVISIBLE` 列,额外可省略列可保留原生属性。 + +MongoDB 集合列表和 ensure 接口限定普通集合;describe 返回指定命名空间的原生元数据,drop 按原生语义删除明确名称。ensure 和 drop 都拒绝未确认写关注;自定义身份索引的复用仍需要目录查询和创建索引权限。分片唯一性遵循原生规则。 + +## 2. 基线、分支与工作树 + +- 本轮起点:`ce740b3aefa554c7e4f0babb7c09b9b53f520ea0`。 +- 集成分支:`feature/database-crud`,工作树为 `.`。 +- `develop` 始终为 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 +- 公共接口检查点:`bdc326d41fb43eb893ea9c85166be8ed474242a6`。 +- 公共清理修复:`ca247e3e80ea66ce633fcd516b3bec00df4a6491`。 +- 最终代码检查点:`938c2a4d5cfe3a09a8b7bc782ce6131b1e407dbd`;随后文档提交的完整 SHA 可用 `git rev-parse feature/database-crud` 查看。 + +任务开始时,工作区已有八份计划及相对路径文档修改。本轮按原意保留并提交为 `6d5abb030e239d0887966db714810a6578e0701f`,未恢复、覆盖或混入其他草稿。 + +| 工作者 | 分支 | 工作树(相对根目录) | 首次交付 SHA | 首次合并后的主线 SHA | +| --- | --- | --- | --- | --- | +| MySQL | `feature/database-crud-mysql-schema` | `../database-worktrees/crud-mysql-schema` | `dc963d2b777a0acda75b65a058bde14920eed576` | `87f1ef1b2d43df43c851edba8cae76912d704edf` | +| MongoDB | `feature/database-crud-mongodb-schema` | `../database-worktrees/crud-mongodb-schema` | `2e4c14752a65f061f1d022e8889142695fbbfd7a` | `f2d870f7ea793eb6ed73509f37857bab85c615c1` | +| Redis | `feature/database-crud-redis-strings` | `../database-worktrees/crud-redis-strings` | `808bd908715061cd5f8f0baaaeef6310bd89c110` | `055f8c8819d7749d7ed6cf6d1f56caf9dbcf8d43` | + +MySQL 最终复核增加声明列可见性检查:`c9d0ee79b4b94f1fe94c64db553cb813de76cefe`,再次合入后的主线为 `938c2a4d5cfe3a09a8b7bc782ce6131b1e407dbd`。原始模块提交及合并边界均保留。 + +## 3. Git 操作顺序 + +以下命令以仓库根目录为路径基准。并行 worker 内部各自串行执行;不同 worker 的完成先后不表示依赖关系。所有操作均为本地操作。 + +| 顺序 | 执行者与位置 | 操作与结果 | +| --- | --- | --- | +| 1 | 协调者,`feature/database-crud`,`.` | `git status --short --branch`、`git diff --cached`、`git worktree list --porcelain`、`git rev-parse HEAD`、`git rev-parse develop`;记录起点、八份已有文档改动及 22 个已有工作树。 | +| 2 | 协调者,同上 | 按路径暂存八份计划文档,核对暂存差异后 `git commit -m "docs(crud): 确定基础使用计划并统一相对路径"`,得到 `6d5abb0`。 | +| 3 | 协调者,同上 | 提交 Schema 绑定、异常上下文、String 结果类型及契约测试,得到 `bdc326d`;185 项离线测试通过。 | +| 4 | 三位 worker,各自工作树 | 从 `bdc326d` 分别执行 `git worktree add -b <表中分支> <表中工作树> bdc326d`,开始模块实现。新增三个工作树,不复用其他任务暂存区。 | +| 5 | 协调者,主线 | 提交清理异常保真修复 `ca247e3`。MySQL、MongoDB worker 分别在自己工作树执行 `git merge --ff-only ca247e3`,继承这一公共检查点;Redis 使用既有连接清理辅助逻辑。 | +| 6 | 三位 worker,各自分支 | 分别按所属实现和测试路径 `git add -- <路径>`、`git diff --cached --stat`、`git diff --cached --check`,创建中文提交。交付 SHA 如上表,不改写其他 worker 分支。 | +| 7 | 协调者,主线 | 核对 MySQL 分支 HEAD 与交付 SHA、`git diff HEAD...dc963d2`,执行 `git merge --no-ff -m "feat(schema): 合入 MySQL 基础库表管理" dc963d2`;记录 HEAD `87f1ef1`,25 项定向测试通过。 | +| 8 | 协调者,主线 | 同样核对 MongoDB 的 `2e4c147` 后,`git merge --no-ff -m "feat(schema): 合入 MongoDB 集合管理与身份约束" 2e4c147`;HEAD `f2d870f`,19 项定向测试通过。 | +| 9 | 协调者,主线 | 同样核对 Redis 的 `808bd90` 后,`git merge --no-ff -m "feat(redis): 合入同步异步 String 与批量操作" 808bd90`;HEAD `055f8c8`,18 项定向测试通过。 | +| 10 | 协调者,主线 | 提交示例、显式隔离验收脚本、脚本清理测试及公共无驱动导入测试,得到 `13091647d28093d12f1557239d744ea80b71d6df`,主题为 `test(crud): 补充完整使用示例与隔离服务验收`。 | +| 11 | MySQL worker;随后协调者 | worker 在原模块分支增量提交 `c9d0ee79b4b94f1fe94c64db553cb813de76cefe`;协调者核对后 `--no-ff` 合入,得到 `938c2a4d5cfe3a09a8b7bc782ce6131b1e407dbd`。没有 amend、rebase 或 squash。 | +| 12 | 协调者,主线 | 复验完整矩阵,按路径提交 README、设计、使用指南、测试指南、发布日志与本记录;最后检查 `git status`、`git diff --check`、`git log --first-parent` 和本地 `develop` SHA。 | + +没有执行 fetch、pull、远端查询或 push。没有切换、提交、合并或更新 `develop`。保留全部 25 个工作树,包括原有 detached 工作树;本轮三位 worker 的工作区干净。 + +## 4. 自动化验证 + +解释器为 Python 3.10.20,Ruff 0.17.5。显式设置 `PYTHONPATH` 指向本工作树 `src`,并实际核对 `database.__file__`,确保加载当前源码。矩阵目录只用于隔离依赖版本,未修改项目依赖声明。 + +| 环境 | PyMongo | Connector/Python | redis-py | 最终完整离线结果 | +| --- | --- | --- | --- | --- | +| 选定当前版本 | 4.18.3 | 26.10.0 | 8.1.0 | 256 项通过 | +| 最低同步版本 | 4.6.0 | 8.3.0 | 5.0.0 | 256 项通过 | +| 最低异步版本 | 4.9 | 8.3.0 | 5.0.0 | 256 项通过 | + +完整测试包含原生工厂、连接管理、原记录 CRUD、新 Schema/String、批量部分结果、清理错误及取消传播。无驱动导入测试以 `python -S` 子进程确认公共入口、表定义与结果类型不强制加载驱动。PyMongo 4.6 环境的异步功能用替身验证协议,真实驱动只支持同步;并未把测试数量解释为该版本具备原生异步能力。 + +各模块额外版本验证:MongoDB 在 PyMongo 4.6、4.9、4.13 各通过 19 项定向测试;MySQL 初次交付在 8.3.0、26.10.0 各通过 211 项该工作树全量测试,Redis 初次交付在 5.0.0、8.1.0 各通过 203 项该工作树全量测试。模块工作树的历史数量不同于最终集成数量。 + +从仓库根目录执行: + +```powershell +$env:PYTHONPATH = (Join-Path (Get-Location) "src") +python -B -m unittest discover -s tests -q +python -m ruff check --no-cache src/database/schema src/database/crud/_strings.py src/database/crud/_string_types.py src/database/crud/_types.py src/database/crud/__init__.py tests/test_schema_binding.py tests/test_schema_mysql.py tests/test_schema_mongodb.py tests/test_crud_strings.py tests/test_basic_verifier.py scripts/verify_basics.py examples/basic_usage.py +python -m ruff format --check --no-cache src/database/schema src/database/crud/_strings.py src/database/crud/_string_types.py src/database/crud/_types.py src/database/crud/__init__.py tests/test_schema_binding.py tests/test_schema_mysql.py tests/test_schema_mongodb.py tests/test_crud_strings.py tests/test_basic_verifier.py scripts/verify_basics.py examples/basic_usage.py +git diff --check +``` + +上述 lint、格式、差异检查通过。完整示例通过语法和导入检查;Markdown 本地链接及 Python 示例语法已核对。文档中的项目路径均使用相对路径。 + +一次受限只读沙箱测试因无法创建临时目录而失败;该次进程也未成功覆盖最低依赖路径,因此不计入版本验收。随后在获准的隔离测试环境重新执行,输出核对版本及源码路径,以上三组结果以该次成功运行及最终修复后复验为准。 + +## 5. 隔离真实服务验证 + +测试使用独立临时服务、显式 JSON 参数和随机目标名,不加载项目私密配置。验收材料保存在仓库外的隔离 QA 目录;此处仅记录文件名与用途,不记录本机绝对路径、密码或完整私密 URI。 + +- `basic-verification-config.json`:本轮 MySQL 与 MongoDB 两个隔离实例。 +- `basic-verification-pure-config.json`:纯 Python MySQL 同步路径。 +- `basic-verification-mongo-config.json`:MongoDB 最低同步版本。 +- `scripts/verify_basics.py`:仓库内可复用的验收脚本。使用 `--config <调用方明确指定的配置文件> --mode both`;最低同步版本使用 `--mode sync`。 + +MySQL 配置必须省略业务数据库并明确 `autocommit=True`。脚本创建完整随机 UUID 命名的测试库,业务连接先核对实际选中的库再进行 CRUD;使用完仅清理本次创建的对象。两个同类实例均先完成探测,再关闭第一个并检查第二个。全部被测后端在同一作用域保持连接,最终再次探测。测试断言不能用 Python `-O` 关闭,脚本对此显式拒绝。失败时仅输出异常类别、操作名和代码位置;清理错误附加到原错误。 + +| 服务与路径 | 实际结果 | +| --- | --- | +| MySQL 8.4.11,Connector/Python 26.10.0,C 扩展、纯 Python、异步 | 通过空库建表、重复初始化、结构查看、单条/批量 CRUD、投影、原生 SELECT 1 和清理。 | +| MySQL 8.4.11,Connector/Python 8.3.0,纯 Python与异步 | 同上通过。该隔离环境没有 8.3.0 的 C 扩展,未声称验证该扩展。 | +| MongoDB 8.0.24,PyMongo 4.18.3、4.9、4.13,同步与异步 | 通过普通集合、自定义身份索引、重复初始化、完整 CRUD 与清理;4.13 在 worker 定向真实场景验证。 | +| MongoDB 8.0.24,PyMongo 4.6.0,同步 | 通过集合与唯一约束准备、完整 CRUD、结构读取及清理。 | +| MySQL、MongoDB 同一进程同时使用,同类双连接 | 当前驱动和最低异步驱动组合的同步/异步均通过,关闭一个不会破坏另一个。 | +| Redis 单机、Sentinel、Cluster | 未完成真实服务验收;实现适配、离线驱动替身及 fakeredis 结果见下文。 | + +MySQL worker 真实场景还覆盖十种字段类型与默认值、额外可省略列、生成列、视图冲突、活动事务拒绝、重复删除、同定义并发创建及不兼容定义竞争。末轮在五条驱动路径复验不可见列:声明列冲突、额外可省略列兼容,27 项定向测试分别在两个驱动版本通过。真实 MySQL 8.4 的类型弃用警告促使建表使用 `BOOLEAN` 而非显式整数显示宽度,复验通过。 + +MongoDB worker 真实场景包括同名视图、capped/时序集合拒绝,索引排序规则和隐藏/降序选项复用,部分索引不充当完整唯一约束,重复数据导致的原始索引错误、集合保留、未确认写关注拒绝及并发初始化。等待索引完成的控制流程和权限异常传播由离线替身覆盖,未人为暂停真实索引构建或配置真实拒绝权限账号。 + +Redis 的额外语义验证使用 fakeredis 2.40.0,覆盖同步异步 String、TTL、str/bytes、WRONGTYPE 与批量结果。Cluster 使用原生 Pipeline 配合两个替身节点,验证跨槽命令路由及错误/清理路径;Sentinel 资源归属由替身验证。这些结果不能替代真实拓扑。 + +本轮随机测试库均已清理,协调者核对两个隔离实例中 `sdk_` 前缀测试库数量为零。两个自行启动的服务已正常关停,并复核原 PID 和测试端口均不再运行;隔离数据目录及历史验收材料保留,未删除其他工作树或用户文件。 + +## 6. 待验证事项与审查入口 + +1. 真实 Redis 单机、Sentinel、Cluster 的同步异步 Hash 和 String 操作,以及 Cluster 跨槽批量。当前没有 Redis 服务、Docker 或 Podman;WSL 因虚拟化条件不可用。未修改主机虚拟化配置,也未用替身冒充真实服务。 +2. 三个真实数据库同时使用的完整示例。当前真实端到端完成的是 MySQL 与 MongoDB 同时使用;三个后端的协作已有离线覆盖。 +3. 本轮新增结构接口在真实权限不足账号、真实网络中断和暂停索引构建条件下的故障验证。离线测试覆盖异常分类、原对象、清理失败、未知批量状态和取消传播,既有连接层历史结果另见连接验收记录。 +4. Connector/Python 8.3.0 C 扩展,以及 Python 3.10 以外解释器的本轮真实服务矩阵。 + +审查可先阅读 [使用说明](crud-usage.md) 和 [完整示例](../examples/basic_usage.py),再查看 `git log --first-parent ce740b3..feature/database-crud` 及 `git diff ce740b3..feature/database-crud`。代码已交付可审查,以上缺口保留为待完成验收;合入 `develop` 由用户决定。 diff --git a/docs/crud-design.md b/docs/crud-design.md new file mode 100644 index 0000000000000000000000000000000000000000..e46d76d8f2b3ccaa855bc1161b2af9dd85c54e21 --- /dev/null +++ b/docs/crud-design.md @@ -0,0 +1,769 @@ +# DATVIA 基础使用封装设计与开发计划 + +日期:2026-11-16 + +更新日期:2026-11-17 + +基础结构管理、记录操作和 Redis String 接口随 [DATVIA v0.1.0](../CHANGELOG.md#v010) 提供。接口用法见 [基础结构与记录操作使用说明](crud-usage.md)。 + +文档位置:`docs\crud-design.md`。 + +本文替代上一版仅面向记录 CRUD 的设计,保留已经实现的连接管理和记录 CRUD,并增加基础结构管理与 Redis String 操作。实际提交、验证环境和未验证事项见 [本轮实施与验收记录](crud-basic-implementation.md);文档中的目标范围不等同于全部真实拓扑均已验收。 + +## 一、目标与实现范围 + +### 1.1 交付目标 + +开发者安装本 SDK 后,应当能够仅使用本 SDK 的公开接口,完成以下基础流程: + +**显式配置 → 创建连接 → 准备存储结构 → 绑定操作对象 → 单条与批量 CRUD → 关闭资源。** + +同时满足: + +- 一个项目同时使用 MySQL、MongoDB、Redis。 +- 同一种数据库可以配置多个连接,分别面向不同业务、实例或数据库。 +- 同步和异步接口覆盖相同的基础能力。 +- 原生客户端始终可用,高级功能继续使用原生驱动。 +- 初始化由调用方显式执行,CRUD 不自动建表、改表、建立索引或迁移结构。 +- SDK 不读取环境变量、`.env` 或其他隐式配置来源。 + +数据库服务、集群部署和初始管理凭据由调用方准备。后续账号与授权管理由独立的 [datvia.accounts 工具](account-management-design.md) 提供。 + +### 1.2 三种数据库的基础封装边界 + +统一开发者的使用流程,同时保留各数据库真实存在的概念。 + +| 层次 | MySQL | MongoDB | Redis | +| --- | --- | --- | --- | +| 连接 | 连接实例,明确选择业务数据库 | 连接实例或集群 | 连接单机、Sentinel 或 Cluster | +| 初始化 | 创建数据库、声明并创建基础表 | 显式创建普通集合,由此完成数据库初始化 | 选择配置中的编号库,绑定 Hash 命名空间或使用 String 键 | +| 结构查看 | 列出、检查数据库与表,查看表结构 | 列出、检查数据库与集合,查看集合信息 | 查看指定键的存在性和 TTL | +| 基础结构删除 | 显式删除表、数据库 | 显式删除集合、数据库 | 删除指定键或已知记录 | +| 记录操作 | 行记录 CRUD、条件查询与批量操作 | 文档 CRUD、条件查询与批量操作 | Hash 记录 CRUD、按已知键批量操作 | +| 普通键值操作 | 不新增此抽象 | 不新增此抽象 | String 单键与批量读写、删除、过期控制 | +| 高级能力 | 通过原生客户端 | 通过原生客户端 | 通过原生客户端 | + +MongoDB 创建第一个真实集合即可完成初始化,不提供没有实际服务端意义的“创建空数据库”操作。[MongoDB 数据库与集合](https://www.mongodb.com/docs/manual/core/databases-and-collections/) + +Redis 不增加虚构的表或集合。编号库由部署提供,Cluster 仅支持数据库 0;Hash 命名空间是 SDK 的键命名约定,不是服务端容器。[Redis SELECT](https://redis.io/docs/latest/commands/select/) + +### 1.3 当前实现与本轮增量 + +集成和交付分支为 `feature/database-crud`。以下提交为本轮设计与开发的历史起点,完整演进见实施记录。 + +- 当前实现起点:`ce740b3aefa554c7e4f0babb7c09b9b53f520ea0`。 +- 本地 `develop`:`b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 + +现有连接管理、记录 CRUD、条件表达式、投影、批量结果和异常处理继续复用。本计划主要增加: + +1. MySQL 基础数据库与表管理。 +2. MongoDB 基础数据库与集合管理,以及必要的记录身份约束。 +3. Redis String 基础操作和批量操作。 +4. 从初始化到 CRUD 的完整示例与验收。 + +原有记录 CRUD 阶段的测试结果继续作为历史记录,不能替代新增功能的验收。已有实际接口参见 [CRUD 使用说明](crud-usage.md),历史结果参见 [CRUD 实施与验收记录](crud-implementation.md),连接层约定参见 [连接管理设计](connection-management-design.md)。 + +## 二、公共接口与行为约定 + +### 2.1 初始化与基础结构管理 + +新增 `datvia.schema`,提供: + +```python +from datvia.schema import AsyncSchema +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import TableSpec +``` + +绑定入口为: + +```python +mysql_schema = Schema.mysql(mysql_connection) +mongo_schema = Schema.mongodb(mongo_connection) + +async_mysql_schema = AsyncSchema.mysql(async_mysql_connection) +async_mongo_schema = AsyncSchema.mongodb(async_mongo_connection) +``` + +绑定本身不执行初始化。异步绑定不需要 `await`,异步管理方法需要 `await`。 + +管理对象借用连接,不拥有连接,也不要求再嵌套一层管理上下文。方法内部创建的游标由方法负责关闭。 + +#### MySQL + +| 方法 | 行为与返回值 | +| --- | --- | +| `ensure_database(name, *, charset="utf8mb4")` | 不存在则创建;存在则核对声明的字符集;成功返回 `None` | +| `list_databases()` | 返回当前账号可见的数据库名称列表 | +| `database_exists(name)` | 判断数据库是否在当前账号可见目录中 | +| `ensure_table(database, table, spec)` | 创建基础表,或校验已有表的兼容性;成功返回 `None` | +| `list_tables(database)` | 返回普通基础表名称列表 | +| `table_exists(database, table)` | 判断指定基础表是否可见 | +| `describe_table(database, table)` | 返回表信息字典;在可见目录中不存在时返回 `None` | +| `drop_table(database, table)` | 显式删除指定表;不存在时不报错 | +| `drop_database(name)` | 显式删除指定数据库;不存在时不报错 | + +`describe_table()` 返回数据库名、表名、字段信息、主键顺序、引擎及字符集等基础信息。字段信息保留实际类型、可空性、默认值和生成属性,不将复杂默认表达式伪装成 Python 值。 + +基础建表模型: + +```python +user_table = TableSpec( + columns={ + "id": Column("varchar", length=32), + "name": Column("varchar", length=100), + "age": Column("int"), + }, + primary_key=("id",), +) +``` + +本期约定: + +- `Column` 支持 `int`、`bigint`、`boolean`、`decimal`、`double`、`varchar`、`text`、`date`、`datetime`、`blob`。 +- 支持适用的长度、精度、小数位、可空性、字面量默认值,以及单一整数主键的自增。 +- `varchar` 必须声明长度;`decimal` 未指定精度时为 `(10, 0)`,`datetime` 支持 0–6 位小数精度,默认 0。`text`、`blob` 本期只支持省略默认值,或在可空时显式指定 `None`;不生成默认表达式。 +- `nullable` 默认 `False`;未提供默认值表示不声明默认值。 +- 不接受任意 SQL 类型片段、默认值表达式或约束表达式。 +- 新表使用 InnoDB,并继承目标数据库的字符集与排序规则。 +- 必须声明主键,可以是单字段或组合主键。 + +已有表采用已经确认的“基础可用兼容”规则: + +- 声明的字段必须存在,规范化后的类型、长度或精度、可空性及明确声明的默认值、自增属性需要匹配。 +- 主键字段及顺序需要匹配。 +- 声明字段不能是生成列或 `INVISIBLE` 列,避免基础写入失败或默认读取漏字段;额外可省略字段仍可保留这些原生属性。 +- 允许额外字段,但额外字段必须允许省略,例如可空、有默认值或由数据库生成。 +- 保留已有表的其他索引、引擎及高级约束,不擅自修改。 +- 不兼容时抛出 `SchemaConflictError`,列出差异,不执行 `ALTER TABLE`。 + +这项检查确认声明结构的兼容性,不保证任意业务数据都能绕过已有的外键、触发器或检查约束。 + +MySQL 创建、删除结构可能隐式提交。因此,结构写操作要求连接启用 `autocommit` 且没有活动事务;示例使用独立初始化连接。SDK 不暗中调整事务状态或执行 `USE` 切换业务库。[MySQL 隐式提交规则](https://dev.mysql.com/doc/refman/8.4/en/implicit-commit.html) + +#### MongoDB + +| 方法 | 行为与返回值 | +| --- | --- | +| `ensure_collection(database, collection, *, key_fields=("_id",))` | 显式准备普通集合及必要身份约束;成功返回 `None` | +| `list_databases()` / `database_exists(name)` | 查看当前账号可见的数据库 | +| `list_collections(database)` / `collection_exists(database, collection)` | 查看普通集合 | +| `describe_collection(database, collection)` | 返回集合类型、选项等原生元数据;不可见或不存在时返回 `None` | +| `drop_collection(database, collection)` | 显式删除集合;不存在时不报错 | +| `drop_database(name)` | 显式删除数据库;不存在时不报错 | + +集合准备遵循以下规则: + +- 默认使用 `_id`,不额外创建索引。 +- 声明自定义 `key_fields` 时,仅准备该身份所必需的完整唯一索引。 +- 已有等价约束可复用:索引字段及顺序与声明一致、具有唯一性、不使用 sparse 或 partial 过滤,且排序规则与记录查询一致;不自动删除、转换或重建其他索引。 +- 已有同名视图、时序集合或 capped 集合不作为普通记录集合接受。 +- 集合创建和必要索引建立必须得到确认,不能仅凭目录中出现索引定义就认定构建完成。 +- 如果集合已创建而索引创建失败,保留集合及原始异常,不自动删除集合补偿。 +- 初始化和删除均要求可确认的写关注;不使用 `w=0` 宣称结构操作成功。 +- 自定义身份约束需要目录查询和创建索引权限;复用已有索引也需取得原生命令的完成应答。 +- `describe_collection()` 保留指定命名空间的原生元数据,包含视图等类型;`drop_collection()` 按原生语义删除显式指定的命名空间。普通记录类型限制由 `ensure_collection()` 与集合列表执行。 +- 本期不处理分片规划;分片集合的全局唯一性继续遵守原生分片键规则。 + +MongoDB 自定义唯一索引可能因已有重复数据而建立失败,这属于应向调用方保留的真实错误。[MongoDB 唯一索引](https://www.mongodb.com/docs/manual/core/index-unique/) + +#### 共同管理规则 + +- `ensure_*` 不返回推测的 `created=True`,避免并发初始化时误报创建者。 +- 相同定义重复执行应成功;遇到并发创建,只在核实最终结构兼容后视为成功。 +- `drop_*` 只处理显式名称,不支持通配删除,不在上下文退出时自动调用。 +- “不存在”与“权限不足”分开处理;明确的权限异常继续抛出。 +- 列表和存在性检查反映当前账号的目录可见性,不承诺全局可见性。 + +### 2.2 记录 CRUD + +保留已有入口: + +```python +Records.mysql(connection, table=..., key_fields=...) +Records.mongodb(connection, database=..., collection=..., key_fields=...) +Records.redis_hash(connection, namespace=..., key_fields=..., codec=...) +``` + +`AsyncRecords` 提供相同绑定方式,操作方法使用 `await`。 + +| 能力 | 方法 | MySQL | MongoDB | Redis Hash | +| --- | --- | --- | --- | --- | +| 按键读取 | `get`、`get_many` | 支持 | 支持 | 支持 | +| 新增 | `insert`、`insert_many` | 支持 | 支持 | 支持 | +| 指定键修改 | `update`、`bulk_update` | 支持 | 支持 | 支持 | +| 指定键删除 | `delete`、`delete_many` | 支持 | 支持 | 支持 | +| 条件查询、排序、分页 | `find` | 支持公共条件范围 | 支持公共条件范围 | 不支持 | +| 条件计数 | `count` | 支持 | 支持 | 不支持 | +| 相同修改作用于匹配记录 | `update_many` | 支持 | 支持 | 不支持 | +| 条件删除 | `delete_where` | 支持 | 支持 | 不支持 | + +不支持的操作明确抛出 `UnsupportedOperationError`。Redis 不通过隐式扫描整个键空间来模拟条件查询和条件修改。 + +批量修改明确区分两种需求: + +```python +# MySQL / MongoDB:匹配的记录全部改为同一个值。 +records.update_many( + where={"status": "pending"}, + set_values={"status": "ready"}, +) + +# 三种记录后端:不同记录分别修改不同值。 +records.bulk_update([ + {"key": {"id": "u1"}, "set_values": {"age": 31}}, + {"key": {"id": "u2"}, "set_values": {"age": 42}}, +]) +``` + +MySQL 的 `update_many` 使用条件 UPDATE;`bulk_update` 按记录执行参数化修改并保留逐项结果。本期不引入 CASE SQL 生成器或批量执行优化框架。 + +其他约定: + +- `insert` 为仅新增;`update` 为仅修改已有记录。 +- `set_values` 只修改列出的字段,不修改身份字段。 +- 无条件批量修改或删除必须显式设置 `all_records=True`。 +- `find` 必须给出 `limit`,分页稳定性由明确排序及数据库实际并发语义决定。 +- 事务提交、回滚和重试仍由调用方决定。 + +#### 字段投影 + +| 参数 | 返回约定 | +| --- | --- | +| `fields=None` | 返回全部业务字段 | +| `fields=["name", "age"]` | 仅返回指定字段 | +| `fields=[]` | 已有记录返回 `{}`;记录不存在返回 `None` | + +三种后端分别使用 SQL 列选择、MongoDB 投影和 Redis Hash 字段读取完成投影。 + +- MongoDB 未请求 `_id` 时不附带 `_id`。 +- MongoDB、Redis 缺失字段省略;明确空值保留为 `None`。 +- MySQL 不存在的列保留为查询错误。 +- 不自动把 `Decimal`、`ObjectId` 等原生值转成字符串。 +- Redis String 不支持字段投影,也不自动解析其中的 JSON。 + +#### 返回结果与批量失败 + +保留已有结果结构: + +- 单条读取:字典或 `None`。 +- 批量读取:按输入位置对应的 `ReadItem` 列表。 +- 单次写入:`WriteResult`。 +- 批量写入:`BatchResult`。 +- 批量失败:`BatchOperationError.partial_result`。 + +返回值遵循证据: + +- 无法确定的匹配数、修改数或生成键使用 `None`。 +- 保留原生结果或游标关闭前取得的统计快照。 +- 不把 MySQL `rowcount` 无条件解释为修改条数。 +- 不推算连续自增 ID,不把执行成功等同于事务已提交。 +- 批量结果区分已确认、失败、未发送、未执行和结果未知。 +- 网络异常后的未知项不能当成失败后直接安全重试。 +- 已派发的同一批次中,某项失败不意味着其他项尚未执行。 + +### 2.3 Redis String + +在 `datvia.crud` 中新增: + +```python +from datvia.crud import AsyncRedisStrings +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +``` + +使用方式: + +```python +with RedisStrings(connection) as strings: + strings.set("app:message", "hello", ttl_s=60) + value = strings.get("app:message") +``` + +| 方法 | 返回值 | +| --- | --- | +| `get(key)` | `str`、`bytes` 或 `None` | +| `set(key, value, *, ttl_s=None)` | 成功确认后返回 `True` | +| `delete(key)` | `0` 或 `1` | +| `exists(key)` | `bool` | +| `expire(key, seconds)` / `persist(key)` | `bool` | +| `ttl(key)` | 原生 TTL 整数 | +| `get_many(keys, *, batch_size=500)` | 保留输入顺序、重复项和缺失位置的值列表 | +| `set_many(items, *, batch_size=500)` | `StringBatchResult` | +| `delete_many(keys, *, batch_size=500)` | `StringBatchResult` | + +`StringSetItem(key, value, ttl_s=None)` 支持每项独立设置过期时间。 + +具体语义: + +- 键和值只接受 `str` 或 `bytes`,不自动 JSON 编解码。 +- 使用调用方指定的完整键名,不隐式添加命名空间。 +- 返回类型遵循原生连接的 `decode_responses`。 +- 普通 `set` 为创建或覆写;未指定 TTL 时清除已有 TTL。 +- 指定 TTL 使用单条 `SET EX`,不拆成两次命令。 +- TTL 和 `expire` 接受正整数秒;移除过期使用 `persist`,立即删除使用 `delete`。 +- `ttl` 保留 `-2` 表示键不存在、`-1` 表示没有过期时间的语义。 +- `SET` 和 `DEL` 保留原生命令对同名其他类型键的作用;`GET` 遇到错误类型保留驱动错误。 + +这些行为与 Redis 原生语义一致。[Redis SET](https://redis.io/docs/latest/commands/set/) · [Redis TTL](https://redis.io/docs/latest/commands/ttl/) + +批量实现采用非事务流水线中的逐键命令: + +- 同步、异步均覆盖单机、Sentinel、Cluster。 +- Cluster 按驱动能力路由,不要求所有键处于同一槽。 +- 不承诺批量整体原子性、全局执行顺序或跨节点一致快照。 +- 写批次在发送前完成参数校验,按驱动实际编码后的键拒绝重复写入目标。 +- 新增轻量 `StringBatchItem`、`StringBatchResult`,保存逐项状态、实际返回值、错误及原生结果。 +- 复用 `BatchState`、`BatchOperationError`,不把 String 结果伪装成记录字典。 +- SET 成功不能推断是新增还是覆盖,不编造新增数、修改数。 +- 取消仍传播原取消对象,并附带已知部分结果。 + +Redis Hash 保持现有记录格式、显式 codec 和 TTL 约定。Hash 更新未传 TTL 时保留原 TTL,与 String 普通 SET 的默认行为分别说明。 + +### 2.4 原生能力、异常与资源 + +基础封装范围内的操作不要求调用方直接使用原生驱动。 + +以下能力继续通过 `connection.native` 使用: + +| 数据库 | 原生高级能力示例 | +| --- | --- | +| MySQL | 表结构变更、复杂索引、外键、视图、触发器、事件、存储过程、JOIN、复杂 SQL、事务 | +| MongoDB | 通用索引管理、聚合、嵌套查询、复杂更新操作符、验证器、时序集合、分片、事务、Change Stream | +| Redis | List、Set、Sorted Set、Stream、Pub/Sub、锁、专用原子命令、自定义 Lua、事务、扫描与整库操作 | + +新增结构管理层沿用现有异常上下文,记录操作名和目标;`SchemaConflictError` 用于 SDK 检测到的结构不兼容。驱动错误保留原对象、因果链和已有分类规则。 + +原生调用在 `connection.errors(...)` 外保持原生异常行为;调用方可以显式选择转换范围。 + +资源管理继续遵循: + +- 连接由连接句柄关闭。 +- 记录和 String 上下文不关闭借用的连接。 +- Sentinel 操作上下文创建的数据客户端及其自有资源由该上下文清理。 +- 外部连接池不被误关闭。 +- 清理错误不覆盖正在传播的业务异常。 +- 异步对象在创建它的事件循环内使用和关闭。 + +本 SDK 不提供跨数据库事务、自动同步、双写补偿或资金账户模型。金额精度由字段类型和 codec 决定;Redis 默认 JSON codec 不承担 `Decimal` 的精确编码,不能把浮点操作或批量成功当作账务一致性保证。 + +## 三、从初始化到 CRUD 的同步、异步示例 + +以下示例使用 DATVIA 的公开接口,完整源码见 [同步异步示例](../examples/basic_usage.py)。由调用方准备隔离服务和凭据,先复用公共配置与表定义,再选择同步或异步入口。MySQL、MongoDB 已完成真实服务端到端验收;Redis 真实拓扑验收仍待补齐。示例会写入并清理指定演示记录,应使用专用演示库和命名空间。 + +示例假设三个数据库服务和账号已经准备好。MySQL 初始化连接具有建库建表权限;生产项目可以为初始化与业务操作配置不同账号。 + +### 3.1 公共配置和表定义 + +```python +from datvia import ConnectionConfig +from datvia import Connections +from datvia.schema import Column +from datvia.schema import TableSpec + + +USER_TABLE = TableSpec( + columns={ + "id": Column("varchar", length=32), + "name": Column("varchar", length=100), + "age": Column("int"), + }, + primary_key=("id",), +) + + +def make_connections( + mysql_password: str, + mongodb_uri: str, + redis_password: str | None = None, +) -> Connections: + return Connections({ + # 初始化时不能选择一个尚未创建的数据库。 + "mysql_setup": ConnectionConfig( + backend="mysql", + host="127.0.0.1", + username="sdk_demo", + password=mysql_password, + options={"autocommit": True}, + ), + "mysql_app": ConnectionConfig( + backend="mysql", + host="127.0.0.1", + username="sdk_demo", + password=mysql_password, + options={ + "database": "demo_app", + "autocommit": True, + }, + ), + "documents": ConnectionConfig( + backend="mongodb", + host=mongodb_uri, + ), + "cache": ConnectionConfig( + backend="redis", + host="127.0.0.1", + password=redis_password, + options={"db": 0, "decode_responses": True}, + ), + }) +``` + +密码和 URI 由调用方传入。调用方可以自行读取环境变量或密钥服务,SDK 不参与配置来源选择。 + +增加第二个 MySQL 业务库或第二个 Redis 实例时,增加一个命名配置即可;不会覆盖现有连接或形成按后端共享的单例。 + +### 3.2 同步完整流程 + +```python +from contextlib import ExitStack + +from datvia.crud import JsonFieldCodec +from datvia.crud import Records +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +from datvia.schema import Schema + + +def run_sync(connections: Connections) -> None: + # 一、显式准备 MySQL 数据库与表。 + with connections.create("mysql_setup") as setup: + schema = Schema.mysql(setup) + schema.ensure_database("demo_app") + schema.ensure_table("demo_app", "users", USER_TABLE) + + # 二、三个数据库连接在同一业务作用域内同时使用。 + with ExitStack() as stack: + mysql = stack.enter_context(connections.create("mysql_app")) + mongo = stack.enter_context(connections.create("documents")) + redis = stack.enter_context(connections.create("cache")) + + for connection in (mysql, mongo, redis): + connection.check() + + # 三、显式准备 MongoDB 普通集合。 + Schema.mongodb(mongo).ensure_collection( + "demo_app", + "users", + ) + + mysql_users = stack.enter_context( + Records.mysql( + mysql, + table="users", + key_fields=USER_TABLE.primary_key, + ) + ) + mongo_users = stack.enter_context( + Records.mongodb( + mongo, + database="demo_app", + collection="users", + ) + ) + + # Redis 不建表;绑定记录命名空间及 codec 即可开始使用。 + redis_users = stack.enter_context( + Records.redis_hash( + redis, + namespace="demo_app:users", + key_fields=("id",), + codec=JsonFieldCodec(), + ) + ) + + # 四、分别对三个后端执行相同的基础记录流程。 + for users, key_field in ( + (mysql_users, "id"), + (mongo_users, "_id"), + (redis_users, "id"), + ): + key = {key_field: "u1"} + + users.insert({**key, "name": "Alice", "age": 30}) + print(users.get(key, fields=["name", "age"])) + users.update(key, set_values={"age": 31}) + + users.insert_many([ + {key_field: "u2", "name": "Bob", "age": 20}, + {key_field: "u3", "name": "Carol", "age": 40}, + ]) + users.bulk_update([ + { + "key": {key_field: "u2"}, + "set_values": {"age": 21}, + }, + { + "key": {key_field: "u3"}, + "set_values": {"age": 41}, + }, + ]) + print(users.get_many( + [{key_field: "u2"}, {key_field: "u3"}], + fields=["name", "age"], + )) + + users.delete(key) + users.delete_many([ + {key_field: "u2"}, + {key_field: "u3"}, + ]) + + # 五、Redis String 键值操作。 + strings = stack.enter_context(RedisStrings(redis)) + + strings.set("demo:message", "hello", ttl_s=60) + print(strings.get("demo:message")) + strings.set("demo:message", "updated", ttl_s=120) + print(strings.ttl("demo:message")) + strings.delete("demo:message") + + strings.set_many([ + StringSetItem("demo:a", "A", ttl_s=60), + StringSetItem("demo:b", "B", ttl_s=60), + ]) + print(strings.get_many(["demo:a", "demo:b", "demo:missing"])) + strings.delete_many(["demo:a", "demo:b"]) + + # 上下文退出:先释放操作对象自有资源,再关闭连接。 + # 不自动删除数据库、表或集合。 +``` + +此示例对三个后端分别写入演示数据,不表示 SDK 自动复制数据,也不形成一个跨数据库事务。示例中的固定键用于独立演示目标,实际业务应使用自己的身份值。 + +### 3.3 异步完整流程 + +异步流程复用上面的配置和表定义;所有数据库操作显式 `await`。 + +```python +from contextlib import AsyncExitStack + +from datvia.crud import AsyncRecords +from datvia.crud import AsyncRedisStrings +from datvia.crud import JsonFieldCodec +from datvia.crud import StringSetItem +from datvia.schema import AsyncSchema + + +async def run_async(connections: Connections) -> None: + # 一、异步准备 MySQL 数据库与表。 + async with await connections.create_async("mysql_setup") as setup: + schema = AsyncSchema.mysql(setup) + await schema.ensure_database("demo_app") + await schema.ensure_table("demo_app", "users", USER_TABLE) + + # 二、三个异步连接同时存在,分别管理各自资源。 + async with AsyncExitStack() as stack: + mysql = await stack.enter_async_context( + await connections.create_async("mysql_app") + ) + mongo = await stack.enter_async_context( + await connections.create_async("documents") + ) + redis = await stack.enter_async_context( + await connections.create_async("cache") + ) + + for connection in (mysql, mongo, redis): + await connection.check() + + # 三、异步准备 MongoDB 集合。 + await AsyncSchema.mongodb(mongo).ensure_collection( + "demo_app", + "users", + ) + + mysql_users = await stack.enter_async_context( + AsyncRecords.mysql( + mysql, + table="users", + key_fields=USER_TABLE.primary_key, + ) + ) + mongo_users = await stack.enter_async_context( + AsyncRecords.mongodb( + mongo, + database="demo_app", + collection="users", + ) + ) + redis_users = await stack.enter_async_context( + AsyncRecords.redis_hash( + redis, + namespace="demo_app:users", + key_fields=("id",), + codec=JsonFieldCodec(), + ) + ) + + # 四、三个后端分别执行异步单条与批量 CRUD。 + for users, key_field in ( + (mysql_users, "id"), + (mongo_users, "_id"), + (redis_users, "id"), + ): + key = {key_field: "u1"} + + await users.insert({**key, "name": "Alice", "age": 30}) + print(await users.get(key, fields=["name", "age"])) + await users.update(key, set_values={"age": 31}) + + await users.insert_many([ + {key_field: "u2", "name": "Bob", "age": 20}, + {key_field: "u3", "name": "Carol", "age": 40}, + ]) + await users.bulk_update([ + { + "key": {key_field: "u2"}, + "set_values": {"age": 21}, + }, + { + "key": {key_field: "u3"}, + "set_values": {"age": 41}, + }, + ]) + print(await users.get_many( + [{key_field: "u2"}, {key_field: "u3"}], + fields=["name", "age"], + )) + + await users.delete(key) + await users.delete_many([ + {key_field: "u2"}, + {key_field: "u3"}, + ]) + + # 五、Redis String 异步操作。 + strings = await stack.enter_async_context( + AsyncRedisStrings(redis) + ) + + await strings.set("demo:message", "hello", ttl_s=60) + print(await strings.get("demo:message")) + await strings.set("demo:message", "updated", ttl_s=120) + print(await strings.ttl("demo:message")) + await strings.delete("demo:message") + + await strings.set_many([ + StringSetItem("demo:a", "A", ttl_s=60), + StringSetItem("demo:b", "B", ttl_s=60), + ]) + print(await strings.get_many([ + "demo:a", + "demo:b", + "demo:missing", + ])) + await strings.delete_many(["demo:a", "demo:b"]) +``` + +结构查看与删除也采用对应同步方法或异步 `await` 方法。删除库表属于单独的显式管理操作,不放进普通 CRUD 示例的自动退出逻辑。 + +## 四、开发组织与实施顺序 + +以下为本轮遵循的实施流程。三个模块已独立开发并串行合入功能分支;交付状态与验收缺口以实施记录为准。 + +### 4.1 集成策略 + +继续使用 `feature/database-crud` 作为本功能集成主线,保留已经完成的记录 CRUD 历史,不重新从 `develop` 开始实现。 + +开发遵循: + +- 不操作 `develop`,最终由用户审核后决定合入。 +- 只进行本地 Git 操作,不检查或同步远端。 +- 不改写既有提交,不混入 `temp/save` 或无关工作区文件。 +- 实现分支使用 `feature/`,缺陷修复使用 `fix/`,不使用 `docs/` 或 Codex 前缀承载代码实现。 +- 独立 worker 使用独立 worktree 与暂存区。 +- 集成采用串行 `--no-ff` 合并,中文 Conventional Commits。 +- 工作树和分支保留至用户审查,不擅自清理已有工作树或草稿。 + +### 4.2 实施阶段 + +| 阶段 | 工作内容 | 完成条件 | +| --- | --- | --- | +| A:设计与公共契约 | 审核本文;固定 Schema、表定义、String 结果类型和异常接口,建立公共契约提交检查点 | 公共接口无待定项,下游使用同一已提交检查点 | +| B:三个模块并行 | MySQL 结构管理、MongoDB 集合管理、Redis String 操作分别实现 | 各模块同步异步定向测试通过,提交模块交付记录 | +| C:串行集成 | 按 MySQL、MongoDB、Redis 顺序合入;协调者维护公共导出及共享结果类型 | 每次合并复验相关功能,公共接口一致 | +| D:完整验收 | 从空结构执行示例,验证多数据库、多实例及故障场景 | 实际结果与文档一致,未验证事项明确 | +| E:审查交付 | 更新使用文档、README、CHANGELOG 和验收记录 | 提供差异、提交关系、测试证据,交由用户审查 | + +并行实现分支: + +- `feature/database-crud-mysql-schema` +- `feature/database-crud-mongodb-schema` +- `feature/database-crud-redis-strings` + +三个 worker 从阶段 A 的同一检查点创建分支,分别拥有自己的实现和测试文件。公共文件由协调者负责,避免并行修改同一文件。最多同时运行三个实现 worker,协调者负责总集成,不让不同工作者交叉操作同一工作树。 + +工作者交付模块范围、提交 SHA、测试结果和 Git 操作记录。协调者在合并前核对分支仍指向已审查 SHA,检查差异范围,再串行合并并记录集成后的 HEAD。不使用 squash、强制改写历史或把整个功能分支压为一次 cherry-pick。 + +本轮不新增 ORM、自动迁移系统、后端插件注册框架、SQL 优化器、跨库事务协调器或新的运行时依赖。 + +## 五、测试、文档与交付标准 + +### 5.1 自动化测试 + +沿用 `unittest`,异步使用 `IsolatedAsyncioTestCase`。 + +重点覆盖: + +| 范围 | 必须验证的场景 | +| --- | --- | +| 初始化 | 空目标创建成功;重复执行不破坏数据;并发相同初始化不虚报成功 | +| 结构查看与删除 | 同步异步 list、exists、describe、drop;仅作用于指定目标;重复及不存在删除;权限错误 | +| MySQL 结构 | 必要字段和主键匹配;允许可省略的额外字段;结构不兼容时不改表 | +| MySQL 事务 | 结构写操作拒绝活动事务;不暗中提交、回滚、切库或修改 autocommit | +| MongoDB 集合 | 首个集合完成初始化;同名非普通集合冲突;自定义唯一键生效 | +| MongoDB 失败 | 索引建立失败保留集合和原异常;不把构建中索引当成就绪 | +| 元数据权限 | 权限异常不被转换为“目标不存在” | +| Redis String | 空值与缺失区分、bytes/str、TTL、错误类型、编码后重复键 | +| 批量结果 | 逐项成功、部分失败、发送状态未知、取消及未派发项 | +| 字段投影 | 三种记录后端只返回指定字段;空投影、缺失字段、原生值类型 | +| 多连接 | 三种数据库同时使用;两个同类实例相互隔离;关闭一个不影响另一个 | +| 生命周期 | 游标、流水线和 Sentinel 自有客户端清理;外部池保护 | +| 兼容性 | 原生工厂、已有连接接口和记录 CRUD 测试继续通过 | + +对 Python 改动执行定向测试、Ruff lint 和格式检查;涉及公共接口与共享结果类型后执行完整离线回归,并运行 `git diff --check`。 + +```bash +python -B -m unittest discover -s tests -q +python -m ruff check <本次改动的Python文件> +python -m ruff format --check <本次改动的Python文件> +git diff --check +``` + +执行前核对实际导入的是当前工作树源码。文档中分别记录离线测试、驱动兼容性测试和真实服务验收,不混合表述。 + +### 5.2 真实服务验收 + +使用显式测试配置和隔离实例,不读取生产配置。 + +验收范围包括: + +- MySQL 8.4:同步、异步从建库建表到完整 CRUD。 +- MongoDB:同步、异步集合准备、必要唯一约束及完整 CRUD。 +- Redis 单机、Sentinel、Cluster:同步、异步 Hash 与 String 操作。 +- Redis Cluster 批量包含不同槽的键。 +- 同一进程同时打开三个后端,并创建两个同类连接。 +- 原生高级操作与基础封装可以在同一连接上按驱动规则使用。 +- 认证失败、权限不足、重复键、结构冲突、网络中断及清理失败保留诊断信息。 + +驱动范围保持现有约定:Python 3.10+、PyMongo 4.6+,MongoDB 异步要求 4.9+;MySQL Connector/Python 8.3+;redis-py 5.0+。验证最低支持版本与选定当前版本,记录实际环境。 + +现有真实 Redis 拓扑验证缺口必须单独补齐。若隔离服务暂不可用,明确标为未完成验收,不能用替身测试宣称真实拓扑已经验证。 + +### 5.3 文档与最终验收标准 + +设计审查通过、进入实施后: + +- 在本文中更新实际接口和验收状态,不把计划能力写成已实现。 +- 使用文档补齐初始化、结构管理、同步异步、多数据库示例。 +- 保留此前记录 CRUD 阶段的历史验收证据。 +- README 展示最短完整使用流程。 +- CHANGELOG 的“未发布”部分登记新增能力。 + +**功能完成标准:开发者面对已经部署好的三个数据库服务,可以仅通过本 SDK 完成基础存储准备、单条与批量读写、字段投影以及资源关闭;高级功能可以继续直接使用原生客户端。** + +最终交付提供集成分支、基线与最终提交、worker 合并关系、按执行顺序记录的 Git 操作、测试结果和未验证事项,停留在功能分支供用户审查。 + +Git 操作记录注明执行者、分支、工作树、命令和结果;合并记录写明被合入 SHA 与集成后的 HEAD。核对 `develop` 未改变,未混入无关文件,未访问或同步远端。本轮实现与验收证据另存于 [基础使用实施记录](crud-basic-implementation.md)。 diff --git a/docs/crud-implementation.md b/docs/crud-implementation.md new file mode 100644 index 0000000000000000000000000000000000000000..e709353b0cdb60c1f9ff9214c0ea89a5fa9a99dd --- /dev/null +++ b/docs/crud-implementation.md @@ -0,0 +1,156 @@ +# CRUD 实施与验收记录 + +日期:2026-11-16。 +集成与交付分支:`feature/database-crud`。按用户要求,功能代码使用 `feature/*` 前缀,`docs/*` 仅用于纯文档任务。 +正式功能基线:本地 `develop` 的 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`。 +本次只操作本地任务分支;没有检查、同步或推送远端,也不将本功能合入 `develop`。 + +使用接口见 [使用说明](crud-usage.md),语义边界见 [设计](crud-design.md)。 + +## 1. 模块和工作树 + +协调者在仓库根目录 `.` 集成。 +各 worker 在相对于仓库根目录的 `../database-worktrees/` 下独立开发,使用独立分支、目录和暂存区。 +旧工作树及用户其他分支全部保留。 + +| 模块 | 分支 | 工作树目录(相对仓库根目录) | 交付提交 | +| --- | --- | --- | --- | +| 公共协议、记录层 | `feature/database-crud-core` | `../database-worktrees/crud-core` | `0ba757a925247f632d32b9d236457fad5e8fbe00` | +| MySQL | `feature/database-crud-mysql` | `../database-worktrees/crud-mysql` | `1e7f52ee5f4646498cca57632a99bb2ca2a3704b` | +| Redis Hash / codec | `feature/database-crud-redis` | `../database-worktrees/crud-redis` | `32c990422a3fdd972b5e8d089b13932dc7f33d29` | +| MongoDB | `feature/database-crud-mongodb` | `../database-worktrees/crud-mongodb` | `c0e4246b54274d3222ed75ab57141e492da6131f` | +| MongoDB 取消修复 | `fix/database-mongodb-cancellation` | `../database-worktrees/crud-mongodb-cancellation` | `a6d778f2a919695ae77012e2426e8c26ce57ea0d` | +| 分块派发与取消链 | `fix/database-crud-batch-evidence` | `../database-worktrees/crud-batch-evidence` | `a64d43861366726176a70cf242c54825d1712049` | + +公共协议检查点先集成,再启动各数据库模块;公共层完成后,各 worker 可以合入明确的集成检查点进行公开 API 验证。 +各 worker 不交叉合并彼此的分支,不操作总线暂存区。 +协调者串行使用 `--no-ff` 合并核对过的提交,保留分支边界。 + +## 2. Git 操作记录 + +以下按执行顺序记录协调者的 Git 写操作及必要核对;命令均带明确工作目录执行。 +没有使用 squash、rebase、amend、reset、强制推送或删除分支、工作树。 + +| 顺序 | 执行者、分支及工作树 | 操作 | 结果 | +| --- | --- | --- | --- | +| 1 | 协调者,主目录 `develop` | `git status --short --branch`、`git diff --cached`、`git worktree list --porcelain`、`git rev-parse HEAD develop` | 主目录干净;确认本地基线 `b670712`,设计工作树存在待保存文档 | +| 2 | 协调者,`crud-design` 工作树 | 按路径暂存 `docs/crud-design.md` 和 `docs/README.md`,核对暂存差异后 `git commit` | `80fd877360787309bf4a679bd94b1baa76c733ea`,保存既有设计文档 | +| 3 | 协调者,`crud-design` 工作树及主目录 | 旧文档工作树 `git switch --detach 80fd877`;主目录 `git switch docs/database-crud-design` | 主目录切至用户指定开发主线,旧文档工作树保留 | +| 4 | 协调者,主目录 | 按路径暂存设计及索引,核对后中文 `git commit` | `da1103d`,登记实施基线 | +| 5 | 公共层 worker,`crud-core` | `git worktree add -b feature/database-crud-core ... 80fd877`,按模块路径暂存、检查并提交 | `da0f2d36b368002d3497d9bc763002b5654c418b`,共享类型与协议 | +| 6 | 协调者,主目录 | 核对 SHA/范围后 `git merge --no-ff -m "feat(crud): 合入结果类型与适配器协议" da0f2d36...` | 集成 HEAD `63d94d3b10392ef0d9a5514db30e185eb85e22de`;91 项回归通过 | +| 7 | MySQL / Redis worker,各自工作树 | 从 `63d94d3...` 用 `git worktree add -b` 创建各自分支 | 独立并行实现,无共享暂存区 | +| 8 | 公共层 worker,`crud-core` | 模块实现及测试分两次中文提交 | `b5fc99c9f30bd00785ac6c4757c04d00e49c611b`、`0ba757a925247f632d32b9d236457fad5e8fbe00`;106 项回归通过 | +| 9 | 协调者,主目录 | 核对 SHA/范围后 `git merge --no-ff -m "feat(crud): 合入同步异步公共记录层" 0ba757a...` | 集成 HEAD `5140bbee593a323b713674c483faa087feeb7c88` | +| 10 | MongoDB worker,`crud-mongodb` | 从 `5140bbe...` 创建独立分支和工作树 | 使用已集成公共接口开展实现 | +| 11 | 协调者,主目录 | 按路径暂存 `tests/test_crud_integration.py`、检查后中文提交 | `d3a93cf35e531ec7311072c7a6ed4d091911e1e7`;多后端/同类多实例的 2 项测试通过 | + +每次提交前使用 `git diff --cached --stat` / `git diff --cached --check` 检查范围和空白。 +后续后端与审查修复集成按以下顺序执行;每次合并前再次核对交付分支指向的 SHA 及三点差异范围。 + +| 顺序 | 执行者、分支及工作树 | 操作及交付 SHA | 集成结果 | +| --- | --- | --- | --- | +| 12 | Redis worker,`crud-redis` | 提交 codec `e732605`;`git merge --no-ff 5140bbe` 得到 `ac7c5ac`;提交适配与测试 `32c9904` | 工作树干净,19 项定向测试在 redis-py 5.0 / 8.1 通过 | +| 13 | 协调者,主目录 | `git merge --no-ff` 合入 `32c990422a3fdd972b5e8d089b13932dc7f33d29` | `123869eb385da7dcf3307861dbc56b447509ee32`;127 项回归与 Lua 模拟验收通过 | +| 14 | 协调者,主目录 | 投影预检修复按指定源文件、测试及契约路径暂存后提交 | `87abaae6e6c4be69bd095070dac9fa83fcfe6cc9`;130 项回归通过 | +| 15 | MySQL worker,`crud-mysql` | `git merge --ff-only 5140bbe`、`git merge --ff-only 87abaae`;仅暂存 MySQL 模块和测试后提交 `1e7f52e` | 两个驱动版本各 152 项回归通过,真实 MySQL 8.4 CRUD/事务验证通过 | +| 16 | MongoDB worker,`crud-mongodb` | 提交 `27c3c33`;`git merge --no-ff 87abaae` 得到 `23cc533`;提交清理修复 `c0e4246` | 五个驱动版本的真实 MongoDB 验证通过,21 项定向测试通过 | +| 17 | 协调者,主目录 | Int64 修复提交 `ab20cef`;`git merge --no-ff` 合入 `1e7f52ee5f4646498cca57632a99bb2ca2a3704b` | `887cd77305435a17dafd8943e63624a3253f94ab`;153 项回归及两个 MySQL 驱动版本真实复验通过 | +| 18 | 协调者,主目录 | `git merge --no-ff` 合入 `c0e4246b54274d3222ed75ab57141e492da6131f` | `b650039b9a7f130d93a45be62db5e2c237e87f1b`;最低同步、最低异步及当前 MongoDB 真实复验通过 | +| 19 | 审查修复 worker,各自工作树 | 分块修复从 `87abaae`,MongoDB 取消修复从 `b650039`,分别 `git worktree add -b fix/...`;按模块路径暂存、检查、中文提交 | 独立保留审查发现及回归修复历史 | +| 20 | 协调者,主目录 | Decimal128 修复提交 `f32db99`;`git merge --no-ff` 合入 `a6d778f2a919695ae77012e2426e8c26ce57ea0d` | `22586247614e3dd68bfd972ad14fad5ba63f6772`;BSON 精确类型真实回读验证通过 | +| 21 | 分块修复 worker,`crud-batch-evidence` | 对两项可复现问题提交 `a2b2227`,再补只读取消对象的回归修复 `a64d438` | 模块及测试两文件,工作树干净,134 项分支回归通过 | +| 22 | 协调者,主目录 | 核对 SHA、差异后 `git merge --no-ff` 合入 `a64d43861366726176a70cf242c54825d1712049` | `3594f21ab214b144e9d14579697bd5dcf9264fad`,作为最终代码验收基点 | +| 23 | 协调者,主目录 | 使用说明、设计状态、README、CHANGELOG、测试指南和本记录按路径暂存;核对暂存范围及文档检查后中文提交 | `09faa013d24e18188042bf951a785a4f4e3c2e0b`;代码保持上述验收基点内容 | +| 24 | 协调者,主目录 | 核对工作区、暂存区和工作树后执行 `git branch -m feature/database-crud` | 主目录改用功能分支名,HEAD 仍为 `09faa013d24e18188042bf951a785a4f4e3c2e0b`,全部提交保留 | +| 25 | 协调者,主目录 `feature/database-crud` | 按路径暂存贡献指南及三份 CRUD 文档,核对差异后以 `docs(crud): 规范功能开发分支命名` 提交 | 本次仅记录命名规范与当前分支,保留历史命令,产品代码未改动 | + +最终分支 HEAD 随文档提交前进,可用 `git rev-parse feature/database-crud` 获取;代码验收绑定于 `3594f21ab214b144e9d14579697bd5dcf9264fad`。本次更名基线为 `09faa013d24e18188042bf951a785a4f4e3c2e0b`,历史记录中的旧分支名称保持原样。 + +## 3. 验证环境和命令 + +源码通过 `PYTHONPATH=<当前工作树>/src` 指向当前工作树。 +验证解释器为独立 QA 环境的 Python 3.10.20。 +当前驱动组合:PyMongo 4.18.3、MySQL Connector/Python 26.10.0、redis-py 8.1.0。 +最低版本分别使用隔离的依赖目录验证,导入后核对实际版本,避免无意回退至当前版本。 +Ruff 使用 0.16.8;运行依赖未增加。 + +```powershell +$env:PYTHONPATH = (Join-Path (Get-Location) 'src') +python -B -m unittest discover -s tests -q +ruff check src/database/crud tests/test_crud_*.py +ruff format --check src/database/crud tests/test_crud_*.py +git diff --check +``` + +`python` 和 `ruff` 在本次实际执行中使用 QA 解释器及工具的绝对路径。 +各后端定向命令使用 `-p test_crud_<模块>.py`。 +标准自动测试使用驱动对象、离线替身和 `unittest.IsolatedAsyncioTestCase`,不接触外部业务数据库。 +最终完整回归为 179 项,三套依赖组合各自全量通过,未跳过测试。 + +| 组合 | PyMongo | MySQL Connector/Python | redis-py | 结果 | +| --- | --- | --- | --- | --- | +| 当前版本 | 4.18.3 | 26.10.0 | 8.1.0 | 179 / 179 | +| 最低同步依赖 | 4.6.0 | 8.3.0 | 5.0.0 | 179 / 179 | +| 最低异步依赖 | 4.9 | 8.3.0 | 5.0.0 | 179 / 179 | + +每个独立进程先断言实际导入版本,再由 `unittest.defaultTestLoader.discover('tests')` 执行完整套件。 +公共接口无驱动导入的独立子进程回归仍通过。 +最终 16 个新增或修改的 Python 文件通过 Ruff lint 和格式检查,Git 空白差异检查通过。 + +## 4. 服务与模拟验证边界 + +真实 CRUD 只在本任务隔离的临时数据库中执行,服务只绑定本机回环地址,不复用用户生产连接。 +测试凭据、数据目录和临时运行日志不进入仓库。 +故障场景的运行时限由测试进程控制,不表示 SDK 提供整体硬超时。 + +Redis 验证包含真实 redis-py 对象的单机 / Sentinel / Cluster 调度和资源清理测试,以及独立 QA 依赖中的 fakeredis / Lua 执行验证。 +Lua 模拟器验证不等于 Redis 服务器、Sentinel 故障切换或 Cluster 多槽网络验收。 +本环境尚无可用 Redis 真实拓扑,相关实测必须独立列为待验证项。 + +| 验证类型 | 环境 | 已验证内容 | +| --- | --- | --- | +| 真实 MySQL | MySQL 8.4.11,Connector 8.3.0 / 26.10.0 | 同步、异步 CRUD;当前版同步 C / pure;复合键、默认/空/包含投影、500 键读取、排序规则、部分失败、原始错误、独立连接的提交/回滚/autocommit | +| 真实 MongoDB | MongoDB 8.0.24 单节点副本集 | PyMongo 4.6 / 4.8 同步,4.9 / 4.13 / 4.18.3 同步及异步;条件与数组/缺失隔离、投影、复合键、排序规则、ordered 部分失败、w=0、原生 session 与事务 abort | +| 原生 BSON 数值 | 同一 MongoDB,PyMongo 4.6 / 4.9 / 4.18.3 | Int64 / Decimal128 读回后重新作为键、条件及重复批量输入使用,保持精确类型 | +| Redis 原生驱动离线 | redis-py 5.0 / 8.1 | 单机 / Sentinel / Cluster 同步与异步调度、所有权和清理;真实 ClusterPipeline 栈使用替代节点及 socket,验证跨槽路由和逐项回复 | +| Redis Lua 模拟器 | fakeredis 2.40.0 / lupa 2.8,redis-py 5.0 / 8.1 | bytes 与 decode_responses、空值/缺失/空投影、560 字段读取、TTL 保留/重置/清除/过期、类型保护、部分失败、8 个并发创建请求仅一个成功 | + +上述 MySQL、MongoDB 各版本及 Redis 模拟器验收均在最终代码验收基点复跑。 +Redis 5.0 的临时验收脚本最初使用了仅新版存在的 `aclose()`;改为适配 `close()` 后完整通过,此处属于测试脚本兼容修正,不是 SDK 清理逻辑变更。 + +历史 QA 材料保存在仓库外的临时目录,实际位置由运行者指定,本文不保留本机绝对地址。以下仅记录验收脚本的文件名和用途,不表示它们位于仓库根目录,也不纳入运行依赖或仓库产品代码。 + +- MySQL:`verify_crud.py`、`verify_transactions.py`,参数依次为预期驱动版本、`current` 或隔离驱动目录、QA 根。 +- MongoDB:`verify_mongo_crud.py`,参数为同目录 `fixture.json` 与预期驱动版本;由 `PYTHONPATH` 选当前源码和驱动。 +- BSON 数值:`verify_int64_main.py`,与 MongoDB 验收脚本位于同一隔离目录,使用同样参数,包含 Int64 与 Decimal128。 +- Redis:`crud-redis-smoke.py`,使用显式 `PYTHONPATH` 加载当前源码、选定 redis-py 及独立 `crud-redis-lua` 模拟器依赖。 + +测试配置仅指向上述隔离服务;凭据不写入本文,也不直接输出配置内容。 +验收完成后,本任务 MySQL 进程与 MongoDB 副本集均已正常关闭,专属回环端口已无监听;临时数据、脚本、配置和日志保留以便复核。 +重跑真实服务脚本需要先按测试夹具恢复已停止的隔离服务,不能替换成业务数据库配置。 + +## 5. 实现选择 + +- MongoDB `get_many` 对去重后的每个键执行原生有界查询,保留数据库的排序规则、数值和时间匹配语义,不用 Python 值重排猜测匹配。 +- MySQL 批量读取由 SQL 标记输入位置;批量写入只报告驱动能证明的组级或逐项信息,不推算连续生成主键。 +- Redis Cluster 的 Pipeline 用原生命令分发单键 EVAL;不调用某些版本未实现的 `pipeline.eval()`,不把异步 Cluster 的服务器 RESET 命令当成客户端资源清理。 +- 新增 `InsertItem(record, ttl_s=...)` 明确区分批量插入的 TTL 控制项与名为 `ttl_s` 的业务字段。 +- `JsonFieldCodec` 是显式的字段编码器,不是自动类型转换层。Redis 身份键只接受字符串或整数;其他身份类型由调用方显式转换。 +- MySQL C 扩展可读取已协商会话标志时映射 UPDATE 统计;纯 Python / 异步连接只保留原始 `rowcount`,统一 matched / modified 计数为 `None`,不依据可变的下次建连标志猜测。 +- BSON `Int64`、`Decimal128` 读回后仍能用作键和条件;精确小数去重使用原生二进制身份,不转成 float。 + +## 6. 最终验收状态 + +本地功能实现、回归、真实 MySQL/MongoDB 以及 Redis 模拟验收完成,代码和使用文档交付用户审查。 +任务工作树保留;用户主目录停在 `feature/database-crud`。 +本地共 22 个工作树,包含本任务新增的 7 个独立工作树;既有其他工作树没有被删除、搬移或改写。 +`develop` 仍为初始基线 `b6707125620d1cea28cabbc7a72e72c84ae73bbd`,没有被提交、合并或改写。 +没有执行远端查询、同步、推送或远端分支清理。 + +仍待验证:真实 Redis 单机、Sentinel 与多主节点 Cluster 的网络/故障切换验收,以及这些真实拓扑中与 MySQL/MongoDB 同进程的完整端到端场景。 +多数据库和同类多实例同时使用已由公共层离线集成回归覆盖,这不替代真实三后端联合验收。 +未将 Lua 模拟结果标成真实 Redis 服务验证;此次也未进行吞吐量基准或生产负载测试。 + +复核可使用 `git log --first-parent --oneline b670712..feature/database-crud` 查看交付顺序,再进入每个 `--no-ff` 合并查看模块内提交。 +用户审查通过后自行决定何时合入 `develop`。 diff --git a/docs/crud-usage.md b/docs/crud-usage.md new file mode 100644 index 0000000000000000000000000000000000000000..43d4ab0ccb2e684c33a958dd9f6aeaa857880bbf --- /dev/null +++ b/docs/crud-usage.md @@ -0,0 +1,361 @@ +# 基础结构与记录操作使用说明 + +`datvia.crud` 在已有连接之上提供可选的记录操作层。 +MySQL 对应表,MongoDB 对应集合,Redis 对应采用本库格式的 Hash 记录。 +它们共享调用方式、字段投影和结果结构;事务、类型系统和查询能力仍遵循各数据库的真实语义。 + +记录操作的受测版本和场景见 [CRUD 验收记录](crud-implementation.md),结构管理与 Redis String 的验证见 [基础使用验收记录](crud-basic-implementation.md)。 +完整设计背景见 [CRUD 设计](crud-design.md),内部模块协作见 [适配器契约](crud-adapter-contract.md)。 + + +## 0. 从初始化开始使用 + +先显式传入连接参数,然后准备数据库或集合,最后绑定记录对象。MySQL 初始化连接应不选择尚未存在的业务库,并显式设置 `options={"autocommit": True}`;业务连接在初始化完成后通过 `options={"database": "app", "autocommit": True}` 选择目标库。MongoDB 使用显式集合准备;Redis 使用配置中的编号库以及 Hash 命名空间或 String 键,不创建虚构的表。 + +```python +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import TableSpec + +users = TableSpec( + columns={ + "id": Column("varchar", length=32), + "name": Column("varchar", length=100), + "age": Column("int"), + }, + primary_key=("id",), +) + +with connections.create("mysql_setup") as setup: + schema = Schema.mysql(setup) + schema.ensure_database("app") + schema.ensure_table("app", "users", users) + +with connections.create("documents") as documents: + Schema.mongodb(documents).ensure_collection("app", "users") +``` + +`connections` 由调用方通过 `ConnectionConfig` 与 `Connections` 显式建立。上面的结构准备不提交业务事务,不改变连接默认数据库,不缓存客户端。已有表只校验声明字段和主键;额外列必须能够省略。不兼容时抛出 `SchemaConflictError`,其 `differences` 保存差异。声明字段不能是生成列或 `INVISIBLE` 列;额外的可省略字段可保留这些原生属性。 + +异步使用 `AsyncSchema.mysql(connection)`、`AsyncSchema.mongodb(connection)`,绑定不需要 await,管理方法需要 await;连接创建使用 `await connections.create_async(name)`。三种数据库同时使用的完整同步、异步流程见 [完整示例](../examples/basic_usage.py) 和 [设计中的使用示例](crud-design.md#三从初始化到-crud-的同步异步示例)。 + +### 基础结构管理 + +MySQL 提供 `ensure_database`、`ensure_table`、`list_databases`、`database_exists`、`list_tables`、`table_exists`、`describe_table`、`drop_table`、`drop_database`。查看方法反映当前账号可见目录,权限错误不会作为“不存在”处理;结构写操作要求 autocommit 且没有活动事务。显式删除按指定名称执行,不在上下文退出时自动删库表。 + +`Column` 支持 int、bigint、boolean、decimal、double、varchar、text、date、datetime、blob;VARCHAR 必须指定 length,DECIMAL 默认精度 10、小数位 0,DATETIME 默认精度 0。TEXT/BLOB 的显式默认值仅支持可空列的 None;其他复杂默认表达式使用原生 API。自增仅支持单列整数主键,通用索引、外键和结构变更仍由原生客户端处理。 + +MongoDB 提供 `ensure_collection`、`list_databases`、`database_exists`、`list_collections`、`collection_exists`、`describe_collection`、`drop_collection`、`drop_database`。准备和普通集合列表排除视图、时序、capped 及系统集合。describe 保留原生目录信息;drop 按原生命名空间语义精确删除,也可以删除明确指定的视图,不先查询是否为普通集合。 + +默认 `key_fields=("_id",)` 复用内建约束。声明自定义键时,准备完整且排序规则兼容的唯一索引;需要目录读取及索引创建权限,已存在的索引也会通过原生命令确认就绪。不能把部分、稀疏或构建中的索引当作完整身份保证。结构写入和删除要求 acknowledged 写关注;失败保留已有集合及原异常,不补偿删库或重建其他索引。分片唯一性仍受原生分片键约束。 + + +## 1. 绑定已有连接 + +安装方式和连接配置见 [README](../README.md)。 +配置始终由调用方显式传入;记录层也不会读取环境变量或 `.env`。 + +```python +from datvia.crud import Records + +def read_orders(connections): + with connections.create('orders') as connection: + with Records.mysql( + connection, table='orders', key_fields=('tenant_id', 'id') + ) as orders: + return orders.get( + {'tenant_id': 7, 'id': 1001}, fields=['status', 'total'] + ) +``` + +`key_fields` 声明数据库中真实的唯一身份,支持复合键。 +调用方负责对应的唯一索引或主键;SDK 不创建表、不迁移集合、不自动建立索引。 +字段名使用普通字段,不接受 SQL 片段、点路径或 MongoDB 操作符。 +数据库名、表名和集合名是绑定参数,SQL 值使用参数绑定。 +MySQL 的表名和列名限 1–64 个字母、数字或下划线;高级标识符和表达式通过原生 SQL 使用。 +MongoDB 插入必须提供除可由驱动生成的 `_id` 之外的所有声明键字段。 + +| 后端 | 同步绑定入口 | 后端参数 | +| --- | --- | --- | +| MySQL | `Records.mysql(connection, ...)` | `table`、`key_fields` | +| MongoDB | `Records.mongodb(connection, ...)` | `database`、`collection`、`key_fields`,可选原生 `session` | +| Redis | `Records.redis_hash(connection, ...)` | `namespace`、`key_fields`、显式 `codec` | + +记录对象必须在 `with` 中使用,退出后不能继续调用。 +需要继续使用时,可以基于仍然打开的连接重新绑定记录对象。 +记录层只释放自己创建的游标、Pipeline 或 Sentinel 数据客户端,不关闭借用的连接和 session。 +它不改变原生客户端的线程安全或并发约束,也不隐式共享会话。 + +## 2. 同步与异步 + +异步使用 `AsyncRecords`;绑定本身是普通函数,进入上下文和数据库操作使用 `await`。 +异步方法调用原生异步驱动,参数与返回结构和同步方法一致。 + +```python +from datvia.crud import AsyncRecords + +async def read_documents(connections, document_id): + async with await connections.create_async('documents') as connection: + async with AsyncRecords.mongodb( + connection, database='app', collection='orders', key_fields=('_id',) + ) as documents: + return await documents.get({'_id': document_id}, fields=['status']) +``` + +MongoDB 异步仍要求 PyMongo 4.9 及以上版本。 +异步连接和记录对象在创建连接的事件循环中使用。 +一个项目可以同时打开三种数据库,也可以创建两个同类连接;使用 `ExitStack` 或 `AsyncExitStack` 管理多个上下文即可。 +关闭某个记录对象不会替应用关闭其他连接;跨库成功、回滚和补偿由应用协调。 + +## 3. 方法范围 + +| 方法 | MySQL / MongoDB | Redis Hash | +| --- | --- | --- | +| `get(key, fields=...)` | 唯一键读取 | 指定键读取 | +| `get_many(keys, fields=..., batch_size=500)` | 多键读取 | 多键 Pipeline | +| `find(where=..., fields=..., limit=..., offset=0, order_by=...)` | 有界查询 | 不支持 | +| `count(where=...)` | 条件计数 | 不支持 | +| `insert(record)` / `insert_many(records)` | 创建记录 | 创建专用 Hash,不覆盖已有键 | +| `update(key, set_values=...)` | 按键赋值 | 只更新已有记录 | +| `update_many(where=..., set_values=...)` | 一次集合条件更新 | 不支持 | +| `bulk_update(items, batch_size=500)` | 每条记录分别赋值 | 单键脚本组成 Pipeline | +| `delete(key)` / `delete_many(keys)` | 按键删除 | 单键脚本删除 | +| `delete_where(where=...)` | 条件删除 | 不支持 | + +批量写入接受有限序列,先检查整批可静态判断的结构和编码,再按输入顺序分块。 +默认每块 500 项;一次批量写入不接受重复的已知键。 +空批次返回空结果,不访问数据库。 +Redis 不通过 `KEYS` 或 `SCAN` 模拟条件筛选;不支持的请求在发送前报 `UnsupportedOperationError`。 + +## 4. 字段投影、空值和批量读取 + +| `fields` | 行为 | +| --- | --- | +| `None` | 返回全部业务字段 | +| `['status', 'total']` | 只返回这些字段,保留原生值类型 | +| `[]` | 只确认存在性:存在返回 `{}`,不存在返回 `None` | + +MongoDB 的包含投影只在显式选择 `_id` 时返回 `_id`;`fields=None` 保留完整文档。 +Redis 的内部格式标记不会出现在业务记录中。 +MySQL 选择不存在的列时保留原始驱动错误。 +MongoDB / Redis 缺少被选字段时,返回字典省略该字段;明确存储的空值保留为 `None`。 +因此空字典不等于未找到记录。 + +```python +items = orders.get_many( + [{'tenant_id': 7, 'id': 1001}, + {'tenant_id': 7, 'id': 1002}, + {'tenant_id': 7, 'id': 1001}], + fields=['status'], +) +for item in items: + print(item.index, item.key, item.record) +``` + +返回 `ReadItem` 列表,与输入的位置和数量完全对应。 +完全相同的重复键在一次调用中只读取一次,并复用该次观察;返回结果不依赖数据库返回顺序。 +没有读取成功的项不会用 `record=None` 冒充“不存在”;部分失败通过批量异常表达。 +一次分块读取不是跨节点、跨批次的一致性快照。 +MongoDB 当前对每个去重后的键执行一次 `limit=2` 的有界查询,以保留原生排序规则、数值和时间匹配语义,并发现声明键实际不唯一的情况。 +因此该接口统一多键结果,但不承诺所有后端都只产生一次网络请求;MySQL 使用按输入位置标记的联合查询,Redis 使用 Pipeline。 + +## 5. 条件与批量修改 + +普通字典表示多个字段的等值 AND 条件。 +需要更明确的结构时,使用 `Eq`、`In`、`Gt`、`Gte`、`Lt`、`Lte`、`And`、`Or`、`IsNull`;MongoDB 另支持 `Exists`。 +只支持普通标量条件;原生数组、嵌套路径、SQL 表达式和聚合继续使用原生客户端。 + +```python +from datvia.crud import And +from datvia.crud import Gte +from datvia.crud import In + +pending = orders.find( + where=And(In('status', ['pending', 'paid']), Gte('total', 100)), + fields=['id', 'status'], limit=50, order_by=[('id', 'asc')], +) +changed = orders.update_many( + where={'status': 'pending'}, set_values={'status': 'closed'}, +) +individual = orders.bulk_update([ + {'key': {'tenant_id': 7, 'id': 1001}, 'set_values': {'status': 'paid'}}, + {'key': {'tenant_id': 7, 'id': 1002}, 'set_values': {'status': 'cancelled'}}, +]) +``` + +`update_many` 是服务器端一次集合操作:MySQL 使用一条参数化 UPDATE,MongoDB 使用原生 `update_many`。 +`bulk_update` 则是逐键不同赋值:MySQL 按连续的相同字段形状组织 `executemany`,MongoDB 使用集合级 ordered bulk write,Redis 使用非事务 Pipeline。 +它们不是先读出全部记录,再逐个修改。 +`set_values` 是赋值,不是自动原子加减;不能修改声明的键字段。 + +`Eq('field', None)` / `{'field': None}` 表示明确空值,不包含 MongoDB 的缺失字段。 +`In` 不接受 `None` 元素;可用 `Or(In('field', values), IsNull('field'))` 表达两者。 +`In('field', [])` 匹配零项。 +相同条件在三种数据库上不承诺相同排序规则、数值转换或时区行为。 + +`find` 必须传正整数 `limit`;`offset` 为非负整数。 +分页需要稳定顺序时,应包括唯一键排序;并发写入仍会影响分页观察。 +条件更新和条件删除默认拒绝空筛选;确需全表或全集合时显式使用 `all_records=True`。 + +## 6. 写入结果与部分失败 + +单次写入返回 `WriteResult`,批量写入返回 `BatchResult`。 +写方法不接受 `fields` / `returning`;需要重读时显式调用 `get`,并理解并发写入可能发生在两次操作之间。 + +| 字段 | 解释 | +| --- | --- | +| `acknowledged` | 收到执行确认;不表示事务已经提交 | +| `matched_count` / `modified_count` | 只有可靠依据时填写,否则为 `None` | +| `inserted_count` / `deleted_count` | 确认可知的记录数,不能把未知当作 0 | +| `generated_keys` | 可以映射为完整身份的生成键,否则为 `None` | +| `native_result` | 原生结果对象,或游标关闭前保存的统计快照 | + +MySQL 的 UPDATE `rowcount` 可能表示改变数或匹配数;不能把一个数同时填到两个字段,也不额外查询猜测。 +MySQL C 扩展能够提供当前会话的已协商标志时,填写对应的 matched 或 modified 计数。 +纯 Python / 异步等无法提供该证据的连接,统一计数保留 `None`,原始 `rowcount` 及未知解释模式保存在 `native_result`。 +连接上的可变客户端标志可能是下次建连配置,不能据此证明当前会话采用的计数模式。 +MongoDB 保留 matched / modified 的差别;未确认写入不虚构计数。 +Redis HSET 的新字段数不等于修改的记录数。 +MySQL 不根据 `lastrowid` 推算一批连续主键。 + +`BatchResult.items` 按输入位置保留状态;`counts` 只汇总可证实的统计,`native_results` 保存原生批次结果。 + +| 状态 | 已知事实 | +| --- | --- | +| `acknowledged` | 已收到该项可解释的执行确认 | +| `unacknowledged` | 驱动使用未确认写入 | +| `failed` | 明确报告该项失败 | +| `not_sent` | SDK 尚未发送该项 | +| `not_executed` | 驱动明确报告未执行,例如有序批次失败后的项 | +| `unknown` | 可能已经发送,现有证据无法确定结果 | + +```python +from datvia.crud import BatchOperationError + +try: + result = orders.bulk_update(changes, batch_size=500) +except BatchOperationError as error: + partial = error.partial_result + # 根据各项状态、业务幂等性和原始错误决定后续处理。 + # error.original 保留最初的驱动或明确的 SDK 错误;不要直接整批重试。 + raise +``` + +失败后 SDK 不再发送后续分块;已进入驱动的命令可能继续执行。 +`asyncio.CancelledError` 保持原类型、原对象,附加 `database_partial_result` 描述已知进度。 +清理失败附加在主异常的 `database_cleanup_errors` 中,不替换业务错误。 +批量异常可以同时包含确定成功、确定失败和不确定项;统一类别本身不能决定是否安全重试。 + +## 7. Redis Hash 格式与边界 + +```python +from datvia.crud import InsertItem +from datvia.crud import JsonFieldCodec + +with connections.create('cache') as connection: + with Records.redis_hash( + connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as cache: + cache.insert({'id': 1001, 'status': 'pending'}, ttl_s=60) + cache.update({'id': 1001}, set_values={'status': 'paid'}) + cache.insert_many([ + InsertItem({'id': 1002, 'status': 'pending'}, ttl_s=120), + {'id': 1003, 'status': 'pending'}, + ]) +``` + +只管理本库专用格式,不把已有 String、List 或任意 Hash 自动当成记录。 +键采用带版本的命名空间、声明的键字段顺序及类型编码,字符串 `'1'` 和整数 `1` 不发生碰撞。 +当前键类型限字符串和整数,排除布尔值;其他身份类型由调用方显式转换。 +每个 Hash 保存保留字段 `__database_record__=1`,业务记录不能使用这个名称。 +详细持久键格式在适配模块中定义,升级格式必须显式处理迁移。 +当前格式为 `database:hash:v1::<身份编码>`。 +两段均使用无填充的 URL-safe Base64;namespace 先编码为 UTF-8,身份先按声明顺序编码为紧凑的 ASCII JSON `[[字段名, 值], ...]`,非 ASCII 字符使用 JSON 转义。 + +`JsonFieldCodec` 对每个字段分别编码,支持标准 JSON 值及有限浮点数。 +不将 Decimal、日期、ObjectId 或 bytes 静默转成字符串或 float;需要这些类型时传入自定义 `FieldCodec`。 +自定义 codec 的 `encode(value)` 返回 str / bytes,`decode(value)` 必须兼容连接提供的 bytes / str。 +字段 codec 不负责身份键编码,也不更改连接的 `decode_responses` 设置。 + +| 场景 | `ttl_s` | +| --- | --- | +| 插入时省略或 `None` | 不设置过期时间 | +| 插入或更新传正整数 | 在单键操作中设置相应秒数 | +| 更新时省略 | 保留原 TTL | +| 更新时显式 `None` | 清除 TTL | +| 更新时记录已过期 | 未匹配,不重新创建 | + +0、负数、布尔值和超过 `2**53 - 1` 秒的 TTL 均拒绝。 +批量插入以 `InsertItem` 单独表达 TTL,普通记录中名为 `ttl_s` 的业务字段不会被当作控制参数。 +MySQL 和 MongoDB 不接受这一 TTL 选项。 + +存在性、格式检查、投影、创建或更新通过单键 Lua 脚本完成。 +运行账户需要对应命令权限;SDK 不在权限不足时降级成存在竞态的先查后写。 +Cluster 还需要原生驱动进行节点发现和命令元数据查询所需的权限,具体随 redis-py 版本而异。 +脚本的串行执行不等于错误回滚,也不等于跨键事务。 +Sentinel 使用所配置服务的主节点数据客户端,退出记录上下文后清理;原 Sentinel 对象仍可访问。 +Cluster 按单键路由,批次使用 `transaction=False`,不强制同槽,不承诺整批原子提交。 + +金额应由应用选择精确单位,例如整数最小货币单位,或支持精确小数的 codec。 +本层不把金额映射为浮点增量,不实现资金扣减、防重账、账本、分布式事务或余额一致性协议。 +执行确认也不是 Redis 持久化、复制或故障切换中不丢数据的保证。 + +## 8. 原生能力与事务仍由调用方控制 + +```python +with connections.create('orders') as connection: + with Records.mysql(connection, table='orders', key_fields=('id',)) as orders: + orders.update({'id': 1001}, set_values={'status': 'paid'}) + with connection.errors(operation='commit_orders'): + connection.native.commit() +``` + +SDK 不在记录上下文退出时额外提交、回滚或自动重试。 +MySQL autocommit 和连接关闭行为按原生驱动规则执行;需要明确回滚时调用方自己管理。 +MongoDB 可绑定原生 session,事务及 session 生命周期由调用方管理。 +Redis 其他结构、MongoDB 聚合、SQL JOIN、数据库专有原子操作都通过 `connection.native` 使用。 +上下文外原生异常行为不变;需要转换原生操作错误时显式使用 `connection.errors(operation=...)`。 + +## 9. Redis String 键值与批量操作 + +`RedisStrings` / `AsyncRedisStrings` 借用既有 Redis 连接,支持单机、Sentinel 和 Cluster。String 使用调用方提供的完整键,不自动加前缀,也不解析 JSON。 + +```python +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem + +with connections.create("cache") as connection: + with RedisStrings(connection) as strings: + strings.set("app:message", "hello", ttl_s=60) + print(strings.get("app:message")) + strings.set_many([ + StringSetItem("app:a", "A", ttl_s=60), + StringSetItem("app:b", b"B"), + ]) + print(strings.get_many(["app:b", "app:a", "app:missing"])) + strings.delete_many(["app:a", "app:b", "app:message"]) +``` + +```python +from datvia.crud import AsyncRedisStrings +from datvia.crud import StringSetItem + +async with await connections.create_async("cache") as connection: + async with AsyncRedisStrings(connection) as strings: + await strings.set("app:message", "hello", ttl_s=60) + print(await strings.get("app:message")) + await strings.set_many([ + StringSetItem("app:a", "A", ttl_s=60), + StringSetItem("app:b", b"B"), + ]) + print(await strings.get_many(["app:b", "app:a", "app:missing"])) + await strings.delete_many(["app:a", "app:b", "app:message"]) +``` + +键和值仅接受 str/bytes;get 返回类型遵循连接的 decode_responses,缺失返回 None,空字符串保留。set 成功返回 True,delete 返回 0/1;exists、expire、persist 返回布尔值。TTL 正整数秒由单条 SET EX 设置;普通 set 不指定 TTL 会清除原 TTL,expire 修改正整数秒过期,persist 清除过期。ttl 保留 -2(缺失)、-1(不过期)及剩余秒数。SET/DEL 保留对同名其他类型键的原生命令作用,GET 遇到类型不符抛出保留原始信息的异常。String 不提供字段投影。 + +批量输入为有限序列,默认每批 500。get_many 使用逐键 GET 保留顺序和重复位置,不保证重复读取来自同一时刻,也不会将 WRONGTYPE 当成缺失。set_many 接受 StringSetItem 序列,delete_many 接受键列表;写批次按实际编码后的 bytes 拒绝重复目标,所有输入先校验再发送。 + +批量写入返回 StringBatchResult,包含 requested_count、items、deleted_count、native_results;StringBatchItem 包含 index、key、state、result、error。失败通过 BatchOperationError.partial_result 保留上述结构,已派发但未收到明确回复的项标为 unknown,后续未派发项标为 not_sent。SET 不推测新增数或修改数;Cluster 跨槽批量不提供整体原子性。异步取消保留原对象及 database_partial_result,清理错误不覆盖主错误。 + +String 与 Hash 上下文只清理自身资源。Sentinel 使用配置的 service_name 创建临时数据客户端;外部池和借用的连接由原所有者管理。 diff --git a/examples/basic_usage.py b/examples/basic_usage.py new file mode 100644 index 0000000000000000000000000000000000000000..b6fdae061e299ce23d5f450cc86e4bb0e8747b90 --- /dev/null +++ b/examples/basic_usage.py @@ -0,0 +1,296 @@ +"""Complete explicit sync/async usage; supply credentials from the calling application.""" + +from contextlib import AsyncExitStack +from contextlib import ExitStack + +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRecords +from datvia.crud import AsyncRedisStrings +from datvia.crud import JsonFieldCodec +from datvia.crud import Records +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +from datvia.schema import AsyncSchema +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import TableSpec + +USER_TABLE = TableSpec( + columns={ + 'id': Column('varchar', length=32), + 'name': Column('varchar', length=100), + 'age': Column('int'), + }, + primary_key=('id',), +) + + +def make_connections( + mysql_password: str, + mongodb_uri: str, + redis_password: str | None = None, +) -> Connections: + return Connections( + { + # 初始化时不能选择一个尚未创建的数据库。 + 'mysql_setup': ConnectionConfig( + backend='mysql', + host='127.0.0.1', + username='sdk_demo', + password=mysql_password, + options={'autocommit': True}, + ), + 'mysql_app': ConnectionConfig( + backend='mysql', + host='127.0.0.1', + username='sdk_demo', + password=mysql_password, + options={ + 'database': 'demo_app', + 'autocommit': True, + }, + ), + 'documents': ConnectionConfig( + backend='mongodb', + host=mongodb_uri, + ), + 'cache': ConnectionConfig( + backend='redis', + host='127.0.0.1', + password=redis_password, + options={'db': 0, 'decode_responses': True}, + ), + } + ) + + +def run_sync(connections: Connections) -> None: + # 一、显式准备 MySQL 数据库与表。 + with connections.create('mysql_setup') as setup: + schema = Schema.mysql(setup) + schema.ensure_database('demo_app') + schema.ensure_table('demo_app', 'users', USER_TABLE) + + # 二、三个数据库连接在同一业务作用域内同时使用。 + with ExitStack() as stack: + mysql = stack.enter_context(connections.create('mysql_app')) + mongo = stack.enter_context(connections.create('documents')) + redis = stack.enter_context(connections.create('cache')) + + for connection in (mysql, mongo, redis): + connection.check() + + # 三、显式准备 MongoDB 普通集合。 + Schema.mongodb(mongo).ensure_collection( + 'demo_app', + 'users', + ) + + mysql_users = stack.enter_context( + Records.mysql( + mysql, + table='users', + key_fields=USER_TABLE.primary_key, + ) + ) + mongo_users = stack.enter_context( + Records.mongodb( + mongo, + database='demo_app', + collection='users', + ) + ) + + # Redis 不建表;绑定记录命名空间及 codec 即可开始使用。 + redis_users = stack.enter_context( + Records.redis_hash( + redis, + namespace='demo_app:users', + key_fields=('id',), + codec=JsonFieldCodec(), + ) + ) + + # 四、分别对三个后端执行相同的基础记录流程。 + for users, key_field in ( + (mysql_users, 'id'), + (mongo_users, '_id'), + (redis_users, 'id'), + ): + key = {key_field: 'u1'} + + users.insert({**key, 'name': 'Alice', 'age': 30}) + print(users.get(key, fields=['name', 'age'])) + users.update(key, set_values={'age': 31}) + + users.insert_many( + [ + {key_field: 'u2', 'name': 'Bob', 'age': 20}, + {key_field: 'u3', 'name': 'Carol', 'age': 40}, + ] + ) + users.bulk_update( + [ + { + 'key': {key_field: 'u2'}, + 'set_values': {'age': 21}, + }, + { + 'key': {key_field: 'u3'}, + 'set_values': {'age': 41}, + }, + ] + ) + print( + users.get_many( + [{key_field: 'u2'}, {key_field: 'u3'}], + fields=['name', 'age'], + ) + ) + + users.delete(key) + users.delete_many( + [ + {key_field: 'u2'}, + {key_field: 'u3'}, + ] + ) + + # 五、Redis String 键值操作。 + strings = stack.enter_context(RedisStrings(redis)) + + strings.set('demo:message', 'hello', ttl_s=60) + print(strings.get('demo:message')) + strings.set('demo:message', 'updated', ttl_s=120) + print(strings.ttl('demo:message')) + strings.delete('demo:message') + + strings.set_many( + [ + StringSetItem('demo:a', 'A', ttl_s=60), + StringSetItem('demo:b', 'B', ttl_s=60), + ] + ) + print(strings.get_many(['demo:a', 'demo:b', 'demo:missing'])) + strings.delete_many(['demo:a', 'demo:b']) + + # 上下文退出:先释放操作对象自有资源,再关闭连接。 + # 不自动删除数据库、表或集合。 + + +async def run_async(connections: Connections) -> None: + # 一、异步准备 MySQL 数据库与表。 + async with await connections.create_async('mysql_setup') as setup: + schema = AsyncSchema.mysql(setup) + await schema.ensure_database('demo_app') + await schema.ensure_table('demo_app', 'users', USER_TABLE) + + # 二、三个异步连接同时存在,分别管理各自资源。 + async with AsyncExitStack() as stack: + mysql = await stack.enter_async_context(await connections.create_async('mysql_app')) + mongo = await stack.enter_async_context(await connections.create_async('documents')) + redis = await stack.enter_async_context(await connections.create_async('cache')) + + for connection in (mysql, mongo, redis): + await connection.check() + + # 三、异步准备 MongoDB 集合。 + await AsyncSchema.mongodb(mongo).ensure_collection( + 'demo_app', + 'users', + ) + + mysql_users = await stack.enter_async_context( + AsyncRecords.mysql( + mysql, + table='users', + key_fields=USER_TABLE.primary_key, + ) + ) + mongo_users = await stack.enter_async_context( + AsyncRecords.mongodb( + mongo, + database='demo_app', + collection='users', + ) + ) + redis_users = await stack.enter_async_context( + AsyncRecords.redis_hash( + redis, + namespace='demo_app:users', + key_fields=('id',), + codec=JsonFieldCodec(), + ) + ) + + # 四、三个后端分别执行异步单条与批量 CRUD。 + for users, key_field in ( + (mysql_users, 'id'), + (mongo_users, '_id'), + (redis_users, 'id'), + ): + key = {key_field: 'u1'} + + await users.insert({**key, 'name': 'Alice', 'age': 30}) + print(await users.get(key, fields=['name', 'age'])) + await users.update(key, set_values={'age': 31}) + + await users.insert_many( + [ + {key_field: 'u2', 'name': 'Bob', 'age': 20}, + {key_field: 'u3', 'name': 'Carol', 'age': 40}, + ] + ) + await users.bulk_update( + [ + { + 'key': {key_field: 'u2'}, + 'set_values': {'age': 21}, + }, + { + 'key': {key_field: 'u3'}, + 'set_values': {'age': 41}, + }, + ] + ) + print( + await users.get_many( + [{key_field: 'u2'}, {key_field: 'u3'}], + fields=['name', 'age'], + ) + ) + + await users.delete(key) + await users.delete_many( + [ + {key_field: 'u2'}, + {key_field: 'u3'}, + ] + ) + + # 五、Redis String 异步操作。 + strings = await stack.enter_async_context(AsyncRedisStrings(redis)) + + await strings.set('demo:message', 'hello', ttl_s=60) + print(await strings.get('demo:message')) + await strings.set('demo:message', 'updated', ttl_s=120) + print(await strings.ttl('demo:message')) + await strings.delete('demo:message') + + await strings.set_many( + [ + StringSetItem('demo:a', 'A', ttl_s=60), + StringSetItem('demo:b', 'B', ttl_s=60), + ] + ) + print( + await strings.get_many( + [ + 'demo:a', + 'demo:b', + 'demo:missing', + ] + ) + ) + await strings.delete_many(['demo:a', 'demo:b']) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..140e8db35fdc02c4563d9cab24d447d9b5fcdddc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[build-system] +requires = [ + "setuptools>=61.0", + "wheel", + "setuptools-scm>=8.0", +] +build-backend = "setuptools.build_meta" + +[project] +name = "datvia" +dynamic = ["version"] +description = "Data via one Python API." +readme = "README.md" +license = { text = "MIT" } +authors = [ + { name = "MSR", email = "msr20666@foxmail.com" }, +] +requires-python = ">=3.10" +keywords = [ + "database", + "MongoDB", + "MySQL", + "Redis", +] +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", +] +dependencies = [ + "loongspears.utils @ git+https://gitee.com/loongspears/utils", + "pymongo>=4.6.0", + "mysql-connector-python>=8.3.0", + "redis>=5.0.0", +] + +[project.urls] +Homepage = "https://gitee.com/loongspears/datvia" +Repository = "https://gitee.com/loongspears/datvia.git" + +[tool.setuptools_scm] +write_to = "src/datvia/_version.py" diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..f8d66c6e57f34305bc1356c3dbc630c1a802a3b0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +# common +loongspears.utils @ git+https://gitee.com/loongspears/utils + +# MongoDB +pymongo>=4.6.0 + +# MySQL +mysql-connector-python>=8.3.0 + +# Redis +redis>=5.0.0 diff --git a/scripts/verify_accounts_mongodb.py b/scripts/verify_accounts_mongodb.py new file mode 100644 index 0000000000000000000000000000000000000000..72ab80f2f63839aff35573dfac941f0c16d304b2 --- /dev/null +++ b/scripts/verify_accounts_mongodb.py @@ -0,0 +1,252 @@ +"""Verify MongoDB account management against an explicitly configured isolated deployment.""" + +from __future__ import annotations + +import argparse +import asyncio +import inspect +import json +from contextlib import asynccontextmanager +from pathlib import Path +from secrets import token_urlsafe +from uuid import uuid4 + +import pymongo +from pymongo.errors import OperationFailure + +from datvia import ConnectionConfig +from datvia import Connections +from datvia.accounts import Accounts +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import AsyncAccounts +from datvia.accounts import MongoPrivilege +from datvia.accounts import MongoRole +from datvia.accounts import MongoUser +from datvia.errors import DriverError +from datvia.errors import PermissionDeniedError +from datvia.errors import attach_cleanup_errors + + +def settings(path: Path) -> tuple[dict, ConnectionConfig]: + config = json.loads(path.read_text(encoding='utf-8-sig')) + raw = dict( + host=config['host'], + port=config['port'], + username=config['username'], + password=config['password'], + authSource=config['auth_db'], + serverSelectionTimeoutMS=6000, + ) + sdk = ConnectionConfig( + backend='mongodb', + host=raw['host'], + port=raw['port'], + username=raw['username'], + password=raw['password'], + options={'authSource': raw['authSource'], 'serverSelectionTimeoutMS': 6000}, + ) + return raw, sdk + + +def check(condition: bool, label: str) -> None: + if not condition: + raise AssertionError(label) + + +async def result(value): + """Use identical assertions for both native driver execution modes.""" + return await value if inspect.isawaitable(value) else value + + +@asynccontextmanager +async def owned(client): + try: + yield client + finally: + await result(client.close()) + + +async def create_fixture(admin, identity, cleanup, **options) -> None: + """Record a unique fixture after success or after reconciling an ambiguous creation.""" + if isinstance(identity, MongoUser): + get, create, remove = admin.get_user, admin.create_user, admin.drop_user + else: + get, create, remove = admin.get_role, admin.create_role, admin.drop_role + if await result(get(identity)) is not None: + raise RuntimeError(f'Isolated fixture already exists: {identity!r}') + try: + await result(create(identity, **options)) + except (AccountWriteOutcomeUnknown, asyncio.CancelledError) as primary: + try: + observed = await result(get(identity)) + except Exception as lookup_error: + attach_cleanup_errors(primary, (lookup_error,)) + else: + if observed is not None: + cleanup.append((remove, identity)) + raise + cleanup.append((remove, identity)) + + +async def run_suite(raw: dict, sdk: ConnectionConfig, *, managed: bool, asynchronous: bool) -> None: + reg = Connections({'admin': sdk}) + factory = AsyncAccounts if asynchronous else Accounts + client_type = pymongo.AsyncMongoClient if asynchronous else pymongo.MongoClient + if managed: + owner = await reg.create_async('admin') if asynchronous else reg.create('admin') + else: + owner = client_type(**raw) + client = owner.native if managed else owner + admin = factory.mongodb(owner) + stem = 'qa_' + uuid4().hex + staff = stem + 'staff' + data = stem + 'data' + user = MongoUser(staff, 'analyst') + twin = MongoUser(stem + 'other', 'analyst') + reader = MongoUser(staff, 'viewer') + parent = MongoRole(data, 'parent') + child = MongoRole(data, 'child') + audit = MongoRole(data, 'viewer') + read = MongoPrivilege({'db': data, 'collection': 'reports'}, ['find']) + update = MongoPrivilege({'db': data, 'collection': 'reports'}, ['update']) + password, replacement, viewer_secret = token_urlsafe(24), token_urlsafe(24), token_urlsafe(24) + cleanup = [] + primary = None + + def login(account, pwd): + return client_type(**{**raw, 'username': account.name, 'password': pwd, 'authSource': account.auth_db}) + + try: + await create_fixture(admin, parent, cleanup, privileges=[read]) + await create_fixture(admin, child, cleanup) + await result(admin.grant_roles_to_role(child, roles=[parent])) + await result(admin.grant_privileges(parent, privileges=[update])) + await result(admin.revoke_privileges(parent, privileges=[update])) + check( + tuple((await result(admin.get_role(child, expand_privileges=True))).inherited_privileges) == (read,), + 'inherited privilege and incremental removal', + ) + check(child in [item.role for item in await result(admin.list_roles(db=data))], 'role list') + check( + any(item.builtin for item in await result(admin.list_roles(db=data, include_builtin=True))), + 'built-in roles', + ) + await create_fixture(admin, user, cleanup, password=password) + await create_fixture(admin, twin, cleanup, password=token_urlsafe(24)) + try: + await result(admin.create_user(user, password=replacement)) + except DriverError: + pass + else: + raise AssertionError('duplicate identity unexpectedly created') + await result(admin.grant_roles(user, roles=[child])) + view = await result(admin.get_user(user, expand_privileges=True)) + check(view.roles == (child,) and read in view.inherited_privileges, 'user details') + check((await result(admin.get_user(user))).inherited_privileges is None, 'explicit expansion') + check(user in [v.user for v in await result(admin.list_users(auth_db=staff))], 'per-db list') + check({user, twin}.issubset({v.user for v in await result(admin.list_users_all_databases())}), 'cross-db list') + async with owned(login(user, password)) as restricted: + await result(restricted[data].reports.find_one({})) + try: + await result(restricted[data].private.find_one({})) + except OperationFailure as exc: + check(exc.code == 13, 'unexpected read rejection') + else: + raise AssertionError('unrelated collection readable') + await result(admin.change_password(user, password=replacement)) + async with owned(login(user, password)) as old: + try: + await result(old.admin.command('ping')) + except OperationFailure as exc: + check(exc.code == 18, 'expected authentication failure') + else: + raise AssertionError('old password accepted') + await result(admin.revoke_roles_from_role(child, roles=[parent])) + async with owned(login(user, replacement)) as restricted: + try: + await result(restricted[data].reports.find_one({})) + except OperationFailure as exc: + check(exc.code == 13, 'expected revoked access') + else: + raise AssertionError('read survived revocation') + await result(admin.grant_roles_to_role(child, roles=[parent])) + await result(admin.revoke_roles(user, roles=[child])) + check(not (await result(admin.get_user(user))).roles, 'user role revocation') + await result(admin.grant_roles(user, roles=[parent])) + await result(admin.drop_role(parent)) + cleanup.remove((admin.drop_role, parent)) + check( + not (await result(admin.get_role(child))).roles and not (await result(admin.get_user(user))).roles, + 'dropped role references', + ) + await create_fixture( + admin, audit, cleanup, privileges=[MongoPrivilege({'db': data, 'collection': ''}, ['viewUser'])] + ) + await create_fixture(admin, reader, cleanup, password=viewer_secret, roles=[audit]) + async with owned(login(reader, viewer_secret)) as viewer_client: + viewer = factory.mongodb(viewer_client) + await result(viewer.list_users(auth_db=data)) + try: + await result(viewer.drop_user(user)) + except PermissionDeniedError: + pass + else: + raise AssertionError('read-only manager could delete a user') + await result(admin.drop_user(user)) + cleanup.remove((admin.drop_user, user)) + check( + await result(admin.get_user(user)) is None and await result(admin.get_user(twin)) is not None, + 'exact deletion', + ) + async with owned(login(user, replacement)) as removed: + try: + await result(removed.admin.command('ping')) + except OperationFailure as exc: + check(exc.code == 18, 'deleted user authentication') + else: + raise AssertionError('deleted account can authenticate') + await result(client.admin.command('ping')) + except BaseException as exc: + primary = exc + raise + finally: + failures = [] + for remove, item in reversed(cleanup): + try: + await result(remove(item)) + except Exception as exc: + failures.append(exc) + try: + await result(owner.close()) + except Exception as exc: + failures.append(exc) + if failures: + if primary is None: + attach_cleanup_errors(failures[0], failures[1:]) + raise failures[0] + attach_cleanup_errors(primary, failures) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--config', + required=True, + type=Path, + help='Dedicated test service JSON: host, port, username, password, auth_db', + ) + parser.add_argument('--mode', choices=['sync', 'async', 'both'], default='both') + args = parser.parse_args() + raw, sdk = settings(args.config) + with pymongo.MongoClient(**raw) as client: + version = client.server_info()['version'] + for mode in ('sync', 'async'): + if args.mode not in (mode, 'both'): + continue + for managed in (False, True): + asyncio.run(run_suite(raw, sdk, managed=managed, asynchronous=mode == 'async')) + print(f'PASS MongoDB={version} PyMongo={pymongo.version} mode={mode} managed={managed}') + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_accounts_mysql.py b/scripts/verify_accounts_mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..b55dba0a9714fdde6ac3552da22b827ec292a80a --- /dev/null +++ b/scripts/verify_accounts_mysql.py @@ -0,0 +1,263 @@ +"""Run MySQL account acceptance on an explicitly supplied disposable instance.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import secrets +import uuid +from contextlib import ExitStack +from contextlib import closing +from contextlib import contextmanager +from pathlib import Path + +import mysql.connector +from mysql.connector import aio +from mysql.connector import errors + +from datvia import ConnectionConfig +from datvia import Connections +from datvia.accounts import Accounts +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import AsyncAccounts +from datvia.accounts import MySQLAccount +from datvia.accounts import MySQLScope +from datvia.errors import ConfigurationError +from datvia.errors import DependencyError +from datvia.errors import PermissionDeniedError +from datvia.errors import attach_cleanup_errors + + +def statement(conn, sql: str, args: tuple = ()) -> list[tuple]: + with closing(conn.cursor()) as cur: + cur.execute(sql, args or None) + return cur.fetchall() if cur.with_rows else [] + + +def require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def denied(action, codes=(1044, 1045, 1142, 3118)): + try: + action() + except errors.Error as err: + if err.errno in codes: + return + raise + raise AssertionError('Expected server denial') + + +def credentials(cfg, user, password): + return {**cfg, 'user': user.user, 'password': password} + + +def managed(cfg): + values = dict(cfg) + host, port = values.pop('host'), values.pop('port') + user, password = values.pop('user'), values.pop('password') + return Connections( + { + 'admin': ConnectionConfig( + backend='mysql', host=host, port=port, username=user, password=password, options=values + ) + } + ) + + +@contextmanager +def track_create(stack, cfg, target, *, kind): + """Reconcile only a prechecked, unique identity in the caller's isolated instance.""" + + def exists(conn): + if kind == 'database': + return bool( + statement(conn, 'SELECT SCHEMA_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME=%s', (target,)) + ) + return Accounts.mysql(conn).get_account(target) is not None + + def release(exc_type, primary, traceback): + try: + with closing(mysql.connector.connect(**cfg)) as conn: + if exists(conn): + if kind == 'database': + statement(conn, f'DROP DATABASE `{target}`') + elif kind == 'role': + Accounts.mysql(conn).drop_role(target) + else: + Accounts.mysql(conn).drop_user(target) + except Exception as cleanup_error: + if primary is None: + raise + attach_cleanup_errors(primary, (cleanup_error,)) + + with closing(mysql.connector.connect(**cfg)) as check: + require(not exists(check), 'collision in isolated resource namespace') + try: + yield + except (AccountWriteOutcomeUnknown, asyncio.CancelledError, errors.OperationalError, errors.InterfaceError): + stack.push(release) + raise + else: + stack.push(release) + + +def verify_sync(cfg: dict) -> None: + token = 'qa' + uuid.uuid4().hex[:12] + a, b = token + '_a', token + 'Xa' + with closing(mysql.connector.connect(**cfg)) as root, ExitStack() as cleanup: + tool = Accounts.mysql(root) + user = MySQLAccount(token + "'u", '%') + sibling = MySQLAccount(user.user, '192.0.2.77') + role = MySQLAccount(token + 'r', '%') + child = MySQLAccount(token + 'c', '%') + password = secrets.token_urlsafe(20) + replacement = secrets.token_urlsafe(20) + with managed(cfg).create('admin') as handle: + require(Accounts.mysql(handle).get_account(user) is None, 'unexpected account collision') + percent, percent_other = token + '%schema', token + 'Zschema' + for db in (a, b, percent, percent_other): + with track_create(cleanup, cfg, db, kind='database'): + statement(root, f'CREATE DATABASE `{db}`') + statement(root, f'CREATE TABLE `{db}`.t (id INT)') + statement(root, f'CREATE PROCEDURE `{a}`.testproc() SELECT 1') + statement(root, f'CREATE FUNCTION `{a}`.testfn() RETURNS INTEGER DETERMINISTIC RETURN 1') + for account in (user, sibling): + with track_create(cleanup, cfg, account, kind='user'): + tool.create_user(account, password=password) + require(tool.get_account(user).account == user and tool.get_account(sibling).account == sibling, 'get identity') + require({user, sibling} <= {r.account for r in tool.list_accounts()}, 'list identity') + for r in (role, child): + with track_create(cleanup, cfg, r, kind='role'): + tool.create_role(r) + tool.grant_privileges(child, MySQLScope.database(a), ['SELECT']) + tool.grant_roles(role, roles=[child]) + tool.grant_roles(user, roles=[role]) + tool.set_default_roles(user, roles=[role]) + require(tool.get_default_roles(user) == (role,), 'default role') + require(any(b.role == role for b in tool.list_role_bindings(user)), 'role binding') + require(any('SELECT' in g for g in tool.show_grants(user, using_roles=[role]).statements), 'expanded grants') + require(not any('SELECT' in g for g in tool.show_grants(user).statements), 'direct grants') + for scope, priv in [ + (MySQLScope.columns(a, 't', ['id']), 'UPDATE'), + (MySQLScope.table(a, 't'), 'INSERT'), + (MySQLScope.procedure(a, 'testproc'), 'EXECUTE'), + (MySQLScope.function(a, 'testfn'), 'EXECUTE'), + (MySQLScope.global_(), 'CONNECTION_ADMIN'), + ]: + tool.grant_privileges(role, scope, [priv]) + tool.revoke_privileges(role, scope, [priv]) + with closing(mysql.connector.connect(**credentials(cfg, user, password))) as limited: + statement(limited, f'SELECT * FROM `{a}`.t') + denied(lambda: statement(limited, f'SELECT * FROM `{b}`.t')) + try: + Accounts.mysql(limited).list_accounts() + except PermissionDeniedError: + pass + else: + raise AssertionError('limited user accessed account catalog') + tool.grant_privileges(child, MySQLScope.database(percent), ['SELECT']) + with closing(mysql.connector.connect(**credentials(cfg, user, password))) as limited: + statement(limited, f'SELECT * FROM `{percent}`.t') + denied(lambda: statement(limited, f'SELECT * FROM `{percent_other}`.t')) + tool.revoke_privileges(child, MySQLScope.database(percent), ['SELECT']) + if statement(root, 'SELECT @@GLOBAL.partial_revokes')[0][0]: + tool.grant_privileges(sibling, MySQLScope.global_(), ['SELECT']) + tool.revoke_privileges(sibling, MySQLScope.database(a), ['SELECT']) + require(any(line.startswith('REVOKE') for line in tool.show_grants(sibling).statements), 'partial revoke') + tool.revoke_privileges(child, MySQLScope.database(a), ['SELECT']) + with closing(mysql.connector.connect(**credentials(cfg, user, password))) as limited: + denied(lambda: statement(limited, f'SELECT * FROM `{a}`.t')) + tool.revoke_roles(user, roles=[role]) + tool.grant_privileges(user, MySQLScope.database(a), ['INSERT']) + tool.grant_privileges(user, MySQLScope.database(a), ['SELECT'], grant_option=True) + with closing(mysql.connector.connect(**credentials(cfg, user, password))) as delegated: + Accounts.mysql(delegated).grant_privileges(child, MySQLScope.database(a), ['INSERT']) + require(any('INSERT' in line for line in tool.show_grants(child).statements), 'static grant option scope') + tool.lock_user(user) + require(tool.get_account(user).locked, 'lock state') + denied(lambda: mysql.connector.connect(**credentials(cfg, user, password)).close()) + tool.unlock_user(user) + tool.change_password(user, password=replacement) + denied(lambda: mysql.connector.connect(**credentials(cfg, user, password)).close()) + with closing(mysql.connector.connect(**credentials(cfg, user, replacement))) as limited: + statement(limited, 'SELECT 1') + # An account write must never commit this row implicitly. + root.start_transaction() + statement(root, f'INSERT INTO `{a}`.t VALUES (1)') + try: + tool.drop_user(sibling) + except ConfigurationError: + pass + else: + raise AssertionError('account statement accepted active transaction') + root.rollback() + require(not statement(root, f'SELECT * FROM `{a}`.t'), 'transaction committed') + require(tool.get_account(sibling) is not None, 'sibling changed') + # Check the more demanding SQL mode. Some older connector versions cannot encode it. + statement(root, "SET SESSION sql_mode='STRICT_ALL_TABLES,NO_BACKSLASH_ESCAPES'") + try: + tool.change_password(user, password=replacement + "'\\x") + except DependencyError: + with closing(mysql.connector.connect(**credentials(cfg, user, replacement))) as limited: + statement(limited, 'SELECT 1') + else: + with closing(mysql.connector.connect(**credentials(cfg, user, replacement + "'\\x"))) as limited: + statement(limited, 'SELECT 1') + finally: + statement(root, 'SET SESSION sql_mode=DEFAULT') + with closing(mysql.connector.connect(**cfg)) as root: + require(Accounts.mysql(root).get_account(user) is None, 'account survived cleanup') + + +async def verify_async(sync_cfg: dict) -> None: + cfg = {k: v for k, v in sync_cfg.items() if k != 'use_pure'} + conn = await aio.connect(**cfg) + user = MySQLAccount('qa' + uuid.uuid4().hex[:14], '%') + role = MySQLAccount(user.user + 'r', '%') + pw = secrets.token_urlsafe(20) + try: + admin = AsyncAccounts.mysql(conn) + with ExitStack() as cleanup: + with track_create(cleanup, sync_cfg, user, kind='user'): + await admin.create_user(user, password=pw) + with track_create(cleanup, sync_cfg, role, kind='role'): + await admin.create_role(role) + async with await managed(cfg).create_async('admin') as handle: + bound = AsyncAccounts.mysql(handle) + await bound.grant_roles(user, roles=[role]) + require(any(b.role == role for b in await bound.list_role_bindings(user)), 'async grant') + await bound.revoke_roles(user, roles=[role]) + async with await aio.connect(**credentials(cfg, user, pw)) as limited: + async with await limited.cursor() as cur: + await cur.execute('SELECT 1') + require((await cur.fetchone())[0] == 1, 'async login') + await admin.drop_role(role) + await admin.drop_user(user) + require(await admin.get_account(user) is None, 'async deletion') + finally: + await conn.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + '--config', required=True, type=Path, help='JSON Connector/Python parameters for an isolated test server' + ) + args = parser.parse_args() + cfg = json.loads(args.config.read_text(encoding='utf-8-sig')) + for field in ('host', 'port', 'user', 'password'): + if field not in cfg: + parser.error(f'Missing required configuration key {field}') + cfg['autocommit'] = True + with closing(mysql.connector.connect(**cfg)) as conn: + print('MySQL', statement(conn, 'SELECT VERSION()')[0][0], 'Connector/Python', mysql.connector.__version__) + verify_sync(cfg) + asyncio.run(verify_async(cfg)) + print('MySQL account acceptance passed (sync, async, managed, native).') + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_accounts_redis.py b/scripts/verify_accounts_redis.py new file mode 100644 index 0000000000000000000000000000000000000000..5d3011e8d741f9bd094a7392370be6b8cd8bf65c --- /dev/null +++ b/scripts/verify_accounts_redis.py @@ -0,0 +1,220 @@ +"""Run Redis account acceptance on a fresh private loopback instance of a supplied server. + +Example: python -B scripts/verify_accounts_redis.py --server /path/to/redis-server +The optional --protocol 3 exercises RESP3. No existing Redis connection is accepted. +""" + +from __future__ import annotations + +import argparse +import asyncio +import gc +import secrets +import socket +import subprocess +import sys +import tempfile +import time +from contextlib import contextmanager +from pathlib import Path + +import redis +import redis.asyncio + +from datvia import ConnectionConfig +from datvia import Connections +from datvia.accounts import Accounts +from datvia.accounts import AclRule +from datvia.accounts import AsyncAccounts +from datvia.errors import DependencyError +from datvia.errors import DriverError +from datvia.errors import PermissionDeniedError + + +def require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def expect(error_type, action) -> None: + try: + action() + except error_type: + return + raise AssertionError(f'Expected {error_type.__name__}') + + +def client(port, password, username='default', protocol=2, *, decode=False): + return redis.Redis( + host='127.0.0.1', + port=port, + username=username, + password=password, + protocol=protocol, + decode_responses=decode, + socket_connect_timeout=2, + socket_timeout=3, + ) + + +@contextmanager +def server(binary: Path, folder: Path, port: int, password: str, *, persistent: bool): + acl_file = folder / 'users.acl' + if persistent and not acl_file.exists(): + acl_file.write_text(f'user default on >{password} ~* +@all\n', encoding='ascii') + acl_config = 'aclfile users.acl' if persistent else f'requirepass {password}' + config = folder / 'redis.conf' + config.write_text(f'bind 127.0.0.1\nport {port}\nsave ""\nappendonly no\n{acl_config}\n', encoding='ascii') + flags = getattr(subprocess, 'CREATE_NO_WINDOW', 0) + with (folder / 'server.log').open('ab') as log: + process = subprocess.Popen([str(binary), config.name], cwd=folder, stdout=log, stderr=log, creationflags=flags) + try: + deadline = time.monotonic() + 20 + with client(port, password) as admin: + while True: + if process.poll() is not None: + raise RuntimeError(f'Disposable Redis exited with code {process.returncode}.') + try: + admin.ping() + break + except redis.ConnectionError: + if time.monotonic() >= deadline: + raise RuntimeError('Disposable Redis did not become ready.') from None + time.sleep(0.1) + yield admin + finally: + # This test owns this child and explicitly persists ACLs with ACL SAVE. + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + + +def fresh_port(): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + return sock.getsockname()[1] + + +async def check_async(port, password, protocol, name, config, major): + raw = redis.asyncio.Redis(host='127.0.0.1', port=port, password=password, protocol=protocol) + try: + acl = AsyncAccounts.redis(raw) + require(name in await acl.list_users(), 'Async native inventory missing target.') + require((await acl.get_user(name)).enabled, 'Async native target missing.') + async with await Connections({'node': config}).create_async('node') as handle: + acl = AsyncAccounts.redis(handle) + await acl.apply_user(name, rules=[AclRule.allow_command('exists')]) + require('+exists' in (await acl.get_user(name)).commands, 'Async grant not applied.') + if major >= 7: + require((await acl.dry_run(name, 'exists', 'qa:read')).allowed, 'Async simulation failed.') + await acl.save_acl() + require(not await acl.delete_user(name + '-missing'), 'Async missing delete count.') + require(await raw.ping(), 'Borrowed async connection was closed.') + finally: + closer = getattr(raw, 'aclose', None) or raw.close + await closer() + + +def check_runtime(admin, name, password, port, protocol): + version = admin.info('server')['redis_version'] + major, minor = [int(i) for i in version.split('.')[:2]] + old, new = secrets.token_hex(16), secrets.token_hex(16) + config = ConnectionConfig( + backend='redis', host='127.0.0.1', port=port, password=password, options={'protocol': protocol} + ) + with Connections({'node': config}).create('node') as handle: + acl = Accounts.redis(handle) + require(acl.get_user(name) is None, 'Unexpected existing account.') + rules = [ + AclRule.enable(), + AclRule.add_password(old), + AclRule.allow_command('get'), + AclRule.key_pattern('qa:*'), + AclRule.allow_command('ping'), + ] + if (major, minor) >= (6, 2): + rules.extend([AclRule.clear_channels(), AclRule.channel_pattern('events:*')]) + acl.apply_user(name, rules=rules) + data = acl.get_user(name) + require(data.enabled and name in acl.list_users(), 'User missing or disabled.') + if (major, minor) >= (6, 2): + require(any('events:*' in p for p in data.channels), 'Missing channel rule.') + else: + expect(DependencyError, lambda: acl.apply_user(name, rules=[AclRule.clear_channels()])) + if major >= 7: + acl.apply_user( + name, + rules=[AclRule.selector(AclRule.allow_command('set'), AclRule.key_pattern('out:*', access='write'))], + ) + require(acl.get_user(name).selectors, 'Missing selector.') + require(acl.dry_run(name, 'set', 'out:result', 'sample').allowed, 'Selector simulation failed.') + require(not acl.dry_run(name, 'set', 'qa:read', 'sample').allowed, 'Unexpected permission.') + expect(DriverError, lambda: acl.dry_run(name + '-missing', 'ping')) + expect(DriverError, lambda: acl.dry_run(name, 'invalid-command-for-qa')) + else: + expect(DependencyError, lambda: acl.dry_run(name, 'ping')) + expect(DependencyError, lambda: acl.apply_user(name, rules=[AclRule.clear_selectors()])) + with client(port, old, name, protocol) as limited: + require(limited.get('qa:read') is None, 'Authorized read failed.') + expect(redis.exceptions.NoPermissionError, lambda: limited.get('forbidden:read')) + expect(PermissionDeniedError, lambda: Accounts.redis(limited).get_user(name)) + if major >= 7: + expect(PermissionDeniedError, lambda: Accounts.redis(limited).dry_run(name, 'ping')) + acl.apply_user(name, rules=[AclRule.add_password(new), AclRule.deny_command('get')]) + expect(redis.exceptions.NoPermissionError, lambda: limited.get('qa:read')) + with client(port, new, name, protocol) as migrated: + require(migrated.ping(), 'Rotated password rejected.') + acl.apply_user(name, rules=[AclRule.remove_password(old), AclRule.allow_command('get')]) + with client(port, old, name, protocol) as previous: + expect(redis.AuthenticationError, previous.ping) + acl.apply_user(name, rules=[AclRule.disable()]) + require(limited.ping(), 'Disabling unexpectedly revoked an existing session.') + with client(port, new, name, protocol) as disabled: + expect(redis.AuthenticationError, disabled.ping) + acl.apply_user(name, rules=[AclRule.enable(), AclRule.replace_passwords([new])]) + asyncio.run(check_async(port, password, protocol, name, config, major)) + require(acl.delete_user(name), 'Existing user was not deleted.') + expect(redis.ConnectionError, limited.ping) + require(acl.get_user(name) is None and not acl.delete_user(name), 'Deletion did not converge.') + expect(DriverError, lambda: acl.delete_user('default')) + require(admin.ping(), 'Independent admin connection lost.') + return version + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--server', type=Path, required=True) + parser.add_argument('--protocol', type=int, choices=(2, 3), default=2) + args = parser.parse_args() + binary = args.server.resolve(strict=True) + password, name = secrets.token_hex(24), 'qa-' + secrets.token_hex(8) + port = fresh_port() + with tempfile.TemporaryDirectory(prefix='redis-accounts-live-') as tmp: + folder = Path(tmp) + with server(binary, folder, port, password, persistent=True) as admin: + version = check_runtime(admin, name, password, port, args.protocol) + with client(port, password, protocol=args.protocol, decode=True) as strings: + a = Accounts.redis(strings) + require(a.get_user('default').enabled, 'decode_responses=True failed.') + a.apply_user(name, rules=[AclRule.disable()]) + a.save_acl() + with server(binary, folder, port, password, persistent=True) as admin: + require(Accounts.redis(admin).get_user(name) is not None, 'ACL file did not survive restart.') + with server(binary, folder, port, password, persistent=False) as admin: + a = Accounts.redis(admin) + a.apply_user(name, rules=[AclRule.disable()]) + expect(DriverError, a.save_acl) + require(a.get_user(name) is not None, 'Failed ACL SAVE discarded runtime changes.') + gc.collect() + print( + f'PASS Redis {version} / redis-py {redis.__version__} / RESP{args.protocol} / Python ' + f'{sys.version_info.major}.{sys.version_info.minor}: lifecycle, ACL, sync/async, persistence, cleanup' + ) + + +if __name__ == '__main__': + main() diff --git a/scripts/verify_basics.py b/scripts/verify_basics.py new file mode 100644 index 0000000000000000000000000000000000000000..4ac879be18410af82cbe94ff840a675508568e57 --- /dev/null +++ b/scripts/verify_basics.py @@ -0,0 +1,353 @@ +"""Verify basic initialization and CRUD against explicit isolated fixtures. + +Run from the repository root with --config and optionally --mode sync/async. +The fixture is a mapping of names to ConnectionConfig keyword arguments. +MySQL fixtures must omit database/db and explicitly enable autocommit. +This script creates and drops randomly named test databases/keys. Use isolated +services only. It never loads environment variables or prints credentials. +""" + +import argparse +import asyncio +import json +from collections.abc import AsyncIterator +from collections.abc import Awaitable +from collections.abc import Callable +from collections.abc import Iterator +from collections.abc import Mapping +from contextlib import AsyncExitStack +from contextlib import ExitStack +from contextlib import asynccontextmanager +from contextlib import contextmanager +from dataclasses import replace +from pathlib import Path +from traceback import extract_tb +from typing import Any +from uuid import uuid4 + +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRecords +from datvia.crud import AsyncRedisStrings +from datvia.crud import JsonFieldCodec +from datvia.crud import Records +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +from datvia.errors import attach_cleanup_errors +from datvia.schema import AsyncSchema +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import TableSpec + +TABLE = TableSpec( + columns={'id': Column('varchar', length=32), 'name': Column('varchar', length=100), 'age': Column('int')}, + primary_key=('id',), +) + + +@contextmanager +def cleaning(cleanup: Callable[[], Any]) -> Iterator[None]: + """Always attempt test-object cleanup without masking the initial failure.""" + try: + yield + except BaseException as primary: + try: + cleanup() + except BaseException as secondary: + attach_cleanup_errors(primary, (secondary,)) + raise + else: + cleanup() + + +@asynccontextmanager +async def cleaning_async(cleanup: Callable[[], Awaitable[Any]]) -> AsyncIterator[None]: + try: + yield + except BaseException as primary: + try: + await cleanup() + except BaseException as secondary: + attach_cleanup_errors(primary, (secondary,)) + raise + else: + await cleanup() + + +def record_cycle(records: Any) -> None: + keys = [{'id': key} for key in ('one', 'two', 'three')] + with cleaning(lambda: records.delete_many(keys)): + _record_cycle_steps(records) + + +async def record_cycle_async(records: Any) -> None: + keys = [{'id': key} for key in ('one', 'two', 'three')] + async with cleaning_async(lambda: records.delete_many(keys)): + await _record_cycle_steps_async(records) + + +def _record_cycle_steps(records: Any) -> None: + records.insert({'id': 'one', 'name': 'Alice', 'age': 30}) + assert records.get({'id': 'one'}, fields=['name']) == {'name': 'Alice'} + assert records.get({'id': 'one'}, fields=[]) == {} + assert records.get({'id': 'missing'}) is None + records.update({'id': 'one'}, set_values={'age': 31}) + records.insert_many([{'id': 'two', 'name': 'Bob', 'age': 20}, {'id': 'three', 'name': 'Carol', 'age': 40}]) + records.bulk_update( + [ + {'key': {'id': 'two'}, 'set_values': {'age': 21}}, + {'key': {'id': 'three'}, 'set_values': {'age': 41}}, + ] + ) + result = records.get_many([{'id': 'three'}, {'id': 'one'}, {'id': 'three'}, {'id': 'two'}], fields=['age']) + assert [item.record for item in result] == [{'age': 41}, {'age': 31}, {'age': 41}, {'age': 21}] + records.delete({'id': 'one'}) + records.delete_many([{'id': 'two'}, {'id': 'three'}]) + assert all(records.get({'id': key}) is None for key in ('one', 'two', 'three')) + + +async def _record_cycle_steps_async(records: Any) -> None: + await records.insert({'id': 'one', 'name': 'Alice', 'age': 30}) + assert await records.get({'id': 'one'}, fields=['name']) == {'name': 'Alice'} + assert await records.get({'id': 'one'}, fields=[]) == {} + assert await records.get({'id': 'missing'}) is None + await records.update({'id': 'one'}, set_values={'age': 31}) + await records.insert_many([{'id': 'two', 'name': 'Bob', 'age': 20}, {'id': 'three', 'name': 'Carol', 'age': 40}]) + await records.bulk_update( + [ + {'key': {'id': 'two'}, 'set_values': {'age': 21}}, + {'key': {'id': 'three'}, 'set_values': {'age': 41}}, + ] + ) + result = await records.get_many([{'id': 'three'}, {'id': 'one'}, {'id': 'three'}, {'id': 'two'}], fields=['age']) + assert [item.record for item in result] == [{'age': 41}, {'age': 31}, {'age': 41}, {'age': 21}] + await records.delete({'id': 'one'}) + await records.delete_many([{'id': 'two'}, {'id': 'three'}]) + for key in ('one', 'two', 'three'): + assert await records.get({'id': key}) is None + + +def strings_cycle(connection: Any, prefix: str) -> None: + keys = [prefix + ':a', prefix + ':b', prefix + ':empty'] + with RedisStrings(connection) as strings: + with cleaning(lambda: strings.delete_many(keys)): + strings.set(keys[0], 'hello', ttl_s=60) + assert strings.exists(keys[0]) and 0 <= strings.ttl(keys[0]) <= 60 + strings.set(keys[0], 'updated') + assert strings.ttl(keys[0]) == -1 + assert strings.expire(keys[0], 60) + assert strings.persist(keys[0]) + strings.set(keys[2], '') + assert strings.get(keys[2]) in ('', b'') + strings.set_many([StringSetItem(keys[0], 'A', 60), StringSetItem(keys[1], b'B', 60)]) + values = strings.get_many([keys[1], keys[0], keys[1], prefix + ':missing']) + assert values in ([b'B', b'A', b'B', None], ['B', 'A', 'B', None]) + assert all(strings.ttl(key) == -2 for key in keys) + + +async def strings_cycle_async(connection: Any, prefix: str) -> None: + keys = [prefix + ':a', prefix + ':b', prefix + ':empty'] + async with AsyncRedisStrings(connection) as strings: + async with cleaning_async(lambda: strings.delete_many(keys)): + await strings.set(keys[0], 'hello', ttl_s=60) + assert await strings.exists(keys[0]) + assert 0 <= await strings.ttl(keys[0]) <= 60 + await strings.set(keys[0], 'updated') + assert await strings.ttl(keys[0]) == -1 + assert await strings.expire(keys[0], 60) + assert await strings.persist(keys[0]) + await strings.set(keys[2], '') + assert await strings.get(keys[2]) in ('', b'') + await strings.set_many([StringSetItem(keys[0], 'A', 60), StringSetItem(keys[1], b'B', 60)]) + values = await strings.get_many([keys[1], keys[0], keys[1], prefix + ':missing']) + assert values in ([b'B', b'A', b'B', None], ['B', 'A', 'B', None]) + for key in keys: + assert await strings.ttl(key) == -2 + + +def verify_sync(fixtures: Mapping[str, ConnectionConfig]) -> None: + connections = Connections(fixtures) + with ExitStack() as stack: + handles = {} + for name in fixtures: + first = stack.enter_context(connections.create(name)) + second = stack.enter_context(connections.create(name)) + assert first.native is not second.native + first.check() + second.check() + first.close() + second.check() + handles[name] = second + for name, config in fixtures.items(): + token = 'sdk_basic_' + uuid4().hex + connection = handles[name] + backend = config.backend.value + if backend == 'mysql': + schema = Schema.mysql(connection) + with cleaning(lambda: schema.drop_database(token)): + schema.ensure_database(token) + schema.ensure_table(token, 'users', TABLE) + schema.ensure_table(token, 'users', TABLE) + assert schema.database_exists(token) and token in schema.list_databases() + assert schema.table_exists(token, 'users') and 'users' in schema.list_tables(token) + assert schema.describe_table(token, 'users') is not None + runtime = Connections({'data': replace(config, options={**config.options, 'database': token})}) + with ( + runtime.create('data') as data, + Records.mysql(data, table='users', key_fields=('id',)) as records, + ): + with data.native.cursor() as selected: + selected.execute('SELECT DATABASE()') + assert selected.fetchone()[0] in (token, token.encode('ascii')) + record_cycle(records) + with data.native.cursor() as cursor: + cursor.execute('SELECT 1') + assert cursor.fetchone()[0] == 1 + schema.drop_table(token, 'users') + schema.drop_table(token, 'users') + assert not schema.table_exists(token, 'users') + schema.drop_database(token) + elif backend == 'mongodb': + schema = Schema.mongodb(connection) + with cleaning(lambda: schema.drop_database(token)): + schema.ensure_collection(token, 'users', key_fields=('id',)) + schema.ensure_collection(token, 'users', key_fields=('id',)) + assert schema.database_exists(token) and token in schema.list_databases() + assert schema.collection_exists(token, 'users') and 'users' in schema.list_collections(token) + assert schema.describe_collection(token, 'users') is not None + with Records.mongodb(connection, database=token, collection='users', key_fields=('id',)) as records: + record_cycle(records) + assert connection.native[token].command('ping')['ok'] == 1 + schema.drop_collection(token, 'users') + schema.drop_collection(token, 'users') + assert not schema.collection_exists(token, 'users') + schema.drop_database(token) + else: + with Records.redis_hash( + connection, namespace=token, key_fields=('id',), codec=JsonFieldCodec() + ) as records: + record_cycle(records) + strings_cycle(connection, token) + print(json.dumps({'connection': name, 'backend': backend, 'mode': 'sync', 'result': 'passed'})) + for connection in handles.values(): + connection.check() + print(json.dumps({'mode': 'sync', 'simultaneous_connections': len(handles), 'result': 'passed'})) + + +async def verify_async(fixtures: Mapping[str, ConnectionConfig]) -> None: + connections = Connections(fixtures) + async with AsyncExitStack() as stack: + handles = {} + for name in fixtures: + first = await stack.enter_async_context(await connections.create_async(name)) + second = await stack.enter_async_context(await connections.create_async(name)) + assert first.native is not second.native + await first.check() + await second.check() + await first.close() + await second.check() + handles[name] = second + for name, config in fixtures.items(): + token = 'sdk_basic_' + uuid4().hex + connection = handles[name] + backend = config.backend.value + if backend == 'mysql': + schema = AsyncSchema.mysql(connection) + async with cleaning_async(lambda: schema.drop_database(token)): + await schema.ensure_database(token) + await schema.ensure_table(token, 'users', TABLE) + await schema.ensure_table(token, 'users', TABLE) + assert await schema.database_exists(token) and token in await schema.list_databases() + assert await schema.table_exists(token, 'users') and 'users' in await schema.list_tables(token) + assert await schema.describe_table(token, 'users') is not None + runtime = Connections({'data': replace(config, options={**config.options, 'database': token})}) + async with await runtime.create_async('data') as data: + async with await data.native.cursor() as selected: + await selected.execute('SELECT DATABASE()') + assert (await selected.fetchone())[0] in (token, token.encode('ascii')) + async with AsyncRecords.mysql(data, table='users', key_fields=('id',)) as records: + await record_cycle_async(records) + async with await data.native.cursor() as cursor: + await cursor.execute('SELECT 1') + assert (await cursor.fetchone())[0] == 1 + await schema.drop_table(token, 'users') + await schema.drop_table(token, 'users') + assert not await schema.table_exists(token, 'users') + await schema.drop_database(token) + elif backend == 'mongodb': + schema = AsyncSchema.mongodb(connection) + async with cleaning_async(lambda: schema.drop_database(token)): + await schema.ensure_collection(token, 'users', key_fields=('id',)) + await schema.ensure_collection(token, 'users', key_fields=('id',)) + assert await schema.database_exists(token) and token in await schema.list_databases() + assert await schema.collection_exists(token, 'users') and 'users' in await schema.list_collections( + token + ) + assert await schema.describe_collection(token, 'users') is not None + async with AsyncRecords.mongodb( + connection, database=token, collection='users', key_fields=('id',) + ) as records: + await record_cycle_async(records) + assert (await connection.native[token].command('ping'))['ok'] == 1 + await schema.drop_collection(token, 'users') + await schema.drop_collection(token, 'users') + assert not await schema.collection_exists(token, 'users') + await schema.drop_database(token) + else: + async with AsyncRecords.redis_hash( + connection, namespace=token, key_fields=('id',), codec=JsonFieldCodec() + ) as records: + await record_cycle_async(records) + await strings_cycle_async(connection, token) + print(json.dumps({'connection': name, 'backend': backend, 'mode': 'async', 'result': 'passed'})) + for connection in handles.values(): + await connection.check() + print(json.dumps({'mode': 'async', 'simultaneous_connections': len(handles), 'result': 'passed'})) + + +def load_fixtures(path: Path) -> dict[str, ConnectionConfig]: + fixtures = { + name: ConnectionConfig(**parameters) + for name, parameters in json.loads(path.read_text(encoding='utf-8-sig')).items() + } + if not fixtures: + raise ValueError('Provide at least one explicit isolated service configuration.') + for config in fixtures.values(): + if config.backend.value == 'mysql': + if 'database' in config.options or 'db' in config.options or config.options.get('autocommit') is not True: + raise ValueError('MySQL fixtures must omit database/db and explicitly set autocommit=True.') + return fixtures + + +def main() -> None: + if not __debug__: + raise RuntimeError('Run verification without Python optimization flags.') + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--config', required=True, type=Path) + parser.add_argument('--mode', choices=('sync', 'async', 'both'), default='both') + args = parser.parse_args() + fixtures = load_fixtures(args.config) + if args.mode in ('sync', 'both'): + verify_sync(fixtures) + if args.mode in ('async', 'both'): + asyncio.run(verify_async(fixtures)) + + +if __name__ == '__main__': + try: + main() + except Exception as error: + trace = extract_tb(error.__traceback__) + print( + json.dumps( + { + 'result': 'failed', + 'error_type': type(error).__name__, + 'operation': getattr(error, 'operation', None), + 'file': Path(trace[-1].filename).name if trace else None, + 'line': trace[-1].lineno if trace else None, + 'cleanup_errors': len(getattr(error, 'database_cleanup_errors', ())), + } + ) + ) + raise SystemExit(1) from None diff --git a/scripts/verify_connections.py b/scripts/verify_connections.py new file mode 100644 index 0000000000000000000000000000000000000000..e82400bebe788c620459d1d58568345a63b6c40e --- /dev/null +++ b/scripts/verify_connections.py @@ -0,0 +1,91 @@ +"""Run read-only probes against explicitly supplied isolated test services. + +Usage: python scripts/verify_connections.py --config path/to/fixtures.json +Fixture format: {"connection_name": {ConnectionConfig keyword arguments...}}. +Configuration loading belongs to this test caller, never to the SDK. +""" +import argparse +import asyncio +import json +from contextlib import AsyncExitStack, ExitStack +from pathlib import Path + +from datvia import ConnectionConfig, Connections + + +def verify_sync(connections, name, backend): + with connections.create(name) as first, connections.create(name) as second: + assert first.native is not second.native + result = first.check() + first.close() + second.check() + if backend == "mysql": + with second.errors(operation="select_one"): + with second.native.cursor() as cursor: + cursor.execute("SELECT 1") + assert cursor.fetchone()[0] == 1 + return {"name": name, "mode": "sync", "scope": result.scope, + "target_count": len(result.targets), "result": "passed"} + + +async def verify_async(connections, name, backend): + async with await connections.create_async(name) as first: + async with await connections.create_async(name) as second: + assert first.native is not second.native + result = await first.check() + await first.close() + await second.check() + if backend == "mysql": + with second.errors(operation="select_one"): + async with await second.native.cursor() as cursor: + await cursor.execute("SELECT 1") + assert (await cursor.fetchone())[0] == 1 + return {"name": name, "mode": "async", "scope": result.scope, + "target_count": len(result.targets), "result": "passed"} + + +def verify_together(connections, names): + with ExitStack() as stack: + handles = [stack.enter_context(connections.create(name)) for name in names] + for handle in handles: + handle.check() + if handles: + handles[0].close() + for handle in handles[1:]: + handle.check() + return {"mode": "sync", "simultaneous_connections": len(handles), "result": "passed"} + + +async def verify_together_async(connections, names): + async with AsyncExitStack() as stack: + handles = [await stack.enter_async_context(await connections.create_async(name)) + for name in names] + for handle in handles: + await handle.check() + if handles: + await handles[0].close() + for handle in handles[1:]: + await handle.check() + return {"mode": "async", "simultaneous_connections": len(handles), "result": "passed"} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", required=True, type=Path) + args = parser.parse_args() + fixtures = json.loads(args.config.read_text(encoding="utf-8-sig")) + connections = Connections({ + name: ConnectionConfig(**parameters) + for name, parameters in fixtures.items() + }) + for name, parameters in fixtures.items(): + print(json.dumps(verify_sync(connections, name, parameters["backend"]))) + print(json.dumps(asyncio.run( + verify_async(connections, name, parameters["backend"]) + ))) + print(json.dumps(verify_together(connections, fixtures))) + print(json.dumps(asyncio.run(verify_together_async(connections, fixtures)))) + + +if __name__ == "__main__": + main() diff --git a/src/datvia/__init__.py b/src/datvia/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0a7783a53a4164f2f16431f6ded417ce585c505e --- /dev/null +++ b/src/datvia/__init__.py @@ -0,0 +1,15 @@ +try: + from ._version import __version__ +except ImportError: + __version__ = '0.0.0+unknown' + +__description__ = 'Data via one Python API.' + +from .config import ConnectionConfig, DatabaseType +from .connections import AsyncConnection, CheckResult, Connection, Connections +from .errors import DatabaseError + +__all__ = [ + 'ConnectionConfig', 'DatabaseType', 'Connections', 'Connection', + 'AsyncConnection', 'CheckResult', 'DatabaseError', +] diff --git a/src/datvia/_adapters/__init__.py b/src/datvia/_adapters/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..34b6f5890a10015fcf9bcf26075392c01730928e --- /dev/null +++ b/src/datvia/_adapters/__init__.py @@ -0,0 +1 @@ +"""Backend adapters loaded only when a connection is created.""" diff --git a/src/datvia/_adapters/mongodb.py b/src/datvia/_adapters/mongodb.py new file mode 100644 index 0000000000000000000000000000000000000000..5eb78039d6e92c29f51cbd3b2693ff341e325c4b --- /dev/null +++ b/src/datvia/_adapters/mongodb.py @@ -0,0 +1,82 @@ +"""MongoDB operations for managed connections.""" + +from pymongo import errors as mongo_errors + +from ..config import ConnectionConfig +from ..errors import ( + AuthenticationError, + ConfigurationError, + DatabaseConnectionError, + DatabaseError, + DatabaseTimeoutError, + DependencyError, + DriverError, + PermissionDeniedError, + error_context, +) +from ..mongodb import Client + + +class Adapter: + def __init__(self, config: ConnectionConfig, name: str, is_async: bool = False): + self._config = config + self._name = name + + def create(self): + return Client.create(**self._config.driver_kwargs()) + + async def create_async(self): + try: + return Client.create_async(**self._config.driver_kwargs()) + except ImportError as original: + raise DependencyError( + "Install PyMongo >= 4.9 for asynchronous MongoDB connections.", + connection_name=self._name, + backend="mongodb", + phase="create", + original=original, + ) from original + + @staticmethod + def classify(error: Exception) -> type[DatabaseError] | None: + if not isinstance(error, mongo_errors.PyMongoError): + return None + if isinstance(error, mongo_errors.OperationFailure): + if error.code == 18: + return AuthenticationError + if error.code == 13: + return PermissionDeniedError + if error.timeout: + return DatabaseTimeoutError + if isinstance(error, mongo_errors.ConnectionFailure): + return DatabaseConnectionError + if isinstance(error, mongo_errors.ConfigurationError): + return ConfigurationError + return DriverError + + def _ping_errors(self): + return error_context( + connection_name=self._name, + backend="mongodb", + phase="check", + operation="ping", + target="selected_server", + classifier=self.classify, + owned=True, + ) + + def check(self, native) -> tuple[str, tuple[str, ...]]: + with self._ping_errors(): + native.admin.command("ping", read_preference=native.read_preference) + return "selected_server", () + + async def check_async(self, native) -> tuple[str, tuple[str, ...]]: + with self._ping_errors(): + await native.admin.command("ping", read_preference=native.read_preference) + return "selected_server", () + + def close(self, native) -> None: + native.close() + + async def close_async(self, native) -> None: + await native.close() diff --git a/src/datvia/_adapters/mysql.py b/src/datvia/_adapters/mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..444bfea84f36e8f0f58bb55ebd44e8732b69be74 --- /dev/null +++ b/src/datvia/_adapters/mysql.py @@ -0,0 +1,151 @@ +"""MySQL Connector/Python adapter for managed connections.""" + +import asyncio +from ipaddress import IPv6Address + +from mysql.connector import errorcode +from mysql.connector import errors as driver_errors + +from ..config import ConnectionConfig +from ..errors import ( + AuthenticationError, + DatabaseConnectionError, + DatabaseError, + DatabaseTimeoutError, + DependencyError, + DriverError, + PermissionDeniedError, + error_context, +) +from ..mysql import Client + + +_PERMISSION_CODES = frozenset({ + errorcode.ER_DBACCESS_DENIED_ERROR, + errorcode.ER_TABLEACCESS_DENIED_ERROR, + errorcode.ER_COLUMNACCESS_DENIED_ERROR, + errorcode.ER_SPECIFIC_ACCESS_DENIED_ERROR, +}) +_CONNECTION_CODES = frozenset({ + errorcode.CR_CONNECTION_ERROR, + errorcode.CR_CONN_HOST_ERROR, + errorcode.CR_IPSOCK_ERROR, + errorcode.CR_UNKNOWN_HOST, + errorcode.CR_SERVER_GONE_ERROR, + errorcode.CR_TCP_CONNECTION, + errorcode.CR_SERVER_HANDSHAKE_ERR, + errorcode.CR_SERVER_LOST, + errorcode.CR_SSL_CONNECTION_ERROR, + errorcode.CR_SERVER_LOST_EXTENDED, +}) +# Connector/Python 8.3 does not define the driver-specific timeout types. +_NATIVE_TIMEOUTS = tuple( + timeout_type for timeout_type in ( + getattr(driver_errors, "ConnectionTimeoutError", None), + getattr(driver_errors, "ReadTimeoutError", None), + getattr(driver_errors, "WriteTimeoutError", None), + ) if timeout_type is not None +) +_TRANSPORT_TIMEOUTS = (TimeoutError, asyncio.TimeoutError) + + +def _connection_target(native) -> str | None: + if getattr(native, "unix_socket", None): + return "unix_socket" + + host = getattr(native, "server_host", None) + port = getattr(native, "server_port", None) + if not isinstance(host, str) or not host or type(port) is not int or not 1 <= port <= 65535: + return None + if any(character.isspace() or character in "@/?#\\" for character in host): + return None + if ":" in host: + try: + host = f"[{IPv6Address(host.removeprefix('[').removesuffix(']'))}]" + except ValueError: + return None + return f"{host}:{port}" + + +class Adapter: + def __init__(self, config: ConnectionConfig, name: str, is_async: bool = False): + self._config = config + self._name = name + + def create(self): + kwargs = self._config.driver_kwargs() + with self._dependency_context(): + return Client.create(**kwargs) + + async def create_async(self): + kwargs = self._config.driver_kwargs() + with self._dependency_context(): + return await Client.create_async(**kwargs) + + def _dependency_context(self): + return error_context( + connection_name=self._name, + backend="mysql", + phase="create", + classifier=lambda error: DependencyError if isinstance(error, ImportError) else None, + ) + + @staticmethod + def classify(error: BaseException | None) -> type[DatabaseError] | None: + if not isinstance(error, driver_errors.Error): + return None + if error.errno == errorcode.ER_ACCESS_DENIED_ERROR: + return AuthenticationError + if error.errno in _PERMISSION_CODES: + return PermissionDeniedError + if isinstance(error, _NATIVE_TIMEOUTS): + return DatabaseTimeoutError + if error.errno in _CONNECTION_CODES: + if isinstance(error.__cause__, _TRANSPORT_TIMEOUTS): + return DatabaseTimeoutError + return DatabaseConnectionError + return DriverError + + def _classify_check_error(self, error: Exception) -> type[DatabaseError] | None: + # ping() can replace a transport failure with an unnumbered InterfaceError. + if not isinstance(error, driver_errors.InterfaceError) or error.errno not in (-1, None): + return self.classify(error) + cause = error.__cause__ + if isinstance(cause, _TRANSPORT_TIMEOUTS): + return DatabaseTimeoutError + if cause is None or isinstance(cause, ConnectionError): + return DatabaseConnectionError + if isinstance(cause, driver_errors.OperationalError) and cause.errno in (-1, None): + return DatabaseConnectionError + return self.classify(cause) or DriverError + + def _check_context(self, target): + return error_context( + connection_name=self._name, + backend="mysql", + phase="check", + operation="ping", + target=target, + classifier=self._classify_check_error, + owned=True, + ) + + def check(self, native) -> tuple[str, tuple[str, ...]]: + target = _connection_target(native) + with self._check_context(target): + native.ping(reconnect=False) + return "current_connection", (target,) if target else () + + async def check_async(self, native) -> tuple[str, tuple[str, ...]]: + target = _connection_target(native) + with self._check_context(target): + await native.ping(reconnect=False) + return "current_connection", (target,) if target else () + + @staticmethod + def close(native): + native.close() + + @staticmethod + async def close_async(native): + await native.close() diff --git a/src/datvia/_adapters/redis.py b/src/datvia/_adapters/redis.py new file mode 100644 index 0000000000000000000000000000000000000000..d49f07e2a5fd00afbb19b7647489eb49e8abeab8 --- /dev/null +++ b/src/datvia/_adapters/redis.py @@ -0,0 +1,338 @@ +"""Managed Redis clients for standalone, Sentinel, and Cluster deployments.""" + +from copy import copy +from functools import partial + +from redis import asyncio as async_redis +from redis import cluster, exceptions + +from ..config import ConnectionConfig +from ..errors import ( + AuthenticationError, + DatabaseError, + ConfigurationError, + DatabaseConnectionError, + DatabaseTimeoutError, + DriverError, + PermissionDeniedError, + attach_cleanup_errors, + convert_error, + error_context, +) +from ..redis import Client + + +def _endpoint(host, port) -> str | None: + if not isinstance(host, str) or not host or any(char in host for char in "/@?#\r\n"): + return None + if isinstance(port, str) and port.isdecimal(): + port = int(port) + if not isinstance(port, int) or isinstance(port, bool): + return None + if ":" in host and not host.startswith("["): + host = f"[{host}]" + return f"{host}:{port}" + + +def _single_target(native) -> str | None: + settings = native.connection_pool.connection_kwargs + path = settings.get("path") + if isinstance(path, str) and path and "://" not in path: + return path + return _endpoint(settings.get("host"), settings.get("port")) + + +def _master_target(native) -> str | None: + address = native.connection_pool.master_address + if isinstance(address, tuple) and len(address) == 2: + return _endpoint(*address) + return None + + +def _async_closer(native): + close = getattr(native, "aclose", None) + return close if close is not None else native.close + + +class Adapter: + """Dispatch to the native Redis factories and manage owned resources.""" + + def __init__(self, config: ConnectionConfig, name: str, is_async: bool = False): + self._config = config + self._name = name + self._is_async = is_async + self._external_pool = config.options.get("connection_pool") is not None + self._pending_cluster_nodes = {} + + @staticmethod + def classify(error: Exception) -> type[DatabaseError] | None: + if isinstance(error, ( + exceptions.AuthenticationError, + exceptions.AuthenticationWrongNumberOfArgsError, + )): + return AuthenticationError + if isinstance(error, (exceptions.NoPermissionError, exceptions.AuthorizationError)): + return PermissionDeniedError + if isinstance(error, exceptions.TimeoutError): + return DatabaseTimeoutError + if isinstance(error, exceptions.ConnectionError): + return DatabaseConnectionError + if isinstance(error, (exceptions.RedisError, exceptions.RedisClusterException)): + return DriverError + return None + + def create(self): + return self._create() + + async def create_async(self): + # Async redis-py clients can be awaitable. Construction leaves I/O to callers. + return self._create() + + def _create(self): + kwargs = self._config.driver_kwargs() + mode = self._config.mode + if mode == "sentinel" and self._config.nodes: + kwargs["sentinels"] = list(self._config.nodes) + if mode == "cluster": + node_type = async_redis.cluster.ClusterNode if self._is_async else cluster.ClusterNode + if self._config.nodes: + kwargs["startup_nodes"] = [node_type(host, port) for host, port in self._config.nodes] + elif not self._is_async and kwargs.get("startup_nodes"): + kwargs["startup_nodes"] = self._sync_startup_nodes(kwargs["startup_nodes"]) + return Client.connect(mode, is_async=self._is_async, **kwargs) + + def _sync_startup_nodes(self, startup_nodes) -> list: + nodes = [] + for node in startup_nodes: + if getattr(node, "redis_connection", None) is not None: + raise ConfigurationError( + "Use Cluster startup nodes without an existing client.", + connection_name=self._name, backend="redis", phase="create", + ) + # redis-py adds a client in place during discovery. Keep reusable + # configured nodes independent of that client's ownership. + if isinstance(node, cluster.ClusterNode): + node = copy(node) + nodes.append(node) + return nodes + + def _context(self, phase: str, *, operation=None, target=None): + return error_context( + connection_name=self._name, + backend="redis", + phase=phase, + operation=operation, + target=target, + classifier=self.classify, + owned=True, + ) + + def _require_pong(self, result, target: str | None) -> None: + if result is not True: + raise DatabaseConnectionError( + "Redis did not acknowledge PING.", + connection_name=self._name, backend="redis", phase="check", + operation="PING", target=target, + ) + + def check(self, native) -> tuple[str, tuple[str, ...]]: + if self._config.mode == "sentinel": + return self._check_sentinel(native) + if self._config.mode == "cluster": + targets = [] + for node in self._primaries(native): + target = _endpoint(node.host, node.port) + with self._context("check", operation="PING", target=target): + self._require_pong(native.ping(target_nodes=node), target) + if target is not None: + targets.append(target) + return "cluster_primaries", tuple(targets) + + target = _single_target(native) + with self._context("check", operation="PING", target=target): + self._require_pong(native.ping(), target) + return "single", (target,) if target is not None else () + + async def check_async(self, native) -> tuple[str, tuple[str, ...]]: + if self._config.mode == "sentinel": + return await self._check_sentinel_async(native) + if self._config.mode == "cluster": + # redis-py makes initialize() idempotent. + with self._context("check", operation="initialize"): + await native.initialize() + targets = [] + for node in self._primaries(native): + target = _endpoint(node.host, node.port) + with self._context("check", operation="PING", target=target): + self._require_pong(await native.ping(target_nodes=node), target) + if target is not None: + targets.append(target) + return "cluster_primaries", tuple(targets) + + target = _single_target(native) + with self._context("check", operation="PING", target=target): + self._require_pong(await native.ping(), target) + return "single", (target,) if target is not None else () + + def _primaries(self, native) -> tuple: + with self._context("check", operation="get_primaries"): + primaries = tuple(native.get_primaries()) + if not primaries: + raise DatabaseConnectionError( + "Redis Cluster reported no primary nodes.", + connection_name=self._name, backend="redis", phase="check", + operation="get_primaries", + ) + return primaries + + def _check_sentinel(self, native) -> tuple[str, tuple[str, ...]]: + with self._context("check", operation="master_for", target="sentinel_master"): + primary = native.master_for(self._config.service_name) + target = None + error = None + try: + try: + result = primary.ping() + except BaseException as caught: + target = _master_target(primary) + self._raise_probe_error(caught, target) + target = _master_target(primary) + self._require_pong(result, target or "sentinel_master") + except BaseException as caught: + error = caught + raise + finally: + errors = self._close_single(primary, True, target=target or "sentinel_master") + self._finish_cleanup(errors, error) + return "sentinel_master", (target,) if target is not None else () + + async def _check_sentinel_async(self, native) -> tuple[str, tuple[str, ...]]: + with self._context("check", operation="master_for", target="sentinel_master"): + primary = native.master_for(self._config.service_name) + target = None + error = None + try: + try: + result = await primary.ping() + except BaseException as caught: + target = _master_target(primary) + self._raise_probe_error(caught, target) + target = _master_target(primary) + self._require_pong(result, target or "sentinel_master") + except BaseException as caught: + error = caught + raise + finally: + errors = await self._close_single_async(primary, True, target=target or "sentinel_master") + self._finish_cleanup(errors, error) + return "sentinel_master", (target,) if target is not None else () + + def _raise_probe_error(self, error: BaseException, target: str | None): + converted = convert_error( + error, connection_name=self._name, backend="redis", phase="check", + operation="PING", target=target or "sentinel_master", + classifier=self.classify, owned=True, + ) + if converted is error: + raise error + raise converted from error + + def _release(self, action, target, errors) -> None: + try: + with self._context("close", target=target): + action() + except BaseException as error: + errors.append(error) + + async def _release_async(self, action, target, errors) -> None: + try: + with self._context("close", target=target): + await action() + except BaseException as error: + errors.append(error) + + @staticmethod + def _finish_cleanup(errors, primary=None) -> None: + if not errors: + return + if primary is not None: + attach_cleanup_errors(primary, errors) + return + first, *remainder = errors + attach_cleanup_errors(first, remainder) + raise first + + def _close_single(self, native, owned: bool, *, target=None) -> list[BaseException]: + errors = [] + target = target or _single_target(native) + driver_closes_pool = getattr(native, "auto_close_connection_pool", False) + if not owned and driver_closes_pool: + # External pools must remain borrowed even if caller code changed + # the driver's ownership flag on the exposed client. + native.auto_close_connection_pool = False + self._release(native.close, target, errors) + if owned and (not driver_closes_pool or errors): + self._release(native.connection_pool.disconnect, target, errors) + return errors + + async def _close_single_async(self, native, owned: bool, *, target=None) -> list[BaseException]: + errors = [] + target = target or _single_target(native) + close = _async_closer(native) + # Keep client release and owned-pool release independent on all versions. + await self._release_async(partial(close, close_connection_pool=False), target, errors) + if owned: + pool = native.connection_pool + cleanup = getattr(pool, "aclose", None) or pool.disconnect + await self._release_async(cleanup, target, errors) + return errors + + @staticmethod + def _cluster_nodes(native): + # Include startup nodes that never became part of the discovered topology. + nodes = (*native.get_nodes(), *native.nodes_manager.startup_nodes.values()) + return {id(node): node for node in nodes}.values() + + def close(self, native) -> None: + errors = [] + if self._config.mode == "sentinel": + for client in tuple(native.sentinels): + errors.extend(self._close_single(client, True)) + elif self._config.mode == "cluster": + topology_clients = {id(node.redis_connection) for node in native.get_nodes()} + clients = { + id(node.redis_connection): node.redis_connection + for node in self._cluster_nodes(native) if node.redis_connection is not None + } + self._release(native.close, None, errors) + cluster_close_failed = bool(errors) + for client in clients.values(): + if cluster_close_failed or id(client) not in topology_clients: + errors.extend(self._close_single(client, True)) + elif not getattr(client, "auto_close_connection_pool", False): + self._release(client.connection_pool.disconnect, _single_target(client), errors) + else: + errors.extend(self._close_single(native, not self._external_pool)) + self._finish_cleanup(errors) + + async def close_async(self, native) -> None: + errors = [] + if self._config.mode == "sentinel": + for client in tuple(native.sentinels): + errors.extend(await self._close_single_async(client, True)) + elif self._config.mode == "cluster": + nodes = self._cluster_nodes(native) + await self._release_async(_async_closer(native), None, errors) + if errors: + # Native close marks the client uninitialized before disconnecting. + # Later closes may be no-ops, so retain nodes until cleanup succeeds. + self._pending_cluster_nodes.update((id(node), node) for node in nodes) + for node_id, node in tuple(self._pending_cluster_nodes.items()): + target = _endpoint(node.host, node.port) + error_count = len(errors) + await self._release_async(node.disconnect, target, errors) + if len(errors) == error_count: + del self._pending_cluster_nodes[node_id] + else: + errors.extend(await self._close_single_async(native, not self._external_pool)) + self._finish_cleanup(errors) diff --git a/src/datvia/accounts/__init__.py b/src/datvia/accounts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c79a32da62e0ff5ac2c36ff1792c8a49ecd2a710 --- /dev/null +++ b/src/datvia/accounts/__init__.py @@ -0,0 +1,94 @@ +"""Database user and privilege management on borrowed connections.""" + +from __future__ import annotations + +from importlib import import_module +from typing import TYPE_CHECKING +from typing import Any + +from ._base import AccountWriteOutcomeUnknown +from ._base import Binding + +if TYPE_CHECKING: + from ._mongodb import AsyncMongoDBAccounts + from ._mongodb import MongoDBAccounts + from ._mongodb import MongoPrivilege + from ._mongodb import MongoRole + from ._mongodb import MongoUser + from ._mysql import AsyncMySQLAccounts + from ._mysql import MySQLAccount + from ._mysql import MySQLAccounts + from ._mysql import MySQLScope + from ._redis import AclRule + from ._redis import AsyncRedisAccounts + from ._redis import RedisAccounts + +__all__ = [ + 'AccountWriteOutcomeUnknown', + 'Accounts', + 'AclRule', + 'AsyncAccounts', + 'MongoPrivilege', + 'MongoRole', + 'MongoUser', + 'MySQLAccount', + 'MySQLScope', +] + + +class Accounts: + """Bind synchronous account operations without network I/O.""" + + @staticmethod + def mongodb(connection: object) -> MongoDBAccounts: + from ._mongodb import MongoDBAccounts + + return MongoDBAccounts(Binding(connection, 'mongodb', is_async=False)) + + @staticmethod + def mysql(connection: object) -> MySQLAccounts: + from ._mysql import MySQLAccounts + + return MySQLAccounts(Binding(connection, 'mysql', is_async=False)) + + @staticmethod + def redis(connection: object) -> RedisAccounts: + from ._redis import RedisAccounts + + return RedisAccounts(Binding(connection, 'redis', is_async=False)) + + +class AsyncAccounts: + """Bind asynchronous account operations without network I/O.""" + + @staticmethod + def mongodb(connection: object) -> AsyncMongoDBAccounts: + from ._mongodb import AsyncMongoDBAccounts + + return AsyncMongoDBAccounts(Binding(connection, 'mongodb', is_async=True)) + + @staticmethod + def mysql(connection: object) -> AsyncMySQLAccounts: + from ._mysql import AsyncMySQLAccounts + + return AsyncMySQLAccounts(Binding(connection, 'mysql', is_async=True)) + + @staticmethod + def redis(connection: object) -> AsyncRedisAccounts: + from ._redis import AsyncRedisAccounts + + return AsyncRedisAccounts(Binding(connection, 'redis', is_async=True)) + + +def __getattr__(name: str) -> Any: + modules = { + 'MongoUser': 'mongodb', + 'MongoRole': 'mongodb', + 'MongoPrivilege': 'mongodb', + 'MySQLAccount': 'mysql', + 'MySQLScope': 'mysql', + 'AclRule': 'redis', + } + if name in modules: + return getattr(import_module(f'{__name__}._{modules[name]}'), name) + raise AttributeError(name) diff --git a/src/datvia/accounts/_base.py b/src/datvia/accounts/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..80e567761e71e9838647bf3b5f7c79d26da1e8f4 --- /dev/null +++ b/src/datvia/accounts/_base.py @@ -0,0 +1,124 @@ +"""Binding and operation boundaries for borrowed account-management clients.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from importlib import import_module +from typing import Any + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DatabaseConnectionError +from ..errors import DatabaseError +from ..errors import DatabaseTimeoutError +from ..errors import DependencyError +from ..errors import attach_cleanup_errors +from ..errors import convert_error + + +class AccountWriteOutcomeUnknown(DatabaseError): + """A mutation was submitted but its result could not be confirmed.""" + + +def _matches_native(client: object, backend: str, is_async: bool) -> bool: + if backend == 'mongodb': + import pymongo + + if is_async and not hasattr(pymongo, 'AsyncMongoClient'): + raise DependencyError('Async MongoDB requires PyMongo >= 4.9.', backend=backend, phase='config') + return isinstance(client, pymongo.AsyncMongoClient if is_async else pymongo.MongoClient) + if backend == 'mysql': + prefix = 'mysql.connector.aio' if is_async else 'mysql.connector' + abstract = import_module(f'{prefix}.abstracts').MySQLConnectionAbstract + if isinstance(client, abstract): + return True + try: + pooled = import_module(f'{prefix}.pooling').PooledMySQLConnection + except (ModuleNotFoundError, AttributeError): + return False + return isinstance(client, pooled) + + redis = import_module('redis.asyncio' if is_async else 'redis') + return isinstance(client, redis.Redis) + + +def _fixed_redis_node(client: Any, is_async: bool) -> bool: + prefix = 'redis.asyncio' if is_async else 'redis' + pipeline = import_module(f'{prefix}.client').Pipeline + sentinel_pool = import_module(f'{prefix}.sentinel').SentinelConnectionPool + # Pipelines subclass Redis but queue commands instead of returning an acknowledgement. + return not isinstance(client, pipeline) and not isinstance(client.connection_pool, sentinel_pool) + + +class Binding: + """Validate a managed handle or native client without creating a connection.""" + + def __init__(self, connection: object, backend: str, *, is_async: bool) -> None: + self.backend = backend + self._managed = connection if isinstance(connection, Connection) else None + self.connection_name = self._managed.name if self._managed is not None else None + invalid = ConfigurationError( + 'Expected a matching account-management client and connection mode.', + backend=backend, + connection_name=self.connection_name, + phase='config', + ) + try: + if self._managed is not None: + if self._managed.backend != backend or isinstance(self._managed, AsyncConnection) != is_async: + raise invalid + if backend == 'redis' and self._managed.mode != 'single': + raise invalid + self._client = self._managed.native + self._classify = self._managed._adapter.classify + else: + if not _matches_native(connection, backend, is_async): + raise invalid + self._client = connection + self._classify = import_module(f'datvia._adapters.{backend}').Adapter.classify + if backend == 'redis' and not _fixed_redis_node(self._client, is_async): + raise invalid + except ImportError as original: + raise DependencyError( + 'The selected database driver is unavailable.', backend=backend, phase='config', original=original + ) from original + + @property + def native(self) -> Any: + if self._managed is not None: + if isinstance(self._managed, AsyncConnection): + self._managed._ensure_loop('operation') + return self._managed.native + return self._client + + @contextmanager + def operation(self, operation: str, target: str | None = None, *, write: bool = False) -> Iterator[None]: + """Mark only the actual write dispatch as possibly having changed server state.""" + if self._managed is not None: + if isinstance(self._managed, AsyncConnection): + self._managed._ensure_loop('operation') + self._managed._ensure_open('operation') + context = dict( + connection_name=self.connection_name, + backend=self.backend, + phase='operation', + operation=operation, + target=target, + ) + try: + yield + except Exception as original: + converted = convert_error(original, **context, classifier=self._classify, owned=True) + if converted is original: + raise + if write and isinstance(converted, (DatabaseConnectionError, DatabaseTimeoutError)): + converted = AccountWriteOutcomeUnknown( + 'The database did not confirm the account change.', **context, original=original + ) + attributes = BaseException.__dict__['__dict__'].__get__(original) + cleanup_errors = attributes.get('database_cleanup_errors', ()) + if isinstance(cleanup_errors, tuple): + attach_cleanup_errors(converted, cleanup_errors) + raise converted from original diff --git a/src/datvia/accounts/_mongodb.py b/src/datvia/accounts/_mongodb.py new file mode 100644 index 0000000000000000000000000000000000000000..9f99aabd46515ef8a629beddf17d14d0437aa188 --- /dev/null +++ b/src/datvia/accounts/_mongodb.py @@ -0,0 +1,405 @@ +"""Account and role operations for a self-managed MongoDB deployment.""" + +from __future__ import annotations + +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import field +from types import MappingProxyType +from typing import Any +from typing import Generic +from typing import TypeVar + +from ..errors import ConfigurationError +from ._base import AccountWriteOutcomeUnknown +from ._base import Binding + + +def _name(value: str) -> str: + if not isinstance(value, str) or not value or '\0' in value: + raise ValueError('Expected a nonempty MongoDB name without a null character.') + return value + + +@dataclass(frozen=True) +class MongoUser: + auth_db: str + name: str + + def __post_init__(self) -> None: + _name(self.auth_db) + _name(self.name) + + +@dataclass(frozen=True) +class MongoRole: + db: str + name: str + + def __post_init__(self) -> None: + _name(self.db) + _name(self.name) + + +@dataclass(frozen=True, init=False) +class MongoPrivilege: + resource: Mapping[str, str | bool] + actions: tuple[str, ...] + + def __init__(self, resource: Mapping[str, str | bool], actions: Iterable[str]) -> None: + if not isinstance(resource, Mapping) or not resource: + raise ValueError('Expected a MongoDB resource document.') + snapshot = dict(resource) + if not all(isinstance(k, str) and isinstance(v, (str, bool)) for k, v in snapshot.items()): + raise ValueError('Invalid MongoDB resource field.') + if isinstance(actions, (str, bytes, Mapping)): + raise TypeError('Expected an iterable of action names.') + names = tuple(_name(action) for action in actions) + if not names: + raise ValueError('Expected at least one MongoDB privilege action.') + object.__setattr__(self, 'resource', MappingProxyType(snapshot)) + object.__setattr__(self, 'actions', names) + + +@dataclass(frozen=True) +class MongoUserInfo: + user: MongoUser + roles: tuple[MongoRole, ...] + inherited_roles: tuple[MongoRole, ...] | None + inherited_privileges: tuple[MongoPrivilege, ...] | None + + +@dataclass(frozen=True) +class MongoRoleInfo: + role: MongoRole + builtin: bool + roles: tuple[MongoRole, ...] + privileges: tuple[MongoPrivilege, ...] | None + inherited_roles: tuple[MongoRole, ...] + inherited_privileges: tuple[MongoPrivilege, ...] | None + + +def _roles(items: Iterable[MongoRole]) -> list[dict[str, str]]: + if isinstance(items, (str, bytes, Mapping)): + raise TypeError('Expected a sequence of MongoRole objects.') + result = [] + for role in items: + if not isinstance(role, MongoRole): + raise TypeError('Expected a MongoRole.') + result.append({'role': role.name, 'db': role.db}) + return result + + +def _privileges(items: Iterable[MongoPrivilege]) -> list[dict[str, Any]]: + if isinstance(items, (str, bytes, Mapping)): + raise TypeError('Expected a sequence of MongoPrivilege objects.') + result = [] + for privilege in items: + if not isinstance(privilege, MongoPrivilege): + raise TypeError('Expected a MongoPrivilege.') + result.append({'resource': dict(privilege.resource), 'actions': list(privilege.actions)}) + return result + + +def _password(password: str) -> str: + if not isinstance(password, str) or not password: + raise ValueError('Expected a nonempty password.') + return password + + +def _read_roles(document: dict[str, Any], field: str) -> tuple[MongoRole, ...]: + return tuple(MongoRole(r['db'], r['role']) for r in document[field]) + + +def _read_privileges(document: dict[str, Any], field: str) -> tuple[MongoPrivilege, ...]: + return tuple(MongoPrivilege(p['resource'], p['actions']) for p in document[field]) + + +def _user_info(row: dict[str, Any], expanded: bool = False) -> MongoUserInfo: + return MongoUserInfo( + MongoUser(row['db'], row['user']), + _read_roles(row, 'roles'), + _read_roles(row, 'inheritedRoles') if expanded else None, + _read_privileges(row, 'inheritedPrivileges') if expanded else None, + ) + + +def _role_info(row: dict[str, Any], expanded: bool = False) -> MongoRoleInfo: + return MongoRoleInfo( + MongoRole(row['db'], row['role']), + row['isBuiltin'], + _read_roles(row, 'roles'), + _read_privileges(row, 'privileges') if expanded else None, + _read_roles(row, 'inheritedRoles'), + _read_privileges(row, 'inheritedPrivileges') if expanded else None, + ) + + +_USERS = {'showCredentials': False, 'showCustomData': False} + + +class _MongoAccounts: + def __init__(self, binding: Binding) -> None: + self._binding = binding + + @contextmanager + def _dispatch(self, operation: str, target: str, write: bool) -> Iterator[None]: + from pymongo.errors import ServerSelectionTimeoutError + from pymongo.errors import WaitQueueTimeoutError + from pymongo.errors import WriteConcernError + + with self._binding.operation(operation, target, write=write): + try: + yield + except (ServerSelectionTimeoutError, WaitQueueTimeoutError): + # No connection was selected or checked out for this command. + with self._binding.operation(operation, target): + raise + except WriteConcernError as error: + if not write: + raise + raise AccountWriteOutcomeUnknown( + 'MongoDB did not confirm the requested write concern.', + connection_name=self._binding.connection_name, + backend='mongodb', + phase='operation', + operation=operation, + target=target, + original=error, + ) from error + + def _prepare(self, db: str, command: dict[str, Any], write: bool) -> Any: + database = self._binding.native[_name(db)] + if write: + concern = database.write_concern + if not concern.acknowledged: + raise ConfigurationError('MongoDB account changes require acknowledged writes.', backend='mongodb') + if concern.document: + command['writeConcern'] = concern.document + return database + + @staticmethod + def _confirm(reply: dict[str, Any]) -> None: + from pymongo.errors import WriteConcernError + + if 'writeConcernError' in reply: + detail = reply['writeConcernError'] + raise WriteConcernError('MongoDB write concern failed.', detail.get('code'), detail) + + +_T = TypeVar('_T') + + +@dataclass(frozen=True) +class _Request(Generic[_T]): + operation: str + db: str + spec: dict[str, Any] = field(repr=False) + decode: Callable[[dict[str, Any]], _T] = field(repr=False) + target: str + write: bool = False + + +def _users(db: str, all_databases: bool = False) -> _Request[tuple[MongoUserInfo, ...]]: + spec = {'usersInfo': {'forAllDBs': True} if all_databases else 1, **_USERS} + op = 'list_users_all_databases' if all_databases else 'list_users' + return _Request(op, db, spec, lambda reply: tuple(_user_info(u) for u in reply['users']), db) + + +def _user(user: MongoUser, expanded: bool) -> _Request[MongoUserInfo | None]: + if not isinstance(user, MongoUser): + raise TypeError('Expected a MongoUser.') + spec = {'usersInfo': user.name, 'showPrivileges': expanded, **_USERS} + return _Request( + 'get_user', + user.auth_db, + spec, + lambda reply: _user_info(reply['users'][0], expanded) if reply['users'] else None, + f'{user.auth_db}.{user.name}', + ) + + +def _list_roles(db: str, builtin: bool) -> _Request[tuple[MongoRoleInfo, ...]]: + return _Request( + 'list_roles', + db, + {'rolesInfo': 1, 'showBuiltinRoles': builtin}, + lambda reply: tuple(_role_info(r) for r in reply['roles']), + db, + ) + + +def _role(role: MongoRole, expanded: bool) -> _Request[MongoRoleInfo | None]: + if not isinstance(role, MongoRole): + raise TypeError('Expected a MongoRole.') + return _Request( + 'get_role', + role.db, + {'rolesInfo': role.name, 'showPrivileges': expanded}, + lambda reply: _role_info(reply['roles'][0], expanded) if reply['roles'] else None, + f'{role.db}.{role.name}', + ) + + +def _user_write(operation: str, command: str, user: MongoUser, **fields: Any) -> _Request[None]: + if not isinstance(user, MongoUser): + raise TypeError('Expected a MongoUser.') + return _Request( + operation, + user.auth_db, + {command: user.name, **fields}, + _MongoAccounts._confirm, + f'{user.auth_db}.{user.name}', + write=True, + ) + + +def _role_write(operation: str, command: str, role: MongoRole, **fields: Any) -> _Request[None]: + if not isinstance(role, MongoRole): + raise TypeError('Expected a MongoRole.') + return _Request( + operation, + role.db, + {command: role.name, **fields}, + _MongoAccounts._confirm, + f'{role.db}.{role.name}', + write=True, + ) + + +class MongoDBAccounts(_MongoAccounts): + """Manage MongoDB users and roles on a borrowed synchronous client.""" + + def _run(self, request: _Request[_T]) -> _T: + database = self._prepare(request.db, request.spec, request.write) + with self._dispatch(request.operation, request.target, request.write): + reply = database.command(request.spec) + return request.decode(reply) + + def list_users(self, *, auth_db: str) -> tuple[MongoUserInfo, ...]: + return self._run(_users(auth_db)) + + def list_users_all_databases(self) -> tuple[MongoUserInfo, ...]: + return self._run(_users('admin', all_databases=True)) + + def get_user(self, user: MongoUser, *, expand_privileges: bool = False) -> MongoUserInfo | None: + return self._run(_user(user, expand_privileges)) + + def list_roles(self, *, db: str, include_builtin: bool = False) -> tuple[MongoRoleInfo, ...]: + return self._run(_list_roles(db, include_builtin)) + + def get_role(self, role: MongoRole, *, expand_privileges: bool = False) -> MongoRoleInfo | None: + return self._run(_role(role, expand_privileges)) + + def create_user(self, user: MongoUser, *, password: str, roles: Iterable[MongoRole] = ()) -> None: + return self._run(_user_write('create_user', 'createUser', user, pwd=_password(password), roles=_roles(roles))) + + def drop_user(self, user: MongoUser) -> None: + return self._run(_user_write('drop_user', 'dropUser', user)) + + def change_password(self, user: MongoUser, *, password: str) -> None: + return self._run(_user_write('change_password', 'updateUser', user, pwd=_password(password))) + + def grant_roles(self, user: MongoUser, *, roles: Iterable[MongoRole]) -> None: + return self._run(_user_write('grant_roles', 'grantRolesToUser', user, roles=_roles(roles))) + + def revoke_roles(self, user: MongoUser, *, roles: Iterable[MongoRole]) -> None: + return self._run(_user_write('revoke_roles', 'revokeRolesFromUser', user, roles=_roles(roles))) + + def create_role( + self, role: MongoRole, *, privileges: Iterable[MongoPrivilege] = (), roles: Iterable[MongoRole] = () + ) -> None: + return self._run( + _role_write('create_role', 'createRole', role, privileges=_privileges(privileges), roles=_roles(roles)) + ) + + def drop_role(self, role: MongoRole) -> None: + return self._run(_role_write('drop_role', 'dropRole', role)) + + def grant_privileges(self, role: MongoRole, *, privileges: Iterable[MongoPrivilege]) -> None: + return self._run( + _role_write('grant_privileges', 'grantPrivilegesToRole', role, privileges=_privileges(privileges)) + ) + + def revoke_privileges(self, role: MongoRole, *, privileges: Iterable[MongoPrivilege]) -> None: + return self._run( + _role_write('revoke_privileges', 'revokePrivilegesFromRole', role, privileges=_privileges(privileges)) + ) + + def grant_roles_to_role(self, role: MongoRole, *, roles: Iterable[MongoRole]) -> None: + return self._run(_role_write('grant_roles_to_role', 'grantRolesToRole', role, roles=_roles(roles))) + + def revoke_roles_from_role(self, role: MongoRole, *, roles: Iterable[MongoRole]) -> None: + return self._run(_role_write('revoke_roles_from_role', 'revokeRolesFromRole', role, roles=_roles(roles))) + + +class AsyncMongoDBAccounts(_MongoAccounts): + """Manage MongoDB users and roles on a borrowed asynchronous client.""" + + async def _run(self, request: _Request[_T]) -> _T: + database = self._prepare(request.db, request.spec, request.write) + with self._dispatch(request.operation, request.target, request.write): + reply = await database.command(request.spec) + return request.decode(reply) + + async def list_users(self, *, auth_db: str) -> tuple[MongoUserInfo, ...]: + return await self._run(_users(auth_db)) + + async def list_users_all_databases(self) -> tuple[MongoUserInfo, ...]: + return await self._run(_users('admin', all_databases=True)) + + async def get_user(self, user: MongoUser, *, expand_privileges: bool = False) -> MongoUserInfo | None: + return await self._run(_user(user, expand_privileges)) + + async def list_roles(self, *, db: str, include_builtin: bool = False) -> tuple[MongoRoleInfo, ...]: + return await self._run(_list_roles(db, include_builtin)) + + async def get_role(self, role: MongoRole, *, expand_privileges: bool = False) -> MongoRoleInfo | None: + return await self._run(_role(role, expand_privileges)) + + async def create_user(self, user: MongoUser, *, password: str, roles: Iterable[MongoRole] = ()) -> None: + return await self._run( + _user_write('create_user', 'createUser', user, pwd=_password(password), roles=_roles(roles)) + ) + + async def drop_user(self, user: MongoUser) -> None: + return await self._run(_user_write('drop_user', 'dropUser', user)) + + async def change_password(self, user: MongoUser, *, password: str) -> None: + return await self._run(_user_write('change_password', 'updateUser', user, pwd=_password(password))) + + async def grant_roles(self, user: MongoUser, *, roles: Iterable[MongoRole]) -> None: + return await self._run(_user_write('grant_roles', 'grantRolesToUser', user, roles=_roles(roles))) + + async def revoke_roles(self, user: MongoUser, *, roles: Iterable[MongoRole]) -> None: + return await self._run(_user_write('revoke_roles', 'revokeRolesFromUser', user, roles=_roles(roles))) + + async def create_role( + self, role: MongoRole, *, privileges: Iterable[MongoPrivilege] = (), roles: Iterable[MongoRole] = () + ) -> None: + return await self._run( + _role_write('create_role', 'createRole', role, privileges=_privileges(privileges), roles=_roles(roles)) + ) + + async def drop_role(self, role: MongoRole) -> None: + return await self._run(_role_write('drop_role', 'dropRole', role)) + + async def grant_privileges(self, role: MongoRole, *, privileges: Iterable[MongoPrivilege]) -> None: + return await self._run( + _role_write('grant_privileges', 'grantPrivilegesToRole', role, privileges=_privileges(privileges)) + ) + + async def revoke_privileges(self, role: MongoRole, *, privileges: Iterable[MongoPrivilege]) -> None: + return await self._run( + _role_write('revoke_privileges', 'revokePrivilegesFromRole', role, privileges=_privileges(privileges)) + ) + + async def grant_roles_to_role(self, role: MongoRole, *, roles: Iterable[MongoRole]) -> None: + return await self._run(_role_write('grant_roles_to_role', 'grantRolesToRole', role, roles=_roles(roles))) + + async def revoke_roles_from_role(self, role: MongoRole, *, roles: Iterable[MongoRole]) -> None: + return await self._run(_role_write('revoke_roles_from_role', 'revokeRolesFromRole', role, roles=_roles(roles))) diff --git a/src/datvia/accounts/_mysql.py b/src/datvia/accounts/_mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..fb960d2c1c49f4f2d66c04597457bdd9b3d839c4 --- /dev/null +++ b/src/datvia/accounts/_mysql.py @@ -0,0 +1,414 @@ +"""Account administration through borrowed MySQL Connector/Python connections.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from ..errors import ConfigurationError +from ..errors import DependencyError +from ..errors import attach_cleanup_errors +from ._base import Binding +from ._mysql_models import AccountInfo +from ._mysql_models import Grants +from ._mysql_models import MySQLAccount +from ._mysql_models import MySQLScope +from ._mysql_models import PrivilegeChange +from ._mysql_models import RoleBinding +from ._mysql_models import _tuple + +_ACCOUNTS = 'SELECT User, Host, account_locked FROM mysql.user' +_BINDINGS = ( + 'SELECT FROM_USER, FROM_HOST, TO_USER, TO_HOST, WITH_ADMIN_OPTION FROM mysql.role_edges ' + 'WHERE TO_USER = %s AND TO_HOST = %s ORDER BY FROM_USER, FROM_HOST' +) +_DEFAULTS = ( + 'SELECT DEFAULT_ROLE_USER, DEFAULT_ROLE_HOST FROM mysql.default_roles ' + 'WHERE USER = %s AND HOST = %s ORDER BY DEFAULT_ROLE_USER, DEFAULT_ROLE_HOST' +) +_ESCAPE_PROBE = "account'probe\\end" + + +def _account(value: MySQLAccount) -> MySQLAccount: + if not isinstance(value, MySQLAccount): + raise ConfigurationError('Expected MySQLAccount with explicit user and host.', phase='config') + return value + + +def _roles(values: Iterable[MySQLAccount], *, empty: bool = False) -> tuple[MySQLAccount, ...]: + result = tuple(_account(value) for value in _tuple(values, 'roles')) + if not result and not empty: + raise ConfigurationError('At least one role is required.', phase='config') + return result + + +def _role_params(roles: tuple[MySQLAccount, ...]) -> tuple[str, ...]: + return tuple(value for role in roles for value in role.params()) + + +def _markers(roles: tuple[MySQLAccount, ...]) -> str: + return ', '.join('%s@%s' for _ in roles) + + +def _password(value: str) -> str: + if not isinstance(value, str): + raise ConfigurationError('Expected password text.', phase='config') + return value + + +def _str(value: Any) -> str: + if isinstance(value, (bytes, bytearray, memoryview)): + return bytes(value).decode('utf-8') + return str(value) + + +def _decode_rows(rows: list[tuple], encoding: str) -> list[tuple]: + # Raw cursors and use_unicode=False expose result bytes in the negotiated charset. + return [ + tuple( + bytes(value).decode(encoding) if isinstance(value, (bytes, memoryview, bytearray)) else value + for value in row + ) + for row in rows + ] + + +def _info(row: tuple) -> AccountInfo: + user, host, locked = map(_str, row) + return AccountInfo(MySQLAccount(user, host), locked == 'Y') + + +def _needs_probe(params: tuple[str, ...]) -> bool: + return any(any(char in s for char in '\'\\\x00\r\n\x1a"') for s in params) + + +def _transaction_error() -> ConfigurationError: + return ConfigurationError( + 'Account changes require autocommit and no active transaction.', backend='mysql', phase='operation' + ) + + +def _escape_error() -> DependencyError: + return DependencyError( + 'The connector cannot safely bind values in this session SQL mode.', backend='mysql', phase='operation' + ) + + +def _check_registered(change: PrivilegeChange, rows: list[tuple]) -> None: + names = {_str(row[0]).upper() for row in rows} + if not change.dynamic <= names: + raise ConfigurationError('Unregistered MySQL global privilege.', backend='mysql', phase='config') + + +def _target(account: MySQLAccount | None) -> str | None: + return f'{account.user}@{account.host}' if account is not None else None + + +class MySQLAccounts: + """Manage users and roles on a borrowed synchronous MySQL connection.""" + + def __init__(self, binding: Binding) -> None: + self._binding = binding + + def _execute( + self, + operation: str, + account: MySQLAccount | None, + sql: str, + params: tuple[str, ...] = (), + *, + write: bool = False, + change: PrivilegeChange | None = None, + ) -> list[tuple]: + target = _target(account) + with self._binding.operation(operation, target): + native = self._binding.native + if write and (native.in_transaction or not native.autocommit or native.in_transaction): + raise _transaction_error() + cursor = native.cursor() + primary = None + try: + if _needs_probe(params): + try: + cursor.execute('SELECT %s', (_ESCAPE_PROBE,)) + except Exception as error: + if getattr(error, 'errno', None) == 1064: + raise _escape_error() from error + raise + probe = cursor.fetchone() + if probe is None or _str(probe[0]) != _ESCAPE_PROBE: + raise _escape_error() + if change is not None: + literal_wildcards = False + if change.scope.kind == 'database' and any(c in change.scope.db for c in '_%\\\\'): + try: + cursor.execute('SELECT @@GLOBAL.partial_revokes') + except Exception as error: + if getattr(error, 'errno', None) != 1193: + raise + else: + row = cursor.fetchone() + literal_wildcards = bool(int(row[0])) + if change.dynamic: + cursor.execute('SHOW PRIVILEGES') + _check_registered(change, cursor.fetchall()) + sql, params = change.statement(literal_wildcards, params) + with self._binding.operation(operation, target, write=write): + cursor.execute(sql, params or None) + return [] if write else _decode_rows(cursor.fetchall(), native.python_charset) + except BaseException as error: + primary = error + raise + finally: + try: + cursor.close() + except BaseException as cleanup: + if primary is None: + raise + attach_cleanup_errors(primary, (cleanup,)) + + def list_accounts(self) -> tuple[AccountInfo, ...]: + rows = self._execute('list_accounts', None, _ACCOUNTS + ' ORDER BY User, Host') + return tuple(_info(row) for row in rows) + + def get_account(self, account: MySQLAccount) -> AccountInfo | None: + rows = self._execute( + 'get_account', account, _ACCOUNTS + ' WHERE User = %s AND Host = %s', _account(account).params() + ) + return _info(rows[0]) if rows else None + + def show_grants(self, account: MySQLAccount, *, using_roles: Iterable[MySQLAccount] = ()) -> Grants: + roles = _roles(using_roles, empty=True) + sql = 'SHOW GRANTS FOR %s@%s' + (' USING ' + _markers(roles) if roles else '') + rows = self._execute('show_grants', account, sql, _account(account).params() + _role_params(roles)) + return Grants(account, roles, tuple(_str(row[0]) for row in rows)) + + def list_role_bindings(self, account: MySQLAccount) -> tuple[RoleBinding, ...]: + rows = self._execute('list_role_bindings', account, _BINDINGS, _account(account).params()) + return tuple( + RoleBinding(MySQLAccount(_str(u), _str(h)), MySQLAccount(_str(m), _str(s)), _str(flag) == 'Y') + for u, h, m, s, flag in rows + ) + + def get_default_roles(self, account: MySQLAccount) -> tuple[MySQLAccount, ...]: + rows = self._execute('get_default_roles', account, _DEFAULTS, _account(account).params()) + return tuple(MySQLAccount(_str(u), _str(h)) for u, h in rows) + + def create_user(self, account: MySQLAccount, *, password: str) -> None: + self._execute( + 'create_user', + account, + 'CREATE USER %s@%s IDENTIFIED BY %s', + _account(account).params() + (_password(password),), + write=True, + ) + + def drop_user(self, account: MySQLAccount) -> None: + self._execute('drop_user', account, 'DROP USER %s@%s', _account(account).params(), write=True) + + def change_password(self, account: MySQLAccount, *, password: str) -> None: + self._execute( + 'change_password', + account, + 'ALTER USER %s@%s IDENTIFIED BY %s', + _account(account).params() + (_password(password),), + write=True, + ) + + def lock_user(self, account: MySQLAccount) -> None: + self._execute('lock_user', account, 'ALTER USER %s@%s ACCOUNT LOCK', _account(account).params(), write=True) + + def unlock_user(self, account: MySQLAccount) -> None: + self._execute('unlock_user', account, 'ALTER USER %s@%s ACCOUNT UNLOCK', _account(account).params(), write=True) + + def create_role(self, role: MySQLAccount) -> None: + self._execute('create_role', role, 'CREATE ROLE %s@%s', _account(role).params(), write=True) + + def drop_role(self, role: MySQLAccount) -> None: + self._execute('drop_role', role, 'DROP ROLE %s@%s', _account(role).params(), write=True) + + def grant_privileges( + self, account: MySQLAccount, scope: MySQLScope, privileges: Iterable[str], *, grant_option: bool = False + ) -> None: + change = PrivilegeChange.build(scope, privileges, True, grant_option) + self._execute('grant_privileges', account, '', _account(account).params(), write=True, change=change) + + def revoke_privileges(self, account: MySQLAccount, scope: MySQLScope, privileges: Iterable[str]) -> None: + change = PrivilegeChange.build(scope, privileges, False, False) + self._execute('revoke_privileges', account, '', _account(account).params(), write=True, change=change) + + def grant_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount], admin_option: bool = False) -> None: + if type(admin_option) is not bool: + raise ConfigurationError('Expected a Boolean admin option.', phase='config') + items = _roles(roles) + sql = 'GRANT ' + _markers(items) + ' TO %s@%s' + (' WITH ADMIN OPTION' if admin_option else '') + self._execute('grant_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + def revoke_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount]) -> None: + items = _roles(roles) + sql = 'REVOKE ' + _markers(items) + ' FROM %s@%s' + self._execute('revoke_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + def set_default_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount]) -> None: + items = _roles(roles, empty=True) + sql = 'SET DEFAULT ROLE ' + (_markers(items) if items else 'NONE') + ' TO %s@%s' + self._execute('set_default_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + +class AsyncMySQLAccounts: + """Manage users and roles on a borrowed asynchronous MySQL connection.""" + + def __init__(self, binding: Binding) -> None: + self._binding = binding + + async def _execute( + self, + operation: str, + account: MySQLAccount | None, + sql: str, + params: tuple[str, ...] = (), + *, + write: bool = False, + change: PrivilegeChange | None = None, + ) -> list[tuple]: + target = _target(account) + with self._binding.operation(operation, target): + native = self._binding.native + if write and (native.in_transaction or not (await native.get_autocommit()) or native.in_transaction): + raise _transaction_error() + cursor = await native.cursor() + primary = None + try: + if _needs_probe(params): + try: + await cursor.execute('SELECT %s', (_ESCAPE_PROBE,)) + except Exception as error: + if getattr(error, 'errno', None) == 1064: + raise _escape_error() from error + raise + probe = await cursor.fetchone() + if probe is None or _str(probe[0]) != _ESCAPE_PROBE: + raise _escape_error() + if change is not None: + literal_wildcards = False + if change.scope.kind == 'database' and any(c in change.scope.db for c in '_%\\\\'): + try: + await cursor.execute('SELECT @@GLOBAL.partial_revokes') + except Exception as error: + if getattr(error, 'errno', None) != 1193: + raise + else: + row = await cursor.fetchone() + literal_wildcards = bool(int(row[0])) + if change.dynamic: + await cursor.execute('SHOW PRIVILEGES') + _check_registered(change, await cursor.fetchall()) + sql, params = change.statement(literal_wildcards, params) + with self._binding.operation(operation, target, write=write): + await cursor.execute(sql, params or None) + return [] if write else _decode_rows(await cursor.fetchall(), native.python_charset) + except BaseException as error: + primary = error + raise + finally: + try: + await cursor.close() + except BaseException as cleanup: + if primary is None: + raise + attach_cleanup_errors(primary, (cleanup,)) + + async def list_accounts(self) -> tuple[AccountInfo, ...]: + rows = await self._execute('list_accounts', None, _ACCOUNTS + ' ORDER BY User, Host') + return tuple(_info(row) for row in rows) + + async def get_account(self, account: MySQLAccount) -> AccountInfo | None: + rows = await self._execute( + 'get_account', account, _ACCOUNTS + ' WHERE User = %s AND Host = %s', _account(account).params() + ) + return _info(rows[0]) if rows else None + + async def show_grants(self, account: MySQLAccount, *, using_roles: Iterable[MySQLAccount] = ()) -> Grants: + roles = _roles(using_roles, empty=True) + sql = 'SHOW GRANTS FOR %s@%s' + (' USING ' + _markers(roles) if roles else '') + rows = await self._execute('show_grants', account, sql, _account(account).params() + _role_params(roles)) + return Grants(account, roles, tuple(_str(row[0]) for row in rows)) + + async def list_role_bindings(self, account: MySQLAccount) -> tuple[RoleBinding, ...]: + rows = await self._execute('list_role_bindings', account, _BINDINGS, _account(account).params()) + return tuple( + RoleBinding(MySQLAccount(_str(u), _str(h)), MySQLAccount(_str(m), _str(s)), _str(flag) == 'Y') + for u, h, m, s, flag in rows + ) + + async def get_default_roles(self, account: MySQLAccount) -> tuple[MySQLAccount, ...]: + rows = await self._execute('get_default_roles', account, _DEFAULTS, _account(account).params()) + return tuple(MySQLAccount(_str(u), _str(h)) for u, h in rows) + + async def create_user(self, account: MySQLAccount, *, password: str) -> None: + await self._execute( + 'create_user', + account, + 'CREATE USER %s@%s IDENTIFIED BY %s', + _account(account).params() + (_password(password),), + write=True, + ) + + async def drop_user(self, account: MySQLAccount) -> None: + await self._execute('drop_user', account, 'DROP USER %s@%s', _account(account).params(), write=True) + + async def change_password(self, account: MySQLAccount, *, password: str) -> None: + await self._execute( + 'change_password', + account, + 'ALTER USER %s@%s IDENTIFIED BY %s', + _account(account).params() + (_password(password),), + write=True, + ) + + async def lock_user(self, account: MySQLAccount) -> None: + await self._execute( + 'lock_user', account, 'ALTER USER %s@%s ACCOUNT LOCK', _account(account).params(), write=True + ) + + async def unlock_user(self, account: MySQLAccount) -> None: + await self._execute( + 'unlock_user', account, 'ALTER USER %s@%s ACCOUNT UNLOCK', _account(account).params(), write=True + ) + + async def create_role(self, role: MySQLAccount) -> None: + await self._execute('create_role', role, 'CREATE ROLE %s@%s', _account(role).params(), write=True) + + async def drop_role(self, role: MySQLAccount) -> None: + await self._execute('drop_role', role, 'DROP ROLE %s@%s', _account(role).params(), write=True) + + async def grant_privileges( + self, account: MySQLAccount, scope: MySQLScope, privileges: Iterable[str], *, grant_option: bool = False + ) -> None: + change = PrivilegeChange.build(scope, privileges, True, grant_option) + await self._execute('grant_privileges', account, '', _account(account).params(), write=True, change=change) + + async def revoke_privileges(self, account: MySQLAccount, scope: MySQLScope, privileges: Iterable[str]) -> None: + change = PrivilegeChange.build(scope, privileges, False, False) + await self._execute('revoke_privileges', account, '', _account(account).params(), write=True, change=change) + + async def grant_roles( + self, account: MySQLAccount, *, roles: Iterable[MySQLAccount], admin_option: bool = False + ) -> None: + if type(admin_option) is not bool: + raise ConfigurationError('Expected a Boolean admin option.', phase='config') + items = _roles(roles) + sql = 'GRANT ' + _markers(items) + ' TO %s@%s' + (' WITH ADMIN OPTION' if admin_option else '') + await self._execute('grant_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + async def revoke_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount]) -> None: + items = _roles(roles) + sql = 'REVOKE ' + _markers(items) + ' FROM %s@%s' + await self._execute('revoke_roles', account, sql, _role_params(items) + _account(account).params(), write=True) + + async def set_default_roles(self, account: MySQLAccount, *, roles: Iterable[MySQLAccount]) -> None: + items = _roles(roles, empty=True) + sql = 'SET DEFAULT ROLE ' + (_markers(items) if items else 'NONE') + ' TO %s@%s' + await self._execute( + 'set_default_roles', account, sql, _role_params(items) + _account(account).params(), write=True + ) diff --git a/src/datvia/accounts/_mysql_models.py b/src/datvia/accounts/_mysql_models.py new file mode 100644 index 0000000000000000000000000000000000000000..83c19fe4219cb115dcd9733bb1d628a3c3bb7757 --- /dev/null +++ b/src/datvia/accounts/_mysql_models.py @@ -0,0 +1,197 @@ +"""MySQL account identities, privilege scopes and server-independent SQL composition.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass + +from ..errors import ConfigurationError + + +def _text(value: str, name: str, *, empty: bool = False) -> str: + if not isinstance(value, str) or (not empty and not value) or '\x00' in value: + raise ConfigurationError(f'Invalid {name}.', backend='mysql', phase='config') + return value + + +@dataclass(frozen=True) +class MySQLAccount: + user: str + host: str + + def __post_init__(self) -> None: + _text(self.user, 'user name', empty=True) + _text(self.host, 'account host', empty=True) + + def params(self) -> tuple[str, str]: + return self.user, self.host + + +def _tuple(items: Iterable, name: str) -> tuple: + if isinstance(items, (bytes, str)) or not isinstance(items, Iterable): + raise ConfigurationError(f'Expected an iterable of {name}.', phase='config') + return tuple(items) + + +@dataclass(frozen=True) +class MySQLScope: + kind: str + db: str = '' + name: str = '' + column_names: tuple[str, ...] = () + + def __post_init__(self) -> None: + columns = _tuple(self.column_names, 'column names') + object.__setattr__(self, 'column_names', columns) + if self.kind not in {'global', 'database', 'table', 'columns', 'procedure', 'function'}: + raise ConfigurationError('Unknown MySQL privilege scope.', phase='config') + if self.kind != 'global': + _text(self.db, 'database name') + if self.kind not in ('global', 'database'): + _text(self.name, 'object name') + if ( + (self.kind == 'global' and self.db) + or (self.kind in ('global', 'database') and self.name) + or bool(columns) != (self.kind == 'columns') + ): + raise ConfigurationError('Invalid fields for this MySQL privilege scope.', phase='config') + for column in columns: + _text(column, 'column name') + + @classmethod + def global_(cls) -> MySQLScope: + return cls('global') + + @classmethod + def database(cls, name: str) -> MySQLScope: + return cls('database', name) + + @classmethod + def table(cls, db: str, name: str) -> MySQLScope: + return cls('table', db, name) + + @classmethod + def columns(cls, db: str, table: str, names: Iterable[str]) -> MySQLScope: + return cls('columns', db, table, _tuple(names, 'column names')) + + @classmethod + def procedure(cls, db: str, name: str) -> MySQLScope: + return cls('procedure', db, name) + + @classmethod + def function(cls, db: str, name: str) -> MySQLScope: + return cls('function', db, name) + + +@dataclass(frozen=True) +class AccountInfo: + account: MySQLAccount + locked: bool + + +@dataclass(frozen=True) +class Grants: + account: MySQLAccount + using_roles: tuple[MySQLAccount, ...] + statements: tuple[str, ...] + + +@dataclass(frozen=True) +class RoleBinding: + role: MySQLAccount + member: MySQLAccount + admin_option: bool + + +_COLUMN = {'SELECT', 'INSERT', 'UPDATE', 'REFERENCES'} +_TABLE = _COLUMN | {'ALTER', 'CREATE', 'CREATE VIEW', 'DELETE', 'DROP', 'INDEX', 'SHOW VIEW', 'TRIGGER'} +_ROUTINE = {'ALTER ROUTINE', 'EXECUTE'} +_DATABASE = _TABLE | _ROUTINE | {'CREATE ROUTINE', 'CREATE TEMPORARY TABLES', 'EVENT', 'LOCK TABLES'} +_GLOBAL = _DATABASE | { + 'CREATE ROLE', + 'CREATE TABLESPACE', + 'CREATE USER', + 'DROP ROLE', + 'FILE', + 'PROCESS', + 'RELOAD', + 'REPLICATION CLIENT', + 'REPLICATION SLAVE', + 'SHOW DATABASES', + 'SHUTDOWN', + 'SUPER', + 'USAGE', +} +_SCOPES = { + 'global': _GLOBAL, + 'database': _DATABASE, + 'table': _TABLE, + 'columns': _COLUMN, + 'procedure': _ROUTINE, + 'function': _ROUTINE, +} + + +def identifiers(name: str) -> str: + return '`' + name.replace('`', '``') + '`' + + +def scope_sql(scope: MySQLScope, literal_wildcards: bool) -> str: + if scope.kind == 'global': + return '*.*' + db = scope.db + if scope.kind == 'database': + if not literal_wildcards: + db = db.replace('\\', '\\\\').replace('_', '\\_').replace('%', '\\%') + return identifiers(db) + '.*' + prefix = scope.kind.upper() + ' ' if scope.kind in ('procedure', 'function') else '' + return prefix + identifiers(db) + '.' + identifiers(scope.name) + + +@dataclass(frozen=True) +class PrivilegeChange: + scope: MySQLScope + privileges: tuple[str, ...] + grant: bool + grant_option: bool = False + + @classmethod + def build(cls, scope: MySQLScope, privileges: Iterable[str], grant: bool, grant_option: bool) -> PrivilegeChange: + if not isinstance(scope, MySQLScope) or type(grant_option) is not bool: + raise ConfigurationError('Expected a typed scope and Boolean grant option.', phase='config') + values = tuple(_text(p, 'privilege name').upper() for p in _tuple(privileges, 'privileges')) + if not values: + raise ConfigurationError('At least one privilege is required.', phase='config') + for name in values: + if name in _SCOPES[scope.kind]: + continue + if scope.kind != 'global' or not re.fullmatch('[A-Z][A-Z0-9_]*', name) or name in {'ALL', 'PROXY', 'GRANT'}: + raise ConfigurationError('Invalid privilege for the selected scope.', phase='config') + return cls(scope, values, grant, grant_option) + + @property + def dynamic(self) -> set[str]: + return set(self.privileges) - _GLOBAL + + def statement( + self, literal_wildcards: bool, params: tuple[str, ...] + ) -> tuple[str, tuple[str, ...] | dict[str, str]]: + names = self.privileges + if self.scope.kind == 'columns': + cols = ', '.join(identifiers(c) for c in self.scope.column_names) + names = tuple(f'{name} ({cols})' for name in names) + target = scope_sql(self.scope, literal_wildcards) + verb, preposition = ('GRANT', 'TO') if self.grant else ('REVOKE', 'FROM') + suffix = ' WITH GRANT OPTION' if self.grant_option else '' + prefix = f'{verb} {", ".join(names)} ON {target} {preposition} ' + if '%s' not in prefix: + return prefix + '%s@%s' + suffix, params + # Connector/Python scans quoted identifiers for parameter markers too. + # Switch placeholder styles when the object name contains the positional marker. + if '%(' in prefix: + raise ConfigurationError('Identifier combines MySQL placeholder syntaxes.', phase='config') + user_key, host_key = 'account_user', 'account_host' + while any(f'%({host_key})s' in value for value in params): + host_key += '_' + return prefix + f'%({user_key})s@%({host_key})s' + suffix, dict(zip((user_key, host_key), params)) diff --git a/src/datvia/accounts/_redis.py b/src/datvia/accounts/_redis.py new file mode 100644 index 0000000000000000000000000000000000000000..4e03910f263ed4ee69ee2e5fc1c7450a17971d6a --- /dev/null +++ b/src/datvia/accounts/_redis.py @@ -0,0 +1,434 @@ +"""Ordered ACL changes and read-only permission views for a single Redis data node.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping +from collections.abc import Sequence +from dataclasses import dataclass +from dataclasses import field +from typing import Any +from typing import Literal + +from ..errors import ConfigurationError +from ..errors import DatabaseError +from ..errors import DependencyError +from ..errors import DriverError +from ._base import AccountWriteOutcomeUnknown +from ._base import Binding + + +def _argument(value: str, what: str) -> str: + if not isinstance(value, str) or not value or any(c.isspace() or c == '\x00' for c in value): + raise ConfigurationError(f'Expected a nonempty Redis {what} without whitespace or NUL.') + return value + + +def _command(value: str) -> str: + if not re.fullmatch(r'[a-zA-Z0-9_.:-]+(?:\|[a-zA-Z0-9_.:-]+)?', _argument(value, 'command')): + raise ConfigurationError('Invalid Redis command name.') + return value + + +def _category(value: str) -> str: + if not re.fullmatch(r'[a-zA-Z0-9_-]+', _argument(value, 'category')): + raise ConfigurationError('Expected a Redis category name without its ACL prefix.') + return value + + +def _password(value: str | bytes, prefix: str) -> str | bytes: + if not isinstance(value, (bytes, str)): + raise ConfigurationError('Expected a Redis password string or bytes.') + return prefix.encode('ascii') + value if isinstance(value, bytes) else prefix + value + + +@dataclass(frozen=True, init=False, slots=True) +class AclRule: + """One ordered ACL change. Use the named constructors to build rules.""" + + _kind: str + _tokens: tuple[str | bytes, ...] = field(repr=False) + _version: tuple[int, int] = field(repr=False) + + def __init__(self) -> None: + raise TypeError('Use an AclRule constructor.') + + @classmethod + def _new(cls, kind: str, *tokens: str | bytes, version: tuple[int, int] = (6, 0)) -> AclRule: + rule = object.__new__(cls) + object.__setattr__(rule, '_kind', kind) + object.__setattr__(rule, '_tokens', tokens) + object.__setattr__(rule, '_version', version) + return rule + + @classmethod + def enable(cls) -> AclRule: + return cls._new('enable', 'on') + + @classmethod + def disable(cls) -> AclRule: + return cls._new('disable', 'off') + + @classmethod + def allow_command(cls, command: str) -> AclRule: + return cls._new('command', '+' + _command(command)) + + @classmethod + def deny_command(cls, command: str) -> AclRule: + return cls._new('command', '-' + _command(command)) + + @classmethod + def allow_category(cls, category: str) -> AclRule: + return cls._new('category', '+@' + _category(category)) + + @classmethod + def deny_category(cls, category: str) -> AclRule: + return cls._new('category', '-@' + _category(category)) + + @classmethod + def key_pattern(cls, pattern: str, *, access: Literal['read', 'write', 'both'] = 'both') -> AclRule: + prefix = {'read': '%R~', 'write': '%W~', 'both': '~'} + if access not in prefix: + raise ConfigurationError('Expected read, write or both for ACL key access.') + token = prefix[access] + _argument(pattern, 'key pattern') + return cls._new('key', token, version=(6, 0) if access == 'both' else (7, 0)) + + @classmethod + def clear_keys(cls) -> AclRule: + return cls._new('key', 'resetkeys') + + @classmethod + def channel_pattern(cls, pattern: str) -> AclRule: + return cls._new('channel', '&' + _argument(pattern, 'channel pattern'), version=(6, 2)) + + @classmethod + def clear_channels(cls) -> AclRule: + return cls._new('channel', 'resetchannels', version=(6, 2)) + + @classmethod + def selector(cls, *rules: AclRule) -> AclRule: + """Group command, key and channel rules under Redis 7 selector semantics.""" + tokens = [] + for rule in rules: + if not isinstance(rule, AclRule) or rule._kind not in ('command', 'category', 'key', 'channel'): + raise ConfigurationError('Selectors accept only access rules.') + token = rule._tokens[0] + # Redis reparses a selector as a quoted/escaped argument string. + # Preserve patterns literally and make each rule a single argument. + escaped = token.replace('\\', '\\\\').replace('"', '\\"') + tokens.append('"' + escaped + '"') + return cls._new('selector', '(' + ' '.join(tokens) + ')', version=(7, 0)) + + @classmethod + def clear_selectors(cls) -> AclRule: + return cls._new('clear_selectors', 'clearselectors', version=(7, 0)) + + @classmethod + def add_password(cls, password: str | bytes) -> AclRule: + return cls._new('add_password', _password(password, '>')) + + @classmethod + def remove_password(cls, password: str | bytes) -> AclRule: + return cls._new('remove_password', _password(password, '<')) + + @classmethod + def replace_passwords(cls, passwords: Sequence[str | bytes]) -> AclRule: + if not isinstance(passwords, Sequence) or isinstance(passwords, (str, bytes)): + raise ConfigurationError('Expected an ordered sequence of Redis passwords.') + return cls._new('replace_passwords', 'resetpass', *(_password(p, '>') for p in passwords)) + + +@dataclass(frozen=True) +class RedisSelector: + commands: tuple[str, ...] + categories: tuple[str, ...] + keys: tuple[str, ...] + channels: tuple[str, ...] + + +@dataclass(frozen=True) +class RedisUser: + name: str + flags: tuple[str, ...] + commands: tuple[str, ...] + categories: tuple[str, ...] + keys: tuple[str, ...] + channels: tuple[str, ...] | None + selectors: tuple[RedisSelector, ...] | None + + @property + def enabled(self) -> bool: + return 'on' in self.flags + + +@dataclass(frozen=True) +class RedisDryRun: + allowed: bool + reason: str | None + + +def _text(value: str | bytes, encoding: str) -> str: + return value.decode(encoding, errors='surrogateescape') if isinstance(value, bytes) else value + + +def _fields(raw: Any, encoding: str) -> dict[str, Any]: + pairs = raw.items() if isinstance(raw, Mapping) else zip(raw[::2], raw[1::2], strict=True) + return {_text(key, encoding): value for key, value in pairs} + + +def _words(raw: Any, encoding: str) -> tuple[str, ...]: + if isinstance(raw, (bytes, str)): + return tuple(_text(raw, encoding).split()) + return tuple(_text(v, encoding) for v in raw if v) + + +def _access(data: Mapping[str, Any], encoding: str) -> RedisSelector: + words = _words(data['commands'], encoding) + return RedisSelector( + tuple(x for x in words if x[1:2] != '@'), + tuple(x for x in words if x[1:2] == '@') + _words(data.get('categories', ()), encoding), + _words(data['keys'], encoding), + _words(data.get('channels', ()), encoding), + ) + + +def _user(name: str, raw: Any, encoding: str) -> RedisUser | None: + if raw is None: + return None + data = _fields(raw, encoding) + base = _access(data, encoding) + selectors = data.get('selectors') + return RedisUser( + name, + _words(data['flags'], encoding), + base.commands, + base.categories, + base.keys, + base.channels if 'channels' in data else None, + tuple(_access(_fields(x, encoding), encoding) for x in selectors) if selectors is not None else None, + ) + + +def _rules(rules: Sequence[AclRule]) -> tuple[tuple[str | bytes, ...], tuple[int, int]]: + if ( + not isinstance(rules, Sequence) + or isinstance(rules, (bytes, str)) + or not all(isinstance(r, AclRule) for r in rules) + ): + raise ConfigurationError('Expected an ordered sequence of AclRule instances.') + return tuple(t for r in rules for t in r._tokens), max((r._version for r in rules), default=(6, 0)) + + +def _dry_result(raw: Any) -> RedisDryRun: + if raw in ('OK', b'OK', True): + return RedisDryRun(True, None) + if not isinstance(raw, (str, bytes)): + raise DriverError('Unexpected ACL DRYRUN response.') + # The server can interpolate a key/channel (including credentials) in the reply. + return RedisDryRun(False, 'The target user lacks permission for this command or resource.') + + +def _capability( + binding: Binding, + operation: str, + target: str | None, + error: DriverError, + required: tuple[int, int], + version: str | None, +) -> None: + original = error.original + if version is not None: + match = re.match(r'(\d+)\.(\d+)', version) + if match and tuple(map(int, match.groups())) < required: + raise DependencyError( + f'This ACL operation requires Redis {required[0]}.{required[1]} or later.', + backend='redis', + connection_name=binding.connection_name, + phase='operation', + operation=operation, + target=target, + original=original, + ) from original + + +class _Accounts: + def __init__(self, binding: Binding) -> None: + self._binding = binding + + @property + def _encoding(self) -> str: + return self._binding.native.connection_pool.get_encoder().encoding + + +class RedisAccounts(_Accounts): + """Manage ACLs of the connected data node with a borrowed synchronous client.""" + + def _run( + self, + operation: str, + target: str | None, + subcommand: str, + *args: Any, + write: bool = False, + required: tuple[int, int] = (6, 0), + ) -> Any: + try: + with self._binding.operation(operation, target, write=write): + # Two command tokens request the native RESP result. A single + # 'ACL GETUSER' would apply redis-py's version-dependent parser. + return self._binding.native.execute_command('ACL', subcommand, *args) + except DriverError as error: + # Ask for the version only after a server refusal. A limited ACL + # administrator may legitimately lack INFO permission. + from redis.exceptions import ResponseError + + if isinstance(error.original, ResponseError): + try: + with self._binding.operation('acl_version', target): + info = self._binding.native.info('server') + except DatabaseError: + info = {} + _capability(self._binding, operation, target, error, required, info.get('redis_version')) + raise + + def list_users(self) -> tuple[str, ...]: + raw = self._run('list_users', None, 'USERS') + return tuple(_text(name, self._encoding) for name in raw) + + def get_user(self, name: str) -> RedisUser | None: + name = _argument(name, 'username') + raw = self._run('get_user', name, 'GETUSER', name) + return _user(name, raw, self._encoding) + + def apply_user(self, name: str, *, rules: Sequence[AclRule]) -> None: + """Create or update a user by applying these rules in their given order.""" + name = _argument(name, 'username') + tokens, required = _rules(rules) + raw = self._run('apply_user', name, 'SETUSER', name, *tokens, write=True, required=required) + if raw not in ('OK', b'OK', True): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL SETUSER acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='apply_user', + target=name, + ) + + def delete_user(self, name: str) -> bool: + name = _argument(name, 'username') + raw = self._run('delete_user', name, 'DELUSER', name, write=True) + if raw not in (0, 1): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL DELUSER acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='delete_user', + target=name, + ) + return bool(raw) + + def dry_run(self, name: str, command: str, *args: str | bytes | int | float) -> RedisDryRun: + name = _argument(name, 'username') + raw = self._run('dry_run', name, 'DRYRUN', name, _command(command), *args, required=(7, 0)) + return _dry_result(raw) + + def save_acl(self) -> None: + """Persist the entire node's ACL to its configured external ACL file.""" + raw = self._run('save_acl', None, 'SAVE', write=True) + if raw not in ('OK', b'OK', True): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL SAVE acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='save_acl', + ) + + +class AsyncRedisAccounts(_Accounts): + """Manage ACLs of the connected data node with a borrowed asynchronous client.""" + + async def _run( + self, + operation: str, + target: str | None, + subcommand: str, + *args: Any, + write: bool = False, + required: tuple[int, int] = (6, 0), + ) -> Any: + try: + with self._binding.operation(operation, target, write=write): + # Two command tokens request the native RESP result. A single + # 'ACL GETUSER' would apply redis-py's version-dependent parser. + return await self._binding.native.execute_command('ACL', subcommand, *args) + except DriverError as error: + # Ask for the version only after a server refusal. A limited ACL + # administrator may legitimately lack INFO permission. + from redis.exceptions import ResponseError + + if isinstance(error.original, ResponseError): + try: + with self._binding.operation('acl_version', target): + info = await self._binding.native.info('server') + except DatabaseError: + info = {} + _capability(self._binding, operation, target, error, required, info.get('redis_version')) + raise + + async def list_users(self) -> tuple[str, ...]: + raw = await self._run('list_users', None, 'USERS') + return tuple(_text(name, self._encoding) for name in raw) + + async def get_user(self, name: str) -> RedisUser | None: + name = _argument(name, 'username') + raw = await self._run('get_user', name, 'GETUSER', name) + return _user(name, raw, self._encoding) + + async def apply_user(self, name: str, *, rules: Sequence[AclRule]) -> None: + """Create or update a user by applying these rules in their given order.""" + name = _argument(name, 'username') + tokens, required = _rules(rules) + raw = await self._run('apply_user', name, 'SETUSER', name, *tokens, write=True, required=required) + if raw not in ('OK', b'OK', True): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL SETUSER acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='apply_user', + target=name, + ) + + async def delete_user(self, name: str) -> bool: + name = _argument(name, 'username') + raw = await self._run('delete_user', name, 'DELUSER', name, write=True) + if raw not in (0, 1): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL DELUSER acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='delete_user', + target=name, + ) + return bool(raw) + + async def dry_run(self, name: str, command: str, *args: str | bytes | int | float) -> RedisDryRun: + name = _argument(name, 'username') + raw = await self._run('dry_run', name, 'DRYRUN', name, _command(command), *args, required=(7, 0)) + return _dry_result(raw) + + async def save_acl(self) -> None: + """Persist the entire node's ACL to its configured external ACL file.""" + raw = await self._run('save_acl', None, 'SAVE', write=True) + if raw not in ('OK', b'OK', True): + raise AccountWriteOutcomeUnknown( + 'Unexpected ACL SAVE acknowledgement.', + backend='redis', + connection_name=self._binding.connection_name, + phase='operation', + operation='save_acl', + ) diff --git a/src/datvia/config.py b/src/datvia/config.py new file mode 100644 index 0000000000000000000000000000000000000000..52dd0ffc39e2ba0e94b7b5f730bc3f3c9390105a --- /dev/null +++ b/src/datvia/config.py @@ -0,0 +1,245 @@ +"""Explicit configuration for managed database connections.""" + +import os +from collections.abc import Mapping +from dataclasses import dataclass, field, replace +from enum import Enum +from typing import Any + +from .errors import ConfigurationError + + +class DatabaseType(str, Enum): + """Backends supported by the connection manager.""" + + MONGODB = "mongodb" + MYSQL = "mysql" + REDIS = "redis" + + def __str__(self) -> str: + return self.value + + +_TIMEOUT_OPTIONS = { + DatabaseType.MONGODB: "connectTimeoutMS", + DatabaseType.MYSQL: "connection_timeout", + DatabaseType.REDIS: "socket_connect_timeout", +} +_NATIVE_ALIASES = { + DatabaseType.MONGODB: {"ssl": "tls"}, + DatabaseType.MYSQL: { + "username": "user", + "passwd": "password", + "password1": "password", + "db": "database", + "connect_timeout": "connection_timeout", + "read_default_file": "option_files", + }, + DatabaseType.REDIS: {"charset": "encoding", "errors": "encoding_errors"}, +} + + +def _copy_containers(value: Any) -> Any: + """Copy built-in containers without copying driver objects or callbacks.""" + value_type = type(value) + if value_type is dict: + return {key: _copy_containers(item) for key, item in value.items()} + if value_type in (list, tuple, set, frozenset): + return value_type(_copy_containers(item) for item in value) + if value_type is bytearray: + return value.copy() + return value + + +def _is_nonempty_string(value: Any) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _is_port(value: Any) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and 1 <= value <= 65535 + + +@dataclass(frozen=True) +class ConnectionConfig: + """Connection inputs supplied by the caller, independent of loaded drivers.""" + + backend: DatabaseType | str + host: str | list[str] | tuple[str, ...] | None = field(default=None, repr=False) + port: int | None = None + username: str | None = field(default=None, repr=False) + password: str | None = field(default=None, repr=False) + connect_timeout_s: int | None = None + options: Mapping[str, Any] = field(default_factory=dict, repr=False) + mode: str = "single" + nodes: tuple[tuple[str, int], ...] | list[tuple[str, int]] = field( + default=(), repr=False + ) + service_name: str | None = field(default=None, repr=False) + + def __post_init__(self) -> None: + try: + backend = DatabaseType(self.backend) + except ValueError: + raise ConfigurationError( + "backend must be mongodb, mysql, or redis", phase="config" + ) from None + object.__setattr__(self, "backend", backend) + + if self.port is not None and not _is_port(self.port): + raise self._error("port must be an integer from 1 to 65535") + timeout = self.connect_timeout_s + if timeout is not None and ( + not isinstance(timeout, int) or isinstance(timeout, bool) or timeout <= 0 + ): + raise self._error("connect_timeout_s must be a positive integer") + for key in ("username", "password"): + value = getattr(self, key) + if value is not None and not isinstance(value, str): + raise self._error(f"{key} must be a string") + if not isinstance(self.options, Mapping) or any( + not isinstance(key, str) for key in self.options + ): + raise self._error("options must be a mapping with string keys") + if not isinstance(self.nodes, (list, tuple)): + raise self._error("nodes must contain (host, port) pairs") + for node in self.nodes: + if ( + not isinstance(node, (list, tuple)) + or len(node) != 2 + or not _is_nonempty_string(node[0]) + or not _is_port(node[1]) + ): + raise self._error("nodes must contain nonempty hosts and valid ports") + + if isinstance(self.host, (list, tuple)): + object.__setattr__(self, "host", self._copy(self.host)) + object.__setattr__(self, "options", self._copy(dict(self.options))) + object.__setattr__(self, "nodes", tuple((host, port) for host, port in self.nodes)) + self._validated_driver_kwargs() + + def snapshot(self) -> "ConnectionConfig": + """Return an independent copy of the configuration containers.""" + return replace(self) + + def driver_kwargs(self) -> dict[str, Any]: + """Return fresh native keyword arguments for this configuration.""" + return self._copy(self._validated_driver_kwargs()) + + def _error(self, message: str) -> ConfigurationError: + return ConfigurationError(message, backend=self.backend, phase="config") + + def _copy(self, value: Any) -> Any: + try: + return _copy_containers(value) + except RecursionError: + raise self._error("configuration containers must be acyclic") from None + + def _native_key(self, key: str) -> str: + if self.backend == DatabaseType.MONGODB: + key = key.lower() + return _NATIVE_ALIASES[self.backend].get(key, key) + + def _validated_driver_kwargs(self) -> dict[str, Any]: + options = dict(self.options) + seen = set() + for key in options: + if not isinstance(key, str): + raise self._error("options must be a mapping with string keys") + native_key = self._native_key(key) + if native_key in seen: + raise self._error("options contains multiple names for the same driver argument") + seen.add(native_key) + + timeout = self.connect_timeout_s + if timeout is not None and self.backend == DatabaseType.MONGODB: + timeout *= 1000 + common = { + "host": self.host, + "port": self.port, + "user" if self.backend == DatabaseType.MYSQL else "username": self.username, + "password": self.password, + _TIMEOUT_OPTIONS[self.backend]: timeout, + } + for key, value in common.items(): + if value is None: + continue + if self._native_key(key) in seen: + raise self._error(f"{key} is specified both directly and in options") + options[key] = value + + host = options.get("host") + if host is not None: + if self.backend == DatabaseType.MONGODB and isinstance(host, (list, tuple)): + if not host or not all(_is_nonempty_string(address) for address in host): + raise self._error("host must contain nonempty MongoDB addresses") + elif not _is_nonempty_string(host): + raise self._error("host must be a nonempty string") + + if self.backend == DatabaseType.REDIS: + self._validate_redis_target(options) + else: + if self.mode != "single" or self.nodes or self.service_name is not None: + raise self._error("mode, nodes, and service_name apply to Redis topology") + # Connector/Python uses its default TCP host for Unix sockets on non-POSIX platforms. + has_socket_target = ( + self.backend == DatabaseType.MYSQL + and os.name == "posix" + and _is_nonempty_string(options.get("unix_socket")) + ) + if host is None and not has_socket_target: + raise self._error("an explicit connection target is required") + return options + + def _validate_redis_target(self, options: dict[str, Any]) -> None: + if self.mode not in ("single", "sentinel", "cluster"): + raise self._error("Redis mode must be single, sentinel, or cluster") + + if self.mode == "sentinel": + self._validate_sentinel_target(options) + return + + if self.service_name is not None or "sentinels" in options: + raise self._error("service_name and sentinels apply to Redis Sentinel") + + pool = options.get("connection_pool") + if self.mode == "cluster": + if pool is not None: + raise self._error("connection_pool applies to Redis single mode") + if self.nodes and "startup_nodes" in options: + raise self._error("nodes and options.startup_nodes cannot both be set") + if not self.nodes and options.get("host") is None and not options.get("startup_nodes"): + raise self._error("Redis Cluster requires host, nodes, or options.startup_nodes") + return + + if self.nodes or "startup_nodes" in options: + raise self._error("Redis single mode cannot specify nodes") + if pool is not None: + pool_conflicts = { + "host", "port", "username", "password", "socket_connect_timeout", + "unix_socket_path", "credential_provider", + } + if pool_conflicts.intersection(options): + raise self._error("connection_pool cannot be combined with connection parameters") + # redis-py creates its default localhost pool when this value is falsy. + if not pool: + raise self._error("connection_pool must supply an explicit target") + elif options.get("host") is None and not _is_nonempty_string( + options.get("unix_socket_path") + ): + raise self._error("Redis single mode requires an explicit connection target") + + def _validate_sentinel_target(self, options: dict[str, Any]) -> None: + if not _is_nonempty_string(self.service_name): + raise self._error("Redis Sentinel requires a nonempty service_name") + if {"host", "port", "startup_nodes"}.intersection(options): + raise self._error("Redis Sentinel requires sentinel addresses through nodes or sentinels") + if self.nodes and "sentinels" in options: + raise self._error("nodes and options.sentinels cannot both be set") + if not self.nodes and not options.get("sentinels"): + raise self._error("Redis Sentinel requires nodes or options.sentinels") + sentinel_kwargs = options.get("sentinel_kwargs") + if options.get("connection_pool") is not None or ( + isinstance(sentinel_kwargs, Mapping) + and sentinel_kwargs.get("connection_pool") is not None + ): + raise self._error("connection_pool applies to Redis single mode") diff --git a/src/datvia/connections.py b/src/datvia/connections.py new file mode 100644 index 0000000000000000000000000000000000000000..a5f05f212db0379421c2938e88edd39f9a510210 --- /dev/null +++ b/src/datvia/connections.py @@ -0,0 +1,206 @@ +"""Named configurations and explicitly managed native clients.""" +import asyncio +import importlib +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime, timezone +from time import perf_counter + +from .config import ConnectionConfig +from .errors import ( + ConfigurationError, ConnectionClosedError, DependencyError, + attach_cleanup_errors, error_context, +) + + +@dataclass(frozen=True) +class CheckResult: + connection_name: str + backend: str + mode: str + checked_at: datetime + elapsed_ms: float + scope: str + targets: tuple[str, ...] + + +class Connection: + """Own a native client without proxying its operations.""" + + def __init__(self, native, adapter, name, backend, mode): + self._native = native + self._adapter = adapter + self.name = name + self.backend = backend + self.mode = mode + self._closed = False + + @property + def closed(self): + return self._closed + + def _ensure_open(self, phase): + if self._closed: + raise ConnectionClosedError( + "The database connection is closed.", connection_name=self.name, + backend=self.backend, phase=phase, + ) + + @property + def native(self): + self._ensure_open("operation") + return self._native + + def _context(self, phase, *, operation=None, owned=True): + return error_context( + connection_name=self.name, backend=self.backend, phase=phase, + operation=operation, classifier=self._adapter.classify, owned=owned, + ) + + def errors(self, *, operation=None): + self._ensure_open("operation") + return self._context("operation", operation=operation, owned=False) + + def check(self): + self._ensure_open("check") + started = perf_counter() + checked_at = datetime.now(timezone.utc) + with self._context("check"): + scope, targets = self._adapter.check(self._native) + return CheckResult( + self.name, self.backend, self.mode, checked_at, + (perf_counter() - started) * 1000, scope, tuple(targets), + ) + + def close(self): + if self._closed: + return + with self._context("close"): + self._adapter.close(self._native) + self._closed = True + + def __enter__(self): + self._ensure_open("create") + return self + + def __exit__(self, exc_type, exc, traceback): + try: + self.close() + except BaseException as cleanup_error: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup_error,)) + return False + + +class AsyncConnection(Connection): + """A native async client tied to its creating event loop.""" + + def __init__(self, native, adapter, name, backend, mode, loop): + super().__init__(native, adapter, name, backend, mode) + self._loop = loop + + def _ensure_loop(self, phase): + if asyncio.get_running_loop() is not self._loop: + raise ConfigurationError( + "Use the connection in its creating event loop.", + connection_name=self.name, backend=self.backend, phase=phase, + ) + + async def check(self): + self._ensure_loop("check") + self._ensure_open("check") + started = perf_counter() + checked_at = datetime.now(timezone.utc) + with self._context("check"): + scope, targets = await self._adapter.check_async(self._native) + return CheckResult( + self.name, self.backend, self.mode, checked_at, + (perf_counter() - started) * 1000, scope, tuple(targets), + ) + + async def close(self): + self._ensure_loop("close") + if self._closed: + return + with self._context("close"): + await self._adapter.close_async(self._native) + self._closed = True + + def __enter__(self): + raise TypeError("Use 'async with' for an asynchronous connection.") + + async def __aenter__(self): + self._ensure_loop("create") + self._ensure_open("create") + return self + + async def __aexit__(self, exc_type, exc, traceback): + try: + await self.close() + except BaseException as cleanup_error: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup_error,)) + return False + + +class Connections: + """Reuse configuration snapshots, never native client instances.""" + + def __init__(self, configs: Mapping[str, ConnectionConfig]): + if not isinstance(configs, Mapping): + raise ConfigurationError("Expected a mapping of named configurations.", phase="config") + self._configs = {} + for name, config in configs.items(): + if not isinstance(name, str) or not name.strip(): + raise ConfigurationError("Connection names must be nonempty strings.", phase="config") + if not isinstance(config, ConnectionConfig): + raise ConfigurationError( + "Expected a ConnectionConfig.", connection_name=name, phase="config", + ) + self._configs[name] = config.snapshot() + + def _config(self, name): + if not isinstance(name, str) or name not in self._configs: + raise ConfigurationError( + "Unknown connection name.", connection_name=name if isinstance(name, str) else None, + phase="config", + ) + return self._configs[name].snapshot() + + def _adapter(self, config, name, is_async): + backend = config.backend.value + try: + module = importlib.import_module("datvia._adapters." + backend) + adapter = module.Adapter(config, name, is_async=is_async) + except ImportError as original: + # Imports happen only after a backend is explicitly selected. + raise DependencyError( + "Install the selected database driver and its required capabilities.", + connection_name=name, backend=backend, phase="create", original=original, + ) from original + return adapter + + def create(self, name): + config = self._config(name) + adapter = self._adapter(config, name, False) + with error_context( + connection_name=name, backend=config.backend.value, phase="create", + classifier=adapter.classify, owned=True, + ): + native = adapter.create() + return Connection(native, adapter, name, config.backend.value, config.mode) + + async def create_async(self, name): + config = self._config(name) + adapter = self._adapter(config, name, True) + loop = asyncio.get_running_loop() + with error_context( + connection_name=name, backend=config.backend.value, phase="create", + classifier=adapter.classify, owned=True, + ): + native = await adapter.create_async() + return AsyncConnection( + native, adapter, name, config.backend.value, config.mode, loop, + ) diff --git a/src/datvia/crud/__init__.py b/src/datvia/crud/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..ecff037f0f58c1e398cfc3b24aef5629bc06c14e --- /dev/null +++ b/src/datvia/crud/__init__.py @@ -0,0 +1,84 @@ +"""Optional record operations that borrow managed database connections.""" + +from typing import TYPE_CHECKING +from typing import Any + +from ._conditions import And +from ._conditions import Eq +from ._conditions import Exists +from ._conditions import Gt +from ._conditions import Gte +from ._conditions import In +from ._conditions import IsNull +from ._conditions import Lt +from ._conditions import Lte +from ._conditions import Or +from ._errors import BatchOperationError +from ._errors import RecordConflictError +from ._errors import RecordFormatError +from ._errors import RecordIdentityError +from ._errors import UnsupportedOperationError +from ._records import AsyncRecords +from ._records import Records +from ._string_types import StringBatchItem +from ._string_types import StringBatchResult +from ._string_types import StringSetItem +from ._types import BatchItem +from ._types import BatchResult +from ._types import BatchState +from ._types import InsertItem +from ._types import ReadItem +from ._types import WriteCounts +from ._types import WriteResult + +if TYPE_CHECKING: + from ._codecs import FieldCodec + from ._codecs import JsonFieldCodec + from ._strings import AsyncRedisStrings + from ._strings import RedisStrings + +__all__ = [ + 'And', + 'AsyncRecords', + 'AsyncRedisStrings', + 'BatchItem', + 'BatchOperationError', + 'BatchResult', + 'BatchState', + 'Eq', + 'Exists', + 'FieldCodec', + 'Gt', + 'Gte', + 'In', + 'InsertItem', + 'IsNull', + 'JsonFieldCodec', + 'Lt', + 'Lte', + 'Or', + 'ReadItem', + 'RecordConflictError', + 'RecordFormatError', + 'RecordIdentityError', + 'Records', + 'RedisStrings', + 'StringBatchItem', + 'StringBatchResult', + 'StringSetItem', + 'UnsupportedOperationError', + 'WriteCounts', + 'WriteResult', +] + + +def __getattr__(name: str) -> Any: + if name in ('FieldCodec', 'JsonFieldCodec'): + from . import _codecs + + return getattr(_codecs, name) + if name in ('RedisStrings', 'AsyncRedisStrings'): + from . import _strings + + return getattr(_strings, name) + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/src/datvia/crud/_batch.py b/src/datvia/crud/_batch.py new file mode 100644 index 0000000000000000000000000000000000000000..a8b1c965f27bd6195ca93612849b4283115bbb5b --- /dev/null +++ b/src/datvia/crud/_batch.py @@ -0,0 +1,105 @@ +"""Preserve fetch ordering and evidence from completed batch chunks.""" + +import sys +from collections.abc import Sequence +from dataclasses import replace +from typing import Any + +from ._types import BatchItem +from ._types import BatchResult +from ._types import BatchState +from ._types import ChunkResult +from ._types import ReadItem +from ._types import WriteCounts + +Record = dict[str, Any] +Positions = Sequence[tuple[int, ...]] + + +def identity(key: Record) -> tuple[tuple[type, Any], ...]: + """Compare previously validated scalar keys without conflating Python types.""" + decimal128 = getattr(sys.modules.get('bson.decimal128'), 'Decimal128', None) + return tuple((type(value), value.bid if type(value) is decimal128 else value) for value in key.values()) + + +def unique_keys(keys: Sequence[Record]) -> tuple[list[Record], list[tuple[int, ...]]]: + groups: dict[tuple[tuple[type, Any], ...], list[int]] = {} + unique = [] + for index, key in enumerate(keys): + token = identity(key) + if token not in groups: + unique.append(key) + groups[token] = [] + groups[token].append(index) + return unique, [tuple(group) for group in groups.values()] + + +class Batch: + """Track the original positions, including read keys queried once.""" + + def __init__(self, keys: Sequence[Record | None]) -> None: + self.items = [BatchItem(index, key, BatchState.NOT_SENT) for index, key in enumerate(keys)] + self.native_results: list[Any] = [] + self._counts: list[WriteCounts] = [] + + def absorb(self, positions: Positions, chunk: ChunkResult | None) -> None: + by_index = {item.index: item for item in chunk.items} if chunk is not None else {} + for offset, group in enumerate(positions): + chunk_item = by_index.get(offset) + for index in group: + original = self.items[index] + if chunk_item is None: + self.items[index] = replace(original, state=BatchState.UNKNOWN) + continue + result = chunk_item.result + if isinstance(result, ReadItem): + result = self._read_result(original, result.record) + self.items[index] = replace(original, state=chunk_item.state, result=result, error=chunk_item.error) + if chunk is not None: + self.native_results.extend(chunk.native_results) + self._counts.append(chunk.counts if chunk is not None else WriteCounts()) + + def absorb_reads(self, positions: Positions, rows: Sequence[Record | None]) -> None: + if len(rows) != len(positions): + raise RuntimeError('Adapter read result does not match chunk size.') + for group, row in zip(positions, rows): + for index in group: + item = self.items[index] + self.items[index] = replace(item, state=BatchState.ACKNOWLEDGED, result=self._read_result(item, row)) + + @staticmethod + def _read_result(item: BatchItem, record: Record | None) -> ReadItem: + if item.key is None: + raise RuntimeError('A batch read result must have an input key.') + return ReadItem(item.index, item.key.copy(), None if record is None else record.copy()) + + def read_items(self) -> list[ReadItem]: + results = [] + for item in self.items: + if not isinstance(item.result, ReadItem): + raise RuntimeError('A complete batch read must include every input position.') + results.append(item.result) + return results + + def result(self, operation: str) -> BatchResult: + if not self.items: + counts = WriteCounts( + matched_count=0 if operation == 'bulk_update' else None, + modified_count=0 if operation == 'bulk_update' else None, + inserted_count=0 if operation == 'insert_many' else None, + deleted_count=0 if operation == 'delete_many' else None, + ) + else: + counts = WriteCounts( + matched_count=self._sum_count('matched_count'), + modified_count=self._sum_count('modified_count'), + inserted_count=self._sum_count('inserted_count'), + deleted_count=self._sum_count('deleted_count'), + ) + return BatchResult(len(self.items), tuple(self.items), counts, tuple(self.native_results)) + + def _sum_count(self, name: str) -> int | None: + values = [getattr(counts, name) for counts in self._counts] + if not values or any(value is None for value in values): + return None + return sum(values) diff --git a/src/datvia/crud/_codecs.py b/src/datvia/crud/_codecs.py new file mode 100644 index 0000000000000000000000000000000000000000..35cb9d21afa1beaa08bc9b4f9bc41fe49368af29 --- /dev/null +++ b/src/datvia/crud/_codecs.py @@ -0,0 +1,52 @@ +"""Field codecs used by the Redis Hash record adapter.""" + +import json +import math +from typing import Any +from typing import Protocol + + +class FieldCodec(Protocol): + """Encode a field as Redis data and decode the reply from the native client.""" + + def encode(self, value: Any) -> str | bytes: + """Return the wire representation for a field value.""" + ... + + def decode(self, value: str | bytes) -> Any: + """Return the field value from a native Redis reply.""" + ... + + +def _check_json_value(value: Any) -> None: + kind = type(value) + if value is None or kind in (str, bool, int): + return + if kind is float and math.isfinite(value): + return + if kind is list: + for item in value: + _check_json_value(item) + return + if kind is dict and all(type(key) is str for key in value): + for item in value.values(): + _check_json_value(item) + return + raise ValueError('Expected a JSON value with finite numbers and string object keys.') + + +class JsonFieldCodec: + """Store standard JSON fields in UTF-8 bytes. + + Accept ordinary JSON types and finite numbers. Decoding accepts the + native client's UTF-8 bytes or decoded string, including JSON null. + """ + + def encode(self, value: Any) -> bytes: + _check_json_value(value) + return json.dumps(value, ensure_ascii=True, allow_nan=False, separators=(',', ':')).encode('ascii') + + def decode(self, value: str | bytes) -> Any: + decoded = json.loads(value) + _check_json_value(decoded) + return decoded diff --git a/src/datvia/crud/_conditions.py b/src/datvia/crud/_conditions.py new file mode 100644 index 0000000000000000000000000000000000000000..b9b52c11f2d53406a45e8d32139d7da4cbae6a45 --- /dev/null +++ b/src/datvia/crud/_conditions.py @@ -0,0 +1,148 @@ +"""Structured conditions for ordinary fields.""" + +import math +import re +import sys +from collections.abc import Mapping +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date +from datetime import datetime +from datetime import time +from decimal import Decimal +from typing import Any +from typing import TypeAlias +from uuid import UUID + +from ..errors import ConfigurationError +from ._errors import UnsupportedOperationError + + +@dataclass(frozen=True) +class Eq: + field: str + value: Any + + +@dataclass(frozen=True) +class In: + field: str + values: Sequence[Any] + + +@dataclass(frozen=True) +class Gt: + field: str + value: Any + + +@dataclass(frozen=True) +class Gte: + field: str + value: Any + + +@dataclass(frozen=True) +class Lt: + field: str + value: Any + + +@dataclass(frozen=True) +class Lte: + field: str + value: Any + + +@dataclass(frozen=True) +class IsNull: + field: str + + +@dataclass(frozen=True) +class Exists: + field: str + + +@dataclass(frozen=True, init=False) +class And: + conditions: tuple['Condition', ...] + + def __init__(self, *conditions: 'Condition') -> None: + object.__setattr__(self, 'conditions', conditions) + + +@dataclass(frozen=True, init=False) +class Or: + conditions: tuple['Condition', ...] + + def __init__(self, *conditions: 'Condition') -> None: + object.__setattr__(self, 'conditions', conditions) + + +Condition: TypeAlias = Eq | In | Gt | Gte | Lt | Lte | IsNull | Exists | And | Or +Where: TypeAlias = Condition | Mapping[str, Any] | None +_FIELD_NAME = re.compile(r'[^.\x00]+\Z') +_SCALARS = {str, int, float, bool, bytes, Decimal, date, datetime, time, UUID} + + +def field_name(name: Any) -> str: + """Validate one plain field name; adapters apply their identifier rules.""" + if not isinstance(name, str) or not name.strip() or name.startswith('$') or not _FIELD_NAME.fullmatch(name): + raise ConfigurationError('Expected a plain, nonempty record field name.', phase='config') + return name + + +def is_sequence(value: Any) -> bool: + return isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray, memoryview)) + + +def validate_scalar(value: Any) -> None: + object_id = getattr(sys.modules.get('bson.objectid'), 'ObjectId', None) + int64 = getattr(sys.modules.get('bson.int64'), 'Int64', None) + decimal128 = getattr(sys.modules.get('bson.decimal128'), 'Decimal128', None) + if type(value) not in _SCALARS and type(value) not in (object_id, int64, decimal128): + raise ConfigurationError('Expected a supported scalar comparison or key value.', phase='config') + if (type(value) is float and not math.isfinite(value)) or (type(value) is Decimal and not value.is_finite()): + raise ConfigurationError('Comparison and key values must be finite.', phase='config') + if type(value) is decimal128 and not value.to_decimal().is_finite(): + raise ConfigurationError('Comparison and key values must be finite.', phase='config') + + +def normalize_where(where: Where, backend: str, operation: str) -> Condition | None: + """Normalize dict equality and validate the complete condition tree.""" + if where is None: + return None + if isinstance(where, Mapping): + if not where: + return None + conditions = tuple(_condition(Eq(name, value), backend, operation) for name, value in where.items()) + return conditions[0] if len(conditions) == 1 else And(*conditions) + return _condition(where, backend, operation) + + +def _condition(node: Condition, backend: str, operation: str) -> Condition: + if type(node) in (And, Or): + if not node.conditions: + raise ConfigurationError('And and Or require at least one condition.', phase='config') + return type(node)(*(_condition(child, backend, operation) for child in node.conditions)) + if type(node) not in (Eq, In, Gt, Gte, Lt, Lte, IsNull, Exists): + raise ConfigurationError('Expected a structured record condition.', phase='config') + name = field_name(node.field) + if type(node) is Exists and backend != 'mongodb': + raise UnsupportedOperationError( + 'Field existence conditions require MongoDB.', backend=backend, operation=operation, phase='config' + ) + if type(node) in (IsNull, Exists): + return node + if type(node) is Eq and node.value is None: + return IsNull(name) + if type(node) is In: + if not is_sequence(node.values): + raise ConfigurationError('In requires a finite sequence of scalar values.', phase='config') + values = tuple(node.values) + for value in values: + validate_scalar(value) + return In(name, values) + validate_scalar(node.value) + return node diff --git a/src/datvia/crud/_errors.py b/src/datvia/crud/_errors.py new file mode 100644 index 0000000000000000000000000000000000000000..168e380947bab99b73895869a3e32b7d7739e1ba --- /dev/null +++ b/src/datvia/crud/_errors.py @@ -0,0 +1,31 @@ +"""Errors shared by the managed CRUD API.""" + +from typing import Any + +from ..errors import DatabaseError +from ._string_types import StringBatchResult +from ._types import BatchResult + + +class UnsupportedOperationError(DatabaseError): + """The selected backend does not support the requested operation.""" + + +class RecordConflictError(DatabaseError): + """A create-only operation conflicts with an existing record.""" + + +class RecordFormatError(DatabaseError): + """A stored record uses an unsupported format.""" + + +class RecordIdentityError(DatabaseError): + """A declared unique key matches more than one record.""" + + +class BatchOperationError(DatabaseError): + """A batch operation failed; the confirmed and uncertain results are retained.""" + + def __init__(self, message: str, *, partial_result: BatchResult | StringBatchResult, **context: Any) -> None: + super().__init__(message, **context) + self.partial_result = partial_result diff --git a/src/datvia/crud/_mongodb.py b/src/datvia/crud/_mongodb.py new file mode 100644 index 0000000000000000000000000000000000000000..700f163c284c9e439dd8cf1bcf421711c9f63797 --- /dev/null +++ b/src/datvia/crud/_mongodb.py @@ -0,0 +1,510 @@ +"""MongoDB record operations on a borrowed collection and optional session.""" + +import asyncio +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import field +from dataclasses import replace +from types import TracebackType +from typing import Any + +from bson import BSON +from bson.errors import BSONError +from pymongo import DeleteOne +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError +from pymongo.errors import DuplicateKeyError +from pymongo.errors import InvalidName + +from ..connections import Connection +from ..errors import ConfigurationError +from ..mongodb._resources import _async_closing +from ..mongodb._resources import _closing +from ._batch import Record +from ._conditions import And +from ._conditions import Condition +from ._conditions import Eq +from ._conditions import Exists +from ._conditions import Gt +from ._conditions import Gte +from ._conditions import In +from ._conditions import IsNull +from ._conditions import Lt +from ._conditions import Lte +from ._conditions import Or +from ._errors import RecordConflictError +from ._errors import RecordIdentityError +from ._types import BatchItem +from ._types import BatchState +from ._types import ChunkError +from ._types import ChunkResult +from ._types import ReadItem +from ._types import WriteCounts +from ._types import WriteInput +from ._types import WriteResult + +_Fields = tuple[str, ...] | None +_Order = tuple[tuple[str, str], ...] +_BACKEND = 'mongodb' +_COMPARISONS = {Eq: '$eq', Gt: '$gt', Gte: '$gte', Lt: '$lt', Lte: '$lte'} + + +def _scalar_predicate(operator: str, value: Any) -> Record: + return {operator: value, '$exists': True, '$not': {'$type': 'array'}} + + +def _condition_filter(where: Condition | None) -> Record | None: + """Translate a normalized condition. Return None only for a known empty match.""" + if where is None: + return {} + if isinstance(where, (And, Or)): + filters = [_condition_filter(child) for child in where.conditions] + if isinstance(where, And): + return None if None in filters else {'$and': filters} + alternatives = [clause for clause in filters if clause is not None] + return {'$or': alternatives} if alternatives else None + if isinstance(where, Exists): + return {where.field: {'$exists': True}} + if isinstance(where, IsNull): + return {where.field: _scalar_predicate('$type', 'null')} + if isinstance(where, In): + return {where.field: _scalar_predicate('$in', where.values)} if where.values else None + return {where.field: _scalar_predicate(_COMPARISONS[type(where)], where.value)} + + +def _key_filter(key: Record) -> Record: + return {field: _scalar_predicate('$eq', value) for field, value in key.items()} + + +def _projection(fields: _Fields) -> Record | None: + if fields is None: + return None + if not fields: + return {'_id': 1} + options = dict.fromkeys(fields, 1) + if '_id' not in options: + options['_id'] = 0 + return options + + +def _record(row: Mapping[str, Any], fields: _Fields) -> Record: + if fields is None: + return dict(row) + return {field: row[field] for field in fields if field in row} + + +@dataclass +class _ReadProgress: + rows: list[Record | None] = field(default_factory=list) + dispatched: int = 0 + + +@contextmanager +def _read_batch(keys: Sequence[Record]) -> Iterator[_ReadProgress]: + progress = _ReadProgress() + try: + yield progress + except (Exception, asyncio.CancelledError) as error: + outcomes = [ + BatchItem(index, keys[index], BatchState.ACKNOWLEDGED, ReadItem(index, keys[index], row)) + for index, row in enumerate(progress.rows) + ] + if isinstance(error, RecordIdentityError) and len(progress.rows) < progress.dispatched: + index = progress.dispatched - 1 + outcomes.append(BatchItem(index, keys[index], BatchState.FAILED, error=error)) + outcomes.extend( + BatchItem(index, keys[index], BatchState.NOT_SENT) for index in range(progress.dispatched, len(keys)) + ) + partial = ChunkResult(tuple(outcomes)) + if isinstance(error, asyncio.CancelledError): + BaseException.__dict__['__dict__'].__get__(error)['database_partial_result'] = partial + raise + raise ChunkError(error, partial) from error + + +def _update_result(operation: str, result: Any) -> WriteResult: + return WriteResult( + _BACKEND, + operation, + acknowledged=result.acknowledged, + matched_count=result.matched_count if result.acknowledged else None, + modified_count=result.modified_count if result.acknowledged else None, + native_result=result, + ) + + +def _delete_result(operation: str, result: Any) -> WriteResult: + return WriteResult( + _BACKEND, + operation, + acknowledged=result.acknowledged, + deleted_count=result.deleted_count if result.acknowledged else None, + native_result=result, + ) + + +def _batch_state(acknowledged: bool) -> BatchState: + return BatchState.ACKNOWLEDGED if acknowledged else BatchState.UNACKNOWLEDGED + + +def _bulk_result(operation: str, items: Sequence[WriteInput], result: Any) -> ChunkResult: + counts = WriteCounts() + if result.acknowledged: + if operation == 'bulk_update': + counts = WriteCounts(matched_count=result.matched_count, modified_count=result.modified_count) + else: + counts = WriteCounts(deleted_count=result.deleted_count) + return ChunkResult( + tuple(BatchItem(index, item.key, _batch_state(result.acknowledged)) for index, item in enumerate(items)), + counts, + (result,), + ) + + +def _partial_bulk(operation: str, items: Sequence[WriteInput], error: BulkWriteError) -> ChunkResult: + details = error.details + write_errors = details.get('writeErrors', ()) + indexed = { + entry['index']: entry + for entry in write_errors + if type(entry.get('index')) is int and 0 <= entry['index'] < len(items) + } + counts = WriteCounts() + stop = None + if len(write_errors) == len(indexed) == 1 and not details.get('writeConcernErrors'): + stop = next(iter(indexed)) + reported = { + 'insert_many': {'inserted_count': 'nInserted'}, + 'bulk_update': {'matched_count': 'nMatched', 'modified_count': 'nModified'}, + 'delete_many': {'deleted_count': 'nRemoved'}, + }[operation] + counts = WriteCounts( + **{ + field: details[metric] + for field, metric in reported.items() + if type(details.get(metric)) is int and 0 <= details[metric] <= stop + } + ) + outcomes = [] + for index, item in enumerate(items): + if index in indexed: + state = BatchState.FAILED + elif stop is None: + state = BatchState.UNKNOWN + elif index < stop: + state = BatchState.ACKNOWLEDGED + else: + state = BatchState.NOT_EXECUTED + outcomes.append(BatchItem(index, item.key, state, error=indexed.get(index))) + return ChunkResult(tuple(outcomes), counts, (details,)) + + +@contextmanager +def _bulk_errors(operation: str, items: Sequence[WriteInput]) -> Iterator[None]: + try: + yield + except BulkWriteError as error: + raise ChunkError(error, _partial_bulk(operation, items, error)) from error + + +class _MongoRecords: + def __init__( + self, + *, + connection: Connection, + key_fields: tuple[str, ...], + database: str, + collection: str, + session: Any = None, + ) -> None: + self._connection = connection + for name in (database, collection): + if not isinstance(name, str) or not name.strip() or '\x00' in name: + raise self._config_error('Expected explicit MongoDB database and collection names.') + self._database = database + self._collection_name = collection + self._key_fields = key_fields + self._session = session + self._collection: Any = None + + def _open(self) -> None: + try: + self._collection = self._connection.native[self._database][self._collection_name] + except InvalidName as error: + raise self._config_error('Invalid MongoDB database or collection name.', original=error) from error + + def _config_error(self, message: str, *, original: BaseException | None = None) -> ConfigurationError: + return ConfigurationError( + message, connection_name=self._connection.name, backend=_BACKEND, phase='config', original=original + ) + + def _validate_bson(self, document: Record) -> None: + try: + BSON.encode(document, codec_options=self._collection.codec_options) + except (BSONError, OverflowError, RecursionError, TypeError, ValueError) as error: + raise self._config_error( + 'The MongoDB collection cannot encode a record value as BSON.', original=error + ) from error + + def _filter(self, where: Condition | None) -> Record | None: + query = _condition_filter(where) + if query: + self._validate_bson(query) + return query + + def _update_document(self, values: Record) -> Record: + if '_id' in values: + raise self._config_error('MongoDB updates cannot modify the _id field.') + document = {'$set': values} + self._validate_bson(document) + return document + + def prepare_key(self, key: Record) -> None: + self._validate_bson(_key_filter(key)) + + def prepare_fields(self, fields: _Fields) -> None: + projection = _projection(fields) + if projection is not None: + self._validate_bson(projection) + + def prepare_write(self, operation: str, item: WriteInput) -> WriteInput: + if operation == 'insert': + if any(field != '_id' and field not in item.values for field in self._key_fields): + raise self._config_error('MongoDB inserts require every declared key field other than _id.') + self._validate_bson(item.values) + return item + if item.key is None: + raise self._config_error('A keyed MongoDB write requires its complete key.') + query = _key_filter(item.key) + if operation == 'update': + return replace(item, prepared=(query, self._update_document(item.values))) + return replace(item, prepared=query) + + def _unique_record(self, rows: list[Record], fields: _Fields, *, operation: str = 'get') -> Record | None: + if len(rows) > 1: + raise RecordIdentityError( + 'The declared MongoDB key matched more than one document.', + connection_name=self._connection.name, + backend=_BACKEND, + phase='operation', + operation=operation, + ) + return _record(rows[0], fields) if rows else None + + def _find_cursor(self, query: Record, fields: _Fields, limit: int, offset: int = 0, order_by: _Order = ()) -> Any: + sort = [(field, 1 if direction == 'asc' else -1) for field, direction in order_by] + if sort: + self._validate_bson(dict(sort)) + return self._collection.find( + query, + projection=_projection(fields), + limit=limit, + skip=offset, + sort=sort or None, + session=self._session, + ) + + @contextmanager + def _create_only(self) -> Iterator[None]: + try: + yield + except DuplicateKeyError as error: + raise RecordConflictError( + 'A MongoDB insert conflicts with a unique key.', + connection_name=self._connection.name, + backend=_BACKEND, + phase='operation', + operation='insert', + original=error, + ) from error + + def _generated_keys(self, item: WriteInput, inserted_id: Any) -> tuple[Record, ...] | None: + if '_id' not in self._key_fields or '_id' in item.values or inserted_id is None: + return None + return ({field: inserted_id if field == '_id' else item.values[field] for field in self._key_fields},) + + def _inserted( + self, item: WriteInput, inserted_id: Any, acknowledged: bool, native_result: Any = None + ) -> WriteResult: + return WriteResult( + _BACKEND, + 'insert', + acknowledged=acknowledged, + inserted_count=1 if acknowledged else None, + generated_keys=self._generated_keys(item, inserted_id), + native_result=native_result, + ) + + def _insert_batch(self, items: Sequence[WriteInput], result: Any) -> ChunkResult: + outcomes = tuple( + BatchItem( + index, + item.key, + _batch_state(result.acknowledged), + self._inserted(item, inserted_id, result.acknowledged), + ) + for index, (item, inserted_id) in enumerate(zip(items, result.inserted_ids, strict=True)) + ) + return ChunkResult(outcomes, WriteCounts(inserted_count=len(items) if result.acknowledged else None), (result,)) + + +class Adapter(_MongoRecords): + def __enter__(self) -> 'Adapter': + self._open() + return self + + def __exit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> bool: + return False + + def get(self, key: Record, fields: _Fields) -> Record | None: + with _closing(self._find_cursor(_key_filter(key), fields, 2)) as cursor: + return self._unique_record(list(cursor), fields) + + def get_many(self, keys: Sequence[Record], fields: _Fields) -> list[Record | None]: + with _read_batch(keys) as progress: + for key in keys: + with _closing(self._find_cursor(_key_filter(key), fields, 2)) as cursor: + progress.dispatched += 1 + progress.rows.append(self._unique_record(list(cursor), fields, operation='get_many')) + return progress.rows + + def find(self, where: Condition | None, fields: _Fields, limit: int, offset: int, order_by: _Order) -> list[Record]: + query = self._filter(where) + if query is None: + return [] + with _closing(self._find_cursor(query, fields, limit, offset, order_by)) as cursor: + return [_record(row, fields) for row in cursor] + + def count(self, where: Condition | None) -> int: + query = self._filter(where) + return 0 if query is None else self._collection.count_documents(query, session=self._session) + + def insert(self, item: WriteInput) -> WriteResult: + with self._create_only(): + result = self._collection.insert_one(item.values.copy(), session=self._session) + return self._inserted(item, result.inserted_id, result.acknowledged, result) + + def update(self, item: WriteInput) -> WriteResult: + result = self._collection.update_one(*item.prepared, session=self._session) + return _update_result('update', result) + + def delete(self, item: WriteInput) -> WriteResult: + return _delete_result('delete', self._collection.delete_one(item.prepared, session=self._session)) + + def insert_many(self, items: Sequence[WriteInput]) -> ChunkResult: + documents = [item.values.copy() for item in items] + with _bulk_errors('insert_many', items): + result = self._collection.insert_many(documents, ordered=True, session=self._session) + return self._insert_batch(items, result) + + def bulk_update(self, items: Sequence[WriteInput]) -> ChunkResult: + operations = [UpdateOne(*item.prepared) for item in items] + with _bulk_errors('bulk_update', items): + result = self._collection.bulk_write(operations, ordered=True, session=self._session) + return _bulk_result('bulk_update', items, result) + + def delete_many(self, items: Sequence[WriteInput]) -> ChunkResult: + operations = [DeleteOne(item.prepared) for item in items] + with _bulk_errors('delete_many', items): + result = self._collection.bulk_write(operations, ordered=True, session=self._session) + return _bulk_result('delete_many', items, result) + + def update_many(self, where: Condition | None, set_values: Record) -> WriteResult: + query = self._filter(where) + document = self._update_document(set_values) + if query is None: + return WriteResult(_BACKEND, 'update_many', matched_count=0, modified_count=0) + return _update_result('update_many', self._collection.update_many(query, document, session=self._session)) + + def delete_where(self, where: Condition | None) -> WriteResult: + query = self._filter(where) + if query is None: + return WriteResult(_BACKEND, 'delete_where', deleted_count=0) + return _delete_result('delete_where', self._collection.delete_many(query, session=self._session)) + + +class AsyncAdapter(_MongoRecords): + async def __aenter__(self) -> 'AsyncAdapter': + self._open() + return self + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> bool: + return False + + async def get(self, key: Record, fields: _Fields) -> Record | None: + async with _async_closing(self._find_cursor(_key_filter(key), fields, 2)) as cursor: + return self._unique_record([row async for row in cursor], fields) + + async def get_many(self, keys: Sequence[Record], fields: _Fields) -> list[Record | None]: + with _read_batch(keys) as progress: + for key in keys: + async with _async_closing(self._find_cursor(_key_filter(key), fields, 2)) as cursor: + progress.dispatched += 1 + matches = [row async for row in cursor] + progress.rows.append(self._unique_record(matches, fields, operation='get_many')) + return progress.rows + + async def find( + self, where: Condition | None, fields: _Fields, limit: int, offset: int, order_by: _Order + ) -> list[Record]: + query = self._filter(where) + if query is None: + return [] + async with _async_closing(self._find_cursor(query, fields, limit, offset, order_by)) as cursor: + return [_record(row, fields) async for row in cursor] + + async def count(self, where: Condition | None) -> int: + query = self._filter(where) + return 0 if query is None else await self._collection.count_documents(query, session=self._session) + + async def insert(self, item: WriteInput) -> WriteResult: + with self._create_only(): + result = await self._collection.insert_one(item.values.copy(), session=self._session) + return self._inserted(item, result.inserted_id, result.acknowledged, result) + + async def update(self, item: WriteInput) -> WriteResult: + result = await self._collection.update_one(*item.prepared, session=self._session) + return _update_result('update', result) + + async def delete(self, item: WriteInput) -> WriteResult: + return _delete_result('delete', await self._collection.delete_one(item.prepared, session=self._session)) + + async def insert_many(self, items: Sequence[WriteInput]) -> ChunkResult: + documents = [item.values.copy() for item in items] + with _bulk_errors('insert_many', items): + result = await self._collection.insert_many(documents, ordered=True, session=self._session) + return self._insert_batch(items, result) + + async def bulk_update(self, items: Sequence[WriteInput]) -> ChunkResult: + operations = [UpdateOne(*item.prepared) for item in items] + with _bulk_errors('bulk_update', items): + result = await self._collection.bulk_write(operations, ordered=True, session=self._session) + return _bulk_result('bulk_update', items, result) + + async def delete_many(self, items: Sequence[WriteInput]) -> ChunkResult: + operations = [DeleteOne(item.prepared) for item in items] + with _bulk_errors('delete_many', items): + result = await self._collection.bulk_write(operations, ordered=True, session=self._session) + return _bulk_result('delete_many', items, result) + + async def update_many(self, where: Condition | None, set_values: Record) -> WriteResult: + query = self._filter(where) + document = self._update_document(set_values) + if query is None: + return WriteResult(_BACKEND, 'update_many', matched_count=0, modified_count=0) + result = await self._collection.update_many(query, document, session=self._session) + return _update_result('update_many', result) + + async def delete_where(self, where: Condition | None) -> WriteResult: + query = self._filter(where) + if query is None: + return WriteResult(_BACKEND, 'delete_where', deleted_count=0) + result = await self._collection.delete_many(query, session=self._session) + return _delete_result('delete_where', result) diff --git a/src/datvia/crud/_mysql.py b/src/datvia/crud/_mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..3ecc1b12ca5d243e16b6ff8297b6a515ada03b9b --- /dev/null +++ b/src/datvia/crud/_mysql.py @@ -0,0 +1,635 @@ +"""MySQL record operations on a borrowed Connector/Python connection.""" + +import asyncio +import re +from collections.abc import AsyncIterator +from collections.abc import Iterator +from collections.abc import Sequence +from contextlib import asynccontextmanager +from contextlib import contextmanager +from dataclasses import dataclass +from dataclasses import replace +from datetime import date +from datetime import datetime +from datetime import time +from datetime import timedelta +from decimal import Decimal +from itertools import groupby +from math import isfinite +from typing import Any + +from mysql.connector.constants import ClientFlag +from mysql.connector.errorcode import ER_DUP_ENTRY +from mysql.connector.errors import Error as MySQLError + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import attach_cleanup_errors +from ._conditions import And +from ._conditions import Condition +from ._conditions import Eq +from ._conditions import Exists +from ._conditions import Gt +from ._conditions import Gte +from ._conditions import In +from ._conditions import IsNull +from ._conditions import Lt +from ._conditions import Lte +from ._conditions import Or +from ._errors import RecordConflictError +from ._errors import RecordIdentityError +from ._errors import UnsupportedOperationError +from ._types import BatchItem +from ._types import BatchState +from ._types import ChunkError +from ._types import ChunkResult +from ._types import ReadItem +from ._types import WriteCounts +from ._types import WriteInput +from ._types import WriteResult + +_IDENTIFIERS = re.compile(r'\w{1,64}') +_COMPARISONS = {Eq: '=', Gt: '>', Gte: '>=', Lt: '<', Lte: '<='} +_VALUE_TYPES = {str, bytes, bytearray, int, bool, float, Decimal, date, datetime, time, timedelta} +_COUNT_FIELDS = ('matched_count', 'modified_count', 'inserted_count', 'deleted_count') + +_Row = dict[str, Any] +_Fields = tuple[str, ...] | None +_Order = tuple[tuple[str, str], ...] + + +def _identifier(name: str) -> str: + if not isinstance(name, str) or not _IDENTIFIERS.fullmatch(name): + raise ConfigurationError( + 'MySQL identifiers require 1–64 letters, digits or underscores.', backend='mysql', phase='config' + ) + return f'`{name}`' + + +def _validate_value(value: Any) -> None: + if value is not None and type(value) not in _VALUE_TYPES: + raise ConfigurationError('Expected a native MySQL scalar value.', backend='mysql', phase='config') + if (type(value) is float and not isfinite(value)) or (type(value) is Decimal and not value.is_finite()): + raise ConfigurationError('Expected a finite MySQL numeric value.', backend='mysql', phase='config') + + +def _columns(names: Sequence[str]) -> tuple[str, ...]: + columns = tuple(_identifier(name) for name in names) + if len({name.casefold() for name in names}) != len(names): + raise ConfigurationError('MySQL requires distinct column names.', backend='mysql', phase='config') + return columns + + +def _projection(fields: _Fields) -> str: + if fields is None: + return '*' + return ', '.join(_columns(fields)) if fields else '1' + + +@dataclass(frozen=True) +class _Statement: + sql: str + params: tuple[Any, ...] = () + + +def _predicate(where: Condition) -> _Statement: + if isinstance(where, (And, Or)): + children = tuple(_predicate(child) for child in where.conditions) + joiner = ' AND ' if isinstance(where, And) else ' OR ' + return _Statement( + '(' + joiner.join(child.sql for child in children) + ')', + tuple(value for child in children for value in child.params), + ) + if isinstance(where, Exists): + raise UnsupportedOperationError('Field existence requires MongoDB.', backend='mysql', phase='config') + + field = _identifier(where.field) + if isinstance(where, IsNull): + return _Statement(f'{field} IS NULL') + if isinstance(where, In): + if not where.values: + return _Statement('0 = 1') + for value in where.values: + _validate_value(value) + placeholders = ', '.join('%s' for _ in where.values) + return _Statement(f'{field} IN ({placeholders})', tuple(where.values)) + _validate_value(where.value) + return _Statement(f'{field} {_COMPARISONS[type(where)]} %s', (where.value,)) + + +def _where_clause(where: Condition | None) -> _Statement: + if where is None: + return _Statement('') + predicate = _predicate(where) + return _Statement(f' WHERE {predicate.sql}', predicate.params) + + +def _is_duplicate(error: BaseException | None) -> bool: + return isinstance(error, MySQLError) and error.errno == ER_DUP_ENTRY + + +def _write_result(operation: str, cursor: Any, native: Any) -> WriteResult: + rowcount = cursor.rowcount + count = rowcount if type(rowcount) is int and rowcount >= 0 else None + stats = {'rowcount': rowcount, 'lastrowid': cursor.lastrowid} + counts = {} + if operation in ('insert', 'insert_many'): + counts['inserted_count'] = count + elif operation in ('delete', 'delete_many', 'delete_where'): + counts['deleted_count'] = count + else: + # The Python driver's client_flags can change before the connection is renegotiated. + # The C extension exposes the session's actual flags. + session_flags = getattr(getattr(native, '_cmysql', None), 'st_client_flag', None) + flags = session_flags() if callable(session_flags) else None + stats['rowcount_mode'] = 'unknown' + if type(flags) is int: + if flags & ClientFlag.FOUND_ROWS: + stats['rowcount_mode'] = 'matched' + counts['matched_count'] = count + else: + stats['rowcount_mode'] = 'changed' + counts['modified_count'] = count + return WriteResult('mysql', operation, acknowledged=True, native_result=stats, **counts) + + +def _read_records(rows: Sequence[tuple[Any, ...]], column_names: Sequence[str], fields: _Fields) -> list[_Row]: + columns = column_names if fields is None else fields + if not columns: + return [{} for _ in rows] + return [dict(zip(columns, row, strict=True)) for row in rows] + + +@dataclass(frozen=True) +class _WriteGroup: + sql: str + params: tuple[Any, ...] | list[tuple[Any, ...]] + size: int + many: bool + + +class _WriteProgress: + """Track completed groups separately from a group that may have been sent.""" + + def __init__(self, operation: str, items: Sequence[WriteInput]) -> None: + self.operation = operation + self.items = items + self.acknowledged: list[BatchItem] = [] + self.results: list[WriteResult] = [] + self.sent = 0 + + def complete(self, result: WriteResult) -> None: + start = len(self.acknowledged) + single = self.sent - start == 1 + self.acknowledged.extend( + BatchItem( + index, + self.items[index].key, + BatchState.ACKNOWLEDGED, + replace(result, operation=self.operation) if single else None, + ) + for index in range(start, self.sent) + ) + self.results.append(result) + + def result(self, error: BaseException | None = None) -> ChunkResult: + acknowledged = len(self.acknowledged) + single_conflict = self.operation == 'insert' and self.sent == acknowledged + 1 and _is_duplicate(error) + items = list(self.acknowledged) + for index in range(acknowledged, len(self.items)): + if index >= self.sent: + state = BatchState.NOT_SENT + elif single_conflict: + state = BatchState.FAILED + else: + state = BatchState.UNKNOWN + items.append( + BatchItem(index, self.items[index].key, state, error=error if state is BatchState.FAILED else None) + ) + + totals = WriteCounts() + if acknowledged == self.sent or single_conflict: + counts = {} + for name in _COUNT_FIELDS: + group_counts = [getattr(result, name) for result in self.results] + if group_counts and all(count is not None for count in group_counts): + counts[name] = sum(group_counts) + if not self.results: + if self.operation == 'insert': + counts['inserted_count'] = 0 + elif self.operation == 'delete': + counts['deleted_count'] = 0 + else: + counts.update(matched_count=0, modified_count=0) + totals = WriteCounts(**counts) + return ChunkResult(tuple(items), totals, tuple(result.native_result for result in self.results)) + + +def _chunk_error(error: BaseException, partial: ChunkResult) -> None: + if isinstance(error, asyncio.CancelledError): + BaseException.__dict__['__dict__'].__get__(error)['database_partial_result'] = partial + elif isinstance(error, Exception): + raise ChunkError(error, partial) from error + + +def _read_chunk(keys: Sequence[_Row], records: list[_Row | None] | None) -> ChunkResult: + if records is None: + items = tuple(BatchItem(index, key, BatchState.NOT_SENT) for index, key in enumerate(keys)) + else: + items = tuple( + BatchItem(index, key, BatchState.ACKNOWLEDGED, ReadItem(index, key, record)) + for index, (key, record) in enumerate(zip(keys, records, strict=True)) + ) + return ChunkResult(items) + + +class _MySQLPlan: + def __init__(self, *, connection: Connection | AsyncConnection, table: str, key_fields: tuple[str, ...]) -> None: + self.connection = connection + self.table = _identifier(table) + self.key_fields = key_fields + if not key_fields: + raise ConfigurationError('MySQL requires key fields.', backend='mysql', phase='config') + key_columns = _columns(key_fields) + self.key_names = {name.casefold(): name for name in key_fields} + self.key_where = ' AND '.join(f'{column} = %s' for column in key_columns) + + def prepare_fields(self, fields: _Fields) -> None: + _projection(fields) + + def prepare_key(self, key: _Row) -> None: + for name in self.key_fields: + _validate_value(key[name]) + + def prepare_write(self, operation: str, item: WriteInput) -> WriteInput: + if item.key is not None: + self.prepare_key(item.key) + if operation == 'insert': + statement = self._insert(item.values) + elif operation == 'update': + statement = self._update(item.values, _Statement(f' WHERE {self.key_where}', self._key_params(item.key))) + else: + statement = _Statement(f'DELETE FROM {self.table} WHERE {self.key_where}', self._key_params(item.key)) + return replace(item, prepared=statement) + + def _value_columns(self, values: _Row, *, update: bool) -> tuple[tuple[str, ...], tuple[Any, ...]]: + names = tuple(sorted(values)) + columns = _columns(names) + for name in names: + declared = self.key_names.get(name.casefold()) + if declared is not None and (update or declared != name): + raise ConfigurationError( + 'Use declared MySQL key names and leave keys unchanged.', + backend='mysql', + phase='config', + ) + params = tuple(values[name] for name in names) + for value in params: + _validate_value(value) + return columns, params + + def _key_params(self, key: _Row | None) -> tuple[Any, ...]: + if key is None: + raise ConfigurationError('Expected a complete MySQL record key.', backend='mysql', phase='config') + return tuple(key[name] for name in self.key_fields) + + def _insert(self, values: _Row) -> _Statement: + columns, params = self._value_columns(values, update=False) + placeholders = ', '.join('%s' for _ in columns) + return _Statement(f'INSERT INTO {self.table} ({", ".join(columns)}) VALUES ({placeholders})', params) + + def _update(self, values: _Row, where: _Statement) -> _Statement: + columns, params = self._value_columns(values, update=True) + assignments = ', '.join(f'{name} = %s' for name in columns) + return _Statement(f'UPDATE {self.table} SET {assignments}{where.sql}', params + where.params) + + def _delete_where(self, where: Condition | None) -> _Statement: + tail = _where_clause(where) + return _Statement(f'DELETE FROM {self.table}{tail.sql}', tail.params) + + def _lookup(self, key: _Row, fields: _Fields) -> _Statement: + return _Statement( + f'SELECT {_projection(fields)} FROM {self.table} WHERE {self.key_where} LIMIT 2', + self._key_params(key), + ) + + def _lookups(self, keys: Sequence[_Row], fields: _Fields) -> _Statement: + parts = [] + params = [] + selection = f'{self.table}.*' if fields is None else _projection(fields) + for index, key in enumerate(keys): + parts.append(f'(SELECT %s, {selection} FROM {self.table} WHERE {self.key_where} LIMIT 2)') + params.extend((index, *self._key_params(key))) + return _Statement(' UNION ALL '.join(parts), tuple(params)) + + def _identity_error(self, operation: str) -> RecordIdentityError: + return RecordIdentityError( + 'The declared MySQL key matched more than one record.', + connection_name=self.connection.name, + backend='mysql', + phase='operation', + operation=operation, + ) + + def _lookup_result(self, rows: Sequence[tuple[Any, ...]], columns: Sequence[str], fields: _Fields) -> _Row | None: + if len(rows) > 1: + raise self._identity_error('get') + return _read_records(rows, columns, fields)[0] if rows else None + + def _lookup_results( + self, keys: Sequence[_Row], rows: Sequence[tuple[Any, ...]], columns: Sequence[str], fields: _Fields + ) -> list[_Row | None]: + records: list[_Row | None] = [None] * len(keys) + duplicates = set() + for row in rows: + # Position is a generated integer literal, even if the native cursor is in raw mode. + index = int(row[0]) + if records[index] is not None: + duplicates.add(index) + else: + records[index] = _read_records((row[1:],), columns[1:], fields)[0] + if duplicates: + error = self._identity_error('get_many') + items = tuple( + BatchItem(index, key, BatchState.FAILED, error=error) + if index in duplicates + else BatchItem(index, key, BatchState.ACKNOWLEDGED, ReadItem(index, key, records[index])) + for index, key in enumerate(keys) + ) + raise ChunkError(error, ChunkResult(items)) from error + return records + + def _find( + self, + where: Condition | None, + fields: _Fields, + limit: int, + offset: int, + order_by: _Order, + ) -> _Statement: + tail = _where_clause(where) + sql = f'SELECT {_projection(fields)} FROM {self.table}{tail.sql}' + if order_by: + names, directions = zip(*order_by) + order = ', '.join( + f'{column} {"DESC" if direction == "desc" else "ASC"}' + for column, direction in zip(_columns(names), directions, strict=True) + ) + sql += f' ORDER BY {order}' + return _Statement(f'{sql} LIMIT %s OFFSET %s', (*tail.params, limit, offset)) + + def _count(self, where: Condition | None) -> _Statement: + tail = _where_clause(where) + return _Statement(f'SELECT COUNT(*) FROM {self.table}{tail.sql}', tail.params) + + def _groups(self, operation: str, items: Sequence[WriteInput]) -> Iterator[_WriteGroup]: + if operation == 'delete': + joined = ' OR '.join(f'({self.key_where})' for _ in items) + params = tuple(value for item in items for value in self._key_params(item.key)) + yield _WriteGroup(f'DELETE FROM {self.table} WHERE {joined}', params, len(items), False) + return + for statement, group in groupby(items, key=lambda item: item.prepared.sql): + params = [item.prepared.params for item in group] + yield _WriteGroup(statement, params, len(params), True) + + def _conflict(self, error: MySQLError) -> RecordConflictError: + return RecordConflictError( + 'The MySQL insert conflicts with an existing unique key.', + connection_name=self.connection.name, + backend='mysql', + phase='operation', + operation='insert', + original=error, + ) + + +class Adapter(_MySQLPlan): + """Synchronous records backed by one caller-owned connection.""" + + def __enter__(self) -> 'Adapter': + return self + + def __exit__(self, exc_type: Any, exc: BaseException | None, traceback: Any) -> None: + return None + + @contextmanager + def _cursor(self) -> Iterator[tuple[Any, Any]]: + native = self.connection.native + cursor = native.cursor(buffered=True) + try: + yield cursor, native + except BaseException as error: + try: + cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(error, (cleanup,)) + raise + else: + cursor.close() + + def _read(self, statement: _Statement) -> tuple[Sequence[tuple[Any, ...]], Sequence[str]]: + with self._cursor() as (cursor, _): + cursor.execute(statement.sql, statement.params) + return cursor.fetchall(), cursor.column_names + + def get(self, key: _Row, fields: _Fields) -> _Row | None: + rows, columns = self._read(self._lookup(key, fields)) + return self._lookup_result(rows, columns, fields) + + def get_many(self, keys: Sequence[_Row], fields: _Fields) -> list[_Row | None]: + if not keys: + return [] + statement = self._lookups(keys, fields) + observations = None + sent = False + try: + with self._cursor() as (cursor, _): + sent = True + cursor.execute(statement.sql, statement.params) + rows = cursor.fetchall() + observations = self._lookup_results(keys, rows, cursor.column_names, fields) + except BaseException as error: + if observations is not None or not sent: + _chunk_error(error, _read_chunk(keys, observations)) + raise + return observations + + def find(self, where: Condition | None, fields: _Fields, limit: int, offset: int, order_by: _Order) -> list[_Row]: + rows, columns = self._read(self._find(where, fields, limit, offset, order_by)) + return _read_records(rows, columns, fields) + + def count(self, where: Condition | None) -> int: + rows, _ = self._read(self._count(where)) + return int(rows[0][0]) + + def _write(self, operation: str, statement: _Statement) -> WriteResult: + with self._cursor() as (cursor, native): + try: + cursor.execute(statement.sql, statement.params) + except MySQLError as error: + if operation == 'insert' and _is_duplicate(error): + raise self._conflict(error) from error + raise + return _write_result(operation, cursor, native) + + def insert(self, item: WriteInput) -> WriteResult: + return self._write('insert', item.prepared) + + def update(self, item: WriteInput) -> WriteResult: + return self._write('update', item.prepared) + + def delete(self, item: WriteInput) -> WriteResult: + return self._write('delete', item.prepared) + + def update_many(self, where: Condition | None, set_values: _Row) -> WriteResult: + return self._write('update_many', self._update(set_values, _where_clause(where))) + + def delete_where(self, where: Condition | None) -> WriteResult: + return self._write('delete_where', self._delete_where(where)) + + def _batch(self, operation: str, native_operation: str, items: Sequence[WriteInput]) -> ChunkResult: + progress = _WriteProgress(operation, items) + if not items: + return progress.result() + try: + with self._cursor() as (cursor, native): + for group in self._groups(operation, items): + progress.sent += group.size + if group.many: + cursor.executemany(group.sql, group.params) + else: + cursor.execute(group.sql, group.params) + progress.complete(_write_result(native_operation, cursor, native)) + except BaseException as error: + _chunk_error(error, progress.result(error)) + raise + return progress.result() + + def insert_many(self, items: Sequence[WriteInput]) -> ChunkResult: + return self._batch('insert', 'insert_many', items) + + def bulk_update(self, items: Sequence[WriteInput]) -> ChunkResult: + return self._batch('update', 'bulk_update', items) + + def delete_many(self, items: Sequence[WriteInput]) -> ChunkResult: + return self._batch('delete', 'delete_many', items) + + +class AsyncAdapter(_MySQLPlan): + """Asynchronous records backed by one caller-owned async connection.""" + + async def __aenter__(self) -> 'AsyncAdapter': + return self + + async def __aexit__(self, exc_type: Any, exc: BaseException | None, traceback: Any) -> None: + return None + + @asynccontextmanager + async def _cursor(self) -> AsyncIterator[tuple[Any, Any]]: + native = self.connection.native + cursor = await native.cursor(buffered=True) + try: + yield cursor, native + except BaseException as error: + try: + await cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(error, (cleanup,)) + raise + else: + await cursor.close() + + async def _read(self, statement: _Statement) -> tuple[Sequence[tuple[Any, ...]], Sequence[str]]: + async with self._cursor() as (cursor, _): + await cursor.execute(statement.sql, statement.params) + return await cursor.fetchall(), cursor.column_names + + async def get(self, key: _Row, fields: _Fields) -> _Row | None: + rows, columns = await self._read(self._lookup(key, fields)) + return self._lookup_result(rows, columns, fields) + + async def get_many(self, keys: Sequence[_Row], fields: _Fields) -> list[_Row | None]: + if not keys: + return [] + statement = self._lookups(keys, fields) + observations = None + sent = False + try: + async with self._cursor() as (cursor, _): + sent = True + await cursor.execute(statement.sql, statement.params) + rows = await cursor.fetchall() + observations = self._lookup_results(keys, rows, cursor.column_names, fields) + except BaseException as error: + if observations is not None or not sent: + _chunk_error(error, _read_chunk(keys, observations)) + raise + return observations + + async def find( + self, + where: Condition | None, + fields: _Fields, + limit: int, + offset: int, + order_by: _Order, + ) -> list[_Row]: + rows, columns = await self._read(self._find(where, fields, limit, offset, order_by)) + return _read_records(rows, columns, fields) + + async def count(self, where: Condition | None) -> int: + rows, _ = await self._read(self._count(where)) + return int(rows[0][0]) + + async def _write(self, operation: str, statement: _Statement) -> WriteResult: + async with self._cursor() as (cursor, native): + try: + await cursor.execute(statement.sql, statement.params) + except MySQLError as error: + if operation == 'insert' and _is_duplicate(error): + raise self._conflict(error) from error + raise + return _write_result(operation, cursor, native) + + async def insert(self, item: WriteInput) -> WriteResult: + return await self._write('insert', item.prepared) + + async def update(self, item: WriteInput) -> WriteResult: + return await self._write('update', item.prepared) + + async def delete(self, item: WriteInput) -> WriteResult: + return await self._write('delete', item.prepared) + + async def update_many(self, where: Condition | None, set_values: _Row) -> WriteResult: + return await self._write('update_many', self._update(set_values, _where_clause(where))) + + async def delete_where(self, where: Condition | None) -> WriteResult: + return await self._write('delete_where', self._delete_where(where)) + + async def _batch(self, operation: str, native_operation: str, items: Sequence[WriteInput]) -> ChunkResult: + progress = _WriteProgress(operation, items) + if not items: + return progress.result() + try: + async with self._cursor() as (cursor, native): + for group in self._groups(operation, items): + progress.sent += group.size + if group.many: + await cursor.executemany(group.sql, group.params) + else: + await cursor.execute(group.sql, group.params) + progress.complete(_write_result(native_operation, cursor, native)) + except BaseException as error: + _chunk_error(error, progress.result(error)) + raise + return progress.result() + + async def insert_many(self, items: Sequence[WriteInput]) -> ChunkResult: + return await self._batch('insert', 'insert_many', items) + + async def bulk_update(self, items: Sequence[WriteInput]) -> ChunkResult: + return await self._batch('update', 'bulk_update', items) + + async def delete_many(self, items: Sequence[WriteInput]) -> ChunkResult: + return await self._batch('delete', 'delete_many', items) diff --git a/src/datvia/crud/_records.py b/src/datvia/crud/_records.py new file mode 100644 index 0000000000000000000000000000000000000000..18f6ed79eae32b31b5249c99e832fb31fe16bea0 --- /dev/null +++ b/src/datvia/crud/_records.py @@ -0,0 +1,574 @@ +"""Typed sync and async record operations on borrowed connection handles.""" + +import asyncio +import importlib +from collections.abc import Mapping +from collections.abc import Sequence +from contextlib import AbstractContextManager +from typing import Any +from typing import ClassVar +from typing import NoReturn +from typing import TypeVar + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DatabaseError +from ..errors import DependencyError +from ..errors import attach_cleanup_errors +from ._batch import Batch +from ._batch import Positions +from ._batch import Record +from ._batch import identity +from ._batch import unique_keys +from ._conditions import Condition +from ._conditions import Where +from ._conditions import field_name +from ._conditions import is_sequence +from ._conditions import normalize_where +from ._conditions import validate_scalar +from ._errors import BatchOperationError +from ._errors import UnsupportedOperationError +from ._types import UNSET +from ._types import BatchResult +from ._types import ChunkError +from ._types import ChunkResult +from ._types import InsertItem +from ._types import ReadItem +from ._types import WriteInput +from ._types import WriteResult +from ._types import _Unset + +_Bound = TypeVar('_Bound', bound='_Records') +_Key = Mapping[str, Any] +_Fields = Sequence[str] | None +_Order = Sequence[tuple[str, str]] +_TTL = int | None | _Unset + + +def _names(value: Sequence[str], *, empty: bool = True) -> tuple[str, ...]: + if not is_sequence(value) or (not empty and not value): + raise ConfigurationError('Expected a finite sequence of field names.', phase='config') + names = tuple(field_name(name) for name in value) + if len(set(names)) != len(names): + raise ConfigurationError('Field names must not be repeated.', phase='config') + return names + + +def _integer(value: int, minimum: int, option: str) -> int: + if type(value) is not int or value < minimum: + raise ConfigurationError(f'{option} must be an integer of at least {minimum}.', phase='config') + return value + + +def _fields(value: _Fields) -> tuple[str, ...] | None: + return None if value is None else _names(value) + + +class _Records: + _async: ClassVar[bool] = False + + def __init__(self, connection: Connection, adapter: Any, key_fields: tuple[str, ...]) -> None: + self._connection = connection + self._adapter = adapter + self._key_fields = key_fields + self._state = 'new' + + @classmethod + def mysql(cls: type[_Bound], connection: Connection, *, table: str, key_fields: Sequence[str]) -> _Bound: + """Bind a MySQL table to its declared unique key.""" + return cls._bind(connection, 'mysql', 'mysql', key_fields, table=table) + + @classmethod + def mongodb( + cls: type[_Bound], + connection: Connection, + *, + database: str, + collection: str, + key_fields: Sequence[str] = ('_id',), + session: Any = None, + ) -> _Bound: + """Bind a MongoDB collection and optionally borrow a caller-managed session.""" + return cls._bind( + connection, 'mongodb', 'mongodb', key_fields, database=database, collection=collection, session=session + ) + + @classmethod + def redis_hash( + cls: type[_Bound], + connection: Connection, + *, + namespace: str, + key_fields: Sequence[str], + codec: Any = None, + ) -> _Bound: + """Bind an explicitly named Redis Hash record namespace.""" + return cls._bind(connection, 'redis', 'redis_hash', key_fields, namespace=namespace, codec=codec) + + @classmethod + def _bind( + cls: type[_Bound], + connection: Connection, + backend: str, + adapter_name: str, + key_fields: Sequence[str], + **options: Any, + ) -> _Bound: + if not isinstance(connection, Connection) or cls._async != isinstance(connection, AsyncConnection): + raise ConfigurationError('Record binding requires a connection with the same sync mode.', phase='config') + if connection.backend != backend: + raise ConfigurationError( + 'Record binding does not match the connection backend.', + connection_name=connection.name, + backend=connection.backend, + phase='config', + ) + connection._ensure_open('operation') + names = _names(key_fields, empty=False) + try: + module = importlib.import_module(f'datvia.crud._{adapter_name}') + except ImportError as error: + raise DependencyError( + 'The CRUD adapter or its database driver is unavailable.', + connection_name=connection.name, + backend=backend, + phase='config', + original=error, + ) from error + factory = module.AsyncAdapter if cls._async else module.Adapter + adapter = factory(connection=connection, key_fields=names, **options) + return cls(connection, adapter, names) + + def _context(self, operation: str) -> AbstractContextManager: + return self._connection._context('operation', operation=operation, owned=True) + + def _error_context(self, operation: str) -> dict[str, str]: + return dict( + connection_name=self._connection.name, + backend=self._connection.backend, + phase='operation', + operation=operation, + ) + + def _check_connection(self) -> None: + if isinstance(self._connection, AsyncConnection): + self._connection._ensure_loop('operation') + self._connection._ensure_open('operation') + + def _start(self) -> None: + self._check_connection() + if self._state != 'new': + raise ConfigurationError('Record contexts can be entered only once.', phase='operation') + self._state = 'closed' + + def _check_active(self) -> None: + self._check_connection() + if self._state != 'active': + raise ConfigurationError('Record operations require an active record context.', phase='operation') + + def _key(self, key: _Key) -> Record: + if not isinstance(key, Mapping) or set(key) != set(self._key_fields): + raise ConfigurationError('A record key must contain exactly the declared key fields.', phase='config') + values = {field: key[field] for field in self._key_fields} + for value in values.values(): + validate_scalar(value) + prepare_key = getattr(self._adapter, 'prepare_key', None) + if prepare_key is not None: + prepare_key(values) + return values + + def _prepare_fields(self, fields: _Fields) -> tuple[str, ...] | None: + projected = _fields(fields) + prepare_fields = getattr(self._adapter, 'prepare_fields', None) + if prepare_fields is not None: + prepare_fields(projected) + return projected + + def _read_batch( + self, keys: Sequence[_Key], fields: _Fields, batch_size: int + ) -> tuple[Batch, list[Record], list[tuple[int, ...]], tuple[str, ...] | None]: + self._check_active() + _integer(batch_size, 1, 'batch_size') + projected = self._prepare_fields(fields) + if not is_sequence(keys): + raise ConfigurationError('Record keys must be a finite sequence.', phase='config') + prepared = [self._key(key) for key in keys] + unique, positions = unique_keys(prepared) + return Batch(prepared), unique, positions, projected + + def _where(self, where: Where, operation: str) -> Condition | None: + self._check_active() + backend = self._connection.backend + if backend == 'redis': + raise UnsupportedOperationError( + 'Redis Hash records only support keyed operations.', **self._error_context(operation) + ) + return normalize_where(where, backend, operation) + + def _find_parameters( + self, where: Where, fields: _Fields, limit: int, offset: int, order_by: _Order + ) -> tuple[Condition | None, tuple[str, ...] | None, int, int, tuple[tuple[str, str], ...]]: + condition = self._where(where, 'find') + projected = self._prepare_fields(fields) + _integer(limit, 1, 'limit') + _integer(offset, 0, 'offset') + if not is_sequence(order_by): + raise ConfigurationError('Expected a finite sequence of sort fields.', phase='config') + order = [] + for pair in order_by: + if not is_sequence(pair) or len(pair) != 2 or pair[1] not in ('asc', 'desc'): + raise ConfigurationError('Each sort must provide a field and asc or desc.', phase='config') + order.append((field_name(pair[0]), pair[1])) + _names([field for field, _ in order]) + return condition, projected, limit, offset, tuple(order) + + def _write_where(self, where: Where, operation: str, all_records: bool) -> Condition | None: + condition = self._where(where, operation) + if type(all_records) is not bool or (condition is None and not all_records): + raise ConfigurationError('A write without conditions requires all_records=True.', phase='config') + return condition + + def _values(self, value: Mapping[str, Any], *, updating: bool = False) -> Record: + if not isinstance(value, Mapping): + raise ConfigurationError('Expected a record mapping.', phase='config') + result = {field_name(field): item for field, item in value.items()} + if updating and (not result or set(result).intersection(self._key_fields)): + raise ConfigurationError('Updates require at least one non-key field.', phase='config') + return result + + def _ttl(self, ttl_s: _TTL, operation: str) -> _TTL: + if ttl_s is UNSET: + return UNSET + if self._connection.backend != 'redis': + raise UnsupportedOperationError('Record TTL requires Redis Hash records.', **self._error_context(operation)) + if ttl_s is not None: + _integer(ttl_s, 1, 'ttl_s') + return ttl_s + + def _write_input( + self, + operation: str, + *, + key: _Key | None = None, + values: Mapping[str, Any] | None = None, + ttl_s: _TTL = UNSET, + ) -> WriteInput: + self._check_active() + ttl_s = self._ttl(ttl_s, operation) + if operation == 'insert': + values = self._values(values) + supplied = {field: values[field] for field in self._key_fields if field in values} + for value in supplied.values(): + validate_scalar(value) + if len(supplied) == len(self._key_fields): + key = self._key(supplied) + elif self._connection.backend == 'redis': + raise ConfigurationError('Redis Hash inserts require all key fields.', phase='config') + else: + key = self._key(key) + values = self._values(values, updating=True) if operation == 'update' else {} + item = WriteInput(key, values, ttl_s) + prepare_write = getattr(self._adapter, 'prepare_write', None) + return item if prepare_write is None else prepare_write(operation, item) + + def _insert_item(self, value: Mapping[str, Any] | InsertItem) -> WriteInput: + if isinstance(value, InsertItem): + return self._write_input('insert', values=value.record, ttl_s=value.ttl_s) + return self._write_input('insert', values=value) + + def _update_item(self, value: Mapping[str, Any]) -> WriteInput: + if ( + not isinstance(value, Mapping) + or not {'key', 'set_values'}.issubset(value) + or not set(value).issubset({'key', 'set_values', 'ttl_s'}) + ): + raise ConfigurationError('Bulk updates require key, set_values and optional ttl_s.', phase='config') + return self._write_input( + 'update', key=value['key'], values=value['set_values'], ttl_s=value.get('ttl_s', UNSET) + ) + + def _write_batch(self, operation: str, values: Sequence[Any], batch_size: int) -> tuple[Batch, list[WriteInput]]: + self._check_active() + _integer(batch_size, 1, 'batch_size') + if not is_sequence(values): + raise ConfigurationError('Batch writes require a finite sequence.', phase='config') + if operation == 'insert_many': + items = [self._insert_item(value) for value in values] + elif operation == 'bulk_update': + items = [self._update_item(value) for value in values] + else: + items = [self._write_input('delete', key=value) for value in values] + seen = set() + for item in items: + if item.key is None: + continue + token = identity(item.key) + if token in seen: + raise ConfigurationError('The same record key appears more than once in a write batch.', phase='config') + seen.add(token) + return Batch([item.key for item in items]), items + + def _batch_failed( + self, operation: str, error: BaseException, batch: Batch, positions: Positions, *, dispatched: bool + ) -> NoReturn: + if isinstance(error, ChunkError): + chunk, source = error.partial_result, error.original + else: + chunk = getattr(error, 'database_partial_result', None) + source = error + if dispatched: + batch.absorb(positions, chunk if isinstance(chunk, ChunkResult) else None) + partial = batch.result(operation) + if isinstance(source, asyncio.CancelledError): + BaseException.__dict__['__dict__'].__get__(source)['database_partial_result'] = partial + if source is not error: + attach_cleanup_errors(source, getattr(error, 'database_cleanup_errors', ())) + context_descriptor = BaseException.__dict__['__context__'] + context = context_descriptor.__get__(source) + try: + raise source + finally: + # The caller may already be handling an unrelated exception. + context_descriptor.__set__(source, context) + original = source.original if isinstance(source, DatabaseError) and source.original is not None else source + wrapped = BatchOperationError( + 'A database record batch failed.', + partial_result=partial, + original=original, + **self._error_context(operation), + ) + for failure in {id(value): value for value in (original, source, error)}.values(): + attach_cleanup_errors(wrapped, getattr(failure, 'database_cleanup_errors', ())) + raise wrapped from original + + +class Records(_Records): + """Synchronous operations within a borrowed connection's record context.""" + + def __enter__(self) -> 'Records': + self._start() + with self._context('open_records'): + self._adapter.__enter__() + self._state = 'active' + return self + + def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: Any) -> bool: + if self._state != 'active': + return False + try: + self._state = 'closed' + with self._context('close_records'): + self._adapter.__exit__(exc_type, exc, traceback) + except BaseException as cleanup_error: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup_error,)) + return False + + def _invoke(self, operation: str, *args: Any) -> Any: + self._check_active() + with self._context(operation): + return getattr(self._adapter, operation)(*args) + + def get(self, key: _Key, *, fields: _Fields = None) -> Record | None: + self._check_active() + return self._invoke('get', self._key(key), self._prepare_fields(fields)) + + def get_many(self, keys: Sequence[_Key], *, fields: _Fields = None, batch_size: int = 500) -> list[ReadItem]: + batch, unique, positions, projected = self._read_batch(keys, fields, batch_size) + for start in range(0, len(unique), batch_size): + end = start + batch_size + group = positions[start:end] + dispatched = False + try: + self._check_active() + with self._context('get_many'): + read = self._adapter.get_many + dispatched = True + rows = read(unique[start:end], projected) + batch.absorb_reads(group, rows) + except (Exception, asyncio.CancelledError) as error: + failure = error + else: + continue + self._batch_failed('get_many', failure, batch, group, dispatched=dispatched) + return batch.read_items() + + def find( + self, + *, + where: Where = None, + fields: _Fields = None, + limit: int, + offset: int = 0, + order_by: _Order = (), + ) -> list[Record]: + return self._invoke('find', *self._find_parameters(where, fields, limit, offset, order_by)) + + def count(self, *, where: Where = None) -> int: + return self._invoke('count', self._where(where, 'count')) + + def insert(self, record: Mapping[str, Any], *, ttl_s: _TTL = UNSET) -> WriteResult: + return self._invoke('insert', self._write_input('insert', values=record, ttl_s=ttl_s)) + + def update(self, key: _Key, *, set_values: Mapping[str, Any], ttl_s: _TTL = UNSET) -> WriteResult: + return self._invoke('update', self._write_input('update', key=key, values=set_values, ttl_s=ttl_s)) + + def delete(self, key: _Key) -> WriteResult: + return self._invoke('delete', self._write_input('delete', key=key)) + + def update_many(self, *, where: Where, set_values: Mapping[str, Any], all_records: bool = False) -> WriteResult: + condition = self._write_where(where, 'update_many', all_records) + return self._invoke('update_many', condition, self._values(set_values, updating=True)) + + def delete_where(self, *, where: Where, all_records: bool = False) -> WriteResult: + return self._invoke('delete_where', self._write_where(where, 'delete_where', all_records)) + + def insert_many(self, records: Sequence[Mapping[str, Any] | InsertItem], *, batch_size: int = 500) -> BatchResult: + return self._execute_batch('insert_many', records, batch_size) + + def bulk_update(self, items: Sequence[Mapping[str, Any]], *, batch_size: int = 500) -> BatchResult: + return self._execute_batch('bulk_update', items, batch_size) + + def delete_many(self, keys: Sequence[_Key], *, batch_size: int = 500) -> BatchResult: + return self._execute_batch('delete_many', keys, batch_size) + + def _execute_batch(self, operation: str, values: Sequence[Any], batch_size: int) -> BatchResult: + batch, items = self._write_batch(operation, values, batch_size) + for start in range(0, len(items), batch_size): + end = min(start + batch_size, len(items)) + positions = [(index,) for index in range(start, end)] + dispatched = False + try: + self._check_active() + with self._context(operation): + write = getattr(self._adapter, operation) + dispatched = True + chunk = write(items[start:end]) + batch.absorb(positions, chunk) + except (Exception, asyncio.CancelledError) as error: + failure = error + else: + continue + self._batch_failed(operation, failure, batch, positions, dispatched=dispatched) + return batch.result(operation) + + +class AsyncRecords(_Records): + """Asynchronous operations within a borrowed connection's record context.""" + + _async = True + + async def __aenter__(self) -> 'AsyncRecords': + self._start() + with self._context('open_records'): + await self._adapter.__aenter__() + self._state = 'active' + return self + + async def __aexit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: Any) -> bool: + if self._state != 'active': + return False + try: + self._connection._ensure_loop('operation') + self._state = 'closed' + with self._context('close_records'): + await self._adapter.__aexit__(exc_type, exc, traceback) + except BaseException as cleanup_error: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup_error,)) + return False + + async def _invoke(self, operation: str, *args: Any) -> Any: + self._check_active() + with self._context(operation): + return await getattr(self._adapter, operation)(*args) + + async def get(self, key: _Key, *, fields: _Fields = None) -> Record | None: + self._check_active() + return await self._invoke('get', self._key(key), self._prepare_fields(fields)) + + async def get_many(self, keys: Sequence[_Key], *, fields: _Fields = None, batch_size: int = 500) -> list[ReadItem]: + batch, unique, positions, projected = self._read_batch(keys, fields, batch_size) + for start in range(0, len(unique), batch_size): + end = start + batch_size + group = positions[start:end] + dispatched = False + try: + self._check_active() + with self._context('get_many'): + read = self._adapter.get_many + dispatched = True + rows = await read(unique[start:end], projected) + batch.absorb_reads(group, rows) + except (Exception, asyncio.CancelledError) as error: + failure = error + else: + continue + self._batch_failed('get_many', failure, batch, group, dispatched=dispatched) + return batch.read_items() + + async def find( + self, + *, + where: Where = None, + fields: _Fields = None, + limit: int, + offset: int = 0, + order_by: _Order = (), + ) -> list[Record]: + return await self._invoke('find', *self._find_parameters(where, fields, limit, offset, order_by)) + + async def count(self, *, where: Where = None) -> int: + return await self._invoke('count', self._where(where, 'count')) + + async def insert(self, record: Mapping[str, Any], *, ttl_s: _TTL = UNSET) -> WriteResult: + return await self._invoke('insert', self._write_input('insert', values=record, ttl_s=ttl_s)) + + async def update(self, key: _Key, *, set_values: Mapping[str, Any], ttl_s: _TTL = UNSET) -> WriteResult: + return await self._invoke('update', self._write_input('update', key=key, values=set_values, ttl_s=ttl_s)) + + async def delete(self, key: _Key) -> WriteResult: + return await self._invoke('delete', self._write_input('delete', key=key)) + + async def update_many( + self, *, where: Where, set_values: Mapping[str, Any], all_records: bool = False + ) -> WriteResult: + condition = self._write_where(where, 'update_many', all_records) + return await self._invoke('update_many', condition, self._values(set_values, updating=True)) + + async def delete_where(self, *, where: Where, all_records: bool = False) -> WriteResult: + return await self._invoke('delete_where', self._write_where(where, 'delete_where', all_records)) + + async def insert_many( + self, records: Sequence[Mapping[str, Any] | InsertItem], *, batch_size: int = 500 + ) -> BatchResult: + return await self._execute_batch('insert_many', records, batch_size) + + async def bulk_update(self, items: Sequence[Mapping[str, Any]], *, batch_size: int = 500) -> BatchResult: + return await self._execute_batch('bulk_update', items, batch_size) + + async def delete_many(self, keys: Sequence[_Key], *, batch_size: int = 500) -> BatchResult: + return await self._execute_batch('delete_many', keys, batch_size) + + async def _execute_batch(self, operation: str, values: Sequence[Any], batch_size: int) -> BatchResult: + batch, items = self._write_batch(operation, values, batch_size) + for start in range(0, len(items), batch_size): + end = min(start + batch_size, len(items)) + positions = [(index,) for index in range(start, end)] + dispatched = False + try: + self._check_active() + with self._context(operation): + write = getattr(self._adapter, operation) + dispatched = True + chunk = await write(items[start:end]) + batch.absorb(positions, chunk) + except (Exception, asyncio.CancelledError) as error: + failure = error + else: + continue + self._batch_failed(operation, failure, batch, positions, dispatched=dispatched) + return batch.result(operation) diff --git a/src/datvia/crud/_redis_hash.py b/src/datvia/crud/_redis_hash.py new file mode 100644 index 0000000000000000000000000000000000000000..5dea28bbe2d88ded029f94961317bff48b9288e2 --- /dev/null +++ b/src/datvia/crud/_redis_hash.py @@ -0,0 +1,508 @@ +"""Atomic single-key CRUD for managed Redis Hash records.""" + +import json +from asyncio import CancelledError +from base64 import urlsafe_b64encode +from contextlib import asynccontextmanager +from contextlib import contextmanager +from dataclasses import replace +from typing import Any + +from redis import exceptions + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DriverError +from ..errors import attach_cleanup_errors +from ._codecs import FieldCodec +from ._errors import RecordConflictError +from ._errors import RecordFormatError +from ._types import UNSET +from ._types import BatchItem +from ._types import BatchState +from ._types import ChunkError +from ._types import ChunkResult +from ._types import ReadItem +from ._types import WriteCounts +from ._types import WriteInput +from ._types import WriteResult + +_MARKER = '__database_record__' +_MAX_TTL_SECONDS = 2**53 - 1 + +# Replies begin with 0 (missing), 1 (success), 2 (wrong format), or 3 (conflict). +# HMGET retains false placeholders in Lua (converted back to null by Redis). +_SCRIPT = """ +local key = KEYS[1] +local operation = ARGV[1] +local qualifier = ARGV[2] +local marker = '__database_record__' + +if redis.call('EXISTS', key) == 0 then + if operation ~= 'insert' then + return {0} + end +else + if redis.call('HGET', key, marker) ~= '1' then + return {2} + end + if operation == 'insert' then + return {3} + end +end + +if operation == 'get' then + if qualifier == 'all' then + return {1, redis.call('HGETALL', key)} + end + local values = {} + for index = 3, #ARGV, 512 do + local last = math.min(index + 511, #ARGV) + local page = redis.call('HMGET', key, unpack(ARGV, index, last)) + for _, value in ipairs(page) do + values[#values + 1] = value + end + end + return {1, values} +end + +if operation == 'delete' then + return {redis.call('DEL', key)} +end + +if operation == 'insert' then + redis.call('HSET', key, marker, '1') +end +-- Limit the Lua unpack size for wide records while keeping one atomic script. +for index = 3, #ARGV, 512 do + local last = math.min(index + 511, #ARGV) + redis.call('HSET', key, unpack(ARGV, index, last)) +end +if qualifier == 'persist' then + redis.call('PERSIST', key) +elseif qualifier ~= 'keep' then + redis.call('EXPIRE', key, qualifier) +end +return {1} +""" + + +def _key_segment(value: bytes) -> bytes: + return urlsafe_b64encode(value).rstrip(b'=') + + +def _queue(pipe, commands) -> None: + # The public eval() method on synchronous ClusterPipeline is a stub. + for command in commands: + pipe.execute_command('EVAL', _SCRIPT, 1, *command) + + +def _pending_chunk(keys, sent: bool) -> ChunkResult: + state = BatchState.UNKNOWN if sent else BatchState.NOT_SENT + return ChunkResult(tuple(BatchItem(index, key, state) for index, key in enumerate(keys))) + + +class _HashRecords: + def __init__( + self, + *, + connection: Connection | AsyncConnection, + key_fields: tuple[str, ...], + namespace: str, + codec: FieldCodec, + ) -> None: + self._connection = connection + self._key_fields = key_fields + self._codec = codec + self._client = None + self._sentinel = connection.mode == 'sentinel' + if not isinstance(namespace, str) or not namespace.strip(): + raise self._error(ConfigurationError, 'Redis Hash namespace must be nonempty.', phase='config') + if _MARKER in key_fields: + raise self._error(ConfigurationError, 'A Redis Hash key field uses the reserved marker.', phase='config') + if not callable(getattr(codec, 'encode', None)) or not callable(getattr(codec, 'decode', None)): + raise self._error(ConfigurationError, 'Redis Hash requires an encode/decode field codec.', phase='config') + try: + self._prefix = b'database:hash:v1:' + _key_segment(namespace.encode('utf-8')) + b':' + except UnicodeError as original: + raise self._error( + ConfigurationError, 'Redis Hash namespace must be valid UTF-8.', phase='config', original=original + ) from original + + def _error(self, kind, message, *, operation=None, phase='operation', original=None): + return kind( + message, + connection_name=self._connection.name, + backend='redis', + phase=phase, + operation=operation, + original=original, + ) + + def _open_client(self) -> None: + native = self._connection.native + if self._sentinel: + service_name = self._connection._adapter._config.service_name + native = native.master_for(service_name) + self._client = native + + def prepare_key(self, key: dict[str, Any]) -> bytes: + """Return this record's canonical Redis key.""" + identity = [[name, key[name]] for name in self._key_fields] + if any(type(value) not in (str, int) for _, value in identity): + raise self._error(ConfigurationError, 'Redis Hash key values must be strings or integers.', phase='config') + try: + encoded = json.dumps(identity, ensure_ascii=True, separators=(',', ':')).encode('ascii') + except ValueError as original: + raise self._error( + ConfigurationError, 'Could not encode the Redis Hash key.', phase='config', original=original + ) from original + return self._prefix + _key_segment(encoded) + + def _field_bytes(self, name: str) -> bytes: + if name == _MARKER: + raise self._error(ConfigurationError, 'The Redis Hash marker is reserved.', phase='config') + try: + return name.encode('utf-8') + except UnicodeError as original: + raise self._error( + ConfigurationError, 'Redis Hash field names must be valid UTF-8.', phase='config', original=original + ) from original + + def prepare_fields(self, fields: tuple[str, ...] | None) -> None: + """Reject invalid projection names before a read batch is dispatched.""" + if fields is not None: + for name in fields: + self._field_bytes(name) + + def prepare_write(self, operation: str, item: WriteInput) -> WriteInput: + """Validate and encode input before any chunk is sent.""" + if item.key is None: + raise self._error(ConfigurationError, 'Redis Hash inserts require a complete key.', phase='config') + key = self.prepare_key(item.key) + if operation == 'delete': + return replace(item, prepared=(key, operation, b'keep')) + ttl_s = item.ttl_s + if ttl_s is UNSET: + ttl = b'keep' + elif ttl_s is None: + ttl = b'keep' if operation == 'insert' else b'persist' + elif type(ttl_s) is int and 1 <= ttl_s <= _MAX_TTL_SECONDS: + # Pass integer seconds verbatim to Redis without a Lua float conversion. + ttl = str(ttl_s).encode('ascii') + else: + raise self._error( + ConfigurationError, + 'Redis Hash ttl_s must be a positive integer within the Redis expiration range.', + phase='config', + ) + command = [key, operation, ttl] + for name, value in item.values.items(): + field = self._field_bytes(name) + try: + data = self._codec.encode(value) + if isinstance(data, str): + data = data.encode('utf-8') + if not isinstance(data, bytes): + raise TypeError('Expected encoded field data as bytes or str.') + except Exception as original: + raise self._error( + ConfigurationError, 'Could not encode a Redis Hash field.', phase='config', original=original + ) from original + command.extend((field, data)) + return replace(item, prepared=tuple(command)) + + def _write_command(self, operation: str, item: WriteInput) -> tuple: + return item.prepared if item.prepared is not None else self.prepare_write(operation, item).prepared + + def _read_command(self, key, fields) -> tuple: + encoded_key = self.prepare_key(key) + if fields is None: + return encoded_key, 'get', b'all' + return (encoded_key, 'get', b'fields', *(self._field_bytes(name) for name in fields)) + + def _unexpected_reply(self, operation): + return self._error(DriverError, 'Unexpected Redis Hash script reply.', operation=operation) + + def _status(self, reply, operation) -> int: + if not isinstance(reply, (list, tuple)) or not reply or type(reply[0]) is not int: + raise self._unexpected_reply(operation) + if reply[0] == 2: + raise self._error(RecordFormatError, 'Invalid Redis Hash record marker.', operation=operation) + if reply[0] == 3 and operation == 'insert': + raise self._error(RecordConflictError, 'Redis Hash record already exists.', operation=operation) + if reply[0] not in (0, 1): + raise self._unexpected_reply(operation) + return reply[0] + + def _read_reply(self, reply, fields, operation) -> dict[str, Any] | None: + if self._status(reply, operation) == 0: + if len(reply) != 1: + raise self._unexpected_reply(operation) + return None + if len(reply) != 2 or not isinstance(reply[1], (list, tuple)): + raise self._unexpected_reply(operation) + values = reply[1] + if fields is None: + if len(values) % 2: + raise self._unexpected_reply(operation) + pairs = zip(values[::2], values[1::2]) + else: + if len(values) != len(fields): + raise self._unexpected_reply(operation) + pairs = zip(fields, values) + + try: + record = {} + for field, value in pairs: + if isinstance(field, bytes): + field = field.decode('utf-8') + if field != _MARKER and value is not None: + record[field] = self._codec.decode(value) + return record + except Exception as original: + raise self._error( + RecordFormatError, 'Could not decode a Redis Hash field.', operation=operation, original=original + ) from original + + def _write_reply(self, reply, operation) -> WriteResult: + status = self._status(reply, operation) + if len(reply) != 1 or (operation == 'insert' and status == 0): + raise self._unexpected_reply(operation) + return WriteResult( + 'redis', + operation, + acknowledged=True, + inserted_count=status if operation == 'insert' else None, + matched_count=status if operation == 'update' else None, + modified_count=0 if operation == 'update' and status == 0 else None, + deleted_count=status if operation == 'delete' else None, + native_result=reply, + ) + + def _batch_reply(self, replies, keys, operation, fields) -> ChunkResult: + if not isinstance(replies, (list, tuple)): + raise ChunkError(self._unexpected_reply(operation), _pending_chunk(keys, True)) + first_error = None + items = [] + for index, key in enumerate(keys): + if index >= len(replies): + item = BatchItem(index, key, BatchState.UNKNOWN, error=self._unexpected_reply(operation)) + else: + item = self._batch_item(replies[index], index, key, operation, fields) + if first_error is None or ( + isinstance(item.error, CancelledError) and not isinstance(first_error, CancelledError) + ): + first_error = item.error + items.append(item) + + if first_error is None and len(replies) != len(keys): + first_error = self._unexpected_reply(operation) + result = ChunkResult( + tuple(items), self._counts(items) if first_error is None else WriteCounts(), native_results=(replies,) + ) + if first_error is not None: + raise ChunkError(first_error, result) from first_error + return result + + def _batch_item(self, reply, index, key, operation, fields) -> BatchItem: + if isinstance(reply, BaseException): + # Only an explicit negative server reply proves failure. Transport, + # routing and other driver failures may follow successful execution. + state = BatchState.UNKNOWN + if isinstance( + reply, (exceptions.ResponseError, exceptions.AuthenticationError, exceptions.AuthorizationError) + ): + state = BatchState.FAILED + return BatchItem(index, key, state, error=reply) + try: + if operation == 'get_many': + result = ReadItem(index, key, self._read_reply(reply, fields, operation)) + else: + result = self._write_reply(reply, operation) + except (RecordConflictError, RecordFormatError) as error: + return BatchItem(index, key, BatchState.FAILED, error=error) + except DriverError as error: + return BatchItem(index, key, BatchState.UNKNOWN, error=error) + return BatchItem(index, key, BatchState.ACKNOWLEDGED, result) + + @staticmethod + def _counts(items: list[BatchItem]) -> WriteCounts: + if not items or not isinstance(items[0].result, WriteResult): + return WriteCounts() + totals = {} + for field in ('inserted_count', 'matched_count', 'modified_count', 'deleted_count'): + values = [getattr(item.result, field) for item in items] + if all(value is not None for value in values): + totals[field] = sum(values) + return WriteCounts(**totals) + + @staticmethod + def _cleanup_primary(primary, cleanup) -> None: + if primary is None: + raise cleanup + original = primary.original if isinstance(primary, ChunkError) else primary + attach_cleanup_errors(original, (cleanup,)) + + +class Adapter(_HashRecords): + """Synchronous Redis Hash adapter.""" + + def __enter__(self): + self._open_client() + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + client, self._client = self._client, None + if self._sentinel: + adapter = self._connection._adapter + errors = adapter._close_single(client, True, target='sentinel_master') + adapter._finish_cleanup(errors, exc) + + def get(self, key, fields): + reply = self._client.execute_command('EVAL', _SCRIPT, 1, *self._read_command(key, fields)) + return self._read_reply(reply, fields, 'get') + + def insert(self, item: WriteInput) -> WriteResult: + return self._write('insert', item) + + def update(self, item: WriteInput) -> WriteResult: + return self._write('update', item) + + def delete(self, item: WriteInput) -> WriteResult: + return self._write('delete', item) + + def _write(self, operation: str, item: WriteInput) -> WriteResult: + reply = self._client.execute_command('EVAL', _SCRIPT, 1, *self._write_command(operation, item)) + return self._write_reply(reply, operation) + + def get_many(self, keys, fields) -> list[dict[str, Any] | None]: + commands = [self._read_command(key, fields) for key in keys] + result = self._run_batch(commands, keys, 'get_many', fields) + return [item.result.record for item in result.items] + + def insert_many(self, items) -> ChunkResult: + return self._write_many('insert', items) + + def bulk_update(self, items) -> ChunkResult: + return self._write_many('update', items) + + def delete_many(self, items) -> ChunkResult: + return self._write_many('delete', items) + + def _write_many(self, operation, items) -> ChunkResult: + commands = [self._write_command(operation, item) for item in items] + return self._run_batch(commands, [item.key for item in items], operation, None) + + @contextmanager + def _pipeline(self, operation): + pipeline = self._client.pipeline(transaction=False) + primary = None + try: + yield pipeline + except BaseException as error: + primary = error + raise + finally: + try: + with self._connection._context('close', operation=operation): + pipeline.__exit__(None, None, None) + except BaseException as cleanup: + self._cleanup_primary(primary, cleanup) + + def _run_batch(self, commands, keys, operation, fields) -> ChunkResult: + sent = False + result = None + try: + with self._pipeline(operation) as pipe: + _queue(pipe, commands) + sent = True + result = self._batch_reply(pipe.execute(raise_on_error=False), keys, operation, fields) + except ChunkError: + raise + except (Exception, CancelledError) as original: + raise ChunkError(original, result or _pending_chunk(keys, sent)) from original + return result + + +class AsyncAdapter(_HashRecords): + """Asynchronous Redis Hash adapter.""" + + async def __aenter__(self): + self._open_client() + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + client, self._client = self._client, None + if self._sentinel: + adapter = self._connection._adapter + errors = await adapter._close_single_async(client, True, target='sentinel_master') + adapter._finish_cleanup(errors, exc) + + async def get(self, key, fields): + reply = await self._client.execute_command('EVAL', _SCRIPT, 1, *self._read_command(key, fields)) + return self._read_reply(reply, fields, 'get') + + async def insert(self, item: WriteInput) -> WriteResult: + return await self._write('insert', item) + + async def update(self, item: WriteInput) -> WriteResult: + return await self._write('update', item) + + async def delete(self, item: WriteInput) -> WriteResult: + return await self._write('delete', item) + + async def _write(self, operation: str, item: WriteInput) -> WriteResult: + reply = await self._client.execute_command('EVAL', _SCRIPT, 1, *self._write_command(operation, item)) + return self._write_reply(reply, operation) + + async def get_many(self, keys, fields) -> list[dict[str, Any] | None]: + commands = [self._read_command(key, fields) for key in keys] + result = await self._run_batch(commands, keys, 'get_many', fields) + return [item.result.record for item in result.items] + + async def insert_many(self, items) -> ChunkResult: + return await self._write_many('insert', items) + + async def bulk_update(self, items) -> ChunkResult: + return await self._write_many('update', items) + + async def delete_many(self, items) -> ChunkResult: + return await self._write_many('delete', items) + + async def _write_many(self, operation, items) -> ChunkResult: + commands = [self._write_command(operation, item) for item in items] + return await self._run_batch(commands, [item.key for item in items], operation, None) + + @asynccontextmanager + async def _pipeline(self, operation): + pipeline = self._client.pipeline(transaction=False) + primary = None + try: + yield pipeline + except BaseException as error: + primary = error + raise + finally: + try: + with self._connection._context('close', operation=operation): + # Avoid extra initialization from __aenter__. redis-py 5.0's + # async ClusterPipeline.reset is the Redis RESET command. + await pipeline.__aexit__(None, None, None) + except BaseException as cleanup: + self._cleanup_primary(primary, cleanup) + + async def _run_batch(self, commands, keys, operation, fields) -> ChunkResult: + sent = False + result = None + try: + async with self._pipeline(operation) as pipe: + _queue(pipe, commands) + sent = True + result = self._batch_reply(await pipe.execute(raise_on_error=False), keys, operation, fields) + except ChunkError: + raise + except (Exception, CancelledError) as original: + raise ChunkError(original, result or _pending_chunk(keys, sent)) from original + return result diff --git a/src/datvia/crud/_string_types.py b/src/datvia/crud/_string_types.py new file mode 100644 index 0000000000000000000000000000000000000000..7975260bb2cc4b77865c44d0023eaaa707b5aee9 --- /dev/null +++ b/src/datvia/crud/_string_types.py @@ -0,0 +1,39 @@ +"""Redis String inputs and evidence-based batch results.""" + +from dataclasses import dataclass +from typing import Any + +from ._types import BatchState + +StringKey = str | bytes +StringValue = str | bytes + + +@dataclass(frozen=True) +class StringSetItem: + """One SET, with an optional positive expiration in seconds.""" + + key: StringKey + value: StringValue + ttl_s: int | None = None + + +@dataclass(frozen=True) +class StringBatchItem: + """A reply or uncertain execution state at one original input position.""" + + index: int + key: StringKey + state: BatchState + result: str | bytes | bool | int | None = None + error: Any = None + + +@dataclass(frozen=True) +class StringBatchResult: + """Per-key evidence without inferred inserted or modified record counts.""" + + requested_count: int + items: tuple[StringBatchItem, ...] + deleted_count: int | None = None + native_results: tuple[Any, ...] = () diff --git a/src/datvia/crud/_strings.py b/src/datvia/crud/_strings.py new file mode 100644 index 0000000000000000000000000000000000000000..0afcc7b34d7df2175f793be2fa37cf81afca706d --- /dev/null +++ b/src/datvia/crud/_strings.py @@ -0,0 +1,503 @@ +"""Redis String operations on a managed connection's original keys.""" + +from asyncio import CancelledError +from collections.abc import Iterator +from collections.abc import Sequence +from contextlib import AbstractContextManager +from dataclasses import dataclass +from dataclasses import replace +from typing import Any +from typing import ClassVar +from typing import NoReturn +from typing import cast + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DatabaseError +from ..errors import DriverError +from ..errors import attach_cleanup_errors +from ._conditions import is_sequence +from ._errors import BatchOperationError +from ._string_types import StringBatchItem +from ._string_types import StringBatchResult +from ._string_types import StringKey +from ._string_types import StringSetItem +from ._string_types import StringValue +from ._types import BatchState + +# Redis represents expiration in signed 64-bit milliseconds. +_MAX_TTL_SECONDS = (2**63 - 1) // 1000 +_Reply = StringValue | bool | int | None + + +@dataclass(frozen=True) +class _Request: + key: StringKey + command: tuple[str | bytes | int, ...] + + +class _Batch: + """Keep proof of each dispatched command in caller input order.""" + + def __init__(self, requests: Sequence[_Request]) -> None: + self.items = [ + StringBatchItem(index, request.key, BatchState.NOT_SENT) for index, request in enumerate(requests) + ] + self.native_results: list[Any] = [] + + def dispatch(self, start: int, size: int) -> None: + for index in range(start, start + size): + self.items[index] = replace(self.items[index], state=BatchState.UNKNOWN) + + def record( + self, index: int, state: BatchState, *, result: _Reply = None, error: BaseException | None = None + ) -> None: + self.items[index] = replace(self.items[index], state=state, result=result, error=error) + + def result(self, operation: str) -> StringBatchResult: + deleted_count = None + if operation == 'delete_many': + sent = [item for item in self.items if item.state != BatchState.NOT_SENT] + if all(item.state == BatchState.ACKNOWLEDGED for item in sent): + deleted_count = sum(cast(int, item.result) for item in sent) + return StringBatchResult(len(self.items), tuple(self.items), deleted_count, tuple(self.native_results)) + + +class _Strings: + _async: ClassVar[bool] = False + + def __init__(self, connection: Connection) -> None: + if not isinstance(connection, Connection) or isinstance(connection, AsyncConnection) != self._async: + raise ConfigurationError( + 'Redis String operations require a connection with the same sync mode.', phase='config' + ) + if connection.backend != 'redis': + raise ConfigurationError( + 'Redis String operations require a Redis connection.', + backend=connection.backend, + connection_name=connection.name, + phase='config', + ) + self._connection = connection + self._sentinel = connection.mode == 'sentinel' + self._client: Any = None + self._entered = False + + def _error( + self, + kind: type[DatabaseError], + message: str, + operation: str, + *, + phase: str = 'operation', + original: BaseException | None = None, + ) -> DatabaseError: + return kind( + message, + connection_name=self._connection.name, + backend='redis', + phase=phase, + operation=operation, + original=original, + ) + + def _context(self, operation: str, *, phase: str = 'operation') -> AbstractContextManager: + return self._connection._context(phase, operation=operation) + + def _check_loop(self, phase: str) -> None: + if isinstance(self._connection, AsyncConnection): + self._connection._ensure_loop(phase) + + def _check_client(self, operation: str) -> None: + self._check_loop('operation') + self._connection._ensure_open('operation') + if self._client is None: + raise self._error(ConfigurationError, 'Redis String operations require an active context.', operation) + + def _open(self) -> None: + operation = 'open_strings' + self._check_loop('operation') + native = self._connection.native + if self._entered: + raise self._error(ConfigurationError, 'Redis String contexts can only be entered once.', operation) + self._entered = True + with self._context(operation): + if self._sentinel: + config = self._connection._adapter._config + native = native.master_for(config.service_name) + self._client = native + + def _encode(self, value: StringKey | StringValue, operation: str) -> bytes: + if not isinstance(value, (str, bytes)): + raise self._error( + ConfigurationError, 'Redis String keys and values require str or bytes.', operation, phase='config' + ) + try: + return self._client.get_encoder().encode(value) + except (LookupError, UnicodeError) as original: + raise self._error( + ConfigurationError, + 'Could not encode a Redis String key or value.', + operation, + phase='config', + original=original, + ) from original + + def _seconds(self, value: int, operation: str) -> int: + if type(value) is not int or not 1 <= value <= _MAX_TTL_SECONDS: + raise self._error( + ConfigurationError, + 'Redis TTL requires representable positive integer seconds.', + operation, + phase='config', + ) + return value + + def _key_request(self, operation: str, command: str, key: StringKey, *args: int) -> _Request: + return _Request(key, (command, self._encode(key, operation), *args)) + + def _set_request(self, operation: str, item: StringSetItem) -> _Request: + key = self._encode(item.key, operation) + value = self._encode(item.value, operation) + ttl = () if item.ttl_s is None else ('EX', self._seconds(item.ttl_s, operation)) + return _Request(item.key, ('SET', key, value, *ttl)) + + def _parse(self, command: str, value: Any, operation: str) -> _Reply: + if command == 'GET' and (value is None or isinstance(value, (str, bytes))): + return value + if command == 'SET' and value is True: + return True + if command == 'DEL' and type(value) is int and value in (0, 1): + return value + if command == 'TTL' and type(value) is int and value >= -2: + return value + if command in ('EXISTS', 'EXPIRE', 'PERSIST') and type(value) in (int, bool) and value in (0, 1): + return bool(value) + raise self._error(DriverError, 'Redis returned an unexpected single-key reply.', operation) + + def _prepare_batch( + self, operation: str, inputs: Sequence[StringKey] | Sequence[StringSetItem], batch_size: int + ) -> list[_Request]: + self._check_client(operation) + if type(batch_size) is not int or batch_size < 1: + raise self._error(ConfigurationError, 'Batch size must be a positive integer.', operation, phase='config') + if not is_sequence(inputs): + raise self._error( + ConfigurationError, 'Redis String batches require a finite input sequence.', operation, phase='config' + ) + + requests = [] + seen: set[bytes] = set() + for item in inputs: + if operation == 'set_many': + if not isinstance(item, StringSetItem): + raise self._error( + ConfigurationError, 'set_many requires StringSetItem inputs.', operation, phase='config' + ) + request = self._set_request(operation, item) + else: + command = 'GET' if operation == 'get_many' else 'DEL' + request = self._key_request(operation, command, item) + if operation != 'get_many': + wire_key = cast(bytes, request.command[1]) + if wire_key in seen: + raise self._error( + ConfigurationError, + 'The same encoded Redis key appears more than once in this batch.', + operation, + phase='config', + ) + seen.add(wire_key) + requests.append(request) + return requests + + @staticmethod + def _chunks(requests: Sequence[_Request], batch_size: int) -> Iterator[tuple[int, Sequence[_Request]]]: + for start in range(0, len(requests), batch_size): + yield start, requests[start : start + batch_size] + + def _consume_replies( + self, batch: _Batch, start: int, requests: Sequence[_Request], replies: Any, operation: str + ) -> BaseException | None: + from redis import exceptions + + batch.native_results.append(replies) + if not isinstance(replies, (list, tuple)): + return self._error(DriverError, 'Redis returned an unexpected batch reply.', operation) + size_error = None + if len(replies) != len(requests): + size_error = self._error( + DriverError, 'Redis batch replies do not match the requested command count.', operation + ) + first_error = None + for offset, request in enumerate(requests[: len(replies)]): + reply = replies[offset] + index = start + offset + error = None + if isinstance(reply, BaseException): + error = reply + state = BatchState.UNKNOWN + if isinstance( + reply, (exceptions.ResponseError, exceptions.AuthenticationError, exceptions.AuthorizationError) + ): + state = BatchState.FAILED + batch.record(index, state, error=error) + else: + try: + output = self._parse(request.command[0], reply, operation) + except DriverError as unexpected: + error = unexpected + batch.record(index, BatchState.UNKNOWN, error=error) + else: + batch.record(index, BatchState.ACKNOWLEDGED, result=output) + if error is not None and (first_error is None or isinstance(error, CancelledError)): + first_error = error + return first_error if first_error is not None else size_error + + def _raise_batch(self, error: BaseException, partial: StringBatchResult, operation: str) -> NoReturn: + if not isinstance(error, Exception): + if isinstance(error, CancelledError): + BaseException.__dict__['__dict__'].__get__(error)['database_partial_result'] = partial + descriptor = BaseException.__dict__['__context__'] + original_context = descriptor.__get__(error) + try: + raise error + finally: + descriptor.__set__(error, original_context) + + original = error.original if isinstance(error, DatabaseError) and error.original is not None else error + wrapped = BatchOperationError( + 'A Redis String batch failed.', + partial_result=partial, + connection_name=self._connection.name, + backend='redis', + phase='operation', + operation=operation, + original=original, + ) + attach_cleanup_errors(wrapped, getattr(original, 'database_cleanup_errors', ())) + if error is not original: + attach_cleanup_errors(wrapped, getattr(error, 'database_cleanup_errors', ())) + raise wrapped from original + + +class RedisStrings(_Strings): + """Synchronous access to Redis String keys within a borrowed connection.""" + + def __enter__(self) -> 'RedisStrings': + self._open() + return self + + def __exit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: Any) -> bool: + try: + self._check_loop('close') + client, self._client = self._client, None + if self._sentinel and client is not None: + adapter = self._connection._adapter + errors = adapter._close_single(client, True, target='sentinel_master') + adapter._finish_cleanup(errors, exc) + except BaseException as cleanup: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup,)) + return False + + def _invoke(self, operation: str, request: _Request) -> _Reply: + with self._context(operation): + reply = self._client.execute_command(*request.command) + return self._parse(request.command[0], reply, operation) + + def get(self, key: StringKey) -> StringValue | None: + """Read a String, returning None only when the key is missing.""" + self._check_client('get') + return cast(StringValue | None, self._invoke('get', self._key_request('get', 'GET', key))) + + def set(self, key: StringKey, value: StringValue, *, ttl_s: int | None = None) -> bool: + """Replace one value and set or clear the expiration in the same command.""" + self._check_client('set') + request = self._set_request('set', StringSetItem(key, value, ttl_s)) + return cast(bool, self._invoke('set', request)) + + def delete(self, key: StringKey) -> int: + """Delete the named key and return the number removed (0 or 1).""" + self._check_client('delete') + return cast(int, self._invoke('delete', self._key_request('delete', 'DEL', key))) + + def exists(self, key: StringKey) -> bool: + """Report whether the named key exists.""" + self._check_client('exists') + return cast(bool, self._invoke('exists', self._key_request('exists', 'EXISTS', key))) + + def expire(self, key: StringKey, seconds: int) -> bool: + """Set positive integer seconds of expiration for one key.""" + self._check_client('expire') + request = self._key_request('expire', 'EXPIRE', key, self._seconds(seconds, 'expire')) + return cast(bool, self._invoke('expire', request)) + + def persist(self, key: StringKey) -> bool: + """Remove an existing expiration, returning whether it was removed.""" + self._check_client('persist') + return cast(bool, self._invoke('persist', self._key_request('persist', 'PERSIST', key))) + + def ttl(self, key: StringKey) -> int: + """Return the remaining seconds, -1 for no expiration, or -2 for a missing key.""" + self._check_client('ttl') + return cast(int, self._invoke('ttl', self._key_request('ttl', 'TTL', key))) + + def get_many(self, keys: Sequence[StringKey], *, batch_size: int = 500) -> list[StringValue | None]: + """Read each input position with its own GET, keeping duplicates and missing values.""" + result = self._run_batch('get_many', keys, batch_size) + return [cast(StringValue | None, item.result) for item in result.items] + + def set_many(self, items: Sequence[StringSetItem], *, batch_size: int = 500) -> StringBatchResult: + """Issue independent SET commands in non-transactional batches.""" + return self._run_batch('set_many', items, batch_size) + + def delete_many(self, keys: Sequence[StringKey], *, batch_size: int = 500) -> StringBatchResult: + """Issue independent DEL commands in non-transactional batches.""" + return self._run_batch('delete_many', keys, batch_size) + + def _run_batch( + self, operation: str, inputs: Sequence[StringKey] | Sequence[StringSetItem], batch_size: int + ) -> StringBatchResult: + requests = self._prepare_batch(operation, inputs, batch_size) + batch = _Batch(requests) + for start, chunk in self._chunks(requests, batch_size): + pipe = None + failure = None + try: + with self._context(operation): + pipe = self._client.pipeline(transaction=False) + for request in chunk: + pipe.execute_command(*request.command) + batch.dispatch(start, len(chunk)) + replies = pipe.execute(raise_on_error=False) + failure = self._consume_replies(batch, start, chunk, replies, operation) + except BaseException as error: + failure = error + finally: + if pipe is not None: + try: + with self._context(operation, phase='close'): + pipe.__exit__(None, None, None) + except BaseException as cleanup: + if failure is None: + failure = cleanup + else: + attach_cleanup_errors(failure, (cleanup,)) + if failure is not None: + self._raise_batch(failure, batch.result(operation), operation) + return batch.result(operation) + + +class AsyncRedisStrings(_Strings): + """Asynchronous access to Redis String keys within a borrowed connection.""" + + _async = True + + async def __aenter__(self) -> 'AsyncRedisStrings': + self._open() + return self + + async def __aexit__(self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: Any) -> bool: + try: + self._check_loop('close') + client, self._client = self._client, None + if self._sentinel and client is not None: + adapter = self._connection._adapter + errors = await adapter._close_single_async(client, True, target='sentinel_master') + adapter._finish_cleanup(errors, exc) + except BaseException as cleanup: + if exc is None: + raise + attach_cleanup_errors(exc, (cleanup,)) + return False + + async def _invoke(self, operation: str, request: _Request) -> _Reply: + with self._context(operation): + reply = await self._client.execute_command(*request.command) + return self._parse(request.command[0], reply, operation) + + async def get(self, key: StringKey) -> StringValue | None: + """Read a String, returning None only when the key is missing.""" + self._check_client('get') + return cast(StringValue | None, await self._invoke('get', self._key_request('get', 'GET', key))) + + async def set(self, key: StringKey, value: StringValue, *, ttl_s: int | None = None) -> bool: + """Replace one value and set or clear the expiration in the same command.""" + self._check_client('set') + request = self._set_request('set', StringSetItem(key, value, ttl_s)) + return cast(bool, await self._invoke('set', request)) + + async def delete(self, key: StringKey) -> int: + """Delete the named key and return the number removed (0 or 1).""" + self._check_client('delete') + return cast(int, await self._invoke('delete', self._key_request('delete', 'DEL', key))) + + async def exists(self, key: StringKey) -> bool: + """Report whether the named key exists.""" + self._check_client('exists') + return cast(bool, await self._invoke('exists', self._key_request('exists', 'EXISTS', key))) + + async def expire(self, key: StringKey, seconds: int) -> bool: + """Set positive integer seconds of expiration for one key.""" + self._check_client('expire') + request = self._key_request('expire', 'EXPIRE', key, self._seconds(seconds, 'expire')) + return cast(bool, await self._invoke('expire', request)) + + async def persist(self, key: StringKey) -> bool: + """Remove an existing expiration, returning whether it was removed.""" + self._check_client('persist') + return cast(bool, await self._invoke('persist', self._key_request('persist', 'PERSIST', key))) + + async def ttl(self, key: StringKey) -> int: + """Return the remaining seconds, -1 for no expiration, or -2 for a missing key.""" + self._check_client('ttl') + return cast(int, await self._invoke('ttl', self._key_request('ttl', 'TTL', key))) + + async def get_many(self, keys: Sequence[StringKey], *, batch_size: int = 500) -> list[StringValue | None]: + """Read each input position with its own GET, keeping duplicates and missing values.""" + result = await self._run_batch('get_many', keys, batch_size) + return [cast(StringValue | None, item.result) for item in result.items] + + async def set_many(self, items: Sequence[StringSetItem], *, batch_size: int = 500) -> StringBatchResult: + """Issue independent SET commands in non-transactional batches.""" + return await self._run_batch('set_many', items, batch_size) + + async def delete_many(self, keys: Sequence[StringKey], *, batch_size: int = 500) -> StringBatchResult: + """Issue independent DEL commands in non-transactional batches.""" + return await self._run_batch('delete_many', keys, batch_size) + + async def _run_batch( + self, operation: str, inputs: Sequence[StringKey] | Sequence[StringSetItem], batch_size: int + ) -> StringBatchResult: + requests = self._prepare_batch(operation, inputs, batch_size) + batch = _Batch(requests) + for start, chunk in self._chunks(requests, batch_size): + pipe = None + failure = None + try: + with self._context(operation): + pipe = self._client.pipeline(transaction=False) + for request in chunk: + pipe.execute_command(*request.command) + batch.dispatch(start, len(chunk)) + replies = await pipe.execute(raise_on_error=False) + failure = self._consume_replies(batch, start, chunk, replies, operation) + except BaseException as error: + failure = error + finally: + if pipe is not None: + try: + with self._context(operation, phase='close'): + # Async ClusterPipeline.reset() in redis-py 5.0 issues Redis RESET. + await pipe.__aexit__(None, None, None) + except BaseException as cleanup: + if failure is None: + failure = cleanup + else: + attach_cleanup_errors(failure, (cleanup,)) + if failure is not None: + self._raise_batch(failure, batch.result(operation), operation) + return batch.result(operation) diff --git a/src/datvia/crud/_types.py b/src/datvia/crud/_types.py new file mode 100644 index 0000000000000000000000000000000000000000..360677c7d95e1d4d2e392f94d1dbf78440426d1f --- /dev/null +++ b/src/datvia/crud/_types.py @@ -0,0 +1,109 @@ +"""Public result types and private data exchanged with backend adapters.""" + +from collections.abc import Mapping +from dataclasses import KW_ONLY +from dataclasses import dataclass +from enum import Enum +from typing import Any + + +class _Unset(Enum): + TOKEN = 'unset' + + +UNSET = _Unset.TOKEN + + +@dataclass(frozen=True) +class ReadItem: + """The observed record (or an observed absence) for one input position.""" + + index: int + key: dict[str, Any] + record: dict[str, Any] | None + + +@dataclass(frozen=True) +class InsertItem: + """An inserted record with explicit per-item options.""" + + record: Mapping[str, Any] + _: KW_ONLY + ttl_s: int | None | _Unset = UNSET + + +@dataclass(frozen=True) +class WriteCounts: + """Counts the backend can establish for a write.""" + + matched_count: int | None = None + modified_count: int | None = None + inserted_count: int | None = None + deleted_count: int | None = None + + +@dataclass(frozen=True) +class WriteResult: + """Write execution information, independent of the final transaction outcome.""" + + backend: str + operation: str + acknowledged: bool | None = None + matched_count: int | None = None + modified_count: int | None = None + inserted_count: int | None = None + deleted_count: int | None = None + generated_keys: tuple[dict[str, Any], ...] | None = None + native_result: Any = None + + +class BatchState(str, Enum): + """States backed by evidence from the SDK or the driver.""" + + ACKNOWLEDGED = 'acknowledged' + UNACKNOWLEDGED = 'unacknowledged' + FAILED = 'failed' + NOT_SENT = 'not_sent' + NOT_EXECUTED = 'not_executed' + UNKNOWN = 'unknown' + + +@dataclass(frozen=True) +class BatchItem: + index: int + key: dict[str, Any] | None + state: BatchState + result: ReadItem | WriteResult | None = None + error: Any = None + + +@dataclass(frozen=True) +class BatchResult: + requested_count: int + items: tuple[BatchItem, ...] + counts: WriteCounts = WriteCounts() + native_results: tuple[Any, ...] = () + + +@dataclass(frozen=True) +class WriteInput: + key: dict[str, Any] | None + values: dict[str, Any] + ttl_s: int | None | _Unset = UNSET + prepared: Any = None + + +@dataclass(frozen=True) +class ChunkResult: + items: tuple[BatchItem, ...] + counts: WriteCounts = WriteCounts() + native_results: tuple[Any, ...] = () + + +class ChunkError(Exception): + """Adapter reports reliable partial data without wrapping the driver cause.""" + + def __init__(self, original: BaseException, partial_result: ChunkResult) -> None: + super().__init__('A database batch chunk failed.') + self.original = original + self.partial_result = partial_result diff --git a/src/datvia/errors.py b/src/datvia/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..a8cf16287a1dceb48381dfb42f80c049bd0dc6a2 --- /dev/null +++ b/src/datvia/errors.py @@ -0,0 +1,183 @@ +"""Shared database errors and driver-independent conversion.""" + +import asyncio +import errno +import socket +from collections.abc import Callable, Iterable, Iterator +from contextlib import contextmanager + + +class DatabaseError(Exception): + """A database failure with connection context and the original exception.""" + + def __init__( + self, + message: str, + *, + connection_name: str | None = None, + backend: str | None = None, + phase: str | None = None, + operation: str | None = None, + target: str | None = None, + original: BaseException | None = None, + ) -> None: + super().__init__(message) + self.message = message + self.connection_name = connection_name + self.backend = backend + self.phase = phase + self.operation = operation + self.target = target + self.original = original + self.database_cleanup_errors: tuple[BaseException, ...] = () + + +class ConfigurationError(DatabaseError): + """Invalid configuration or connection parameters.""" + + +class DependencyError(DatabaseError): + """A required driver or driver capability is unavailable.""" + + +class AuthenticationError(DatabaseError): + """The server rejected the connection credentials.""" + + +class PermissionDeniedError(DatabaseError): + """The authenticated user lacks a required permission.""" + + +class DatabaseTimeoutError(DatabaseError): + """The driver or transport reported a timeout.""" + + +class DatabaseConnectionError(DatabaseError): + """A connection or transport failure.""" + + +class DriverError(DatabaseError): + """A driver error without a more specific classification.""" + + +class ConnectionClosedError(DatabaseError): + """The managed connection handle is closed.""" + + +_Classifier = Callable[[Exception], type[DatabaseError] | None] + +_CONNECTION_ERRNOS = frozenset( + { + errno.EHOSTUNREACH, + errno.ENETDOWN, + errno.ENETRESET, + errno.ENETUNREACH, + errno.ENOTCONN, + } +) + +_MESSAGES = { + ConfigurationError: "Invalid database configuration.", + DependencyError: "The required database driver or capability is unavailable.", + AuthenticationError: "Database authentication failed.", + PermissionDeniedError: "Database permission denied.", + DatabaseTimeoutError: "Database operation timed out.", + DatabaseConnectionError: "Database connection failed.", + DriverError: "The database driver reported an error.", + ConnectionClosedError: "The database connection is closed.", +} + + +def _classify_owned_error(error: Exception, phase: str) -> type[DatabaseError] | None: + if isinstance(error, (TimeoutError, asyncio.TimeoutError)): + return DatabaseTimeoutError + if isinstance(error, (ConnectionError, socket.gaierror)): + return DatabaseConnectionError + if isinstance(error, OSError): + if error.errno == errno.ETIMEDOUT: + return DatabaseTimeoutError + if error.errno in _CONNECTION_ERRNOS: + return DatabaseConnectionError + if phase == "create" and isinstance(error, (TypeError, ValueError)): + return ConfigurationError + return None + + +def convert_error( + error: BaseException, + *, + connection_name: str | None, + backend: str | None, + phase: str, + operation: str | None = None, + target: str | None = None, + classifier: _Classifier | None = None, + owned: bool = False, +) -> BaseException: + """Convert a driver error while retaining the original object and cause.""" + if not isinstance(error, Exception) or isinstance(error, DatabaseError): + return error + + error_type = classifier(error) if classifier is not None else None + if error_type is None and owned: + error_type = _classify_owned_error(error, phase) + if error_type is None: + return error + + converted = error_type( + _MESSAGES.get(error_type, "Database operation failed."), + connection_name=connection_name, + backend=backend, + phase=phase, + operation=operation, + target=target, + original=error, + ) + converted.__cause__ = error + return converted + + +@contextmanager +def error_context( + *, + connection_name: str | None, + backend: str | None, + phase: str, + operation: str | None = None, + target: str | None = None, + classifier: _Classifier | None = None, + owned: bool = False, +) -> Iterator[None]: + """Translate matching exceptions escaping a synchronous or async operation.""" + try: + yield + except Exception as error: + converted = convert_error( + error, + connection_name=connection_name, + backend=backend, + phase=phase, + operation=operation, + target=target, + classifier=classifier, + owned=owned, + ) + if converted is error: + raise + raise converted from error + + +def attach_cleanup_errors( + primary: BaseException, errors: Iterable[BaseException] +) -> None: + """Record cleanup failures without replacing the primary exception.""" + additions = tuple(errors) + if not additions: + return + + # Use the base descriptor to bypass custom attribute hooks on exceptions. + attributes = BaseException.__dict__["__dict__"].__get__(primary) + existing = attributes.get("database_cleanup_errors", ()) + if not isinstance(existing, tuple): + existing = () + attributes["database_cleanup_errors"] = tuple.__add__(existing, additions) diff --git a/src/datvia/mongodb/__init__.py b/src/datvia/mongodb/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bcf70097d4616ae66d01f2e79f5af5ae4c8b7d52 --- /dev/null +++ b/src/datvia/mongodb/__init__.py @@ -0,0 +1,5 @@ +from .client import Client + +__all__ = [ + 'Client', +] diff --git a/src/datvia/mongodb/_async_utils.py b/src/datvia/mongodb/_async_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..e9681706562e88846caad6f821d9b0bacc7c4d00 --- /dev/null +++ b/src/datvia/mongodb/_async_utils.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from asyncio import Lock +from collections.abc import AsyncIterable +from collections.abc import Iterable +from types import TracebackType +from typing import TypeVar + +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError + +from ..errors import attach_cleanup_errors +from ._batch import _BatchBuffer +from ._batch import _Document +from ._batch import _ErrorPolicy +from ._batch import _Item +from ._batch import _Update +from ._batch import _update_operation +from ._batch import _WriteOperation + +_AsyncBatchT = TypeVar('_AsyncBatchT', bound='AsyncBatch') + + +class AsyncBatch(_BatchBuffer[_Item]): + def __init__( + self, collection: object, step: int = 1000, ordered: bool = True, on_error: _ErrorPolicy = 'raise' + ) -> None: + super().__init__(collection, step, ordered, on_error) + self._lock = Lock() + + async def add(self, doc_or_op: _Item) -> None: + """Queue one item, writing the batch when it reaches step.""" + async with self._lock: + self.ops.append(doc_or_op) + if self.count >= self.step: + await self._flush() + + async def extend(self, docs_or_ops: Iterable[_Item] | AsyncIterable[_Item]) -> None: + """Consume items incrementally, leaving any final partial batch pending.""" + if isinstance(docs_or_ops, AsyncIterable): + async for item in docs_or_ops: + await self.add(item) + else: + for item in docs_or_ops: + await self.add(item) + + async def execute(self) -> None: + """Write the pending batch, if any.""" + async with self._lock: + await self._flush() + + async def _flush(self) -> None: + if not self.ops: + return + try: + write = getattr(self.collection, self._write_method) + self._record_result(await write(self.ops, ordered=self.ordered)) + except BulkWriteError as error: + self._record_error(error) + if self.on_error == 'raise': + raise + finally: + self.ops.clear() + + async def discard(self) -> None: + """Release pending items without sending them to MongoDB.""" + async with self._lock: + self.ops.clear() + + async def close(self) -> None: + """Flush pending items. The batch and its collection remain usable.""" + await self.execute() + + async def __aenter__(self: _AsyncBatchT) -> _AsyncBatchT: + return self + + async def __aexit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> None: + try: + await self.close() + except BaseException as error: + if exc is None: + raise + attach_cleanup_errors(exc, (error,)) + + +class AsyncBatchInsert(AsyncBatch[_Document]): + """Buffer documents for batched calls to AsyncCollection.insert_many().""" + + _write_method = 'insert_many' + _success_keys = ('nInserted',) + + +class AsyncBatchUpdate(AsyncBatch[UpdateOne]): + """Buffer PyMongo UpdateOne operations or build them from filter/update pairs.""" + + _success_keys = ('nModified',) + + async def add(self, query_or_op: _Document | UpdateOne, update: _Update | None = None) -> None: + await super().add(_update_operation(query_or_op, update)) + + +class AsyncBatchWrite(AsyncBatch[_WriteOperation]): + """Buffer mixed native write models for AsyncCollection.bulk_write().""" diff --git a/src/datvia/mongodb/_batch.py b/src/datvia/mongodb/_batch.py new file mode 100644 index 0000000000000000000000000000000000000000..a85a5e79c99e5c85955240f0541b4d94100e72e9 --- /dev/null +++ b/src/datvia/mongodb/_batch.py @@ -0,0 +1,90 @@ +"""Shared buffering and write accounting for native MongoDB batch utilities.""" + +from collections.abc import Mapping +from logging import getLogger +from typing import Any +from typing import ClassVar +from typing import Generic +from typing import Literal +from typing import TypeVar + +from pymongo import DeleteMany +from pymongo import DeleteOne +from pymongo import InsertOne +from pymongo import ReplaceOne +from pymongo import UpdateMany +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError +from pymongo.errors import InvalidOperation +from pymongo.results import BulkWriteResult +from pymongo.results import InsertManyResult + +logger = getLogger(__name__) + +_Item = TypeVar('_Item') +_Document = Mapping[str, Any] +_Update = _Document | list[_Document] +_WriteOperation = InsertOne | UpdateOne | UpdateMany | ReplaceOne | DeleteOne | DeleteMany +_ErrorPolicy = Literal['raise', 'log', 'ignore'] + + +def _validate_step(step: int, *, name: str = 'step') -> None: + if not isinstance(step, int) or isinstance(step, bool) or step < 1: + raise ValueError(f'{name} must be a positive integer.') + + +def _update_operation(query_or_op: _Document | UpdateOne, update: _Update | None) -> UpdateOne: + if isinstance(query_or_op, UpdateOne): + return query_or_op + if not update: + raise InvalidOperation('Missing update parameter.') + return UpdateOne(query_or_op, update) + + +class _BatchBuffer(Generic[_Item]): + _write_method: ClassVar[str] = 'bulk_write' + _success_keys: ClassVar[tuple[str, ...]] = ('nInserted', 'nModified', 'nRemoved', 'nUpserted') + + def __init__( + self, collection: object, step: int = 1000, ordered: bool = True, on_error: _ErrorPolicy = 'raise' + ) -> None: + _validate_step(step) + if on_error not in ('raise', 'log', 'ignore'): + raise ValueError("on_error must be 'raise', 'log', or 'ignore'.") + self.ops: list[_Item] = [] + self.collection = collection + self.step = step + self.ordered = ordered + self.on_error = on_error + self.success_count = 0 + self.error_count = 0 + self.unacknowledged_count = 0 + + @property + def count(self) -> int: + """Number of operations currently in the buffer.""" + return len(self.ops) + + def _record_result(self, result: InsertManyResult | BulkWriteResult) -> None: + if not result.acknowledged: + self.unacknowledged_count += self.count + elif isinstance(result, InsertManyResult): + self.success_count += len(result.inserted_ids) + else: + self._record_successes(result.bulk_api_result) + + def _record_successes(self, details: Mapping[str, Any]) -> None: + self.success_count += sum(details.get(key, 0) for key in self._success_keys) + + def _record_error(self, error: BulkWriteError) -> None: + details = error.details or {} + self._record_successes(details) + write_errors = len(details.get('writeErrors') or ()) + self.error_count += write_errors + if self.on_error == 'log': + logger.warning( + '%s: %d write errors, %d write concern errors', + type(self).__name__, + write_errors, + len(details.get('writeConcernErrors') or ()), + ) diff --git a/src/datvia/mongodb/_cursor_batches.py b/src/datvia/mongodb/_cursor_batches.py new file mode 100644 index 0000000000000000000000000000000000000000..2219d9266994c4f267f1802aaea7d720d934e150 --- /dev/null +++ b/src/datvia/mongodb/_cursor_batches.py @@ -0,0 +1,68 @@ +"""Read a native MongoDB cursor in a bounded, explicitly owned scope.""" + +from collections.abc import AsyncGenerator +from collections.abc import AsyncIterator +from collections.abc import Generator +from collections.abc import Iterator +from contextlib import aclosing +from contextlib import asynccontextmanager +from contextlib import closing +from contextlib import contextmanager +from itertools import islice +from typing import Protocol +from typing import TypeVar + +from ._batch import _validate_step +from ._resources import _async_closing +from ._resources import _closing + +_Row = TypeVar('_Row') +_Row_co = TypeVar('_Row_co', covariant=True) + + +class _Cursor(Protocol[_Row_co]): + def __iter__(self) -> Iterator[_Row_co]: ... + + def close(self) -> None: ... + + +class _AsyncCursor(Protocol[_Row_co]): + def __aiter__(self) -> AsyncIterator[_Row_co]: ... + + async def close(self) -> None: ... + + +def _chunks(cursor: _Cursor[_Row], step: int) -> Generator[list[_Row], None, None]: + rows = iter(cursor) + while batch := list(islice(rows, step)): + yield batch + del batch + + +async def _async_chunks(cursor: _AsyncCursor[_Row], step: int) -> AsyncGenerator[list[_Row], None]: + batch: list[_Row] = [] + async for row in cursor: + batch.append(row) + if len(batch) == step: + yield batch + batch = [] + if batch: + yield batch + + +@contextmanager +def cursor_batches(cursor: _Cursor[_Row], *, step: int = 1000) -> Iterator[Iterator[list[_Row]]]: + """Yield a batch iterator and close its native cursor when the scope exits.""" + _validate_step(step) + with _closing(cursor), closing(_chunks(cursor, step)) as batches: + yield batches + + +@asynccontextmanager +async def async_cursor_batches( + cursor: _AsyncCursor[_Row], *, step: int = 1000 +) -> AsyncIterator[AsyncIterator[list[_Row]]]: + """Yield an async batch iterator and await native cursor cleanup on exit.""" + _validate_step(step) + async with _async_closing(cursor), aclosing(_async_chunks(cursor, step)) as batches: + yield batches diff --git a/src/datvia/mongodb/_resources.py b/src/datvia/mongodb/_resources.py new file mode 100644 index 0000000000000000000000000000000000000000..39fd9b0c415de313a1e19624f9f234984fcc69aa --- /dev/null +++ b/src/datvia/mongodb/_resources.py @@ -0,0 +1,37 @@ +"""Close MongoDB cursors while preserving an in-flight error.""" + +from collections.abc import AsyncIterator +from collections.abc import Iterator +from contextlib import asynccontextmanager +from contextlib import contextmanager +from typing import Any + +from ..errors import attach_cleanup_errors + + +@contextmanager +def _closing(cursor: Any) -> Iterator[Any]: + try: + yield cursor + except BaseException as error: + try: + cursor.close() + except BaseException as cleanup_error: + attach_cleanup_errors(error, (cleanup_error,)) + raise + else: + cursor.close() + + +@asynccontextmanager +async def _async_closing(cursor: Any) -> AsyncIterator[Any]: + try: + yield cursor + except BaseException as error: + try: + await cursor.close() + except BaseException as cleanup_error: + attach_cleanup_errors(error, (cleanup_error,)) + raise + else: + await cursor.close() diff --git a/src/datvia/mongodb/_scan.py b/src/datvia/mongodb/_scan.py new file mode 100644 index 0000000000000000000000000000000000000000..fd02093b9e4477fda1ad518527755952b77140af --- /dev/null +++ b/src/datvia/mongodb/_scan.py @@ -0,0 +1,266 @@ +"""Ordered collection traversal with bounded cursor recovery.""" + +from __future__ import annotations + +from collections.abc import AsyncGenerator +from collections.abc import AsyncIterator +from collections.abc import Generator +from collections.abc import Iterator +from collections.abc import Mapping +from contextlib import AsyncExitStack +from contextlib import ExitStack +from contextlib import aclosing +from contextlib import asynccontextmanager +from contextlib import closing +from contextlib import contextmanager +from copy import deepcopy +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar + +from bson import MaxKey +from pymongo.errors import AutoReconnect +from pymongo.errors import CursorNotFound +from pymongo.errors import OperationFailure +from pymongo.errors import PyMongoError + +from ._batch import _validate_step +from ._resources import _async_closing +from ._resources import _closing + +if TYPE_CHECKING: + from pymongo.asynchronous.collection import AsyncCollection + from pymongo.collection import Collection + +_Filter = Mapping[str, Any] +_Document = TypeVar('_Document', bound=Mapping[str, Any]) +_START = object() +_CURSOR_LOST_CODES = frozenset({43, 175, 237}) # CursorNotFound, QueryPlanKilled, CursorKilled + + +def _validate_retries(max_retries: int) -> None: + if not isinstance(max_retries, int) or isinstance(max_retries, bool) or max_retries < 0: + raise ValueError('max_retries must be a non-negative integer.') + + +def _with_id_filter(filter: _Filter | None, id_filter: _Filter) -> _Filter: + return {'$and': [filter, id_filter]} if filter else id_filter + + +def _find_options(filter: _Filter | None, last_id: Any, *, batch_size: int, limit: int = 0) -> dict[str, Any]: + options: dict[str, Any] = { + 'filter': filter, + 'sort': [('_id', 1)], + 'hint': '_id_', + 'batch_size': batch_size, + 'limit': limit, + } + if last_id is not _START: + # The index lower bound is inclusive and follows BSON sort order across types. + options['min'] = {'_id': last_id} + options['filter'] = _with_id_filter(filter, {'_id': {'$ne': last_id}}) + return options + + +def _max_id_options(filter: _Filter | None) -> dict[str, Any]: + # A min-bounded scan has an implicit exclusive upper bound at MaxKey. + return _find_options(_with_id_filter(filter, {'_id': {'$eq': MaxKey()}}), _START, batch_size=1, limit=1) + + +def _can_retry(error: Exception, retries_left: int) -> bool: + if retries_left == 0 or getattr(error, 'database_cleanup_errors', ()): + return False + return isinstance(error, (AutoReconnect, CursorNotFound)) or ( + isinstance(error, OperationFailure) and error.code in _CURSOR_LOST_CODES + ) + + +def _read_page(collection: Collection[_Document], options: dict[str, Any], max_retries: int) -> list[_Document]: + while True: + read_complete = False + try: + with _closing(collection.find(**options)) as cursor: + page = list(cursor) + read_complete = True + return page + except PyMongoError as error: + if read_complete or not _can_retry(error, max_retries): + raise + max_retries -= 1 + + +async def _async_read_page( + collection: AsyncCollection[_Document], options: dict[str, Any], max_retries: int +) -> list[_Document]: + while True: + read_complete = False + try: + async with _async_closing(collection.find(**options)) as cursor: + page = [document async for document in cursor] + read_complete = True + return page + except PyMongoError as error: + if read_complete or not _can_retry(error, max_retries): + raise + max_retries -= 1 + + +def scan_collection( + collection: Collection[_Document], filter: _Filter | None = None, *, step: int = 1000, max_retries: int = 0 +) -> Iterator[_Document]: + """Yield documents in _id order, closing each page cursor before delivery. + + Use a regular collection with a simple collation and globally unique _id. + max_retries counts additional read attempts per page. + """ + _validate_step(step) + _validate_retries(max_retries) + last_id = _START + while True: + page = _read_page(collection, _find_options(filter, last_id, batch_size=step, limit=step), max_retries) + end_id = deepcopy(page[-1]['_id']) if page else last_id + full_page = len(page) == step + yield from page + del page + if isinstance(end_id, MaxKey): + return + if not full_page: + if last_id is not _START: + yield from _read_page(collection, _max_id_options(filter), max_retries) + return + last_id = end_id + + +async def async_scan_collection( + collection: AsyncCollection[_Document], filter: _Filter | None = None, *, step: int = 1000, max_retries: int = 0 +) -> AsyncIterator[_Document]: + """Yield async documents in _id order, closing each page cursor before delivery.""" + _validate_step(step) + _validate_retries(max_retries) + last_id = _START + while True: + page = await _async_read_page( + collection, _find_options(filter, last_id, batch_size=step, limit=step), max_retries + ) + end_id = deepcopy(page[-1]['_id']) if page else last_id + full_page = len(page) == step + for document in page: + yield document + del document + del page + if isinstance(end_id, MaxKey): + return + if not full_page: + if last_id is not _START: + for document in await _async_read_page(collection, _max_id_options(filter), max_retries): + yield document + return + last_id = end_id + + +def _stream( + collection: Collection[_Document], filter: _Filter | None, batch_size: int, max_retries: int, resources: ExitStack +) -> Generator[_Document, None, None]: + cursor = None + bounded = False + last_id = _START + retries_left = max_retries + while True: + try: + if cursor is None: + bounded = last_id is not _START + cursor = resources.enter_context( + _closing(collection.find(**_find_options(filter, last_id, batch_size=batch_size))) + ) + document = next(cursor) + except StopIteration: + resources.close() + if bounded and not isinstance(last_id, MaxKey): + yield from _read_page(collection, _max_id_options(filter), retries_left) + return + except PyMongoError as error: + resources.__exit__(type(error), error, error.__traceback__) + if not _can_retry(error, retries_left): + raise + retries_left -= 1 + cursor = None + continue + last_id = deepcopy(document['_id']) + retries_left = max_retries + yield document + del document + if isinstance(last_id, MaxKey): + resources.close() + return + + +async def _async_stream( + collection: AsyncCollection[_Document], + filter: _Filter | None, + batch_size: int, + max_retries: int, + resources: AsyncExitStack, +) -> AsyncGenerator[_Document, None]: + cursor = None + bounded = False + last_id = _START + retries_left = max_retries + while True: + try: + if cursor is None: + bounded = last_id is not _START + cursor = await resources.enter_async_context( + _async_closing(collection.find(**_find_options(filter, last_id, batch_size=batch_size))) + ) + document = await anext(cursor) + except StopAsyncIteration: + await resources.aclose() + if bounded and not isinstance(last_id, MaxKey): + for document in await _async_read_page(collection, _max_id_options(filter), retries_left): + yield document + return + except PyMongoError as error: + await resources.__aexit__(type(error), error, error.__traceback__) + if not _can_retry(error, retries_left): + raise + retries_left -= 1 + cursor = None + continue + last_id = deepcopy(document['_id']) + retries_left = max_retries + yield document + del document + if isinstance(last_id, MaxKey): + await resources.aclose() + return + + +@contextmanager +def stream_collection( + collection: Collection[_Document], filter: _Filter | None = None, *, batch_size: int = 1, max_retries: int = 3 +) -> Iterator[Iterator[_Document]]: + """Yield a streaming iterator that can reopen at the last delivered _id. + + The context owns the active cursor. max_retries counts additional read + attempts at the same position. + """ + _validate_step(batch_size, name='batch_size') + _validate_retries(max_retries) + # Keep cursor ownership outside the generator so closing it on break + # reports any cursor cleanup failure to the surrounding context. + with ExitStack() as resources, closing(_stream(collection, filter, batch_size, max_retries, resources)) as rows: + yield rows + + +@asynccontextmanager +async def async_stream_collection( + collection: AsyncCollection[_Document], filter: _Filter | None = None, *, batch_size: int = 1, max_retries: int = 3 +) -> AsyncIterator[AsyncIterator[_Document]]: + """Yield an async streaming iterator and close its active cursor on exit.""" + _validate_step(batch_size, name='batch_size') + _validate_retries(max_retries) + async with ( + AsyncExitStack() as resources, + aclosing(_async_stream(collection, filter, batch_size, max_retries, resources)) as rows, + ): + yield rows diff --git a/src/datvia/mongodb/_sync_utils.py b/src/datvia/mongodb/_sync_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5cfcc00d56d0c206b1411dd7df8218d1ada9a327 --- /dev/null +++ b/src/datvia/mongodb/_sync_utils.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from collections.abc import Iterable +from threading import Lock +from types import TracebackType +from typing import TypeVar + +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError + +from ..errors import attach_cleanup_errors +from ._batch import _BatchBuffer +from ._batch import _Document +from ._batch import _ErrorPolicy +from ._batch import _Item +from ._batch import _Update +from ._batch import _update_operation +from ._batch import _WriteOperation + +_BatchT = TypeVar('_BatchT', bound='Batch') + + +class Batch(_BatchBuffer[_Item]): + def __init__( + self, collection: object, step: int = 1000, ordered: bool = True, on_error: _ErrorPolicy = 'raise' + ) -> None: + super().__init__(collection, step, ordered, on_error) + self._lock = Lock() + + def add(self, doc_or_op: _Item) -> None: + """Queue one item, writing the batch when it reaches step.""" + with self._lock: + self.ops.append(doc_or_op) + if self.count >= self.step: + self._flush() + + def extend(self, docs_or_ops: Iterable[_Item]) -> None: + """Consume items incrementally, leaving any final partial batch pending.""" + for item in docs_or_ops: + self.add(item) + + def execute(self) -> None: + """Write the pending batch, if any.""" + with self._lock: + self._flush() + + def _flush(self) -> None: + if not self.ops: + return + try: + write = getattr(self.collection, self._write_method) + self._record_result(write(self.ops, ordered=self.ordered)) + except BulkWriteError as error: + self._record_error(error) + if self.on_error == 'raise': + raise + finally: + self.ops.clear() + + def discard(self) -> None: + """Release pending items without sending them to MongoDB.""" + with self._lock: + self.ops.clear() + + def close(self) -> None: + """Flush pending items. The batch and its collection remain usable.""" + self.execute() + + def __enter__(self: _BatchT) -> _BatchT: + return self + + def __exit__( + self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None + ) -> None: + try: + self.close() + except BaseException as error: + if exc is None: + raise + attach_cleanup_errors(exc, (error,)) + + +class BatchInsert(Batch[_Document]): + """Buffer documents for batched calls to Collection.insert_many().""" + + _write_method = 'insert_many' + _success_keys = ('nInserted',) + + +class BatchUpdate(Batch[UpdateOne]): + """Buffer PyMongo UpdateOne operations or build them from filter/update pairs.""" + + _success_keys = ('nModified',) + + def add(self, query_or_op: _Document | UpdateOne, update: _Update | None = None) -> None: + super().add(_update_operation(query_or_op, update)) + + +class BatchWrite(Batch[_WriteOperation]): + """Buffer mixed native write models for Collection.bulk_write().""" diff --git a/src/datvia/mongodb/client.py b/src/datvia/mongodb/client.py new file mode 100644 index 0000000000000000000000000000000000000000..72bad8c859f7015cdaf91bb52e7d3e2274b1f9f0 --- /dev/null +++ b/src/datvia/mongodb/client.py @@ -0,0 +1,18 @@ +import pymongo + + +class Client: + + def __init__(self): + pass + + @staticmethod + def create(*args, **kwargs) -> pymongo.MongoClient: + return pymongo.MongoClient(*args, **kwargs) + + @staticmethod + def create_async(*args, **kwargs): + async_client = getattr(pymongo, "AsyncMongoClient", None) + if async_client is None: + raise ImportError("MongoDB async client requires PyMongo >= 4.9. Upgrade PyMongo to use create_async().") + return async_client(*args, **kwargs) diff --git a/src/datvia/mongodb/types.py b/src/datvia/mongodb/types.py new file mode 100644 index 0000000000000000000000000000000000000000..e4db8b007e9138ae9bee1e36afe4fb8c69b9e48c --- /dev/null +++ b/src/datvia/mongodb/types.py @@ -0,0 +1,25 @@ +from bson import ( + ObjectId, + Binary, + Code, + DBRef, + Decimal128, + Int64, + MaxKey, + MinKey, + Regex, + Timestamp, +) + +__all__ = [ + "ObjectId", + "Binary", + "Code", + "DBRef", + "Decimal128", + "Int64", + "MaxKey", + "MinKey", + "Regex", + "Timestamp", +] diff --git a/src/datvia/mongodb/utils.py b/src/datvia/mongodb/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..221ec77e2bef0aa83860ad1f45308c3bbc5a00c4 --- /dev/null +++ b/src/datvia/mongodb/utils.py @@ -0,0 +1,29 @@ +"""Batch writes and cursor utilities for native PyMongo collections.""" + +from ._async_utils import AsyncBatchInsert +from ._async_utils import AsyncBatchUpdate +from ._async_utils import AsyncBatchWrite +from ._cursor_batches import async_cursor_batches +from ._cursor_batches import cursor_batches +from ._scan import async_scan_collection +from ._scan import async_stream_collection +from ._scan import scan_collection +from ._scan import stream_collection +from ._sync_utils import BatchInsert +from ._sync_utils import BatchUpdate +from ._sync_utils import BatchWrite + +__all__ = [ + 'AsyncBatchInsert', + 'AsyncBatchUpdate', + 'AsyncBatchWrite', + 'BatchInsert', + 'BatchUpdate', + 'BatchWrite', + 'async_cursor_batches', + 'async_scan_collection', + 'async_stream_collection', + 'cursor_batches', + 'scan_collection', + 'stream_collection', +] diff --git a/src/datvia/mysql/__init__.py b/src/datvia/mysql/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..19790f0eb723342403b58a089f12a0963d7d5895 --- /dev/null +++ b/src/datvia/mysql/__init__.py @@ -0,0 +1,5 @@ +from .client import Client + +__all__ = [ + "Client", +] diff --git a/src/datvia/mysql/client.py b/src/datvia/mysql/client.py new file mode 100644 index 0000000000000000000000000000000000000000..38f75c1c79e233a0023c82e8b6008abc71ca3f8c --- /dev/null +++ b/src/datvia/mysql/client.py @@ -0,0 +1,14 @@ +from mysql import connector +from mysql.connector import aio + + +class Client: + """Factory for native MySQL Connector/Python connections.""" + + @staticmethod + def create(*args, **kwargs): + return connector.connect(*args, **kwargs) + + @staticmethod + async def create_async(*args, **kwargs): + return await aio.connect(*args, **kwargs) diff --git a/src/datvia/redis/__init__.py b/src/datvia/redis/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..bcf70097d4616ae66d01f2e79f5af5ae4c8b7d52 --- /dev/null +++ b/src/datvia/redis/__init__.py @@ -0,0 +1,5 @@ +from .client import Client + +__all__ = [ + 'Client', +] diff --git a/src/datvia/redis/client.py b/src/datvia/redis/client.py new file mode 100644 index 0000000000000000000000000000000000000000..a6303b32afad3358926a7975d6705d21b2fc9870 --- /dev/null +++ b/src/datvia/redis/client.py @@ -0,0 +1,77 @@ +import redis + + +class Client: + + def __init__(self): + pass + + @classmethod + def create(cls, *args, **kwargs) -> redis.Redis: + return cls.single(*args, **kwargs) + + @classmethod + def create_async(cls, *args, **kwargs) -> redis.asyncio.Redis: + return cls.single_async(*args, **kwargs) + + @staticmethod + def single(*args, **kwargs) -> redis.Redis: + return redis.Redis(*args, **kwargs) + + @staticmethod + def single_async(*args, **kwargs) -> redis.asyncio.Redis: + return redis.asyncio.Redis(*args, **kwargs) + + @staticmethod + def sentinel(*args, **kwargs) -> redis.Sentinel: + return redis.Sentinel(*args, **kwargs) + + @staticmethod + def sentinel_async(*args, **kwargs) -> redis.asyncio.Sentinel: + return redis.asyncio.Sentinel(*args, **kwargs) + + @staticmethod + def cluster(*args, **kwargs) -> redis.RedisCluster: + return redis.RedisCluster(*args, **kwargs) + + @staticmethod + def cluster_async(*args, **kwargs) -> redis.asyncio.RedisCluster: + return redis.asyncio.RedisCluster(*args, **kwargs) + + @classmethod + def connect(cls, mode='single', is_async=False, *args, **kwargs): + """统一连接入口 + :param mode: single[default], sentinel, cluster; + "param mode type: str; + :param is_async: False[default]; + "param is_async type: bool; + """ + if mode == 'single': + return cls._connect_single(is_async, *args, **kwargs) + elif mode == 'sentinel': + return cls._connect_sentinel(is_async, *args, **kwargs) + elif mode == 'cluster': + return cls._connect_cluster(is_async, *args, **kwargs) + else: + raise ValueError(f"Invalid mode: '{mode}'. Expected one of: 'single', 'sentinel', 'cluster'") + + @classmethod + def _connect_single(cls, is_async, *args, **kwargs): + if is_async: + return cls.single_async(*args, **kwargs) + else: + return cls.single(*args, **kwargs) + + @classmethod + def _connect_sentinel(cls, is_async, *args, **kwargs): + if is_async: + return cls.sentinel_async(*args, **kwargs) + else: + return cls.sentinel(*args, **kwargs) + + @classmethod + def _connect_cluster(cls, is_async, *args, **kwargs): + if is_async: + return cls.cluster_async(*args, **kwargs) + else: + return cls.cluster(*args, **kwargs) diff --git a/src/datvia/schema/__init__.py b/src/datvia/schema/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..95d8d6d89d3a0af93fc97ff7e5e6f9da932532b1 --- /dev/null +++ b/src/datvia/schema/__init__.py @@ -0,0 +1,67 @@ +"""Explicit basic schema preparation on borrowed managed connections.""" + +from typing import TYPE_CHECKING +from typing import Any + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import DependencyError +from ._base import SchemaConflictError + +if TYPE_CHECKING: + from ._models import Column + from ._models import TableSpec + from ._mongodb import AsyncMongoDBSchema + from ._mongodb import MongoDBSchema + from ._mysql import AsyncMySQLSchema + from ._mysql import MySQLSchema + +__all__ = ['AsyncSchema', 'Column', 'Schema', 'SchemaConflictError', 'TableSpec'] + + +def _factory(connection: Connection, backend: str, class_name: str) -> Any: + from importlib import import_module + + try: + module = import_module(f'{__name__}._{backend}') + except ImportError as original: + raise DependencyError( + 'The selected schema adapter or database driver is unavailable.', + connection_name=getattr(connection, 'name', None), + backend=backend, + phase='config', + original=original, + ) from original + return getattr(module, class_name)(connection) + + +class Schema: + """Bind synchronous schema operations; binding performs no network I/O.""" + + @staticmethod + def mysql(connection: Connection) -> 'MySQLSchema': + return _factory(connection, 'mysql', 'MySQLSchema') + + @staticmethod + def mongodb(connection: Connection) -> 'MongoDBSchema': + return _factory(connection, 'mongodb', 'MongoDBSchema') + + +class AsyncSchema: + """Bind asynchronous operations; await methods, not this factory.""" + + @staticmethod + def mysql(connection: AsyncConnection) -> 'AsyncMySQLSchema': + return _factory(connection, 'mysql', 'AsyncMySQLSchema') + + @staticmethod + def mongodb(connection: AsyncConnection) -> 'AsyncMongoDBSchema': + return _factory(connection, 'mongodb', 'AsyncMongoDBSchema') + + +def __getattr__(name: str) -> Any: + if name in ('Column', 'TableSpec'): + from . import _models + + return getattr(_models, name) + raise AttributeError(f'module {__name__!r} has no attribute {name!r}') diff --git a/src/datvia/schema/_base.py b/src/datvia/schema/_base.py new file mode 100644 index 0000000000000000000000000000000000000000..d91f74699b3915df9f3763b53765071ea953da76 --- /dev/null +++ b/src/datvia/schema/_base.py @@ -0,0 +1,71 @@ +"""Driver-independent binding and error context for explicit schema operations.""" + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import DatabaseError +from ..errors import attach_cleanup_errors +from ..errors import error_context + + +class SchemaConflictError(DatabaseError): + """An existing object does not satisfy the explicitly requested structure.""" + + def __init__(self, message: str, *, differences: tuple[str, ...] = (), **context: Any) -> None: + super().__init__(message, **context) + self.differences = tuple(differences) + + +class BoundSchema: + """Borrow a connection without owning its lifecycle or starting network I/O.""" + + def __init__(self, connection: Connection, *, backend: str, is_async: bool) -> None: + if ( + not isinstance(connection, Connection) + or isinstance(connection, AsyncConnection) != is_async + or connection.backend != backend + ): + raise ConfigurationError('Schema binding requires the matching backend and sync mode.', phase='config') + connection._ensure_open('operation') + self._connection = connection + + def _context_fields(self, operation: str, target: str | None = None) -> dict[str, Any]: + return dict( + connection_name=self._connection.name, + backend=self._connection.backend, + phase='operation', + operation=operation, + target=target, + ) + + @contextmanager + def _operation(self, operation: str, target: str | None = None) -> Iterator[None]: + connection = self._connection + if isinstance(connection, AsyncConnection): + connection._ensure_loop('operation') + connection._ensure_open('operation') + try: + with error_context( + **self._context_fields(operation, target), + classifier=connection._adapter.classify, + owned=True, + ): + yield + except DatabaseError as error: + if error.original is not None: + attributes = BaseException.__dict__['__dict__'].__get__(error.original) + cleanup_errors = attributes.get('database_cleanup_errors', ()) + if isinstance(cleanup_errors, tuple): + attach_cleanup_errors( + error, + tuple( + cleanup + for cleanup in cleanup_errors + if not any(cleanup is existing for existing in error.database_cleanup_errors) + ), + ) + raise diff --git a/src/datvia/schema/_models.py b/src/datvia/schema/_models.py new file mode 100644 index 0000000000000000000000000000000000000000..795380b0b41432258713c69f670cd51ea86dc55b --- /dev/null +++ b/src/datvia/schema/_models.py @@ -0,0 +1,182 @@ +"""Driver-independent declarations for basic MySQL table creation.""" + +import re +from collections.abc import Mapping +from collections.abc import Sequence +from dataclasses import dataclass +from dataclasses import field +from datetime import date +from datetime import datetime +from decimal import Context +from decimal import Decimal +from enum import Enum +from math import isfinite +from types import MappingProxyType + +from ..errors import ConfigurationError + +_IDENTIFIERS = re.compile(r'\w{1,64}') +_TYPES = frozenset({'int', 'bigint', 'boolean', 'decimal', 'double', 'varchar', 'text', 'date', 'datetime', 'blob'}) +_INT_BITS = {'int': 32, 'bigint': 64} + + +class _Default(Enum): + UNSPECIFIED = 0 + + +_UNSET = _Default.UNSPECIFIED + + +def _invalid(message: str) -> ConfigurationError: + return ConfigurationError(message, backend='mysql', phase='config') + + +def _identifier(name: str) -> str: + if type(name) is not str or _IDENTIFIERS.fullmatch(name) is None: + raise _invalid('MySQL identifiers require 1–64 letters, digits or underscores.') + return f'`{name}`' + + +def _integer_in_range(value: object, minimum: int, maximum: int) -> bool: + return type(value) is int and minimum <= value <= maximum + + +@dataclass(frozen=True) +class Column: + """MySQL column type, dimensions, nullability, literal default and auto-increment.""" + + type: str + length: int | None = None + precision: int | None = None + scale: int | None = None + nullable: bool = False + default: str | int | float | Decimal | date | datetime | None | _Default = field(default=_UNSET, repr=False) + auto_increment: bool = False + + def __post_init__(self) -> None: + if type(self.type) is not str or self.type.lower() not in _TYPES: + raise _invalid('Unsupported MySQL column type.') + object.__setattr__(self, 'type', self.type.lower()) + if type(self.nullable) is not bool or type(self.auto_increment) is not bool: + raise _invalid('Column nullable and auto_increment must be boolean.') + self._validate_dimensions() + if self.auto_increment and (self.type not in _INT_BITS or self.nullable or self.default is not _UNSET): + raise _invalid('AUTO_INCREMENT requires a non-nullable integer column with no explicit default.') + if self.default is _UNSET: + return + if self.default is None: + if not self.nullable: + raise _invalid('A NULL default requires a nullable MySQL column.') + return + object.__setattr__(self, 'default', self._literal_default()) + + def _validate_dimensions(self) -> None: + if self.type == 'varchar': + if not _integer_in_range(self.length, 1, 65535): + raise _invalid('VARCHAR requires a length between 1 and 65535.') + elif self.length is not None: + raise _invalid('Column length is supported for VARCHAR only.') + + if self.precision is not None: + if self.type == 'decimal': + valid = _integer_in_range(self.precision, 1, 65) + else: + valid = self.type == 'datetime' and _integer_in_range(self.precision, 0, 6) + if not valid: + raise _invalid('Precision requires DECIMAL (1–65) or DATETIME (0–6).') + if self.scale is not None and ( + self.type != 'decimal' or not _integer_in_range(self.scale, 0, min(30, self.precision or 10)) + ): + raise _invalid('DECIMAL scale must be between 0 and 30 and cannot exceed precision.') + + def _literal_default(self) -> str | int | float | Decimal | date | datetime: + value = self.default + if self.type in _INT_BITS: + limit = 2 ** (_INT_BITS[self.type] - 1) + if _integer_in_range(value, -limit, limit - 1): + return value + elif self.type == 'boolean': + if type(value) is bool or _integer_in_range(value, 0, 1): + return bool(value) + elif self.type == 'decimal': + if type(value) in (Decimal, float, int): + number = Decimal(str(value)) if type(value) is float else Decimal(value) + scale = self.scale or 0 + quantized = number.quantize( + Decimal((0, (1,), -scale)), context=Context(prec=self.precision or 10, traps=[]) + ) + if number.is_finite() and quantized == number: + return quantized + elif self.type == 'double': + if type(value) in (Decimal, float, int): + try: + number = float(value) + if isfinite(number): + return number + except (OverflowError, ValueError): + pass + elif self.type == 'varchar': + if type(value) is str and len(value) <= self.length: + try: + value.encode('utf-8') + except UnicodeEncodeError: + pass + else: + return value + elif self.type == 'date': + if type(value) is date and value.year >= 1000: + return value + elif self.type == 'datetime': + if ( + type(value) is datetime + and value.year >= 1000 + and value.tzinfo is None + and value.microsecond % (10 ** (6 - (self.precision or 0))) == 0 + ): + return value + elif self.type in ('text', 'blob'): + raise _invalid('TEXT and BLOB support only a NULL explicit default.') + raise _invalid('Expected a literal default that fits the declared MySQL column.') + + +@dataclass(frozen=True) +class TableSpec: + """Ordered MySQL columns and an explicit simple or composite primary key.""" + + columns: Mapping[str, Column] + primary_key: Sequence[str] + + def __post_init__(self) -> None: + if not isinstance(self.columns, Mapping) or not self.columns: + raise _invalid('A MySQL table requires a nonempty mapping of columns.') + columns = dict(self.columns) + names = {} + for name, column in columns.items(): + _identifier(name) + if not isinstance(column, Column): + raise _invalid('MySQL table columns must be Column objects.') + folded = name.casefold() + if folded in names: + raise _invalid('MySQL column names must be distinct regardless of case.') + names[folded] = name + + if isinstance(self.primary_key, (str, bytes)) or not isinstance(self.primary_key, Sequence): + raise _invalid('A MySQL primary key requires a sequence of column names.') + primary_key = [] + for name in self.primary_key: + _identifier(name) + declared = names.get(name.casefold()) + if declared is None: + raise _invalid('Every MySQL primary key field must be a declared column.') + if declared in primary_key: + raise _invalid('MySQL primary key fields must be distinct.') + if columns[declared].nullable or columns[declared].type in ('text', 'blob'): + raise _invalid('MySQL primary key fields must be non-nullable and support an unprefixed index.') + primary_key.append(declared) + if not primary_key: + raise _invalid('A MySQL table requires an explicit primary key.') + if any(column.auto_increment and primary_key != [name] for name, column in columns.items()): + raise _invalid('AUTO_INCREMENT is supported on a single-column integer primary key.') + + object.__setattr__(self, 'columns', MappingProxyType(columns)) + object.__setattr__(self, 'primary_key', tuple(primary_key)) diff --git a/src/datvia/schema/_mongodb.py b/src/datvia/schema/_mongodb.py new file mode 100644 index 0000000000000000000000000000000000000000..26d7e371bf876601f36bef6e848939055a9b25c4 --- /dev/null +++ b/src/datvia/schema/_mongodb.py @@ -0,0 +1,430 @@ +"""Explicit MongoDB collection and database operations.""" + +from collections.abc import Mapping +from collections.abc import Sequence +from hashlib import sha256 +from typing import Any + +from pymongo.errors import InvalidName +from pymongo.errors import OperationFailure + +from ..connections import AsyncConnection +from ..connections import Connection +from ..crud._records import _names +from ..errors import ConfigurationError +from ..errors import attach_cleanup_errors +from ._base import BoundSchema +from ._base import SchemaConflictError + +_Document = Mapping[str, Any] +_IndexRequest = tuple[list[tuple[str, int]], dict[str, Any]] +_ID_FIELDS = ('_id',) +_NAMESPACE_EXISTS = 48 +_INDEX_CONFLICTS = (85, 86) + + +def _collection_type(document: _Document) -> str: + if document['type'] != 'collection': + return document['type'] + options = document['options'] + if options.get('capped'): + return 'capped collection' + if 'timeseries' in options: + return 'timeseries' + if document['name'].startswith('system.'): + return 'system collection' + return 'collection' + + +def _collation(document: _Document) -> _Document: + collation = document.get('collation', {}) + return {} if collation.get('locale') == 'simple' else dict(collation) + + +def _has_id_index(document: _Document) -> bool: + options = document['options'] + index = document.get('idIndex', {}) + clustered = options.get('clusteredIndex', {}) + return (index.get('key') == {'_id': 1} and _collation(index) == _collation(options)) or ( + clustered.get('key') == {'_id': 1} and clustered.get('unique') is True + ) + + +def _equivalent_index(indexes: list[_Document], fields: tuple[str, ...], collation: _Document) -> _Document | None: + for index in indexes: + key = index['key'] + if ( + tuple(key) == fields + and all(direction in (1, -1) for direction in key.values()) + and index.get('unique') is True + and not index.get('sparse', False) + and 'partialFilterExpression' not in index + and _collation(index) == collation + ): + return index + return None + + +def _index_request(fields: tuple[str, ...], existing: _Document | None) -> _IndexRequest: + if existing is None: + # Use a separate stable name so an ordinary index with the driver's default name can coexist. + digest = sha256(repr(fields).encode('utf-8')).hexdigest()[:16] + return [(name, 1) for name in fields], {'name': f'database_unique_{digest}', 'unique': True} + + # Reissue the matching definition, preserving its name and options. The collection supplies + # the already-verified collation. A successful createIndexes confirms the index is ready. + options = {name: value for name, value in existing.items() if name not in ('key', 'ns', 'collation')} + return list(existing['key'].items()), options + + +def _target(database: str, collection: str) -> str | None: + if isinstance(database, str) and isinstance(collection, str): + return f'{database}.{collection}' + return None + + +def _rows(cursor: Any) -> list[_Document]: + try: + rows = list(cursor) + except BaseException as original: + try: + cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(original, (cleanup,)) + raise + cursor.close() + return rows + + +async def _async_rows(cursor: Any) -> list[_Document]: + try: + rows = [row async for row in cursor] + except BaseException as original: + try: + await cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(original, (cleanup,)) + raise + await cursor.close() + return rows + + +def _describe(database: Any, collection: str) -> _Document | None: + rows = _rows(database.list_collections(filter={'name': collection})) + return rows[0] if rows else None + + +async def _async_describe(database: Any, collection: str) -> _Document | None: + rows = await _async_rows(await database.list_collections(filter={'name': collection})) + return rows[0] if rows else None + + +class _MongoDBSchema(BoundSchema): + def _name(self, name: str, operation: str, target: str | None) -> None: + if not isinstance(name, str) or not name.strip() or '\x00' in name: + raise ConfigurationError( + 'Expected an explicit MongoDB database or collection name.', + **self._context_fields(operation, target), + ) + + def _database(self, name: str, operation: str, target: str | None) -> Any: + self._name(name, operation, target) + try: + return self._connection.native.get_database(name) + except InvalidName as original: + raise ConfigurationError( + 'Invalid MongoDB database name.', original=original, **self._context_fields(operation, target) + ) from original + + def _collection(self, database: Any, name: str, operation: str, target: str | None) -> Any: + self._name(name, operation, target) + try: + return database.get_collection(name) + except InvalidName as original: + raise ConfigurationError( + 'Invalid MongoDB collection name.', original=original, **self._context_fields(operation, target) + ) from original + + def _require_acknowledged(self, database: Any, operation: str, target: str | None) -> None: + if not database.write_concern.acknowledged: + raise ConfigurationError( + 'MongoDB schema writes require acknowledged write concern.', + **self._context_fields(operation, target), + ) + + def _key_fields(self, key_fields: Sequence[str], target: str | None) -> tuple[str, ...]: + try: + fields = _names(key_fields, empty=False) + for name in fields: + name.encode('utf-8') + except (ConfigurationError, UnicodeError) as original: + raise ConfigurationError( + 'Expected distinct, nonempty MongoDB record key fields.', + original=original, + **self._context_fields('ensure_collection', target), + ) from original + return fields + + def _conflict( + self, difference: str, target: str | None, original: OperationFailure | None = None + ) -> SchemaConflictError: + return SchemaConflictError( + 'MongoDB collection does not satisfy the requested structure.', + differences=(difference,), + original=original, + **self._context_fields('ensure_collection', target), + ) + + def _require_collection( + self, + metadata: _Document | None, + fields: tuple[str, ...], + target: str | None, + original: OperationFailure | None = None, + ) -> _Document: + if metadata is None: + raise self._conflict('Collection is absent from the visible catalog after creation.', target) + kind = _collection_type(metadata) + if kind != 'collection': + raise self._conflict(f'Expected an ordinary collection, found {kind}.', target, original) from original + if fields == _ID_FIELDS and not _has_id_index(metadata): + raise self._conflict('No compatible built-in _id identity exists.', target, original) from original + return metadata + + +class MongoDBSchema(_MongoDBSchema): + """Manage MongoDB schema using a borrowed synchronous connection.""" + + def __init__(self, connection: Connection) -> None: + super().__init__(connection, backend='mongodb', is_async=False) + + def list_databases(self) -> list[str]: + """Return database names visible to the current account.""" + with self._operation('list_databases'): + return self._connection.native.list_database_names() + + def database_exists(self, name: str) -> bool: + """Check whether a database is visible to the current account.""" + operation = 'database_exists' + with self._operation(operation, name): + self._database(name, operation, name) + return name in self._connection.native.list_database_names() + + def list_collections(self, database: str) -> list[str]: + """Return visible ordinary collections in the named database.""" + operation = 'list_collections' + with self._operation(operation, database): + native = self._database(database, operation, database) + rows = _rows(native.list_collections(filter={'type': 'collection'})) + return [row['name'] for row in rows if _collection_type(row) == 'collection'] + + def collection_exists(self, database: str, collection: str) -> bool: + """Check whether the named ordinary collection is visible.""" + operation = 'collection_exists' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + metadata = _describe(native, collection) + return metadata is not None and _collection_type(metadata) == 'collection' + + def describe_collection(self, database: str, collection: str) -> dict[str, Any] | None: + """Return native collection metadata when visible, including for views.""" + operation = 'describe_collection' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + metadata = _describe(native, collection) + return None if metadata is None else dict(metadata) + + def ensure_collection(self, database: str, collection: str, *, key_fields: Sequence[str] = _ID_FIELDS) -> None: + """Prepare an ordinary collection and confirm its record identity. + + Notes + ----- + The default identity uses the built-in _id index. Other key fields + require a complete unique index. An index failure leaves the collection + intact. Writes require acknowledged write concern. + """ + operation = 'ensure_collection' + target = _target(database, collection) + with self._operation(operation, target): + fields = self._key_fields(key_fields, target) + native = self._database(database, operation, target) + handle = self._collection(native, collection, operation, target) + self._require_acknowledged(native, operation, target) + metadata = _describe(native, collection) + creation_error = None + if metadata is None: + try: + native.create_collection(collection, check_exists=False) + except OperationFailure as original: + if original.code != _NAMESPACE_EXISTS: + raise + metadata = _describe(native, collection) + if metadata is None: + raise + creation_error = original + else: + metadata = _describe(native, collection) + metadata = self._require_collection(metadata, fields, target, creation_error) + if fields != _ID_FIELDS: + self._ensure_index(handle, fields, _collation(metadata['options']), target) + + def _ensure_index(self, collection: Any, fields: tuple[str, ...], collation: _Document, target: str | None) -> None: + existing = _equivalent_index(_rows(collection.list_indexes()), fields, collation) + keys, options = _index_request(fields, existing) + try: + collection.create_index(keys, **options) + except OperationFailure as original: + if original.code not in _INDEX_CONFLICTS: + raise + concurrent = _equivalent_index(_rows(collection.list_indexes()), fields, collation) + if concurrent is None: + raise self._conflict( + 'No compatible complete unique index could be created.', target, original + ) from original + confirmed_keys, confirmed_options = _index_request(fields, concurrent) + if (confirmed_keys, confirmed_options) == (keys, options): + raise + collection.create_index(confirmed_keys, **confirmed_options) + + def drop_collection(self, database: str, collection: str) -> None: + """Drop the exact collection namespace using the native driver.""" + operation = 'drop_collection' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + self._require_acknowledged(native, operation, target) + native.drop_collection(collection) + + def drop_database(self, name: str) -> None: + """Drop the exact named database using the native driver.""" + operation = 'drop_database' + with self._operation(operation, name): + native = self._database(name, operation, name) + self._require_acknowledged(native, operation, name) + self._connection.native.drop_database(name) + + +class AsyncMongoDBSchema(_MongoDBSchema): + """Manage MongoDB schema using a borrowed asynchronous connection.""" + + def __init__(self, connection: AsyncConnection) -> None: + super().__init__(connection, backend='mongodb', is_async=True) + + async def list_databases(self) -> list[str]: + """Return database names visible to the current account.""" + with self._operation('list_databases'): + return await self._connection.native.list_database_names() + + async def database_exists(self, name: str) -> bool: + """Check whether a database is visible to the current account.""" + operation = 'database_exists' + with self._operation(operation, name): + self._database(name, operation, name) + return name in await self._connection.native.list_database_names() + + async def list_collections(self, database: str) -> list[str]: + """Return visible ordinary collections in the named database.""" + operation = 'list_collections' + with self._operation(operation, database): + native = self._database(database, operation, database) + rows = await _async_rows(await native.list_collections(filter={'type': 'collection'})) + return [row['name'] for row in rows if _collection_type(row) == 'collection'] + + async def collection_exists(self, database: str, collection: str) -> bool: + """Check whether the named ordinary collection is visible.""" + operation = 'collection_exists' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + metadata = await _async_describe(native, collection) + return metadata is not None and _collection_type(metadata) == 'collection' + + async def describe_collection(self, database: str, collection: str) -> dict[str, Any] | None: + """Return native collection metadata when visible, including for views.""" + operation = 'describe_collection' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + metadata = await _async_describe(native, collection) + return None if metadata is None else dict(metadata) + + async def ensure_collection( + self, database: str, collection: str, *, key_fields: Sequence[str] = _ID_FIELDS + ) -> None: + """Prepare an ordinary collection and confirm its record identity. + + Notes + ----- + The default identity uses the built-in _id index. Other key fields + require a complete unique index. An index failure leaves the collection + intact. Writes require acknowledged write concern. + """ + operation = 'ensure_collection' + target = _target(database, collection) + with self._operation(operation, target): + fields = self._key_fields(key_fields, target) + native = self._database(database, operation, target) + handle = self._collection(native, collection, operation, target) + self._require_acknowledged(native, operation, target) + metadata = await _async_describe(native, collection) + creation_error = None + if metadata is None: + try: + await native.create_collection(collection, check_exists=False) + except OperationFailure as original: + if original.code != _NAMESPACE_EXISTS: + raise + metadata = await _async_describe(native, collection) + if metadata is None: + raise + creation_error = original + else: + metadata = await _async_describe(native, collection) + metadata = self._require_collection(metadata, fields, target, creation_error) + if fields != _ID_FIELDS: + await self._ensure_index(handle, fields, _collation(metadata['options']), target) + + async def _ensure_index( + self, collection: Any, fields: tuple[str, ...], collation: _Document, target: str | None + ) -> None: + existing = _equivalent_index(await _async_rows(await collection.list_indexes()), fields, collation) + keys, options = _index_request(fields, existing) + try: + await collection.create_index(keys, **options) + except OperationFailure as original: + if original.code not in _INDEX_CONFLICTS: + raise + concurrent = _equivalent_index(await _async_rows(await collection.list_indexes()), fields, collation) + if concurrent is None: + raise self._conflict( + 'No compatible complete unique index could be created.', target, original + ) from original + confirmed_keys, confirmed_options = _index_request(fields, concurrent) + if (confirmed_keys, confirmed_options) == (keys, options): + raise + await collection.create_index(confirmed_keys, **confirmed_options) + + async def drop_collection(self, database: str, collection: str) -> None: + """Drop the exact collection namespace using the native driver.""" + operation = 'drop_collection' + target = _target(database, collection) + with self._operation(operation, target): + native = self._database(database, operation, target) + self._collection(native, collection, operation, target) + self._require_acknowledged(native, operation, target) + await native.drop_collection(collection) + + async def drop_database(self, name: str) -> None: + """Drop the exact named database using the native driver.""" + operation = 'drop_database' + with self._operation(operation, name): + native = self._database(name, operation, name) + self._require_acknowledged(native, operation, name) + await self._connection.native.drop_database(name) diff --git a/src/datvia/schema/_mysql.py b/src/datvia/schema/_mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..2ddc6535d42a244f215c0abf92e928798300392c --- /dev/null +++ b/src/datvia/schema/_mysql.py @@ -0,0 +1,466 @@ +"""Basic MySQL schema preparation on a borrowed Connector/Python connection.""" + +import re +from collections.abc import AsyncIterator +from collections.abc import Iterator +from contextlib import asynccontextmanager +from contextlib import contextmanager +from datetime import date +from datetime import datetime +from decimal import Decimal +from decimal import InvalidOperation +from typing import Any +from typing import NoReturn + +from mysql.connector.errorcode import ER_BAD_TABLE_ERROR +from mysql.connector.errorcode import ER_DB_CREATE_EXISTS +from mysql.connector.errorcode import ER_DB_DROP_EXISTS +from mysql.connector.errorcode import ER_TABLE_EXISTS_ERROR +from mysql.connector.errors import Error as MySQLError + +from ..connections import AsyncConnection +from ..connections import Connection +from ..errors import ConfigurationError +from ..errors import attach_cleanup_errors +from ._base import BoundSchema +from ._base import SchemaConflictError +from ._models import _UNSET +from ._models import Column +from ._models import TableSpec +from ._models import _identifier +from ._models import _invalid + +_CHARSET = re.compile(r'[A-Za-z][A-Za-z0-9_]{0,63}') +_LEGACY_INTEGER_WIDTH = re.compile(r'(int|bigint)\(\d+\)') +_DATABASES = 'SELECT SCHEMA_NAME FROM INFORMATION_SCHEMA.SCHEMATA ORDER BY SCHEMA_NAME' +_DATABASE = 'SELECT DEFAULT_CHARACTER_SET_NAME FROM INFORMATION_SCHEMA.SCHEMATA WHERE SCHEMA_NAME = %s' +_TABLES = """ +SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES +WHERE TABLE_SCHEMA = %s AND TABLE_TYPE = 'BASE TABLE' ORDER BY TABLE_NAME +""" +_TABLE = """ +SELECT t.TABLE_SCHEMA, t.TABLE_NAME, t.TABLE_TYPE, t.ENGINE, c.CHARACTER_SET_NAME, t.TABLE_COLLATION +FROM INFORMATION_SCHEMA.TABLES AS t +LEFT JOIN INFORMATION_SCHEMA.COLLATION_CHARACTER_SET_APPLICABILITY AS c + ON t.TABLE_COLLATION = c.COLLATION_NAME +WHERE t.TABLE_SCHEMA = %s AND t.TABLE_NAME = %s +""" +_COLUMNS = """ +SELECT COLUMN_NAME, COLUMN_TYPE, IS_NULLABLE, COLUMN_DEFAULT, EXTRA, GENERATION_EXPRESSION +FROM INFORMATION_SCHEMA.COLUMNS +WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s ORDER BY ORDINAL_POSITION +""" +_PRIMARY_KEY = """ +SELECT COLUMN_NAME FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE +WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND CONSTRAINT_NAME = 'PRIMARY' +ORDER BY ORDINAL_POSITION +""" + +_Row = tuple[Any, ...] + + +def _charset_name(value: str) -> str: + if type(value) is not str or _CHARSET.fullmatch(value) is None: + raise _invalid('Expected a MySQL character set name.') + name = value.lower() + return 'utf8mb3' if name == 'utf8' else name + + +def _qualified(database: str, table: str) -> str: + return f'{_identifier(database)}.{_identifier(table)}' + + +def _column_type(column: Column) -> str: + if column.type == 'decimal': + return f'decimal({column.precision or 10},{column.scale or 0})' + if column.type == 'datetime' and column.precision: + return f'datetime({column.precision})' + if column.type == 'varchar': + return f'varchar({column.length})' + if column.type == 'boolean': + return 'tinyint(1)' + return column.type + + +def _table_definition(qualified: str, spec: TableSpec) -> tuple[str, tuple[Any, ...]]: + if not isinstance(spec, TableSpec): + raise _invalid('MySQL table creation requires a TableSpec.') + columns = [] + literals = [] + for name, column in spec.columns.items(): + sql_type = 'BOOLEAN' if column.type == 'boolean' else _column_type(column).upper() + definitions = [_identifier(name), sql_type, 'NULL' if column.nullable else 'NOT NULL'] + if column.default is None: + definitions.append('DEFAULT NULL') + elif column.default is not _UNSET: + definitions.append('DEFAULT %s') + literals.append(column.default) + if column.auto_increment: + definitions.append('AUTO_INCREMENT') + columns.append(' '.join(definitions)) + key = ', '.join(map(_identifier, spec.primary_key)) + columns.append(f'PRIMARY KEY ({key})') + return f'CREATE TABLE IF NOT EXISTS {qualified} ({", ".join(columns)}) ENGINE=InnoDB', tuple(literals) + + +def _describe(table: _Row, columns: list[_Row], key: list[_Row]) -> dict[str, Any]: + database, name, _, engine, charset, collation = table + fields = {} + for column, native_type, nullable, default, extra, generated in columns: + flags = (extra or '').lower() + fields[column] = dict( + type=native_type, + nullable=nullable == 'YES', + default=default, + auto_increment='auto_increment' in flags.split(), + generated=bool(generated) or 'virtual generated' in flags or 'stored generated' in flags, + generation_expression=generated, + extra=extra, + ) + return dict( + database=database, + table=name, + columns=fields, + primary_key=tuple(column for (column,) in key), + engine=engine, + charset=charset, + collation=collation, + ) + + +def _compare_default(column: Column, actual: dict[str, Any]) -> bool: + if column.default is _UNSET: + return True + if 'default_generated' in (actual['extra'] or '').lower().split(): + return False + value = actual['default'] + if value is None or column.default is None: + return value is column.default + + # INFORMATION_SCHEMA exposes even numeric and temporal literal defaults as text. + try: + if column.type in ('int', 'bigint', 'boolean'): + value = int(value) + elif column.type == 'decimal': + value = Decimal(value) + elif column.type == 'double': + value = float(value) + elif column.type == 'date' and isinstance(value, str): + value = date.fromisoformat(value) + elif column.type == 'datetime' and isinstance(value, str): + value = datetime.fromisoformat(value) + return value == column.default + except (InvalidOperation, TypeError, ValueError): + return False + + +def _compare_table(spec: TableSpec, record: dict[str, Any]) -> tuple[str, ...]: + differences = [] + native_columns = {name.casefold(): (name, field) for name, field in record['columns'].items()} + for name, column in spec.columns.items(): + native = native_columns.pop(name.casefold(), None) + if native is None: + differences.append(f'Missing column {name!r}.') + continue + _, actual = native + native_type = actual['type'].lower() + width = _LEGACY_INTEGER_WIDTH.fullmatch(native_type) + if width is not None: + native_type = width[1] + elif native_type == 'datetime(0)': + native_type = 'datetime' + expected_type = _column_type(column) + if expected_type != native_type: + differences.append(f'Column {name!r} has type {actual["type"]}, expected {expected_type}.') + if actual['nullable'] != column.nullable: + differences.append(f'Column {name!r} has different nullability.') + if actual['generated']: + differences.append(f'Column {name!r} is generated.') + if 'invisible' in (actual['extra'] or '').lower().split(): + differences.append(f'Column {name!r} is invisible in default projections.') + if actual['auto_increment'] != column.auto_increment: + differences.append(f'Column {name!r} has different auto-increment behavior.') + if not _compare_default(column, actual): + differences.append(f'Column {name!r} has a different default.') + + expected_key = tuple(name.casefold() for name in spec.primary_key) + native_key = tuple(name.casefold() for name in record['primary_key']) + if expected_key != native_key: + differences.append(f'Primary key {record["primary_key"]!r} differs from {tuple(spec.primary_key)!r}.') + for name, column in native_columns.values(): + if not (column['nullable'] or column['default'] is not None or column['auto_increment'] or column['generated']): + differences.append(f'Extra column {name!r} cannot be omitted.') + return tuple(differences) + + +def _decode_rows(native: Any, rows: list[_Row]) -> list[_Row]: + charset = getattr(native, 'python_charset', 'utf8') + return [tuple(value.decode(charset) if isinstance(value, bytes) else value for value in row) for row in rows] + + +class _MySQLSchema(BoundSchema): + def _require_ddl(self, allowed: bool, operation: str, target: str) -> None: + if not allowed: + raise ConfigurationError( + 'MySQL schema writes require autocommit and no active transaction.', + **self._context_fields(operation, target), + ) + + def _check_database(self, name: str, charset: str, rows: list[_Row]) -> None: + if not rows: + differences = ('Database is not visible after creation.',) + elif rows[0][0].lower() != charset: + differences = (f'Database has character set {rows[0][0]}, expected {charset}.',) + else: + return + self._conflict('ensure_database', name, differences) + + def _check_table_kind(self, target: str, rows: list[_Row]) -> None: + if not rows: + self._conflict('ensure_table', target, ('Table is not visible after creation.',)) + if rows[0][2] != 'BASE TABLE': + self._conflict('ensure_table', target, ('Existing object is not a base table.',)) + + def _check_table(self, target: str, spec: TableSpec, actual: dict[str, Any]) -> None: + differences = _compare_table(spec, actual) + if differences: + self._conflict('ensure_table', target, differences) + + def _conflict(self, operation: str, target: str, differences: tuple[str, ...]) -> NoReturn: + raise SchemaConflictError( + 'MySQL schema does not satisfy the declared structure.', + differences=differences, + **self._context_fields(operation, target), + ) + + +class MySQLSchema(_MySQLSchema): + """Manage MySQL databases and base tables synchronously.""" + + def __init__(self, connection: Connection) -> None: + super().__init__(connection, backend='mysql', is_async=False) + + @contextmanager + def _cursor(self) -> Iterator[Any]: + cursor = self._connection.native.cursor(buffered=True, raw=False) + try: + yield cursor + except BaseException as error: + try: + cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(error, (cleanup,)) + raise + else: + cursor.close() + + def _read(self, sql: str, params: tuple[Any, ...] = ()) -> list[_Row]: + with self._cursor() as cursor: + cursor.execute(sql, params) + return _decode_rows(self._connection.native, cursor.fetchall()) + + def _write(self, sql: str, params: tuple[Any, ...], *, ignore_errno: int) -> MySQLError | None: + with self._cursor() as cursor: + try: + cursor.execute(sql, params) + except MySQLError as error: + if error.errno != ignore_errno: + raise + return error + return None + + def _check_writable(self, operation: str, target: str) -> None: + native = self._connection.native + autocommit = not native.in_transaction and native.autocommit + self._require_ddl(autocommit and not native.in_transaction, operation, target) + + def ensure_database(self, name: str, *, charset: str = 'utf8mb4') -> None: + quoted = _identifier(name) + charset = _charset_name(charset) + with self._operation('ensure_database', name): + self._check_writable('ensure_database', name) + rows = self._read(_DATABASE, (name,)) + if not rows: + error = self._write( + f'CREATE DATABASE IF NOT EXISTS {quoted} DEFAULT CHARACTER SET {charset}', + (), + ignore_errno=ER_DB_CREATE_EXISTS, + ) + rows = self._read(_DATABASE, (name,)) + if not rows and error is not None: + raise error + self._check_database(name, charset, rows) + + def list_databases(self) -> list[str]: + with self._operation('list_databases'): + return [name for (name,) in self._read(_DATABASES)] + + def database_exists(self, name: str) -> bool: + _identifier(name) + with self._operation('database_exists', name): + return bool(self._read(_DATABASE, (name,))) + + def ensure_table(self, database: str, table: str, spec: TableSpec) -> None: + statement, literals = _table_definition(_qualified(database, table), spec) + target = f'{database}.{table}' + params = (database, table) + with self._operation('ensure_table', target): + self._check_writable('ensure_table', target) + rows = self._read(_TABLE, params) + if not rows: + error = self._write(statement, literals, ignore_errno=ER_TABLE_EXISTS_ERROR) + rows = self._read(_TABLE, params) + if not rows and error is not None: + raise error + self._check_table_kind(target, rows) + actual = _describe(rows[0], self._read(_COLUMNS, params), self._read(_PRIMARY_KEY, params)) + self._check_table(target, spec, actual) + + def list_tables(self, database: str) -> list[str]: + _identifier(database) + with self._operation('list_tables', database): + return [table for (table,) in self._read(_TABLES, (database,))] + + def table_exists(self, database: str, table: str) -> bool: + _qualified(database, table) + with self._operation('table_exists', f'{database}.{table}'): + rows = self._read(_TABLE, (database, table)) + return bool(rows and rows[0][2] == 'BASE TABLE') + + def describe_table(self, database: str, table: str) -> dict[str, Any] | None: + _qualified(database, table) + params = (database, table) + with self._operation('describe_table', f'{database}.{table}'): + rows = self._read(_TABLE, params) + if not rows or rows[0][2] != 'BASE TABLE': + return None + return _describe(rows[0], self._read(_COLUMNS, params), self._read(_PRIMARY_KEY, params)) + + def drop_table(self, database: str, table: str) -> None: + qualified = _qualified(database, table) + target = f'{database}.{table}' + with self._operation('drop_table', target): + self._check_writable('drop_table', target) + self._write(f'DROP TABLE IF EXISTS {qualified}', (), ignore_errno=ER_BAD_TABLE_ERROR) + + def drop_database(self, name: str) -> None: + quoted = _identifier(name) + with self._operation('drop_database', name): + self._check_writable('drop_database', name) + self._write(f'DROP DATABASE IF EXISTS {quoted}', (), ignore_errno=ER_DB_DROP_EXISTS) + + +class AsyncMySQLSchema(_MySQLSchema): + """Manage MySQL databases and base tables using an async connection.""" + + def __init__(self, connection: AsyncConnection) -> None: + super().__init__(connection, backend='mysql', is_async=True) + + @asynccontextmanager + async def _cursor(self) -> AsyncIterator[Any]: + cursor = await self._connection.native.cursor(buffered=True, raw=False) + try: + yield cursor + except BaseException as error: + try: + await cursor.close() + except BaseException as cleanup: + attach_cleanup_errors(error, (cleanup,)) + raise + else: + await cursor.close() + + async def _read(self, sql: str, params: tuple[Any, ...] = ()) -> list[_Row]: + async with self._cursor() as cursor: + await cursor.execute(sql, params) + return _decode_rows(self._connection.native, await cursor.fetchall()) + + async def _write(self, sql: str, params: tuple[Any, ...], *, ignore_errno: int) -> MySQLError | None: + async with self._cursor() as cursor: + try: + await cursor.execute(sql, params) + except MySQLError as error: + if error.errno != ignore_errno: + raise + return error + return None + + async def _check_writable(self, operation: str, target: str) -> None: + native = self._connection.native + autocommit = not native.in_transaction and await native.get_autocommit() + self._require_ddl(autocommit and not native.in_transaction, operation, target) + + async def ensure_database(self, name: str, *, charset: str = 'utf8mb4') -> None: + quoted = _identifier(name) + charset = _charset_name(charset) + with self._operation('ensure_database', name): + await self._check_writable('ensure_database', name) + rows = await self._read(_DATABASE, (name,)) + if not rows: + error = await self._write( + f'CREATE DATABASE IF NOT EXISTS {quoted} DEFAULT CHARACTER SET {charset}', + (), + ignore_errno=ER_DB_CREATE_EXISTS, + ) + rows = await self._read(_DATABASE, (name,)) + if not rows and error is not None: + raise error + self._check_database(name, charset, rows) + + async def list_databases(self) -> list[str]: + with self._operation('list_databases'): + return [name for (name,) in await self._read(_DATABASES)] + + async def database_exists(self, name: str) -> bool: + _identifier(name) + with self._operation('database_exists', name): + return bool(await self._read(_DATABASE, (name,))) + + async def ensure_table(self, database: str, table: str, spec: TableSpec) -> None: + statement, literals = _table_definition(_qualified(database, table), spec) + target = f'{database}.{table}' + params = (database, table) + with self._operation('ensure_table', target): + await self._check_writable('ensure_table', target) + rows = await self._read(_TABLE, params) + if not rows: + error = await self._write(statement, literals, ignore_errno=ER_TABLE_EXISTS_ERROR) + rows = await self._read(_TABLE, params) + if not rows and error is not None: + raise error + self._check_table_kind(target, rows) + actual = _describe(rows[0], await self._read(_COLUMNS, params), await self._read(_PRIMARY_KEY, params)) + self._check_table(target, spec, actual) + + async def list_tables(self, database: str) -> list[str]: + _identifier(database) + with self._operation('list_tables', database): + return [table for (table,) in await self._read(_TABLES, (database,))] + + async def table_exists(self, database: str, table: str) -> bool: + _qualified(database, table) + with self._operation('table_exists', f'{database}.{table}'): + rows = await self._read(_TABLE, (database, table)) + return bool(rows and rows[0][2] == 'BASE TABLE') + + async def describe_table(self, database: str, table: str) -> dict[str, Any] | None: + _qualified(database, table) + params = (database, table) + with self._operation('describe_table', f'{database}.{table}'): + rows = await self._read(_TABLE, params) + if not rows or rows[0][2] != 'BASE TABLE': + return None + return _describe(rows[0], await self._read(_COLUMNS, params), await self._read(_PRIMARY_KEY, params)) + + async def drop_table(self, database: str, table: str) -> None: + qualified = _qualified(database, table) + target = f'{database}.{table}' + with self._operation('drop_table', target): + await self._check_writable('drop_table', target) + await self._write(f'DROP TABLE IF EXISTS {qualified}', (), ignore_errno=ER_BAD_TABLE_ERROR) + + async def drop_database(self, name: str) -> None: + quoted = _identifier(name) + with self._operation('drop_database', name): + await self._check_writable('drop_database', name) + await self._write(f'DROP DATABASE IF EXISTS {quoted}', (), ignore_errno=ER_DB_DROP_EXISTS) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dc5a03ae5f055f1fb9979f8f36ea416794a9c57c --- /dev/null +++ b/tests/README.md @@ -0,0 +1,56 @@ +# 测试指南 + +## 离线回归 + +记录操作由 `tests/test_crud_*.py` 覆盖,包含公共条件与生命周期、三个后端适配、多后端及同类多实例的资源隔离。 +批量状态、取消信号、字段投影、原始类型和资源归属均覆盖同步与异步。 +该功能的版本矩阵、Lua 模拟器及隔离服务结果见 [CRUD 实施与验收记录](../docs/crud-implementation.md)。 + +按 [贡献指南](../CONTRIBUTING.md) 在独立环境运行 `python -m pip install -e .`,以 editable 方式安装当前工作区及三个后端的驱动。从仓库根目录运行以下命令。 + +```bash +python -B -c "import datvia; print(datvia.__file__)" +python -B -m unittest discover -s tests -q +``` + +源码回归时,第一条命令的路径应指向当前工作区的 `src/datvia/__init__.py`。自动测试使用固定数据、驱动替身或打桩的原生驱动,保持离线。 + +## 按改动范围定位测试 + +| 变更范围 | 测试文件 | +| --- | --- | +| 显式配置、参数映射和配置隔离 | [test_connection_config.py](test_connection_config.py) | +| 公共连接句柄和同步、异步生命周期 | [test_connections.py](test_connections.py) | +| 异常分类、异常链和清理错误 | [test_connection_errors.py](test_connection_errors.py) | +| MongoDB、MySQL 原生工厂 | [test_mongodb_client.py](test_mongodb_client.py)、[test_mysql_client.py](test_mysql_client.py) | +| 高层后端适配 | [test_managed_mongodb.py](test_managed_mongodb.py)、[test_managed_mysql.py](test_managed_mysql.py)、[test_managed_redis.py](test_managed_redis.py) | + +用 `-p` 指定文件名可以运行定向测试。例如,以下命令覆盖 Redis 单机、Sentinel 和 Cluster 的高层适配。 + +```bash +python -B -m unittest discover -s tests -p "test_managed_redis.py" -q +``` + +新增回归沿用 `unittest.TestCase` 和 `unittest.IsolatedAsyncioTestCase`。涉及后端适配时,按改动覆盖可用的同步和异步路径,并核对失败条件、清理和兼容性。驱动兼容测试在独立环境中运行,记录实际导入的版本。 + +## 账号与权限管理验证 + +`python -B -m unittest discover -s tests -p "test_accounts*.py" -v` 运行账号管理离线测试。真实授权验证使用三个后端各自的验收脚本,配置格式、服务端和驱动组合以及实测结果见[账号管理验收记录](../docs/account-management-validation.md)。 + +## 隔离服务验证 + +[scripts/verify_connections.py](../scripts/verify_connections.py) 使用 `--config` 读取测试环境的 JSON 配置。JSON 的键为连接名称,值为 `ConnectionConfig` 的关键字参数。配置文件保存在仓库外,并指向隔离测试实例。 + +```bash +python -B scripts/verify_connections.py --config "<测试配置文件路径>" +``` + +脚本探测同步与异步连接,验证实例独立关闭,并对 MySQL 执行只读的 `SELECT 1`。分别记录离线回归与真实服务验证的结果,具体环境和实际结果见 [连接管理验收记录](../docs/connection-management-validation.md)。 + +## 基础结构与键值封装验收 + +新增测试覆盖 `tests/test_schema_binding.py`、`tests/test_schema_mysql.py`、`tests/test_schema_mongodb.py`、`tests/test_crud_strings.py` 和 `tests/test_basic_verifier.py`。依次验证公共异常与生命周期、基础库表与集合、Redis String 及验收清理规则。 + +`scripts/verify_basics.py --config <隔离配置文件> --mode both` 使用显式 ConnectionConfig 参数映射,创建并清理随机命名的验收数据库与键,覆盖初始化、投影、单条/批量 CRUD、原生访问、两个同类连接和多个后端同时在线。该脚本会写入数据,配置必须指向隔离服务;MySQL 配置不选择业务数据库,且 options.autocommit 必须为 true。可用 --mode sync 验证仅支持同步的 PyMongo 4.6。 + +脚本不读取环境变量、不输出配置或原始异常详情;失败摘要仅包含错误类型、操作名、文件名、行号和清理错误数量。不要使用 Python -O 运行验收。版本矩阵与真实服务边界见 [基础使用封装验收记录](../docs/crud-basic-implementation.md)。 diff --git a/tests/test_accounts_binding.py b/tests/test_accounts_binding.py new file mode 100644 index 0000000000000000000000000000000000000000..c0ba90eb8c4f460b5428bd05ac78df2cb60c7be8 --- /dev/null +++ b/tests/test_accounts_binding.py @@ -0,0 +1,106 @@ +"""Account factories must not own connections or perform implicit probes.""" + +import asyncio +import os +import subprocess +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts._base import Binding +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import ConnectionClosedError +from datvia.errors import DatabaseConnectionError + + +class BindingTests(unittest.TestCase): + def test_import_without_drivers(self): + source = str(Path(__file__).resolve().parents[1] / 'src') + code = ( + 'import sys; from datvia.accounts import *; ' + "assert not {'pymongo', 'redis', 'mysql.connector'}.intersection(sys.modules)" + ) + subprocess.run([sys.executable, '-B', '-S', '-c', code], env={**os.environ, 'PYTHONPATH': source}, check=True) + + def test_borrow_and_close(self): + client = object() + handle = Connection( + client, SimpleNamespace(classify=lambda exc: None, close=lambda native: None), 'test', 'mongodb', 'single' + ) + bound = Binding(handle, 'mongodb', is_async=False) + self.assertIs(bound.native, client) + with self.assertRaises(ConfigurationError): + Binding(handle, 'mysql', is_async=False) + handle.close() + with self.assertRaises(ConnectionClosedError): + with bound.operation('read'): + self.fail('closed handle entered the operation') + + def test_write_uncertainty_and_safe_error(self): + handle = Connection(object(), SimpleNamespace(classify=lambda exc: None), 'test', 'mysql', 'single') + bound = Binding(handle, 'mysql', is_async=False) + for write, error_type in [(False, DatabaseConnectionError), (True, AccountWriteOutcomeUnknown)]: + with self.subTest(write=write): + original = ConnectionResetError('private payload') + cleanup_error = RuntimeError('cursor close failed') + original.database_cleanup_errors = (cleanup_error,) + with self.assertRaises(error_type) as caught: + with bound.operation('grant_roles', 'target', write=write): + raise original + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.target, 'target') + self.assertEqual(caught.exception.database_cleanup_errors, (cleanup_error,)) + self.assertNotIn('private payload', str(caught.exception)) + + def test_native_binding_checks_mode_and_sentinel_pools(self): + import redis + from redis.asyncio.sentinel import Sentinel as AsyncSentinel + from redis.sentinel import Sentinel + + for client, is_async in ((redis.Redis(), False), (redis.asyncio.Redis(), True)): + with self.subTest(is_async=is_async): + self.assertIs(Binding(client, 'redis', is_async=is_async).native, client) + with self.assertRaises(ConfigurationError): + Binding(client, 'redis', is_async=not is_async) + with self.assertRaises(ConfigurationError): + Binding(client.pipeline(), 'redis', is_async=is_async) + for cls, is_async in ((Sentinel, False), (AsyncSentinel, True)): + primary = cls([('127.0.0.1', 26379)]).master_for('test') + handle = Connection(primary, SimpleNamespace(classify=lambda exc: None), 'r', 'redis', 'single') + with self.assertRaises(ConfigurationError): + Binding(primary, 'redis', is_async=is_async) + if not is_async: + with self.assertRaises(ConfigurationError): + Binding(handle, 'redis', is_async=False) + + def test_cancelled_write_is_propagated(self): + handle = Connection(object(), SimpleNamespace(classify=lambda exc: None), 'test', 'mysql', 'single') + bound = Binding(handle, 'mysql', is_async=False) + with self.assertRaises(asyncio.CancelledError): + with bound.operation('write', write=True): + raise asyncio.CancelledError() + + def test_foreign_loop(self): + async def make_binding(): + handle = AsyncConnection( + object(), + SimpleNamespace(classify=lambda exc: None), + 'async', + 'mongodb', + 'single', + asyncio.get_running_loop(), + ) + return Binding(handle, 'mongodb', is_async=True) + + async def use(binding): + with binding.operation('read'): + self.fail('entered using a different loop') + + binding = asyncio.run(make_binding()) + with self.assertRaises(ConfigurationError): + asyncio.run(use(binding)) diff --git a/tests/test_accounts_mongodb.py b/tests/test_accounts_mongodb.py new file mode 100644 index 0000000000000000000000000000000000000000..e68ea9c572e2d49b000bcea5a9c914fde029e3e7 --- /dev/null +++ b/tests/test_accounts_mongodb.py @@ -0,0 +1,245 @@ +"""MongoDB account commands, secret projection, and dispatch boundaries.""" + +import asyncio +import importlib.util +import unittest +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +from datvia.accounts import Accounts +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import AsyncAccounts +from datvia.accounts import MongoPrivilege +from datvia.accounts import MongoRole +from datvia.accounts import MongoUser +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import DatabaseTimeoutError +from datvia.errors import PermissionDeniedError + +HAS_DRIVER = importlib.util.find_spec('pymongo') is not None + + +def fixture(asynchronous=False): + from pymongo.write_concern import WriteConcern + + from datvia._adapters.mongodb import Adapter + + client = MagicMock() + database = client.__getitem__.return_value + database.write_concern = WriteConcern() + if asynchronous: + database.command = AsyncMock() + handle = AsyncConnection(client, Adapter, 'mg', 'mongodb', 'single', asyncio.get_running_loop()) + admin = AsyncAccounts.mongodb(handle) + else: + handle = Connection(client, Adapter, 'mg', 'mongodb', 'single') + admin = Accounts.mongodb(handle) + return client, database, admin + + +class ModelTests(unittest.TestCase): + def test_owns_permissions_and_validates_names(self): + resource = {'db': '', 'collection': ''} + actions = ['find'] + value = MongoPrivilege(resource, actions) + resource['db'] = 'changed' + actions.clear() + self.assertEqual(dict(value.resource), {'db': '', 'collection': ''}) + self.assertEqual(value.actions, ('find',)) + with self.assertRaises(TypeError): + value.resource['db'] = 'mutated' + for args in [({}, ['find']), ({'cluster': 1}, ['find']), ({'cluster': True}, [])]: + with self.subTest(args=args), self.assertRaises(ValueError): + MongoPrivilege(*args) + with self.assertRaises(TypeError): + MongoPrivilege({'cluster': True}, 'find') + for value in [None, '', '\0']: + with self.subTest(value=value), self.assertRaises(ValueError): + MongoRole('admin', value) + + +@unittest.skipUnless(HAS_DRIVER, 'requires PyMongo') +class MongoTests(unittest.TestCase): + def setUp(self): + self.client, self.db, self.admin = fixture() + self.user = MongoUser('staff', 'reader') + self.role = MongoRole('reports', 'read') + + def test_selection_and_projection(self): + self.db.command.return_value = { + 'ok': 1, + 'users': [ + { + 'db': 'staff', + 'user': 'reader', + 'roles': [{'db': 'reports', 'role': 'read'}], + 'credentials': {'password': 'private'}, + 'customData': {'code': 'secret'}, + } + ], + } + info = self.admin.list_users(auth_db='staff')[0] + self.client.__getitem__.assert_called_once_with('staff') + self.db.command.assert_called_once_with({'usersInfo': 1, 'showCustomData': False, 'showCredentials': False}) + self.assertEqual(info.user, self.user) + self.assertEqual(info.roles, (self.role,)) + self.assertIsNone(info.inherited_privileges) + self.assertNotIn('private', repr(info)) + self.assertNotIn('secret', repr(info)) + self.admin.list_users_all_databases() + self.client.__getitem__.assert_called_with('admin') + self.assertEqual(self.db.command.call_args.args[0]['usersInfo'], {'forAllDBs': True}) + self.db.command.return_value = {'users': []} + self.assertIsNone(self.admin.get_user(self.user)) + self.db.command.return_value = {'roles': []} + self.assertIsNone(self.admin.get_role(self.role, expand_privileges=True)) + + def test_invalid_inputs_do_not_dispatch(self): + with self.assertRaises(TypeError): + self.admin.drop_role(self.user) + with self.assertRaises(TypeError): + self.admin.grant_roles(self.user, roles=['read']) + with self.assertRaises(ValueError): + self.admin.create_user(self.user, password='') + self.db.command.assert_not_called() + + def test_explicit_mutations_and_concern(self): + from pymongo.write_concern import WriteConcern + + self.db.command.return_value = {'ok': 1} + self.db.write_concern = WriteConcern(w='majority') + self.admin.grant_roles(self.user, roles=[self.role]) + self.db.command.assert_called_once_with( + { + 'grantRolesToUser': 'reader', + 'roles': [{'db': 'reports', 'role': 'read'}], + 'writeConcern': {'w': 'majority'}, + } + ) + self.admin.change_password(self.user, password='replacement') + self.db.command.assert_called_with( + {'updateUser': 'reader', 'pwd': 'replacement', 'writeConcern': {'w': 'majority'}} + ) + self.db.write_concern = WriteConcern(w=0) + with self.assertRaises(ConfigurationError): + self.admin.drop_user(self.user) + self.assertEqual(self.db.command.call_count, 2) + + def test_failures_distinguish_point_of_dispatch(self): + from pymongo.errors import AutoReconnect + from pymongo.errors import OperationFailure + from pymongo.errors import ServerSelectionTimeoutError + from pymongo.errors import WriteConcernError + + for exc, expected in [ + (ServerSelectionTimeoutError('no target'), DatabaseTimeoutError), + (AutoReconnect('lost response'), AccountWriteOutcomeUnknown), + (WriteConcernError('concern', 64), AccountWriteOutcomeUnknown), + (OperationFailure('secret detail', 13), PermissionDeniedError), + ]: + with self.subTest(error=exc): + self.db.command.side_effect = exc + with self.assertRaises(expected) as caught: + self.admin.drop_user(self.user) + self.assertIs(caught.exception.__cause__, exc) + self.assertEqual(caught.exception.operation, 'drop_user') + self.assertNotIn('secret detail', str(caught.exception)) + self.assertEqual(self.db.command.call_count, 4) + self.db.command.side_effect = None + self.db.command.return_value = {'ok': 1, 'writeConcernError': {'code': 64, 'errmsg': 'not acknowledged'}} + with self.assertRaises(AccountWriteOutcomeUnknown) as caught: + self.admin.drop_user(self.user) + self.assertEqual(caught.exception.original.code, 64) + self.assertEqual(caught.exception.target, 'staff.reader') + + +@unittest.skipUnless(HAS_DRIVER, 'requires PyMongo') +class AsyncMongoTests(unittest.IsolatedAsyncioTestCase): + async def test_async_roles_read_write_and_cancel(self): + _, db, admin = fixture(True) + role = MongoRole('db', 'reader') + privilege = MongoPrivilege({'db': 'db', 'collection': 'c'}, ['find']) + db.command.return_value = { + 'roles': [ + { + 'db': 'db', + 'role': 'reader', + 'isBuiltin': False, + 'roles': [], + 'privileges': [], + 'inheritedRoles': [], + 'inheritedPrivileges': [], + 'credentials': 'secret', + } + ] + } + detail = await admin.get_role(role, expand_privileges=True) + self.assertEqual(detail.privileges, ()) + self.assertEqual(detail.inherited_privileges, ()) + self.assertNotIn('secret', repr(detail)) + await admin.grant_privileges(role, privileges=[privilege]) + db.command.assert_awaited_with( + { + 'grantPrivilegesToRole': 'reader', + 'privileges': [{'resource': {'db': 'db', 'collection': 'c'}, 'actions': ['find']}], + } + ) + db.command.side_effect = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError): + await admin.drop_role(role) + self.assertEqual(db.command.await_count, 3) + + +@unittest.skipUnless(HAS_DRIVER, 'requires PyMongo') +class MongoVerifierTests(unittest.IsolatedAsyncioTestCase): + async def test_uncertain_creation_is_reconciled_before_cleanup(self): + from scripts.verify_accounts_mongodb import create_fixture + + for applied in (True, False): + for identity, what in [(MongoUser('unique', 'analyst'), 'user'), (MongoRole('unique', 'reader'), 'role')]: + with self.subTest(applied=applied, what=what): + admin = AsyncMock() + create = getattr(admin, f'create_{what}') + get = getattr(admin, f'get_{what}') + remove = getattr(admin, f'drop_{what}') + error = AccountWriteOutcomeUnknown('lost response') + create.side_effect = error + get.side_effect = [None, object() if applied else None] + cleanup = [] + with self.assertRaises(AccountWriteOutcomeUnknown) as caught: + await create_fixture(admin, identity, cleanup) + self.assertIs(caught.exception, error) + self.assertEqual(cleanup, [(remove, identity)] if applied else []) + + async def test_preexisting_or_conflicting_target_never_enters_cleanup(self): + from scripts.verify_accounts_mongodb import create_fixture + + admin = AsyncMock() + identity = MongoUser('unique', 'analyst') + cleanup = [] + admin.get_user.return_value = object() + with self.assertRaises(RuntimeError): + await create_fixture(admin, identity, cleanup) + admin.create_user.assert_not_awaited() + admin.get_user.return_value = None + admin.create_user.side_effect = PermissionDeniedError('definite rejection') + with self.assertRaises(PermissionDeniedError): + await create_fixture(admin, identity, cleanup) + self.assertEqual(cleanup, []) + + async def test_readback_failure_retains_uncertain_creation(self): + from scripts.verify_accounts_mongodb import create_fixture + + admin = AsyncMock() + error = AccountWriteOutcomeUnknown('lost response') + read_error = DatabaseTimeoutError('disconnected') + admin.create_role.side_effect = error + admin.get_role.side_effect = [None, read_error] + cleanup = [] + with self.assertRaises(AccountWriteOutcomeUnknown) as caught: + await create_fixture(admin, MongoRole('unique', 'reader'), cleanup) + self.assertIs(caught.exception, error) + self.assertEqual(caught.exception.database_cleanup_errors, (read_error,)) + self.assertEqual(cleanup, []) diff --git a/tests/test_accounts_mysql.py b/tests/test_accounts_mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..c055d5c01787ceb24f63c6594647bc103f624693 --- /dev/null +++ b/tests/test_accounts_mysql.py @@ -0,0 +1,159 @@ +"""Offline checks for MySQL account SQL, resource boundaries and fault classification.""" + +import asyncio +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import MagicMock + +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import MySQLAccount +from datvia.accounts import MySQLScope +from datvia.accounts._base import Binding +from datvia.accounts._mysql import AsyncMySQLAccounts +from datvia.accounts._mysql import MySQLAccounts +from datvia.accounts._mysql_models import PrivilegeChange +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import DatabaseConnectionError +from datvia.errors import DependencyError + + +class MySQLAccountTests(unittest.TestCase): + def setUp(self): + self.user = MySQLAccount('user', 'localhost') + self.role = MySQLAccount('reader', '%') + self.native = MagicMock(in_transaction=False, autocommit=True, python_charset='utf8') + self.cur = self.native.cursor.return_value + bound = Binding( + Connection(self.native, SimpleNamespace(classify=lambda _: None), 'test', 'mysql', 'single'), + 'mysql', + is_async=False, + ) + self.admin = MySQLAccounts(bound) + + def test_exact_database_scopes(self): + change = PrivilegeChange.build(MySQLScope.database('team_a%'), ['select'], True, False) + stmt, params = change.statement(False, self.user.params()) + self.assertIn(r'`team\_a\%`.*', stmt) + self.assertIn('`team_a%`.*', change.statement(True, self.user.params())[0]) + self.assertEqual(params, ('user', 'localhost')) + change = PrivilegeChange.build(MySQLScope.table('db', 'report%s'), ['select'], True, False) + stmt, params = change.statement(False, self.user.params()) + self.assertIn('`report%s`', stmt) + self.assertEqual(params, {'account_user': 'user', 'account_host': 'localhost'}) + + def test_scope_validation(self): + for scope, privilege in [ + (MySQLScope.columns('db', 't', ['a']), 'DELETE'), + (MySQLScope.function('db', 'f'), 'DROP'), + (MySQLScope.table('db', 't'), 'CREATE USER'), + (MySQLScope.database('db'), 'FILE'), + ]: + with self.subTest(scope=scope), self.assertRaises(ConfigurationError): + self.admin.grant_privileges(self.user, scope, [privilege]) + self.native.cursor.assert_not_called() + with self.assertRaises(ConfigurationError): + MySQLScope.columns('db', 't', 'abc') + with self.assertRaises(ConfigurationError): + self.admin.grant_roles(self.user, roles=[]) + with self.assertRaises(ConfigurationError): + self.admin.grant_roles(self.user, roles=[self.role], admin_option='false') + + def test_punctuation_and_columns(self): + self.admin.grant_privileges(self.user, MySQLScope.columns('d', 'a`b', ['id']), ['select', 'update']) + sql, params = self.cur.execute.call_args.args + self.assertEqual(sql, 'GRANT SELECT (`id`), UPDATE (`id`) ON `d`.`a``b` TO %s@%s') + self.assertEqual(params, self.user.params()) + + def test_check_registration_of_dynamic_global_only(self): + self.cur.fetchall.return_value = [('CONNECTION_ADMIN', 'Server Admin', '')] + self.admin.grant_privileges(self.user, MySQLScope.global_(), ['connection_admin']) + self.assertEqual(self.cur.execute.call_args_list[0].args, ('SHOW PRIVILEGES',)) + self.assertIn('CONNECTION_ADMIN ON *.*', self.cur.execute.call_args.args[0]) + self.cur.execute.reset_mock() + with self.assertRaises(ConfigurationError): + self.admin.grant_privileges(self.user, MySQLScope.global_(), ['FAKE_ADMIN']) + self.assertEqual(self.cur.execute.call_count, 1) + + def test_explicit_empty_host_survives_catalog_projection(self): + self.cur.fetchall.return_value = [('analyst', '', 'N'), ('analyst', '%', 'N')] + identities = tuple(item.account for item in self.admin.list_accounts()) + self.assertEqual(identities, (MySQLAccount('analyst', ''), MySQLAccount('analyst', '%'))) + self.admin.get_account(identities[0]) + self.assertEqual(self.cur.execute.call_args.args[1], ('analyst', '')) + + def test_grant_context_and_role_membership(self): + self.cur.fetchall.return_value = [('GRANT ...',), ('REVOKE ...',)] + grants = self.admin.show_grants(self.user, using_roles=[self.role]) + self.assertEqual(grants.using_roles, (self.role,)) + self.assertEqual(grants.statements, ('GRANT ...', 'REVOKE ...')) + self.assertEqual( + self.cur.execute.call_args.args, + ('SHOW GRANTS FOR %s@%s USING %s@%s', self.user.params() + self.role.params()), + ) + self.admin.grant_roles(self.user, roles=[self.role], admin_option=True) + self.assertIn('WITH ADMIN OPTION', self.cur.execute.call_args.args[0]) + self.admin.set_default_roles(self.user, roles=[]) + self.assertIn('SET DEFAULT ROLE NONE', self.cur.execute.call_args.args[0]) + + def test_guard_leaves_transaction_to_caller(self): + for active, auto in [(True, True), (False, False)]: + self.native.in_transaction, self.native.autocommit = active, auto + with self.assertRaises(ConfigurationError): + self.admin.create_user(self.user, password='secret') + self.native.cursor.assert_not_called() + self.native.rollback.assert_not_called() + + def test_probe_failure_blocks_account_write(self): + self.cur.fetchone.return_value = ('wrong',) + with self.assertRaises(DependencyError): + self.admin.create_user(self.user, password="val'ue") + self.assertEqual(self.cur.execute.call_count, 1) + self.assertNotIn("val'ue", repr(self.cur.execute.call_args)) + self.cur.close.assert_called_once() + + def test_decodes_raw_non_utf8_metadata(self): + self.native.python_charset = 'latin1' + self.cur.fetchall.return_value = [(b'caf\xe9', b'%', b'N')] + self.assertEqual(self.admin.list_accounts()[0].account, MySQLAccount('caf\u00e9', '%')) + + def test_results_and_missing_account(self): + self.cur.fetchall.return_value = [(b'user', b'localhost', b'Y')] + info = self.admin.get_account(self.user) + self.assertEqual((info.account, info.locked), (self.user, True)) + self.cur.fetchall.return_value = [] + self.assertIsNone(self.admin.get_account(self.user)) + + def test_write_and_read_failures_preserve_stage_and_cause(self): + err = ConnectionResetError('sensitive') + self.cur.execute.side_effect = err + with self.assertRaises(AccountWriteOutcomeUnknown) as raised: + self.admin.change_password(self.user, password='private') + self.assertIs(raised.exception.original, err) + self.assertNotIn('sensitive', str(raised.exception)) + self.cur.close.assert_called_once() + with self.assertRaises(DatabaseConnectionError): + self.admin.show_grants(self.user) + self.cur.close.side_effect = RuntimeError('cursor') + with self.assertRaises(AccountWriteOutcomeUnknown) as dual: + self.admin.drop_user(self.user) + self.assertEqual(len(dual.exception.database_cleanup_errors), 1) + + +class AsyncMySQLTests(unittest.IsolatedAsyncioTestCase): + async def test_cancellation_and_cursor_cleanup(self): + cursor = AsyncMock() + cursor.execute.side_effect = asyncio.CancelledError() + native = SimpleNamespace( + cursor=AsyncMock(return_value=cursor), in_transaction=False, get_autocommit=AsyncMock(return_value=True) + ) + conn = AsyncConnection( + native, SimpleNamespace(classify=lambda _: None), 'qa', 'mysql', 'single', asyncio.get_running_loop() + ) + admin = AsyncMySQLAccounts(Binding(conn, 'mysql', is_async=True)) + with self.assertRaises(asyncio.CancelledError): + await admin.create_role(MySQLAccount('qa', 'localhost')) + cursor.close.assert_awaited_once() + self.assertFalse(conn.closed) diff --git a/tests/test_accounts_redis.py b/tests/test_accounts_redis.py new file mode 100644 index 0000000000000000000000000000000000000000..174ccc1f0b78393a066be68cca64cc6948ecf5d1 --- /dev/null +++ b/tests/test_accounts_redis.py @@ -0,0 +1,185 @@ +"""Redis account boundaries, rule compilation, and RESP permission views.""" + +import asyncio +import unittest +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import Mock + +from datvia.accounts import Accounts +from datvia.accounts import AccountWriteOutcomeUnknown +from datvia.accounts import AclRule +from datvia.accounts._base import Binding +from datvia.accounts._redis import AsyncRedisAccounts +from datvia.accounts._redis import RedisAccounts +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import DatabaseConnectionError +from datvia.errors import DriverError + + +def fake_binding(native): + native.connection_pool.get_encoder.return_value.encoding = 'utf-8' + adapter = SimpleNamespace(classify=lambda error: None) + handle = Connection(native, adapter, 'cache', 'redis', 'single') + return Binding(handle, 'redis', is_async=False) + + +class RedisRulesTests(unittest.TestCase): + def test_order_and_secret_hygiene(self): + native = Mock() + native.execute_command.return_value = b'OK' + acl = RedisAccounts(fake_binding(native)) + rules = [ + AclRule.clear_keys(), + AclRule.key_pattern('team:*'), + AclRule.allow_category('read'), + AclRule.deny_command('get'), + AclRule.replace_passwords([b'private-a', 'private-b']), + AclRule.remove_password('private-old'), + ] + acl.apply_user('reader', rules=rules) + native.execute_command.assert_called_once_with( + 'ACL', + 'SETUSER', + 'reader', + 'resetkeys', + '~team:*', + '+@read', + '-get', + 'resetpass', + b'>private-a', + '>private-b', + ' %s AND ({quote}value{quote} = %s OR {quote}note{quote} IS NULL) ' + f'AND {quote}tenant{quote} IN (%s, %s)) ORDER BY {quote}id{quote} DESC LIMIT %s OFFSET %s', + (4, literal, 5, 9, 2, 3), + ) + self.cursor.fetchall.assert_called_once_with() + self.cursor.close.assert_called_once_with() + self.native.close.assert_not_called() + + def test_get_distinguishes_empty_projection_absence_and_invalid_cardinality(self): + self.cursor.fetchall.side_effect = [[(1,)], [], [(1,), (1,)]] + + with sync_records(self.connection) as records: + self.assertEqual(records.get(key(1), fields=[]), {}) + self.assertIsNone(records.get(key(2), fields=[])) + with self.assertRaises(RecordIdentityError): + records.get(key(3), fields=[]) + + self.assertEqual(self.cursor.close.call_count, 3) + self.assertEqual(self.cursor.execute.call_args.args[1], (1, 3)) + self.assertTrue(self.cursor.execute.call_args.args[0].endswith('LIMIT 2')) + + def test_get_many_validates_projection_before_sending_any_chunk(self): + with sync_records(self.connection) as records: + with self.assertRaises(ConfigurationError): + records.get_many([key(1), key(2)], fields=('bad%s',), batch_size=1) + + self.native.cursor.assert_not_called() + + def test_get_many_uses_sql_position_for_composite_keys_and_projection(self): + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(2, 'one'), (0, 'other tenant')] + keys = [key(2, tenant=2), key(9), key(1), key(2, tenant=2)] + + with sync_records(self.connection) as records: + result = records.get_many(keys, fields=('value',)) + + self.assertEqual([item.index for item in result], list(range(4))) + self.assertEqual([item.key for item in result], keys) + self.assertEqual( + [item.record for item in result], + [{'value': 'other tenant'}, None, {'value': 'one'}, {'value': 'other tenant'}], + ) + sql, parameters = self.cursor.execute.call_args.args + self.assertEqual(parameters, (0, 2, 2, 1, 1, 9, 2, 1, 1)) + self.assertEqual(sql.count('LIMIT 2'), 3) + self.assertEqual(sql.count(' UNION ALL '), 2) + self.cursor.close.assert_called_once_with() + + def test_get_many_preserves_other_observations_when_key_matches_twice(self): + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(0, 'a'), (0, 'b'), (2, 'c')] + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.get_many([key(1), key(2), key(3), key(1)]) + + error = caught.exception + self.assertIsInstance(error.original, RecordIdentityError) + self.assertIs(error.__cause__, error.original) + self.assertIn('SELECT %s, ' + chr(96) + 'sample' + chr(96) + '.*', self.cursor.execute.call_args.args[0]) + items = error.partial_result.items + self.assertEqual( + [item.state for item in items], + [ + BatchState.FAILED, + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.FAILED, + ], + ) + self.assertIsNone(items[1].result.record) + self.assertEqual(items[2].result.record, {'value': 'c'}) + + def test_read_group_keeps_confirmed_observations_when_cursor_close_fails(self): + original = RuntimeError('close failed') + self.cursor.close.side_effect = original + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(0, 'confirmed')] + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.get_many([key(1), key(9), key(1), key(7)], fields=('value',), batch_size=2) + + error = caught.exception + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + items = error.partial_result.items + self.assertEqual( + [item.state for item in items], + [ + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.NOT_SENT, + ], + ) + self.assertEqual(items[0].result.record, {'value': 'confirmed'}) + self.assertIsNone(items[1].result.record) + self.assertEqual(items[2].result.record, {'value': 'confirmed'}) + self.cursor.execute.assert_called_once() + + def test_preflight_rejects_bad_later_value_and_key_alias_before_any_write(self): + with sync_records(self.connection) as records: + with self.assertRaises(ConfigurationError): + records.insert_many( + [{'tenant': 1, 'id': 1, 'value': 1}, {'tenant': 1, 'id': 2, 'value': ['unsupported']}], + batch_size=1, + ) + for unsupported in (float('inf'), float('nan'), Decimal('NaN')): + with self.subTest(unsupported=unsupported), self.assertRaises(ConfigurationError): + records.insert_many( + [{'tenant': 1, 'id': 1, 'value': 1}, {'tenant': 1, 'id': 2, 'value': unsupported}], + batch_size=1, + ) + with self.assertRaises(ConfigurationError): + records.update(key(1), set_values={'ID': 2}) + with self.assertRaises(ConfigurationError): + records.bulk_update([{'key': key(1), 'set_values': {'ID': 2}}]) + with self.assertRaises(ConfigurationError): + records.update_many(where={'tenant': 1}, set_values={'ID': 2}) + with self.assertRaises(ConfigurationError): + records.find(fields=('value%s',), limit=1) + with self.assertRaises(ConfigurationError): + records.find(fields=('value', 'VALUE'), limit=1) + with self.assertRaises(ConfigurationError): + records.find(order_by=(('value', 'asc'), ('VALUE', 'desc')), limit=1) + with self.assertRaises(UnsupportedOperationError): + records.count(where=Exists('value')) + + self.native.cursor.assert_not_called() + + def test_insert_preserves_lastrowid_without_inferred_identity_or_transaction_control(self): + self.cursor.lastrowid = 37 + + with sync_records(self.connection) as records: + result = records.insert({'value': Decimal('12.3400')}) + + self.assertEqual(result.inserted_count, 1) + self.assertEqual(result.native_result, {'rowcount': 1, 'lastrowid': 37}) + self.assertIsNone(result.generated_keys) + self.assertEqual(self.cursor.execute.call_args.args[1], (Decimal('12.3400'),)) + self.native.commit.assert_not_called() + self.native.rollback.assert_not_called() + + def test_update_preserves_changed_matched_and_unknown_modes(self): + self.native.isset_client_flag.return_value = True + self.native._cmysql.st_client_flag.side_effect = [0, ClientFlag.FOUND_ROWS, None] + self.cursor.rowcount = 2 + + with sync_records(self.connection) as records: + results = [records.update_many(where=In('id', (1, 2)), set_values={'value': 0}) for _ in range(3)] + + self.assertEqual( + [(item.matched_count, item.modified_count, item.native_result['rowcount_mode']) for item in results], + [(None, 2, 'changed'), (2, None, 'matched'), (None, None, 'unknown')], + ) + self.assertEqual(self.native._cmysql.st_client_flag.call_args_list, [call()] * 3) + self.native.isset_client_flag.assert_not_called() + + def test_bulk_update_groups_only_contiguous_equivalent_columns(self): + items = [ + {'key': key(1), 'set_values': {'value': 'a', 'note': 'first'}}, + {'key': key(2), 'set_values': {'note': 'second', 'value': 'b'}}, + {'key': key(3), 'set_values': {'value': 'c'}}, + {'key': key(4), 'set_values': {'value': 'd', 'note': 'last'}}, + ] + + with sync_records(self.connection) as records: + result = records.bulk_update(items) + + groups = self.cursor.executemany.call_args_list + self.assertEqual(len(groups), 3) + self.assertEqual(groups[0].args[1], [('first', 'a', 1, 1), ('second', 'b', 1, 2)]) + self.assertEqual(groups[1].args[1], [('c', 1, 3)]) + self.assertEqual(groups[2].args[1], [('last', 'd', 1, 4)]) + self.assertEqual(result.counts.modified_count, 3) + self.assertTrue(all(item.state is BatchState.ACKNOWLEDGED for item in result.items)) + self.assertIsNone(result.items[0].result) + self.assertEqual(result.items[2].result.modified_count, 1) + self.native.commit.assert_not_called() + + def test_delete_many_uses_whole_composite_keys_in_one_statement(self): + self.cursor.rowcount = 1 + + with sync_records(self.connection) as records: + result = records.delete_many([key(1, tenant=1), key(2, tenant=2)]) + + self.assertEqual(result.counts.deleted_count, 1) + sql, parameters = self.cursor.execute.call_args.args + quote = chr(96) + predicate = f'({quote}tenant{quote} = %s AND {quote}id{quote} = %s)' + self.assertEqual(sql, f'DELETE FROM {quote}sample{quote} WHERE {predicate} OR {predicate}') + self.assertEqual(parameters, (1, 1, 2, 2)) + self.cursor.executemany.assert_not_called() + self.assertTrue(all(item.result is None for item in result.items)) + + def test_failed_subgroup_is_unknown_and_remaining_statements_are_not_sent(self): + original = IntegrityError(errno=ER_DUP_ENTRY) + cleanup = RuntimeError('cursor close failed') + self.cursor.executemany.side_effect = [None, original] + self.cursor.close.side_effect = cleanup + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.insert_many( + [ + {'tenant': 1, 'id': 1, 'value': 'ok'}, + {'tenant': 1, 'id': 2, 'note': 'unknown'}, + {'tenant': 1, 'id': 3, 'note': 'unknown'}, + {'tenant': 1, 'id': 4, 'value': 'not sent'}, + ] + ) + + error = caught.exception + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + self.assertEqual(error.database_cleanup_errors, (cleanup,)) + self.assertEqual( + [item.state for item in error.partial_result.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.UNKNOWN, + BatchState.UNKNOWN, + BatchState.NOT_SENT, + ], + ) + self.assertEqual(error.partial_result.native_results[0]['rowcount'], 1) + self.assertIsNone(error.partial_result.counts.inserted_count) + self.assertEqual(self.cursor.executemany.call_count, 2) + + def test_cursor_acquisition_failure_keeps_precise_preceding_counts(self): + original = ProgrammingError('could not allocate cursor') + self.native.cursor.side_effect = [self.cursor, original] + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.insert_many([{'tenant': 1, 'id': 1}, {'tenant': 1, 'id': 2}], batch_size=1) + + partial = caught.exception.partial_result + self.assertEqual([item.state for item in partial.items], [BatchState.ACKNOWLEDGED, BatchState.NOT_SENT]) + self.assertEqual(partial.counts.inserted_count, 1) + self.assertIs(caught.exception.original, original) + + def test_cursor_acquisition_failure_has_no_sent_writes(self): + original = ProgrammingError('could not allocate cursor') + self.native.cursor.side_effect = original + + with sync_records(self.connection) as records: + with self.assertRaises(BatchOperationError) as caught: + records.delete_many([key(1), key(2)]) + + self.assertIs(caught.exception.original, original) + self.assertTrue(all(item.state is BatchState.NOT_SENT for item in caught.exception.partial_result.items)) + + def test_record_conflict_preserves_the_driver_error_and_close_failure(self): + original = IntegrityError(errno=ER_DUP_ENTRY) + cleanup = RuntimeError('close failed') + self.cursor.execute.side_effect = original + self.cursor.close.side_effect = cleanup + + with sync_records(self.connection) as records: + with self.assertRaises(RecordConflictError) as caught: + records.insert({'id': 1, 'tenant': 1}) + + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.database_cleanup_errors, (cleanup,)) + + def test_unknown_column_remains_a_driver_error_and_closes_cursor(self): + original = ProgrammingError(errno=1054) + self.cursor.execute.side_effect = original + + with sync_records(self.connection) as records: + with self.assertRaises(DriverError) as caught: + records.get(key(1), fields=('unknown_column',)) + + self.assertIs(caught.exception.original, original) + self.cursor.close.assert_called_once_with() + + +class AsyncMySQLCRUDTests(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.connection, self.native, self.cursor = make_handle(is_async=True) + + async def test_get_many_validates_projection_before_sending_any_chunk(self): + async with async_records(self.connection) as records: + with self.assertRaises(ConfigurationError): + await records.get_many([key(1), key(2)], fields=('bad%s',), batch_size=1) + + self.native.cursor.assert_not_awaited() + + async def test_async_reads_consume_and_close_the_native_cursor(self): + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(1, 'second'), (0, 'first')] + + async with async_records(self.connection) as records: + result = await records.get_many([key(1), key(2)], fields=('value',)) + + self.assertEqual([item.record for item in result], [{'value': 'first'}, {'value': 'second'}]) + self.native.cursor.assert_awaited_once_with(buffered=True) + self.cursor.execute.assert_awaited_once() + self.cursor.fetchall.assert_awaited_once_with() + self.cursor.close.assert_awaited_once_with() + self.native.close.assert_not_called() + + async def test_read_group_preserves_results_if_close_is_cancelled(self): + cancellation = asyncio.CancelledError() + self.cursor.close.side_effect = cancellation + self.cursor.column_names = ('position', 'value') + self.cursor.fetchall.return_value = [(1, 'confirmed')] + + async with async_records(self.connection) as records: + with self.assertRaises(asyncio.CancelledError) as caught: + await records.get_many([key(1), key(2), key(3)], fields=('value',), batch_size=2) + + self.assertIs(caught.exception, cancellation) + items = cancellation.database_partial_result.items + self.assertEqual( + [item.state for item in items], + [ + BatchState.ACKNOWLEDGED, + BatchState.ACKNOWLEDGED, + BatchState.NOT_SENT, + ], + ) + self.assertIsNone(items[0].result.record) + self.assertEqual(items[1].result.record, {'value': 'confirmed'}) + self.cursor.execute.assert_awaited_once() + + async def test_async_writes_preserve_rowcount_and_use_native_coroutines(self): + self.cursor.rowcount = 2 + self.native._cmysql = None + self.native.isset_client_flag.return_value = True + + async with async_records(self.connection) as records: + batch = await records.bulk_update( + [ + {'key': key(1), 'set_values': {'value': 'first'}}, + {'key': key(2), 'set_values': {'value': 'second'}}, + ] + ) + delete = await records.delete_where(where={'tenant': 1}) + + self.assertEqual((batch.counts.matched_count, batch.counts.modified_count), (None, None)) + self.assertEqual(batch.native_results[0]['rowcount_mode'], 'unknown') + self.native.isset_client_flag.assert_not_called() + self.assertEqual(delete.deleted_count, 2) + self.cursor.executemany.assert_awaited_once() + self.cursor.execute.assert_awaited_once() + self.assertEqual(self.cursor.close.await_count, 2) + self.native.commit.assert_not_called() + self.native.rollback.assert_not_called() + + async def test_cancelled_write_preserves_partial_progress_and_cleanup_error(self): + cancellation = asyncio.CancelledError() + cleanup = RuntimeError('close failed') + self.cursor.executemany.side_effect = [None, cancellation] + self.cursor.close.side_effect = cleanup + + async with async_records(self.connection) as records: + with self.assertRaises(asyncio.CancelledError) as caught: + await records.insert_many( + [ + {'tenant': 1, 'id': 1, 'value': 'acknowledged'}, + {'tenant': 1, 'id': 2, 'note': 'unknown'}, + {'tenant': 1, 'id': 3, 'value': 'not sent'}, + ] + ) + + self.assertIs(caught.exception, cancellation) + self.assertEqual(cancellation.database_cleanup_errors, (cleanup,)) + partial = cancellation.database_partial_result + self.assertEqual( + [item.state for item in partial.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.UNKNOWN, + BatchState.NOT_SENT, + ], + ) + self.assertEqual(partial.native_results[0]['rowcount'], 1) + self.assertIsNone(partial.counts.inserted_count) + self.cursor.close.assert_awaited_once() diff --git a/tests/test_crud_redis.py b/tests/test_crud_redis.py new file mode 100644 index 0000000000000000000000000000000000000000..daafc5b8d67b31e29e4a8cbf15b590b1c2e8deba --- /dev/null +++ b/tests/test_crud_redis.py @@ -0,0 +1,532 @@ +"""Verify Redis Hash wire commands, result state, and native ownership.""" + +import asyncio +import json +import unittest +from base64 import urlsafe_b64decode +from contextlib import ExitStack +from decimal import Decimal +from types import SimpleNamespace +from unittest.mock import AsyncMock +from unittest.mock import Mock +from unittest.mock import call +from unittest.mock import patch + +import redis + +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRecords +from datvia.crud import BatchOperationError +from datvia.crud import Records +from datvia.crud._codecs import JsonFieldCodec +from datvia.crud._errors import RecordConflictError +from datvia.crud._errors import RecordFormatError +from datvia.crud._errors import UnsupportedOperationError +from datvia.crud._redis_hash import _SCRIPT +from datvia.crud._redis_hash import Adapter +from datvia.crud._redis_hash import AsyncAdapter +from datvia.crud._types import UNSET +from datvia.crud._types import BatchState +from datvia.crud._types import ChunkError +from datvia.crud._types import WriteInput +from datvia.errors import ConfigurationError + + +def _open(backend='single', **kwargs): + if backend == 'sentinel': + kwargs.update(nodes=[('sentinel.test', 26379)], service_name='main') + else: + kwargs.setdefault('host', 'redis.test') + return Connections({'cache': ConnectionConfig('redis', mode=backend, **kwargs)}) + + +def _bind(connection, is_async=False, *, key_fields=('id',), namespace='orders', codec=None): + adapter_type = AsyncAdapter if is_async else Adapter + return adapter_type( + connection=connection, + key_fields=key_fields, + namespace=namespace, + codec=JsonFieldCodec() if codec is None else codec, + ) + + +def _write(identity, values=None, ttl_s=UNSET): + return WriteInput({'id': identity}, {'id': identity} if values is None else values, ttl_s) + + +def _encoded(segment): + return urlsafe_b64decode(segment + b'=' * (-len(segment) % 4)) + + +class RedisFieldCodecTests(unittest.TestCase): + def test_json_codec_preserves_nested_values_and_explicit_null(self): + value = {'city': '上海', 'active': True, 'details': [None, 2**60, 0.75]} + codec = JsonFieldCodec() + encoded = codec.encode(value) + self.assertEqual(codec.encode(None), b'null') + self.assertEqual(codec.decode(encoded), value) + self.assertEqual(codec.decode(encoded.decode('ascii')), value) + + def test_json_codec_rejects_types_and_nonfinite_numbers_that_lose_information(self): + codec = JsonFieldCodec() + for value in (Decimal('2.0'), b'bytes', (1, 2), {'nested': {3: 'a'}}, float('nan'), [float('inf')]): + with self.subTest(value=value): + with self.assertRaises(ValueError): + codec.encode(value) + for wire in (b'NaN', b'1e999', 'Infinity'): + with self.subTest(wire=wire): + with self.assertRaises(ValueError): + codec.decode(wire) + + +class RedisHashTests(unittest.TestCase): + def setUp(self): + self.connection = _open().create('cache') + self.addCleanup(self.connection.close) + + def test_binding_has_no_network_and_key_is_canonical_and_typed(self): + with patch.object(self.connection.native, 'execute_command') as execute: + records = _bind(self.connection, key_fields=('shop', 'id'), namespace='order:{用户}') + fields_a = {'shop': 'a:1:b', 'id': 1} + fields_b = {'id': 1, 'shop': 'a:1:b'} + native_key = records.prepare_key(fields_a) + self.assertEqual(records.prepare_key(fields_b), native_key) + self.assertNotEqual(records.prepare_key({'shop': 'a:1:b', 'id': '1'}), native_key) + self.assertNotEqual(records.prepare_key({'shop': 'a', 'id': '1:b:1'}), native_key) + self.assertNotIn(b'{', native_key) + self.assertNotIn(b'}', native_key) + _, _, _, namespace, identity = native_key.split(b':') + self.assertEqual(_encoded(namespace).decode('utf-8'), 'order:{用户}') + self.assertEqual(json.loads(_encoded(identity)), [['shop', 'a:1:b'], ['id', 1]]) + with self.assertRaises(ConfigurationError): + records.prepare_key({'shop': 'a', 'id': True}) + self.assertNotEqual( + _bind(self.connection, key_fields=('shop', 'id'), namespace='other').prepare_key(fields_a), + native_key, + ) + execute.assert_not_called() + + def test_reserved_field_is_rejected_for_binding_writes_and_projection(self): + with patch.object(self.connection.native, 'execute_command') as execute: + with self.assertRaises(ConfigurationError): + _bind(self.connection, key_fields=('__database_record__',)) + with _bind(self.connection) as records: + for operation, outgoing in ( + ('insert', _write(1, {'id': 1, '__database_record__': 1})), + ('update', _write(1, {'__database_record__': 1})), + ): + with self.subTest(operation=operation): + with self.assertRaises(ConfigurationError): + records.prepare_write(operation, outgoing) + with self.assertRaises(ConfigurationError): + records.get({'id': 1}, ('__database_record__',)) + execute.assert_not_called() + + def test_public_batch_projection_error_is_configuration_error_before_pipeline(self): + with patch.object(self.connection.native, 'pipeline') as pipeline: + with Records.redis_hash( + self.connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records: + for field in ('__database_record__', '\ud800'): + with self.subTest(field=repr(field)): + with self.assertRaises(ConfigurationError): + records.get_many([{'id': 1}, {'id': 2}], fields=[field], batch_size=1) + pipeline.assert_not_called() + + def test_projected_slotted_replies_distinguish_missing_null_and_presence_only(self): + with ( + patch.object( + self.connection.native, + 'execute_command', + side_effect=[ + [0], + [1, [None, b'null', '"ready"']], + [1, []], + [1, [b'__database_record__', b'1', b'id', b'1', b'name', b'"test"']], + ], + ) as execute, + _bind(self.connection) as records, + ): + key = {'id': 1} + self.assertIsNone(records.get(key, ('status',))) + self.assertEqual( + records.get(key, ('missing', 'empty', 'status')), + {'empty': None, 'status': 'ready'}, + ) + self.assertEqual(records.get(key, ()), {}) + self.assertEqual(records.get(key, None), {'id': 1, 'name': 'test'}) + native_key = records.prepare_key(key) + self.assertEqual( + execute.call_args_list, + [ + call('EVAL', _SCRIPT, 1, native_key, 'get', b'fields', b'status'), + call('EVAL', _SCRIPT, 1, native_key, 'get', b'fields', b'missing', b'empty', b'status'), + call('EVAL', _SCRIPT, 1, native_key, 'get', b'fields'), + call('EVAL', _SCRIPT, 1, native_key, 'get', b'all'), + ], + ) + + def test_write_counts_expire_modes_and_script_status(self): + scripts = [[1], [1], [0], [1], [3], [2], [1], [0]] + with ( + patch.object(self.connection.native, 'execute_command', side_effect=scripts) as execute, + _bind(self.connection) as records, + ): + created = records.insert(_write(1, ttl_s=10)) + self.assertEqual(created.inserted_count, 1) + self.assertIs(created.native_result, scripts[0]) + self.assertEqual(execute.call_args.args[-3:], (b'10', b'id', b'1')) + updated = records.update(_write(1, {'status': 'a'})) + self.assertEqual(updated.matched_count, 1) + self.assertIsNone(updated.modified_count) + self.assertEqual(execute.call_args.args[-3:], (b'keep', b'status', b'"a"')) + missing = records.update(_write(8, {'status': 1}, ttl_s=20)) + self.assertEqual((missing.matched_count, missing.modified_count), (0, 0)) + self.assertEqual(execute.call_args.args[-3], b'20') + records.update(_write(1, {'status': 'b'}, ttl_s=None)) + self.assertEqual(execute.call_args.args[-3], b'persist') + with self.assertRaises(RecordConflictError): + records.insert(_write(1)) + with self.assertRaises(RecordFormatError): + records.delete(_write(8, {})) + self.assertEqual(records.delete(_write(1, {})).deleted_count, 1) + self.assertEqual(records.delete(_write(8, {})).deleted_count, 0) + + def test_prepare_all_batch_fields_before_opening_a_pipeline(self): + with patch.object(self.connection.native, 'pipeline') as pipeline, _bind(self.connection) as records: + with self.assertRaises(ConfigurationError): + records.insert_many([_write(1), _write(2, {'id': 2, 'cost': Decimal('2.99')})]) + for ttl in (0, -1, True, 1.0, 2**63): + with self.subTest(ttl=ttl): + with self.assertRaises(ConfigurationError): + records.prepare_write('insert', _write(1, ttl_s=ttl)) + pipeline.assert_not_called() + + def test_custom_field_codec_preserves_binary_data_and_types(self): + codec = SimpleNamespace(encode=Mock(return_value=b'\x00\xff'), decode=Mock(return_value=Decimal('0.10'))) + with ( + patch.object( + self.connection.native, + 'execute_command', + side_effect=[ + [1], + [1, [b'\x00\xff']], + ], + ) as execute, + _bind(self.connection, codec=codec) as records, + ): + source = Decimal('0.10') + records.update(_write(1, {'balance': source})) + codec.encode.assert_called_once_with(source) + self.assertEqual(execute.call_args.args[-1], b'\x00\xff') + self.assertEqual(records.get({'id': 1}, ('balance',)), {'balance': source}) + codec.decode.assert_called_once_with(b'\x00\xff') + + def test_corrupt_payload_retains_decoder_cause(self): + with ( + patch.object(self.connection.native, 'execute_command', return_value=[1, [b'{?']]), + _bind(self.connection) as records, + ): + with self.assertRaises(RecordFormatError) as failed: + records.get({'id': 1}, ('x',)) + self.assertIsInstance(failed.exception.original, json.JSONDecodeError) + self.assertIs(failed.exception.__cause__, failed.exception.original) + + def test_driver_reply_list_proves_items_independently(self): + pipe = Mock() + denied = redis.exceptions.NoPermissionError('denied') + disconnected = redis.exceptions.ConnectionError('connection dropped') + replies = [[1], denied, [0], disconnected] + pipe.execute.return_value = replies + pipe.__exit__ = Mock() + with ( + patch.object(self.connection.native, 'pipeline', return_value=pipe) as factory, + _bind(self.connection) as records, + ): + with self.assertRaises(ChunkError) as failed: + records.delete_many([_write(index, {}) for index in range(1, 5)]) + keys = [records.prepare_key({'id': index}) for index in range(1, 5)] + partial = failed.exception.partial_result + self.assertIs(failed.exception.original, denied) + self.assertIs(partial.native_results[0], replies) + self.assertEqual( + [item.state for item in partial.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.FAILED, + BatchState.ACKNOWLEDGED, + BatchState.UNKNOWN, + ], + ) + self.assertEqual(partial.items[0].result.deleted_count, 1) + self.assertEqual(partial.items[2].result.deleted_count, 0) + self.assertIs(partial.items[1].error, denied) + self.assertIs(partial.items[3].error, disconnected) + factory.assert_called_once_with(transaction=False) + self.assertEqual( + [args.args[:4] for args in pipe.execute_command.call_args_list], [('EVAL', _SCRIPT, 1, key) for key in keys] + ) + pipe.execute.assert_called_once_with(raise_on_error=False) + pipe.__exit__.assert_called_once_with(None, None, None) + + def test_queue_error_leaves_chunk_not_sent_and_execute_error_is_unknown(self): + error = redis.exceptions.ConnectionError('transient') + for step, state in (('execute_command', BatchState.NOT_SENT), ('execute', BatchState.UNKNOWN)): + pipe = Mock() + pipe.__exit__ = Mock() + getattr(pipe, step).side_effect = error + with ( + self.subTest(step=step), + patch.object(self.connection.native, 'pipeline', return_value=pipe), + _bind(self.connection) as records, + ): + with self.assertRaises(ChunkError) as failed: + records.insert_many([_write(1), _write(2)]) + self.assertIs(failed.exception.original, error) + self.assertEqual([item.state for item in failed.exception.partial_result.items], [state, state]) + pipe.__exit__.assert_called_once() + if step == 'execute_command': + pipe.execute.assert_not_called() + + def test_partial_read_saves_only_confirmed_observations(self): + missing = [0] + found = [1, [b'null']] + corrupt = [2] + pipe = Mock() + pipe.__exit__ = Mock() + pipe.execute.return_value = [missing, corrupt, found] + with patch.object(self.connection.native, 'pipeline', return_value=pipe), _bind(self.connection) as records: + with self.assertRaises(ChunkError) as failed: + records.get_many([{'id': index} for index in (1, 2, 3, 4)], ('field',)) + items = failed.exception.partial_result.items + self.assertIsNone(items[0].result.record) + self.assertIsInstance(items[1].error, RecordFormatError) + self.assertEqual(items[2].result.record, {'field': None}) + self.assertIsNone(items[3].result) + self.assertIs(items[3].state, BatchState.UNKNOWN) + + def test_scope_exits_without_closing_borrowed_native(self): + with ( + patch.object(self.connection.native, 'close') as native_close, + Records.redis_hash( + self.connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records, + ): + calls = ( + ('find', lambda: records.find(limit=1)), + ('count', records.count), + ('update_many', lambda: records.update_many(where={'id': 1}, set_values={'x': 1})), + ('delete_where', lambda: records.delete_where(where={'id': 1})), + ) + for operation, invoke in calls: + with self.subTest(operation=operation): + with self.assertRaises(UnsupportedOperationError) as failed: + invoke() + self.assertEqual(failed.exception.operation, operation) + native_close.assert_not_called() + self.assertFalse(self.connection.closed) + + def test_sentinel_scope_opens_one_owned_master_and_releases_its_pool(self): + with _open('sentinel').create('cache') as connection: + sentinel = connection.native + master = sentinel.master_for('main') + with ( + patch.object(sentinel, 'master_for', return_value=master) as factory, + patch.object(master, 'execute_command', return_value=[0]) as command, + patch.object(master.connection_pool, 'disconnect') as disconnect, + ): + records = _bind(connection) + factory.assert_not_called() + with records: + self.assertIsNone(records.get({'id': 1}, None)) + self.assertIsNone(records.get({'id': 2}, None)) + factory.assert_called_once_with('main') + self.assertEqual(command.call_count, 2) + disconnect.assert_called_once_with() + self.assertIs(connection.native, sentinel) + self.assertFalse(connection.closed) + + +class AsyncRedisHashTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.connection = await _open().create_async('cache') + self.addAsyncCleanup(self.connection.close) + + async def test_public_async_batch_projection_error_precedes_pipeline(self): + with patch.object(self.connection.native, 'pipeline') as pipeline: + async with AsyncRecords.redis_hash( + self.connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records: + with self.assertRaises(ConfigurationError): + await records.get_many([{'id': 1}, {'id': 2}], fields=['__database_record__'], batch_size=1) + pipeline.assert_not_called() + + async def test_async_commands_and_batch_await_native_execution(self): + pipe = Mock() + pipe.__aexit__ = AsyncMock() + pipe.execute = AsyncMock(return_value=[[1], [1], [0]]) + with ( + patch.object( + self.connection.native, + 'execute_command', + new_callable=AsyncMock, + side_effect=[ + [1, [b'null']], + [1], + [0], + [1], + ], + ) as command, + patch.object(self.connection.native, 'pipeline', return_value=pipe), + ): + async with _bind(self.connection, is_async=True) as records: + self.assertEqual(await records.get({'id': 1}, ('field',)), {'field': None}) + self.assertEqual((await records.insert(_write(1))).inserted_count, 1) + self.assertEqual((await records.update(_write(2, {'x': 1}))).matched_count, 0) + self.assertEqual((await records.delete(_write(1, {}))).deleted_count, 1) + result = await records.bulk_update([_write(index, {'x': 1}) for index in (1, 2, 3)]) + self.assertEqual(result.counts.matched_count, 2) + self.assertIsNone(result.counts.modified_count) + self.assertEqual(command.await_count, 4) + self.assertEqual(pipe.execute_command.call_count, 3) + pipe.execute.assert_awaited_once_with(raise_on_error=False) + pipe.__aexit__.assert_awaited_once_with(None, None, None) + self.assertFalse(self.connection.closed) + + async def test_async_batch_cancellation_preserves_original_and_cleanup_error(self): + original = asyncio.CancelledError() + cleanup = redis.exceptions.ConnectionError('cleanup failed') + pipe = Mock() + pipe.__aexit__ = AsyncMock(side_effect=cleanup) + pipe.execute = AsyncMock(side_effect=original) + with patch.object(self.connection.native, 'pipeline', return_value=pipe): + async with _bind(self.connection, is_async=True) as records: + with self.assertRaises(ChunkError) as failed: + await records.insert_many([_write(1)]) + self.assertIs(failed.exception.original, original) + self.assertIs(failed.exception.partial_result.items[0].state, BatchState.UNKNOWN) + self.assertIs(original.database_cleanup_errors[0].original, cleanup) + pipe.__aexit__.assert_awaited_once() + + async def test_async_sentinel_context_closes_owned_pool_and_preserves_primary_error(self): + connection = await _open('sentinel').create_async('cache') + self.addAsyncCleanup(connection.close) + sentinel = connection.native + master = sentinel.master_for('main') + original = asyncio.CancelledError() + cleanup = redis.exceptions.ConnectionError('cleanup failed') + with ( + patch.object(sentinel, 'master_for', return_value=master) as factory, + patch.object(master, 'execute_command', new_callable=AsyncMock, side_effect=original), + patch.object( + master.connection_pool, 'disconnect', new_callable=AsyncMock, side_effect=cleanup + ) as disconnect, + ): + with self.assertRaises(asyncio.CancelledError) as failed: + async with _bind(connection, is_async=True) as records: + await records.get({'id': 1}, None) + self.assertIs(failed.exception, original) + self.assertIs(original.database_cleanup_errors[0].original, cleanup) + factory.assert_called_once_with('main') + disconnect.assert_awaited_once_with() + self.assertIs(connection.native, sentinel) + self.assertFalse(connection.closed) + + +def _cluster_topology(manager, nodes, keys=()): + manager.nodes_cache = {node.name: node for node in nodes} + manager.default_node = nodes[0] + slots = [redis.crc.key_slot(key) for key in keys] + manager.slots_cache = {slot: [node] for slot, node in zip(slots, nodes)} + + +def _assert_cluster_partial(test, result, error): + test.assertIs(result.original, error) + test.assertEqual( + [item.state for item in result.partial_result.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.FAILED, + ], + ) + + +class RedisNativeClusterTests(unittest.IsolatedAsyncioTestCase): + def test_sync_native_cluster_pipeline_executes_scripts_across_slots(self): + clients = [redis.Redis(host=f'primary{index}.test') for index in (1, 2)] + nodes = [ + redis.cluster.ClusterNode(f'primary{index}.test', 7000, server_type='primary', redis_connection=client) + for index, client in enumerate(clients, 1) + ] + error = redis.exceptions.NoPermissionError('denied') + with ExitStack() as cleanup: + cleanup.enter_context( + patch.object( + redis.cluster.NodesManager, + 'initialize', + lambda manager, *args, **opts: _cluster_topology(manager, nodes), + ) + ) + cleanup.enter_context(patch.object(redis.cluster.CommandsParser, 'initialize')) + connection = cleanup.enter_context(_open('cluster').create('cache')) + key_encoder = _bind(connection) + keys = [key_encoder.prepare_key({'id': index}) for index in (1, 2)] + self.assertNotEqual(redis.crc.key_slot(keys[0]), redis.crc.key_slot(keys[1])) + _cluster_topology(connection.native.nodes_manager, nodes, keys) + connection.native.commands_parser.commands['eval'] = {'flags': ['movablekeys']} + sockets = [] + for index, client in enumerate(clients): + socket = Mock() + socket.pack_commands.side_effect = lambda commands: commands + cleanup.enter_context(patch.object(client.connection_pool, 'get_connection', return_value=socket)) + cleanup.enter_context(patch.object(client.connection_pool, 'release')) + # Current redis-py queries COMMAND GETKEYS for EVAL in a sync Pipeline. + cleanup.enter_context( + patch.object( + client, + 'execute_command', + side_effect=lambda command, *args: args[3:4], + ) + ) + reply = {'return_value': [1]} if index == 0 else {'side_effect': error} + cleanup.enter_context(patch.object(client, 'parse_response', **reply)) + sockets.append(socket) + with Records.redis_hash( + connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records: + with self.assertRaises(BatchOperationError) as failure: + records.insert_many([{'id': 1}, {'id': 2}]) + _assert_cluster_partial(self, failure.exception, error) + self.assertEqual([sock.send_packed_command.call_args.args[0][0][3] for sock in sockets], keys) + + async def test_async_native_cluster_pipeline_executes_scripts_across_slots(self): + async with await _open('cluster').create_async('cache') as connection: + key_encoder = _bind(connection, is_async=True) + keys = [key_encoder.prepare_key({'id': index}) for index in (1, 2)] + self.assertNotEqual(redis.crc.key_slot(keys[0]), redis.crc.key_slot(keys[1])) + nodes = [ + redis.asyncio.cluster.ClusterNode(f'primary{index}.test', 7000, server_type='primary') + for index in (1, 2) + ] + _cluster_topology(connection.native.nodes_manager, nodes, keys) + connection.native._initialize = False + error = redis.exceptions.NoPermissionError('denied') + routed = {} + + async def execute(node, commands): + for command in commands: + routed[node.host] = command.args[3] + command.result = [1] if node is nodes[0] else error + return node is nodes[1] + + with patch.object( + redis.asyncio.cluster.ClusterNode, 'execute_pipeline', autospec=True, side_effect=execute + ): + async with AsyncRecords.redis_hash( + connection, namespace='orders', key_fields=('id',), codec=JsonFieldCodec() + ) as records: + with self.assertRaises(BatchOperationError) as failure: + await records.insert_many([{'id': 1}, {'id': 2}]) + _assert_cluster_partial(self, failure.exception, error) + self.assertEqual(routed, {node.host: key for node, key in zip(nodes, keys)}) diff --git a/tests/test_crud_strings.py b/tests/test_crud_strings.py new file mode 100644 index 0000000000000000000000000000000000000000..99bc036316ff67ae4b8b1bac5db1953b8aa9222a --- /dev/null +++ b/tests/test_crud_strings.py @@ -0,0 +1,439 @@ +"""Redis String commands, lifetimes, and evidence from mixed batch replies.""" + +import asyncio +import unittest +from contextlib import ExitStack +from unittest.mock import AsyncMock +from unittest.mock import Mock +from unittest.mock import call +from unittest.mock import patch + +import redis + +from datvia import ConnectionConfig +from datvia import Connections +from datvia.crud import AsyncRedisStrings +from datvia.crud import BatchOperationError +from datvia.crud import BatchState +from datvia.crud import RedisStrings +from datvia.crud import StringSetItem +from datvia.errors import ConfigurationError +from datvia.errors import ConnectionClosedError +from datvia.errors import DriverError + + +def registry(mode='single', **options): + endpoint = {'nodes': [('sentinel.test', 26379)], 'service_name': 'main'} + if mode != 'sentinel': + endpoint = {} if 'connection_pool' in options else {'host': 'cache.test'} + config = ConnectionConfig('redis', mode=mode, options=options, **endpoint) + return Connections({'cache': config}) + + +def pipeline(replies=None, *, asynchronous=False): + pipe = Mock() + if asynchronous: + pipe.execute = AsyncMock(return_value=replies) + pipe.__aexit__ = AsyncMock() + else: + pipe.execute.return_value = replies + pipe.__exit__ = Mock() + return pipe + + +class RedisStringTests(unittest.TestCase): + def setUp(self): + self.connection = registry().create('cache') + self.addCleanup(self.connection.close) + self.native = self.connection.native + + def test_single_operations_preserve_wire_semantics_and_empty_strings(self): + replies = [b'', None, True, True, 1, -1, -2, 0, True, False, 1, 0] + with patch.object(self.native, 'execute_command', side_effect=replies) as execute: + with RedisStrings(self.connection) as strings: + self.assertEqual(strings.get('empty'), b'') + self.assertIsNone(strings.get(b'missing')) + self.assertTrue(strings.set('msg', '你好', ttl_s=20)) + self.assertTrue(strings.set('msg', b'new')) + self.assertTrue(strings.exists('msg')) + self.assertEqual(strings.ttl('msg'), -1) + self.assertEqual(strings.ttl('missing'), -2) + self.assertEqual(strings.ttl('expiring'), 0) + self.assertTrue(strings.expire('msg', 15)) + self.assertFalse(strings.persist('msg')) + self.assertEqual(strings.delete('msg'), 1) + self.assertEqual(strings.delete('msg'), 0) + self.assertEqual( + execute.call_args_list, + [ + call('GET', b'empty'), + call('GET', b'missing'), + call('SET', b'msg', '你好'.encode(), 'EX', 20), + call('SET', b'msg', b'new'), + call('EXISTS', b'msg'), + call('TTL', b'msg'), + call('TTL', b'missing'), + call('TTL', b'expiring'), + call('EXPIRE', b'msg', 15), + call('PERSIST', b'msg'), + call('DEL', b'msg'), + call('DEL', b'msg'), + ], + ) + + def test_get_preserves_driver_decoded_value_and_original_wrong_type(self): + wrong_type = redis.exceptions.ResponseError('WRONGTYPE') + with registry(decode_responses=True).create('cache') as connection: + with ( + patch.object(connection.native, 'execute_command', side_effect=['', wrong_type]), + RedisStrings(connection) as strings, + ): + self.assertEqual(strings.get('empty'), '') + with self.assertRaises(DriverError) as caught: + strings.get('hash') + self.assertIs(caught.exception.original, wrong_type) + self.assertIs(caught.exception.__cause__, wrong_type) + self.assertEqual(caught.exception.operation, 'get') + + def test_invalid_single_arguments_make_no_native_calls(self): + with patch.object(self.native, 'execute_command') as execute, RedisStrings(self.connection) as strings: + invalid = (True, 0, -1, 1.5, 2**63) + for seconds in invalid: + with self.subTest(ttl=seconds), self.assertRaises(ConfigurationError): + strings.set('key', 'value', ttl_s=seconds) + with self.subTest(seconds=seconds), self.assertRaises(ConfigurationError): + strings.expire('key', seconds) + for invalid_value in (True, 1, ['list'], None): + with self.subTest(value=invalid_value), self.assertRaises(ConfigurationError): + strings.set('key', invalid_value) + with self.assertRaises(ConfigurationError): + strings.get(42) + execute.assert_not_called() + + def test_batch_validates_complete_wire_keys_and_values_before_pipeline(self): + with registry(encoding='latin-1').create('cache') as connection: + with patch.object(connection.native, 'pipeline') as factory, RedisStrings(connection) as strings: + with self.assertRaises(ConfigurationError): + strings.delete_many(['café', b'caf\xe9'], batch_size=1) + with self.assertRaises(ConfigurationError): + strings.set_many([StringSetItem('é', 'a'), StringSetItem(b'\xe9', 'b')], batch_size=1) + with self.assertRaises(ConfigurationError) as caught: + strings.set_many([StringSetItem('valid', 'ok'), StringSetItem('bad', '€')], batch_size=1) + self.assertIsInstance(caught.exception.original, UnicodeEncodeError) + self.assertIs(caught.exception.__cause__, caught.exception.original) + with self.assertRaises(ConfigurationError): + strings.get_many(['ok'], batch_size=False) + self.assertEqual(strings.get_many([]), []) + self.assertEqual(strings.set_many([]).items, ()) + self.assertEqual(strings.delete_many([]).deleted_count, 0) + factory.assert_not_called() + + def test_batched_get_uses_separate_commands_and_preserves_positions(self): + first = pipeline([b'', b'changed']) + last = pipeline([None]) + with ( + patch.object(self.native, 'pipeline', side_effect=[first, last]) as factory, + RedisStrings(self.connection) as strings, + ): + self.assertEqual(strings.get_many(['key', b'key', 'absent'], batch_size=2), [b'', b'changed', None]) + self.assertEqual(factory.call_args_list, [call(transaction=False), call(transaction=False)]) + self.assertEqual(first.execute_command.call_args_list, [call('GET', b'key'), call('GET', b'key')]) + self.assertEqual(last.execute_command.call_args_list, [call('GET', b'absent')]) + last.execute.assert_called_once_with(raise_on_error=False) + + def test_batches_keep_set_acknowledgment_and_delete_reply_counts(self): + first = pipeline([True, True]) + last = pipeline([1, 0]) + with patch.object(self.native, 'pipeline', side_effect=[first, last]), RedisStrings(self.connection) as strings: + batch = strings.set_many([StringSetItem('a', 'a', ttl_s=99), StringSetItem('b', b'b')]) + self.assertEqual([item.result for item in batch.items], [True, True]) + self.assertEqual(batch.requested_count, 2) + self.assertIsNone(batch.deleted_count) + deleted = strings.delete_many(['a', 'missing']) + self.assertEqual(deleted.deleted_count, 1) + self.assertEqual([item.result for item in deleted.items], [1, 0]) + self.assertEqual( + first.execute_command.call_args_list, [call('SET', b'a', b'a', 'EX', 99), call('SET', b'b', b'b')] + ) + self.assertEqual(last.execute_command.call_args_list, [call('DEL', b'a'), call('DEL', b'missing')]) + + def test_batch_exception_keeps_server_failures_and_unknowns_distinct(self): + wrong_type = redis.exceptions.ResponseError('WRONGTYPE') + disconnected = redis.exceptions.ConnectionError('connection dropped') + replies = [None, wrong_type, disconnected, b'valid'] + pipe = pipeline(replies) + with ( + patch.object(self.native, 'pipeline', return_value=pipe) as factory, + RedisStrings(self.connection) as strings, + ): + with self.assertRaises(BatchOperationError) as caught: + strings.get_many(['missing', 'hash', 'maybe', 'last', 'not_sent'], batch_size=4) + error = caught.exception + self.assertIs(error.original, wrong_type) + self.assertIs(error.__cause__, wrong_type) + self.assertIs(error.partial_result.native_results[0], replies) + self.assertEqual( + [item.state for item in error.partial_result.items], + [ + BatchState.ACKNOWLEDGED, + BatchState.FAILED, + BatchState.UNKNOWN, + BatchState.ACKNOWLEDGED, + BatchState.NOT_SENT, + ], + ) + self.assertIsNone(error.partial_result.items[0].result) + self.assertIs(error.partial_result.items[2].error, disconnected) + self.assertEqual(error.partial_result.items[3].result, b'valid') + factory.assert_called_once() + + def test_queue_and_transport_failures_preserve_dispatch_evidence_and_cleanup(self): + for failure_point, state in (('execute_command', BatchState.NOT_SENT), ('execute', BatchState.UNKNOWN)): + pipe = pipeline() + driver_error = redis.exceptions.ConnectionError('transport failed') + cleanup_error = RuntimeError('pipeline cleanup failed') + getattr(pipe, failure_point).side_effect = driver_error + pipe.__exit__.side_effect = cleanup_error + with ( + self.subTest(step=failure_point), + patch.object(self.native, 'pipeline', return_value=pipe), + RedisStrings(self.connection) as strings, + self.assertRaises(BatchOperationError) as caught, + ): + strings.delete_many(['a', 'b'], batch_size=1) + error = caught.exception + self.assertIs(error.original, driver_error) + self.assertIn(cleanup_error, error.database_cleanup_errors) + self.assertEqual([item.state for item in error.partial_result.items], [state, BatchState.NOT_SENT]) + + def test_cleanup_only_failure_keeps_acknowledged_write_results(self): + first = pipeline([1]) + cleanup_error = OSError('close failed') + first.__exit__.side_effect = cleanup_error + with patch.object(self.native, 'pipeline', return_value=first), RedisStrings(self.connection) as strings: + with self.assertRaises(BatchOperationError) as caught: + strings.delete_many(['completed', 'not_queued'], batch_size=1) + self.assertIs(caught.exception.original, cleanup_error) + partial = caught.exception.partial_result + self.assertEqual(partial.deleted_count, 1) + self.assertEqual([item.state for item in partial.items], [BatchState.ACKNOWLEDGED, BatchState.NOT_SENT]) + + def test_single_scope_never_closes_borrowed_native_or_external_pool(self): + external_pool = redis.ConnectionPool(host='cache.test') + with registry(connection_pool=external_pool).create('cache') as connection: + native = connection.native + wrapper = RedisStrings(connection) + with patch.object(native, 'close') as close, patch.object(external_pool, 'disconnect') as disconnect: + with self.assertRaises(ConfigurationError): + wrapper.get('inactive') + with wrapper: + self.assertFalse(connection.closed) + with self.assertRaises(ConfigurationError): + wrapper.get('outside') + self.assertIs(connection.native, native) + close.assert_not_called() + disconnect.assert_not_called() + with self.assertRaises(ConnectionClosedError): + with RedisStrings(connection): + self.fail('a closed connection cannot enter a String context') + + def test_sentinel_scope_borrows_wrapper_but_owns_one_data_client(self): + with registry('sentinel').create('cache') as connection: + sentinel = connection.native + master = sentinel.master_for('main') + with ( + patch.object(sentinel, 'master_for', return_value=master) as factory, + patch.object(master, 'execute_command', return_value=b'value') as execute, + patch.object(master.connection_pool, 'disconnect') as disconnect, + RedisStrings(connection) as strings, + ): + self.assertEqual(strings.get('key'), b'value') + self.assertEqual(strings.get('key'), b'value') + self.assertIs(connection.native, sentinel) + factory.assert_called_once_with('main') + self.assertEqual(execute.call_count, 2) + disconnect.assert_called_once_with() + + +class AsyncRedisStringTests(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.connection = await registry().create_async('cache') + self.addAsyncCleanup(self.connection.close) + self.native = self.connection.native + + async def test_uses_native_async_commands_and_nontransactional_pipeline(self): + pipe = pipeline([True, True], asynchronous=True) + with ( + patch.object(self.native, 'execute_command', new_callable=AsyncMock) as execute, + patch.object(self.native, 'pipeline', return_value=pipe) as factory, + ): + execute.side_effect = ['value', True, 1, 1, 1, 0, -2] + async with AsyncRedisStrings(self.connection) as strings: + self.assertEqual(await strings.get('a'), 'value') + self.assertTrue(await strings.set('a', 'x', ttl_s=10)) + self.assertTrue(await strings.exists('a')) + self.assertEqual(await strings.delete('a'), 1) + self.assertTrue(await strings.expire('a', 5)) + self.assertFalse(await strings.persist('a')) + self.assertEqual(await strings.ttl('a'), -2) + result = await strings.set_many([StringSetItem('a', 'a'), StringSetItem('b', 'b')]) + self.assertEqual(execute.await_count, 7) + self.assertEqual([item.result for item in result.items], [True, True]) + factory.assert_called_once_with(transaction=False) + pipe.execute.assert_awaited_once_with(raise_on_error=False) + pipe.__aexit__.assert_awaited_once_with(None, None, None) + + async def test_cancelled_pipeline_keeps_identity_and_partial_delivery(self): + completed = pipeline([b'confirmed'], asynchronous=True) + cancelled = pipeline(asynchronous=True) + + class ReadOnlyCancellation(asyncio.CancelledError): + def __setattr__(self, name, value): + raise AttributeError('read only') + + original = ReadOnlyCancellation() + cancelled.execute.side_effect = original + cleanup = OSError('cancel cleanup failed') + cancelled.__aexit__.side_effect = cleanup + with patch.object(self.native, 'pipeline', side_effect=[completed, cancelled]): + async with AsyncRedisStrings(self.connection) as strings: + with self.assertRaises(ReadOnlyCancellation) as caught: + await strings.get_many(['first', 'second', 'third'], batch_size=1) + self.assertIs(caught.exception, original) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + partial = original.database_partial_result + self.assertEqual(partial.items[0].result, b'confirmed') + self.assertEqual( + [item.state for item in partial.items], + [BatchState.ACKNOWLEDGED, BatchState.UNKNOWN, BatchState.NOT_SENT], + ) + + async def test_pipeline_cancellation_reply_keeps_prior_exception_context(self): + original = asyncio.CancelledError() + initial_context = ValueError('initial context') + original.__context__ = initial_context + pipe = pipeline([original], asynchronous=True) + with patch.object(self.native, 'pipeline', return_value=pipe): + async with AsyncRedisStrings(self.connection) as strings: + try: + raise RuntimeError('caller exception') + except RuntimeError: + with self.assertRaises(asyncio.CancelledError) as caught: + await strings.delete_many(['a']) + self.assertIs(caught.exception, original) + self.assertIs(original.__context__, initial_context) + self.assertEqual(original.database_partial_result.items[0].state, BatchState.UNKNOWN) + + async def test_async_scope_rejects_wrong_loop_and_bind_mode(self): + with self.assertRaises(ConfigurationError): + RedisStrings(self.connection) + async with AsyncRedisStrings(self.connection) as strings: + + def run_in_other_loop(): + return asyncio.run(strings.get('illegal')) + + with patch.object(self.native, 'execute_command', new_callable=AsyncMock) as execute: + with self.assertRaises(ConfigurationError): + await asyncio.to_thread(run_in_other_loop) + execute.assert_not_awaited() + + async def test_async_sentinel_closes_owned_pool_without_hiding_primary_error(self): + connection = await registry('sentinel').create_async('cache') + self.addAsyncCleanup(connection.close) + sentinel = connection.native + master = sentinel.master_for('main') + original = ValueError('user body exception') + cleanup = redis.exceptions.ConnectionError('data pool cleanup failed') + with ( + patch.object(sentinel, 'master_for', return_value=master) as factory, + patch.object(master.connection_pool, 'disconnect', new_callable=AsyncMock, side_effect=cleanup), + ): + with self.assertRaises(ValueError) as caught: + async with AsyncRedisStrings(connection): + raise original + self.assertIs(caught.exception, original) + self.assertIs(original.database_cleanup_errors[0].original, cleanup) + self.assertIs(connection.native, sentinel) + factory.assert_called_once_with('main') + + +def cluster_topology(manager, nodes, keys=()): + manager.nodes_cache = {node.name: node for node in nodes} + manager.default_node = nodes[0] + manager.slots_cache = {redis.crc.key_slot(key): [node] for key, node in zip(keys, nodes)} + + +class RedisStringNativeClusterTests(unittest.IsolatedAsyncioTestCase): + def test_sync_driver_routes_each_set_to_its_own_slot(self): + keys = [b'{first}:string', b'{second}:string'] + clients = [redis.Redis(host=f'primary-{index}.test') for index in (1, 2)] + nodes = [ + redis.cluster.ClusterNode(client.connection_pool.connection_kwargs['host'], 7000, 'primary', client) + for client in clients + ] + with ExitStack() as stack: + stack.enter_context( + patch.object( + redis.cluster.NodesManager, 'initialize', lambda manager, *a, **k: cluster_topology(manager, nodes) + ) + ) + stack.enter_context(patch.object(redis.cluster.CommandsParser, 'initialize')) + connection = stack.enter_context(registry('cluster').create('cache')) + cluster_topology(connection.native.nodes_manager, nodes, keys) + stack.enter_context( + patch.object( + connection.native.commands_parser, 'get_keys', side_effect=lambda client, *args: (args[1],) + ) + ) + sockets = [] + for client in clients: + socket = Mock() + socket.pack_commands.side_effect = list + sockets.append(socket) + stack.enter_context(patch.object(client.connection_pool, 'get_connection', return_value=socket)) + stack.enter_context(patch.object(client.connection_pool, 'release')) + stack.enter_context(patch.object(client, 'parse_response', return_value=True)) + with RedisStrings(connection) as strings: + result = strings.set_many([StringSetItem(key, b'value') for key in keys]) + self.assertNotEqual(redis.crc.key_slot(keys[0]), redis.crc.key_slot(keys[1])) + self.assertEqual([item.result for item in result.items], [True, True]) + self.assertEqual( + [socket.send_packed_command.call_args.args[0][0][:2] for socket in sockets], + [('SET', keys[0]), ('SET', keys[1])], + ) + + async def test_async_driver_routes_get_to_multiple_slots_and_keeps_per_item_error(self): + async with await registry('cluster').create_async('cache') as connection: + keys = [b'{first}:string', b'{second}:string'] + nodes = [ + redis.asyncio.cluster.ClusterNode(f'primary-{index}.test', 7000, server_type='primary') + for index in (1, 2) + ] + cluster_topology(connection.native.nodes_manager, nodes, keys) + connection.native._initialize = False + wrong_type = redis.exceptions.ResponseError('WRONGTYPE') + routed = {} + + async def execute(node, commands): + for command in commands: + routed[node.host] = command.args[1] + command.result = b'first' if node is nodes[0] else wrong_type + return node is nodes[1] + + with ( + patch.object( + connection.native.commands_parser, + 'get_keys', + new_callable=AsyncMock, + side_effect=lambda *args: (args[1],), + ), + patch.object(redis.asyncio.cluster.ClusterNode, 'execute_pipeline', autospec=True, side_effect=execute), + ): + async with AsyncRedisStrings(connection) as strings: + with self.assertRaises(BatchOperationError) as caught: + await strings.get_many(keys) + self.assertIs(caught.exception.original, wrong_type) + self.assertEqual( + [item.state for item in caught.exception.partial_result.items], [BatchState.ACKNOWLEDGED, BatchState.FAILED] + ) + self.assertEqual(routed, {node.host: key for node, key in zip(nodes, keys)}) diff --git a/tests/test_managed_mongodb.py b/tests/test_managed_mongodb.py new file mode 100644 index 0000000000000000000000000000000000000000..948500ba99ff78d34a7a5659b63ffe24548852a1 --- /dev/null +++ b/tests/test_managed_mongodb.py @@ -0,0 +1,175 @@ +import asyncio +from types import SimpleNamespace +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, Mock, patch + +from pymongo import ReadPreference +from pymongo import errors as mongo_errors + +from datvia import ConnectionConfig, Connections +from datvia.errors import ( + AuthenticationError, + ConfigurationError, + DatabaseConnectionError, + DatabaseTimeoutError, + DependencyError, + DriverError, + PermissionDeniedError, +) + + +class MongoDBConnectionTests(TestCase): + def setUp(self): + self.connections = Connections({ + "reports": ConnectionConfig( + "mongodb", + host=["server.test", "replica.test"], + port=27018, + username="reporter", + password="sample-password", + connect_timeout_s=2, + options={"authSource": "admin", "read_preference": ReadPreference.SECONDARY}, + ) + }) + self.native = SimpleNamespace( + admin=SimpleNamespace(command=Mock()), + read_preference=ReadPreference.SECONDARY, + close=Mock(), + ) + + @patch("datvia.mongodb.Client.create") + def test_create_preserves_driver_options_and_check_uses_read_preference(self, create): + create.return_value = self.native + with self.connections.create("reports") as connection: + self.assertIs(connection.native, self.native) + self.native.admin.command.assert_not_called() + + result = connection.check() + self.assertEqual(result.scope, "selected_server") + self.assertEqual(result.targets, ()) + self.native.admin.command.assert_called_once_with( + "ping", read_preference=ReadPreference.SECONDARY + ) + + create.assert_called_once_with( + host=["server.test", "replica.test"], + port=27018, + username="reporter", + password="sample-password", + connectTimeoutMS=2000, + authSource="admin", + read_preference=ReadPreference.SECONDARY, + ) + self.native.close.assert_called_once_with() + + @patch("datvia.mongodb.Client.create") + def test_check_translates_native_authentication_failure(self, create): + create.return_value = self.native + original = mongo_errors.OperationFailure( + "credentials in original error", code=18, details={"errorLabels": ["test"]} + ) + self.native.admin.command.side_effect = original + with self.connections.create("reports") as connection: + with self.assertRaises(AuthenticationError) as caught: + connection.check() + + error = caught.exception + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + self.assertTrue(error.original.has_error_label("test")) + self.assertEqual(error.original.code, 18) + self.assertEqual(error.phase, "check") + self.assertEqual(error.operation, "ping") + self.assertEqual(error.target, "selected_server") + self.assertNotIn("credentials in original error", str(error)) + + @patch("datvia.mongodb.Client.create") + def test_native_operations_classify_supported_driver_errors(self, create): + create.return_value = self.native + errors = ( + (mongo_errors.OperationFailure("denied", code=13), PermissionDeniedError), + (mongo_errors.NetworkTimeout("timeout"), DatabaseTimeoutError), + (mongo_errors.OperationFailure("timeout", code=50), DatabaseTimeoutError), + (mongo_errors.ConnectionFailure("unavailable"), DatabaseConnectionError), + (mongo_errors.InvalidURI("invalid"), ConfigurationError), + (mongo_errors.OperationFailure("unknown", code=42), DriverError), + ) + with self.connections.create("reports") as connection: + for original, expected in errors: + with self.subTest(error=type(original), expected=expected): + with self.assertRaises(expected) as caught: + with connection.errors(operation="aggregate"): + raise original + self.assertIs(caught.exception.original, original) + self.assertEqual(caught.exception.phase, "operation") + self.assertEqual(caught.exception.operation, "aggregate") + + @patch("datvia.mongodb.Client.create") + def test_business_wrappers_are_preserved_in_operation_context(self, create): + create.return_value = self.native + original = RuntimeError("business rule") + with self.connections.create("reports") as connection: + with self.assertRaises(RuntimeError) as caught: + with connection.errors(operation="build_query"): + raise original from mongo_errors.OperationFailure("denied", code=13) + self.assertIs(caught.exception, original) + + +class AsyncMongoDBConnectionTests(IsolatedAsyncioTestCase): + def setUp(self): + self.connections = Connections({ + "reports": ConnectionConfig("mongodb", host="server.test") + }) + + @patch("datvia.mongodb.Client.create_async") + async def test_native_object_is_returned_directly_and_probe_is_awaited(self, create): + class NativeClient(SimpleNamespace): + def __await__(self): + raise AssertionError("Construction awaited the native client") + + native = NativeClient( + admin=SimpleNamespace(command=AsyncMock()), + read_preference=ReadPreference.SECONDARY, + close=AsyncMock(), + ) + create.return_value = native + async with await self.connections.create_async("reports") as connection: + self.assertIs(connection.native, native) + native.admin.command.assert_not_awaited() + + result = await connection.check() + self.assertEqual(result.scope, "selected_server") + self.assertEqual(result.targets, ()) + native.admin.command.assert_awaited_once_with( + "ping", read_preference=ReadPreference.SECONDARY + ) + + create.assert_called_once_with(host="server.test") + native.close.assert_awaited_once_with() + + @patch("datvia.mongodb.Client.create_async") + async def test_missing_async_driver_capability_preserves_import_error(self, create): + original = ImportError("missing AsyncMongoClient") + create.side_effect = original + with self.assertRaises(DependencyError) as caught: + await self.connections.create_async("reports") + + error = caught.exception + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + self.assertEqual(error.phase, "create") + self.assertEqual(error.connection_name, "reports") + self.assertIn("PyMongo >= 4.9", str(error)) + + @patch("datvia.mongodb.Client.create_async") + async def test_ping_cancellation_is_preserved(self, create): + cancellation = asyncio.CancelledError() + create.return_value = SimpleNamespace( + admin=SimpleNamespace(command=AsyncMock(side_effect=cancellation)), + read_preference=ReadPreference.PRIMARY, + close=AsyncMock(), + ) + async with await self.connections.create_async("reports") as connection: + with self.assertRaises(asyncio.CancelledError) as caught: + await connection.check() + self.assertIs(caught.exception, cancellation) diff --git a/tests/test_managed_mysql.py b/tests/test_managed_mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..64f5ddcbb8831305a0267094a730c51617c30b22 --- /dev/null +++ b/tests/test_managed_mysql.py @@ -0,0 +1,186 @@ +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, Mock, patch + +from mysql.connector import errorcode +from mysql.connector import errors as driver_errors + +from datvia import ConnectionConfig, Connections +from datvia.errors import ( + AuthenticationError, + DatabaseConnectionError, + DatabaseTimeoutError, + DependencyError, + DriverError, + PermissionDeniedError, +) + + +def registry(): + config = ConnectionConfig( + backend="mysql", + host="example.test", + port=3307, + username="test", + connect_timeout_s=3, + options={"database": "sample"}, + ) + return Connections({"mysql": config}) + + +class ManagedMySQLTests(TestCase): + @patch("datvia.mysql.client.connector.connect") + def test_create_wraps_missing_driver_capability(self, connect): + original = ImportError("MySQL C extension is unavailable") + connect.side_effect = original + + with self.assertRaises(DependencyError) as result: + registry().create("mysql") + + error = result.exception + self.assertEqual((error.connection_name, error.backend, error.phase), ("mysql", "mysql", "create")) + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + + @patch("datvia.mysql.client.connector.connect") + def test_operation_preserves_unrelated_import_error(self, connect): + original = ImportError("unrelated import") + + with registry().create("mysql") as handle: + with self.assertRaises(ImportError) as result: + with handle.errors(operation="read"): + raise original + + self.assertIs(result.exception, original) + + @patch("datvia.mysql.client.connector.connect") + def test_create_checks_current_endpoint_and_closes_native_connection(self, connect): + native = Mock(server_host="2001:db8::1", server_port=3308, unix_socket=None) + connect.return_value = native + + with registry().create("mysql") as handle: + self.assertIs(handle.native, native) + native.ping.assert_not_called() + result = handle.check() + + connect.assert_called_once_with( + host="example.test", port=3307, user="test", connection_timeout=3, database="sample", + ) + native.ping.assert_called_once_with(reconnect=False) + native.close.assert_called_once_with() + self.assertEqual(result.scope, "current_connection") + self.assertEqual(result.targets, ("[2001:db8::1]:3308",)) + + @patch("datvia.mysql.client.connector.connect") + def test_check_preserves_wrapped_driver_error(self, connect): + native = Mock(server_host="example.test", server_port=3307, unix_socket=None) + connect.return_value = native + inner = driver_errors.OperationalError(errno=errorcode.CR_SERVER_LOST) + outer = driver_errors.InterfaceError("driver failure") + outer.__cause__ = inner + native.ping.side_effect = outer + + with registry().create("mysql") as handle: + with self.assertRaises(DatabaseConnectionError) as result: + handle.check() + + error = result.exception + self.assertEqual((error.phase, error.operation, error.target), ("check", "ping", "example.test:3307")) + self.assertIs(error.original, outer) + self.assertIs(error.__cause__, outer) + self.assertIs(error.original.__cause__, inner) + native.ping.assert_called_once_with(reconnect=False) + + @patch("datvia.mysql.client.connector.connect") + def test_classifies_only_structured_driver_failures(self, connect): + cases = ( + (errorcode.ER_ACCESS_DENIED_ERROR, AuthenticationError), + (errorcode.ER_DBACCESS_DENIED_ERROR, PermissionDeniedError), + (errorcode.ER_TABLEACCESS_DENIED_ERROR, PermissionDeniedError), + (errorcode.ER_COLUMNACCESS_DENIED_ERROR, PermissionDeniedError), + (errorcode.ER_SPECIFIC_ACCESS_DENIED_ERROR, PermissionDeniedError), + (errorcode.CR_CONN_HOST_ERROR, DatabaseConnectionError), + (errorcode.ER_PARSE_ERROR, DriverError), + ) + + with registry().create("mysql") as handle: + for errno, expected_type in cases: + with self.subTest(errno=errno): + original = driver_errors.ProgrammingError(errno=errno, msg="driver message") + with self.assertRaises(expected_type) as result: + with handle.errors(operation="read"): + raise original + self.assertIs(type(result.exception), expected_type) + self.assertEqual(result.exception.operation, "read") + self.assertIs(result.exception.original, original) + self.assertIs(result.exception.__cause__, original) + original = driver_errors.OperationalError(msg="connection refused") + with self.assertRaises(DriverError): + with handle.errors(): + raise original + + @patch("datvia.mysql.client.connector.connect") + def test_classifies_typed_transport_timeout_wrapped_by_driver(self, connect): + original = driver_errors.InterfaceError(errno=errorcode.CR_CONN_HOST_ERROR) + original.__cause__ = TimeoutError() + connect.side_effect = original + + with self.assertRaises(DatabaseTimeoutError) as result: + registry().create("mysql") + + self.assertIs(result.exception.original, original) + + +class AsyncManagedMySQLTests(IsolatedAsyncioTestCase): + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_create_async_wraps_missing_driver_capability(self, connect): + original = ImportError("async driver capability is unavailable") + connect.side_effect = original + + with self.assertRaises(DependencyError) as result: + await registry().create_async("mysql") + + error = result.exception + self.assertEqual((error.connection_name, error.backend, error.phase), ("mysql", "mysql", "create")) + self.assertIs(error.original, original) + self.assertIs(error.__cause__, original) + + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_awaits_create_check_and_close(self, connect): + native = Mock( + server_host="example.test", server_port=3307, unix_socket=None, + ping=AsyncMock(), close=AsyncMock(), + ) + connect.return_value = native + + async with await registry().create_async("mysql") as handle: + self.assertIs(handle.native, native) + native.ping.assert_not_awaited() + result = await handle.check() + + connect.assert_awaited_once_with( + host="example.test", port=3307, user="test", connection_timeout=3, database="sample", + ) + native.ping.assert_awaited_once_with(reconnect=False) + native.close.assert_awaited_once_with() + self.assertEqual(result.scope, "current_connection") + self.assertEqual(result.targets, ("example.test:3307",)) + + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_check_failure_omits_a_non_host_target(self, connect): + original = driver_errors.InterfaceError() + native = Mock( + server_host="mysql://user:secret@example.test", + server_port=3307, + unix_socket=None, + ping=AsyncMock(side_effect=original), + close=AsyncMock(), + ) + connect.return_value = native + + async with await registry().create_async("mysql") as handle: + with self.assertRaises(DatabaseConnectionError) as result: + await handle.check() + + self.assertIsNone(result.exception.target) + self.assertIs(result.exception.original, original) + native.close.assert_awaited_once_with() diff --git a/tests/test_managed_redis.py b/tests/test_managed_redis.py new file mode 100644 index 0000000000000000000000000000000000000000..25d06a0cacb4e17f5de79a8a9e16ca9b1ff0f69f --- /dev/null +++ b/tests/test_managed_redis.py @@ -0,0 +1,437 @@ +"""Managed Redis tests using installed driver classes without Redis servers.""" + +from importlib.util import find_spec +import unittest +from unittest.mock import AsyncMock, Mock, call, patch + +from datvia import ConnectionConfig, Connections +from datvia.errors import ( + AuthenticationError, + ConfigurationError, + DatabaseConnectionError, + DatabaseTimeoutError, + DriverError, + PermissionDeniedError, +) + +if find_spec("redis") is not None: + import redis + from datvia._adapters.redis import Adapter +else: + redis = None + + +def _config(mode="single", **kwargs): + if mode == "single": + kwargs.setdefault("host", "redis.test") + else: + kwargs.setdefault("nodes", [("redis.test", 7000)]) + if mode == "sentinel": + kwargs.setdefault("service_name", "main") + return ConnectionConfig("redis", mode=mode, **kwargs) + + +def _registry(config=None): + return Connections({"test": config or _config()}) + + +def _configure_sync_cluster(manager): + # Supply a discovered topology to the actual Cluster constructor. + nodes = [ + redis.cluster.ClusterNode("first.test", 7001, server_type="primary"), + redis.cluster.ClusterNode("second.test", 7002, server_type="primary"), + redis.cluster.ClusterNode("replica.test", 7003, server_type="replica"), + ] + for node in nodes: + node.redis_connection = redis.Redis(host=node.host, port=node.port) + manager.nodes_cache = {node.name: node for node in nodes} + manager.default_node = nodes[0] + + +def _configure_async_cluster(client): + nodes = [ + redis.asyncio.cluster.ClusterNode("first.test", 7001, server_type="primary"), + redis.asyncio.cluster.ClusterNode("second.test", 7002, server_type="primary"), + redis.asyncio.cluster.ClusterNode("replica.test", 7003, server_type="replica"), + ] + client.nodes_manager.nodes_cache = {node.name: node for node in nodes} + client.nodes_manager.default_node = nodes[0] + + +@unittest.skipIf(redis is None, "redis driver is not installed") +class ManagedRedisTests(unittest.TestCase): + def test_single_returns_native_and_pings_only_when_requested(self): + with patch.object(redis.Redis, "ping", return_value=True) as ping: + with _registry().create("test") as handle: + self.assertIsInstance(handle.native, redis.Redis) + ping.assert_not_called() + result = handle.check() + self.assertEqual(result.scope, "single") + self.assertEqual(result.targets, ("redis.test:6379",)) + ping.assert_called_once_with() + + def test_external_pool_is_preserved_and_owned_pool_is_disconnected(self): + pool = redis.ConnectionPool(host="external.test") + config = ConnectionConfig("redis", options={"connection_pool": pool}) + shared = _registry(config).create("test") + private = _registry().create("test") + self.assertIs(shared.native.connection_pool, pool) + with patch.object(pool, "disconnect") as shared_disconnect, patch.object( + private.native.connection_pool, "disconnect" + ) as private_disconnect: + shared.close() + private.close() + private.close() + shared_disconnect.assert_not_called() + private_disconnect.assert_called_once_with() + + def test_single_false_ping_and_driver_errors_are_not_healthy(self): + handle = _registry().create("test") + self.addCleanup(handle.close) + with patch.object(handle.native, "ping", return_value=False): + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertEqual(failed.exception.target, "redis.test:6379") + self.assertEqual(failed.exception.operation, "PING") + original = redis.exceptions.NoPermissionError("server detail") + with patch.object(handle.native, "ping", side_effect=original): + with self.assertRaises(PermissionDeniedError) as failed: + handle.check() + self.assertIs(failed.exception.original, original) + self.assertIs(failed.exception.__cause__, original) + self.assertEqual(failed.exception.phase, "check") + + def test_sentinel_uses_native_manager_and_closes_temporary_master(self): + config = _config( + "sentinel", username="data", password="data-secret", + options={"sentinel_kwargs": {"username": "sentinel", "password": "sentinel-secret"}}, + ) + with _registry(config).create("test") as handle: + self.assertIsInstance(handle.native, redis.Sentinel) + sentinel_client, = handle.native.sentinels + self.assertEqual(sentinel_client.connection_pool.connection_kwargs["username"], "sentinel") + self.assertEqual(handle.native.connection_kwargs["username"], "data") + master = handle.native.master_for("main") + master.connection_pool.proxy.master_address = ("master.test", 6380) + with patch.object(handle.native, "master_for", return_value=master) as factory, patch.object( + master, "ping", return_value=True + ) as ping, patch.object(master.connection_pool, "disconnect") as disconnect: + result = handle.check() + self.assertEqual(result.scope, "sentinel_master") + self.assertEqual(result.targets, ("master.test:6380",)) + factory.assert_called_once_with("main") + ping.assert_called_once_with() + disconnect.assert_called_once_with() + + def test_sentinel_original_ping_error_survives_cleanup_errors(self): + handle = _registry(_config("sentinel")).create("test") + self.addCleanup(handle.close) + master = handle.native.master_for("main") + master.connection_pool.proxy.master_address = ("master.test", 6380) + original = redis.exceptions.ConnectionError("unreachable") + release_error = redis.exceptions.ResponseError("release failed") + pool_error = redis.exceptions.TimeoutError("pool release timed out") + with patch.object(handle.native, "master_for", return_value=master), patch.object( + master, "ping", side_effect=original + ), patch.object(master, "close", side_effect=release_error), patch.object( + master.connection_pool, "disconnect", side_effect=pool_error + ) as disconnect: + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertEqual(failed.exception.target, "master.test:6380") + self.assertIs(failed.exception.original, original) + self.assertIs(failed.exception.__cause__, original) + errors = failed.exception.database_cleanup_errors + self.assertEqual(tuple(type(error) for error in errors), (DriverError, DatabaseTimeoutError)) + self.assertEqual(tuple(error.original for error in errors), (release_error, pool_error)) + self.assertEqual(tuple(error.phase for error in errors), ("close", "close")) + disconnect.assert_called_once_with() + + def test_sentinel_unknown_service_preserves_driver_error(self): + with _registry(_config("sentinel")).create("test") as handle: + master = handle.native.master_for("main") + error = redis.sentinel.MasterNotFoundError("not found") + with patch.object(handle.native, "master_for", return_value=master), patch.object( + master, "ping", side_effect=error + ): + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertIs(failed.exception.original, error) + self.assertEqual(failed.exception.target, "sentinel_master") + + def test_sentinel_close_attempts_all_node_clients(self): + config = _config("sentinel", nodes=[("first.test", 26379), ("second.test", 26379)]) + handle = _registry(config).create("test") + first, second = handle.native.sentinels + self.addCleanup(handle.close) + first_error = redis.exceptions.ConnectionError("first") + second_error = redis.exceptions.TimeoutError("second") + with patch.object(first, "close", side_effect=first_error), patch.object( + first.connection_pool, "disconnect" + ) as first_disconnect, patch.object(second, "close", side_effect=second_error) as second_close, patch.object( + second.connection_pool, "disconnect" + ) as second_disconnect: + with self.assertRaises(DatabaseConnectionError) as failed: + handle.close() + self.assertIs(failed.exception.original, first_error) + self.assertIs(failed.exception.database_cleanup_errors[0].original, second_error) + first_disconnect.assert_called_once_with() + second_disconnect.assert_called_once_with() + second_close.assert_called_once_with() + + def test_cluster_targets_actual_primaries_in_order_and_stops_on_failure(self): + with patch.object(redis.cluster.NodesManager, "initialize", autospec=True, side_effect=_configure_sync_cluster), patch.object( + redis.cluster.CommandsParser, "initialize" + ): + handle = _registry(_config("cluster")).create("test") + self.addCleanup(handle.close) + self.assertIsInstance(handle.native, redis.RedisCluster) + first, second = handle.native.get_primaries() + with patch.object(handle.native, "_execute_command", return_value=True) as command: + result = handle.check() + self.assertEqual(result.scope, "cluster_primaries") + self.assertEqual(result.targets, ("first.test:7001", "second.test:7002")) + self.assertEqual(command.call_args_list, [call(first, "PING"), call(second, "PING")]) + with patch.object(handle.native, "_execute_command", return_value=False) as command: + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertEqual(failed.exception.target, "first.test:7001") + command.assert_called_once_with(first, "PING") + + def test_cluster_rejects_empty_primary_snapshot(self): + with patch.object(redis.cluster.NodesManager, "initialize", autospec=True, side_effect=_configure_sync_cluster), patch.object( + redis.cluster.CommandsParser, "initialize" + ): + handle = _registry(_config("cluster")).create("test") + self.addCleanup(handle.close) + with patch.object(handle.native, "get_primaries", return_value=[]), patch.object( + handle.native, "ping" + ) as ping: + with self.assertRaises(DatabaseConnectionError) as failed: + handle.check() + self.assertEqual(failed.exception.operation, "get_primaries") + ping.assert_not_called() + + def test_cluster_pool_from_an_existing_node_stays_external(self): + original_client = redis.Redis(host="external.test") + self.addCleanup(original_client.close) + node = redis.cluster.ClusterNode("external.test", 7000, redis_connection=original_client) + config = ConnectionConfig("redis", mode="cluster", options={"startup_nodes": [node]}) + with patch.object(redis.cluster.NodesManager, "initialize") as initialize: + with self.assertRaises(ConfigurationError) as failed: + _registry(config).create("test") + self.assertEqual(failed.exception.phase, "create") + initialize.assert_not_called() + self.assertIs(node.redis_connection, original_client) + + def test_cluster_reuses_configured_host_addresses_without_sharing_driver_nodes(self): + node = redis.cluster.ClusterNode("redis.test", 7000) + config = ConnectionConfig("redis", mode="cluster", options={"startup_nodes": [node]}) + with patch.object(redis.cluster.NodesManager, "initialize", autospec=True, side_effect=_configure_sync_cluster), patch.object( + redis.cluster.CommandsParser, "initialize" + ): + registry = _registry(config) + first = registry.create("test") + second = registry.create("test") + with first, second: + first_node, = first.native.nodes_manager.startup_nodes.values() + second_node, = second.native.nodes_manager.startup_nodes.values() + self.assertEqual((first_node.host, first_node.port), (node.host, node.port)) + self.assertIsNot(first_node, node) + self.assertIsNot(first_node, second_node) + + def test_redis_classification_uses_native_types_and_prioritizes_subclasses(self): + for error_type, expected in ( + (redis.exceptions.AuthenticationError, AuthenticationError), + (redis.exceptions.AuthenticationWrongNumberOfArgsError, AuthenticationError), + (redis.exceptions.NoPermissionError, PermissionDeniedError), + (redis.exceptions.AuthorizationError, PermissionDeniedError), + (redis.exceptions.TimeoutError, DatabaseTimeoutError), + (redis.sentinel.MasterNotFoundError, DatabaseConnectionError), + (redis.exceptions.RedisClusterException, DriverError), + (redis.exceptions.ResponseError, DriverError), + (TypeError, None), + ): + with self.subTest(error_type=error_type): + error = error_type("irrelevant authentication timeout message") + self.assertIs(Adapter.classify(error), expected) + + +@unittest.skipIf(redis is None, "redis driver is not installed") +class ManagedAsyncRedisTests(unittest.IsolatedAsyncioTestCase): + async def test_single_returns_raw_awaitable_client_without_awaiting_it(self): + with patch.object(redis.asyncio.Redis, "initialize", new_callable=AsyncMock) as initialize: + handle = await _registry().create_async("test") + self.assertIsInstance(handle.native, redis.asyncio.Redis) + initialize.assert_not_called() + async with handle: + with patch.object(handle.native, "ping", new_callable=AsyncMock, return_value=True) as ping: + result = await handle.check() + self.assertEqual(result.targets, ("redis.test:6379",)) + ping.assert_awaited_once_with() + + async def test_async_external_pool_is_preserved_and_owned_pool_is_disconnected(self): + pool = redis.asyncio.ConnectionPool(host="external.test") + config = ConnectionConfig("redis", options={"connection_pool": pool}) + shared = await _registry(config).create_async("test") + private = await _registry().create_async("test") + self.assertIs(shared.native.connection_pool, pool) + with patch.object(pool, "disconnect", new_callable=AsyncMock) as shared_disconnect, patch.object( + private.native.connection_pool, "disconnect", new_callable=AsyncMock + ) as private_disconnect: + await shared.close() + await private.close() + await private.close() + shared_disconnect.assert_not_called() + private_disconnect.assert_awaited_once_with() + + async def test_async_sentinel_releases_temporary_master_and_node_pools(self): + handle = await _registry(_config("sentinel")).create_async("test") + self.assertIsInstance(handle.native, redis.asyncio.Sentinel) + master = handle.native.master_for("main") + master.connection_pool.master_address = ("2001:db8::1", 6380) + sentinel_client, = handle.native.sentinels + with patch.object(handle.native, "master_for", return_value=master) as factory, patch.object( + master, "ping", new_callable=AsyncMock, return_value=True + ) as ping, patch.object(master.connection_pool, "disconnect", new_callable=AsyncMock) as disconnect, patch.object( + sentinel_client.connection_pool, "disconnect", new_callable=AsyncMock + ) as sentinel_disconnect: + result = await handle.check() + await handle.close() + self.assertEqual(result.scope, "sentinel_master") + self.assertEqual(result.targets, ("[2001:db8::1]:6380",)) + factory.assert_called_once_with("main") + ping.assert_awaited_once_with() + disconnect.assert_awaited_once_with() + sentinel_disconnect.assert_awaited_once_with() + + async def test_async_sentinel_keeps_cancelled_error_when_cleanup_fails(self): + from asyncio import CancelledError + + handle = await _registry(_config("sentinel")).create_async("test") + self.addAsyncCleanup(handle.close) + master = handle.native.master_for("main") + original = CancelledError("task cancelled") + cleanup = redis.exceptions.ConnectionError("close failed") + with patch.object(handle.native, "master_for", return_value=master), patch.object( + master, "ping", new_callable=AsyncMock, side_effect=original + ), patch.object(master.connection_pool, "disconnect", new_callable=AsyncMock, side_effect=cleanup): + with self.assertRaises(CancelledError) as failed: + await handle.check() + self.assertIs(failed.exception, original) + self.assertIs(failed.exception.database_cleanup_errors[0].original, cleanup) + + async def test_async_cluster_initializes_then_pings_each_primary(self): + with patch.object(redis.asyncio.RedisCluster, "initialize", new_callable=AsyncMock) as initialize: + handle = await _registry(_config("cluster")).create_async("test") + initialize.assert_not_called() + self.assertIsInstance(handle.native, redis.asyncio.RedisCluster) + self.addAsyncCleanup(handle.close) + _configure_async_cluster(handle.native) + first, second = handle.native.get_primaries() + + async def initialize_cluster(): + handle.native._initialize = False + + with patch.object(handle.native, "initialize", new_callable=AsyncMock, side_effect=initialize_cluster) as initialize, patch.object( + handle.native, "_execute_command", new_callable=AsyncMock, return_value=True + ) as command: + result = await handle.check() + initialize.assert_awaited_once_with() + self.assertEqual(result.targets, ("first.test:7001", "second.test:7002")) + self.assertEqual(command.await_args_list, [call(first, "PING"), call(second, "PING")]) + + async def test_async_cluster_recreates_native_startup_nodes_per_handle(self): + node = redis.asyncio.cluster.ClusterNode("seed.test", 7000) + config = ConnectionConfig("redis", mode="cluster", options={"startup_nodes": [node]}) + registry = _registry(config) + first = await registry.create_async("test") + second = await registry.create_async("test") + self.addAsyncCleanup(first.close) + self.addAsyncCleanup(second.close) + first_node, = first.native.nodes_manager.startup_nodes.values() + second_node, = second.native.nodes_manager.startup_nodes.values() + self.assertEqual((first_node.host, first_node.port), (node.host, node.port)) + self.assertIsNot(first_node, node) + self.assertIsNot(second_node, node) + self.assertIsNot(first_node, second_node) + + # Both clients reached initialization independently. Closing one can + # disconnect its private seed without touching the other client's seed. + first.native._initialize = False + second.native._initialize = False + with patch.object(type(node), "disconnect", autospec=True) as disconnect: + await first.close() + disconnect.assert_awaited_once_with(first_node) + self.assertFalse(second.closed) + + async def test_async_cluster_close_tries_topology_and_seed_nodes_after_failure(self): + handle = await _registry(_config("cluster")).create_async("test") + self.addAsyncCleanup(handle.close) + _configure_async_cluster(handle.native) + first, second = handle.native.get_primaries() + seed, = handle.native.nodes_manager.startup_nodes.values() + native_close = "aclose" if hasattr(handle.native, "aclose") else "close" + primary = redis.exceptions.ConnectionError("cluster close failed") + fallback = redis.exceptions.TimeoutError("first node close failed") + async def disconnect_node(node): + if node is first: + raise fallback + + with patch.object( + handle.native, native_close, new_callable=AsyncMock, side_effect=primary + ), patch.object( + type(first), "disconnect", autospec=True, side_effect=disconnect_node + ) as disconnect: + with self.assertRaises(DatabaseConnectionError) as failed: + await handle.close() + self.assertIs(failed.exception.original, primary) + self.assertIs(failed.exception.database_cleanup_errors[0].original, fallback) + self.assertIn(call(second), disconnect.await_args_list) + self.assertIn(call(seed), disconnect.await_args_list) + + async def test_async_cluster_close_retries_failed_nodes_until_disconnected(self): + for failing_node_kind in ("primary", "seed"): + with self.subTest(failing_node_kind=failing_node_kind): + handle = await _registry(_config("cluster")).create_async("test") + self.addAsyncCleanup(handle.close) + native = handle.native + _configure_async_cluster(native) + seed, = native.nodes_manager.startup_nodes.values() + failing_node = native.get_primaries()[0] if failing_node_kind == "primary" else seed + # Use native close(), which marks the client uninitialized before cleanup. + native._initialize = False + failure = redis.exceptions.ConnectionError("node disconnect failed") + should_fail = True + disconnected = set() + + async def disconnect_node(node): + if node is failing_node and should_fail: + raise failure + disconnected.add(id(node)) + + with patch.object( + type(failing_node), "disconnect", autospec=True, side_effect=disconnect_node + ) as disconnect: + with self.assertRaises(DatabaseConnectionError) as first: + await handle.close() + self.assertIs(first.exception.original, failure) + self.assertTrue(native._initialize) + self.assertFalse(handle.closed) + self.assertNotIn(id(failing_node), disconnected) + + disconnect.reset_mock() + with self.assertRaises(DatabaseConnectionError) as retry: + await handle.close() + self.assertIs(retry.exception.original, failure) + self.assertFalse(handle.closed) + disconnect.assert_awaited_once_with(failing_node) + + should_fail = False + disconnect.reset_mock() + await handle.close() + self.assertTrue(handle.closed) + self.assertIn(id(failing_node), disconnected) + disconnect.assert_awaited_once_with(failing_node) + await handle.close() + disconnect.assert_awaited_once_with(failing_node) diff --git a/tests/test_mongodb_client.py b/tests/test_mongodb_client.py new file mode 100644 index 0000000000000000000000000000000000000000..1ab3a29fdb087d5bfb5f13c1dedd7a77615c1720 --- /dev/null +++ b/tests/test_mongodb_client.py @@ -0,0 +1,39 @@ +import unittest +from types import SimpleNamespace +from unittest.mock import Mock, patch + +from datvia.mongodb import Client + + +class MongoDBClientTests(unittest.TestCase): + + def test_driver_without_async_client(self): + native_client = object() + driver = SimpleNamespace(MongoClient=Mock(return_value=native_client)) + + with patch("datvia.mongodb.client.pymongo", driver): + self.assertIs(Client.create("mongodb://localhost", connect=False), native_client) + with self.assertRaisesRegex(ImportError, r"PyMongo >= 4\.9"): + Client.create_async("mongodb://localhost") + + driver.MongoClient.assert_called_once_with("mongodb://localhost", connect=False) + + def test_async_factory_returns_native_client(self): + native_client = object() + driver = SimpleNamespace(AsyncMongoClient=Mock(return_value=native_client)) + + with patch("datvia.mongodb.client.pymongo", driver): + result = Client.create_async("mongodb://localhost", serverSelectionTimeoutMS=500) + + self.assertIs(result, native_client) + driver.AsyncMongoClient.assert_called_once_with("mongodb://localhost", serverSelectionTimeoutMS=500) + + def test_async_factory_propagates_driver_error(self): + error = AttributeError("driver constructor failed") + driver = SimpleNamespace(AsyncMongoClient=Mock(side_effect=error)) + + with patch("datvia.mongodb.client.pymongo", driver): + with self.assertRaises(AttributeError) as caught: + Client.create_async("mongodb://localhost") + + self.assertIs(caught.exception, error) diff --git a/tests/test_mongodb_scan.py b/tests/test_mongodb_scan.py new file mode 100644 index 0000000000000000000000000000000000000000..b0df01b6aaa6fb6c6455a20d6498b8ce8737f6d5 --- /dev/null +++ b/tests/test_mongodb_scan.py @@ -0,0 +1,318 @@ +import asyncio +import unittest + +from bson import MaxKey +from bson import ObjectId +from pymongo.errors import AutoReconnect +from pymongo.errors import CursorNotFound +from pymongo.errors import ExecutionTimeout +from pymongo.errors import NetworkTimeout +from pymongo.errors import OperationFailure +from pymongo.errors import ServerSelectionTimeoutError + +from datvia.mongodb.utils import async_scan_collection +from datvia.mongodb.utils import async_stream_collection +from datvia.mongodb.utils import scan_collection +from datvia.mongodb.utils import stream_collection + + +class Cursor: + def __init__(self, *results, close_error=None): + self.results = iter(results) + self.read = 0 + self.closed = 0 + self.close_error = close_error + + def __iter__(self): + return self + + def __next__(self): + result = next(self.results) + if isinstance(result, BaseException): + raise result + self.read += 1 + return {'_id': result} + + def close(self): + self.closed += 1 + if self.close_error is not None: + raise self.close_error + + +class AsyncCursor(Cursor): + def __aiter__(self): + return self + + async def __anext__(self): + try: + return next(self) + except StopIteration: + raise StopAsyncIteration from None + + async def close(self): + super().close() + + +class Collection: + def __init__(self, *results): + self.results = iter(results) + self.calls = [] + self.cursors = [] + + def find(self, **options): + if self.cursors and self.cursors[-1].closed != 1: + raise AssertionError('The previous cursor must be closed before opening another.') + self.calls.append(options) + try: + result = next(self.results) + except StopIteration: + raise AssertionError('Unexpected find call.') from None + if isinstance(result, BaseException): + raise result + self.cursors.append(result) + return result + + +class TestScanCollection(unittest.TestCase): + def test_default_page_is_closed_before_first_delivery(self): + cursor = Cursor(1, 2) + collection = Collection(cursor) + rows = scan_collection(collection) + self.assertEqual(next(rows), {'_id': 1}) + self.assertEqual((cursor.read, cursor.closed), (2, 1)) + self.assertEqual(list(rows), [{'_id': 2}]) + self.assertEqual(collection.calls[0]['limit'], 1000) + self.assertEqual(collection.calls[0]['sort'], [('_id', 1)]) + self.assertEqual(collection.calls[0]['hint'], '_id_') + + def test_each_page_preserves_filter_and_bson_boundary(self): + oid = ObjectId() + query = {'_id': {'$ne': 'hidden'}, '$or': [{'ready': True}, {'urgent': True}]} + collection = Collection(Cursor(None), Cursor('abc'), Cursor(oid), Cursor(), Cursor(MaxKey())) + self.assertEqual( + [doc['_id'] for doc in scan_collection(collection, query, step=1)], + [None, 'abc', oid, MaxKey()], + ) + self.assertNotIn('min', collection.calls[0]) + for call, previous in zip(collection.calls[1:], [None, 'abc', oid]): + self.assertEqual(call['min'], {'_id': previous}) + self.assertEqual(call['filter'], {'$and': [query, {'_id': {'$ne': previous}}]}) + self.assertEqual(collection.calls[-1]['filter'], {'$and': [query, {'_id': {'$eq': MaxKey()}}]}) + self.assertEqual(query, {'_id': {'$ne': 'hidden'}, '$or': [{'ready': True}, {'urgent': True}]}) + self.assertTrue(all(cursor.closed == 1 for cursor in collection.cursors)) + + def test_retry_is_off_by_default_even_after_partial_read(self): + error = CursorNotFound('expired', code=43) + cursor = Cursor(1, error) + collection = Collection(cursor) + with self.assertRaises(CursorNotFound) as raised: + next(scan_collection(collection)) + self.assertIs(raised.exception, error) + self.assertEqual((len(collection.calls), cursor.read, cursor.closed), (1, 1, 1)) + + def test_opt_in_rereads_entire_failed_page_at_same_boundary(self): + collection = Collection( + Cursor(1, 2), Cursor(3, NetworkTimeout('socket timeout')), Cursor(3, 4), Cursor(5), Cursor() + ) + self.assertEqual( + [doc['_id'] for doc in scan_collection(collection, step=2, max_retries=1)], + [1, 2, 3, 4, 5], + ) + self.assertEqual(collection.calls[1], collection.calls[2]) + self.assertEqual(collection.calls[2]['min'], {'_id': 2}) + self.assertTrue(all(cursor.closed == 1 for cursor in collection.cursors)) + + def test_close_error_is_preserved_without_retrying_a_successfully_read_page(self): + error = AutoReconnect('close') + collection = Collection(Cursor(1, close_error=error)) + with self.assertRaises(AutoReconnect) as raised: + next(scan_collection(collection, max_retries=1)) + self.assertIs(raised.exception, error) + self.assertEqual(len(collection.calls), 1) + + def test_invalid_arguments_start_no_query(self): + collection = Collection() + with self.assertRaisesRegex(ValueError, 'step'): + next(scan_collection(collection, step=0)) + with self.assertRaisesRegex(ValueError, 'max_retries'): + next(scan_collection(collection, max_retries=True)) + self.assertFalse(collection.calls) + + +class TestStreamCollection(unittest.TestCase): + def test_evaluates_only_on_demand_and_closes_on_early_exit(self): + cursor = Cursor(1, 2, 3) + collection = Collection(cursor) + with stream_collection(collection) as rows: + self.assertFalse(collection.calls) + self.assertEqual(next(rows), {'_id': 1}) + self.assertEqual((cursor.read, cursor.closed), (1, 0)) + self.assertEqual(collection.calls[0]['batch_size'], 1) + self.assertEqual(collection.calls[0]['limit'], 0) + self.assertEqual(cursor.closed, 1) + self.assertEqual(list(rows), []) + + def test_reopens_after_last_delivery_and_resets_retry_budget_on_progress(self): + oid = ObjectId() + query = {'ready': True} + collection = Collection( + Cursor(None, CursorNotFound('expired')), + Cursor('abc', OperationFailure('cursor killed', code=237)), + Cursor(oid, NetworkTimeout('connection timed out')), + Cursor(4), + Cursor(MaxKey()), + ) + with stream_collection(collection, query, batch_size=5, max_retries=1) as rows: + self.assertEqual([doc['_id'] for doc in rows], [None, 'abc', oid, 4, MaxKey()]) + for call, previous in zip(collection.calls[1:], [None, 'abc', oid]): + self.assertEqual(call['min'], {'_id': previous}) + self.assertEqual(call['filter'], {'$and': [query, {'_id': {'$ne': previous}}]}) + self.assertEqual(call['batch_size'], 5) + self.assertEqual(collection.calls[-1]['filter'], {'$and': [query, {'_id': {'$eq': MaxKey()}}]}) + self.assertTrue(all(cursor.closed == 1 for cursor in collection.cursors)) + + def test_embedded_id_checkpoint_is_captured_before_document_delivery(self): + collection = Collection(Cursor({'part': 1}, OperationFailure('plan killed', code=175)), Cursor(), Cursor()) + with stream_collection(collection) as rows: + next(rows)['_id']['part'] = 2 + self.assertEqual(list(rows), []) + self.assertEqual(collection.calls[1]['min'], {'_id': {'part': 1}}) + + def test_exhausted_retries_raise_last_native_error(self): + error = ServerSelectionTimeoutError('selection') + collection = Collection(Cursor(1, AutoReconnect('unavailable')), error) + with self.assertRaises(ServerSelectionTimeoutError) as raised: + with stream_collection(collection, max_retries=1) as rows: + self.assertEqual(next(rows), {'_id': 1}) + next(rows) + self.assertIs(raised.exception, error) + self.assertEqual(collection.calls[1]['min'], {'_id': 1}) + self.assertEqual(collection.cursors[0].closed, 1) + + def test_initial_network_failure_retries_without_a_checkpoint(self): + collection = Collection(AutoReconnect('initial'), Cursor(1)) + with stream_collection(collection) as rows: + self.assertEqual(list(rows), [{'_id': 1}]) + self.assertEqual(collection.calls[0], collection.calls[1]) + + def test_permission_and_query_timeout_errors_are_propagated(self): + for error in (OperationFailure('unauthorized', code=13), ExecutionTimeout('server time limit', code=50)): + with self.subTest(error=type(error)): + cursor = Cursor(1, error) + collection = Collection(cursor) + with self.assertRaises(type(error)) as raised: + with stream_collection(collection) as rows: + list(rows) + self.assertIs(raised.exception, error) + self.assertEqual((len(collection.calls), cursor.closed), (1, 1)) + + def test_read_error_retains_failed_cleanup(self): + error = CursorNotFound('read') + cleanup = RuntimeError('close') + collection = Collection(Cursor(error, close_error=cleanup)) + with self.assertRaises(CursorNotFound) as raised: + with stream_collection(collection) as rows: + next(rows) + self.assertIs(raised.exception, error) + self.assertEqual(error.database_cleanup_errors, (cleanup,)) + self.assertEqual(len(collection.calls), 1) + + def test_body_error_retains_failed_cleanup_and_never_retries(self): + error = CursorNotFound('from consumer') + cleanup = AutoReconnect('close') + collection = Collection(Cursor(1, close_error=cleanup)) + with self.assertRaises(CursorNotFound) as raised: + with stream_collection(collection) as rows: + next(rows) + raise error + self.assertIs(raised.exception, error) + self.assertEqual(error.database_cleanup_errors, (cleanup,)) + self.assertEqual(len(collection.calls), 1) + + def test_early_close_error_is_visible_without_a_body_error(self): + error = RuntimeError('close') + collection = Collection(Cursor(1, close_error=error)) + with self.assertRaises(RuntimeError) as raised: + with stream_collection(collection) as rows: + next(rows) + self.assertIs(raised.exception, error) + + def test_invalid_batch_size_starts_no_query(self): + collection = Collection() + with self.assertRaisesRegex(ValueError, 'batch_size'): + with stream_collection(collection, batch_size=0): + pass + self.assertFalse(collection.calls) + + +class TestAsyncCollectionReads(unittest.IsolatedAsyncioTestCase): + async def test_scan_delivers_only_after_close_and_rereads_partial_page(self): + collection = Collection( + AsyncCursor(1, 2), + AsyncCursor(3, NetworkTimeout('timeout')), + AsyncCursor(3, 4), + AsyncCursor(), + AsyncCursor(), + ) + rows = async_scan_collection(collection, step=2, max_retries=1) + self.assertEqual(await anext(rows), {'_id': 1}) + self.assertEqual(collection.cursors[0].closed, 1) + self.assertEqual([doc['_id'] async for doc in rows], [2, 3, 4]) + self.assertEqual(collection.calls[1], collection.calls[2]) + self.assertTrue(all(cursor.closed == 1 for cursor in collection.cursors)) + + async def test_async_scan_does_not_retry_by_default(self): + error = CursorNotFound('expired') + collection = Collection(AsyncCursor(1, error)) + with self.assertRaises(CursorNotFound) as raised: + await anext(async_scan_collection(collection)) + self.assertIs(raised.exception, error) + self.assertEqual((len(collection.calls), collection.cursors[0].closed), (1, 1)) + + async def test_stream_reuses_cursor_until_interruption_then_recovers(self): + first = AsyncCursor(1, 2, OperationFailure('plan killed', code=175)) + second = AsyncCursor(3, 4) + collection = Collection(first, second) + async with async_stream_collection(collection) as rows: + self.assertEqual(await anext(rows), {'_id': 1}) + self.assertEqual((first.read, first.closed, len(collection.calls)), (1, 0, 1)) + self.assertEqual(await anext(rows), {'_id': 2}) + self.assertEqual(await anext(rows), {'_id': 3}) + self.assertEqual(collection.calls[1]['min'], {'_id': 2}) + self.assertEqual((first.closed, second.closed), (1, 1)) + self.assertEqual([row async for row in rows], []) + + async def test_async_stream_cleanup_error_is_reported(self): + error = RuntimeError('close') + collection = Collection(AsyncCursor(1, close_error=error)) + with self.assertRaises(RuntimeError) as raised: + async with async_stream_collection(collection) as rows: + await anext(rows) + self.assertIs(raised.exception, error) + + async def test_cancellation_during_read_closes_cursor_without_retrying(self): + for read_type in ('scan', 'stream'): + with self.subTest(read_type=read_type): + reading = asyncio.Event() + + class WaitingCursor(AsyncCursor): + async def __anext__(self): + reading.set() + await asyncio.Future() + + cursor = WaitingCursor() + collection = Collection(cursor) + + async def consume(): + if read_type == 'scan': + await anext(async_scan_collection(collection, max_retries=1)) + else: + async with async_stream_collection(collection) as rows: + await anext(rows) + + task = asyncio.create_task(consume()) + await asyncio.wait_for(reading.wait(), timeout=2) + task.cancel() + with self.assertRaises(asyncio.CancelledError): + await task + self.assertEqual((len(collection.calls), cursor.closed), (1, 1)) diff --git a/tests/test_mongodb_utils.py b/tests/test_mongodb_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..428386ae621da88899de6e99614bff0faa7b6029 --- /dev/null +++ b/tests/test_mongodb_utils.py @@ -0,0 +1,275 @@ +import asyncio +import unittest + +from pymongo import DeleteOne +from pymongo import InsertOne +from pymongo import ReplaceOne +from pymongo import UpdateMany +from pymongo import UpdateOne +from pymongo.errors import BulkWriteError +from pymongo.errors import InvalidOperation +from pymongo.results import BulkWriteResult +from pymongo.results import InsertManyResult + +from datvia.mongodb.utils import AsyncBatchInsert +from datvia.mongodb.utils import AsyncBatchUpdate +from datvia.mongodb.utils import AsyncBatchWrite +from datvia.mongodb.utils import BatchInsert +from datvia.mongodb.utils import BatchUpdate +from datvia.mongodb.utils import BatchWrite +from datvia.mongodb.utils import async_cursor_batches +from datvia.mongodb.utils import cursor_batches + + +class Collection: + def __init__(self, *, error=None, acknowledged=True): + self.calls = [] + self.error = error + self.acknowledged = acknowledged + + def insert_many(self, documents, *, ordered): + self.calls.append((list(documents), ordered)) + if self.error is not None: + raise self.error + return InsertManyResult([doc['_id'] for doc in documents], self.acknowledged) + + def bulk_write(self, operations, *, ordered): + self.calls.append((list(operations), ordered)) + if self.error is not None: + raise self.error + return BulkWriteResult({'nInserted': 1, 'nModified': 2, 'nRemoved': 1, 'nUpserted': 1}, self.acknowledged) + + +class AsyncCollection(Collection): + async def insert_many(self, documents, *, ordered): + return super().insert_many(documents, ordered=ordered) + + async def bulk_write(self, operations, *, ordered): + return super().bulk_write(operations, ordered=ordered) + + +class Cursor: + def __init__(self, rows=range(7), *, close_error=None): + self.rows = iter(rows) + self.read = 0 + self.closed = 0 + self.close_error = close_error + + def __iter__(self): + for row in self.rows: + self.read += 1 + yield row + + def close(self): + self.closed += 1 + if self.close_error is not None: + raise self.close_error + + +class AsyncCursor(Cursor): + async def __aiter__(self): + for row in self: + yield row + + async def close(self): + super().close() + + +class TestBatch(unittest.TestCase): + def test_incremental_extend_flushes_at_the_threshold(self): + collection = Collection() + + def documents(): + for index in range(7): + self.assertEqual(len(collection.calls), index // 3) + yield {'_id': index} + + batch = BatchInsert(collection, step=3, ordered=False) + with batch: + batch.extend(documents()) + self.assertEqual(batch.count, 1) + batch.execute() + batch.close() + self.assertEqual([len(docs) for docs, _ in collection.calls], [3, 3, 1]) + self.assertTrue(all(not ordered for _, ordered in collection.calls)) + self.assertEqual(batch.success_count, 7) + self.assertFalse(batch.ops) + + def test_discard_releases_tail_and_leaves_batch_usable(self): + collection = Collection() + batch = BatchInsert(collection, step=2) + batch.extend({'_id': index} for index in range(3)) + batch.discard() + batch.close() + self.assertEqual(collection.calls, [([{'_id': 0}, {'_id': 1}], True)]) + batch.add({'_id': 3}) + batch.close() + self.assertEqual(collection.calls[-1], ([{'_id': 3}], True)) + + def test_update_shortcut_and_mixed_operations(self): + collection = Collection() + native = UpdateOne({'_id': 1}, {'$set': {'ready': True}}, upsert=True) + with BatchUpdate(collection, step=2) as batch: + batch.add({'_id': 2}, [{'$set': {'ready': True}}]) + batch.extend(iter([native])) + with self.assertRaises(InvalidOperation): + batch.add({'_id': 3}) + self.assertEqual(batch.success_count, 2) + self.assertIs(collection.calls[0][0][1], native) + models = [InsertOne({'_id': 1}), UpdateMany({}, {'$inc': {'n': 1}}), ReplaceOne({}, {}), DeleteOne({})] + with BatchWrite(collection, step=3, ordered=False) as mixed: + mixed.extend(models) + self.assertEqual(collection.calls[1:], [(models[:3], False), (models[3:], False)]) + + def test_bulk_failure_has_raw_evidence_and_releases_buffer(self): + error = BulkWriteError({'nInserted': 1, 'writeErrors': [{'op': {'value': 'private'}}]}) + batch = BatchInsert(Collection(error=error), step=2) + with self.assertRaises(BulkWriteError) as caught: + batch.extend([{'_id': 1}, {'_id': 2}]) + self.assertIs(caught.exception, error) + self.assertEqual((batch.success_count, batch.error_count, batch.count), (1, 1, 0)) + batch.close() + + batch = BatchInsert(Collection(error=error), step=1, on_error='log') + with self.assertLogs('datvia.mongodb._batch', level='WARNING') as logs: + batch.add({'_id': 1}) + self.assertNotIn('private', '\n'.join(logs.output)) + + def test_context_preserves_primary_error_when_tail_write_fails(self): + error = RuntimeError('driver') + primary = ValueError('producer') + collection = Collection(error=error) + with self.assertRaises(ValueError) as caught: + with BatchInsert(collection) as batch: + batch.add({'_id': 1}) + raise primary + self.assertIs(caught.exception, primary) + self.assertEqual(primary.database_cleanup_errors, (error,)) + self.assertEqual(collection.calls, [([{'_id': 1}], True)]) + self.assertFalse(batch.ops) + + def test_validation_and_unacknowledged_counts(self): + with self.assertRaises(ValueError): + BatchInsert(Collection(), step=0) + with self.assertRaises(ValueError): + BatchUpdate(Collection(), on_error='invalid') + collection = Collection(acknowledged=False) + with BatchWrite(collection) as batch: + batch.add(DeleteOne({})) + self.assertEqual((batch.success_count, batch.error_count, batch.unacknowledged_count), (0, 0, 1)) + + +class TestAsyncBatch(unittest.IsolatedAsyncioTestCase): + async def test_extend_consumes_both_kinds_of_iterable(self): + collection = AsyncCollection() + + async def documents(): + for index in range(5): + self.assertEqual(len(collection.calls), index // 2) + yield {'_id': index} + + async with AsyncBatchInsert(collection, step=2) as batch: + await batch.extend(documents()) + await batch.extend(iter([{'_id': 5}, {'_id': 6}])) + await batch.discard() + self.assertEqual([len(docs) for docs, _ in collection.calls], [2, 2, 2]) + self.assertEqual(batch.success_count, 6) + + async def test_update_and_mixed_chunk_tail(self): + collection = AsyncCollection() + update = UpdateOne({'_id': 1}, {'$inc': {'n': 1}}) + async with AsyncBatchUpdate(collection, step=2) as batch: + await batch.extend([update]) + await batch.add({'_id': 2}, {'$unset': {'n': True}}) + self.assertIs(collection.calls[0][0][0], update) + self.assertEqual(batch.success_count, 2) + delete = DeleteOne({'_id': 3}) + async with AsyncBatchWrite(collection, step=2) as mixed: + await mixed.add(delete) + self.assertEqual(collection.calls[-1], ([delete], True)) + + async def test_concurrent_add_and_cancelled_flush_do_not_replay_or_drop_other_items(self): + first_started = asyncio.Event() + + class PausedCollection(AsyncCollection): + async def insert_many(self, documents, *, ordered): + if documents[0]['_id'] == 0: + first_started.set() + await asyncio.Future() + return await super().insert_many(documents, ordered=ordered) + + collection = PausedCollection() + batch = AsyncBatchInsert(collection, step=1) + first = asyncio.create_task(batch.add({'_id': 0})) + await first_started.wait() + second = asyncio.create_task(batch.add({'_id': 1})) + await asyncio.sleep(0) + self.assertFalse(second.done()) + first.cancel() + with self.assertRaises(asyncio.CancelledError): + await first + await second + await batch.close() + self.assertEqual(collection.calls, [([{'_id': 1}], True)]) + self.assertFalse(batch.ops) + + +class TestCursorBatches(unittest.TestCase): + def test_normal_consumption_and_early_exit(self): + cursor = Cursor() + with cursor_batches(cursor, step=3) as batches: + self.assertEqual(list(batches), [[0, 1, 2], [3, 4, 5], [6]]) + self.assertEqual(cursor.closed, 1) + cursor = Cursor() + with cursor_batches(cursor, step=3) as batches: + self.assertEqual(next(batches), [0, 1, 2]) + self.assertEqual(cursor.read, 3) + self.assertEqual(list(batches), []) + self.assertEqual(cursor.closed, 1) + + def test_cursor_iteration_and_close_failures_preserve_primary_error(self): + primary = ValueError('read') + cleanup = RuntimeError('close') + + def rows(): + yield 1 + raise primary + + cursor = Cursor(rows(), close_error=cleanup) + with self.assertRaises(ValueError) as caught: + with cursor_batches(cursor, step=2) as batches: + next(batches) + self.assertEqual(cursor.closed, 1) + self.assertIs(caught.exception, primary) + self.assertEqual(primary.database_cleanup_errors, (cleanup,)) + + +class TestAsyncCursorBatches(unittest.IsolatedAsyncioTestCase): + async def test_normal_consumption_and_early_exit(self): + cursor = AsyncCursor() + async with async_cursor_batches(cursor, step=3) as batches: + self.assertEqual([batch async for batch in batches], [[0, 1, 2], [3, 4, 5], [6]]) + self.assertEqual(cursor.closed, 1) + cursor = AsyncCursor() + async with async_cursor_batches(cursor, step=3) as batches: + self.assertEqual(await anext(batches), [0, 1, 2]) + self.assertEqual(cursor.read, 3) + self.assertEqual([batch async for batch in batches], []) + self.assertEqual(cursor.closed, 1) + + async def test_consumer_cancellation_closes_cursor(self): + cursor = AsyncCursor() + entered = asyncio.Event() + + async def consume(): + async with async_cursor_batches(cursor, step=3) as batches: + await anext(batches) + entered.set() + await asyncio.Future() + + consumer = asyncio.create_task(consume()) + await entered.wait() + consumer.cancel() + with self.assertRaises(asyncio.CancelledError): + await consumer + self.assertEqual(cursor.closed, 1) diff --git a/tests/test_mysql_client.py b/tests/test_mysql_client.py new file mode 100644 index 0000000000000000000000000000000000000000..377590b720fad8a16dfdda97aafdc9a9f1f2d237 --- /dev/null +++ b/tests/test_mysql_client.py @@ -0,0 +1,50 @@ +from unittest import IsolatedAsyncioTestCase, TestCase +from unittest.mock import AsyncMock, patch + +from mysql.connector import Error + +from datvia.mysql import Client + + +class MySQLClientTests(TestCase): + @patch("datvia.mysql.client.connector.connect") + def test_create_returns_the_driver_connection(self, connect): + connection = object() + connect.return_value = connection + + result = Client.create("option", host="127.0.0.1", port=3306) + + self.assertIs(result, connection) + connect.assert_called_once_with("option", host="127.0.0.1", port=3306) + + @patch("datvia.mysql.client.connector.connect") + def test_create_propagates_the_driver_error(self, connect): + error = Error("connection failed") + connect.side_effect = error + + with self.assertRaises(Error) as result: + Client.create() + + self.assertIs(result.exception, error) + + +class AsyncMySQLClientTests(IsolatedAsyncioTestCase): + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_create_async_returns_the_driver_connection(self, connect): + connection = object() + connect.return_value = connection + + result = await Client.create_async("option", host="127.0.0.1", port=3306) + + self.assertIs(result, connection) + connect.assert_awaited_once_with("option", host="127.0.0.1", port=3306) + + @patch("datvia.mysql.client.aio.connect", new_callable=AsyncMock) + async def test_create_async_propagates_the_driver_error(self, connect): + error = Error("connection failed") + connect.side_effect = error + + with self.assertRaises(Error) as result: + await Client.create_async() + + self.assertIs(result.exception, error) diff --git a/tests/test_schema_binding.py b/tests/test_schema_binding.py new file mode 100644 index 0000000000000000000000000000000000000000..e442e53fb96cafe56184f0311793c9eccc5abd68 --- /dev/null +++ b/tests/test_schema_binding.py @@ -0,0 +1,104 @@ +"""Lifecycle and exception contracts independent of optional database drivers.""" + +import asyncio +import subprocess +import sys +import unittest +from pathlib import Path +from types import SimpleNamespace + +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import ConnectionClosedError +from datvia.errors import DriverError +from datvia.schema import SchemaConflictError +from datvia.schema._base import BoundSchema + + +class NativeFailure(Exception): + pass + + +def adapter(): + return SimpleNamespace(classify=lambda exc: DriverError if isinstance(exc, NativeFailure) else None) + + +class SchemaBindingTests(unittest.TestCase): + def test_public_import_does_not_import_drivers(self): + source = str(Path(__file__).resolve().parents[1] / 'src') + code = ( + f'import sys; sys.path.insert(0, {source!r}); ' + 'from datvia.schema import Column, TableSpec; ' + 'from datvia.crud import RedisStrings, AsyncRedisStrings, StringSetItem; ' + 'assert not any(k == n or k.startswith(n + ".") ' + 'for n in ("pymongo", "mysql.connector", "redis") for k in sys.modules)' + ) + subprocess.run([sys.executable, '-B', '-S', '-c', code], check=True, capture_output=True) + + def test_bind_checks_backend_and_closed_handle(self): + conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') + with self.assertRaises(ConfigurationError): + BoundSchema(conn, backend='mongodb', is_async=False) + conn._closed = True + with self.assertRaises(ConnectionClosedError): + BoundSchema(conn, backend='mysql', is_async=False) + + def test_exception_preserves_original_and_target(self): + conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') + schema = BoundSchema(conn, backend='mysql', is_async=False) + original = NativeFailure('driver detail') + with self.assertRaises(DriverError) as raised: + with schema._operation('ensure_table', 'app.users'): + raise original + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.assertEqual(raised.exception.target, 'app.users') + self.assertEqual(raised.exception.connection_name, 'orders') + self.assertEqual(raised.exception.operation, 'ensure_table') + + def test_cleanup_errors_survive_conversion_without_duplicates(self): + conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') + schema = BoundSchema(conn, backend='mysql', is_async=False) + original = NativeFailure('driver detail') + cleanup = OSError('cursor close failed') + original.database_cleanup_errors = (cleanup,) + with self.assertRaises(DriverError) as raised: + with schema._operation('ensure_table', 'app.users'): + with schema._operation('inspect_table', 'app.users'): + raise original + self.assertIs(raised.exception.original, original) + self.assertEqual(raised.exception.database_cleanup_errors, (cleanup,)) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + + def test_business_and_unified_errors_keep_identity(self): + conn = Connection(object(), adapter(), 'orders', 'mysql', 'single') + schema = BoundSchema(conn, backend='mysql', is_async=False) + for original in (ValueError('business'), SchemaConflictError('mismatch', differences=('primary_key',))): + with self.assertRaises(type(original)) as raised: + with schema._operation('ensure_table'): + raise original + self.assertIs(raised.exception, original) + + +class AsyncSchemaBindingTests(unittest.IsolatedAsyncioTestCase): + async def test_sync_mode_and_loop_are_enforced(self): + conn = AsyncConnection(object(), adapter(), 'documents', 'mongodb', 'single', asyncio.get_running_loop()) + with self.assertRaises(ConfigurationError): + BoundSchema(conn, backend='mongodb', is_async=False) + schema = BoundSchema(conn, backend='mongodb', is_async=True) + with schema._operation('list_collections'): + pass + conn._loop = object() + with self.assertRaises(ConfigurationError): + with schema._operation('list_collections'): + pass + + async def test_cancellation_is_not_converted(self): + conn = AsyncConnection(object(), adapter(), 'documents', 'mongodb', 'single', asyncio.get_running_loop()) + schema = BoundSchema(conn, backend='mongodb', is_async=True) + original = asyncio.CancelledError() + with self.assertRaises(asyncio.CancelledError) as raised: + with schema._operation('ensure_collection'): + raise original + self.assertIs(raised.exception, original) diff --git a/tests/test_schema_mongodb.py b/tests/test_schema_mongodb.py new file mode 100644 index 0000000000000000000000000000000000000000..316b6c960680b573b9821bf196334fc065f3720b --- /dev/null +++ b/tests/test_schema_mongodb.py @@ -0,0 +1,411 @@ +"""MongoDB schema behavior with synchronous and asynchronous driver doubles.""" + +import asyncio +from unittest import IsolatedAsyncioTestCase +from unittest import TestCase +from unittest.mock import AsyncMock +from unittest.mock import Mock + +from pymongo.errors import DuplicateKeyError +from pymongo.errors import OperationFailure +from pymongo.errors import WriteConcernError +from pymongo.write_concern import WriteConcern + +from datvia._adapters.mongodb import Adapter +from datvia.connections import AsyncConnection +from datvia.connections import Connection +from datvia.errors import ConfigurationError +from datvia.errors import DriverError +from datvia.errors import PermissionDeniedError +from datvia.schema import AsyncSchema +from datvia.schema import Schema +from datvia.schema import SchemaConflictError + + +def _metadata(name='orders', kind='collection', **options): + document = {'name': name, 'type': kind, 'options': options, 'info': {'readOnly': kind == 'view'}} + if kind == 'collection': + document['idIndex'] = {'key': {'_id': 1}, 'name': '_id_'} + if 'collation' in options: + document['idIndex']['collation'] = options['collation'] + return document + + +def _index(name='order_key', **options): + return {'name': name, 'key': {'tenant': 1, 'number': -1}, 'v': 2, 'unique': True, **options} + + +class _Cursor: + def __init__(self, rows=(), *, error=None, cleanup=None): + self.rows = rows + self.error = error + self.close = Mock(side_effect=cleanup) + + def __iter__(self): + yield from self.rows + if self.error is not None: + raise self.error + + +class _AsyncCursor: + def __init__(self, rows=(), *, error=None, cleanup=None): + self.rows = rows + self.error = error + self.close = AsyncMock(side_effect=cleanup) + + async def __aiter__(self): + for row in self.rows: + yield row + if self.error is not None: + raise self.error + + +class MongoDBSchemaTests(TestCase): + def setUp(self): + self.collection = Mock(list_indexes=Mock(return_value=_Cursor())) + self.database = Mock(write_concern=WriteConcern(), get_collection=Mock(return_value=self.collection)) + self.client = Mock(get_database=Mock(return_value=self.database)) + self.adapter = Mock(classify=Adapter.classify) + self.connection = Connection(self.client, self.adapter, 'app-mongo', 'mongodb', 'single') + self.schema = Schema.mongodb(self.connection) + self.client.get_database.assert_not_called() + + def test_visible_catalog_metadata_and_exact_drop(self): + ordinary = _metadata() + capped = _metadata('queue', capped=True) + view = _metadata('daily', 'view', viewOn='orders', pipeline=[]) + series = _metadata('events', 'timeseries', timeseries={'timeField': 'at'}) + buckets = _metadata('system.buckets.events', timeseries={'timeField': 'at'}) + system = _metadata('system.views') + list_cursor = _Cursor([ordinary, capped, view, series, buckets, system]) + view_cursor = _Cursor([view]) + absent_cursor = _Cursor() + self.client.list_database_names.return_value = ['sales'] + self.database.list_collections.side_effect = [ + list_cursor, + _Cursor([ordinary]), + _Cursor([capped]), + view_cursor, + absent_cursor, + ] + + self.assertEqual(self.schema.list_databases(), ['sales']) + self.assertTrue(self.schema.database_exists('sales')) + self.assertFalse(self.schema.database_exists('invisible')) + self.assertEqual(self.schema.list_collections('sales'), ['orders']) + self.assertTrue(self.schema.collection_exists('sales', 'orders')) + self.assertFalse(self.schema.collection_exists('sales', 'queue')) + self.assertEqual(self.schema.describe_collection('sales', 'daily'), view) + self.assertIsNone(self.schema.describe_collection('sales', 'missing')) + self.assertIsNone(self.schema.drop_collection('sales', 'daily')) + self.assertIsNone(self.schema.drop_database('missing')) + + for cursor in (list_cursor, view_cursor, absent_cursor): + cursor.close.assert_called_once_with() + self.assertEqual(self.database.list_collections.call_args_list[0].kwargs, {'filter': {'type': 'collection'}}) + self.assertEqual(self.database.list_collections.call_args_list[3].kwargs, {'filter': {'name': 'daily'}}) + self.database.drop_collection.assert_called_once_with('daily') + self.client.drop_database.assert_called_once_with('missing') + self.adapter.close.assert_not_called() + self.client.close.assert_not_called() + + def test_ensure_default_creates_once_and_uses_builtin_id(self): + metadata = _metadata() + self.database.list_collections.side_effect = [_Cursor(), _Cursor([metadata]), _Cursor([metadata])] + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders')) + self.assertIsNone(self.schema.ensure_collection('sales', 'orders')) + + self.database.create_collection.assert_called_once_with('orders', check_exists=False) + self.collection.list_indexes.assert_not_called() + self.collection.create_index.assert_not_called() + + def test_ensure_accepts_clustered_builtin_id_but_rejects_missing_id(self): + clustered = _metadata(clusteredIndex={'key': {'_id': 1}, 'unique': True}) + clustered.pop('idIndex') + unindexed = _metadata(autoIndexId=False) + unindexed.pop('idIndex') + self.database.list_collections.side_effect = [_Cursor([clustered]), _Cursor([unindexed])] + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders')) + with self.assertRaises(SchemaConflictError) as raised: + self.schema.ensure_collection('sales', 'orders') + self.assertIn('_id', raised.exception.differences[0]) + self.collection.create_index.assert_not_called() + + def test_nonordinary_collection_is_not_changed(self): + for metadata in ( + _metadata(kind='view'), + _metadata(kind='timeseries', timeseries={'timeField': 'at'}), + _metadata(capped=True), + ): + with self.subTest(metadata=metadata): + self.database.list_collections.return_value = _Cursor([metadata]) + with self.assertRaises(SchemaConflictError) as raised: + self.schema.ensure_collection('sales', 'orders') + self.assertEqual(raised.exception.operation, 'ensure_collection') + self.assertEqual(raised.exception.target, 'sales.orders') + self.assertTrue(raised.exception.differences) + self.database.create_collection.assert_not_called() + self.collection.create_index.assert_not_called() + + def test_concurrent_collection_creation_is_verified(self): + original = OperationFailure('namespace exists', 48) + self.database.create_collection.side_effect = original + self.database.list_collections.side_effect = [ + _Cursor(), + _Cursor([_metadata()]), + _Cursor(), + _Cursor([_metadata(kind='view')]), + _Cursor(), + _Cursor(), + ] + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders')) + with self.assertRaises(SchemaConflictError) as conflict: + self.schema.ensure_collection('sales', 'orders') + with self.assertRaises(DriverError) as disappeared: + self.schema.ensure_collection('sales', 'orders') + for raised in (conflict, disappeared): + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + + def test_acknowledged_create_without_visible_metadata_is_unconfirmed(self): + self.database.list_collections.side_effect = [_Cursor(), _Cursor()] + with self.assertRaises(SchemaConflictError) as raised: + self.schema.ensure_collection('sales', 'orders') + self.assertIn('visible catalog', raised.exception.differences[0]) + + def test_custom_index_reuses_and_confirms_equivalent_definition(self): + collation = {'locale': 'en', 'strength': 2, 'version': '57.1'} + matching = _index(collation=collation, hidden=True, ns='sales.orders') + wrong_collation = _index('binary') + cursor = _Cursor([wrong_collation, matching]) + self.database.list_collections.return_value = _Cursor([_metadata(collation=collation)]) + self.collection.list_indexes.return_value = cursor + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders', key_fields=['tenant', 'number'])) + + self.collection.create_index.assert_called_once_with( + [('tenant', 1), ('number', -1)], name='order_key', unique=True, v=2, hidden=True + ) + cursor.close.assert_called_once_with() + self.database.create_collection.assert_not_called() + + def test_nonmatching_index_does_not_replace_full_unique_index(self): + indexes = [ + _index('sparse', sparse=True), + _index('partial', partialFilterExpression={'number': {'$exists': True}}), + _index('nonunique', unique=False), + _index('reversed', key={'number': 1, 'tenant': 1}), + _index('nonbinary', collation={'locale': 'en'}), + _index('hashed', key={'tenant': 1, 'number': 'hashed'}), + _index('tenant_1_number_1', key={'tenant': 1, 'number': 1}, unique=False), + ] + self.database.list_collections.return_value = _Cursor([_metadata()]) + self.collection.list_indexes.return_value = _Cursor(indexes) + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number'))) + + keys = self.collection.create_index.call_args.args[0] + options = self.collection.create_index.call_args.kwargs + self.assertEqual(keys, [('tenant', 1), ('number', 1)]) + self.assertIs(options['unique'], True) + self.assertNotIn(options['name'], {index['name'] for index in indexes}) + self.collection.drop_index.assert_not_called() + + def test_index_conflict_requires_matching_definition_and_acknowledgment(self): + original = OperationFailure('conflicting index name', 85) + self.database.list_collections.return_value = _Cursor([_metadata()]) + self.collection.list_indexes.side_effect = [_Cursor(), _Cursor([_index()]), _Cursor(), _Cursor()] + self.collection.create_index.side_effect = [original, 'order_key', original] + + self.assertIsNone(self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number'))) + self.assertEqual(self.collection.create_index.call_args.kwargs['name'], 'order_key') + with self.assertRaises(SchemaConflictError) as raised: + self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number')) + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.assertEqual(self.collection.create_index.call_count, 3) + + def test_known_index_rejection_does_not_reissue_unchanged_definition(self): + original = OperationFailure('conflicting index options', 85) + self.database.list_collections.return_value = _Cursor([_metadata()]) + self.collection.list_indexes.side_effect = [_Cursor([_index()]), _Cursor([_index()])] + self.collection.create_index.side_effect = original + + with self.assertRaises(DriverError) as raised: + self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number')) + + self.assertIs(raised.exception.original, original) + self.collection.create_index.assert_called_once() + + def test_duplicate_data_keeps_new_collection_and_native_failure(self): + original = DuplicateKeyError('duplicate value', 11000) + self.database.list_collections.side_effect = [_Cursor(), _Cursor([_metadata()])] + self.collection.create_index.side_effect = original + + with self.assertRaises(DriverError) as raised: + self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number')) + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.assertEqual(raised.exception.target, 'sales.orders') + self.database.create_collection.assert_called_once() + self.database.drop_collection.assert_not_called() + self.collection.drop_index.assert_not_called() + + def test_write_concern_error_cannot_become_success(self): + original = WriteConcernError('replication incomplete', 64) + self.database.list_collections.return_value = _Cursor() + self.database.create_collection.side_effect = original + with self.assertRaises(DriverError) as raised: + self.schema.ensure_collection('sales', 'orders') + self.assertIs(raised.exception.original, original) + + def test_w0_and_invalid_keys_fail_before_catalog_or_write(self): + for fields in ('tenant', (), ('tenant', 'tenant'), ('nested.key',), ('$expr',), ('\ud800',)): + with self.subTest(fields=fields): + with self.assertRaises(ConfigurationError) as raised: + self.schema.ensure_collection('sales', 'orders', key_fields=fields) + self.assertEqual(raised.exception.operation, 'ensure_collection') + self.database.write_concern = WriteConcern(w=0) + with self.assertRaises(ConfigurationError): + self.schema.ensure_collection('sales', 'orders') + with self.assertRaises(ConfigurationError): + self.schema.drop_collection('sales', 'orders') + self.database.list_collections.assert_not_called() + self.database.create_collection.assert_not_called() + self.database.drop_collection.assert_not_called() + + def test_catalog_permission_error_is_not_reported_as_missing(self): + original = OperationFailure('not authorized', 13) + self.database.list_collections.side_effect = original + with self.assertRaises(PermissionDeniedError) as raised: + self.schema.collection_exists('sales', 'orders') + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.assertEqual(raised.exception.operation, 'collection_exists') + + def test_cursor_cleanup_preserves_primary_and_cleanup(self): + original = OperationFailure('getMore failed', 43) + cleanup = OperationFailure('killCursors failed', 13) + cursor = _Cursor([_metadata()], error=original, cleanup=cleanup) + self.database.list_collections.return_value = cursor + + with self.assertRaises(DriverError) as raised: + self.schema.describe_collection('sales', 'orders') + + self.assertIs(raised.exception.original, original) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + self.assertEqual(raised.exception.database_cleanup_errors, (cleanup,)) + cursor.close.assert_called_once_with() + + +class AsyncMongoDBSchemaTests(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.collection = Mock(list_indexes=AsyncMock(return_value=_AsyncCursor()), create_index=AsyncMock()) + self.database = Mock( + write_concern=WriteConcern(), + get_collection=Mock(return_value=self.collection), + list_collections=AsyncMock(), + create_collection=AsyncMock(), + drop_collection=AsyncMock(), + ) + self.client = Mock( + get_database=Mock(return_value=self.database), + list_database_names=AsyncMock(return_value=['sales']), + drop_database=AsyncMock(), + ) + self.adapter = Mock(classify=Adapter.classify) + connection = AsyncConnection( + self.client, self.adapter, 'app-mongo', 'mongodb', 'single', asyncio.get_running_loop() + ) + self.schema = AsyncSchema.mongodb(connection) + self.client.get_database.assert_not_called() + + async def test_async_catalog_default_identity_and_exact_drop(self): + metadata = _metadata() + cursor = _AsyncCursor([metadata, _metadata('capped', capped=True)]) + self.database.list_collections.side_effect = [ + _AsyncCursor(), + _AsyncCursor([metadata]), + cursor, + _AsyncCursor([metadata]), + _AsyncCursor([metadata]), + _AsyncCursor(), + ] + + self.assertIsNone(await self.schema.ensure_collection('sales', 'orders')) + self.assertEqual(await self.schema.list_databases(), ['sales']) + self.assertTrue(await self.schema.database_exists('sales')) + self.assertFalse(await self.schema.database_exists('missing')) + self.assertEqual(await self.schema.list_collections('sales'), ['orders']) + self.assertTrue(await self.schema.collection_exists('sales', 'orders')) + self.assertEqual(await self.schema.describe_collection('sales', 'orders'), metadata) + self.assertIsNone(await self.schema.describe_collection('sales', 'missing')) + self.assertIsNone(await self.schema.drop_collection('sales', 'orders')) + self.assertIsNone(await self.schema.drop_database('sales')) + + self.database.create_collection.assert_awaited_once_with('orders', check_exists=False) + self.collection.create_index.assert_not_awaited() + self.database.drop_collection.assert_awaited_once_with('orders') + self.client.drop_database.assert_awaited_once_with('sales') + cursor.close.assert_awaited_once_with() + self.adapter.close_async.assert_not_called() + + async def test_concurrent_index_listing_does_not_bypass_acknowledgment(self): + self.database.list_collections.return_value = _AsyncCursor([_metadata()]) + self.collection.list_indexes.side_effect = [_AsyncCursor([_index()]), _AsyncCursor([_index()])] + both_started = asyncio.Event() + ready = asyncio.Event() + started = 0 + + async def create_index(*args, **kwargs): + nonlocal started + started += 1 + if started == 2: + both_started.set() + await ready.wait() + return 'order_key' + + self.collection.create_index.side_effect = create_index + tasks = [ + asyncio.create_task(self.schema.ensure_collection('sales', 'orders', key_fields=('tenant', 'number'))) + for _ in range(2) + ] + try: + await asyncio.wait_for(both_started.wait(), timeout=2) + self.assertTrue(all(not task.done() for task in tasks)) + finally: + ready.set() + results = await asyncio.gather(*tasks) + self.assertEqual(results, [None, None]) + self.assertEqual( + [call.kwargs['name'] for call in self.collection.create_index.await_args_list], + ['order_key', 'order_key'], + ) + + async def test_async_cleanup_cannot_replace_cancellation(self): + original = asyncio.CancelledError('read canceled') + cleanup = OperationFailure('failed to close cursor', 43) + cursor = _AsyncCursor([_metadata()], error=original, cleanup=cleanup) + self.database.list_collections.return_value = cursor + + with self.assertRaises(asyncio.CancelledError) as raised: + await self.schema.collection_exists('sales', 'orders') + + self.assertIs(raised.exception, original) + self.assertEqual(original.database_cleanup_errors, (cleanup,)) + cursor.close.assert_awaited_once_with() + + async def test_async_conflicting_concurrent_namespace_has_native_cause(self): + original = OperationFailure('namespace exists', 48) + self.database.create_collection.side_effect = original + self.database.list_collections.side_effect = [_AsyncCursor(), _AsyncCursor([_metadata(capped=True)])] + + with self.assertRaises(SchemaConflictError) as raised: + await self.schema.ensure_collection('sales', 'orders', key_fields=['tenant']) + + self.assertIs(raised.exception.original, original) + self.assertIs(raised.exception.__cause__, original) + self.collection.create_index.assert_not_awaited() diff --git a/tests/test_schema_mysql.py b/tests/test_schema_mysql.py new file mode 100644 index 0000000000000000000000000000000000000000..8bbbad52ea03f36ad1e7a683db9a2bcdf250fb2b --- /dev/null +++ b/tests/test_schema_mysql.py @@ -0,0 +1,505 @@ +"""Offline checks for the public MySQL Schema bindings and declarations.""" + +import asyncio +import subprocess +import sys +from datetime import date +from datetime import datetime +from datetime import timezone +from decimal import Decimal +from pathlib import Path +from types import SimpleNamespace +from unittest import IsolatedAsyncioTestCase +from unittest import TestCase +from unittest.mock import AsyncMock +from unittest.mock import Mock +from unittest.mock import call + +from mysql.connector.errorcode import ER_BAD_TABLE_ERROR +from mysql.connector.errorcode import ER_DB_CREATE_EXISTS +from mysql.connector.errorcode import ER_DB_DROP_EXISTS +from mysql.connector.errorcode import ER_TABLE_EXISTS_ERROR +from mysql.connector.errorcode import ER_TABLEACCESS_DENIED_ERROR +from mysql.connector.errors import ProgrammingError + +from datvia import AsyncConnection +from datvia import Connection +from datvia._adapters.mysql import Adapter as ConnectionAdapter +from datvia.errors import ConfigurationError +from datvia.errors import DriverError +from datvia.errors import PermissionDeniedError +from datvia.schema import AsyncSchema +from datvia.schema import Column +from datvia.schema import Schema +from datvia.schema import SchemaConflictError +from datvia.schema import TableSpec + +_LITERAL = "quote'; DROP DATABASE victim; --" +_STAMP = datetime(2026, 1, 2, 12, 15, 0, 230000) +_DATE = date(2026, 1, 2) +_BASE = ('Catalog', 'Users', 'BASE TABLE', 'InnoDB', 'utf8mb4', 'utf8mb4_0900_ai_ci') +_VIEW = ('Catalog', 'Users', 'VIEW', None, None, None) +_SPEC = TableSpec( + { + 'tenant': Column('bigint'), + 'id': Column('int'), + 'name': Column('varchar', length=100, default=_LITERAL), + 'balance': Column('decimal', precision=12, scale=2, default=Decimal('32.50')), + 'subscribed': Column('boolean', default=True), + 'score': Column('double', default=1.25), + 'born': Column('date', default=_DATE), + 'created': Column('datetime', precision=3, default=_STAMP), + 'note': Column('text', nullable=True, default=None), + 'payload': Column('blob', nullable=True), + }, + primary_key=('tenant', 'id'), +) +_COLUMNS = [ + ('tenant', 'bigint(20)', 'NO', None, '', ''), + ('id', 'int', 'NO', None, '', ''), + ('Name', 'varchar(100)', 'NO', _LITERAL, '', ''), + ('balance', 'decimal(12,2)', 'NO', '32.50', '', ''), + ('subscribed', 'tinyint(1)', 'NO', '1', '', ''), + ('score', 'double', 'NO', '1.25', '', ''), + ('born', 'date', 'NO', '2026-01-02', '', ''), + ('created', 'datetime(3)', 'NO', '2026-01-02 12:15:00.230', '', ''), + ('note', 'text', 'YES', None, '', ''), + ('payload', 'blob', 'YES', None, '', ''), +] +_KEYS = [('tenant',), ('id',)] + + +def _handle(*, is_async=False): + cursor = Mock() + native = Mock(in_transaction=False, autocommit=True, python_charset='utf8') + adapter = SimpleNamespace(classify=ConnectionAdapter.classify) + if is_async: + cursor.execute = AsyncMock() + cursor.fetchall = AsyncMock() + cursor.close = AsyncMock() + native.cursor = AsyncMock(return_value=cursor) + native.get_autocommit = AsyncMock(return_value=True) + connection = AsyncConnection(native, adapter, 'setup', 'mysql', 'single', asyncio.get_running_loop()) + return AsyncSchema.mysql(connection), native, cursor + native.cursor.return_value = cursor + connection = Connection(native, adapter, 'setup', 'mysql', 'single') + return Schema.mysql(connection), native, cursor + + +def _executed_sql(cursor): + return [entry.args[0].strip() for entry in cursor.execute.call_args_list] + + +class MySQLTableSpecTests(TestCase): + def test_model_import_has_no_driver_dependency(self): + source = str(Path(__file__).resolve().parents[1] / 'src') + code = ( + f'import sys; sys.path.insert(0, {source!r}); ' + 'from datvia.schema import Column, TableSpec; ' + 'TableSpec({"id": Column("int")}, ("id",)); ' + 'assert "mysql.connector" not in sys.modules' + ) + subprocess.run([sys.executable, '-B', '-S', '-c', code], check=True, capture_output=True) + + def test_accepts_supported_types_and_normalizes_defaults(self): + self.assertEqual(Column('BIGINT', default=-(2**63)).type, 'bigint') + self.assertEqual(Column('decimal', default=42).default, Decimal('42')) + self.assertEqual(Column('decimal', precision=8, scale=2, default=1.25).default, Decimal('1.25')) + self.assertEqual(Column('boolean', default=1).default, True) + self.assertEqual(Column('varchar', length=32, default='你好').default, '你好') + self.assertIsNone(Column('blob', nullable=True, default=None).default) + self.assertNotEqual(Column('text', nullable=True), Column('text', nullable=True, default=None)) + self.assertEqual(Column('datetime', default=_STAMP.replace(microsecond=0)).precision, None) + + def test_rejects_invalid_types_dimensions_and_defaults(self): + cases = [ + ('int unsigned', {}), + ('double); DROP TABLE t', {}), + ('int', {'length': 4}), + ('int', {'nullable': 1}), + ('int', {'default': None}), + ('int', {'default': 2**31}), + ('int', {'default': True}), + ('boolean', {'default': 2}), + ('varchar', {}), + ('varchar', {'length': True}), + ('varchar', {'length': 3, 'default': 'four'}), + ('decimal', {'precision': 66}), + ('decimal', {'precision': 2, 'scale': 3}), + ('decimal', {'precision': 8, 'scale': 2, 'default': Decimal('1.003')}), + ('decimal', {'precision': 8, 'scale': 2, 'default': Decimal('NaN')}), + ('double', {'default': float('inf')}), + ('date', {'default': _STAMP}), + ('datetime', {'precision': 7}), + ('datetime', {'precision': 1, 'default': _STAMP}), + ('datetime', {'precision': 6, 'default': _STAMP.replace(tzinfo=timezone.utc)}), + ('datetime', {'default': 'NOW()'}), + ('text', {'default': 'ordinary string'}), + ('blob', {'default': b'binary'}), + ('bigint', {'auto_increment': True, 'default': 0}), + ('boolean', {'auto_increment': True}), + ] + for kind, options in cases: + with self.subTest(kind=kind, options=options), self.assertRaises(ConfigurationError): + Column(kind, **options) + + def test_table_snapshots_columns_and_ordered_case_insensitive_key(self): + columns = {'tenant': Column('int'), 'Id': Column('bigint')} + key = ['ID', 'tenant'] + spec = TableSpec(columns, key) + columns.clear() + key.reverse() + self.assertEqual(list(spec.columns), ['tenant', 'Id']) + self.assertEqual(spec.primary_key, ('Id', 'tenant')) + with self.assertRaises(TypeError): + spec.columns['id'] = Column('int') + + def test_rejects_invalid_keys_columns_and_auto_increment(self): + identity = Column('bigint', auto_increment=True) + value = Column('int') + cases = [ + ({}, ('id',)), + ({'id': object()}, ('id',)), + ({'id': value}, 'id'), + ({'id': value}, ()), + ({'id': value}, ('id', 'ID')), + ({'id': value}, ('missing',)), + ({'id;DROP': value}, ('id;DROP',)), + ({'id': value, 'ID': value}, ('id',)), + ({'id': Column('int', nullable=True)}, ('id',)), + ({'id': Column('text')}, ('id',)), + ({'id': identity, 'tenant': value}, ('id', 'tenant')), + ({'id': identity, 'tenant': value}, ('tenant',)), + ] + for columns, key in cases: + with self.subTest(columns=columns, key=key), self.assertRaises(ConfigurationError): + TableSpec(columns, key) + self.assertEqual(TableSpec({'id': identity}, ('id',)).columns['id'], identity) + + +class SyncMySQLSchemaTests(TestCase): + def setUp(self): + self.schema, self.native, self.cursor = _handle() + + def test_create_database_rechecks_visibility_and_charset(self): + self.native.cursor.assert_not_called() + self.cursor.fetchall.side_effect = [[], [('utf8mb4',)], [('utf8mb4',)]] + self.assertIsNone(self.schema.ensure_database('Catalog')) + self.assertIsNone(self.schema.ensure_database('Catalog', charset='UTF8MB4')) + + qualified = chr(96) + 'Catalog' + chr(96) + self.assertEqual( + self.cursor.execute.call_args_list[1], + call(f'CREATE DATABASE IF NOT EXISTS {qualified} DEFAULT CHARACTER SET utf8mb4', ()), + ) + self.assertEqual(sum(sql.startswith('CREATE DATABASE') for sql in _executed_sql(self.cursor)), 1) + self.assertEqual(self.cursor.close.call_count, 4) + self.native.close.assert_not_called() + + def test_database_charset_conflict_is_read_only(self): + self.cursor.fetchall.return_value = [('latin1',)] + with self.assertRaises(SchemaConflictError) as caught: + self.schema.ensure_database('Catalog') + self.assertEqual(caught.exception.differences, ('Database has character set latin1, expected utf8mb4.',)) + self.assertEqual((caught.exception.connection_name, caught.exception.target), ('setup', 'Catalog')) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_create_table_binds_literals_and_checks_compatible_shape(self): + self.cursor.fetchall.side_effect = [[], [_BASE], _COLUMNS, _KEYS] + self.schema.ensure_table('Catalog', 'Users', _SPEC) + create = self.cursor.execute.call_args_list[1].args + quote = chr(96) + self.assertTrue(create[0].startswith(f'CREATE TABLE IF NOT EXISTS {quote}Catalog{quote}.{quote}Users{quote}')) + self.assertIn(f'{quote}name{quote} VARCHAR(100) NOT NULL DEFAULT %s', create[0]) + self.assertIn(f'{quote}subscribed{quote} BOOLEAN NOT NULL DEFAULT %s', create[0]) + self.assertIn(f'{quote}note{quote} TEXT NULL DEFAULT NULL', create[0]) + self.assertIn(f'PRIMARY KEY ({quote}tenant{quote}, {quote}id{quote})', create[0]) + self.assertTrue(create[0].endswith('ENGINE=InnoDB')) + self.assertNotIn(_LITERAL, create[0]) + self.assertEqual(create[1], (_LITERAL, Decimal('32.50'), True, 1.25, _DATE, _STAMP)) + self.assertEqual(self.cursor.close.call_count, 5) + self.native.close.assert_not_called() + + def test_compatible_table_allows_omittable_extra_columns_and_preserves_engine(self): + columns = _COLUMNS + [ + ('optional', 'int', 'YES', None, '', ''), + ('defaulted', 'int', 'NO', '(2 + 2)', 'DEFAULT_GENERATED INVISIBLE', ''), + ('derived', 'int', 'NO', None, 'STORED GENERATED', 'tenant + id'), + ] + other_engine = (*_BASE[:3], 'MyISAM', *_BASE[4:]) + self.cursor.fetchall.side_effect = [[other_engine], columns, _KEYS] + self.schema.ensure_table('Catalog', 'Users', _SPEC) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_declared_invisible_column_conflicts_without_writing(self): + columns = _COLUMNS.copy() + columns[2] = ('Name', 'varchar(100)', 'NO', _LITERAL, 'INVISIBLE', '') + self.cursor.fetchall.side_effect = [[_BASE], columns, _KEYS] + + with self.assertRaises(SchemaConflictError) as caught: + self.schema.ensure_table('Catalog', 'Users', _SPEC) + + self.assertEqual(caught.exception.differences, ("Column 'name' is invisible in default projections.",)) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_conflicting_columns_default_expression_and_key_order_report_differences(self): + columns = [ + ('tenant', 'bigint unsigned', 'NO', None, '', ''), + ('id', 'int', 'NO', None, 'auto_increment', ''), + ('Name', 'varchar(100)', 'YES', _LITERAL, 'DEFAULT_GENERATED', ''), + ('balance', 'decimal(13,2)', 'NO', '32.60', '', ''), + ('required', 'int', 'NO', None, 'INVISIBLE', ''), + *_COLUMNS[4:9], + ] + self.cursor.fetchall.side_effect = [[_BASE], columns, list(reversed(_KEYS))] + with self.assertRaises(SchemaConflictError) as caught: + self.schema.ensure_table('Catalog', 'Users', _SPEC) + self.assertEqual(caught.exception.operation, 'ensure_table') + self.assertEqual(caught.exception.target, 'Catalog.Users') + self.assertEqual(len(caught.exception.differences), 9) + for word in ('auto-increment', 'default', 'Extra column', 'Missing column', 'nullability', 'Primary key'): + self.assertTrue(any(word in item for item in caught.exception.differences), caught.exception.differences) + self.assertNotIn(_LITERAL, ' '.join(caught.exception.differences)) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_database_and_table_catalog_read_only(self): + self.cursor.fetchall.side_effect = [ + [(b'Catalog',), ('system',)], + [('utf8mb4',)], + [], + [('Users',)], + [_BASE], + [_VIEW], + [_BASE], + _COLUMNS, + _KEYS, + [], + ] + self.assertEqual(self.schema.list_databases(), ['Catalog', 'system']) + self.assertTrue(self.schema.database_exists('Catalog')) + self.assertFalse(self.schema.database_exists('Missing')) + self.assertEqual(self.schema.list_tables('Catalog'), ['Users']) + self.assertTrue(self.schema.table_exists('Catalog', 'Users')) + self.assertFalse(self.schema.table_exists('Catalog', 'View')) + record = self.schema.describe_table('Catalog', 'Users') + self.assertEqual(record['table'], 'Users') + self.assertEqual(record['database'], 'Catalog') + self.assertEqual(record['primary_key'], ('tenant', 'id')) + self.assertEqual(record['charset'], 'utf8mb4') + self.assertEqual(record['collation'], 'utf8mb4_0900_ai_ci') + self.assertEqual(record['engine'], 'InnoDB') + self.assertEqual(record['columns']['Name']['default'], _LITERAL) + self.assertEqual(record['columns']['balance']['type'], 'decimal(12,2)') + self.assertIsNone(self.schema.describe_table('Catalog', 'Missing')) + self.assertIn("TABLE_TYPE = 'BASE TABLE'", _executed_sql(self.cursor)[3]) + self.assertEqual(self.cursor.execute.call_args_list[6].args[1], ('Catalog', 'Users')) + self.native.close.assert_not_called() + + def test_existing_view_is_a_conflict_and_not_described_as_table(self): + self.cursor.fetchall.return_value = [_VIEW] + with self.assertRaises(SchemaConflictError) as caught: + self.schema.ensure_table('Catalog', 'Users', _SPEC) + self.assertEqual(caught.exception.differences, ('Existing object is not a base table.',)) + self.assertIsNone(self.schema.describe_table('Catalog', 'Users')) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + def test_concurrent_database_creation_rechecks_winning_charset(self): + self.cursor.execute.side_effect = [None, ProgrammingError(errno=ER_DB_CREATE_EXISTS), None] + self.cursor.fetchall.side_effect = [[], [('latin1',)]] + with self.assertRaises(SchemaConflictError): + self.schema.ensure_database('Catalog') + self.assertEqual(self.cursor.close.call_count, 3) + + def test_drop_is_explicit_idempotent_and_qualified(self): + self.cursor.execute.side_effect = [ + None, + ProgrammingError(errno=ER_BAD_TABLE_ERROR), + None, + ProgrammingError(errno=ER_DB_DROP_EXISTS), + ] + for _ in range(2): + self.schema.drop_table('Catalog', 'Users') + for _ in range(2): + self.schema.drop_database('Catalog') + + quote = chr(96) + drop_table = f'DROP TABLE IF EXISTS {quote}Catalog{quote}.{quote}Users{quote}' + drop_database = f'DROP DATABASE IF EXISTS {quote}Catalog{quote}' + self.assertEqual(_executed_sql(self.cursor), [drop_table, drop_table, drop_database, drop_database]) + self.cursor.fetchall.assert_not_called() + self.native.close.assert_not_called() + + def test_database_and_table_ddl_refuse_active_transactions_or_disabled_autocommit(self): + for in_transaction, autocommit in [(True, True), (False, False)]: + self.native.in_transaction = in_transaction + self.native.autocommit = autocommit + for operation, args in [ + ('ensure_database', ('Catalog',)), + ('ensure_table', ('Catalog', 'Users', _SPEC)), + ('drop_table', ('Catalog', 'Users')), + ('drop_database', ('Catalog',)), + ]: + with self.subTest(operation=operation), self.assertRaises(ConfigurationError) as caught: + getattr(self.schema, operation)(*args) + self.assertEqual(caught.exception.operation, operation) + self.assertTrue(caught.exception.target.startswith('Catalog')) + self.assertEqual(self.native.in_transaction, in_transaction) + self.assertEqual(self.native.autocommit, autocommit) + self.native.cursor.assert_not_called() + self.native.commit.assert_not_called() + self.native.rollback.assert_not_called() + + def test_invalid_names_and_non_table_spec_only_fail_locally(self): + for operation, args, kwargs in [ + ('ensure_database', ('evil;DROP',), {}), + ('ensure_database', ('app',), {'charset': 'utf8mb4 /* unsafe */'}), + ('database_exists', ('db dot',), {}), + ('ensure_table', ('app', 'safe', object()), {}), + ('drop_table', ('app', 'dangerous' + chr(96)), {}), + ]: + with self.subTest(operation=operation), self.assertRaises(ConfigurationError): + getattr(self.schema, operation)(*args, **kwargs) + self.native.cursor.assert_not_called() + + def test_metadata_permission_failure_and_cursor_close_failure_are_preserved(self): + original = ProgrammingError(errno=ER_TABLEACCESS_DENIED_ERROR) + failure = OSError('cursor close failed') + self.cursor.execute.side_effect = original + self.cursor.close.side_effect = failure + with self.assertRaises(PermissionDeniedError) as caught: + self.schema.table_exists('Catalog', 'Users') + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.target, 'Catalog.Users') + self.assertEqual(caught.exception.database_cleanup_errors, (failure,)) + self.assertEqual(original.database_cleanup_errors, (failure,)) + self.native.close.assert_not_called() + + +class AsyncMySQLSchemaTests(IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.schema, self.native, self.cursor = _handle(is_async=True) + + async def test_async_create_database_and_table_and_recheck(self): + self.native.cursor.assert_not_awaited() + self.cursor.fetchall.side_effect = [[], [('utf8mb4',)], [], [_BASE], _COLUMNS, _KEYS] + self.assertIsNone(await self.schema.ensure_database('Catalog')) + self.assertIsNone(await self.schema.ensure_table('Catalog', 'Users', _SPEC)) + self.assertEqual(self.native.get_autocommit.await_count, 2) + self.assertEqual(self.cursor.close.await_count, 8) + self.assertEqual(sum(sql.startswith('CREATE') for sql in _executed_sql(self.cursor)), 2) + self.assertEqual( + self.cursor.execute.await_args_list[4].args[1], + (_LITERAL, Decimal('32.50'), True, 1.25, _DATE, _STAMP), + ) + self.native.close.assert_not_called() + + async def test_async_catalog_and_drop(self): + self.cursor.fetchall.side_effect = [ + [('Catalog',)], + [('utf8mb4',)], + [('Users',)], + [_BASE], + [_BASE], + _COLUMNS, + _KEYS, + [], + ] + self.assertEqual(await self.schema.list_databases(), ['Catalog']) + self.assertTrue(await self.schema.database_exists('Catalog')) + self.assertEqual(await self.schema.list_tables('Catalog'), ['Users']) + self.assertTrue(await self.schema.table_exists('Catalog', 'Users')) + record = await self.schema.describe_table('Catalog', 'Users') + self.assertEqual(record['primary_key'], ('tenant', 'id')) + self.assertIsNone(await self.schema.describe_table('Catalog', 'Missing')) + self.native.get_autocommit.assert_not_awaited() + + self.cursor.execute.side_effect = [ + ProgrammingError(errno=ER_BAD_TABLE_ERROR), + ProgrammingError(errno=ER_DB_DROP_EXISTS), + ] + await self.schema.drop_table('Catalog', 'Users') + await self.schema.drop_database('Catalog') + self.assertEqual(self.cursor.close.await_count, 10) + self.native.close.assert_not_called() + + async def test_async_transaction_guard_uses_async_method(self): + self.native.in_transaction = True + with self.assertRaises(ConfigurationError): + await self.schema.ensure_database('Catalog') + self.native.get_autocommit.assert_not_awaited() + self.native.in_transaction = False + self.native.get_autocommit.return_value = False + with self.assertRaises(ConfigurationError) as caught: + await self.schema.drop_table('Catalog', 'Users') + self.assertEqual(caught.exception.operation, 'drop_table') + self.native.get_autocommit.assert_awaited_once_with() + self.native.cursor.assert_not_awaited() + self.native.commit.assert_not_called() + self.native.rollback.assert_not_called() + + async def test_async_transaction_state_is_checked_after_autocommit_query(self): + def get_autocommit(): + self.native.in_transaction = True + return True + + self.native.get_autocommit.side_effect = get_autocommit + with self.assertRaises(ConfigurationError): + await self.schema.drop_database('Catalog') + + self.native.cursor.assert_not_awaited() + + async def test_async_declared_generated_invisible_column_conflicts_for_both_attributes(self): + spec = TableSpec({'id': Column('int'), 'derived': Column('int', nullable=True)}, ('id',)) + columns = [ + ('id', 'int', 'NO', None, '', ''), + ('derived', 'int', 'YES', None, 'STORED GENERATED INVISIBLE', '(id + 1)'), + ] + self.cursor.fetchall.side_effect = [[_BASE], columns, [('id',)]] + + with self.assertRaises(SchemaConflictError) as caught: + await self.schema.ensure_table('Catalog', 'Users', spec) + + self.assertEqual( + caught.exception.differences, + ("Column 'derived' is generated.", "Column 'derived' is invisible in default projections."), + ) + self.assertTrue(all(sql.startswith('SELECT') for sql in _executed_sql(self.cursor))) + + async def test_async_concurrent_create_checks_final_matching_table(self): + self.cursor.execute.side_effect = [None, ProgrammingError(errno=ER_TABLE_EXISTS_ERROR), None, None, None] + self.cursor.fetchall.side_effect = [[], [_BASE], _COLUMNS, _KEYS] + self.assertIsNone(await self.schema.ensure_table('Catalog', 'Users', _SPEC)) + self.assertEqual(self.cursor.close.await_count, 5) + + async def test_async_hidden_concurrent_table_does_not_report_success(self): + original = ProgrammingError(errno=ER_TABLE_EXISTS_ERROR) + self.cursor.execute.side_effect = [None, original, None] + self.cursor.fetchall.side_effect = [[], []] + with self.assertRaises(DriverError) as caught: + await self.schema.ensure_table('Catalog', 'Users', _SPEC) + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(self.cursor.close.await_count, 3) + + async def test_async_drop_does_not_ignore_permission_error(self): + original = ProgrammingError(errno=ER_TABLEACCESS_DENIED_ERROR) + failure = RuntimeError('cursor close failed') + self.cursor.execute.side_effect = original + self.cursor.close.side_effect = failure + with self.assertRaises(PermissionDeniedError) as caught: + await self.schema.drop_table('Catalog', 'Users') + self.assertIs(caught.exception.original, original) + self.assertIs(caught.exception.__cause__, original) + self.assertEqual(caught.exception.database_cleanup_errors, (failure,)) + self.cursor.close.assert_awaited_once_with() + self.native.close.assert_not_called() + + async def test_async_cancelled_metadata_closes_cursor_and_propagates_cancellation(self): + cancellation = asyncio.CancelledError('cancelled') + self.cursor.fetchall.side_effect = cancellation + with self.assertRaises(asyncio.CancelledError) as caught: + await self.schema.list_tables('Catalog') + self.assertIs(caught.exception, cancellation) + self.cursor.close.assert_awaited_once_with() + self.native.close.assert_not_called()