From a90191e8d8735481878253d1bbf98ee71829cb36 Mon Sep 17 00:00:00 2001 From: lewist4cj Date: Tue, 5 May 2026 10:47:51 +0800 Subject: [PATCH] Add Cangjie binding with service layer, tests, and docs Cangjie ip2region xdb searcher binding with: - Low-level Searcher API (content buff / vector index / file only) - Multi-threaded service layer (Config, SearcherPool, Ip2Region) - 41 unit tests covering xdb and service packages - Example project with 4 usage demos + performance benchmark - Full documentation in README.md --- binding/cangjie/.gitignore | 5 + binding/cangjie/README.md | 172 +++++++++++++ binding/cangjie/cjpm.toml | 15 ++ binding/cangjie/example/cjpm.toml | 10 + binding/cangjie/example/src/main.cj | 196 +++++++++++++++ binding/cangjie/src/main.cj | 225 ++++++++++++++++++ binding/cangjie/src/service/config.cj | 72 ++++++ binding/cangjie/src/service/ip2region.cj | 161 +++++++++++++ binding/cangjie/src/service/searcher_pool.cj | 111 +++++++++ .../cangjie/src/tests/service/config_test.cj | 50 ++++ .../src/tests/service/ip2region_test.cj | 108 +++++++++ .../src/tests/service/searcher_pool_test.cj | 55 +++++ binding/cangjie/src/tests/tests.cj | 1 + binding/cangjie/src/tests/xdb/header_test.cj | 83 +++++++ .../cangjie/src/tests/xdb/searcher_test.cj | 132 ++++++++++ binding/cangjie/src/tests/xdb/util_test.cj | 133 +++++++++++ binding/cangjie/src/xdb/header.cj | 77 ++++++ binding/cangjie/src/xdb/searcher.cj | 146 ++++++++++++ binding/cangjie/src/xdb/util.cj | 108 +++++++++ binding/cangjie/src/xdb/version.cj | 41 ++++ 20 files changed, 1901 insertions(+) create mode 100644 binding/cangjie/.gitignore create mode 100644 binding/cangjie/README.md create mode 100644 binding/cangjie/cjpm.toml create mode 100644 binding/cangjie/example/cjpm.toml create mode 100644 binding/cangjie/example/src/main.cj create mode 100644 binding/cangjie/src/main.cj create mode 100644 binding/cangjie/src/service/config.cj create mode 100644 binding/cangjie/src/service/ip2region.cj create mode 100644 binding/cangjie/src/service/searcher_pool.cj create mode 100644 binding/cangjie/src/tests/service/config_test.cj create mode 100644 binding/cangjie/src/tests/service/ip2region_test.cj create mode 100644 binding/cangjie/src/tests/service/searcher_pool_test.cj create mode 100644 binding/cangjie/src/tests/tests.cj create mode 100644 binding/cangjie/src/tests/xdb/header_test.cj create mode 100644 binding/cangjie/src/tests/xdb/searcher_test.cj create mode 100644 binding/cangjie/src/tests/xdb/util_test.cj create mode 100644 binding/cangjie/src/xdb/header.cj create mode 100644 binding/cangjie/src/xdb/searcher.cj create mode 100644 binding/cangjie/src/xdb/util.cj create mode 100644 binding/cangjie/src/xdb/version.cj diff --git a/binding/cangjie/.gitignore b/binding/cangjie/.gitignore new file mode 100644 index 0000000..f01a540 --- /dev/null +++ b/binding/cangjie/.gitignore @@ -0,0 +1,5 @@ +# build cache +.cache/ + +# dependency lock +cjpm.lock diff --git a/binding/cangjie/README.md b/binding/cangjie/README.md new file mode 100644 index 0000000..575ed70 --- /dev/null +++ b/binding/cangjie/README.md @@ -0,0 +1,172 @@ +# ip2region xdb searcher — Cangjie Binding + +ip2region (v2.0/v3.0) xdb 数据库的仓颉语言查询库，支持 IPv4 和 IPv6，提供多线程安全的服务层。 + +## Architecture + +``` +binding/cangjie/ +├── src/ +│ ├── xdb/ # Low-level xdb query engine +│ │ ├── searcher.cj # Searcher (file-only / vector-index / content-buff modes) +│ │ ├── util.cj # IP parsing/comparison, byte readers +│ │ ├── header.cj # xdb header + version parsing +│ │ ├── version.cj # IPv4/IPv6 version definitions +│ │ ├── *_test.cj # Unit tests (24 cases) +│ ├── service/ # High-level thread-safe service layer +│ │ ├── config.cj # Config (cache policy, pool size, pre-loaded data) +│ │ ├── searcher_pool.cj # SearcherPool (Semaphore + ConcurrentLinkedQueue) +│ │ ├── ip2region.cj # Ip2Region unified API +│ │ ├── *_test.cj # Unit tests (17 cases) +├── example/ # Executable CLI demo (separate project) +│ ├── cjpm.toml +│ └── src/main.cj # 5 usage demos + benchmarks +├── cjpm.toml # Static library project +└── README.md +``` + +**Two API layers:** + +- **`ip2region.xdb`** — Low-level Searcher. Not thread-safe (mutable `ioCount` + `File` seek/read state). Supports three cache modes. +- **`ip2region.service`** — High-level `Ip2Region` class wrapping `SearcherPool`. Thread-safe, handles v4/v6 dispatch automatically. + +## Build + +```bash +cd binding/cangjie +cjpm build # static library +cd example +cjpm build # example executable +``` + +## Test + +41 tests across both packages (24 xdb + 17 service): + +```bash +cd binding/cangjie +cjpm test +``` + +## Cache Policies + +| Policy | Memory | Speed | Thread-safe* | +|--------|--------|-------|-------------| +| `FileOnly` (0) | ~0 MB | ~21 µs/op (v4) | Via SearcherPool | +| `VectorIndex` (1) | ~4 MB | ~20 µs/op (v4) | Via SearcherPool | +| `ContentBuff` (2) | ~full xdb | ~5 µs/op (v4) | Via SearcherPool | + +\* Low-level `Searcher` is NOT thread-safe. The service layer (`SearcherPool` / `Ip2Region`) provides thread safety by pooling searchers (FileOnly/VectorIndex) or sharing a read-only buffer (ContentBuff). + +## Benchmark Results + +Benchmarked on Windows 11, AMD Ryzen 7, 487k IPv4 records / 638k IPv6 records: + +### IPv4 (`ip2region_v4.xdb`, 487,167 records) + +| Cache Policy | Total | Time | Avg | +|-------------|-------|------|-----| +| ContentBuff | 487,167 | 3,059 ms | **5 µs/op** | +| VectorIndex | 487,167 | 10,413 ms | **20 µs/op** | +| FileOnly | 487,167 | 11,271 ms | **21 µs/op** | + +### IPv6 (`ip2region_v6.xdb`, 638,953 records) + +| Cache Policy | Total | Time | Avg | +|-------------|-------|------|-----| +| ContentBuff | 638,953 | 37,080 ms | **56 µs/op** | +| VectorIndex | 638,953 | 38,443 ms | **58 µs/op** | +| FileOnly | 638,953 | 19,148 ms | **29 µs/op** | + +Run your own: + +```bash +cd example +cjpm run -- bench --db ../../../data/ip2region_v4.xdb --src ../../../data/ipv4_source.txt --cache-policy content +cjpm run -- bench --db ../../../data/ip2region_v6.xdb --src ../../../data/ipv6_source.txt --cache-policy file +``` + +## Usage + +### Low-level API (`ip2region.xdb`) + +```cangjie +import ip2region.xdb.* + +// Content buffer mode (fastest) +let content = File.readFrom(Path("ip2region_v4.xdb")) +let header = newHeaderFromBytes(content) +let version = versionFromHeader(header) +let searcher = Searcher(version, content, "ip2region_v4.xdb") +let region = searcher.search(parseIP("220.181.108.183")) +searcher.close() + +// Vector index mode +let vIndex = loadVectorIndex(content) +let searcher2 = Searcher(version, "ip2region_v4.xdb", vIndex) +searcher2.close() + +// File only mode +let searcher3 = Searcher(version, "ip2region_v4.xdb") +searcher3.close() +``` + +### High-level API (`ip2region.service`) + +```cangjie +import ip2region.service.* + +// Via factory (VectorIndex, 20 searchers) +let region = newIp2Region("ip2region_v4.xdb", "ip2region_v6.xdb") +let result = region.search("220.181.108.183") +region.close() + +// Custom config +let v4Cfg = Config(ContentBuff, IPv4, "ip2region_v4.xdb", 10) +let v6Cfg = Config(VectorIndex, IPv6, "ip2region_v6.xdb", 20) +let svc = Ip2Region(v4Cfg, v6Cfg) +let result = svc.search("2408:8266:100:1000::") +svc.close() +``` + +### Single-version API + +```cangjie +import ip2region.service.* + +// IPv4 only — Ip2Region(config, IPv4) +let v4Cfg = Config(ContentBuff, IPv4, "ip2region_v4.xdb", 10) +let v4 = Ip2Region(v4Cfg, IPv4) +let r4 = v4.search("220.181.108.183") +v4.close() + +// IPv6 only — Ip2Region(config, IPv6) +let v6Cfg = Config(ContentBuff, IPv6, "ip2region_v6.xdb", 10) +let v6 = Ip2Region(v6Cfg, IPv6) +let r6 = v6.search("2408:8266:100:1000::") +v6.close() + +// Factory functions +let v4Only = newIp2RegionV4(v4Cfg) +let v6Only = newIp2RegionV6(v6Cfg) +v4Only.close(); v6Only.close() +``` + +### CLI + +```bash +cd example +cjpm run +``` + +## Thread Safety + +- **`xdb.Searcher`** — NOT thread-safe. Keep one per thread or use a pool. +- **`service.SearcherPool`** — Thread-safe. Pre-allocates N searchers, uses `Semaphore` for backpressure and `ConcurrentLinkedQueue` for storage. `borrow()` blocks until a searcher is available. +- **`service.Ip2Region`** — Thread-safe. Delegates to pool or shared in-mem searcher based on cache policy. ContentBuff mode shares a single Searcher (Array is read-only), FileOnly/VectorIndex use the pool. +- **`service.Config`** — Immutable after construction. Safe to share. + +## Requirements + +- Cangjie SDK 1.1.0+ +- `cjpm` build tool diff --git a/binding/cangjie/cjpm.toml b/binding/cangjie/cjpm.toml new file mode 100644 index 0000000..6078a2f --- /dev/null +++ b/binding/cangjie/cjpm.toml @@ -0,0 +1,15 @@ +[package] + cjc-version = "1.1.0" + name = "ip2region" + organization = "" + description = "ip2region xdb searcher library for Cangjie" + version = "1.0.0" + target-dir = "" + script-dir = "" + output-type = "static" + compile-option = "-Woff unused" + override-compile-option = "" + link-option = "" + package-configuration = {} + +[dependencies] diff --git a/binding/cangjie/example/cjpm.toml b/binding/cangjie/example/cjpm.toml new file mode 100644 index 0000000..ae6c95c --- /dev/null +++ b/binding/cangjie/example/cjpm.toml @@ -0,0 +1,10 @@ +[package] + cjc-version = "1.1.0" + name = "ip2regionExample" + version = "1.0.0" + output-type = "executable" + compile-option = "-Woff unused" + description = "ip2region Cangjie CLI example" + +[dependencies] + ip2region = { path = ".." } diff --git a/binding/cangjie/example/src/main.cj b/binding/cangjie/example/src/main.cj new file mode 100644 index 0000000..def748d --- /dev/null +++ b/binding/cangjie/example/src/main.cj @@ -0,0 +1,196 @@ +package ip2regionExample + +import std.fs.* +import std.time.* +import std.collection.* +import ip2region.xdb.* +import ip2region.service.* + +// ip2region Cangjie 使用示例 +// +// 构建: cjpm build +// 运行: cjpm run +// +// 默认从 ../../../data/ 查找 xdb 文件， +// 也可通过命令行参数指定路径: +// cjpm run -- --v4-db --v6-db + +func printHelp() { + println("ip2region Cangjie 使用示例") + println("用法: cjpm run -- [options]") + println("选项:") + println(" --v4-db IPv4 xdb 文件路径 (默认: ../../../data/ip2region_v4.xdb)") + println(" --v6-db IPv6 xdb 文件路径 (默认: ../../../data/ip2region_v6.xdb)") + println(" --help 显示此帮助") +} + +// ============================================================ +// 方式一: 使用底层 Searcher API (ip2region.xdb) +// 可自行选择缓存策略，单线程使用 +// ============================================================ +func demoXdbSearcher(v4Path: String, v6Path: String) { + println("\n=== 方式一: 底层 Searcher API ===") + + // 1. 读取 xdb 文件到内存 + let v4Content = File.readFrom(Path(v4Path)) + + // 2. 解析文件头，获取版本信息 + let v4Header = newHeaderFromBytes(v4Content) + let v4Version = versionFromHeader(v4Header) + + // 3. 选择缓存策略创建 Searcher + // 支持三种模式: ContentBuff(最快), VectorIndex(均衡), FileOnly(省内存) + let vIndex = loadVectorIndex(v4Content) + let searcher = Searcher(v4Version, v4Path, vIndex) // VectorIndex 模式 + + // 4. 查询 IP + let ip = parseIP("220.181.108.183") + let region = searcher.search(ip) + println(" IP: 220.181.108.183") + println(" 结果: ${region}") + + searcher.close() +} + +// ============================================================ +// 方式二: 使用 Ip2Region 服务 API (ip2region.service) +// 自动管理连接池，支持多线程安全访问 +// ============================================================ +func demoIp2Region(v4Path: String, v6Path: String) { + println("\n=== 方式二: Ip2Region 服务 API ===") + + // 1. 创建配置 (缓存策略 + 连接池大小) + let v4Cfg = Config(VectorIndex, IPv4, v4Path, 10) + let v6Cfg = Config(VectorIndex, IPv6, v6Path, 10) + + // 2. 创建服务 (同时支持 IPv4 + IPv6) + let region = Ip2Region(v4Cfg, v6Cfg) + + // 3. 查询 IPv4 + let r1 = region.search("220.181.108.183") + println(" IPv4: 220.181.108.183 -> ${r1}") + + // 4. 查询 IPv6 + let r2 = region.search("2408:8266:100:1000::") + println(" IPv6: 2408:8266:100:1000:: -> ${r2}") + + region.close() +} + +// ============================================================ +// 方式三: 单版本服务 (仅 IPv4 或 仅 IPv6) +// 使用 init(config, ipVersion) 构造函数 +// ============================================================ +func demoSingleVersion(v4Path: String, v6Path: String) { + println("\n=== 方式三: 单版本服务 ===") + + // IPv4 only + let v4Cfg = Config(ContentBuff, IPv4, v4Path, 3) + let v4Region = Ip2Region(v4Cfg, IPv4) + let r4 = v4Region.search("220.181.108.183") + println(" IPv4 only: 220.181.108.183 -> ${r4}") + v4Region.close() + + // IPv6 only + let v6Cfg = Config(ContentBuff, IPv6, v6Path, 3) + let v6Region = Ip2Region(v6Cfg, IPv6) + let r6 = v6Region.search("2408:8266:100:1000::") + println(" IPv6 only: 2408:8266:100:1000:: -> ${r6}") + v6Region.close() +} + +// ============================================================ +// 方式四: 使用简便工厂函数 (ip2region.service) +// 一行创建，默认 VectorIndex + 20 连接池 +// ============================================================ +func demoFactory(v4Path: String, v6Path: String) { + println("\n=== 方式四: 简便工厂函数 ===") + + let region = newIp2Region(v4Path, v6Path) + let result = region.search("220.181.108.183") + println(" 220.181.108.183 -> ${result}") + region.close() +} + +// ============================================================ +// 性能测试: 测试三种缓存策略的查询速度 +// ============================================================ +func bench(v4Path: String) { + println("\n=== 性能测试 (IPv4, 487k 条记录) ===") + + let content = File.readFrom(Path(v4Path)) + let header = newHeaderFromBytes(content) + let version = versionFromHeader(header) + let vIndex = loadVectorIndex(content) + + // 从数据文件读取测试 IP 列表 + let srcPath = v4Path.replace("ip2region_v4.xdb", "ipv4_source.txt") + let srcContent = File.readFrom(Path(srcPath)) + let lines = String.fromUtf8(srcContent).split("\n") + + var ipList = ArrayList>() + for (line in lines) { + let trimmed = line.trimAsciiStart().trimAsciiEnd() + if (trimmed.size == 0) { continue } + let parts = trimmed.split("|") + if (parts.size < 3) { continue } + try { ipList.add(parseIP(parts[0])) } catch (_) { continue } + } + println(" 加载了 ${ipList.size} 个 IP") + + // 测试 ContentBuff 模式 + println("\n [ContentBuff 模式]") + let s1 = Searcher(version, content, v4Path) + benchSearcher(s1, ipList) + + // 测试 VectorIndex 模式 + println(" [VectorIndex 模式]") + let s2 = Searcher(version, v4Path, vIndex) + benchSearcher(s2, ipList) + + // 测试 FileOnly 模式 + println(" [FileOnly 模式]") + let s3 = Searcher(version, v4Path) + benchSearcher(s3, ipList) + + s1.close(); s2.close(); s3.close() +} + +func benchSearcher(searcher: Searcher, ipList: ArrayList>) { + let count = ipList.size + let start = MonoTime.now() + for (i in 0..count) { + let _ = searcher.search(ipList[i]) + } + let elapsed = MonoTime.now() - start + let avg = elapsed.toMicroseconds() / count + println(" 总量: ${count}, 耗时: ${elapsed.toMilliseconds()} ms, 平均: ${avg} us/op") +} + +main(args: Array): Int64 { + var v4Path = "../../../data/ip2region_v4.xdb" + var v6Path = "../../../data/ip2region_v6.xdb" + + var i = 0 + while (i < args.size) { + if (args[i] == "--v4-db" && i + 1 < args.size) { + v4Path = args[i + 1]; i = i + 2 + } else if (args[i] == "--v6-db" && i + 1 < args.size) { + v6Path = args[i + 1]; i = i + 2 + } else if (args[i] == "--help") { + printHelp(); return 0 + } else { i = i + 1 } + } + + println("ip2region Cangjie 使用示例") + println("xdb 路径: ${v4Path}") + + demoXdbSearcher(v4Path, v6Path) + demoIp2Region(v4Path, v6Path) + demoSingleVersion(v4Path, v6Path) + demoFactory(v4Path, v6Path) + bench(v4Path) + + println("\n完成!") + return 0 +} diff --git a/binding/cangjie/src/main.cj b/binding/cangjie/src/main.cj new file mode 100644 index 0000000..577f377 --- /dev/null +++ b/binding/cangjie/src/main.cj @@ -0,0 +1,225 @@ +package ip2region + +import std.fs.* +import std.time.* +import std.env.* +import ip2region.xdb.* + +func printHelp() { + println("ip2region xdb searcher") + println("Usage: ip2region [command] [command options]") + println("Command:") + println(" search search input test") + println(" bench search bench test") +} + +func runSearch(args: Array) { + var v4DbPath = "../../data/ip2region_v4.xdb" + var v6DbPath = "../../data/ip2region_v6.xdb" + var v4CachePolicy = "vectorIndex" + var v6CachePolicy = "vectorIndex" + var showHelp = false + + var i = 1 // skip command name + while (i < args.size) { + let arg = args[i] + if (arg == "--v4-db" && i + 1 < args.size) { + v4DbPath = args[i + 1] + i = i + 2 + } else if (arg == "--v6-db" && i + 1 < args.size) { + v6DbPath = args[i + 1] + i = i + 2 + } else if (arg == "--v4-cache-policy" && i + 1 < args.size) { + v4CachePolicy = args[i + 1] + i = i + 2 + } else if (arg == "--v6-cache-policy" && i + 1 < args.size) { + v6CachePolicy = args[i + 1] + i = i + 2 + } else if (arg == "--help") { + showHelp = true + i = i + 1 + } else { + i = i + 1 + } + } + + if (showHelp) { + println("ip2region search [command options]") + println("options:") + println(" --v4-db ip2region v4 binary xdb file path") + println(" --v4-cache-policy v4 cache policy: file/vectorIndex/content") + println(" --v6-db ip2region v6 binary xdb file path") + println(" --v6-cache-policy v6 cache policy: file/vectorIndex/content") + println(" --help print this help") + return + } + + // Load v4 xdb + print("Loading v4 xdb ... ") + let v4Content = File.readFrom(Path(v4DbPath)) + let v4Header = newHeaderFromBytes(v4Content) + let v4Version = versionFromHeader(v4Header) + let v4Searcher = createSearcher(v4Version, v4Content, v4DbPath, v4CachePolicy) + println("done") + + // Load v6 xdb + print("Loading v6 xdb ... ") + let v6Content = File.readFrom(Path(v6DbPath)) + let v6Header = newHeaderFromBytes(v6Content) + let v6Version = versionFromHeader(v6Header) + let v6Searcher = createSearcher(v6Version, v6Content, v6DbPath, v6CachePolicy) + println("done") + + println("ip2region search service test program") + println("type 'quit' to exit") + + let reader = getStdIn() + while (true) { + print("ip2region>> ") + let lineOpt = reader.readln() + if (lineOpt == None) { + break + } + let line = lineOpt.getOrThrow() + if (line == "quit" || line == "exit") { + break + } + if (line.size == 0) { + continue + } + + let start = MonoTime.now() + let region = searchIP(line, v4Searcher, v6Searcher) + let elapsed = MonoTime.now() - start + println("${line} -> ${region} (took: ${elapsed.toMicroseconds()} us)") + } + + v4Searcher.close() + v6Searcher.close() +} + +func runBench(args: Array) { + var dbFile = "" + var srcFile = "" + var cachePolicy = "content" + + var i = 1 // skip command name + while (i < args.size) { + let arg = args[i] + if (arg == "--db" && i + 1 < args.size) { + dbFile = args[i + 1] + i = i + 2 + } else if (arg == "--src" && i + 1 < args.size) { + srcFile = args[i + 1] + i = i + 2 + } else if (arg == "--cache-policy" && i + 1 < args.size) { + cachePolicy = args[i + 1] + i = i + 2 + } else { + i = i + 1 + } + } + + if (dbFile == "" || srcFile == "") { + println("ip2region bench [command options]") + println("options:") + println(" --db ip2region binary xdb file path") + println(" --src source ip text file path") + println(" --cache-policy cache policy: file/vectorIndex/content") + return + } + + println("Loading xdb from: ${dbFile}") + let content = File.readFrom(Path(dbFile)) + let header = newHeaderFromBytes(content) + let version = versionFromHeader(header) + let searcher = createSearcher(version, content, dbFile, cachePolicy) + + println("Loading source data from: ${srcFile}") + let srcContent = File.readFrom(Path(srcFile)) + let srcStr = String.fromUtf8(srcContent) + let lines = srcStr.split("\n") + + var count: Int64 = 0 + var totalCost: Int64 = 0 + let tStart = MonoTime.now() + + for (line in lines) { + let trimmed = line.trimAsciiStart().trimAsciiEnd() + if (trimmed.size == 0) { + continue + } + + let parts = trimmed.split("|") + if (parts.size < 3) { + continue + } + + let ipBytes = parseIP(parts[0]) + let t0 = MonoTime.now() + let _ = searcher.search(ipBytes) + let t1 = MonoTime.now() + totalCost = totalCost + (t1 - t0).toNanoseconds() + count = count + 1 + + if (count % 10000 == 0) { + print(".") + } + } + + let tEnd = MonoTime.now() + let totalTime = tEnd - tStart + + println("") + println("Bench finished:") + println(" cachePolicy: ${cachePolicy}") + println(" total: ${count}") + println(" took: ${totalTime.toMilliseconds()} ms") + if (count > 0) { + println(" avg: ${totalCost / count / 1000} us/op") + } + + searcher.close() +} + +func createSearcher(version: Version, content: Array, dbPath: String, cachePolicy: String): Searcher { + if (cachePolicy == "content") { + println("Using content buffer mode") + return Searcher(version, content, dbPath) + } else if (cachePolicy == "vectorIndex") { + println("Using vector index mode") + let vIndex = loadVectorIndex(content) + return Searcher(version, dbPath, vIndex) + } else { + println("Using file only mode") + return Searcher(version, dbPath) + } +} + +func searchIP(ipStr: String, v4Searcher: Searcher, v6Searcher: Searcher): String { + let ipBytes = parseIP(ipStr) + if (ipBytes.size == 4) { + return v4Searcher.search(ipBytes) + } else if (ipBytes.size == 16) { + return v6Searcher.search(ipBytes) + } + return "Invalid IP" +} + +main(args: Array): Int64 { + if (args.size == 0) { + printHelp() + return 0 + } + + let cmd = args[0] + if (cmd == "search") { + runSearch(args) + } else if (cmd == "bench") { + runBench(args) + } else { + printHelp() + } + + return 0 +} diff --git a/binding/cangjie/src/service/config.cj b/binding/cangjie/src/service/config.cj new file mode 100644 index 0000000..2241dcd --- /dev/null +++ b/binding/cangjie/src/service/config.cj @@ -0,0 +1,72 @@ +package ip2region.service + +import std.fs.* +import ip2region.xdb.* + +// Cache policy constants - match Searcher modes +public let FileOnly: Int64 = 0 +public let VectorIndex: Int64 = 1 +public let ContentBuff: Int64 = 2 + +// Config holds xdb file configuration and pre-loaded data for creating Searcher instances +public class Config { + public let cachePolicy: Int64 + public let ipVersion: Version + public let xdbPath: String + public let header: Header + public let vIndex: Array + public let cBuffer: Array + public let searchers: Int64 + + public init(cachePolicy: Int64, ipVersion: Version, xdbPath: String, searchers: Int64) { + if (searchers <= 0) { + throw Exception("searchers must be > 0") + } + + this.cachePolicy = cachePolicy + this.ipVersion = ipVersion + this.xdbPath = xdbPath + this.searchers = searchers + + let content = File.readFrom(Path(xdbPath)) + this.header = newHeaderFromBytes(content) + + // Verify IP version matches + let detectedVersion = versionFromHeader(this.header) + if (detectedVersion.id != ipVersion.id) { + throw Exception("xdb file IP version mismatch: expected ${ipVersion.name}, got ${detectedVersion.name}") + } + + if (cachePolicy == VectorIndex) { + this.vIndex = loadVectorIndex(content) + this.cBuffer = Array(0, repeat: 0) + } else if (cachePolicy == ContentBuff) { + this.vIndex = Array(0, repeat: 0) + this.cBuffer = content + } else { + this.vIndex = Array(0, repeat: 0) + this.cBuffer = Array(0, repeat: 0) + } + } + + public init(cachePolicy: Int64, ipVersion: Version, xdbPath: String) { + this(cachePolicy, ipVersion, xdbPath, 20) + } +} + +// Static factories for IPv4 and IPv6 +public func newV4Config(cachePolicy: Int64, xdbPath: String, searchers: Int64): Config { + return Config(cachePolicy, IPv4, xdbPath, searchers) +} + +public func newV6Config(cachePolicy: Int64, xdbPath: String, searchers: Int64): Config { + return Config(cachePolicy, IPv6, xdbPath, searchers) +} + +public func newV4Config(cachePolicy: Int64, xdbPath: String): Config { + return Config(cachePolicy, IPv4, xdbPath, 20) +} + +public func newV6Config(cachePolicy: Int64, xdbPath: String): Config { + return Config(cachePolicy, IPv6, xdbPath, 20) +} diff --git a/binding/cangjie/src/service/ip2region.cj b/binding/cangjie/src/service/ip2region.cj new file mode 100644 index 0000000..8a42099 --- /dev/null +++ b/binding/cangjie/src/service/ip2region.cj @@ -0,0 +1,161 @@ +package ip2region.service + +import ip2region.xdb.* + +// Ip2Region is the high-level API for searching IP addresses. +// It supports both IPv4 and IPv6 with thread-safe access via SearcherPool. +public class Ip2Region { + let v4Pool: SearcherPool + let v6Pool: SearcherPool + let v4InMemSearcher: Searcher + let v6InMemSearcher: Searcher + let hasV4: Bool + let hasV6: Bool + let v4IsInMem: Bool + let v6IsInMem: Bool + + // Create Ip2Region with both v4 and v6 configurations. + public init(v4Config: Config, v6Config: Config) { + // v4 setup + if (v4Config.cachePolicy == ContentBuff) { + this.hasV4 = true + this.v4IsInMem = true + this.v4InMemSearcher = Searcher(v4Config.ipVersion, v4Config.cBuffer, v4Config.xdbPath) + this.v4Pool = SearcherPool(v4Config) + } else { + this.hasV4 = true + this.v4IsInMem = false + this.v4InMemSearcher = Searcher(v4Config.ipVersion, Array(0, repeat: 0), v4Config.xdbPath) + this.v4Pool = SearcherPool(v4Config) + } + + // v6 setup + if (v6Config.cachePolicy == ContentBuff) { + this.hasV6 = true + this.v6IsInMem = true + this.v6InMemSearcher = Searcher(v6Config.ipVersion, v6Config.cBuffer, v6Config.xdbPath) + this.v6Pool = SearcherPool(v6Config) + } else { + this.hasV6 = true + this.v6IsInMem = false + this.v6InMemSearcher = Searcher(v6Config.ipVersion, Array(0, repeat: 0), v6Config.xdbPath) + this.v6Pool = SearcherPool(v6Config) + } + } + + // Single-version: provide Config and IP version. + // Examples: + // Ip2Region(v4Config, IPv4) — IPv4 only + // Ip2Region(v6Config, IPv6) — IPv6 only + public init(config: Config, ipVersion: Version) { + if (ipVersion.id == IPv4VersionNo) { + this.hasV4 = true + this.v4IsInMem = (config.cachePolicy == ContentBuff) + if (config.cachePolicy == ContentBuff) { + this.v4InMemSearcher = Searcher(config.ipVersion, config.cBuffer, config.xdbPath) + } else { + this.v4InMemSearcher = Searcher(config.ipVersion, Array(0, repeat: 0), config.xdbPath) + } + this.v4Pool = SearcherPool(config) + // IPv6 disabled + this.hasV6 = false + this.v6IsInMem = false + this.v6InMemSearcher = Searcher(IPv4, Array(0, repeat: 0), config.xdbPath) + this.v6Pool = SearcherPool(Config(ContentBuff, IPv4, config.xdbPath, 1)) + } else { + this.hasV6 = true + this.v6IsInMem = (config.cachePolicy == ContentBuff) + if (config.cachePolicy == ContentBuff) { + this.v6InMemSearcher = Searcher(config.ipVersion, config.cBuffer, config.xdbPath) + } else { + this.v6InMemSearcher = Searcher(config.ipVersion, Array(0, repeat: 0), config.xdbPath) + } + this.v6Pool = SearcherPool(config) + // IPv4 disabled + this.hasV4 = false + this.v4IsInMem = false + this.v4InMemSearcher = Searcher(IPv6, Array(0, repeat: 0), config.xdbPath) + this.v4Pool = SearcherPool(Config(ContentBuff, IPv6, config.xdbPath, 1)) + } + } + + // Search an IP string and return the region info. + public func search(ipStr: String): String { + try { + let ipBytes = parseIP(ipStr) + + if (ipBytes.size == 4) { + return this.searchV4(ipBytes) + } else if (ipBytes.size == 16) { + return this.searchV6(ipBytes) + } + } catch (_) { + return "Invalid IP" + } + return "Invalid IP" + } + + func searchV4(ip: Array): String { + if (!this.hasV4) { + return "" + } + if (this.v4IsInMem) { + return this.v4InMemSearcher.search(ip) + } + let searcher = this.v4Pool.borrow() + let region = searcher.search(ip) + this.v4Pool.return_(searcher) + return region + } + + func searchV6(ip: Array): String { + if (!this.hasV6) { + return "" + } + if (this.v6IsInMem) { + return this.v6InMemSearcher.search(ip) + } + let searcher = this.v6Pool.borrow() + let region = searcher.search(ip) + this.v6Pool.return_(searcher) + return region + } + + // close releases all resources. + public func close() { + if (this.hasV4) { + this.v4Pool.close() + } + if (this.hasV6) { + this.v6Pool.close() + } + } + + // closeTimeout releases all resources with the specified timeout. + public func closeTimeout(timeout: Duration) { + if (this.hasV4) { + this.v4Pool.closeTimeout(timeout) + } + if (this.hasV6) { + this.v6Pool.closeTimeout(timeout) + } + } +} + +// Create an Ip2Region instance using xdb file paths. +// Uses VectorIndex cache policy with 20 searchers by default. +public func newIp2Region(v4XdbPath: String, v6XdbPath: String): Ip2Region { + let v4Config = Config(VectorIndex, IPv4, v4XdbPath, 20) + let v6Config = Config(VectorIndex, IPv6, v6XdbPath, 20) + return Ip2Region(v4Config, v6Config) +} + +// Create an IPv4-only Ip2Region service. +public func newIp2RegionV4(config: Config): Ip2Region { + return Ip2Region(config, IPv4) +} + +// Create an IPv6-only Ip2Region service. +public func newIp2RegionV6(config: Config): Ip2Region { + return Ip2Region(config, IPv6) +} diff --git a/binding/cangjie/src/service/searcher_pool.cj b/binding/cangjie/src/service/searcher_pool.cj new file mode 100644 index 0000000..cfefaca --- /dev/null +++ b/binding/cangjie/src/service/searcher_pool.cj @@ -0,0 +1,111 @@ +package ip2region.service + +import std.sync.* +import std.core.* +import std.time.* +import std.collection.concurrent.* +import ip2region.xdb.* + +// SearcherPool is a thread-safe pool of xdb.Searcher instances. +// Uses Semaphore for backpressure and ConcurrentLinkedQueue for storage. +public class SearcherPool { + let config: Config + let queue: ConcurrentLinkedQueue + let semaphore: Semaphore + let closing: AtomicBool + let loanCnt: AtomicInt64 + + public init(config: Config) { + this.config = config + this.queue = ConcurrentLinkedQueue() + this.semaphore = Semaphore(config.searchers) + this.closing = AtomicBool(false) + this.loanCnt = AtomicInt64(0) + + let poolSize = config.searchers + for (_ in 0..poolSize) { + let searcher = createPoolSearcher(config) + this.queue.add(searcher) + this.semaphore.release(amount: 1) + } + } + + // borrow takes a Searcher from the pool, blocking until one is available. + // Throws if the pool is closing or closed. + public func borrow(): Searcher { + while (!this.closing.load()) { + if (this.semaphore.tryAcquire(amount: 1)) { + let opt = this.queue.remove() + if (opt.isNone()) { + this.semaphore.release(amount: 1) + continue + } + let s = opt.getOrThrow() + this.loanCnt.fetchAdd(1) + return s + } + sleep(Duration.millisecond * 5) + } + throw Exception("SearcherPool is closing") + } + + // return_ returns a Searcher to the pool, or closes it if the pool is shutting down. + public func return_(searcher: Searcher) { + if (this.closing.load()) { + searcher.close() + } else { + this.queue.add(searcher) + this.semaphore.release(amount: 1) + } + this.loanCnt.fetchSub(1) + } + + // close closes the pool with a default 10 second timeout. + public func close() { + this.closeTimeout(Duration.second * 10) + } + + // closeTimeout closes the pool with the specified timeout. + public func closeTimeout(timeout: Duration) { + this.closing.store(true) + + // Drain the queue: acquire all permits and close searchers + while (true) { + if (!this.semaphore.tryAcquire(amount: 1)) { + break + } + try { + let opt = this.queue.remove() + if (opt.isSome()) { + opt.getOrThrow().close() + } + } catch (_) { + break + } + } + + // Wait for outstanding loans to return (with timeout) + let deadline = MonoTime.now() + timeout + while (this.loanCnt.load() > 0) { + if (MonoTime.now() >= deadline) { + break + } + sleep(Duration.millisecond * 10) + } + } + + public func getLoanCount(): Int64 { + return this.loanCnt.load() + } +} + +// Helper to create a Searcher based on Config's cache policy +func createPoolSearcher(config: Config): Searcher { + if (config.cachePolicy == ContentBuff) { + return Searcher(config.ipVersion, config.cBuffer, config.xdbPath) + } else if (config.cachePolicy == VectorIndex) { + return Searcher(config.ipVersion, config.xdbPath, config.vIndex) + } else { + return Searcher(config.ipVersion, config.xdbPath) + } +} diff --git a/binding/cangjie/src/tests/service/config_test.cj b/binding/cangjie/src/tests/service/config_test.cj new file mode 100644 index 0000000..70fe1c0 --- /dev/null +++ b/binding/cangjie/src/tests/service/config_test.cj @@ -0,0 +1,50 @@ +package ip2region.tests.service + +import ip2region.xdb.* +import ip2region.service.* + +@Test +class ConfigV4Tests { + @TestCase + func testCreateFileOnly() { + let cfg = Config(FileOnly, IPv4, "../../data/ip2region_v4.xdb", 5) + if (cfg.cachePolicy != FileOnly) { throw Exception("wrong cache policy") } + if (cfg.searchers != 5) { throw Exception("wrong searchers count") } + if (cfg.ipVersion.id != IPv4VersionNo) { throw Exception("wrong version") } + } + + @TestCase + func testCreateVectorIndex() { + let cfg = Config(VectorIndex, IPv4, "../../data/ip2region_v4.xdb", 10) + if (cfg.vIndex.size == 0) { throw Exception("vIndex should be loaded") } + let expectedSize = VectorIndexRows * VectorIndexCols * VectorIndexSize + if (cfg.vIndex.size != expectedSize) { + throw Exception("vIndex size mismatch: ${cfg.vIndex.size} vs ${expectedSize}") + } + } + + @TestCase + func testCreateContentBuff() { + let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 3) + if (cfg.cBuffer.size == 0) { throw Exception("cBuffer should be loaded") } + } + + @TestCase + func testFactoryV4() { + let cfg = newV4Config(VectorIndex, "../../data/ip2region_v4.xdb", 8) + if (cfg.ipVersion.id != IPv4VersionNo) { throw Exception("should be IPv4") } + if (cfg.searchers != 8) { throw Exception("wrong searchers") } + } + + @TestCase + func testFactoryV6() { + let cfg = newV6Config(VectorIndex, "../../data/ip2region_v6.xdb", 8) + if (cfg.ipVersion.id != IPv6VersionNo) { throw Exception("should be IPv6") } + } + + @TestCase + func testFactoryDefaultSearchers() { + let cfg = newV4Config(ContentBuff, "../../data/ip2region_v4.xdb") + if (cfg.searchers != 20) { throw Exception("default searchers should be 20") } + } +} diff --git a/binding/cangjie/src/tests/service/ip2region_test.cj b/binding/cangjie/src/tests/service/ip2region_test.cj new file mode 100644 index 0000000..3f054da --- /dev/null +++ b/binding/cangjie/src/tests/service/ip2region_test.cj @@ -0,0 +1,108 @@ +package ip2region.tests.service + +import ip2region.xdb.* +import ip2region.service.* + +@Test +class Ip2RegionBasicTests { + @TestCase + func testSearchWithContentBuff() { + let v4Cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 1) + let v6Cfg = Config(ContentBuff, IPv6, "../../data/ip2region_v6.xdb", 1) + let region = Ip2Region(v4Cfg, v6Cfg) + let r1 = region.search("220.181.108.183") + if (r1.size == 0) { + throw Exception("v4 search should return result") + } + let r2 = region.search("2408:8266:100:1000::") + if (r2.size == 0) { + throw Exception("v6 search should return result") + } + region.close() + } + + @TestCase + func testSearchWithVectorIndex() { + let v4Cfg = Config(VectorIndex, IPv4, "../../data/ip2region_v4.xdb", 5) + let v6Cfg = Config(VectorIndex, IPv6, "../../data/ip2region_v6.xdb", 5) + let region = Ip2Region(v4Cfg, v6Cfg) + let r = region.search("220.181.108.183") + if (r.size == 0) { + throw Exception("v4 search should return result") + } + region.close() + } + + @TestCase + func testInvalidIP() { + let v4Cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 1) + let v6Cfg = Config(ContentBuff, IPv6, "../../data/ip2region_v6.xdb", 1) + let region = Ip2Region(v4Cfg, v6Cfg) + let r = region.search("invalid") + if (r != "Invalid IP") { + throw Exception("expected 'Invalid IP', got '${r}'") + } + region.close() + } + + @TestCase + func testClose() { + let v4Cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 1) + let v6Cfg = Config(ContentBuff, IPv6, "../../data/ip2region_v6.xdb", 1) + let region = Ip2Region(v4Cfg, v6Cfg) + region.search("220.181.108.183") + region.close() + } + + @TestCase + func testV4OnlyInit() { + let cfg = Config(VectorIndex, IPv4, "../../data/ip2region_v4.xdb", 3) + let region = Ip2Region(cfg, IPv4) + let r = region.search("220.181.108.183") + if (r.size == 0) { + throw Exception("v4-only region should find v4 IP") + } + let r6 = region.search("2408:8266:100:1000::") + if (r6.size != 0) { + throw Exception("v4-only region should not find v6 IP") + } + region.close() + } + + @TestCase + func testV6OnlyInit() { + let cfg = Config(VectorIndex, IPv6, "../../data/ip2region_v6.xdb", 3) + let region = Ip2Region(cfg, IPv6) + let r = region.search("2408:8266:100:1000::") + if (r.size == 0) { + throw Exception("v6-only region should find v6 IP") + } + let r4 = region.search("220.181.108.183") + if (r4.size != 0) { + throw Exception("v6-only region should not find v4 IP") + } + region.close() + } + + @TestCase + func testNewV4Factory() { + let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 2) + let region = newIp2RegionV4(cfg) + let r = region.search("220.181.108.183") + if (r.size == 0) { + throw Exception("newIp2RegionV4 should work") + } + region.close() + } + + @TestCase + func testNewV6Factory() { + let cfg = Config(ContentBuff, IPv6, "../../data/ip2region_v6.xdb", 2) + let region = newIp2RegionV6(cfg) + let r = region.search("2408:8266:100:1000::") + if (r.size == 0) { + throw Exception("newIp2RegionV6 should work") + } + region.close() + } +} diff --git a/binding/cangjie/src/tests/service/searcher_pool_test.cj b/binding/cangjie/src/tests/service/searcher_pool_test.cj new file mode 100644 index 0000000..e2340b6 --- /dev/null +++ b/binding/cangjie/src/tests/service/searcher_pool_test.cj @@ -0,0 +1,55 @@ +package ip2region.tests.service + +import ip2region.xdb.* +import ip2region.service.* + +@Test +class SearcherPoolTests { + @TestCase + func testBorrowAndReturn() { + let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 3) + let pool = SearcherPool(cfg) + let s = pool.borrow() + if (pool.getLoanCount() != 1) { + throw Exception("loan count should be 1, got ${pool.getLoanCount()}") + } + pool.return_(s) + if (pool.getLoanCount() != 0) { + throw Exception("loan count should be 0 after return") + } + pool.close() + } + + @TestCase + func testBorrowAll() { + let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 5) + let pool = SearcherPool(cfg) + let s1 = pool.borrow() + let s2 = pool.borrow() + let s3 = pool.borrow() + let s4 = pool.borrow() + let s5 = pool.borrow() + if (pool.getLoanCount() != 5) { + throw Exception("should have 5 loans, got ${pool.getLoanCount()}") + } + pool.return_(s5) + pool.return_(s4) + pool.return_(s3) + pool.return_(s2) + pool.return_(s1) + if (pool.getLoanCount() != 0) { + throw Exception("all should be returned, got ${pool.getLoanCount()}") + } + pool.close() + } + + @TestCase + func testClosePool() { + let cfg = Config(ContentBuff, IPv4, "../../data/ip2region_v4.xdb", 2) + let pool = SearcherPool(cfg) + let s = pool.borrow() + pool.return_(s) + pool.close() + pool.close() + } +} diff --git a/binding/cangjie/src/tests/tests.cj b/binding/cangjie/src/tests/tests.cj new file mode 100644 index 0000000..03ddf43 --- /dev/null +++ b/binding/cangjie/src/tests/tests.cj @@ -0,0 +1 @@ +package ip2region.tests diff --git a/binding/cangjie/src/tests/xdb/header_test.cj b/binding/cangjie/src/tests/xdb/header_test.cj new file mode 100644 index 0000000..f3a3f3a --- /dev/null +++ b/binding/cangjie/src/tests/xdb/header_test.cj @@ -0,0 +1,83 @@ +package ip2region.tests.xdb + +import ip2region.xdb.* + +@Test +class HeaderTests { + func makeHeaderBytes(): Array { + let buf = Array(256, repeat: 0) + var v: Byte + v = 3; buf[0] = v; v = 0; buf[1] = v + v = 1; buf[2] = v; v = 0; buf[3] = v + v = 0xA0; buf[4] = v; v = 0x86; buf[5] = v; v = 0x01; buf[6] = v; v = 0x00; buf[7] = v + v = 0xE8; buf[8] = v; v = 0x03; buf[9] = v; v = 0x00; buf[10] = v; v = 0x00; buf[11] = v + v = 0xD0; buf[12] = v; v = 0x07; buf[13] = v; v = 0x00; buf[14] = v; v = 0x00; buf[15] = v + v = 4; buf[16] = v; v = 0; buf[17] = v + v = 0; buf[18] = v; v = 0; buf[19] = v + return buf + } + + @TestCase + func testNewHeaderFromBytes() { + let buf = makeHeaderBytes() + let h = newHeaderFromBytes(buf) + if (h.version != UInt16(3)) { + throw Exception("expected version 3, got ${h.version}") + } + if (h.ipVersion != UInt16(4)) { + throw Exception("expected ipVersion 4, got ${h.ipVersion}") + } + } + + @TestCase + func testHeaderToString() { + let buf = makeHeaderBytes() + let h = newHeaderFromBytes(buf) + let s = h.toString() + if (s.size == 0) { + throw Exception("toString should not be empty") + } + } + + @TestCase + func testVersionFromHeaderV4() { + let buf = makeHeaderBytes() + let h = newHeaderFromBytes(buf) + let v = versionFromHeader(h) + if (v.id != IPv4VersionNo) { + throw Exception("expected IPv4, got ${v.name}") + } + } + + @TestCase + func testVersionFromHeaderV6() { + let buf = makeHeaderBytes() + var v: Byte = 6; buf[16] = v + let h = newHeaderFromBytes(buf) + let ver = versionFromHeader(h) + if (ver.id != IPv6VersionNo) { + throw Exception("expected IPv6, got ${ver.name}") + } + } + + @TestCase + func testVersionFromHeader20() { + let buf = makeHeaderBytes() + var v: Byte = 2; buf[0] = v + v = 0; buf[16] = v + let h = newHeaderFromBytes(buf) + let ver = versionFromHeader(h) + if (ver.id != IPv4VersionNo) { + throw Exception("expected IPv4 for v2.0") + } + } + + @TestCase + func testHeaderConstants() { + if (HeaderInfoLength != 256) { throw Exception("expected 256") } + if (VectorIndexRows != 256) { throw Exception("expected 256") } + if (VectorIndexSize != 8) { throw Exception("expected 8") } + if (IPv4SegmentIndexSize != 14) { throw Exception("expected 14") } + if (IPv6SegmentIndexSize != 38) { throw Exception("expected 38") } + } +} diff --git a/binding/cangjie/src/tests/xdb/searcher_test.cj b/binding/cangjie/src/tests/xdb/searcher_test.cj new file mode 100644 index 0000000..ae5246c --- /dev/null +++ b/binding/cangjie/src/tests/xdb/searcher_test.cj @@ -0,0 +1,132 @@ +package ip2region.tests.xdb + +import std.fs.* +import ip2region.xdb.* + +@Test +class SearcherV4Tests { + var v4Content: Array = Array(0, repeat: 0) + var initialized: Bool = false + + @BeforeAll + func setup() { + this.v4Content = File.readFrom(Path("../../data/ip2region_v4.xdb")) + this.initialized = true + } + + @TestCase + func testSearchContentMode() { + if (!this.initialized) { return } + let header = newHeaderFromBytes(this.v4Content) + let version = versionFromHeader(header) + let searcher = Searcher(version, this.v4Content, "../../data/ip2region_v4.xdb") + let ip = parseIP("220.181.108.183") + let region = searcher.search(ip) + if (region.size == 0) { + throw Exception("should find region for known IP") + } + searcher.close() + } + + @TestCase + func testSearchVectorIndexMode() { + if (!this.initialized) { return } + let header = newHeaderFromBytes(this.v4Content) + let version = versionFromHeader(header) + let vIndex = loadVectorIndex(this.v4Content) + let searcher = Searcher(version, "../../data/ip2region_v4.xdb", vIndex) + let ip = parseIP("220.181.108.183") + let region = searcher.search(ip) + if (region.size == 0) { + throw Exception("should find region for known IP") + } + searcher.close() + } + + @TestCase + func testSearchFileMode() { + if (!this.initialized) { return } + let header = newHeaderFromBytes(this.v4Content) + let version = versionFromHeader(header) + let searcher = Searcher(version, "../../data/ip2region_v4.xdb") + let ip = parseIP("220.181.108.183") + let region = searcher.search(ip) + if (region.size == 0) { + throw Exception("should find region for known IP") + } + searcher.close() + } + + @TestCase + func testAllModesSameResult() { + if (!this.initialized) { return } + let header = newHeaderFromBytes(this.v4Content) + let version = versionFromHeader(header) + let vIndex = loadVectorIndex(this.v4Content) + let s1 = Searcher(version, this.v4Content, "../../data/ip2region_v4.xdb") + let s2 = Searcher(version, "../../data/ip2region_v4.xdb", vIndex) + let s3 = Searcher(version, "../../data/ip2region_v4.xdb") + let ip = parseIP("1.2.3.4") + let r1 = s1.search(ip) + let r2 = s2.search(ip) + let r3 = s3.search(ip) + s1.close(); s2.close(); s3.close() + if (r1 != r2 || r2 != r3) { + throw Exception("all modes should return same result") + } + } + + @TestCase + func testSearchByString() { + if (!this.initialized) { return } + let header = newHeaderFromBytes(this.v4Content) + let version = versionFromHeader(header) + let searcher = Searcher(version, this.v4Content, "../../data/ip2region_v4.xdb") + let region = searcher.searchByString("220.181.108.183") + if (region.size == 0) { + throw Exception("searchByString should find region") + } + searcher.close() + } +} + +@Test +class SearcherV6Tests { + var v6Content: Array = Array(0, repeat: 0) + var initialized: Bool = false + + @BeforeAll + func setup() { + this.v6Content = File.readFrom(Path("../../data/ip2region_v6.xdb")) + this.initialized = true + } + + @TestCase + func testSearchV6ContentMode() { + if (!this.initialized) { return } + let header = newHeaderFromBytes(this.v6Content) + let version = versionFromHeader(header) + let searcher = Searcher(version, this.v6Content, "../../data/ip2region_v6.xdb") + let ip = parseIP("2408:8266:100:1000::") + let region = searcher.search(ip) + if (region.size == 0) { + throw Exception("should find region for known IPv6") + } + searcher.close() + } + + @TestCase + func testSearchV6VectorIndexMode() { + if (!this.initialized) { return } + let header = newHeaderFromBytes(this.v6Content) + let version = versionFromHeader(header) + let vIndex = loadVectorIndex(this.v6Content) + let searcher = Searcher(version, "../../data/ip2region_v6.xdb", vIndex) + let ip = parseIP("2408:8266:100:1000::") + let region = searcher.search(ip) + if (region.size == 0) { + throw Exception("should find region for known IPv6") + } + searcher.close() + } +} diff --git a/binding/cangjie/src/tests/xdb/util_test.cj b/binding/cangjie/src/tests/xdb/util_test.cj new file mode 100644 index 0000000..eecebf9 --- /dev/null +++ b/binding/cangjie/src/tests/xdb/util_test.cj @@ -0,0 +1,133 @@ +package ip2region.tests.xdb + +import ip2region.xdb.* + +@Test +class UtilTests { + @TestCase + func testParseIPv4() { + let ip = parseIP("1.2.3.4") + if (ip.size != 4) { + throw Exception("expected 4 bytes, got ${ip.size}") + } + if (Int64(ip[0]) != 1 || Int64(ip[1]) != 2 || Int64(ip[2]) != 3 || Int64(ip[3]) != 4) { + throw Exception("IPv4 parse mismatch") + } + } + + @TestCase + func testParseIPv6() { + let ip = parseIP("::1") + if (ip.size != 16) { + throw Exception("expected 16 bytes for IPv6, got ${ip.size}") + } + } + + @TestCase + func testIPCompareEqual() { + let a = Array(4, repeat: 0) + var v: Byte = 1; a[0] = v; v = 2; a[1] = v; v = 3; a[2] = v; v = 4; a[3] = v + let b = Array(4, repeat: 0) + v = 1; b[0] = v; v = 2; b[1] = v; v = 3; b[2] = v; v = 4; b[3] = v + let r = ipCompare(a, b) + if (r != 0) { + throw Exception("expected 0, got ${r}") + } + } + + @TestCase + func testIPCompareLess() { + let a = Array(4, repeat: 0) + var v: Byte = 1; a[0] = v; v = 2; a[1] = v; v = 3; a[2] = v; v = 4; a[3] = v + let b = Array(4, repeat: 0) + v = 1; b[0] = v; v = 2; b[1] = v; v = 3; b[2] = v; v = 5; b[3] = v + let r = ipCompare(a, b) + if (r != -1) { + throw Exception("expected -1, got ${r}") + } + } + + @TestCase + func testIPCompareGreater() { + let a = Array(4, repeat: 0) + var v: Byte = 1; a[0] = v; v = 2; a[1] = v; v = 3; a[2] = v; v = 5; a[3] = v + let b = Array(4, repeat: 0) + v = 1; b[0] = v; v = 2; b[1] = v; v = 3; b[2] = v; v = 4; b[3] = v + let r = ipCompare(a, b) + if (r != 1) { + throw Exception("expected 1, got ${r}") + } + } + + @TestCase + func testReadLEUint32() { + let buf = Array(4, repeat: 0) + var v: Byte = 0x78; buf[0] = v; v = 0x56; buf[1] = v; v = 0x34; buf[2] = v; v = 0x12; buf[3] = v + let val = readLEUint32(buf, 0) + let expected: Int64 = 0x12345678 + if (val != expected) { + throw Exception("expected 0x12345678, got ${val}") + } + } + + @TestCase + func testReadLEUint16() { + let buf = Array(2, repeat: 0) + var v: Byte = 0x34; buf[0] = v; v = 0x12; buf[1] = v + let val = readLEUint16(buf, 0) + if (val != 0x1234) { + throw Exception("expected 0x1234, got ${val}") + } + } + + @TestCase + func testIpToString() { + let ip = Array(4, repeat: 0) + var v: Byte = 127; ip[0] = v; v = 0; ip[1] = v; v = 0; ip[2] = v; v = 1; ip[3] = v + let s = ipToString(ip) + if (s != "127.0.0.1") { + throw Exception("expected 127.0.0.1, got ${s}") + } + } + + @TestCase + func testCompareIPv4() { + let ip = Array(4, repeat: 0) + var v: Byte = 1; ip[0] = v; v = 2; ip[1] = v; v = 3; ip[2] = v; v = 4; ip[3] = v + let entry = Array(8, repeat: 0) + v = 4; entry[0] = v; v = 3; entry[1] = v; v = 2; entry[2] = v; v = 1; entry[3] = v + v = 8; entry[4] = v; v = 7; entry[5] = v; v = 6; entry[6] = v; v = 5; entry[7] = v + let r = compareIPv4(ip, entry, 0) + if (r != 0) { + throw Exception("expected equal, got ${r}") + } + } + + @TestCase + func testCompareIPv6() { + let ip = Array(16, repeat: 0) + var v: Byte = 0x20; ip[0] = v; v = 0x01; ip[1] = v; v = 0x0d; ip[2] = v; v = 0xb8; ip[3] = v + let entry = Array(16, repeat: 0) + v = 0x20; entry[0] = v; v = 0x01; entry[1] = v; v = 0x0d; entry[2] = v; v = 0xb8; entry[3] = v + let r = compareIPv6(ip, entry, 0) + if (r != 0) { + throw Exception("expected equal, got ${r}") + } + } + + @TestCase + func testLoadVectorIndex() { + let viSize = VectorIndexRows * VectorIndexCols * VectorIndexSize + let cBuff = Array(HeaderInfoLength + viSize, repeat: 0) + var v: Byte = 0x42 + cBuff[HeaderInfoLength] = v + let vi = loadVectorIndex(cBuff) + if (vi.size != viSize) { + throw Exception("expected size ${viSize}, got ${vi.size}") + } + let firstVal = Int64(vi[0]) + if (firstVal != 0x42) { + throw Exception("expected first byte 0x42, got ${firstVal}") + } + } +} diff --git a/binding/cangjie/src/xdb/header.cj b/binding/cangjie/src/xdb/header.cj new file mode 100644 index 0000000..b324eed --- /dev/null +++ b/binding/cangjie/src/xdb/header.cj @@ -0,0 +1,77 @@ +package ip2region.xdb + +// xdb file format constants +public let HeaderInfoLength: Int64 = 256 +public let VectorIndexRows: Int64 = 256 +public let VectorIndexCols: Int64 = 256 +public let VectorIndexSize: Int64 = 8 +public let IPv4SegmentIndexSize: Int64 = 14 // 4 + 4 + 2 + 4 +public let IPv6SegmentIndexSize: Int64 = 38 // 16 + 16 + 2 + 4 + +public let Structure20: Int64 = 2 +public let Structure30: Int64 = 3 + +// xdb header parsed from the first 256 bytes of the xdb file +public class Header { + public let version: UInt16 + public let indexPolicy: UInt16 + public let createdAt: UInt32 + public let startIndexPtr: UInt32 + public let endIndexPtr: UInt32 + public let ipVersion: UInt16 + public let runtimePtrBytes: UInt16 + + public init( + ver: UInt16, + idxPolicy: UInt16, + createAt: UInt32, + sPtr: UInt32, + ePtr: UInt32, + ipVer: UInt16, + rtPtrBytes: UInt16 + ) { + this.version = ver + this.indexPolicy = idxPolicy + this.createdAt = createAt + this.startIndexPtr = sPtr + this.endIndexPtr = ePtr + this.ipVersion = ipVer + this.runtimePtrBytes = rtPtrBytes + } + + public func toString(): String { + return "{version: ${this.version}, index_policy: ${this.indexPolicy}, " + + "created_at: ${this.createdAt}, start_index_ptr: ${this.startIndexPtr}, " + + "end_index_ptr: ${this.endIndexPtr}, ip_version: ${this.ipVersion}, " + + "runtime_ptr_bytes: ${this.runtimePtrBytes}}" + } +} + +// Read a little-endian UInt16 from byte array at offset +func readHeaderUInt16(buf: Array, offset: Int64): UInt16 { + let b0 = UInt16(buf[offset]) + let b1 = UInt16(buf[offset + 1]) << 8 + return b0 | b1 +} + +// Read a little-endian UInt32 from byte array at offset +func readHeaderUInt32(buf: Array, offset: Int64): UInt32 { + let b0 = UInt32(buf[offset]) + let b1 = UInt32(buf[offset + 1]) << 8 + let b2 = UInt32(buf[offset + 2]) << 16 + let b3 = UInt32(buf[offset + 3]) << 24 + return b0 | b1 | b2 | b3 +} + +// Create a Header from raw byte buffer (first 256 bytes of xdb file) +public func newHeaderFromBytes(buf: Array): Header { + return Header( + readHeaderUInt16(buf, 0), + readHeaderUInt16(buf, 2), + readHeaderUInt32(buf, 4), + readHeaderUInt32(buf, 8), + readHeaderUInt32(buf, 12), + readHeaderUInt16(buf, 16), + readHeaderUInt16(buf, 18) + ) +} diff --git a/binding/cangjie/src/xdb/searcher.cj b/binding/cangjie/src/xdb/searcher.cj new file mode 100644 index 0000000..050a829 --- /dev/null +++ b/binding/cangjie/src/xdb/searcher.cj @@ -0,0 +1,146 @@ +package ip2region.xdb + +import std.fs.* +import std.io.SeekPosition + +// Searcher for ip2region xdb database +public class Searcher { + let version: Version + let handle: File + var ioCount: Int64 + let vectorIndex: Array + let contentBuff: Array + let mode: Int64 + + public let FileOnlyMode: Int64 = 0 + public let VectorIndexMode: Int64 = 1 + public let ContentBuffMode: Int64 = 2 + + public init(ver: Version, dbFile: String) { + this.version = ver + this.handle = File(dbFile, Read) + this.vectorIndex = Array(0, repeat: 0) + this.contentBuff = Array(0, repeat: 0) + this.mode = FileOnlyMode + this.ioCount = 0 + } + + public init(ver: Version, dbFile: String, vIndex: Array) { + this.version = ver + this.handle = File(dbFile, Read) + this.vectorIndex = vIndex + this.contentBuff = Array(0, repeat: 0) + this.mode = VectorIndexMode + this.ioCount = 0 + } + + public init(ver: Version, cBuff: Array, dbPath: String) { + this.version = ver + this.handle = File(dbPath, Read) + this.vectorIndex = Array(0, repeat: 0) + this.contentBuff = cBuff + this.mode = ContentBuffMode + this.ioCount = 0 + } + + public func close() { + this.handle.close() + } + + public func getIOCount(): Int64 { + return this.ioCount + } + + public func searchByString(ipStr: String): String { + let ipBytes = parseIP(ipStr) + return this.search(ipBytes) + } + + public func search(ip: Array): String { + this.ioCount = 0 + + let il0 = Int64(ip[0]) + let il1 = Int64(ip[1]) + let idx = il0 * VectorIndexCols * VectorIndexSize + il1 * VectorIndexSize + + var sPtr: Int64 = 0 + var ePtr: Int64 = 0 + + if (this.mode == VectorIndexMode) { + sPtr = readLEUint32(this.vectorIndex, idx) + ePtr = readLEUint32(this.vectorIndex, idx + 4) + } else if (this.mode == ContentBuffMode) { + sPtr = readLEUint32(this.contentBuff, HeaderInfoLength + idx) + ePtr = readLEUint32(this.contentBuff, HeaderInfoLength + idx + 4) + } else { + let buff = Array(VectorIndexSize, repeat: 0) + this.readFromFile(HeaderInfoLength + idx, buff) + sPtr = readLEUint32(buff, 0) + ePtr = readLEUint32(buff, 4) + } + + if (sPtr == 0 || ePtr == 0) { + return "" + } + + let segIndexSize = this.version.segmentIndexSize + let bytes = this.version.bytes + let dBytes = bytes * 2 + var dataLen: Int64 = 0 + var dataPtr: Int64 = 0 + + var l: Int64 = 0 + var h: Int64 = (ePtr - sPtr) / segIndexSize + let buff = Array(segIndexSize, repeat: 0) + + while (l <= h) { + let m = (l + h) / 2 + let p = sPtr + m * segIndexSize + + this.read(p, buff) + this.ioCount = this.ioCount + 1 + + if (this.compareIP(ip, buff, 0) < 0) { + h = m - 1 + } else if (this.compareIP(ip, buff, bytes) > 0) { + l = m + 1 + } else { + dataLen = readLEUint16(buff, dBytes) + dataPtr = readLEUint32(buff, dBytes + 2) + break + } + } + + if (dataLen == 0) { + return "" + } + + let regionBuff = Array(dataLen, repeat: 0) + this.read(dataPtr, regionBuff) + return String.fromUtf8(regionBuff) + } + + func read(offset: Int64, buff: Array) { + if (this.mode == ContentBuffMode) { + for (i in 0..buff.size) { + buff[i] = this.contentBuff[offset + i] + } + } else { + this.readFromFile(offset, buff) + () + } + } + + func readFromFile(offset: Int64, buff: Array) { + this.handle.seek(SeekPosition.Begin(offset)) + this.handle.read(buff) + () + } + + func compareIP(ip: Array, entry: Array, offset: Int64): Int64 { + if (this.version.id == IPv4VersionNo) { + return compareIPv4(ip, entry, offset) + } + return compareIPv6(ip, entry, offset) + } +} diff --git a/binding/cangjie/src/xdb/util.cj b/binding/cangjie/src/xdb/util.cj new file mode 100644 index 0000000..212d2d1 --- /dev/null +++ b/binding/cangjie/src/xdb/util.cj @@ -0,0 +1,108 @@ +package ip2region.xdb + +import std.net.IPAddress + +// --- IP address parsing and formatting + +// Parse IP string to byte array +// For IPv4, returns 4 bytes in big-endian order (network byte order) +// For IPv6, returns 16 bytes in big-endian order +public func parseIP(ip: String): Array { + let addr = IPAddress.parse(ip) + return addr.getAddressBytes() +} + +// Convert IP byte array back to string +public func ipToString(ip: Array): String { + if (ip.size == 4) { + return "${Int64(ip[0])}.${Int64(ip[1])}.${Int64(ip[2])}.${Int64(ip[3])}" + } + // IPv6 simplified format + var result = "" + for (i in 0..ip.size) { + if (i > 0 && i % 2 == 0) { + result = result + ":" + } + result = result + "${Int64(ip[i])}" + } + return result +} + +// Compare two IP byte arrays. +// Returns: -1 if ip1 < ip2, 0 if equal, 1 if ip1 > ip2 +public func ipCompare(ip1: Array, ip2: Array): Int64 { + var len = ip1.size + if (ip2.size < len) { + len = ip2.size + } + for (i in 0..len) { + if (ip1[i] < ip2[i]) { + return -1 + } else if (ip1[i] > ip2[i]) { + return 1 + } + } + if (ip1.size < ip2.size) { + return -1 + } else if (ip1.size > ip2.size) { + return 1 + } + return 0 +} + +// --- Binary data reading helpers for xdb format + +// Read a little-endian UInt32 from a byte buffer at the given offset +public func readLEUint32(buf: Array, offset: Int64): Int64 { + let b0 = Int64(buf[offset]) + let b1 = Int64(buf[offset + 1]) << 8 + let b2 = Int64(buf[offset + 2]) << 16 + let b3 = Int64(buf[offset + 3]) << 24 + return b0 | b1 | b2 | b3 +} + +// Read a little-endian UInt16 from a byte buffer at the given offset +public func readLEUint16(buf: Array, offset: Int64): Int64 { + let b0 = Int64(buf[offset]) + let b1 = Int64(buf[offset + 1]) << 8 + return b0 | b1 +} + +// Compare IPv4 address (big-endian) with xdb stored IPv4 (little-endian) +// xdb stores IPv4 in little-endian, so we compare ip[i] with xdbEntry[3-i] +public func compareIPv4(ip: Array, xdbEntry: Array, entryOffset: Int64): Int64 { + for (i in 0..4) { + let ipByte = Int64(ip[i]) + let xdbByte = Int64(xdbEntry[entryOffset + 3 - i]) + if (ipByte < xdbByte) { + return -1 + } else if (ipByte > xdbByte) { + return 1 + } + } + return 0 +} + +// Load the vector index from a content buffer +// The vector index starts at HeaderInfoLength and has VectorIndexRows * VectorIndexCols * VectorIndexSize bytes +public func loadVectorIndex(cBuff: Array): Array { + let start = HeaderInfoLength + let len = VectorIndexRows * VectorIndexCols * VectorIndexSize + let vi = Array(len, repeat: 0) + for (i in 0..len) { + vi[i] = cBuff[start + i] + } + return vi +} + +// Compare IPv6 address (big-endian) with xdb stored IPv6 (big-endian) +public func compareIPv6(ip: Array, xdbEntry: Array, entryOffset: Int64): Int64 { + for (i in 0..16) { + if (ip[i] < xdbEntry[entryOffset + i]) { + return -1 + } else if (ip[i] > xdbEntry[entryOffset + i]) { + return 1 + } + } + return 0 +} diff --git a/binding/cangjie/src/xdb/version.cj b/binding/cangjie/src/xdb/version.cj new file mode 100644 index 0000000..d8bcf11 --- /dev/null +++ b/binding/cangjie/src/xdb/version.cj @@ -0,0 +1,41 @@ +package ip2region.xdb + +// IP version definitions for ip2region xdb searcher +public class Version { + public let id: Int64 + public let name: String + public let bytes: Int64 + public let segmentIndexSize: Int64 + + public init(id: Int64, name: String, bytes: Int64, segmentIndexSize: Int64) { + this.id = id + this.name = name + this.bytes = bytes + this.segmentIndexSize = segmentIndexSize + } + + public func toString(): String { + return "{id: ${this.id}, name: ${this.name}, bytes: ${this.bytes}, " + + "segment_index_size: ${this.segmentIndexSize}}" + } +} + +public let IPv4VersionNo: Int64 = 4 +public let IPv6VersionNo: Int64 = 6 + +// Pre-defined IP versions +public let IPv4 = Version(4, "IPv4", 4, 14) +public let IPv6 = Version(6, "IPv6", 16, 38) + +// Determine IP version from header info +public func versionFromHeader(header: Header): Version { + // Old structure (2.0) with IPv4 only + if (Int64(header.version) == Structure20) { + return IPv4 + } + // Structure 3.0+ + if (Int64(header.ipVersion) == IPv4VersionNo) { + return IPv4 + } + return IPv6 +} -- Gitee