feat: 新增数据库持久化模块(Persistence),实现 Cursor + Retry 双层架构
## 核心功能 ### 1. 数据库持久化支持 - 新增完整的 Persistence 模块 (api/persistence/) - 支持三种持久化策略: * StrategyDBOnly - 仅落库,不存证 * StrategyDBAndTrustlog - 既落库又存证(推荐) * StrategyTrustlogOnly - 仅存证,不落库 - 支持多数据库:PostgreSQL, MySQL, SQLite ### 2. Cursor + Retry 双层架构 - CursorWorker:第一道防线,快速发现新记录并尝试存证 * 增量扫描 operation 表(基于时间戳游标) * 默认 10 秒扫描间隔,批量处理 100 条 * 成功更新状态,失败转入重试队列 - RetryWorker:第二道防线,处理失败记录 * 指数退避重试(1m → 2m → 4m → 8m → 16m) * 默认最多重试 5 次 * 超限自动标记为死信 ### 3. 数据库表设计 - operation 表:存储操作记录,支持可空 IP 字段 - trustlog_cursor 表:Key-Value 模式,支持多游标 - trustlog_retry 表:重试队列,支持指数退避 ### 4. 异步最终一致性 - 应用调用立即返回(仅落库) - CursorWorker 异步扫描并存证 - RetryWorker 保障失败重试 - 完整的监控和死信处理机制 ## 修改文件 ### 核心代码(11个文件) - api/persistence/cursor_worker.go - Cursor 工作器(新增) - api/persistence/repository.go - 数据仓储层(新增) - api/persistence/schema.go - 数据库 Schema(新增) - api/persistence/strategy.go - 策略管理器(新增) - api/persistence/client.go - 客户端封装(新增) - api/persistence/retry_worker.go - Retry 工作器(新增) - api/persistence/config.go - 配置管理(新增) ### 修复内部包引用(5个文件) - api/adapter/publisher.go - 修复 internal 包引用 - api/adapter/subscriber.go - 修复 internal 包引用 - api/model/envelope.go - 修复 internal 包引用 - api/model/operation.go - 修复 internal 包引用 - api/model/record.go - 修复 internal 包引用 ### 单元测试(8个文件) - api/persistence/*_test.go - 完整的单元测试 - 测试覆盖率:28.5% - 测试通过率:49/49 (100%) ### SQL 脚本(4个文件) - api/persistence/sql/postgresql.sql - PostgreSQL 建表脚本 - api/persistence/sql/mysql.sql - MySQL 建表脚本 - api/persistence/sql/sqlite.sql - SQLite 建表脚本 - api/persistence/sql/test_data.sql - 测试数据 ### 文档(2个文件) - README.md - 更新主文档,新增 Persistence 使用指南 - api/persistence/README.md - 完整的 Persistence 文档 - api/persistence/sql/README.md - SQL 脚本说明 ## 技术亮点 1. **充分利用 Cursor 游标表** - 作为任务发现队列,非简单的位置记录 - Key-Value 模式,支持多游标并发扫描 - 时间戳天然有序,增量扫描高效 2. **双层保障机制** - Cursor:正常流程,快速处理 - Retry:异常流程,可靠重试 - 职责分离,监控清晰 3. **可空 IP 字段支持** - ClientIP 和 ServerIP 使用 *string 类型 - 支持 NULL 值,符合数据库最佳实践 - 使用 sql.NullString 正确处理 4. **完整的监控支持** - 未存证记录数监控 - Cursor 延迟监控 - 重试队列长度监控 - 死信队列监控 ## 测试结果 - ✅ 单元测试:49/49 通过 (100%) - ✅ 代码覆盖率:28.5% - ✅ 编译状态:无错误 - ✅ 支持数据库:PostgreSQL, MySQL, SQLite ## Breaking Changes 无破坏性变更。Persistence 模块作为可选功能,不影响现有代码。 ## 版本信息 - 版本:v2.1.0 - Go 版本要求:1.21+ - 更新日期:2025-12-23
This commit is contained in:
203
api/persistence/sql/test_data.sql
Normal file
203
api/persistence/sql/test_data.sql
Normal file
@@ -0,0 +1,203 @@
|
||||
-- 测试数据插入脚本
|
||||
-- 用于验证数据库表结构和功能
|
||||
|
||||
-- ============================================
|
||||
-- 1. 插入测试操作记录
|
||||
-- ============================================
|
||||
|
||||
-- 测试1: 插入包含 IP 信息的操作记录
|
||||
INSERT INTO operation (
|
||||
op_id, op_actor, doid, producer_id,
|
||||
request_body_hash, response_body_hash,
|
||||
op_source, op_type, do_prefix, do_repository,
|
||||
client_ip, server_ip, trustlog_status, timestamp
|
||||
) VALUES (
|
||||
'test-op-001',
|
||||
'test-user',
|
||||
'10.1000/test-repo/doc001',
|
||||
'producer-001',
|
||||
'req_hash_001',
|
||||
'resp_hash_001',
|
||||
'DOIP',
|
||||
'Create',
|
||||
'10.1000',
|
||||
'test-repo',
|
||||
'192.168.1.100', -- 客户端IP
|
||||
'10.0.0.50', -- 服务端IP
|
||||
'TRUSTLOGGED', -- 已存证
|
||||
CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- 测试2: 插入 IP 为 NULL 的操作记录
|
||||
INSERT INTO operation (
|
||||
op_id, op_actor, doid, producer_id,
|
||||
request_body_hash, response_body_hash,
|
||||
op_source, op_type, do_prefix, do_repository,
|
||||
client_ip, server_ip, trustlog_status, timestamp
|
||||
) VALUES (
|
||||
'test-op-002',
|
||||
'test-user',
|
||||
'10.1000/test-repo/doc002',
|
||||
'producer-001',
|
||||
'req_hash_002',
|
||||
'resp_hash_002',
|
||||
'DOIP',
|
||||
'Update',
|
||||
'10.1000',
|
||||
'test-repo',
|
||||
NULL, -- IP 为 NULL
|
||||
NULL, -- IP 为 NULL
|
||||
'NOT_TRUSTLOGGED', -- 未存证
|
||||
CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- 测试3: 插入只有客户端 IP 的记录
|
||||
INSERT INTO operation (
|
||||
op_id, op_actor, doid, producer_id,
|
||||
request_body_hash, response_body_hash,
|
||||
op_source, op_type, do_prefix, do_repository,
|
||||
client_ip, server_ip, trustlog_status, timestamp
|
||||
) VALUES (
|
||||
'test-op-003',
|
||||
'test-user',
|
||||
'10.1000/test-repo/doc003',
|
||||
'producer-001',
|
||||
'req_hash_003',
|
||||
'resp_hash_003',
|
||||
'IRP',
|
||||
'Delete',
|
||||
'10.1000',
|
||||
'test-repo',
|
||||
'172.16.0.100', -- 仅客户端IP
|
||||
NULL, -- 服务端IP为NULL
|
||||
'NOT_TRUSTLOGGED',
|
||||
CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- ============================================
|
||||
-- 2. 插入测试重试记录
|
||||
-- ============================================
|
||||
|
||||
-- 测试1: 待重试记录
|
||||
INSERT INTO trustlog_retry (
|
||||
op_id, retry_count, retry_status,
|
||||
last_retry_at, next_retry_at, error_message
|
||||
) VALUES (
|
||||
'test-op-002',
|
||||
0,
|
||||
'PENDING',
|
||||
NULL,
|
||||
CURRENT_TIMESTAMP, -- 立即重试
|
||||
'Initial retry'
|
||||
);
|
||||
|
||||
-- 测试2: 重试中记录
|
||||
INSERT INTO trustlog_retry (
|
||||
op_id, retry_count, retry_status,
|
||||
last_retry_at, next_retry_at, error_message
|
||||
) VALUES (
|
||||
'test-op-003',
|
||||
2,
|
||||
'RETRYING',
|
||||
CURRENT_TIMESTAMP,
|
||||
CURRENT_TIMESTAMP, -- 下次重试时间
|
||||
'Connection timeout'
|
||||
);
|
||||
|
||||
-- ============================================
|
||||
-- 3. 验证查询
|
||||
-- ============================================
|
||||
|
||||
-- 查询所有操作记录
|
||||
SELECT
|
||||
op_id,
|
||||
op_type,
|
||||
client_ip,
|
||||
server_ip,
|
||||
trustlog_status,
|
||||
timestamp
|
||||
FROM operation
|
||||
ORDER BY timestamp DESC;
|
||||
|
||||
-- 查询包含 IP 信息的记录
|
||||
SELECT
|
||||
op_id,
|
||||
client_ip,
|
||||
server_ip,
|
||||
trustlog_status
|
||||
FROM operation
|
||||
WHERE client_ip IS NOT NULL OR server_ip IS NOT NULL;
|
||||
|
||||
-- 查询未存证的记录
|
||||
SELECT
|
||||
op_id,
|
||||
doid,
|
||||
trustlog_status,
|
||||
timestamp
|
||||
FROM operation
|
||||
WHERE trustlog_status = 'NOT_TRUSTLOGGED'
|
||||
ORDER BY timestamp ASC;
|
||||
|
||||
-- 查询重试记录
|
||||
SELECT
|
||||
r.op_id,
|
||||
r.retry_count,
|
||||
r.retry_status,
|
||||
r.error_message,
|
||||
o.doid
|
||||
FROM trustlog_retry r
|
||||
JOIN operation o ON r.op_id = o.op_id
|
||||
ORDER BY r.next_retry_at ASC;
|
||||
|
||||
-- 查询游标状态
|
||||
SELECT * FROM trustlog_cursor WHERE id = 1;
|
||||
|
||||
-- ============================================
|
||||
-- 4. 统计查询
|
||||
-- ============================================
|
||||
|
||||
-- 统计各状态的记录数
|
||||
SELECT
|
||||
trustlog_status,
|
||||
COUNT(*) as count
|
||||
FROM operation
|
||||
GROUP BY trustlog_status;
|
||||
|
||||
-- 统计 IP 字段使用情况
|
||||
SELECT
|
||||
CASE
|
||||
WHEN client_ip IS NOT NULL THEN 'Has Client IP'
|
||||
ELSE 'No Client IP'
|
||||
END as client_ip_status,
|
||||
CASE
|
||||
WHEN server_ip IS NOT NULL THEN 'Has Server IP'
|
||||
ELSE 'No Server IP'
|
||||
END as server_ip_status,
|
||||
COUNT(*) as count
|
||||
FROM operation
|
||||
GROUP BY
|
||||
CASE WHEN client_ip IS NOT NULL THEN 'Has Client IP' ELSE 'No Client IP' END,
|
||||
CASE WHEN server_ip IS NOT NULL THEN 'Has Server IP' ELSE 'No Server IP' END;
|
||||
|
||||
-- 统计重试状态
|
||||
SELECT
|
||||
retry_status,
|
||||
COUNT(*) as count,
|
||||
AVG(retry_count) as avg_retry_count
|
||||
FROM trustlog_retry
|
||||
GROUP BY retry_status;
|
||||
|
||||
-- ============================================
|
||||
-- 5. 清理测试数据
|
||||
-- ============================================
|
||||
|
||||
-- 取消注释以下语句来清理测试数据
|
||||
/*
|
||||
DELETE FROM trustlog_retry WHERE op_id LIKE 'test-op-%';
|
||||
DELETE FROM operation WHERE op_id LIKE 'test-op-%';
|
||||
UPDATE trustlog_cursor SET
|
||||
last_processed_id = NULL,
|
||||
last_processed_at = NULL
|
||||
WHERE id = 1;
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user