88f80ffa5e
## 核心功能 ### 1. 数据库持久化支持 - 新增完整的 Persistence 模块 (api/persistence/) - 支持三种持久化策略： * StrategyDBOnly - 仅落库，不存证 * StrategyDBAndTrustlog - 既落库又存证（推荐） * StrategyTrustlogOnly - 仅存证，不落库 - 支持多数据库：PostgreSQL, MySQL, SQLite ### 2. Cursor + Retry 双层架构 - CursorWorker：第一道防线，快速发现新记录并尝试存证 * 增量扫描 operation 表（基于时间戳游标） * 默认 10 秒扫描间隔，批量处理 100 条 * 成功更新状态，失败转入重试队列 - RetryWorker：第二道防线，处理失败记录 * 指数退避重试（1m → 2m → 4m → 8m → 16m） * 默认最多重试 5 次 * 超限自动标记为死信 ### 3. 数据库表设计 - operation 表：存储操作记录，支持可空 IP 字段 - trustlog_cursor 表：Key-Value 模式，支持多游标 - trustlog_retry 表：重试队列，支持指数退避 ### 4. 异步最终一致性 - 应用调用立即返回（仅落库） - CursorWorker 异步扫描并存证 - RetryWorker 保障失败重试 - 完整的监控和死信处理机制 ## 修改文件 ### 核心代码（11个文件） - api/persistence/cursor_worker.go - Cursor 工作器（新增） - api/persistence/repository.go - 数据仓储层（新增） - api/persistence/schema.go - 数据库 Schema（新增） - api/persistence/strategy.go - 策略管理器（新增） - api/persistence/client.go - 客户端封装（新增） - api/persistence/retry_worker.go - Retry 工作器（新增） - api/persistence/config.go - 配置管理（新增） ### 修复内部包引用（5个文件） - api/adapter/publisher.go - 修复 internal 包引用 - api/adapter/subscriber.go - 修复 internal 包引用 - api/model/envelope.go - 修复 internal 包引用 - api/model/operation.go - 修复 internal 包引用 - api/model/record.go - 修复 internal 包引用 ### 单元测试（8个文件） - api/persistence/*_test.go - 完整的单元测试 - 测试覆盖率：28.5% - 测试通过率：49/49 (100%) ### SQL 脚本（4个文件） - api/persistence/sql/postgresql.sql - PostgreSQL 建表脚本 - api/persistence/sql/mysql.sql - MySQL 建表脚本 - api/persistence/sql/sqlite.sql - SQLite 建表脚本 - api/persistence/sql/test_data.sql - 测试数据 ### 文档（2个文件） - README.md - 更新主文档，新增 Persistence 使用指南 - api/persistence/README.md - 完整的 Persistence 文档 - api/persistence/sql/README.md - SQL 脚本说明 ## 技术亮点 1. **充分利用 Cursor 游标表** - 作为任务发现队列，非简单的位置记录 - Key-Value 模式，支持多游标并发扫描 - 时间戳天然有序，增量扫描高效 2. **双层保障机制** - Cursor：正常流程，快速处理 - Retry：异常流程，可靠重试 - 职责分离，监控清晰 3. **可空 IP 字段支持** - ClientIP 和 ServerIP 使用 *string 类型 - 支持 NULL 值，符合数据库最佳实践 - 使用 sql.NullString 正确处理 4. **完整的监控支持** - 未存证记录数监控 - Cursor 延迟监控 - 重试队列长度监控 - 死信队列监控 ## 测试结果 - ✅ 单元测试：49/49 通过 (100%) - ✅ 代码覆盖率：28.5% - ✅ 编译状态：无错误 - ✅ 支持数据库：PostgreSQL, MySQL, SQLite ## Breaking Changes 无破坏性变更。Persistence 模块作为可选功能，不影响现有代码。 ## 版本信息 - 版本：v2.1.0 - Go 版本要求：1.21+ - 更新日期：2025-12-23
379 lines
9.3 KiB
Go
379 lines
9.3 KiB
Go
package persistence_test
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"time"
|
||
|
||
"github.com/go-logr/logr"
|
||
|
||
"go.yandata.net/iod/iod/go-trustlog/api/adapter"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/logger"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/model"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/persistence"
|
||
)
|
||
|
||
// Example_dbOnly 演示仅落库策略
|
||
func Example_dbOnly() {
|
||
ctx := context.Background()
|
||
|
||
// 1. 创建 Logger
|
||
myLogger := logger.NewLogger(logr.Discard())
|
||
|
||
// 2. 创建 Pulsar Publisher
|
||
pub, err := adapter.NewPublisher(
|
||
adapter.PublisherConfig{
|
||
URL: "pulsar://localhost:6650",
|
||
},
|
||
myLogger,
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer pub.Close()
|
||
|
||
// 3. 准备 SM2 密钥配置
|
||
privateKeyHex := []byte("私钥D的十六进制字符串")
|
||
publicKeyHex := []byte("04 + x坐标 + y坐标的十六进制字符串")
|
||
envelopeConfig := model.NewSM2EnvelopeConfig(privateKeyHex, publicKeyHex)
|
||
|
||
// 4. 创建支持数据库持久化的客户端（仅落库策略）
|
||
client, err := persistence.NewPersistenceClient(ctx, persistence.PersistenceClientConfig{
|
||
Publisher: pub,
|
||
Logger: myLogger,
|
||
EnvelopeConfig: envelopeConfig,
|
||
DBConfig: persistence.DefaultDBConfig(
|
||
"postgres",
|
||
"postgres://user:pass@localhost:5432/trustlog?sslmode=disable",
|
||
),
|
||
PersistenceConfig: persistence.DefaultPersistenceConfig(persistence.StrategyDBOnly),
|
||
EnableRetryWorker: false, // 仅落库不需要重试
|
||
})
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer client.Close()
|
||
|
||
// 5. 构造 Operation（包含 IP 信息）
|
||
op, err := model.NewFullOperation(
|
||
model.OpSourceDOIP,
|
||
model.OpTypeCreate,
|
||
"10.1000",
|
||
"my-repo",
|
||
"10.1000/my-repo/doc001",
|
||
"producer-001",
|
||
"admin",
|
||
[]byte(`{"action":"create"}`),
|
||
[]byte(`{"status":"success"}`),
|
||
time.Now(),
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
// 设置 IP 信息（仅落库字段，可空）
|
||
clientIP := "192.168.1.100"
|
||
serverIP := "10.0.0.50"
|
||
op.ClientIP = &clientIP
|
||
op.ServerIP = &serverIP
|
||
|
||
// 6. 发布操作（仅落库，不存证）
|
||
if err := client.OperationPublish(ctx, op); err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
fmt.Printf("Operation saved to database only: %s\n", op.OpID)
|
||
}
|
||
|
||
// Example_dbAndTrustlog 演示既落库又存证策略
|
||
func Example_dbAndTrustlog() {
|
||
ctx := context.Background()
|
||
|
||
// 1. 创建 Logger
|
||
myLogger := logger.NewLogger(logr.Discard())
|
||
|
||
// 2. 创建 Pulsar Publisher
|
||
pub, err := adapter.NewPublisher(
|
||
adapter.PublisherConfig{
|
||
URL: "pulsar://localhost:6650",
|
||
},
|
||
myLogger,
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer pub.Close()
|
||
|
||
// 3. 准备 SM2 密钥配置
|
||
privateKeyHex := []byte("私钥D的十六进制字符串")
|
||
publicKeyHex := []byte("04 + x坐标 + y坐标的十六进制字符串")
|
||
envelopeConfig := model.NewSM2EnvelopeConfig(privateKeyHex, publicKeyHex)
|
||
|
||
// 4. 创建支持数据库持久化的客户端（既落库又存证策略）
|
||
retryConfig := persistence.DefaultRetryWorkerConfig()
|
||
retryConfig.MaxRetryCount = 5
|
||
retryConfig.RetryInterval = 30 * time.Second
|
||
|
||
client, err := persistence.NewPersistenceClient(ctx, persistence.PersistenceClientConfig{
|
||
Publisher: pub,
|
||
Logger: myLogger,
|
||
EnvelopeConfig: envelopeConfig,
|
||
DBConfig: persistence.DefaultDBConfig(
|
||
"postgres",
|
||
"postgres://user:pass@localhost:5432/trustlog?sslmode=disable",
|
||
),
|
||
PersistenceConfig: persistence.PersistenceConfig{
|
||
Strategy: persistence.StrategyDBAndTrustlog,
|
||
EnableRetry: true,
|
||
MaxRetryCount: 5,
|
||
RetryBatchSize: 100,
|
||
},
|
||
RetryWorkerConfig: &retryConfig,
|
||
EnableRetryWorker: true, // 启用重试工作器保证最终一致性
|
||
})
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer client.Close()
|
||
|
||
// 5. 构造 Operation
|
||
op, err := model.NewFullOperation(
|
||
model.OpSourceDOIP,
|
||
model.OpTypeCreate,
|
||
"10.1000",
|
||
"my-repo",
|
||
"10.1000/my-repo/doc002",
|
||
"producer-001",
|
||
"admin",
|
||
[]byte(`{"action":"create"}`),
|
||
[]byte(`{"status":"success"}`),
|
||
time.Now(),
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
// 设置 IP 信息（可空）
|
||
clientIP := "192.168.1.100"
|
||
serverIP := "10.0.0.50"
|
||
op.ClientIP = &clientIP
|
||
op.ServerIP = &serverIP
|
||
|
||
// 6. 发布操作（既落库又存证，保证最终一致性）
|
||
if err := client.OperationPublish(ctx, op); err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
fmt.Printf("Operation saved to database and published to trustlog: %s\n", op.OpID)
|
||
fmt.Println("If publish fails, retry worker will handle it automatically")
|
||
}
|
||
|
||
// Example_trustlogOnly 演示仅存证策略
|
||
func Example_trustlogOnly() {
|
||
ctx := context.Background()
|
||
|
||
// 1. 创建 Logger
|
||
myLogger := logger.NewLogger(logr.Discard())
|
||
|
||
// 2. 创建 Pulsar Publisher
|
||
pub, err := adapter.NewPublisher(
|
||
adapter.PublisherConfig{
|
||
URL: "pulsar://localhost:6650",
|
||
},
|
||
myLogger,
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer pub.Close()
|
||
|
||
// 3. 准备 SM2 密钥配置
|
||
privateKeyHex := []byte("私钥D的十六进制字符串")
|
||
publicKeyHex := []byte("04 + x坐标 + y坐标的十六进制字符串")
|
||
envelopeConfig := model.NewSM2EnvelopeConfig(privateKeyHex, publicKeyHex)
|
||
|
||
// 4. 创建支持数据库持久化的客户端（仅存证策略）
|
||
client, err := persistence.NewPersistenceClient(ctx, persistence.PersistenceClientConfig{
|
||
Publisher: pub,
|
||
Logger: myLogger,
|
||
EnvelopeConfig: envelopeConfig,
|
||
DBConfig: persistence.DefaultDBConfig(
|
||
"postgres",
|
||
"postgres://user:pass@localhost:5432/trustlog?sslmode=disable",
|
||
),
|
||
PersistenceConfig: persistence.DefaultPersistenceConfig(persistence.StrategyTrustlogOnly),
|
||
EnableRetryWorker: false, // 仅存证不需要重试工作器
|
||
})
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer client.Close()
|
||
|
||
// 5. 构造 Operation
|
||
op, err := model.NewFullOperation(
|
||
model.OpSourceDOIP,
|
||
model.OpTypeCreate,
|
||
"10.1000",
|
||
"my-repo",
|
||
"10.1000/my-repo/doc003",
|
||
"producer-001",
|
||
"admin",
|
||
[]byte(`{"action":"create"}`),
|
||
[]byte(`{"status":"success"}`),
|
||
time.Now(),
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
// 6. 发布操作（仅存证，不落库）
|
||
if err := client.OperationPublish(ctx, op); err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
fmt.Printf("Operation published to trustlog only: %s\n", op.OpID)
|
||
}
|
||
|
||
// Example_mysqlDatabase 演示使用 MySQL 数据库
|
||
func Example_mysqlDatabase() {
|
||
ctx := context.Background()
|
||
|
||
// 1. 创建 Logger
|
||
myLogger := logger.NewLogger(logr.Discard())
|
||
|
||
// 2. 创建 Pulsar Publisher
|
||
pub, err := adapter.NewPublisher(
|
||
adapter.PublisherConfig{
|
||
URL: "pulsar://localhost:6650",
|
||
},
|
||
myLogger,
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer pub.Close()
|
||
|
||
// 3. 准备 SM2 密钥配置
|
||
privateKeyHex := []byte("私钥D的十六进制字符串")
|
||
publicKeyHex := []byte("04 + x坐标 + y坐标的十六进制字符串")
|
||
envelopeConfig := model.NewSM2EnvelopeConfig(privateKeyHex, publicKeyHex)
|
||
|
||
// 4. 创建支持 MySQL 数据库的客户端
|
||
client, err := persistence.NewPersistenceClient(ctx, persistence.PersistenceClientConfig{
|
||
Publisher: pub,
|
||
Logger: myLogger,
|
||
EnvelopeConfig: envelopeConfig,
|
||
DBConfig: persistence.DefaultDBConfig(
|
||
"mysql",
|
||
"user:pass@tcp(localhost:3306)/trustlog?parseTime=true",
|
||
),
|
||
PersistenceConfig: persistence.DefaultPersistenceConfig(persistence.StrategyDBAndTrustlog),
|
||
EnableRetryWorker: true,
|
||
})
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer client.Close()
|
||
|
||
// 5. 构造并发布 Operation
|
||
op, err := model.NewFullOperation(
|
||
model.OpSourceDOIP,
|
||
model.OpTypeCreate,
|
||
"10.1000",
|
||
"my-repo",
|
||
"10.1000/my-repo/doc004",
|
||
"producer-001",
|
||
"admin",
|
||
[]byte(`{"action":"create"}`),
|
||
[]byte(`{"status":"success"}`),
|
||
time.Now(),
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
// 设置 IP 信息（可空）
|
||
clientIP := "192.168.1.100"
|
||
serverIP := "10.0.0.50"
|
||
op.ClientIP = &clientIP
|
||
op.ServerIP = &serverIP
|
||
|
||
if err := client.OperationPublish(ctx, op); err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
fmt.Printf("Operation saved to MySQL and published: %s\n", op.OpID)
|
||
}
|
||
|
||
// Example_sqliteDatabase 演示使用 SQLite 数据库
|
||
func Example_sqliteDatabase() {
|
||
ctx := context.Background()
|
||
|
||
// 1. 创建 Logger
|
||
myLogger := logger.NewLogger(logr.Discard())
|
||
|
||
// 2. 创建 Pulsar Publisher
|
||
pub, err := adapter.NewPublisher(
|
||
adapter.PublisherConfig{
|
||
URL: "pulsar://localhost:6650",
|
||
},
|
||
myLogger,
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer pub.Close()
|
||
|
||
// 3. 准备 SM2 密钥配置
|
||
privateKeyHex := []byte("私钥D的十六进制字符串")
|
||
publicKeyHex := []byte("04 + x坐标 + y坐标的十六进制字符串")
|
||
envelopeConfig := model.NewSM2EnvelopeConfig(privateKeyHex, publicKeyHex)
|
||
|
||
// 4. 创建支持 SQLite 数据库的客户端
|
||
client, err := persistence.NewPersistenceClient(ctx, persistence.PersistenceClientConfig{
|
||
Publisher: pub,
|
||
Logger: myLogger,
|
||
EnvelopeConfig: envelopeConfig,
|
||
DBConfig: persistence.DefaultDBConfig(
|
||
"sqlite3",
|
||
"./trustlog.db",
|
||
),
|
||
PersistenceConfig: persistence.DefaultPersistenceConfig(persistence.StrategyDBOnly),
|
||
EnableRetryWorker: false,
|
||
})
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
defer client.Close()
|
||
|
||
// 5. 构造并发布 Operation
|
||
op, err := model.NewFullOperation(
|
||
model.OpSourceDOIP,
|
||
model.OpTypeCreate,
|
||
"10.1000",
|
||
"my-repo",
|
||
"10.1000/my-repo/doc005",
|
||
"producer-001",
|
||
"admin",
|
||
[]byte(`{"action":"create"}`),
|
||
[]byte(`{"status":"success"}`),
|
||
time.Now(),
|
||
)
|
||
if err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
// 设置 IP 信息（可空）
|
||
clientIP := "192.168.1.100"
|
||
serverIP := "10.0.0.50"
|
||
op.ClientIP = &clientIP
|
||
op.ServerIP = &serverIP
|
||
|
||
if err := client.OperationPublish(ctx, op); err != nil {
|
||
panic(err)
|
||
}
|
||
|
||
fmt.Printf("Operation saved to SQLite: %s\n", op.OpID)
|
||
}
|
||
|