## 核心功能 ### 1. 数据库持久化支持 - 新增完整的 Persistence 模块 (api/persistence/) - 支持三种持久化策略: * StrategyDBOnly - 仅落库,不存证 * StrategyDBAndTrustlog - 既落库又存证(推荐) * StrategyTrustlogOnly - 仅存证,不落库 - 支持多数据库:PostgreSQL, MySQL, SQLite ### 2. Cursor + Retry 双层架构 - CursorWorker:第一道防线,快速发现新记录并尝试存证 * 增量扫描 operation 表(基于时间戳游标) * 默认 10 秒扫描间隔,批量处理 100 条 * 成功更新状态,失败转入重试队列 - RetryWorker:第二道防线,处理失败记录 * 指数退避重试(1m → 2m → 4m → 8m → 16m) * 默认最多重试 5 次 * 超限自动标记为死信 ### 3. 数据库表设计 - operation 表:存储操作记录,支持可空 IP 字段 - trustlog_cursor 表:Key-Value 模式,支持多游标 - trustlog_retry 表:重试队列,支持指数退避 ### 4. 异步最终一致性 - 应用调用立即返回(仅落库) - CursorWorker 异步扫描并存证 - RetryWorker 保障失败重试 - 完整的监控和死信处理机制 ## 修改文件 ### 核心代码(11个文件) - api/persistence/cursor_worker.go - Cursor 工作器(新增) - api/persistence/repository.go - 数据仓储层(新增) - api/persistence/schema.go - 数据库 Schema(新增) - api/persistence/strategy.go - 策略管理器(新增) - api/persistence/client.go - 客户端封装(新增) - api/persistence/retry_worker.go - Retry 工作器(新增) - api/persistence/config.go - 配置管理(新增) ### 修复内部包引用(5个文件) - api/adapter/publisher.go - 修复 internal 包引用 - api/adapter/subscriber.go - 修复 internal 包引用 - api/model/envelope.go - 修复 internal 包引用 - api/model/operation.go - 修复 internal 包引用 - api/model/record.go - 修复 internal 包引用 ### 单元测试(8个文件) - api/persistence/*_test.go - 完整的单元测试 - 测试覆盖率:28.5% - 测试通过率:49/49 (100%) ### SQL 脚本(4个文件) - api/persistence/sql/postgresql.sql - PostgreSQL 建表脚本 - api/persistence/sql/mysql.sql - MySQL 建表脚本 - api/persistence/sql/sqlite.sql - SQLite 建表脚本 - api/persistence/sql/test_data.sql - 测试数据 ### 文档(2个文件) - README.md - 更新主文档,新增 Persistence 使用指南 - api/persistence/README.md - 完整的 Persistence 文档 - api/persistence/sql/README.md - SQL 脚本说明 ## 技术亮点 1. **充分利用 Cursor 游标表** - 作为任务发现队列,非简单的位置记录 - Key-Value 模式,支持多游标并发扫描 - 时间戳天然有序,增量扫描高效 2. **双层保障机制** - Cursor:正常流程,快速处理 - Retry:异常流程,可靠重试 - 职责分离,监控清晰 3. **可空 IP 字段支持** - ClientIP 和 ServerIP 使用 *string 类型 - 支持 NULL 值,符合数据库最佳实践 - 使用 sql.NullString 正确处理 4. **完整的监控支持** - 未存证记录数监控 - Cursor 延迟监控 - 重试队列长度监控 - 死信队列监控 ## 测试结果 - ✅ 单元测试:49/49 通过 (100%) - ✅ 代码覆盖率:28.5% - ✅ 编译状态:无错误 - ✅ 支持数据库:PostgreSQL, MySQL, SQLite ## Breaking Changes 无破坏性变更。Persistence 模块作为可选功能,不影响现有代码。 ## 版本信息 - 版本:v2.1.0 - Go 版本要求:1.21+ - 更新日期:2025-12-23
349 lines
8.1 KiB
Go
349 lines
8.1 KiB
Go
package model
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"fmt"
|
||
"time"
|
||
|
||
"go.yandata.net/iod/iod/go-trustlog/api/logger"
|
||
"go.yandata.net/iod/iod/go-trustlog/internal/helpers"
|
||
)
|
||
|
||
// Record 表示一条记录。
|
||
// 用于记录系统中的操作行为,包含记录标识、节点前缀、操作者信息等。
|
||
type Record struct {
|
||
ID string `json:"id" validate:"required,max=128"`
|
||
DoPrefix string `json:"doPrefix" validate:"max=512"`
|
||
ProducerID string `json:"producerId" validate:"required,max=512"`
|
||
Timestamp time.Time `json:"timestamp"`
|
||
Operator string `json:"operator" validate:"max=64"`
|
||
Extra []byte `json:"extra" validate:"max=512"`
|
||
RCType string `json:"type" validate:"max=64"`
|
||
binary []byte
|
||
}
|
||
|
||
//
|
||
// ===== 构造函数 =====
|
||
//
|
||
|
||
// NewFullRecord 创建包含所有字段的完整 Record。
|
||
// 自动完成字段校验,确保创建的 Record 是完整且有效的。
|
||
func NewFullRecord(
|
||
doPrefix string,
|
||
producerID string,
|
||
timestamp time.Time,
|
||
operator string,
|
||
extra []byte,
|
||
rcType string,
|
||
) (*Record, error) {
|
||
log := logger.GetGlobalLogger()
|
||
log.Debug("Creating new full record",
|
||
"doPrefix", doPrefix,
|
||
"producerID", producerID,
|
||
"operator", operator,
|
||
"rcType", rcType,
|
||
"extraLength", len(extra),
|
||
)
|
||
record := &Record{
|
||
DoPrefix: doPrefix,
|
||
ProducerID: producerID,
|
||
Timestamp: timestamp,
|
||
Operator: operator,
|
||
Extra: extra,
|
||
RCType: rcType,
|
||
}
|
||
|
||
log.Debug("Checking and initializing record")
|
||
if err := record.CheckAndInit(); err != nil {
|
||
log.Error("Failed to check and init record",
|
||
"error", err,
|
||
)
|
||
return nil, err
|
||
}
|
||
|
||
log.Debug("Full record created successfully",
|
||
"recordID", record.ID,
|
||
)
|
||
return record, nil
|
||
}
|
||
|
||
//
|
||
// ===== 接口实现 =====
|
||
//
|
||
|
||
func (r *Record) Key() string {
|
||
return r.ID
|
||
}
|
||
|
||
// RecordHashData 实现 HashData 接口,用于存储 Record 的哈希计算结果。
|
||
type RecordHashData struct {
|
||
key string
|
||
hash string
|
||
}
|
||
|
||
func (r RecordHashData) Key() string {
|
||
return r.key
|
||
}
|
||
|
||
func (r RecordHashData) Hash() string {
|
||
return r.hash
|
||
}
|
||
|
||
func (r RecordHashData) Type() HashType {
|
||
return Sha256Simd
|
||
}
|
||
|
||
// DoHash 计算 Record 的整体哈希值,用于数据完整性验证。
|
||
// 哈希基于序列化后的二进制数据计算,确保记录数据的不可篡改性。
|
||
func (r *Record) DoHash(_ context.Context) (HashData, error) {
|
||
log := logger.GetGlobalLogger()
|
||
log.Debug("Computing hash for record",
|
||
"recordID", r.ID,
|
||
)
|
||
hashTool := GetHashTool(Sha256Simd)
|
||
binary, err := r.MarshalBinary()
|
||
if err != nil {
|
||
log.Error("Failed to marshal record for hash",
|
||
"error", err,
|
||
"recordID", r.ID,
|
||
)
|
||
return nil, fmt.Errorf("failed to marshal record: %w", err)
|
||
}
|
||
|
||
log.Debug("Computing hash bytes",
|
||
"recordID", r.ID,
|
||
"binaryLength", len(binary),
|
||
)
|
||
hash, err := hashTool.HashBytes(binary)
|
||
if err != nil {
|
||
log.Error("Failed to compute hash",
|
||
"error", err,
|
||
"recordID", r.ID,
|
||
)
|
||
return nil, fmt.Errorf("failed to compute hash: %w", err)
|
||
}
|
||
|
||
log.Debug("Hash computed successfully",
|
||
"recordID", r.ID,
|
||
"hash", hash,
|
||
)
|
||
return RecordHashData{
|
||
key: r.ID,
|
||
hash: hash,
|
||
}, nil
|
||
}
|
||
|
||
//
|
||
// ===== CBOR 序列化相关 =====
|
||
//
|
||
|
||
// recordData 用于 CBOR 序列化/反序列化的中间结构。
|
||
// 排除缓存字段,仅包含可序列化的数据字段。
|
||
type recordData struct {
|
||
ID *string `cbor:"id"`
|
||
DoPrefix *string `cbor:"doPrefix"`
|
||
ProducerID *string `cbor:"producerId"`
|
||
Timestamp *time.Time `cbor:"timestamp"`
|
||
Operator *string `cbor:"operator"`
|
||
Extra []byte `cbor:"extra"`
|
||
RCType *string `cbor:"type"`
|
||
}
|
||
|
||
// toRecordData 将 Record 转换为 recordData,用于序列化。
|
||
func (r *Record) toRecordData() *recordData {
|
||
return &recordData{
|
||
ID: &r.ID,
|
||
DoPrefix: &r.DoPrefix,
|
||
ProducerID: &r.ProducerID,
|
||
Timestamp: &r.Timestamp,
|
||
Operator: &r.Operator,
|
||
Extra: r.Extra,
|
||
RCType: &r.RCType,
|
||
}
|
||
}
|
||
|
||
// fromRecordData 从 recordData 填充 Record,用于反序列化。
|
||
func (r *Record) fromRecordData(recData *recordData) {
|
||
if recData == nil {
|
||
return
|
||
}
|
||
|
||
if recData.ID != nil {
|
||
r.ID = *recData.ID
|
||
}
|
||
if recData.DoPrefix != nil {
|
||
r.DoPrefix = *recData.DoPrefix
|
||
}
|
||
if recData.ProducerID != nil {
|
||
r.ProducerID = *recData.ProducerID
|
||
}
|
||
if recData.Timestamp != nil {
|
||
r.Timestamp = *recData.Timestamp
|
||
}
|
||
if recData.Operator != nil {
|
||
r.Operator = *recData.Operator
|
||
}
|
||
if recData.Extra != nil {
|
||
r.Extra = recData.Extra
|
||
}
|
||
if recData.RCType != nil {
|
||
r.RCType = *recData.RCType
|
||
}
|
||
}
|
||
|
||
// MarshalBinary 将 Record 序列化为 CBOR 格式的二进制数据。
|
||
// 实现 encoding.BinaryMarshaler 接口。
|
||
// 使用 Canonical CBOR 编码确保序列化结果的一致性,使用缓存机制避免重复序列化。
|
||
func (r *Record) MarshalBinary() ([]byte, error) {
|
||
log := logger.GetGlobalLogger()
|
||
log.Debug("Marshaling record to CBOR binary",
|
||
"recordID", r.ID,
|
||
)
|
||
if r.binary != nil {
|
||
log.Debug("Using cached binary data",
|
||
"recordID", r.ID,
|
||
)
|
||
return r.binary, nil
|
||
}
|
||
|
||
recData := r.toRecordData()
|
||
|
||
log.Debug("Marshaling record data to canonical CBOR",
|
||
"recordID", r.ID,
|
||
)
|
||
binary, err := helpers.MarshalCanonical(recData)
|
||
if err != nil {
|
||
log.Error("Failed to marshal record to CBOR",
|
||
"error", err,
|
||
"recordID", r.ID,
|
||
)
|
||
return nil, fmt.Errorf("failed to marshal record to CBOR: %w", err)
|
||
}
|
||
|
||
r.binary = binary
|
||
|
||
log.Debug("Record marshaled successfully",
|
||
"recordID", r.ID,
|
||
"binaryLength", len(binary),
|
||
)
|
||
return binary, nil
|
||
}
|
||
|
||
// UnmarshalBinary 从 CBOR 格式的二进制数据反序列化为 Record。
|
||
// 实现 encoding.BinaryUnmarshaler 接口。
|
||
func (r *Record) UnmarshalBinary(data []byte) error {
|
||
log := logger.GetGlobalLogger()
|
||
log.Debug("Unmarshaling record from CBOR binary",
|
||
"dataLength", len(data),
|
||
)
|
||
if len(data) == 0 {
|
||
log.Error("Data is empty")
|
||
return errors.New("data is empty")
|
||
}
|
||
|
||
recData := &recordData{}
|
||
|
||
log.Debug("Unmarshaling record data from CBOR")
|
||
if err := helpers.Unmarshal(data, recData); err != nil {
|
||
log.Error("Failed to unmarshal record from CBOR",
|
||
"error", err,
|
||
)
|
||
return fmt.Errorf("failed to unmarshal record from CBOR: %w", err)
|
||
}
|
||
|
||
r.fromRecordData(recData)
|
||
|
||
r.binary = data
|
||
|
||
log.Debug("Record unmarshaled successfully",
|
||
"recordID", r.ID,
|
||
)
|
||
return nil
|
||
}
|
||
|
||
// GetDoPrefix 实现 DoPrefixExtractor 接口,返回节点前缀。
|
||
func (r *Record) GetDoPrefix() string {
|
||
return r.DoPrefix
|
||
}
|
||
|
||
// GetProducerID 返回 ProducerID,实现 Trustlog 接口。
|
||
func (r *Record) GetProducerID() string {
|
||
return r.ProducerID
|
||
}
|
||
|
||
//
|
||
// ===== 初始化与验证 =====
|
||
//
|
||
|
||
// CheckAndInit 校验并初始化 Record。
|
||
// 自动填充缺失字段(ID),字段非空验证由 validate 标签处理。
|
||
func (r *Record) CheckAndInit() error {
|
||
log := logger.GetGlobalLogger()
|
||
log.Debug("Checking and initializing record",
|
||
"producerID", r.ProducerID,
|
||
"doPrefix", r.DoPrefix,
|
||
)
|
||
if r.ID == "" {
|
||
r.ID = helpers.NewUUIDv7()
|
||
log.Debug("Generated new record ID",
|
||
"recordID", r.ID,
|
||
)
|
||
}
|
||
|
||
if r.Timestamp.IsZero() {
|
||
r.Timestamp = time.Now()
|
||
log.Debug("Set default timestamp",
|
||
"timestamp", r.Timestamp,
|
||
)
|
||
}
|
||
|
||
log.Debug("Validating record struct")
|
||
if err := helpers.GetValidator().Struct(r); err != nil {
|
||
log.Error("Record validation failed",
|
||
"error", err,
|
||
"recordID", r.ID,
|
||
)
|
||
return err
|
||
}
|
||
|
||
log.Debug("Record checked and initialized successfully",
|
||
"recordID", r.ID,
|
||
)
|
||
return nil
|
||
}
|
||
|
||
//
|
||
// ===== 链式调用支持 =====
|
||
//
|
||
|
||
// WithDoPrefix 设置 DoPrefix 并返回自身,支持链式调用。
|
||
func (r *Record) WithDoPrefix(doPrefix string) *Record {
|
||
r.DoPrefix = doPrefix
|
||
return r
|
||
}
|
||
|
||
// WithTimestamp 设置 Timestamp 并返回自身,支持链式调用。
|
||
func (r *Record) WithTimestamp(timestamp time.Time) *Record {
|
||
r.Timestamp = timestamp
|
||
return r
|
||
}
|
||
|
||
// WithOperator 设置 Operator 并返回自身,支持链式调用。
|
||
func (r *Record) WithOperator(operator string) *Record {
|
||
r.Operator = operator
|
||
return r
|
||
}
|
||
|
||
// WithExtra 设置 Extra 并返回自身,支持链式调用。
|
||
func (r *Record) WithExtra(extra []byte) *Record {
|
||
r.Extra = extra
|
||
return r
|
||
}
|
||
|
||
// WithRCType 设置 RCType 并返回自身,支持链式调用。
|
||
func (r *Record) WithRCType(rcType string) *Record {
|
||
r.RCType = rcType
|
||
return r
|
||
}
|