Files
go-trustlog/api/model/record_test.go
ryan 4b72a37120 feat: 完善数据库持久化与存证功能
主要更新:

1. 数据库持久化功能
   - 支持三种策略:仅落库、既落库又存证、仅存证
   - 实现 Cursor Worker 异步扫描和存证机制
   - 实现 Retry Worker 失败重试机制
   - 支持 PostgreSQL、MySQL、SQLite 等多种数据库
   - 添加 ClientIP 和 ServerIP 字段(可空,仅落库)

2. 集群并发安全
   - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理
   - 实现 CAS (Compare-And-Set) 原子状态更新
   - 添加 updated_at 字段支持并发控制

3. Cursor 初始化优化
   - 自动基于历史数据初始化 cursor
   - 确保不遗漏任何历史记录
   - 修复 UPSERT 逻辑

4. 测试完善
   - 添加 E2E 集成测试(含 Pulsar 消费者验证)
   - 添加 PostgreSQL 集成测试
   - 添加 Pulsar 集成测试
   - 添加集群并发安全测试
   - 添加 Cursor 初始化验证测试
   - 补充大量单元测试,提升覆盖率

5. 工具脚本
   - 添加数据库迁移脚本
   - 添加 Cursor 状态检查工具
   - 添加 Cursor 初始化工具
   - 添加 Pulsar 消息验证工具

6. 文档清理
   - 删除冗余文档,只保留根目录 README

测试结果:
- 所有 E2E 测试通过(100%)
- 数据库持久化与异步存证流程验证通过
- 集群环境下的并发安全性验证通过
- Cursor 自动初始化和历史数据处理验证通过
2025-12-24 15:31:11 +08:00

322 lines
6.9 KiB
Go

package model_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.yandata.net/iod/iod/go-trustlog/api/model"
)
func TestRecord_Key(t *testing.T) {
t.Parallel()
rec := &model.Record{
ID: "test-record-id",
}
assert.Equal(t, "test-record-id", rec.Key())
}
func TestNewFullRecord(t *testing.T) {
t.Parallel()
now := time.Now()
rec, err := model.NewFullRecord(
"test-prefix",
"producer-1",
now,
"operator-1",
[]byte("extra"),
"log",
)
require.NoError(t, err)
assert.NotNil(t, rec)
assert.NotEmpty(t, rec.ID)
assert.Equal(t, "test-prefix", rec.DoPrefix)
assert.Equal(t, "producer-1", rec.ProducerID)
assert.Equal(t, now.Unix(), rec.Timestamp.Unix())
assert.Equal(t, "operator-1", rec.Operator)
assert.Equal(t, []byte("extra"), rec.Extra)
assert.Equal(t, "log", rec.RCType)
}
func TestNewFullRecord_Invalid(t *testing.T) {
t.Parallel()
now := time.Now()
// Missing required ProducerID
rec, err := model.NewFullRecord(
"test-prefix",
"", // Empty ProducerID
now,
"operator-1",
[]byte("extra"),
"log",
)
require.Error(t, err)
assert.Nil(t, rec)
}
func TestRecord_CheckAndInit(t *testing.T) {
t.Parallel()
tests := []struct {
name string
rec *model.Record
wantErr bool
}{
{
name: "valid record",
rec: &model.Record{
DoPrefix: "test",
ProducerID: "producer-1",
Timestamp: time.Now(),
Operator: "operator-1",
Extra: []byte("extra"),
RCType: "log",
},
wantErr: false,
},
{
name: "auto generate ID",
rec: &model.Record{
ID: "", // Will be auto-generated
DoPrefix: "test",
ProducerID: "producer-1",
Timestamp: time.Now(),
Operator: "operator-1",
Extra: []byte("extra"),
RCType: "log",
},
wantErr: false,
},
{
name: "missing ProducerID",
rec: &model.Record{
DoPrefix: "test",
ProducerID: "", // Required field
Timestamp: time.Now(),
Operator: "operator-1",
Extra: []byte("extra"),
RCType: "log",
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
err := tt.rec.CheckAndInit()
if tt.wantErr {
require.Error(t, err)
} else {
require.NoError(t, err)
if tt.name == "auto generate ID" {
assert.NotEmpty(t, tt.rec.ID)
}
}
})
}
}
func TestRecord_MarshalUnmarshalBinary(t *testing.T) {
t.Parallel()
original := &model.Record{
ID: "rec-123",
DoPrefix: "test",
ProducerID: "producer-1",
Timestamp: time.Now(),
Operator: "operator-1",
Extra: []byte("extra"),
RCType: "log",
}
err := original.CheckAndInit()
require.NoError(t, err)
// Marshal
data, err := original.MarshalBinary()
require.NoError(t, err)
require.NotNil(t, data)
// Unmarshal
result := &model.Record{}
err = result.UnmarshalBinary(data)
require.NoError(t, err)
// Verify
assert.Equal(t, original.ID, result.ID)
assert.Equal(t, original.DoPrefix, result.DoPrefix)
assert.Equal(t, original.ProducerID, result.ProducerID)
assert.Equal(t, original.Timestamp.Unix(), result.Timestamp.Unix())
// 验证纳秒精度被保留
assert.Equal(t, original.Timestamp.UnixNano(), result.Timestamp.UnixNano(),
"时间戳的纳秒精度应该被保留")
assert.Equal(t, original.Operator, result.Operator)
assert.Equal(t, original.Extra, result.Extra)
assert.Equal(t, original.RCType, result.RCType)
}
func TestRecord_MarshalBinary_Empty(t *testing.T) {
t.Parallel()
rec := &model.Record{
DoPrefix: "test",
ProducerID: "producer-1",
Timestamp: time.Now(),
}
// MarshalBinary should succeed even without CheckAndInit
// It just serializes the data
data, err := rec.MarshalBinary()
require.NoError(t, err)
assert.NotNil(t, data)
}
func TestRecord_UnmarshalBinary_Empty(t *testing.T) {
t.Parallel()
rec := &model.Record{}
err := rec.UnmarshalBinary([]byte{})
require.Error(t, err)
}
func TestRecord_DoHash(t *testing.T) {
t.Parallel()
rec := &model.Record{
ID: "rec-123",
DoPrefix: "test",
ProducerID: "producer-1",
Timestamp: time.Now(),
Operator: "operator-1",
Extra: []byte("extra"),
RCType: "log",
}
err := rec.CheckAndInit()
require.NoError(t, err)
ctx := context.Background()
hashData, err := rec.DoHash(ctx)
require.NoError(t, err)
assert.NotNil(t, hashData)
assert.Equal(t, rec.ID, hashData.Key())
assert.NotEmpty(t, hashData.Hash())
}
func TestRecordHashData(t *testing.T) {
t.Parallel()
// RecordHashData is created through DoHash, test it indirectly
rec := &model.Record{
ID: "rec-123",
DoPrefix: "test",
ProducerID: "producer-1",
Timestamp: time.Now(),
Operator: "operator-1",
Extra: []byte("extra"),
RCType: "log",
}
err := rec.CheckAndInit()
require.NoError(t, err)
ctx := context.Background()
hashData, err := rec.DoHash(ctx)
require.NoError(t, err)
assert.NotNil(t, hashData)
assert.Equal(t, "rec-123", hashData.Key())
assert.NotEmpty(t, hashData.Hash())
assert.Equal(t, model.Sha256Simd, hashData.Type())
}
func TestRecord_GetProducerID(t *testing.T) {
t.Parallel()
rec := &model.Record{
ProducerID: "producer-123",
}
assert.Equal(t, "producer-123", rec.GetProducerID())
}
func TestRecord_GetDoPrefix(t *testing.T) {
t.Parallel()
rec := &model.Record{
DoPrefix: "test-prefix",
}
assert.Equal(t, "test-prefix", rec.GetDoPrefix())
}
func TestRecord_WithDoPrefix(t *testing.T) {
t.Parallel()
rec := &model.Record{}
result := rec.WithDoPrefix("test-prefix")
assert.Equal(t, rec, result)
assert.Equal(t, "test-prefix", rec.DoPrefix)
}
func TestRecord_WithTimestamp(t *testing.T) {
t.Parallel()
rec := &model.Record{}
now := time.Now()
result := rec.WithTimestamp(now)
assert.Equal(t, rec, result)
assert.Equal(t, now.Unix(), rec.Timestamp.Unix())
}
func TestRecord_WithOperator(t *testing.T) {
t.Parallel()
rec := &model.Record{}
result := rec.WithOperator("operator-1")
assert.Equal(t, rec, result)
assert.Equal(t, "operator-1", rec.Operator)
}
func TestRecord_WithExtra(t *testing.T) {
t.Parallel()
rec := &model.Record{}
extra := []byte("extra-data")
result := rec.WithExtra(extra)
assert.Equal(t, rec, result)
assert.Equal(t, extra, rec.Extra)
}
func TestRecord_WithRCType(t *testing.T) {
t.Parallel()
rec := &model.Record{}
result := rec.WithRCType("log")
assert.Equal(t, rec, result)
assert.Equal(t, "log", rec.RCType)
}
func TestRecord_ChainedMethods(t *testing.T) {
t.Parallel()
rec := &model.Record{}
now := time.Now()
result := rec.
WithDoPrefix("prefix").
WithTimestamp(now).
WithOperator("operator").
WithExtra([]byte("extra")).
WithRCType("log")
assert.Equal(t, rec, result)
assert.Equal(t, "prefix", rec.DoPrefix)
assert.Equal(t, now.Unix(), rec.Timestamp.Unix())
assert.Equal(t, "operator", rec.Operator)
assert.Equal(t, []byte("extra"), rec.Extra)
assert.Equal(t, "log", rec.RCType)
}