4b72a37120
主要更新： 1. 数据库持久化功能 - 支持三种策略：仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段（可空，仅落库） 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试（含 Pulsar 消费者验证） - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试，提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档，只保留根目录 README 测试结果： - 所有 E2E 测试通过（100%） - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
594 lines
13 KiB
Go
594 lines
13 KiB
Go
package model_test
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"go.yandata.net/iod/iod/go-trustlog/api/model"
|
|
)
|
|
|
|
func TestOperation_Key(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{
|
|
OpID: "test-op-id",
|
|
}
|
|
assert.Equal(t, "test-op-id", op.Key())
|
|
}
|
|
|
|
func TestOperation_CheckAndInit(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
op *model.Operation
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "valid operation",
|
|
op: &model.Operation{
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "auto generate OpID",
|
|
op: &model.Operation{
|
|
OpID: "", // Will be auto-generated
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "auto set OpActor",
|
|
op: &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "", // Will be set to "SYSTEM"
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "invalid doid format",
|
|
op: &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "invalid/123", // Doesn't start with "test/repo"
|
|
ProducerID: "producer-1",
|
|
},
|
|
wantErr: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
err := tt.op.CheckAndInit()
|
|
if tt.wantErr {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
if tt.name == "auto generate OpID" {
|
|
assert.NotEmpty(t, tt.op.OpID)
|
|
}
|
|
if tt.name == "auto set OpActor" {
|
|
assert.Equal(t, "SYSTEM", tt.op.OpActor)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOperation_RequestBodyFlexible(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
input interface{}
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "string",
|
|
input: "test",
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "bytes",
|
|
input: []byte("test"),
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "nil",
|
|
input: nil,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "empty string",
|
|
input: "",
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "empty bytes",
|
|
input: []byte{},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "invalid type",
|
|
input: 123,
|
|
wantErr: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
op := &model.Operation{}
|
|
err := op.RequestBodyFlexible(tt.input)
|
|
if tt.wantErr {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOperation_ResponseBodyFlexible(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
input interface{}
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "string",
|
|
input: "test",
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "bytes",
|
|
input: []byte("test"),
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "nil",
|
|
input: nil,
|
|
wantErr: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
op := &model.Operation{}
|
|
err := op.ResponseBodyFlexible(tt.input)
|
|
if tt.wantErr {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestOperation_WithRequestBody(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{}
|
|
result := op.WithRequestBody([]byte("test"))
|
|
assert.Equal(t, op, result)
|
|
}
|
|
|
|
func TestOperation_WithResponseBody(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{}
|
|
result := op.WithResponseBody([]byte("test"))
|
|
assert.Equal(t, op, result)
|
|
}
|
|
|
|
func TestOperation_MarshalUnmarshalBinary(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
original := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
// Marshal
|
|
data, err := original.MarshalBinary()
|
|
require.NoError(t, err)
|
|
require.NotNil(t, data)
|
|
|
|
// Unmarshal
|
|
result := &model.Operation{}
|
|
err = result.UnmarshalBinary(data)
|
|
require.NoError(t, err)
|
|
|
|
// Verify
|
|
assert.Equal(t, original.OpID, result.OpID)
|
|
assert.Equal(t, original.OpSource, result.OpSource)
|
|
assert.Equal(t, original.OpType, result.OpType)
|
|
assert.Equal(t, original.DoPrefix, result.DoPrefix)
|
|
assert.Equal(t, original.DoRepository, result.DoRepository)
|
|
assert.Equal(t, original.Doid, result.Doid)
|
|
assert.Equal(t, original.ProducerID, result.ProducerID)
|
|
assert.Equal(t, original.OpActor, result.OpActor)
|
|
// 验证纳秒精度被保留
|
|
assert.Equal(t, original.Timestamp.UnixNano(), result.Timestamp.UnixNano(),
|
|
"时间戳的纳秒精度应该被保留")
|
|
}
|
|
|
|
func TestOperation_MarshalBinary_Empty(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
}
|
|
// MarshalBinary should succeed even without CheckAndInit
|
|
// It just serializes the data
|
|
data, err := op.MarshalBinary()
|
|
require.NoError(t, err)
|
|
assert.NotNil(t, data)
|
|
}
|
|
|
|
func TestOperation_UnmarshalBinary_Empty(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{}
|
|
err := op.UnmarshalBinary([]byte{})
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestOperation_GetProducerID(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{
|
|
ProducerID: "producer-123",
|
|
}
|
|
assert.Equal(t, "producer-123", op.GetProducerID())
|
|
}
|
|
|
|
func TestOperation_DoHash(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
err := op.CheckAndInit()
|
|
require.NoError(t, err)
|
|
|
|
ctx := context.Background()
|
|
hashData, err := op.DoHash(ctx)
|
|
require.NoError(t, err)
|
|
assert.NotNil(t, hashData)
|
|
assert.Equal(t, op.OpID, hashData.Key())
|
|
assert.NotEmpty(t, hashData.Hash())
|
|
}
|
|
|
|
func TestOperationHashData(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// OperationHashData is created through DoHash, test it indirectly
|
|
op := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
err := op.CheckAndInit()
|
|
require.NoError(t, err)
|
|
|
|
ctx := context.Background()
|
|
hashData, err := op.DoHash(ctx)
|
|
require.NoError(t, err)
|
|
assert.NotNil(t, hashData)
|
|
assert.Equal(t, "op-123", hashData.Key())
|
|
assert.NotEmpty(t, hashData.Hash())
|
|
assert.Equal(t, model.Sha256Simd, hashData.Type())
|
|
}
|
|
|
|
func TestOperation_UnmarshalBinary_InvalidData(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{}
|
|
err := op.UnmarshalBinary([]byte("invalid-cbor-data"))
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "failed to unmarshal operation from CBOR")
|
|
}
|
|
|
|
func TestOperation_MarshalTrustlog_EmptyProducerID(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Create an operation with empty ProducerID
|
|
// MarshalBinary will fail validation, but MarshalTrustlog checks ProducerID first
|
|
op := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "", // Empty ProducerID
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
config := model.NewEnvelopeConfig(model.NewNopSigner())
|
|
_, err := model.MarshalTrustlog(op, config)
|
|
// MarshalTrustlog checks ProducerID before calling MarshalBinary
|
|
require.Error(t, err)
|
|
// Error could be from ProducerID check or MarshalBinary validation
|
|
assert.True(t,
|
|
err.Error() == "producerID cannot be empty" ||
|
|
strings.Contains(err.Error(), "ProducerID") ||
|
|
strings.Contains(err.Error(), "producerID"))
|
|
}
|
|
|
|
func TestOperation_MarshalTrustlog_NilSigner(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
op := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: time.Now(),
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
err := op.CheckAndInit()
|
|
require.NoError(t, err)
|
|
|
|
config := model.EnvelopeConfig{Signer: nil}
|
|
_, err = model.MarshalTrustlog(op, config)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "signer is required")
|
|
}
|
|
|
|
func TestGetOpTypesBySource(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
source model.Source
|
|
wantTypes []model.Type
|
|
}{
|
|
{
|
|
name: "IRP操作类型",
|
|
source: model.OpSourceIRP,
|
|
wantTypes: []model.Type{
|
|
model.OpTypeOCCreateHandle,
|
|
model.OpTypeOCDeleteHandle,
|
|
model.OpTypeOCAddValue,
|
|
},
|
|
},
|
|
{
|
|
name: "DOIP操作类型",
|
|
source: model.OpSourceDOIP,
|
|
wantTypes: []model.Type{
|
|
model.OpTypeHello,
|
|
model.OpTypeCreate,
|
|
model.OpTypeDelete,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
opTypes := model.GetOpTypesBySource(tt.source)
|
|
assert.NotNil(t, opTypes)
|
|
// Verify expected types are included
|
|
for _, expectedType := range tt.wantTypes {
|
|
assert.Contains(t, opTypes, expectedType)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestIsValidOpType(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
source model.Source
|
|
opType model.Type
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "IRP有效操作类型",
|
|
source: model.OpSourceIRP,
|
|
opType: model.OpTypeOCCreateHandle,
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "IRP无效操作类型",
|
|
source: model.OpSourceIRP,
|
|
opType: model.OpTypeHello,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "DOIP有效操作类型",
|
|
source: model.OpSourceDOIP,
|
|
opType: model.OpTypeHello,
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "DOIP无效操作类型",
|
|
source: model.OpSourceDOIP,
|
|
opType: model.OpTypeOCCreateHandle,
|
|
expected: false,
|
|
},
|
|
{
|
|
name: "未知来源和类型",
|
|
source: model.Source("unknown"),
|
|
opType: model.Type("unknown"),
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
result := model.IsValidOpType(tt.source, tt.opType)
|
|
assert.Equal(t, tt.expected, result)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewFullOperation(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
opSource model.Source
|
|
opType model.Type
|
|
doPrefix string
|
|
doRepository string
|
|
doid string
|
|
producerID string
|
|
opActor string
|
|
requestBody interface{}
|
|
responseBody interface{}
|
|
timestamp time.Time
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "成功创建完整操作",
|
|
opSource: model.OpSourceIRP,
|
|
opType: model.OpTypeOCCreateHandle,
|
|
doPrefix: "test",
|
|
doRepository: "repo",
|
|
doid: "test/repo/123",
|
|
producerID: "producer-1",
|
|
opActor: "actor-1",
|
|
requestBody: []byte(`{"key": "value"}`),
|
|
responseBody: []byte(`{"status": "ok"}`),
|
|
timestamp: time.Now(),
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "空请求体和响应体",
|
|
opSource: model.OpSourceIRP,
|
|
opType: model.OpTypeOCCreateHandle,
|
|
doPrefix: "test",
|
|
doRepository: "repo",
|
|
doid: "test/repo/123",
|
|
producerID: "producer-1",
|
|
opActor: "actor-1",
|
|
requestBody: nil,
|
|
responseBody: nil,
|
|
timestamp: time.Now(),
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "字符串类型的请求体",
|
|
opSource: model.OpSourceIRP,
|
|
opType: model.OpTypeOCCreateHandle,
|
|
doPrefix: "test",
|
|
doRepository: "repo",
|
|
doid: "test/repo/123",
|
|
producerID: "producer-1",
|
|
opActor: "actor-1",
|
|
requestBody: "string body",
|
|
responseBody: "string response",
|
|
timestamp: time.Now(),
|
|
wantErr: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
op, err := model.NewFullOperation(
|
|
tt.opSource,
|
|
tt.opType,
|
|
tt.doPrefix,
|
|
tt.doRepository,
|
|
tt.doid,
|
|
tt.producerID,
|
|
tt.opActor,
|
|
tt.requestBody,
|
|
tt.responseBody,
|
|
tt.timestamp,
|
|
)
|
|
|
|
if tt.wantErr {
|
|
require.Error(t, err)
|
|
assert.Nil(t, op)
|
|
} else {
|
|
require.NoError(t, err)
|
|
require.NotNil(t, op)
|
|
assert.Equal(t, tt.opSource, op.OpSource)
|
|
assert.Equal(t, tt.opType, op.OpType)
|
|
assert.Equal(t, tt.doPrefix, op.DoPrefix)
|
|
assert.Equal(t, tt.doRepository, op.DoRepository)
|
|
assert.Equal(t, tt.doid, op.Doid)
|
|
assert.Equal(t, tt.producerID, op.ProducerID)
|
|
assert.Equal(t, tt.opActor, op.OpActor)
|
|
assert.NotEmpty(t, op.OpID) // Should be auto-generated
|
|
}
|
|
})
|
|
}
|
|
}
|