4b72a37120
主要更新： 1. 数据库持久化功能 - 支持三种策略：仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段（可空，仅落库） 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试（含 Pulsar 消费者验证） - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试，提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档，只保留根目录 README 测试结果： - 所有 E2E 测试通过（100%） - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
576 lines
14 KiB
Go
576 lines
14 KiB
Go
package model_test
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"google.golang.org/protobuf/types/known/timestamppb"
|
|
|
|
"go.yandata.net/iod/iod/go-trustlog/api/grpc/pb"
|
|
"go.yandata.net/iod/iod/go-trustlog/api/model"
|
|
)
|
|
|
|
func TestFromProtobuf_Nil(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
result, err := model.FromProtobuf(nil)
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "protobuf operation data is nil")
|
|
}
|
|
|
|
func TestFromProtobuf_NoTimestamp(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pbOp := &pb.OperationData{}
|
|
result, err := model.FromProtobuf(pbOp)
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "timestamp is required")
|
|
}
|
|
|
|
func TestFromProtobuf_Basic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
pbOp := &pb.OperationData{
|
|
OpId: "op-123",
|
|
Timestamp: timestamppb.New(now),
|
|
OpSource: "IRP",
|
|
OpType: "OC_CREATE_HANDLE",
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerId: "producer-1",
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
result, err := model.FromProtobuf(pbOp)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, "op-123", result.OpID)
|
|
assert.Equal(t, now.Unix(), result.Timestamp.Unix())
|
|
assert.Equal(t, model.Source("IRP"), result.OpSource)
|
|
assert.Equal(t, model.Type("OC_CREATE_HANDLE"), result.OpType)
|
|
assert.Equal(t, "test", result.DoPrefix)
|
|
assert.Equal(t, "repo", result.DoRepository)
|
|
assert.Equal(t, "test/repo/123", result.Doid)
|
|
assert.Equal(t, "producer-1", result.ProducerID)
|
|
assert.Equal(t, "actor-1", result.OpActor)
|
|
}
|
|
|
|
func TestFromProtobuf_WithHashes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
pbOp := &pb.OperationData{
|
|
OpId: "op-123",
|
|
Timestamp: timestamppb.New(now),
|
|
OpSource: "DOIP",
|
|
OpType: "Create",
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerId: "producer-1",
|
|
OpActor: "actor-1",
|
|
RequestBodyHash: "req-hash",
|
|
ResponseBodyHash: "resp-hash",
|
|
}
|
|
|
|
result, err := model.FromProtobuf(pbOp)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.NotNil(t, result.RequestBodyHash)
|
|
assert.Equal(t, "req-hash", *result.RequestBodyHash)
|
|
assert.NotNil(t, result.ResponseBodyHash)
|
|
assert.Equal(t, "resp-hash", *result.ResponseBodyHash)
|
|
}
|
|
|
|
func TestFromProtobuf_EmptyHashes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
pbOp := &pb.OperationData{
|
|
OpId: "op-123",
|
|
Timestamp: timestamppb.New(now),
|
|
OpSource: "DOIP",
|
|
OpType: "Create",
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerId: "producer-1",
|
|
OpActor: "actor-1",
|
|
RequestBodyHash: "",
|
|
ResponseBodyHash: "",
|
|
}
|
|
|
|
result, err := model.FromProtobuf(pbOp)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Nil(t, result.RequestBodyHash)
|
|
assert.Nil(t, result.ResponseBodyHash)
|
|
}
|
|
|
|
func TestToProtobuf_Nil(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
result, err := model.ToProtobuf(nil)
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "operation is nil")
|
|
}
|
|
|
|
func TestToProtobuf_Basic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
op := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: now,
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
result, err := model.ToProtobuf(op)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, "op-123", result.GetOpId())
|
|
assert.Equal(t, now.Unix(), result.GetTimestamp().AsTime().Unix())
|
|
assert.Equal(t, "IRP", result.GetOpSource())
|
|
assert.Equal(t, "OC_CREATE_HANDLE", result.GetOpType())
|
|
assert.Equal(t, "test", result.GetDoPrefix())
|
|
assert.Equal(t, "repo", result.GetDoRepository())
|
|
assert.Equal(t, "test/repo/123", result.GetDoid())
|
|
assert.Equal(t, "producer-1", result.GetProducerId())
|
|
assert.Equal(t, "actor-1", result.GetOpActor())
|
|
}
|
|
|
|
func TestToProtobuf_WithHashes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
reqHash := "req-hash"
|
|
respHash := "resp-hash"
|
|
now := time.Now()
|
|
op := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: now,
|
|
OpSource: model.OpSourceDOIP,
|
|
OpType: model.OpTypeCreate,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "actor-1",
|
|
RequestBodyHash: &reqHash,
|
|
ResponseBodyHash: &respHash,
|
|
}
|
|
|
|
result, err := model.ToProtobuf(op)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, "req-hash", result.GetRequestBodyHash())
|
|
assert.Equal(t, "resp-hash", result.GetResponseBodyHash())
|
|
}
|
|
|
|
func TestToProtobuf_WithoutHashes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
op := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: now,
|
|
OpSource: model.OpSourceDOIP,
|
|
OpType: model.OpTypeCreate,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
result, err := model.ToProtobuf(op)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Empty(t, result.GetRequestBodyHash())
|
|
assert.Empty(t, result.GetResponseBodyHash())
|
|
}
|
|
|
|
func TestFromProtobufValidationResult_Nil(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
result, err := model.FromProtobufValidationResult(nil)
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "protobuf validation result is nil")
|
|
}
|
|
|
|
func TestFromProtobufValidationResult_Basic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pbRes := &pb.ValidationStreamRes{
|
|
Code: 100,
|
|
Msg: "Processing",
|
|
Progress: "50%",
|
|
}
|
|
|
|
result, err := model.FromProtobufValidationResult(pbRes)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, int32(100), result.Code)
|
|
assert.Equal(t, "Processing", result.Msg)
|
|
assert.Equal(t, "50%", result.Progress)
|
|
assert.Nil(t, result.Data)
|
|
assert.Nil(t, result.Proof)
|
|
}
|
|
|
|
func TestFromProtobufValidationResult_WithProof(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pbRes := &pb.ValidationStreamRes{
|
|
Code: 200,
|
|
Msg: "Completed",
|
|
Progress: "100%",
|
|
Proof: &pb.Proof{
|
|
Sign: "test-signature",
|
|
ColItems: []*pb.MerkleTreeProofItem{
|
|
{Floor: 1, Hash: "hash1", Left: true},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := model.FromProtobufValidationResult(pbRes)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, int32(200), result.Code)
|
|
assert.NotNil(t, result.Proof)
|
|
assert.Equal(t, "test-signature", result.Proof.Sign)
|
|
assert.Len(t, result.Proof.ColItems, 1)
|
|
}
|
|
|
|
func TestFromProtobufValidationResult_WithData(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
pbRes := &pb.ValidationStreamRes{
|
|
Code: 200,
|
|
Msg: "Completed",
|
|
Progress: "100%",
|
|
Data: &pb.OperationData{
|
|
OpId: "op-123",
|
|
Timestamp: timestamppb.New(now),
|
|
OpSource: "IRP",
|
|
OpType: "OC_CREATE_HANDLE",
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerId: "producer-1",
|
|
OpActor: "actor-1",
|
|
},
|
|
}
|
|
|
|
result, err := model.FromProtobufValidationResult(pbRes)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, int32(200), result.Code)
|
|
assert.NotNil(t, result.Data)
|
|
assert.Equal(t, "op-123", result.Data.OpID)
|
|
}
|
|
|
|
func TestFromProtobufValidationResult_WithInvalidData(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pbRes := &pb.ValidationStreamRes{
|
|
Code: 200,
|
|
Msg: "Completed",
|
|
Progress: "100%",
|
|
Data: &pb.OperationData{
|
|
// Missing timestamp
|
|
},
|
|
}
|
|
|
|
result, err := model.FromProtobufValidationResult(pbRes)
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "failed to convert operation data")
|
|
}
|
|
|
|
func TestRecordFromProtobuf_Nil(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
result, err := model.RecordFromProtobuf(nil)
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "protobuf record data is nil")
|
|
}
|
|
|
|
func TestRecordFromProtobuf_Basic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
pbRec := &pb.RecordData{
|
|
Id: "rec-123",
|
|
DoPrefix: "test",
|
|
ProducerId: "producer-1",
|
|
Timestamp: timestamppb.New(now),
|
|
Operator: "operator-1",
|
|
Extra: []byte("extra-data"),
|
|
RcType: "log",
|
|
}
|
|
|
|
result, err := model.RecordFromProtobuf(pbRec)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, "rec-123", result.ID)
|
|
assert.Equal(t, "test", result.DoPrefix)
|
|
assert.Equal(t, "producer-1", result.ProducerID)
|
|
assert.Equal(t, now.Unix(), result.Timestamp.Unix())
|
|
assert.Equal(t, "operator-1", result.Operator)
|
|
assert.Equal(t, []byte("extra-data"), result.Extra)
|
|
assert.Equal(t, "log", result.RCType)
|
|
}
|
|
|
|
func TestRecordFromProtobuf_NoTimestamp(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pbRec := &pb.RecordData{
|
|
Id: "rec-123",
|
|
DoPrefix: "test",
|
|
ProducerId: "producer-1",
|
|
Operator: "operator-1",
|
|
Extra: []byte("extra-data"),
|
|
RcType: "log",
|
|
}
|
|
|
|
result, err := model.RecordFromProtobuf(pbRec)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, "rec-123", result.ID)
|
|
assert.True(t, result.Timestamp.IsZero())
|
|
}
|
|
|
|
func TestRecordToProtobuf_Nil(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
result, err := model.RecordToProtobuf(nil)
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "record is nil")
|
|
}
|
|
|
|
func TestRecordToProtobuf_Basic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
rec := &model.Record{
|
|
ID: "rec-123",
|
|
DoPrefix: "test",
|
|
ProducerID: "producer-1",
|
|
Timestamp: now,
|
|
Operator: "operator-1",
|
|
Extra: []byte("extra-data"),
|
|
RCType: "log",
|
|
}
|
|
|
|
result, err := model.RecordToProtobuf(rec)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, "rec-123", result.GetId())
|
|
assert.Equal(t, "test", result.GetDoPrefix())
|
|
assert.Equal(t, "producer-1", result.GetProducerId())
|
|
assert.Equal(t, now.Unix(), result.GetTimestamp().AsTime().Unix())
|
|
assert.Equal(t, "operator-1", result.GetOperator())
|
|
assert.Equal(t, []byte("extra-data"), result.GetExtra())
|
|
assert.Equal(t, "log", result.GetRcType())
|
|
}
|
|
|
|
func TestRecordFromProtobufValidationResult_Nil(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
result, err := model.RecordFromProtobufValidationResult(nil)
|
|
require.Nil(t, result)
|
|
require.Error(t, err)
|
|
assert.Contains(t, err.Error(), "protobuf record validation result is nil")
|
|
}
|
|
|
|
func TestRecordFromProtobufValidationResult_Basic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pbRes := &pb.RecordValidationStreamRes{
|
|
Code: 100,
|
|
Msg: "Processing",
|
|
Progress: "50%",
|
|
}
|
|
|
|
result, err := model.RecordFromProtobufValidationResult(pbRes)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, int32(100), result.Code)
|
|
assert.Equal(t, "Processing", result.Msg)
|
|
assert.Equal(t, "50%", result.Progress)
|
|
assert.Nil(t, result.Data)
|
|
assert.Nil(t, result.Proof)
|
|
}
|
|
|
|
func TestRecordFromProtobufValidationResult_WithProof(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pbRes := &pb.RecordValidationStreamRes{
|
|
Code: 200,
|
|
Msg: "Completed",
|
|
Progress: "100%",
|
|
Proof: &pb.Proof{
|
|
Sign: "test-signature",
|
|
RawItems: []*pb.MerkleTreeProofItem{
|
|
{Floor: 1, Hash: "hash1", Left: true},
|
|
},
|
|
},
|
|
}
|
|
|
|
result, err := model.RecordFromProtobufValidationResult(pbRes)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, int32(200), result.Code)
|
|
assert.NotNil(t, result.Proof)
|
|
assert.Equal(t, "test-signature", result.Proof.Sign)
|
|
assert.Len(t, result.Proof.RawItems, 1)
|
|
}
|
|
|
|
func TestRecordFromProtobufValidationResult_WithData(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
pbRes := &pb.RecordValidationStreamRes{
|
|
Code: 200,
|
|
Msg: "Completed",
|
|
Progress: "100%",
|
|
Result: &pb.RecordData{
|
|
Id: "rec-123",
|
|
DoPrefix: "test",
|
|
ProducerId: "producer-1",
|
|
Timestamp: timestamppb.New(now),
|
|
Operator: "operator-1",
|
|
Extra: []byte("extra-data"),
|
|
RcType: "log",
|
|
},
|
|
}
|
|
|
|
result, err := model.RecordFromProtobufValidationResult(pbRes)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
assert.Equal(t, int32(200), result.Code)
|
|
assert.NotNil(t, result.Data)
|
|
assert.Equal(t, "rec-123", result.Data.ID)
|
|
}
|
|
|
|
func TestRecordFromProtobufValidationResult_WithInvalidData(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
pbRes := &pb.RecordValidationStreamRes{
|
|
Code: 200,
|
|
Msg: "Completed",
|
|
Progress: "100%",
|
|
Result: &pb.RecordData{
|
|
// Missing required fields to trigger error
|
|
},
|
|
}
|
|
|
|
result, err := model.RecordFromProtobufValidationResult(pbRes)
|
|
// This should succeed even with empty RecordData
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
assert.Equal(t, int32(200), result.Code)
|
|
}
|
|
|
|
func TestRoundTrip_Operation(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
original := &model.Operation{
|
|
OpID: "op-123",
|
|
Timestamp: now,
|
|
OpSource: model.OpSourceIRP,
|
|
OpType: model.OpTypeOCCreateHandle,
|
|
DoPrefix: "test",
|
|
DoRepository: "repo",
|
|
Doid: "test/repo/123",
|
|
ProducerID: "producer-1",
|
|
OpActor: "actor-1",
|
|
}
|
|
|
|
// Convert to protobuf
|
|
pbOp, err := model.ToProtobuf(original)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, pbOp)
|
|
|
|
// Convert back to model
|
|
result, err := model.FromProtobuf(pbOp)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
// Verify round trip
|
|
assert.Equal(t, original.OpID, result.OpID)
|
|
assert.Equal(t, original.OpSource, result.OpSource)
|
|
assert.Equal(t, original.OpType, result.OpType)
|
|
assert.Equal(t, original.DoPrefix, result.DoPrefix)
|
|
assert.Equal(t, original.DoRepository, result.DoRepository)
|
|
assert.Equal(t, original.Doid, result.Doid)
|
|
assert.Equal(t, original.ProducerID, result.ProducerID)
|
|
assert.Equal(t, original.OpActor, result.OpActor)
|
|
}
|
|
|
|
func TestRoundTrip_Record(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
now := time.Now()
|
|
original := &model.Record{
|
|
ID: "rec-123",
|
|
DoPrefix: "test",
|
|
ProducerID: "producer-1",
|
|
Timestamp: now,
|
|
Operator: "operator-1",
|
|
Extra: []byte("extra-data"),
|
|
RCType: "log",
|
|
}
|
|
|
|
// Convert to protobuf
|
|
pbRec, err := model.RecordToProtobuf(original)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, pbRec)
|
|
|
|
// Convert back to model
|
|
result, err := model.RecordFromProtobuf(pbRec)
|
|
require.NoError(t, err)
|
|
require.NotNil(t, result)
|
|
|
|
// Verify round trip
|
|
assert.Equal(t, original.ID, result.ID)
|
|
assert.Equal(t, original.DoPrefix, result.DoPrefix)
|
|
assert.Equal(t, original.ProducerID, result.ProducerID)
|
|
assert.Equal(t, original.Timestamp.Unix(), result.Timestamp.Unix())
|
|
assert.Equal(t, original.Operator, result.Operator)
|
|
assert.Equal(t, original.Extra, result.Extra)
|
|
assert.Equal(t, original.RCType, result.RCType)
|
|
}
|