主要更新: 1. 数据库持久化功能 - 支持三种策略:仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段(可空,仅落库) 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试(含 Pulsar 消费者验证) - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试,提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档,只保留根目录 README 测试结果: - 所有 E2E 测试通过(100%) - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
398 lines
9.8 KiB
Go
398 lines
9.8 KiB
Go
package queryclient_test
|
||
|
||
import (
|
||
"context"
|
||
"errors"
|
||
"testing"
|
||
|
||
"github.com/stretchr/testify/assert"
|
||
"github.com/stretchr/testify/require"
|
||
"google.golang.org/protobuf/types/known/timestamppb"
|
||
|
||
"go.yandata.net/iod/iod/go-trustlog/api/grpc/pb"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/logger"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/model"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/queryclient"
|
||
)
|
||
|
||
// TestNewClient_ErrorCases 测试客户端创建的错误情况
|
||
func TestNewClient_ErrorCases(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
config queryclient.ClientConfig
|
||
wantError bool
|
||
}{
|
||
{
|
||
name: "empty server addresses",
|
||
config: queryclient.ClientConfig{
|
||
ServerAddrs: []string{},
|
||
ServerAddr: "",
|
||
},
|
||
wantError: true,
|
||
},
|
||
{
|
||
name: "invalid dial options",
|
||
config: queryclient.ClientConfig{
|
||
ServerAddr: "invalid://address",
|
||
},
|
||
wantError: false, // 连接错误在拨号时才会发生
|
||
},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
_, err := queryclient.NewClient(tt.config, logger.NewNopLogger())
|
||
if tt.wantError {
|
||
require.Error(t, err)
|
||
} else {
|
||
// 即使配置有问题,NewClient 也可能成功(连接是惰性的)
|
||
t.Log("Client created, connection errors may occur on actual use")
|
||
}
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestListOperations_ErrorHandling 测试 ListOperations 的错误处理
|
||
func TestListOperations_ErrorHandling(t *testing.T) {
|
||
// 由于需要实际的 gRPC 连接,这里主要测试输入验证
|
||
t.Run("verify request construction", func(t *testing.T) {
|
||
req := queryclient.ListOperationsRequest{
|
||
PageSize: 10,
|
||
OpSource: "api",
|
||
OpType: "create",
|
||
DoPrefix: "test",
|
||
DoRepository: "repo",
|
||
}
|
||
|
||
assert.Equal(t, uint64(10), req.PageSize)
|
||
assert.Equal(t, model.Source("api"), req.OpSource)
|
||
assert.Equal(t, model.Type("create"), req.OpType)
|
||
})
|
||
}
|
||
|
||
// TestValidationRequest_Construction 测试 ValidationRequest 构造
|
||
func TestValidationRequest_Construction(t *testing.T) {
|
||
req := queryclient.ValidationRequest{
|
||
OpID: "test-op",
|
||
OpType: "create",
|
||
DoRepository: "test-repo",
|
||
}
|
||
|
||
assert.Equal(t, "test-op", req.OpID)
|
||
assert.Equal(t, "create", req.OpType)
|
||
assert.Equal(t, "test-repo", req.DoRepository)
|
||
}
|
||
|
||
// TestListRecordsRequest_Construction 测试 ListRecordsRequest 构造
|
||
func TestListRecordsRequest_Construction(t *testing.T) {
|
||
req := queryclient.ListRecordsRequest{
|
||
PageSize: 20,
|
||
DoPrefix: "test",
|
||
RCType: "log",
|
||
}
|
||
|
||
assert.Equal(t, uint64(20), req.PageSize)
|
||
assert.Equal(t, "test", req.DoPrefix)
|
||
assert.Equal(t, "log", req.RCType)
|
||
}
|
||
|
||
// TestRecordValidationRequest_Construction 测试 RecordValidationRequest 构造
|
||
func TestRecordValidationRequest_Construction(t *testing.T) {
|
||
req := queryclient.RecordValidationRequest{
|
||
RecordID: "rec-123",
|
||
DoPrefix: "test",
|
||
RCType: "log",
|
||
}
|
||
|
||
assert.Equal(t, "rec-123", req.RecordID)
|
||
assert.Equal(t, "test", req.DoPrefix)
|
||
assert.Equal(t, "log", req.RCType)
|
||
}
|
||
|
||
// mockFailingOperationServer 总是失败的mock服务器
|
||
type mockFailingOperationServer struct {
|
||
pb.UnimplementedOperationValidationServiceServer
|
||
}
|
||
|
||
func (s *mockFailingOperationServer) ListOperations(
|
||
_ context.Context,
|
||
_ *pb.ListOperationReq,
|
||
) (*pb.ListOperationRes, error) {
|
||
return nil, errors.New("mock error: list operations failed")
|
||
}
|
||
|
||
func (s *mockFailingOperationServer) ValidateOperation(
|
||
_ *pb.ValidationReq,
|
||
stream pb.OperationValidationService_ValidateOperationServer,
|
||
) error {
|
||
// 发送错误消息
|
||
_ = stream.Send(&pb.ValidationStreamRes{
|
||
Code: 500,
|
||
Msg: "Validation failed",
|
||
})
|
||
return errors.New("mock error: validation failed")
|
||
}
|
||
|
||
// mockFailingRecordServer 总是失败的mock记录服务器
|
||
type mockFailingRecordServer struct {
|
||
pb.UnimplementedRecordValidationServiceServer
|
||
}
|
||
|
||
func (s *mockFailingRecordServer) ListRecords(
|
||
_ context.Context,
|
||
_ *pb.ListRecordReq,
|
||
) (*pb.ListRecordRes, error) {
|
||
return nil, errors.New("mock error: list records failed")
|
||
}
|
||
|
||
func (s *mockFailingRecordServer) ValidateRecord(
|
||
_ *pb.RecordValidationReq,
|
||
stream pb.RecordValidationService_ValidateRecordServer,
|
||
) error {
|
||
return errors.New("mock error: record validation failed")
|
||
}
|
||
|
||
// mockEmptyOperationServer 返回空数据的mock服务器
|
||
type mockEmptyOperationServer struct {
|
||
pb.UnimplementedOperationValidationServiceServer
|
||
}
|
||
|
||
func (s *mockEmptyOperationServer) ListOperations(
|
||
_ context.Context,
|
||
_ *pb.ListOperationReq,
|
||
) (*pb.ListOperationRes, error) {
|
||
return &pb.ListOperationRes{
|
||
Count: 0,
|
||
Data: []*pb.OperationData{},
|
||
}, nil
|
||
}
|
||
|
||
// mockInvalidDataOperationServer 返回无效数据的mock服务器
|
||
type mockInvalidDataOperationServer struct {
|
||
pb.UnimplementedOperationValidationServiceServer
|
||
}
|
||
|
||
func (s *mockInvalidDataOperationServer) ListOperations(
|
||
_ context.Context,
|
||
_ *pb.ListOperationReq,
|
||
) (*pb.ListOperationRes, error) {
|
||
return &pb.ListOperationRes{
|
||
Count: 1,
|
||
Data: []*pb.OperationData{
|
||
{
|
||
// 缺少必需的 Timestamp 字段
|
||
OpId: "invalid-op",
|
||
OpSource: "test",
|
||
},
|
||
},
|
||
}, nil
|
||
}
|
||
|
||
func (s *mockInvalidDataOperationServer) ValidateOperation(
|
||
_ *pb.ValidationReq,
|
||
stream pb.OperationValidationService_ValidateOperationServer,
|
||
) error {
|
||
// 发送无效数据
|
||
_ = stream.Send(&pb.ValidationStreamRes{
|
||
Code: 200,
|
||
Msg: "Completed",
|
||
Progress: "100%",
|
||
Data: &pb.OperationData{
|
||
// 缺少 Timestamp
|
||
OpId: "invalid",
|
||
},
|
||
})
|
||
return nil
|
||
}
|
||
|
||
// TestValidateOperationSync_ProgressCallback 测试带进度回调的同步验证
|
||
func TestValidateOperationSync_ProgressCallback(t *testing.T) {
|
||
t.Run("verify progress callback structure", func(t *testing.T) {
|
||
progressCalled := false
|
||
progressCallback := func(result *model.ValidationResult) {
|
||
progressCalled = true
|
||
assert.NotNil(t, result)
|
||
}
|
||
|
||
// 验证回调函数签名正确
|
||
assert.NotNil(t, progressCallback)
|
||
|
||
// 模拟调用
|
||
testResult := &model.ValidationResult{
|
||
Code: 100,
|
||
Msg: "Processing",
|
||
Progress: "50%",
|
||
}
|
||
progressCallback(testResult)
|
||
assert.True(t, progressCalled)
|
||
})
|
||
}
|
||
|
||
// TestValidateRecordSync_ProgressCallback 测试记录验证的进度回调
|
||
func TestValidateRecordSync_ProgressCallback(t *testing.T) {
|
||
t.Run("verify record progress callback", func(t *testing.T) {
|
||
called := false
|
||
callback := func(result *model.RecordValidationResult) {
|
||
called = true
|
||
assert.NotNil(t, result)
|
||
}
|
||
|
||
testResult := &model.RecordValidationResult{
|
||
Code: 100,
|
||
Msg: "Processing",
|
||
Progress: "50%",
|
||
}
|
||
callback(testResult)
|
||
assert.True(t, called)
|
||
})
|
||
}
|
||
|
||
// TestClient_MultipleCallsToClose 测试多次调用 Close
|
||
func TestClient_MultipleCallsToClose(t *testing.T) {
|
||
t.Skip("Requires actual gRPC setup")
|
||
// 这个测试需要实际的 gRPC 连接来验证幂等性
|
||
}
|
||
|
||
// TestResponseConversion 测试响应转换逻辑
|
||
func TestResponseConversion(t *testing.T) {
|
||
t.Run("operation response with nil timestamp", func(t *testing.T) {
|
||
pbOp := &pb.OperationData{
|
||
OpId: "test",
|
||
OpSource: "api",
|
||
OpType: "create",
|
||
// Timestamp: nil - 这应该导致转换失败
|
||
}
|
||
|
||
// 验证会失败因为缺少必需字段
|
||
_, err := model.FromProtobuf(pbOp)
|
||
assert.Error(t, err)
|
||
})
|
||
|
||
t.Run("operation response with valid data", func(t *testing.T) {
|
||
pbOp := &pb.OperationData{
|
||
OpId: "test",
|
||
OpSource: "api",
|
||
OpType: "create",
|
||
Timestamp: timestamppb.Now(),
|
||
}
|
||
|
||
op, err := model.FromProtobuf(pbOp)
|
||
require.NoError(t, err)
|
||
assert.NotNil(t, op)
|
||
assert.Equal(t, "test", op.OpID)
|
||
})
|
||
}
|
||
|
||
// TestValidationResult_States 测试验证结果的状态
|
||
func TestValidationResult_States(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
result *model.ValidationResult
|
||
isCompleted bool
|
||
isFailed bool
|
||
}{
|
||
{
|
||
name: "completed",
|
||
result: &model.ValidationResult{
|
||
Code: 200,
|
||
Msg: "Completed",
|
||
},
|
||
isCompleted: true,
|
||
isFailed: false,
|
||
},
|
||
{
|
||
name: "failed",
|
||
result: &model.ValidationResult{
|
||
Code: 500,
|
||
Msg: "Failed",
|
||
},
|
||
isCompleted: false,
|
||
isFailed: true,
|
||
},
|
||
{
|
||
name: "in progress",
|
||
result: &model.ValidationResult{
|
||
Code: 100,
|
||
Msg: "Processing",
|
||
Progress: "50%",
|
||
},
|
||
isCompleted: false,
|
||
isFailed: false,
|
||
},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
assert.Equal(t, tt.isCompleted, tt.result.IsCompleted())
|
||
assert.Equal(t, tt.isFailed, tt.result.IsFailed())
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestRecordValidationResult_States 测试记录验证结果的状态
|
||
func TestRecordValidationResult_States(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
result *model.RecordValidationResult
|
||
isCompleted bool
|
||
isFailed bool
|
||
}{
|
||
{
|
||
name: "completed",
|
||
result: &model.RecordValidationResult{
|
||
Code: 200,
|
||
Msg: "Completed",
|
||
},
|
||
isCompleted: true,
|
||
isFailed: false,
|
||
},
|
||
{
|
||
name: "failed",
|
||
result: &model.RecordValidationResult{
|
||
Code: 500,
|
||
Msg: "Failed",
|
||
},
|
||
isCompleted: false,
|
||
isFailed: true,
|
||
},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
assert.Equal(t, tt.isCompleted, tt.result.IsCompleted())
|
||
assert.Equal(t, tt.isFailed, tt.result.IsFailed())
|
||
})
|
||
}
|
||
}
|
||
|
||
// TestClient_GetLowLevel 测试获取底层客户端
|
||
func TestClient_GetLowLevel(t *testing.T) {
|
||
t.Skip("Requires actual gRPC setup")
|
||
// 需要实际的 gRPC 连接来测试 GetLowLevelOperationClient 和 GetLowLevelRecordClient
|
||
}
|
||
|
||
// TestListOperationsResponse_Structure 测试响应结构
|
||
func TestListOperationsResponse_Structure(t *testing.T) {
|
||
resp := &queryclient.ListOperationsResponse{
|
||
Count: 10,
|
||
Data: []*model.Operation{},
|
||
}
|
||
|
||
assert.Equal(t, int64(10), resp.Count)
|
||
assert.NotNil(t, resp.Data)
|
||
assert.Len(t, resp.Data, 0)
|
||
}
|
||
|
||
// TestListRecordsResponse_Structure 测试记录响应结构
|
||
func TestListRecordsResponse_Structure(t *testing.T) {
|
||
resp := &queryclient.ListRecordsResponse{
|
||
Count: 5,
|
||
Data: []*model.Record{},
|
||
}
|
||
|
||
assert.Equal(t, int64(5), resp.Count)
|
||
assert.NotNil(t, resp.Data)
|
||
assert.Len(t, resp.Data, 0)
|
||
}
|
||
|