4b72a37120
主要更新： 1. 数据库持久化功能 - 支持三种策略：仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段（可空，仅落库） 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试（含 Pulsar 消费者验证） - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试，提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档，只保留根目录 README 测试结果： - 所有 E2E 测试通过（100%） - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
537 lines
15 KiB
Go
537 lines
15 KiB
Go
package highclient_test
|
||
|
||
import (
|
||
"errors"
|
||
"fmt"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/ThreeDotsLabs/watermill/message"
|
||
"github.com/go-logr/logr"
|
||
"github.com/stretchr/testify/assert"
|
||
"github.com/stretchr/testify/mock"
|
||
"github.com/stretchr/testify/require"
|
||
|
||
"go.yandata.net/iod/iod/go-trustlog/api/adapter"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/highclient"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/logger"
|
||
"go.yandata.net/iod/iod/go-trustlog/api/model"
|
||
)
|
||
|
||
// MockPublisher 模拟 message.Publisher.
|
||
type MockPublisher struct {
|
||
mock.Mock
|
||
}
|
||
|
||
func (m *MockPublisher) Publish(topic string, messages ...*message.Message) error {
|
||
args := m.Called(topic, messages)
|
||
return args.Error(0)
|
||
}
|
||
|
||
func (m *MockPublisher) Close() error {
|
||
args := m.Called()
|
||
return args.Error(0)
|
||
}
|
||
|
||
// generateTestKeys 生成测试用的SM2密钥对（DER格式）.
|
||
func generateTestKeys(t testing.TB) ([]byte, []byte) {
|
||
keyPair, err := model.GenerateSM2KeyPair()
|
||
if err != nil {
|
||
if t != nil {
|
||
require.NoError(t, err)
|
||
} else {
|
||
panic(err)
|
||
}
|
||
}
|
||
|
||
// 私钥：DER编码
|
||
privateKeyDER, err := model.MarshalSM2PrivateDER(keyPair.Private)
|
||
if err != nil {
|
||
if t != nil {
|
||
require.NoError(t, err)
|
||
} else {
|
||
panic(err)
|
||
}
|
||
}
|
||
|
||
// 公钥：DER编码
|
||
publicKeyDER, err := model.MarshalSM2PublicDER(keyPair.Public)
|
||
if err != nil {
|
||
if t != nil {
|
||
require.NoError(t, err)
|
||
} else {
|
||
panic(err)
|
||
}
|
||
}
|
||
|
||
return privateKeyDER, publicKeyDER
|
||
}
|
||
|
||
func TestNewClient(t *testing.T) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
require.NotNil(t, client)
|
||
assert.Equal(t, mockPublisher, client.GetLow())
|
||
}
|
||
|
||
func TestClient_GetLow(t *testing.T) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
lowLevelPublisher := client.GetLow()
|
||
assert.Equal(t, mockPublisher, lowLevelPublisher)
|
||
}
|
||
|
||
func TestClient_OperationPublish(t *testing.T) { //nolint:dupl // 测试代码中的重复模式是合理的
|
||
tests := []struct {
|
||
name string
|
||
operation *model.Operation
|
||
setupMock func(*MockPublisher)
|
||
wantErr bool
|
||
errContains string
|
||
}{
|
||
{
|
||
name: "成功发布Operation",
|
||
operation: createTestOperation(t),
|
||
setupMock: func(mp *MockPublisher) {
|
||
mp.On("Publish", adapter.OperationTopic, mock.AnythingOfType("[]*message.Message")).Return(nil).Once()
|
||
},
|
||
wantErr: false,
|
||
},
|
||
{
|
||
name: "发布失败",
|
||
operation: createTestOperation(t),
|
||
setupMock: func(mp *MockPublisher) {
|
||
mp.On("Publish", adapter.OperationTopic, mock.AnythingOfType("[]*message.Message")).
|
||
Return(errors.New("publish failed")).
|
||
Once()
|
||
},
|
||
wantErr: true,
|
||
errContains: "publish failed",
|
||
},
|
||
{
|
||
name: "nil Operation应该失败",
|
||
operation: nil,
|
||
setupMock: func(_ *MockPublisher) {
|
||
// nil operation不会调用Publish，因为会在之前失败
|
||
},
|
||
wantErr: true,
|
||
},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
tt.setupMock(mockPublisher)
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
err := client.OperationPublish(tt.operation)
|
||
if tt.wantErr {
|
||
require.Error(t, err)
|
||
if tt.errContains != "" {
|
||
assert.Contains(t, err.Error(), tt.errContains)
|
||
}
|
||
} else {
|
||
require.NoError(t, err)
|
||
}
|
||
|
||
mockPublisher.AssertExpectations(t)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestClient_RecordPublish(t *testing.T) { //nolint:dupl // 测试代码中的重复模式是合理的
|
||
tests := []struct {
|
||
name string
|
||
record *model.Record
|
||
setupMock func(*MockPublisher)
|
||
wantErr bool
|
||
errContains string
|
||
}{
|
||
{
|
||
name: "成功发布Record",
|
||
record: createTestRecord(t),
|
||
setupMock: func(mp *MockPublisher) {
|
||
mp.On("Publish", adapter.RecordTopic, mock.AnythingOfType("[]*message.Message")).Return(nil).Once()
|
||
},
|
||
wantErr: false,
|
||
},
|
||
{
|
||
name: "发布失败",
|
||
record: createTestRecord(t),
|
||
setupMock: func(mp *MockPublisher) {
|
||
mp.On("Publish", adapter.RecordTopic, mock.AnythingOfType("[]*message.Message")).
|
||
Return(errors.New("record publish failed")).
|
||
Once()
|
||
},
|
||
wantErr: true,
|
||
errContains: "record publish failed",
|
||
},
|
||
{
|
||
name: "nil Record应该失败",
|
||
record: nil,
|
||
setupMock: func(_ *MockPublisher) {
|
||
// nil record不会调用Publish，因为会在之前失败
|
||
},
|
||
wantErr: true,
|
||
},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
tt.setupMock(mockPublisher)
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
err := client.RecordPublish(tt.record)
|
||
if tt.wantErr {
|
||
require.Error(t, err)
|
||
if tt.errContains != "" {
|
||
assert.Contains(t, err.Error(), tt.errContains)
|
||
}
|
||
} else {
|
||
require.NoError(t, err)
|
||
}
|
||
|
||
mockPublisher.AssertExpectations(t)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestClient_Close(t *testing.T) {
|
||
tests := []struct {
|
||
name string
|
||
setupMock func(*MockPublisher)
|
||
wantErr bool
|
||
errContains string
|
||
}{
|
||
{
|
||
name: "成功关闭",
|
||
setupMock: func(mp *MockPublisher) {
|
||
mp.On("Close").Return(nil).Once()
|
||
},
|
||
wantErr: false,
|
||
},
|
||
{
|
||
name: "关闭失败",
|
||
setupMock: func(mp *MockPublisher) {
|
||
mp.On("Close").Return(errors.New("close failed")).Once()
|
||
},
|
||
wantErr: true,
|
||
errContains: "close failed",
|
||
},
|
||
}
|
||
|
||
for _, tt := range tests {
|
||
t.Run(tt.name, func(t *testing.T) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
tt.setupMock(mockPublisher)
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
err := client.Close()
|
||
|
||
if tt.wantErr {
|
||
require.Error(t, err)
|
||
if tt.errContains != "" {
|
||
assert.Contains(t, err.Error(), tt.errContains)
|
||
}
|
||
} else {
|
||
require.NoError(t, err)
|
||
}
|
||
|
||
mockPublisher.AssertExpectations(t)
|
||
})
|
||
}
|
||
}
|
||
|
||
func TestClient_MessageContent(t *testing.T) {
|
||
// 测试发布的消息内容是否正确
|
||
t.Run("Operation消息内容验证", func(t *testing.T) { //nolint:dupl // 测试代码中的重复模式是合理的
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
operation := createTestOperation(t)
|
||
|
||
// 捕获发布的消息
|
||
var capturedMessages []*message.Message
|
||
mockPublisher.On("Publish", adapter.OperationTopic, mock.AnythingOfType("[]*message.Message")).
|
||
Run(func(args mock.Arguments) {
|
||
messages, ok := args.Get(1).([]*message.Message)
|
||
if ok {
|
||
capturedMessages = messages
|
||
}
|
||
}).Return(nil).Once()
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
err := client.OperationPublish(operation)
|
||
require.NoError(t, err)
|
||
|
||
// 验证消息内容
|
||
require.Len(t, capturedMessages, 1)
|
||
msg := capturedMessages[0]
|
||
assert.Equal(t, operation.Key(), msg.UUID)
|
||
assert.NotEmpty(t, msg.Payload)
|
||
|
||
// 验证是Envelope格式，可以反序列化
|
||
unmarshaledOp, err := model.UnmarshalOperation(msg.Payload)
|
||
require.NoError(t, err)
|
||
assert.Equal(t, operation.OpID, unmarshaledOp.OpID)
|
||
|
||
// 验证签名
|
||
verifyConfig := model.NewSM2VerifyConfig(publicKey)
|
||
verifiedEnv, err := model.VerifyEnvelopeWithConfig(msg.Payload, verifyConfig)
|
||
require.NoError(t, err)
|
||
assert.NotNil(t, verifiedEnv)
|
||
})
|
||
|
||
t.Run("Record消息内容验证", func(t *testing.T) { //nolint:dupl // 测试代码中的重复模式是合理的
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
record := createTestRecord(t)
|
||
|
||
// 捕获发布的消息
|
||
var capturedMessages []*message.Message
|
||
mockPublisher.On("Publish", adapter.RecordTopic, mock.AnythingOfType("[]*message.Message")).
|
||
Run(func(args mock.Arguments) {
|
||
messages, ok := args.Get(1).([]*message.Message)
|
||
if ok {
|
||
capturedMessages = messages
|
||
}
|
||
}).Return(nil).Once()
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
err := client.RecordPublish(record)
|
||
require.NoError(t, err)
|
||
|
||
// 验证消息内容
|
||
require.Len(t, capturedMessages, 1)
|
||
msg := capturedMessages[0]
|
||
assert.Equal(t, record.Key(), msg.UUID)
|
||
assert.NotEmpty(t, msg.Payload)
|
||
|
||
// 验证是Envelope格式，可以反序列化
|
||
unmarshaledRecord, err := model.UnmarshalRecord(msg.Payload)
|
||
require.NoError(t, err)
|
||
assert.Equal(t, record.ID, unmarshaledRecord.ID)
|
||
|
||
// 验证签名
|
||
verifyConfig := model.NewSM2VerifyConfig(publicKey)
|
||
verifiedEnv, err := model.VerifyEnvelopeWithConfig(msg.Payload, verifyConfig)
|
||
require.NoError(t, err)
|
||
assert.NotNil(t, verifiedEnv)
|
||
})
|
||
}
|
||
|
||
func TestClient_ConcurrentPublish(t *testing.T) {
|
||
// 测试并发发布
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
|
||
// 设置期望的调用次数
|
||
publishCount := 100
|
||
mockPublisher.On("Publish", adapter.OperationTopic, mock.AnythingOfType("[]*message.Message")).
|
||
Return(nil).Times(publishCount)
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
// 并发发布
|
||
errChan := make(chan error, publishCount)
|
||
for i := range publishCount {
|
||
go func(id int) {
|
||
//nolint:testifylint // 在goroutine中创建测试数据，使用panic处理错误
|
||
operation := createTestOperationWithID(nil, fmt.Sprintf("concurrent-test-%d", id))
|
||
errChan <- client.OperationPublish(operation)
|
||
}(i)
|
||
}
|
||
|
||
// 收集结果
|
||
for range publishCount {
|
||
err := <-errChan
|
||
require.NoError(t, err)
|
||
}
|
||
|
||
mockPublisher.AssertExpectations(t)
|
||
}
|
||
|
||
func TestClient_EdgeCases(t *testing.T) {
|
||
t.Run("发布大型Operation", func(t *testing.T) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
mockPublisher.On("Publish", adapter.OperationTopic, mock.AnythingOfType("[]*message.Message")).
|
||
Return(nil).
|
||
Once()
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
// 创建包含大量数据的Operation
|
||
operation := createTestOperation(t)
|
||
operation.OpActor = string(make([]byte, 1000)) // 1KB数据
|
||
|
||
err := client.OperationPublish(operation)
|
||
require.NoError(t, err)
|
||
|
||
mockPublisher.AssertExpectations(t)
|
||
})
|
||
|
||
t.Run("发布大型Record", func(t *testing.T) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
mockPublisher.On("Publish", adapter.RecordTopic, mock.AnythingOfType("[]*message.Message")).Return(nil).Once()
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
// 创建包含大量数据的Record
|
||
record := createTestRecord(t)
|
||
record.WithExtra(make([]byte, 500)) // 500字节的额外数据
|
||
|
||
err := client.RecordPublish(record)
|
||
require.NoError(t, err)
|
||
|
||
mockPublisher.AssertExpectations(t)
|
||
})
|
||
}
|
||
|
||
func TestClient_Integration(t *testing.T) {
|
||
// 集成测试 - 测试完整的工作流程
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(t)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
|
||
// 设置期望：发布Operation -> 发布Record -> 关闭
|
||
mockPublisher.On("Publish", adapter.OperationTopic, mock.AnythingOfType("[]*message.Message")).Return(nil).Once()
|
||
mockPublisher.On("Publish", adapter.RecordTopic, mock.AnythingOfType("[]*message.Message")).Return(nil).Once()
|
||
mockPublisher.On("Close").Return(nil).Once()
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
|
||
// 发布Operation
|
||
operation := createTestOperation(t)
|
||
err := client.OperationPublish(operation)
|
||
require.NoError(t, err)
|
||
|
||
// 发布Record
|
||
record := createTestRecord(t)
|
||
err = client.RecordPublish(record)
|
||
require.NoError(t, err)
|
||
|
||
// 关闭客户端
|
||
err = client.Close()
|
||
require.NoError(t, err)
|
||
|
||
mockPublisher.AssertExpectations(t)
|
||
}
|
||
|
||
// 性能基准测试.
|
||
func BenchmarkClient_OperationPublish(b *testing.B) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(b)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
mockPublisher.On("Publish", adapter.OperationTopic, mock.AnythingOfType("[]*message.Message")).Return(nil)
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
operation := createTestOperation(b)
|
||
|
||
b.ResetTimer()
|
||
for range b.N {
|
||
err := client.OperationPublish(operation)
|
||
if err != nil {
|
||
b.Fatal(err)
|
||
}
|
||
}
|
||
}
|
||
|
||
func BenchmarkClient_RecordPublish(b *testing.B) {
|
||
mockPublisher := &MockPublisher{}
|
||
testLogger := logger.NewLogger(logr.Discard())
|
||
privateKey, publicKey := generateTestKeys(b)
|
||
config := model.NewSM2EnvelopeConfig(privateKey, publicKey)
|
||
mockPublisher.On("Publish", adapter.RecordTopic, mock.AnythingOfType("[]*message.Message")).Return(nil)
|
||
|
||
client := highclient.NewClient(mockPublisher, testLogger, config)
|
||
record := createTestRecord(b)
|
||
|
||
b.ResetTimer()
|
||
for range b.N {
|
||
err := client.RecordPublish(record)
|
||
if err != nil {
|
||
b.Fatal(err)
|
||
}
|
||
}
|
||
}
|
||
|
||
// 测试辅助函数.
|
||
func createTestOperation(t testing.TB) *model.Operation {
|
||
return createTestOperationWithID(t, "test-operation-001")
|
||
}
|
||
|
||
func createTestOperationWithID(t testing.TB, id string) *model.Operation {
|
||
// 在并发测试中，t可能为nil，这是正常的
|
||
errorHandler := func(err error) {
|
||
if t != nil {
|
||
require.NoError(t, err)
|
||
} else if err != nil {
|
||
panic(err)
|
||
}
|
||
}
|
||
operation, err := model.NewFullOperation(
|
||
model.OpSourceDOIP,
|
||
model.OpTypeRetrieve,
|
||
"test-prefix",
|
||
"test-repo",
|
||
"test-prefix/test-repo/test-object",
|
||
"test-producer-id",
|
||
"test-actor",
|
||
"test request body",
|
||
"test response body",
|
||
time.Now(),
|
||
)
|
||
errorHandler(err)
|
||
operation.OpID = id // 设置自定义ID
|
||
return operation
|
||
}
|
||
|
||
func createTestRecord(t testing.TB) *model.Record {
|
||
record, err := model.NewFullRecord(
|
||
"test-prefix",
|
||
"test-producer-id",
|
||
time.Now(),
|
||
"test-operator",
|
||
[]byte("test extra data"),
|
||
"test-type",
|
||
)
|
||
require.NoError(t, err)
|
||
return record
|
||
}
|