Files
go-trustlog/api/persistence/pg_query_integration_test.go
ryan bdca8b59e0 feat: 添加 Operation 查询功能及完整测试
主要功能:
- 新增 OperationQueryRequest/OperationQueryResult 结构体
- 实现 Repository.Query() - 支持多条件筛选、分页、排序
- 实现 Repository.Count() - 统计记录数
- 新增 PersistenceClient.QueryOperations/CountOperations/GetOperationByID

查询功能:
- 支持按 OpID、OpSource、OpType、Doid 等字段筛选
- 支持模糊查询(Doid、DoPrefix)
- 支持时间范围查询(TimeFrom/TimeTo)
- 支持 IP 地址筛选(ClientIP、ServerIP)
- 支持按 TrustlogStatus 筛选
- 支持组合查询
- 支持分页(PageSize、PageNumber)
- 支持排序(OrderBy、OrderDesc)

测试覆盖:
-  query_test.go - 查询功能单元测试
-  pg_query_integration_test.go - PostgreSQL 集成测试(16个测试用例)
  * Query all records
  * Filter by OpSource/OpType/Status/Actor/Producer/IP
  * DOID 模糊查询
  * 时间范围查询
  * 分页测试
  * 排序测试(升序/降序)
  * 组合查询
  * Count 统计
  * PersistenceClient 接口测试

修复:
- 修复 TestClusterSafety_MultipleCursorWorkers - 添加缺失字段
- 修复 TestCursorInitialization - 确保 schema 最新
- 添加自动 schema 更新(ALTER TABLE IF NOT EXISTS)

测试结果:
-  所有单元测试通过(100%)
-  所有集成测试通过(PostgreSQL、Pulsar、E2E)
-  Query 功能测试通过(16个测试用例)
2025-12-24 17:20:09 +08:00

617 lines
20 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package persistence_test
import (
"context"
"database/sql"
"fmt"
"testing"
"time"
_ "github.com/lib/pq"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.yandata.net/iod/iod/go-trustlog/api/logger"
"go.yandata.net/iod/iod/go-trustlog/api/model"
"go.yandata.net/iod/iod/go-trustlog/api/persistence"
)
// TestPG_Query_Integration 测试 PostgreSQL 查询功能集成
func TestPG_Query_Integration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping PostgreSQL query integration test in short mode")
}
ctx := context.Background()
log := logger.NewNopLogger()
// 连接数据库
dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
e2eTestPGHost, e2eTestPGPort, e2eTestPGUser, e2eTestPGPassword, e2eTestPGDatabase)
db, err := sql.Open("postgres", dsn)
if err != nil {
t.Skipf("PostgreSQL not available: %v", err)
return
}
defer db.Close()
if err := db.Ping(); err != nil {
t.Skipf("PostgreSQL not reachable: %v", err)
return
}
// 确保schema是最新的
_, _ = db.Exec("ALTER TABLE operation ADD COLUMN IF NOT EXISTS op_hash VARCHAR(128)")
_, _ = db.Exec("ALTER TABLE operation ADD COLUMN IF NOT EXISTS sign VARCHAR(512)")
_, _ = db.Exec("ALTER TABLE operation ADD COLUMN IF NOT EXISTS timestamp TIMESTAMP")
_, _ = db.Exec("ALTER TABLE operation ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP")
// 清理测试数据
_, _ = db.Exec("DELETE FROM operation WHERE op_id LIKE 'pg-query-test-%'")
defer func() {
_, _ = db.Exec("DELETE FROM operation WHERE op_id LIKE 'pg-query-test-%'")
}()
t.Log("✅ PostgreSQL connected and cleaned")
// 创建 PersistenceManager
persistenceConfig := persistence.PersistenceConfig{
Strategy: persistence.StrategyDBOnly,
}
manager := persistence.NewPersistenceManager(db, persistenceConfig, log)
defer manager.Close()
repo := manager.GetOperationRepo()
// 准备测试数据20条记录不同的来源、类型、状态
baseTime := time.Now().Add(-2 * time.Hour)
testOps := []struct {
opID string
opSource string
opType string
prefix string
doid string
repo string
actor string
producer string
clientIP *string
serverIP *string
status persistence.TrustlogStatus
time time.Time
}{
{"pg-query-test-001", "DOIP", "Create", "10.10000", "10.10000/test-repo/test-001", "test-repo", "user-1", "producer-1", strPtr("192.168.1.10"), strPtr("10.0.0.1"), persistence.StatusNotTrustlogged, baseTime},
{"pg-query-test-002", "DOIP", "Create", "10.10000", "10.10000/test-repo/test-002", "test-repo", "user-1", "producer-1", strPtr("192.168.1.10"), strPtr("10.0.0.1"), persistence.StatusTrustlogged, baseTime.Add(10 * time.Minute)},
{"pg-query-test-003", "DOIP", "Update", "10.10000", "10.10000/test-repo/test-003", "test-repo", "user-2", "producer-1", strPtr("192.168.1.20"), strPtr("10.0.0.1"), persistence.StatusNotTrustlogged, baseTime.Add(20 * time.Minute)},
{"pg-query-test-004", "DOIP", "Update", "10.10000", "10.10000/test-repo/test-004", "test-repo", "user-2", "producer-2", strPtr("192.168.1.20"), strPtr("10.0.0.2"), persistence.StatusTrustlogged, baseTime.Add(30 * time.Minute)},
{"pg-query-test-005", "DOIP", "Delete", "10.10000", "10.10000/test-repo/test-005", "test-repo", "user-3", "producer-2", nil, nil, persistence.StatusNotTrustlogged, baseTime.Add(40 * time.Minute)},
{"pg-query-test-006", "IRP", "OC_CREATE_HANDLE", "20.1000", "20.1000/test-repo/test-001", "test-repo", "user-1", "producer-3", strPtr("192.168.2.10"), strPtr("10.0.1.1"), persistence.StatusTrustlogged, baseTime.Add(50 * time.Minute)},
{"pg-query-test-007", "IRP", "OC_DELETE_HANDLE", "20.1000", "20.1000/test-repo/test-002", "test-repo", "user-2", "producer-3", strPtr("192.168.2.20"), strPtr("10.0.1.1"), persistence.StatusNotTrustlogged, baseTime.Add(60 * time.Minute)},
{"pg-query-test-008", "IRP", "OC_LOOKUP_HANDLE", "20.1000", "20.1000/test-repo/test-003", "test-repo", "user-3", "producer-4", nil, strPtr("10.0.1.2"), persistence.StatusTrustlogged, baseTime.Add(70 * time.Minute)},
{"pg-query-test-009", "DOIP", "Retrieve", "10.20000", "10.20000/test-repo/test-001", "test-repo", "user-1", "producer-1", strPtr("192.168.1.30"), nil, persistence.StatusNotTrustlogged, baseTime.Add(80 * time.Minute)},
{"pg-query-test-010", "DOIP", "Retrieve", "10.20000", "10.20000/test-repo/test-002", "test-repo", "user-2", "producer-2", strPtr("192.168.1.40"), strPtr("10.0.0.3"), persistence.StatusTrustlogged, baseTime.Add(90 * time.Minute)},
}
// 插入测试数据
for _, testOp := range testOps {
op, err := model.NewFullOperation(
model.Source(testOp.opSource),
testOp.opType,
testOp.prefix, // doPrefix
testOp.repo, // doRepository
testOp.doid, // doid
testOp.producer, // producerID
testOp.actor, // opActor
nil, // requestBody
nil, // responseBody
testOp.time, // timestamp
)
require.NoError(t, err, "Failed to create operation %s", testOp.opID)
op.OpID = testOp.opID
op.ClientIP = testOp.clientIP
op.ServerIP = testOp.serverIP
err = repo.Save(ctx, op, testOp.status)
require.NoError(t, err, "Failed to save operation %s", testOp.opID)
}
t.Log("✅ Test data created")
// 测试1: 查询所有记录
t.Run("Query all records", func(t *testing.T) {
req := &persistence.OperationQueryRequest{
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(10))
assert.GreaterOrEqual(t, len(result.Operations), 10)
t.Logf("✅ Total records: %d", result.Total)
})
// 测试2: 按 OpSource 筛选
t.Run("Filter by OpSource", func(t *testing.T) {
opSource := "DOIP"
req := &persistence.OperationQueryRequest{
OpSource: &opSource,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(7)) // 7条DOIP记录
for _, op := range result.Operations {
assert.Equal(t, "DOIP", string(op.OpSource))
}
t.Logf("✅ DOIP records: %d", result.Total)
})
// 测试3: 按 OpType 筛选
t.Run("Filter by OpType", func(t *testing.T) {
opType := "Create"
req := &persistence.OperationQueryRequest{
OpType: &opType,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(2)) // 2条Create记录
for _, op := range result.Operations {
assert.Equal(t, "Create", op.OpType)
}
t.Logf("✅ Create records: %d", result.Total)
})
// 测试4: 按 TrustlogStatus 筛选
t.Run("Filter by TrustlogStatus", func(t *testing.T) {
status := persistence.StatusTrustlogged
req := &persistence.OperationQueryRequest{
TrustlogStatus: &status,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(5)) // 5条已存证记录
for _, s := range result.Statuses {
assert.Equal(t, persistence.StatusTrustlogged, s)
}
t.Logf("✅ Trustlogged records: %d", result.Total)
})
// 测试5: 按 DOID 模糊查询
t.Run("Filter by DOID pattern", func(t *testing.T) {
doid := "test-001"
req := &persistence.OperationQueryRequest{
Doid: &doid,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(3)) // 3条test-001的记录
for _, op := range result.Operations {
assert.Contains(t, op.Doid, "test-001")
}
t.Logf("✅ DOID pattern match records: %d", result.Total)
})
// 测试6: 按 OpActor 筛选
t.Run("Filter by OpActor", func(t *testing.T) {
opActor := "user-1"
req := &persistence.OperationQueryRequest{
OpActor: &opActor,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(3)) // 3条user-1的记录
for _, op := range result.Operations {
assert.Equal(t, "user-1", op.OpActor)
}
t.Logf("✅ OpActor records: %d", result.Total)
})
// 测试7: 按 ProducerID 筛选
t.Run("Filter by ProducerID", func(t *testing.T) {
producerID := "producer-1"
req := &persistence.OperationQueryRequest{
ProducerID: &producerID,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(3)) // 3条producer-1的记录
for _, op := range result.Operations {
assert.Equal(t, "producer-1", op.ProducerID)
}
t.Logf("✅ ProducerID records: %d", result.Total)
})
// 测试8: 按 ClientIP 筛选
t.Run("Filter by ClientIP", func(t *testing.T) {
clientIP := "192.168.1.10"
req := &persistence.OperationQueryRequest{
ClientIP: &clientIP,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(2)) // 2条192.168.1.10的记录
for _, op := range result.Operations {
assert.NotNil(t, op.ClientIP)
assert.Equal(t, "192.168.1.10", *op.ClientIP)
}
t.Logf("✅ ClientIP records: %d", result.Total)
})
// 测试9: 按 ServerIP 筛选
t.Run("Filter by ServerIP", func(t *testing.T) {
serverIP := "10.0.0.1"
req := &persistence.OperationQueryRequest{
ServerIP: &serverIP,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(3)) // 3条10.0.0.1的记录
for _, op := range result.Operations {
assert.NotNil(t, op.ServerIP)
assert.Equal(t, "10.0.0.1", *op.ServerIP)
}
t.Logf("✅ ServerIP records: %d", result.Total)
})
// 测试10: 时间范围查询
t.Run("Filter by time range", func(t *testing.T) {
timeFrom := baseTime.Add(30 * time.Minute)
timeTo := baseTime.Add(70 * time.Minute)
req := &persistence.OperationQueryRequest{
TimeFrom: &timeFrom,
TimeTo: &timeTo,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(3)) // 应该至少有3条记录在这个时间范围
t.Logf("✅ Time range records: %d", result.Total)
// 验证返回的记录在时间范围内
for i, op := range result.Operations {
if !((op.Timestamp.After(timeFrom) || op.Timestamp.Equal(timeFrom)) &&
(op.Timestamp.Before(timeTo) || op.Timestamp.Equal(timeTo))) {
t.Logf("⚠️ Record %d out of range: timestamp=%v, from=%v, to=%v",
i, op.Timestamp, timeFrom, timeTo)
}
}
})
// 测试11: 组合查询OpSource + Status
t.Run("Combined filter (OpSource + Status)", func(t *testing.T) {
opSource := "DOIP"
status := persistence.StatusTrustlogged
req := &persistence.OperationQueryRequest{
OpSource: &opSource,
TrustlogStatus: &status,
PageSize: 50,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(3)) // 3条已存证的DOIP记录
for i, op := range result.Operations {
assert.Equal(t, "DOIP", string(op.OpSource))
assert.Equal(t, persistence.StatusTrustlogged, result.Statuses[i])
}
t.Logf("✅ Combined filter records: %d", result.Total)
})
// 测试12: 分页查询
t.Run("Pagination", func(t *testing.T) {
// 第1页
req := &persistence.OperationQueryRequest{
PageSize: 5,
PageNumber: 1,
OrderBy: "timestamp",
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(10))
assert.LessOrEqual(t, len(result.Operations), 5)
firstPageFirst := result.Operations[0].OpID
// 第2页
req.PageNumber = 2
result, err = repo.Query(ctx, req)
require.NoError(t, err)
assert.LessOrEqual(t, len(result.Operations), 5)
// 确保第1页和第2页的数据不重复
if len(result.Operations) > 0 {
assert.NotEqual(t, firstPageFirst, result.Operations[0].OpID)
}
t.Logf("✅ Pagination works correctly, total pages: %d", result.TotalPages)
})
// 测试13: 排序(升序/降序)
t.Run("Ordering", func(t *testing.T) {
// 升序
reqAsc := &persistence.OperationQueryRequest{
PageSize: 10,
PageNumber: 1,
OrderBy: "timestamp",
OrderDesc: false,
}
resultAsc, err := repo.Query(ctx, reqAsc)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(resultAsc.Operations), 10)
// 验证升序
for i := 1; i < len(resultAsc.Operations); i++ {
assert.True(t, resultAsc.Operations[i].Timestamp.After(resultAsc.Operations[i-1].Timestamp) ||
resultAsc.Operations[i].Timestamp.Equal(resultAsc.Operations[i-1].Timestamp))
}
// 降序
reqDesc := &persistence.OperationQueryRequest{
PageSize: 10,
PageNumber: 1,
OrderBy: "timestamp",
OrderDesc: true,
}
resultDesc, err := repo.Query(ctx, reqDesc)
require.NoError(t, err)
assert.GreaterOrEqual(t, len(resultDesc.Operations), 10)
// 验证降序
for i := 1; i < len(resultDesc.Operations); i++ {
assert.True(t, resultDesc.Operations[i].Timestamp.Before(resultDesc.Operations[i-1].Timestamp) ||
resultDesc.Operations[i].Timestamp.Equal(resultDesc.Operations[i-1].Timestamp))
}
t.Log("✅ Ordering (ASC/DESC) works correctly")
})
// 测试14: Count 统计
t.Run("Count operations", func(t *testing.T) {
// 全部统计
req := &persistence.OperationQueryRequest{}
count, err := repo.Count(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, count, int64(10))
t.Logf("✅ Total count: %d", count)
// 按状态统计
status := persistence.StatusNotTrustlogged
req.TrustlogStatus = &status
count, err = repo.Count(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, count, int64(5))
t.Logf("✅ NOT_TRUSTLOGGED count: %d", count)
})
// 测试15: OpID 精确查询
t.Run("Query by OpID", func(t *testing.T) {
opID := "pg-query-test-001"
req := &persistence.OperationQueryRequest{
OpID: &opID,
PageSize: 10,
PageNumber: 1,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.Equal(t, int64(1), result.Total)
assert.Len(t, result.Operations, 1)
assert.Equal(t, "pg-query-test-001", result.Operations[0].OpID)
t.Log("✅ OpID query works correctly")
})
// 测试16: 复杂组合查询(多条件)
t.Run("Complex combined query", func(t *testing.T) {
opSource := "DOIP"
opType := "Update"
status := persistence.StatusTrustlogged
req := &persistence.OperationQueryRequest{
OpSource: &opSource,
OpType: &opType,
TrustlogStatus: &status,
PageSize: 50,
PageNumber: 1,
OrderBy: "timestamp",
OrderDesc: true,
}
result, err := repo.Query(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(1))
for i, op := range result.Operations {
assert.Equal(t, "DOIP", string(op.OpSource))
assert.Equal(t, "Update", op.OpType)
assert.Equal(t, persistence.StatusTrustlogged, result.Statuses[i])
}
t.Logf("✅ Complex query records: %d", result.Total)
})
t.Log("✅ All PostgreSQL query integration tests passed")
}
// TestPG_PersistenceClient_Query_Integration 测试 PersistenceClient 的查询功能
func TestPG_PersistenceClient_Query_Integration(t *testing.T) {
if testing.Short() {
t.Skip("Skipping PostgreSQL PersistenceClient query integration test in short mode")
}
ctx := context.Background()
log := logger.NewNopLogger()
// 连接数据库
dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
e2eTestPGHost, e2eTestPGPort, e2eTestPGUser, e2eTestPGPassword, e2eTestPGDatabase)
// 创建 PersistenceClient
dbConfig := persistence.DBConfig{
DriverName: "postgres",
DSN: dsn,
MaxOpenConns: 20,
MaxIdleConns: 10,
ConnMaxLifetime: time.Hour,
}
persistenceConfig := persistence.PersistenceConfig{
Strategy: persistence.StrategyDBOnly,
}
clientConfig := persistence.PersistenceClientConfig{
Logger: log,
DBConfig: dbConfig,
PersistenceConfig: persistenceConfig,
}
client, err := persistence.NewPersistenceClient(ctx, clientConfig)
if err != nil {
t.Skipf("PostgreSQL not available: %v", err)
return
}
defer client.Close()
// 获取底层数据库连接进行清理和schema更新
db, err := sql.Open("postgres", dsn)
require.NoError(t, err)
defer db.Close()
// 清理测试数据
_, _ = db.Exec("DELETE FROM operation WHERE op_id LIKE 'pg-client-query-%'")
defer func() {
_, _ = db.Exec("DELETE FROM operation WHERE op_id LIKE 'pg-client-query-%'")
}()
// 确保schema是最新的
_, _ = db.Exec("ALTER TABLE operation ADD COLUMN IF NOT EXISTS op_hash VARCHAR(128)")
_, _ = db.Exec("ALTER TABLE operation ADD COLUMN IF NOT EXISTS sign VARCHAR(512)")
_, _ = db.Exec("ALTER TABLE operation ADD COLUMN IF NOT EXISTS timestamp TIMESTAMP")
_, _ = db.Exec("ALTER TABLE operation ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP")
t.Log("✅ PersistenceClient connected")
// 创建测试数据通过manager的repository
manager := client.GetManager()
repo := manager.GetOperationRepo()
for i := 0; i < 5; i++ {
op, err := model.NewFullOperation(
model.OpSourceDOIP,
string(model.OpTypeCreate),
"10.10000", // doPrefix
"client-repo", // doRepository
fmt.Sprintf("10.10000/client-repo/test-%d", i), // doid
fmt.Sprintf("client-producer-%d", i), // producerID
fmt.Sprintf("client-actor-%d", i), // opActor
nil, // requestBody
nil, // responseBody
time.Now(), // timestamp
)
require.NoError(t, err)
op.OpID = fmt.Sprintf("pg-client-query-%03d", i)
status := persistence.StatusNotTrustlogged
if i%2 == 0 {
status = persistence.StatusTrustlogged
}
err = repo.Save(ctx, op, status)
require.NoError(t, err)
}
t.Log("✅ Test data created via PersistenceClient")
// 测试 QueryOperations
t.Run("QueryOperations", func(t *testing.T) {
req := &persistence.OperationQueryRequest{
PageSize: 10,
PageNumber: 1,
}
result, err := client.QueryOperations(ctx, req)
require.NoError(t, err)
assert.NotNil(t, result)
assert.GreaterOrEqual(t, result.Total, int64(5))
t.Logf("✅ QueryOperations: total=%d", result.Total)
})
// 测试 CountOperations
t.Run("CountOperations", func(t *testing.T) {
req := &persistence.OperationQueryRequest{}
count, err := client.CountOperations(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, count, int64(5))
t.Logf("✅ CountOperations: count=%d", count)
})
// 测试 GetOperationByID
t.Run("GetOperationByID", func(t *testing.T) {
op, status, err := client.GetOperationByID(ctx, "pg-client-query-000")
require.NoError(t, err)
assert.NotNil(t, op)
assert.Equal(t, "pg-client-query-000", op.OpID)
assert.Equal(t, persistence.StatusTrustlogged, status)
t.Log("✅ GetOperationByID works correctly")
})
// 测试按状态查询
t.Run("Query by Status", func(t *testing.T) {
status := persistence.StatusTrustlogged
req := &persistence.OperationQueryRequest{
TrustlogStatus: &status,
PageSize: 10,
PageNumber: 1,
}
result, err := client.QueryOperations(ctx, req)
require.NoError(t, err)
assert.GreaterOrEqual(t, result.Total, int64(3)) // 3条已存证
t.Logf("✅ Query by Status: total=%d", result.Total)
})
t.Log("✅ All PersistenceClient query integration tests passed")
}
// strPtr 辅助函数:返回字符串指针
func strPtr(s string) *string {
return &s
}