feat: 完善数据库持久化与存证功能
主要更新： 1. 数据库持久化功能 - 支持三种策略：仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段（可空，仅落库） 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试（含 Pulsar 消费者验证） - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试，提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档，只保留根目录 README 测试结果： - 所有 E2E 测试通过（100%） - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
This commit is contained in:
ryan
367
api/persistence/manager_test.go
Normal file
367
api/persistence/manager_test.go
Normal file
@@ -0,0 +1,367 @@
|
||||
package persistence
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
|
||||
"go.yandata.net/iod/iod/go-trustlog/api/logger"
|
||||
)
|
||||
|
||||
func TestPersistenceManager_DBOnly(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
log := logger.GetGlobalLogger()
|
||||
|
||||
config := PersistenceConfig{
|
||||
Strategy: StrategyDBOnly,
|
||||
}
|
||||
|
||||
manager := NewPersistenceManager(db, config, log)
|
||||
if manager == nil {
|
||||
t.Fatal("failed to create PersistenceManager")
|
||||
}
|
||||
|
||||
op := createTestOperation(t, "manager-test-001")
|
||||
|
||||
err := manager.SaveOperation(ctx, op)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to save operation: %v", err)
|
||||
}
|
||||
|
||||
// 验证已保存到数据库
|
||||
var count int
|
||||
err = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation WHERE op_id = ?", "manager-test-001").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to query database: %v", err)
|
||||
}
|
||||
|
||||
if count != 1 {
|
||||
t.Errorf("expected 1 record, got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistenceManager_TrustlogOnly(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
log := logger.GetGlobalLogger()
|
||||
|
||||
config := PersistenceConfig{
|
||||
Strategy: StrategyTrustlogOnly,
|
||||
}
|
||||
|
||||
manager := NewPersistenceManager(db, config, log)
|
||||
if manager == nil {
|
||||
t.Fatal("failed to create PersistenceManager")
|
||||
}
|
||||
|
||||
op := createTestOperation(t, "manager-test-002")
|
||||
|
||||
err := manager.SaveOperation(ctx, op)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to save operation: %v", err)
|
||||
}
|
||||
|
||||
// TrustlogOnly 不会保存到数据库，应该查不到
|
||||
var count int
|
||||
err = db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation WHERE op_id = ?", "manager-test-002").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to query database: %v", err)
|
||||
}
|
||||
|
||||
if count != 0 {
|
||||
t.Errorf("expected 0 records (TrustlogOnly should not save to DB), got %d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistenceManager_DBAndTrustlog(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
log := logger.GetGlobalLogger()
|
||||
|
||||
config := PersistenceConfig{
|
||||
Strategy: StrategyDBAndTrustlog,
|
||||
}
|
||||
|
||||
manager := NewPersistenceManager(db, config, log)
|
||||
if manager == nil {
|
||||
t.Fatal("failed to create PersistenceManager")
|
||||
}
|
||||
|
||||
op := createTestOperation(t, "manager-test-003")
|
||||
|
||||
err := manager.SaveOperation(ctx, op)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to save operation: %v", err)
|
||||
}
|
||||
|
||||
// DBAndTrustlog 会保存到数据库，状态为 NOT_TRUSTLOGGED
|
||||
var status string
|
||||
err = db.QueryRowContext(ctx, "SELECT trustlog_status FROM operation WHERE op_id = ?", "manager-test-003").Scan(&status)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to query database: %v", err)
|
||||
}
|
||||
|
||||
if status != "NOT_TRUSTLOGGED" {
|
||||
t.Errorf("expected status to be NOT_TRUSTLOGGED, got %s", status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistenceManager_GetRepositories(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
log := logger.GetGlobalLogger()
|
||||
|
||||
config := PersistenceConfig{
|
||||
Strategy: StrategyDBOnly,
|
||||
}
|
||||
|
||||
manager := NewPersistenceManager(db, config, log)
|
||||
|
||||
// 测试获取各个 Repository
|
||||
opRepo := manager.GetOperationRepo()
|
||||
if opRepo == nil {
|
||||
t.Error("GetOperationRepo returned nil")
|
||||
}
|
||||
|
||||
cursorRepo := manager.GetCursorRepo()
|
||||
if cursorRepo == nil {
|
||||
t.Error("GetCursorRepo returned nil")
|
||||
}
|
||||
|
||||
retryRepo := manager.GetRetryRepo()
|
||||
if retryRepo == nil {
|
||||
t.Error("GetRetryRepo returned nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistenceManager_Close(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
log := logger.GetGlobalLogger()
|
||||
|
||||
config := PersistenceConfig{
|
||||
Strategy: StrategyDBOnly,
|
||||
}
|
||||
|
||||
manager := NewPersistenceManager(db, config, log)
|
||||
|
||||
err := manager.Close()
|
||||
if err != nil {
|
||||
t.Errorf("Close returned error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPersistenceManager_InitSchema(t *testing.T) {
|
||||
// 创建一个空数据库
|
||||
db, err := sql.Open("sqlite3", ":memory:")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to open database: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
log := logger.GetGlobalLogger()
|
||||
|
||||
config := PersistenceConfig{
|
||||
Strategy: StrategyDBOnly,
|
||||
}
|
||||
|
||||
manager := NewPersistenceManager(db, config, log)
|
||||
|
||||
// 手动调用 InitSchema（如果 NewPersistenceManager 没有自动调用）
|
||||
err = manager.InitSchema(context.Background(), "sqlite3")
|
||||
if err != nil {
|
||||
t.Fatalf("InitSchema failed: %v", err)
|
||||
}
|
||||
|
||||
// 验证表已创建
|
||||
var count int
|
||||
err = db.QueryRow("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='operation'").Scan(&count)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to query schema: %v", err)
|
||||
}
|
||||
|
||||
if count != 1 {
|
||||
t.Errorf("expected operation table to exist, got count=%d", count)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperationRepository_SaveTx(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
log := logger.GetGlobalLogger()
|
||||
repo := NewOperationRepository(db, log)
|
||||
|
||||
// 开始事务
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to begin transaction: %v", err)
|
||||
}
|
||||
|
||||
op := createTestOperation(t, "tx-test-001")
|
||||
|
||||
// 在事务中保存
|
||||
err = repo.SaveTx(ctx, tx, op, StatusNotTrustlogged)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
t.Fatalf("failed to save operation in transaction: %v", err)
|
||||
}
|
||||
|
||||
// 提交事务
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to commit transaction: %v", err)
|
||||
}
|
||||
|
||||
// 验证保存成功
|
||||
savedOp, _, err := repo.FindByID(ctx, "tx-test-001")
|
||||
if err != nil {
|
||||
t.Fatalf("failed to find operation: %v", err)
|
||||
}
|
||||
|
||||
if savedOp.OpID != "tx-test-001" {
|
||||
t.Errorf("expected OpID to be 'tx-test-001', got %s", savedOp.OpID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOperationRepository_SaveTxRollback(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
log := logger.GetGlobalLogger()
|
||||
repo := NewOperationRepository(db, log)
|
||||
|
||||
// 开始事务
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to begin transaction: %v", err)
|
||||
}
|
||||
|
||||
op := createTestOperation(t, "tx-test-002")
|
||||
|
||||
// 在事务中保存
|
||||
err = repo.SaveTx(ctx, tx, op, StatusNotTrustlogged)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
t.Fatalf("failed to save operation in transaction: %v", err)
|
||||
}
|
||||
|
||||
// 回滚事务
|
||||
err = tx.Rollback()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to rollback transaction: %v", err)
|
||||
}
|
||||
|
||||
// 验证未保存
|
||||
_, _, err = repo.FindByID(ctx, "tx-test-002")
|
||||
if err == nil {
|
||||
t.Error("expected error when finding rolled back operation, got nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryRepository_AddRetryTx(t *testing.T) {
|
||||
db := setupTestDB(t)
|
||||
defer db.Close()
|
||||
|
||||
ctx := context.Background()
|
||||
log := logger.GetGlobalLogger()
|
||||
repo := NewRetryRepository(db, log)
|
||||
|
||||
// 开始事务
|
||||
tx, err := db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to begin transaction: %v", err)
|
||||
}
|
||||
|
||||
nextRetry := time.Now().Add(-1 * time.Second)
|
||||
err = repo.AddRetryTx(ctx, tx, "tx-retry-001", "test error", nextRetry)
|
||||
if err != nil {
|
||||
tx.Rollback()
|
||||
t.Fatalf("failed to add retry in transaction: %v", err)
|
||||
}
|
||||
|
||||
// 提交事务
|
||||
err = tx.Commit()
|
||||
if err != nil {
|
||||
t.Fatalf("failed to commit transaction: %v", err)
|
||||
}
|
||||
|
||||
// 验证已保存
|
||||
records, err := repo.FindPendingRetries(ctx, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to find pending retries: %v", err)
|
||||
}
|
||||
|
||||
found := false
|
||||
for _, record := range records {
|
||||
if record.OpID == "tx-retry-001" {
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
t.Error("expected to find retry record 'tx-retry-001'")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDialectDDL_AllDrivers(t *testing.T) {
|
||||
drivers := []string{"sqlite3", "postgres", "mysql"}
|
||||
|
||||
for _, driver := range drivers {
|
||||
t.Run(driver, func(t *testing.T) {
|
||||
opDDL, cursorDDL, retryDDL, err := GetDialectDDL(driver)
|
||||
if err != nil {
|
||||
t.Fatalf("GetDialectDDL(%s) failed: %v", driver, err)
|
||||
}
|
||||
|
||||
if opDDL == "" {
|
||||
t.Errorf("opDDL is empty for driver %s", driver)
|
||||
}
|
||||
|
||||
if cursorDDL == "" {
|
||||
t.Errorf("cursorDDL is empty for driver %s", driver)
|
||||
}
|
||||
|
||||
if retryDDL == "" {
|
||||
t.Errorf("retryDDL is empty for driver %s", driver)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDialectDDL_UnknownDriver(t *testing.T) {
|
||||
// GetDialectDDL 对未知驱动返回通用 SQL（而不是错误）
|
||||
opDDL, cursorDDL, retryDDL, err := GetDialectDDL("unknown-driver")
|
||||
if err != nil {
|
||||
t.Fatalf("GetDialectDDL should not error for unknown driver, got: %v", err)
|
||||
}
|
||||
|
||||
// 应该返回非空的 DDL
|
||||
if opDDL == "" {
|
||||
t.Error("expected non-empty operation DDL")
|
||||
}
|
||||
|
||||
if cursorDDL == "" {
|
||||
t.Error("expected non-empty cursor DDL")
|
||||
}
|
||||
|
||||
if retryDDL == "" {
|
||||
t.Error("expected non-empty retry DDL")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user