feat: 新增数据库持久化模块（Persistence），实现 Cursor + Retry 双层架构

## 核心功能 ### 1. 数据库持久化支持 - 新增完整的 Persistence 模块 (api/persistence/) - 支持三种持久化策略： * StrategyDBOnly - 仅落库，不存证 * StrategyDBAndTrustlog - 既落库又存证（推荐） * StrategyTrustlogOnly - 仅存证，不落库 - 支持多数据库：PostgreSQL, MySQL, SQLite ### 2. Cursor + Retry 双层架构 - CursorWorker：第一道防线，快速发现新记录并尝试存证 * 增量扫描 operation 表（基于时间戳游标） * 默认 10 秒扫描间隔，批量处理 100 条 * 成功更新状态，失败转入重试队列 - RetryWorker：第二道防线，处理失败记录 * 指数退避重试（1m → 2m → 4m → 8m → 16m） * 默认最多重试 5 次 * 超限自动标记为死信 ### 3. 数据库表设计 - operation 表：存储操作记录，支持可空 IP 字段 - trustlog_cursor 表：Key-Value 模式，支持多游标 - trustlog_retry 表：重试队列，支持指数退避 ### 4. 异步最终一致性 - 应用调用立即返回（仅落库） - CursorWorker 异步扫描并存证 - RetryWorker 保障失败重试 - 完整的监控和死信处理机制 ## 修改文件 ### 核心代码（11个文件） - api/persistence/cursor_worker.go - Cursor 工作器（新增） - api/persistence/repository.go - 数据仓储层（新增） - api/persistence/schema.go - 数据库 Schema（新增） - api/persistence/strategy.go - 策略管理器（新增） - api/persistence/client.go - 客户端封装（新增） - api/persistence/retry_worker.go - Retry 工作器（新增） - api/persistence/config.go - 配置管理（新增） ### 修复内部包引用（5个文件） - api/adapter/publisher.go - 修复 internal 包引用 - api/adapter/subscriber.go - 修复 internal 包引用 - api/model/envelope.go - 修复 internal 包引用 - api/model/operation.go - 修复 internal 包引用 - api/model/record.go - 修复 internal 包引用 ### 单元测试（8个文件） - api/persistence/*_test.go - 完整的单元测试 - 测试覆盖率：28.5% - 测试通过率：49/49 (100%) ### SQL 脚本（4个文件） - api/persistence/sql/postgresql.sql - PostgreSQL 建表脚本 - api/persistence/sql/mysql.sql - MySQL 建表脚本 - api/persistence/sql/sqlite.sql - SQLite 建表脚本 - api/persistence/sql/test_data.sql - 测试数据 ### 文档（2个文件） - README.md - 更新主文档，新增 Persistence 使用指南 - api/persistence/README.md - 完整的 Persistence 文档 - api/persistence/sql/README.md - SQL 脚本说明 ## 技术亮点 1. **充分利用 Cursor 游标表** - 作为任务发现队列，非简单的位置记录 - Key-Value 模式，支持多游标并发扫描 - 时间戳天然有序，增量扫描高效 2. **双层保障机制** - Cursor：正常流程，快速处理 - Retry：异常流程，可靠重试 - 职责分离，监控清晰 3. **可空 IP 字段支持** - ClientIP 和 ServerIP 使用 *string 类型 - 支持 NULL 值，符合数据库最佳实践 - 使用 sql.NullString 正确处理 4. **完整的监控支持** - 未存证记录数监控 - Cursor 延迟监控 - 重试队列长度监控 - 死信队列监控 ## 测试结果 - ✅ 单元测试：49/49 通过 (100%) - ✅ 代码覆盖率：28.5% - ✅ 编译状态：无错误 - ✅ 支持数据库：PostgreSQL, MySQL, SQLite ## Breaking Changes 无破坏性变更。Persistence 模块作为可选功能，不影响现有代码。 ## 版本信息 - 版本：v2.1.0 - Go 版本要求：1.21+ - 更新日期：2025-12-23
2025-12-23 18:59:43 +08:00
parent d313449c5c
commit 88f80ffa5e
31 changed files with 6551 additions and 36 deletions
--- a/api/persistence/standalone_test.go
+++ b/api/persistence/standalone_test.go
@@ -0,0 +1,309 @@
+// Package persistence_test provides standalone tests that don't depend on internal packages
+package persistence_test
+
+import (
+	"context"
+	"database/sql"
+	"strings"
+	"testing"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3"
+
+	"go.yandata.net/iod/iod/go-trustlog/api/persistence"
+)
+
+// Standalone tests - 独立测试，不依赖复杂模块
+// Run with: go test -v -run Standalone ./api/persistence/
+
+func TestStandaloneConfig(t *testing.T) {
+	cfg := persistence.DefaultDBConfig("postgres", "test-dsn")
+	if cfg.DriverName != "postgres" {
+		t.Errorf("expected DriverName=postgres, got %s", cfg.DriverName)
+	}
+	if cfg.MaxOpenConns != 25 {
+		t.Error("MaxOpenConns should be 25")
+	}
+}
+
+func TestStandaloneStrategy(t *testing.T) {
+	tests := []struct {
+		strategy persistence.PersistenceStrategy
+		want     string
+	}{
+		{persistence.StrategyDBOnly, "DB_ONLY"},
+		{persistence.StrategyDBAndTrustlog, "DB_AND_TRUSTLOG"},
+		{persistence.StrategyTrustlogOnly, "TRUSTLOG_ONLY"},
+	}
+
+	for _, tt := range tests {
+		got := tt.strategy.String()
+		if got != tt.want {
+			t.Errorf("strategy.String() = %s, want %s", got, tt.want)
+		}
+	}
+}
+
+func TestStandaloneEnums(t *testing.T) {
+	if persistence.StatusNotTrustlogged != "NOT_TRUSTLOGGED" {
+		t.Error("StatusNotTrustlogged value incorrect")
+	}
+	if persistence.StatusTrustlogged != "TRUSTLOGGED" {
+		t.Error("StatusTrustlogged value incorrect")
+	}
+	if persistence.RetryStatusPending != "PENDING" {
+		t.Error("RetryStatusPending value incorrect")
+	}
+	if persistence.RetryStatusRetrying != "RETRYING" {
+		t.Error("RetryStatusRetrying value incorrect")
+	}
+	if persistence.RetryStatusDeadLetter != "DEAD_LETTER" {
+		t.Error("RetryStatusDeadLetter value incorrect")
+	}
+}
+
+func TestStandaloneDDL(t *testing.T) {
+	drivers := []string{"postgres", "mysql", "sqlite3", "unknown"}
+	
+	for _, driver := range drivers {
+		opDDL, cursorDDL, retryDDL, err := persistence.GetDialectDDL(driver)
+		if err != nil {
+			t.Fatalf("GetDialectDDL(%s) failed: %v", driver, err)
+		}
+		
+		if len(opDDL) == 0 {
+			t.Errorf("%s: operation DDL is empty", driver)
+		}
+		if len(cursorDDL) == 0 {
+			t.Errorf("%s: cursor DDL is empty", driver)
+		}
+		if len(retryDDL) == 0 {
+			t.Errorf("%s: retry DDL is empty", driver)
+		}
+		
+		// 验证必需字段
+		if !strings.Contains(opDDL, "op_id") {
+			t.Errorf("%s: missing op_id field", driver)
+		}
+		if !strings.Contains(opDDL, "client_ip") {
+			t.Errorf("%s: missing client_ip field", driver)
+		}
+		if !strings.Contains(opDDL, "server_ip") {
+			t.Errorf("%s: missing server_ip field", driver)
+		}
+		if !strings.Contains(opDDL, "trustlog_status") {
+			t.Errorf("%s: missing trustlog_status field", driver)
+		}
+	}
+}
+
+func TestStandaloneDatabase(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("failed to open database: %v", err)
+	}
+	defer db.Close()
+
+	// 获取 DDL
+	opDDL, cursorDDL, retryDDL, err := persistence.GetDialectDDL("sqlite3")
+	if err != nil {
+		t.Fatalf("GetDialectDDL failed: %v", err)
+	}
+
+	// 创建表
+	if _, err := db.Exec(opDDL); err != nil {
+		t.Fatalf("failed to create operation table: %v", err)
+	}
+	if _, err := db.Exec(cursorDDL); err != nil {
+		t.Fatalf("failed to create cursor table: %v", err)
+	}
+	if _, err := db.Exec(retryDDL); err != nil {
+		t.Fatalf("failed to create retry table: %v", err)
+	}
+
+	// 验证表存在
+	tables := []string{"operation", "trustlog_cursor", "trustlog_retry"}
+	for _, table := range tables {
+		var name string
+		err = db.QueryRow(
+			"SELECT name FROM sqlite_master WHERE type='table' AND name=?",
+			table,
+		).Scan(&name)
+		if err != nil {
+			t.Errorf("table %s not found: %v", table, err)
+		}
+	}
+}
+
+func TestStandaloneIPFields(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("failed to open database: %v", err)
+	}
+	defer db.Close()
+
+	// 创建表
+	opDDL, _, _, _ := persistence.GetDialectDDL("sqlite3")
+	if _, err := db.Exec(opDDL); err != nil {
+		t.Fatalf("failed to create table: %v", err)
+	}
+
+	ctx := context.Background()
+
+	// 测试 NULL IP
+	_, err = db.ExecContext(ctx, `
+		INSERT INTO operation (
+			op_id, doid, producer_id, op_source, op_type,
+			do_prefix, do_repository, trustlog_status, timestamp,
+			client_ip, server_ip
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+	`, "test-001", "10.1000/repo/obj", "producer-001", "DOIP", "Create",
+		"10.1000", "repo", "NOT_TRUSTLOGGED", time.Now(), nil, nil)
+
+	if err != nil {
+		t.Fatalf("failed to insert with NULL IPs: %v", err)
+	}
+
+	var clientIP, serverIP sql.NullString
+	err = db.QueryRowContext(ctx,
+		"SELECT client_ip, server_ip FROM operation WHERE op_id = ?",
+		"test-001",
+	).Scan(&clientIP, &serverIP)
+
+	if err != nil {
+		t.Fatalf("failed to query: %v", err)
+	}
+
+	if clientIP.Valid {
+		t.Error("client_ip should be NULL")
+	}
+	if serverIP.Valid {
+		t.Error("server_ip should be NULL")
+	}
+
+	// 测试非 NULL IP
+	_, err = db.ExecContext(ctx, `
+		INSERT INTO operation (
+			op_id, doid, producer_id, op_source, op_type,
+			do_prefix, do_repository, trustlog_status, timestamp,
+			client_ip, server_ip
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+	`, "test-002", "10.1000/repo/obj2", "producer-001", "DOIP", "Create",
+		"10.1000", "repo", "NOT_TRUSTLOGGED", time.Now(),
+		"192.168.1.100", "10.0.0.50")
+
+	if err != nil {
+		t.Fatalf("failed to insert with IP values: %v", err)
+	}
+
+	err = db.QueryRowContext(ctx,
+		"SELECT client_ip, server_ip FROM operation WHERE op_id = ?",
+		"test-002",
+	).Scan(&clientIP, &serverIP)
+
+	if err != nil {
+		t.Fatalf("failed to query: %v", err)
+	}
+
+	if !clientIP.Valid || clientIP.String != "192.168.1.100" {
+		t.Errorf("client_ip should be '192.168.1.100', got %v", clientIP)
+	}
+	if !serverIP.Valid || serverIP.String != "10.0.0.50" {
+		t.Errorf("server_ip should be '10.0.0.50', got %v", serverIP)
+	}
+}
+
+func TestStandaloneStatusFlow(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("failed to open database: %v", err)
+	}
+	defer db.Close()
+
+	opDDL, _, _, _ := persistence.GetDialectDDL("sqlite3")
+	if _, err := db.Exec(opDDL); err != nil {
+		t.Fatalf("failed to create table: %v", err)
+	}
+
+	ctx := context.Background()
+
+	// 插入未存证记录
+	_, err = db.ExecContext(ctx, `
+		INSERT INTO operation (
+			op_id, doid, producer_id, op_source, op_type,
+			do_prefix, do_repository, trustlog_status, timestamp
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+	`, "test-001", "10.1000/repo/obj", "producer-001", "DOIP", "Create",
+		"10.1000", "repo", "NOT_TRUSTLOGGED", time.Now())
+
+	if err != nil {
+		t.Fatalf("failed to insert: %v", err)
+	}
+
+	// 更新为已存证
+	_, err = db.ExecContext(ctx,
+		"UPDATE operation SET trustlog_status = ? WHERE op_id = ?",
+		"TRUSTLOGGED", "test-001",
+	)
+
+	if err != nil {
+		t.Fatalf("failed to update: %v", err)
+	}
+
+	// 验证状态
+	var status string
+	err = db.QueryRowContext(ctx,
+		"SELECT trustlog_status FROM operation WHERE op_id = ?",
+		"test-001",
+	).Scan(&status)
+
+	if err != nil {
+		t.Fatalf("failed to query: %v", err)
+	}
+
+	if status != "TRUSTLOGGED" {
+		t.Errorf("expected status=TRUSTLOGGED, got %s", status)
+	}
+}
+
+func TestStandaloneCursorInit(t *testing.T) {
+	db, err := sql.Open("sqlite3", ":memory:")
+	if err != nil {
+		t.Fatalf("failed to open database: %v", err)
+	}
+	defer db.Close()
+
+	_, cursorDDL, _, _ := persistence.GetDialectDDL("sqlite3")
+	if _, err := db.Exec(cursorDDL); err != nil {
+		t.Fatalf("failed to create cursor table: %v", err)
+	}
+
+	// 验证表结构（新的 cursor 表不再自动插入初始记录）
+	var count int
+	err = db.QueryRow("SELECT COUNT(*) FROM trustlog_cursor").Scan(&count)
+	if err != nil {
+		t.Fatalf("failed to count: %v", err)
+	}
+
+	if count != 0 {
+		t.Errorf("expected 0 initial cursor records (empty table), got %d", count)
+	}
+	
+	// 测试插入新记录
+	_, err = db.Exec(`INSERT INTO trustlog_cursor (cursor_key, cursor_value) VALUES (?, ?)`,
+		"test-cursor", time.Now().Format(time.RFC3339Nano))
+	if err != nil {
+		t.Fatalf("failed to insert cursor: %v", err)
+	}
+	
+	// 验证插入
+	err = db.QueryRow("SELECT COUNT(*) FROM trustlog_cursor WHERE cursor_key = ?", "test-cursor").Scan(&count)
+	if err != nil {
+		t.Fatalf("failed to count after insert: %v", err)
+	}
+	
+	if count != 1 {
+		t.Errorf("expected 1 cursor record after insert, got %d", count)
+	}
+}
+