feat: 完善数据库持久化与存证功能
主要更新: 1. 数据库持久化功能 - 支持三种策略:仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段(可空,仅落库) 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试(含 Pulsar 消费者验证) - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试,提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档,只保留根目录 README 测试结果: - 所有 E2E 测试通过(100%) - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
This commit is contained in:
112
scripts/init_cursor.go
Normal file
112
scripts/init_cursor.go
Normal file
@@ -0,0 +1,112 @@
|
||||
// 初始化或重置 cursor 的脚本
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
const (
|
||||
pgHost = "localhost"
|
||||
pgPort = 5432
|
||||
pgUser = "postgres"
|
||||
pgPassword = "postgres"
|
||||
pgDatabase = "trustlog"
|
||||
)
|
||||
|
||||
func main() {
|
||||
fmt.Println("🔧 Cursor Initialization Tool")
|
||||
fmt.Println(strings.Repeat("=", 60))
|
||||
|
||||
// 连接数据库
|
||||
dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||||
pgHost, pgPort, pgUser, pgPassword, pgDatabase)
|
||||
|
||||
db, err := sql.Open("postgres", dsn)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to connect: %v", err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
if err := db.Ping(); err != nil {
|
||||
log.Fatalf("Failed to ping: %v", err)
|
||||
}
|
||||
|
||||
fmt.Println("✅ Connected to PostgreSQL")
|
||||
fmt.Println()
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// 查询最早的 NOT_TRUSTLOGGED 记录
|
||||
var earliestTime sql.NullTime
|
||||
err = db.QueryRowContext(ctx,
|
||||
"SELECT MIN(created_at) FROM operation WHERE trustlog_status = 'NOT_TRUSTLOGGED'",
|
||||
).Scan(&earliestTime)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to query earliest record: %v", err)
|
||||
}
|
||||
|
||||
var cursorValue string
|
||||
if earliestTime.Valid {
|
||||
// 设置为最早记录之前 1 秒
|
||||
cursorValue = earliestTime.Time.Add(-1 * time.Second).Format(time.RFC3339Nano)
|
||||
fmt.Printf("📊 Earliest NOT_TRUSTLOGGED record: %v\n", earliestTime.Time)
|
||||
fmt.Printf("📍 Setting cursor to: %s\n", cursorValue)
|
||||
} else {
|
||||
// 如果没有未存证记录,使用一个很早的时间
|
||||
cursorValue = time.Date(2020, 1, 1, 0, 0, 0, 0, time.UTC).Format(time.RFC3339Nano)
|
||||
fmt.Println("📊 No NOT_TRUSTLOGGED records found")
|
||||
fmt.Printf("📍 Setting cursor to default: %s\n", cursorValue)
|
||||
}
|
||||
|
||||
fmt.Println()
|
||||
|
||||
// 插入或更新 cursor
|
||||
_, err = db.ExecContext(ctx, `
|
||||
INSERT INTO trustlog_cursor (cursor_key, cursor_value, last_updated_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (cursor_key)
|
||||
DO UPDATE SET cursor_value = EXCLUDED.cursor_value, last_updated_at = EXCLUDED.last_updated_at
|
||||
`, "operation_scan", cursorValue, time.Now())
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to init cursor: %v", err)
|
||||
}
|
||||
|
||||
fmt.Println("✅ Cursor initialized successfully!")
|
||||
fmt.Println()
|
||||
|
||||
// 验证
|
||||
var savedValue string
|
||||
var updatedAt time.Time
|
||||
err = db.QueryRowContext(ctx,
|
||||
"SELECT cursor_value, last_updated_at FROM trustlog_cursor WHERE cursor_key = 'operation_scan'",
|
||||
).Scan(&savedValue, &updatedAt)
|
||||
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to verify cursor: %v", err)
|
||||
}
|
||||
|
||||
fmt.Println("📊 Cursor Status:")
|
||||
fmt.Printf(" Key: operation_scan\n")
|
||||
fmt.Printf(" Value: %s\n", savedValue)
|
||||
fmt.Printf(" Updated: %v\n", updatedAt)
|
||||
fmt.Println()
|
||||
|
||||
// 统计
|
||||
var notTrustloggedCount int
|
||||
db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation WHERE trustlog_status = 'NOT_TRUSTLOGGED'").Scan(¬TrustloggedCount)
|
||||
|
||||
fmt.Printf("📝 Records to process: %d\n", notTrustloggedCount)
|
||||
fmt.Println()
|
||||
fmt.Println("✅ Cursor Worker 现在会处理这些记录")
|
||||
fmt.Println(strings.Repeat("=", 60))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user