主要更新: 1. 数据库持久化功能 - 支持三种策略:仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段(可空,仅落库) 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试(含 Pulsar 消费者验证) - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试,提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档,只保留根目录 README 测试结果: - 所有 E2E 测试通过(100%) - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
145 lines
4.2 KiB
Go
145 lines
4.2 KiB
Go
// 检查和修复 cursor 表的脚本
|
||
package main
|
||
|
||
import (
|
||
"context"
|
||
"database/sql"
|
||
"fmt"
|
||
"log"
|
||
"strings"
|
||
"time"
|
||
|
||
_ "github.com/lib/pq"
|
||
)
|
||
|
||
const (
|
||
pgHost = "localhost"
|
||
pgPort = 5432
|
||
pgUser = "postgres"
|
||
pgPassword = "postgres"
|
||
pgDatabase = "trustlog"
|
||
)
|
||
|
||
func main() {
|
||
fmt.Println("🔍 Cursor Table Check Tool")
|
||
fmt.Println(strings.Repeat("=", 60))
|
||
|
||
// 连接数据库
|
||
dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
|
||
pgHost, pgPort, pgUser, pgPassword, pgDatabase)
|
||
|
||
db, err := sql.Open("postgres", dsn)
|
||
if err != nil {
|
||
log.Fatalf("Failed to connect: %v", err)
|
||
}
|
||
defer db.Close()
|
||
|
||
if err := db.Ping(); err != nil {
|
||
log.Fatalf("Failed to ping: %v", err)
|
||
}
|
||
|
||
fmt.Println("✅ Connected to PostgreSQL")
|
||
fmt.Println()
|
||
|
||
ctx := context.Background()
|
||
|
||
// 1. 检查 cursor 表数据
|
||
fmt.Println("📊 Current Cursor Table:")
|
||
rows, err := db.QueryContext(ctx, "SELECT cursor_key, cursor_value, last_updated_at FROM trustlog_cursor ORDER BY last_updated_at DESC")
|
||
if err != nil {
|
||
log.Printf("Failed to query cursor table: %v", err)
|
||
} else {
|
||
defer rows.Close()
|
||
|
||
count := 0
|
||
for rows.Next() {
|
||
var key, value string
|
||
var updatedAt time.Time
|
||
rows.Scan(&key, &value, &updatedAt)
|
||
fmt.Printf(" Key: %s\n", key)
|
||
fmt.Printf(" Value: %s\n", value)
|
||
fmt.Printf(" Updated: %v\n", updatedAt)
|
||
fmt.Println()
|
||
count++
|
||
}
|
||
|
||
if count == 0 {
|
||
fmt.Println(" ❌ No cursor records found!")
|
||
fmt.Println()
|
||
fmt.Println(" 问题原因:")
|
||
fmt.Println(" - Cursor Worker 可能没有启动")
|
||
fmt.Println(" - 或者初始化失败")
|
||
fmt.Println()
|
||
}
|
||
}
|
||
|
||
// 2. 检查 operation 表状态
|
||
fmt.Println("📊 Operation Table Status:")
|
||
|
||
var totalCount int
|
||
db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation").Scan(&totalCount)
|
||
fmt.Printf(" Total operations: %d\n", totalCount)
|
||
|
||
var trustloggedCount int
|
||
db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation WHERE trustlog_status = 'TRUSTLOGGED'").Scan(&trustloggedCount)
|
||
fmt.Printf(" Trustlogged: %d\n", trustloggedCount)
|
||
|
||
var notTrustloggedCount int
|
||
db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation WHERE trustlog_status = 'NOT_TRUSTLOGGED'").Scan(¬TrustloggedCount)
|
||
fmt.Printf(" Not trustlogged: %d\n", notTrustloggedCount)
|
||
|
||
// 查询最早的记录
|
||
var earliestTime sql.NullTime
|
||
db.QueryRowContext(ctx, "SELECT MIN(created_at) FROM operation WHERE trustlog_status = 'NOT_TRUSTLOGGED'").Scan(&earliestTime)
|
||
if earliestTime.Valid {
|
||
fmt.Printf(" Earliest NOT_TRUSTLOGGED record: %v\n", earliestTime.Time)
|
||
}
|
||
|
||
fmt.Println()
|
||
|
||
// 3. 检查 cursor 和记录的时间关系
|
||
if notTrustloggedCount > 0 {
|
||
fmt.Println("⚠️ Problem Detected:")
|
||
fmt.Printf(" 有 %d 条记录未存证\n", notTrustloggedCount)
|
||
|
||
var cursorValue sql.NullString
|
||
db.QueryRowContext(ctx, "SELECT cursor_value FROM trustlog_cursor WHERE cursor_key = 'operation_scan'").Scan(&cursorValue)
|
||
|
||
if !cursorValue.Valid {
|
||
fmt.Println(" Cursor 表为空!")
|
||
fmt.Println()
|
||
fmt.Println(" 可能的原因:")
|
||
fmt.Println(" 1. Cursor Worker 从未启动")
|
||
fmt.Println(" 2. PersistenceClient 没有启用 Cursor Worker")
|
||
fmt.Println()
|
||
fmt.Println(" 解决方案:")
|
||
fmt.Println(" 1. 确保 PersistenceClient 配置了 EnableCursorWorker: true")
|
||
fmt.Println(" 2. 手动初始化 cursor:")
|
||
fmt.Println(" go run scripts/init_cursor.go")
|
||
} else {
|
||
cursorTime, _ := time.Parse(time.RFC3339Nano, cursorValue.String)
|
||
fmt.Printf(" Cursor 时间: %v\n", cursorTime)
|
||
|
||
if earliestTime.Valid && earliestTime.Time.Before(cursorTime) {
|
||
fmt.Println()
|
||
fmt.Println(" ❌ 问题:Cursor 时间晚于最早的未存证记录!")
|
||
fmt.Println(" 这些记录不会被处理。")
|
||
fmt.Println()
|
||
fmt.Println(" 解决方案:")
|
||
fmt.Println(" 1. 重置 cursor 到更早的时间:")
|
||
fmt.Printf(" UPDATE trustlog_cursor SET cursor_value = '%s' WHERE cursor_key = 'operation_scan';\n",
|
||
earliestTime.Time.Add(-1*time.Second).Format(time.RFC3339Nano))
|
||
fmt.Println()
|
||
fmt.Println(" 2. 或者使用脚本重置:")
|
||
fmt.Println(" go run scripts/reset_cursor.go")
|
||
}
|
||
}
|
||
} else {
|
||
fmt.Println("✅ All operations are trustlogged!")
|
||
}
|
||
|
||
fmt.Println()
|
||
fmt.Println(strings.Repeat("=", 60))
|
||
}
|
||
|