Files
go-trustlog/scripts/check_cursor.go
ryan 4b72a37120 feat: 完善数据库持久化与存证功能
主要更新:

1. 数据库持久化功能
   - 支持三种策略:仅落库、既落库又存证、仅存证
   - 实现 Cursor Worker 异步扫描和存证机制
   - 实现 Retry Worker 失败重试机制
   - 支持 PostgreSQL、MySQL、SQLite 等多种数据库
   - 添加 ClientIP 和 ServerIP 字段(可空,仅落库)

2. 集群并发安全
   - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理
   - 实现 CAS (Compare-And-Set) 原子状态更新
   - 添加 updated_at 字段支持并发控制

3. Cursor 初始化优化
   - 自动基于历史数据初始化 cursor
   - 确保不遗漏任何历史记录
   - 修复 UPSERT 逻辑

4. 测试完善
   - 添加 E2E 集成测试(含 Pulsar 消费者验证)
   - 添加 PostgreSQL 集成测试
   - 添加 Pulsar 集成测试
   - 添加集群并发安全测试
   - 添加 Cursor 初始化验证测试
   - 补充大量单元测试,提升覆盖率

5. 工具脚本
   - 添加数据库迁移脚本
   - 添加 Cursor 状态检查工具
   - 添加 Cursor 初始化工具
   - 添加 Pulsar 消息验证工具

6. 文档清理
   - 删除冗余文档,只保留根目录 README

测试结果:
- 所有 E2E 测试通过(100%)
- 数据库持久化与异步存证流程验证通过
- 集群环境下的并发安全性验证通过
- Cursor 自动初始化和历史数据处理验证通过
2025-12-24 15:31:11 +08:00

145 lines
4.2 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// 检查和修复 cursor 表的脚本
package main
import (
"context"
"database/sql"
"fmt"
"log"
"strings"
"time"
_ "github.com/lib/pq"
)
const (
pgHost = "localhost"
pgPort = 5432
pgUser = "postgres"
pgPassword = "postgres"
pgDatabase = "trustlog"
)
func main() {
fmt.Println("🔍 Cursor Table Check Tool")
fmt.Println(strings.Repeat("=", 60))
// 连接数据库
dsn := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
pgHost, pgPort, pgUser, pgPassword, pgDatabase)
db, err := sql.Open("postgres", dsn)
if err != nil {
log.Fatalf("Failed to connect: %v", err)
}
defer db.Close()
if err := db.Ping(); err != nil {
log.Fatalf("Failed to ping: %v", err)
}
fmt.Println("✅ Connected to PostgreSQL")
fmt.Println()
ctx := context.Background()
// 1. 检查 cursor 表数据
fmt.Println("📊 Current Cursor Table:")
rows, err := db.QueryContext(ctx, "SELECT cursor_key, cursor_value, last_updated_at FROM trustlog_cursor ORDER BY last_updated_at DESC")
if err != nil {
log.Printf("Failed to query cursor table: %v", err)
} else {
defer rows.Close()
count := 0
for rows.Next() {
var key, value string
var updatedAt time.Time
rows.Scan(&key, &value, &updatedAt)
fmt.Printf(" Key: %s\n", key)
fmt.Printf(" Value: %s\n", value)
fmt.Printf(" Updated: %v\n", updatedAt)
fmt.Println()
count++
}
if count == 0 {
fmt.Println(" ❌ No cursor records found!")
fmt.Println()
fmt.Println(" 问题原因:")
fmt.Println(" - Cursor Worker 可能没有启动")
fmt.Println(" - 或者初始化失败")
fmt.Println()
}
}
// 2. 检查 operation 表状态
fmt.Println("📊 Operation Table Status:")
var totalCount int
db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation").Scan(&totalCount)
fmt.Printf(" Total operations: %d\n", totalCount)
var trustloggedCount int
db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation WHERE trustlog_status = 'TRUSTLOGGED'").Scan(&trustloggedCount)
fmt.Printf(" Trustlogged: %d\n", trustloggedCount)
var notTrustloggedCount int
db.QueryRowContext(ctx, "SELECT COUNT(*) FROM operation WHERE trustlog_status = 'NOT_TRUSTLOGGED'").Scan(&notTrustloggedCount)
fmt.Printf(" Not trustlogged: %d\n", notTrustloggedCount)
// 查询最早的记录
var earliestTime sql.NullTime
db.QueryRowContext(ctx, "SELECT MIN(created_at) FROM operation WHERE trustlog_status = 'NOT_TRUSTLOGGED'").Scan(&earliestTime)
if earliestTime.Valid {
fmt.Printf(" Earliest NOT_TRUSTLOGGED record: %v\n", earliestTime.Time)
}
fmt.Println()
// 3. 检查 cursor 和记录的时间关系
if notTrustloggedCount > 0 {
fmt.Println("⚠️ Problem Detected:")
fmt.Printf(" 有 %d 条记录未存证\n", notTrustloggedCount)
var cursorValue sql.NullString
db.QueryRowContext(ctx, "SELECT cursor_value FROM trustlog_cursor WHERE cursor_key = 'operation_scan'").Scan(&cursorValue)
if !cursorValue.Valid {
fmt.Println(" Cursor 表为空!")
fmt.Println()
fmt.Println(" 可能的原因:")
fmt.Println(" 1. Cursor Worker 从未启动")
fmt.Println(" 2. PersistenceClient 没有启用 Cursor Worker")
fmt.Println()
fmt.Println(" 解决方案:")
fmt.Println(" 1. 确保 PersistenceClient 配置了 EnableCursorWorker: true")
fmt.Println(" 2. 手动初始化 cursor:")
fmt.Println(" go run scripts/init_cursor.go")
} else {
cursorTime, _ := time.Parse(time.RFC3339Nano, cursorValue.String)
fmt.Printf(" Cursor 时间: %v\n", cursorTime)
if earliestTime.Valid && earliestTime.Time.Before(cursorTime) {
fmt.Println()
fmt.Println(" ❌ 问题Cursor 时间晚于最早的未存证记录!")
fmt.Println(" 这些记录不会被处理。")
fmt.Println()
fmt.Println(" 解决方案:")
fmt.Println(" 1. 重置 cursor 到更早的时间:")
fmt.Printf(" UPDATE trustlog_cursor SET cursor_value = '%s' WHERE cursor_key = 'operation_scan';\n",
earliestTime.Time.Add(-1*time.Second).Format(time.RFC3339Nano))
fmt.Println()
fmt.Println(" 2. 或者使用脚本重置:")
fmt.Println(" go run scripts/reset_cursor.go")
}
}
} else {
fmt.Println("✅ All operations are trustlogged!")
}
fmt.Println()
fmt.Println(strings.Repeat("=", 60))
}