4b72a37120
主要更新： 1. 数据库持久化功能 - 支持三种策略：仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段（可空，仅落库） 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试（含 Pulsar 消费者验证） - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试，提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档，只保留根目录 README 测试结果： - 所有 E2E 测试通过（100%） - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
104 lines
2.8 KiB
Go
104 lines
2.8 KiB
Go
// 验证 Pulsar 消息的简单脚本
|
||
// 使用方法: go run scripts/verify_pulsar_messages.go
|
||
|
||
package main
|
||
|
||
import (
|
||
"context"
|
||
"fmt"
|
||
"log"
|
||
"time"
|
||
|
||
"github.com/apache/pulsar-client-go/pulsar"
|
||
)
|
||
|
||
const (
|
||
pulsarURL = "pulsar://localhost:6650"
|
||
topic = "persistent://public/default/operation"
|
||
timeout = 10 * time.Second
|
||
)
|
||
|
||
func main() {
|
||
fmt.Println("🔍 Pulsar Message Verification Tool")
|
||
fmt.Println("=====================================")
|
||
fmt.Printf("Pulsar URL: %s\n", pulsarURL)
|
||
fmt.Printf("Topic: %s\n", topic)
|
||
fmt.Println()
|
||
|
||
// 创建 Pulsar 客户端
|
||
client, err := pulsar.NewClient(pulsar.ClientOptions{
|
||
URL: pulsarURL,
|
||
})
|
||
if err != nil {
|
||
log.Fatalf("❌ Failed to create Pulsar client: %v", err)
|
||
}
|
||
defer client.Close()
|
||
fmt.Println("✅ Connected to Pulsar")
|
||
|
||
// 创建消费者（使用唯一的 subscription）
|
||
subName := fmt.Sprintf("verify-sub-%d", time.Now().Unix())
|
||
consumer, err := client.Subscribe(pulsar.ConsumerOptions{
|
||
Topic: topic,
|
||
SubscriptionName: subName,
|
||
Type: pulsar.Shared,
|
||
// 从最早的未确认消息开始读取
|
||
SubscriptionInitialPosition: pulsar.SubscriptionPositionEarliest,
|
||
})
|
||
if err != nil {
|
||
log.Fatalf("❌ Failed to create consumer: %v", err)
|
||
}
|
||
defer consumer.Close()
|
||
fmt.Printf("✅ Consumer created: %s\n\n", subName)
|
||
|
||
// 接收消息
|
||
fmt.Println("📩 Listening for messages (timeout: 10s)...")
|
||
fmt.Println("----------------------------------------")
|
||
|
||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||
defer cancel()
|
||
|
||
messageCount := 0
|
||
for {
|
||
msg, err := consumer.Receive(ctx)
|
||
if err != nil {
|
||
if ctx.Err() == context.DeadlineExceeded {
|
||
break
|
||
}
|
||
log.Printf("⚠️ Error receiving message: %v", err)
|
||
continue
|
||
}
|
||
|
||
messageCount++
|
||
fmt.Printf("\n📨 Message #%d:\n", messageCount)
|
||
fmt.Printf(" Key: %s\n", msg.Key())
|
||
fmt.Printf(" Payload Size: %d bytes\n", len(msg.Payload()))
|
||
fmt.Printf(" Publish Time: %v\n", msg.PublishTime())
|
||
fmt.Printf(" Topic: %s\n", msg.Topic())
|
||
fmt.Printf(" Message ID: %v\n", msg.ID())
|
||
|
||
// 确认消息
|
||
consumer.Ack(msg)
|
||
|
||
// 最多显示 10 条消息
|
||
if messageCount >= 10 {
|
||
fmt.Println("\n⚠️ Reached 10 messages limit, stopping...")
|
||
break
|
||
}
|
||
}
|
||
|
||
fmt.Println("\n========================================")
|
||
if messageCount == 0 {
|
||
fmt.Println("❌ No messages found in Pulsar")
|
||
fmt.Println("\nPossible reasons:")
|
||
fmt.Println(" 1. No operations have been published yet")
|
||
fmt.Println(" 2. All messages have been consumed by other consumers")
|
||
fmt.Println(" 3. Wrong topic name")
|
||
fmt.Println("\nTo test, run the E2E test:")
|
||
fmt.Println(" go test ./api/persistence -v -run TestE2E_DBAndTrustlog_WithPulsarConsumer")
|
||
} else {
|
||
fmt.Printf("✅ Found %d messages in Pulsar\n", messageCount)
|
||
}
|
||
fmt.Println("========================================")
|
||
}
|
||
|