feat: 完善数据库持久化与存证功能

主要更新:

1. 数据库持久化功能
   - 支持三种策略:仅落库、既落库又存证、仅存证
   - 实现 Cursor Worker 异步扫描和存证机制
   - 实现 Retry Worker 失败重试机制
   - 支持 PostgreSQL、MySQL、SQLite 等多种数据库
   - 添加 ClientIP 和 ServerIP 字段(可空,仅落库)

2. 集群并发安全
   - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理
   - 实现 CAS (Compare-And-Set) 原子状态更新
   - 添加 updated_at 字段支持并发控制

3. Cursor 初始化优化
   - 自动基于历史数据初始化 cursor
   - 确保不遗漏任何历史记录
   - 修复 UPSERT 逻辑

4. 测试完善
   - 添加 E2E 集成测试(含 Pulsar 消费者验证)
   - 添加 PostgreSQL 集成测试
   - 添加 Pulsar 集成测试
   - 添加集群并发安全测试
   - 添加 Cursor 初始化验证测试
   - 补充大量单元测试,提升覆盖率

5. 工具脚本
   - 添加数据库迁移脚本
   - 添加 Cursor 状态检查工具
   - 添加 Cursor 初始化工具
   - 添加 Pulsar 消息验证工具

6. 文档清理
   - 删除冗余文档,只保留根目录 README

测试结果:
- 所有 E2E 测试通过(100%)
- 数据库持久化与异步存证流程验证通过
- 集群环境下的并发安全性验证通过
- Cursor 自动初始化和历史数据处理验证通过
This commit is contained in:
ryan
2025-12-24 15:31:11 +08:00
parent 88f80ffa5e
commit 4b72a37120
60 changed files with 6160 additions and 1313 deletions

View File

@@ -0,0 +1,128 @@
package main
import (
"database/sql"
"fmt"
"log"
_ "github.com/lib/pq"
)
func main() {
dsn := "host=localhost port=5432 user=postgres password=postgres dbname=trustlog sslmode=disable"
db, err := sql.Open("postgres", dsn)
if err != nil {
log.Fatalf("Failed to connect: %v", err)
}
defer db.Close()
if err := db.Ping(); err != nil {
log.Fatalf("Failed to ping: %v", err)
}
fmt.Println("🔄 Migrating PostgreSQL schema...")
// 删除旧表
fmt.Println(" Dropping old tables...")
_, err = db.Exec("DROP TABLE IF EXISTS trustlog_retry")
if err != nil {
log.Printf("Warning: Failed to drop retry table: %v", err)
}
_, err = db.Exec("DROP TABLE IF EXISTS operation")
if err != nil {
log.Printf("Warning: Failed to drop operation table: %v", err)
}
_, err = db.Exec("DROP TABLE IF EXISTS trustlog_cursor")
if err != nil {
log.Printf("Warning: Failed to drop cursor table: %v", err)
}
// 重新创建表
fmt.Println(" Creating new tables...")
_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS operation (
op_id VARCHAR(32) NOT NULL PRIMARY KEY,
op_actor VARCHAR(64),
doid VARCHAR(512),
producer_id VARCHAR(32),
request_body_hash VARCHAR(128),
response_body_hash VARCHAR(128),
op_hash VARCHAR(128),
sign VARCHAR(512),
op_source VARCHAR(10),
op_type VARCHAR(30),
do_prefix VARCHAR(128),
do_repository VARCHAR(64),
client_ip VARCHAR(32),
server_ip VARCHAR(32),
trustlog_status VARCHAR(32),
timestamp TIMESTAMP,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)`)
if err != nil {
log.Fatalf("Failed to create operation table: %v", err)
}
_, err = db.Exec(`
CREATE INDEX IF NOT EXISTS idx_operation_timestamp ON operation(timestamp)`)
if err != nil {
log.Printf("Warning: Failed to create timestamp index: %v", err)
}
_, err = db.Exec(`
CREATE INDEX IF NOT EXISTS idx_operation_trustlog_status ON operation(trustlog_status)`)
if err != nil {
log.Printf("Warning: Failed to create status index: %v", err)
}
_, err = db.Exec(`
CREATE INDEX IF NOT EXISTS idx_operation_created_at ON operation(created_at)`)
if err != nil {
log.Printf("Warning: Failed to create created_at index: %v", err)
}
_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS trustlog_cursor (
cursor_key VARCHAR(64) NOT NULL PRIMARY KEY,
cursor_value TEXT NOT NULL,
last_updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)`)
if err != nil {
log.Fatalf("Failed to create cursor table: %v", err)
}
_, err = db.Exec(`
CREATE TABLE IF NOT EXISTS trustlog_retry (
op_id VARCHAR(32) NOT NULL PRIMARY KEY,
retry_count INTEGER DEFAULT 0,
retry_status VARCHAR(32) DEFAULT 'PENDING',
last_retry_at TIMESTAMP,
next_retry_at TIMESTAMP,
error_message TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)`)
if err != nil {
log.Fatalf("Failed to create retry table: %v", err)
}
_, err = db.Exec(`
CREATE INDEX IF NOT EXISTS idx_retry_next_retry_at ON trustlog_retry(next_retry_at)`)
if err != nil {
log.Printf("Warning: Failed to create retry time index: %v", err)
}
_, err = db.Exec(`
CREATE INDEX IF NOT EXISTS idx_retry_retry_status ON trustlog_retry(retry_status)`)
if err != nil {
log.Printf("Warning: Failed to create retry status index: %v", err)
}
fmt.Println("✅ Schema migration completed!")
}