Files
go-trustlog/api/highclient/client.go
ryan 4b72a37120 feat: 完善数据库持久化与存证功能
主要更新:

1. 数据库持久化功能
   - 支持三种策略:仅落库、既落库又存证、仅存证
   - 实现 Cursor Worker 异步扫描和存证机制
   - 实现 Retry Worker 失败重试机制
   - 支持 PostgreSQL、MySQL、SQLite 等多种数据库
   - 添加 ClientIP 和 ServerIP 字段(可空,仅落库)

2. 集群并发安全
   - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理
   - 实现 CAS (Compare-And-Set) 原子状态更新
   - 添加 updated_at 字段支持并发控制

3. Cursor 初始化优化
   - 自动基于历史数据初始化 cursor
   - 确保不遗漏任何历史记录
   - 修复 UPSERT 逻辑

4. 测试完善
   - 添加 E2E 集成测试(含 Pulsar 消费者验证)
   - 添加 PostgreSQL 集成测试
   - 添加 Pulsar 集成测试
   - 添加集群并发安全测试
   - 添加 Cursor 初始化验证测试
   - 补充大量单元测试,提升覆盖率

5. 工具脚本
   - 添加数据库迁移脚本
   - 添加 Cursor 状态检查工具
   - 添加 Cursor 初始化工具
   - 添加 Pulsar 消息验证工具

6. 文档清理
   - 删除冗余文档,只保留根目录 README

测试结果:
- 所有 E2E 测试通过(100%)
- 数据库持久化与异步存证流程验证通过
- 集群环境下的并发安全性验证通过
- Cursor 自动初始化和历史数据处理验证通过
2025-12-24 15:31:11 +08:00

157 lines
3.9 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package highclient
import (
"errors"
"fmt"
"github.com/ThreeDotsLabs/watermill/message"
"go.yandata.net/iod/iod/go-trustlog/api/adapter"
"go.yandata.net/iod/iod/go-trustlog/api/logger"
"go.yandata.net/iod/iod/go-trustlog/api/model"
)
type Client struct {
publisher message.Publisher
logger logger.Logger
envelopeConfig model.EnvelopeConfig
}
// NewClient 创建HighClient,使用Envelope序列化方式.
// publisher可以使用任意包含forwarder创建的publisher,但是我们所有的订阅者必须可以处理Envelope格式的消息.
// 参数:
// - publisher: 消息发布器
// - logger: 日志记录器
// - envelopeConfig: SM2密钥配置,用于签名和序列化
func NewClient(publisher message.Publisher, logger logger.Logger, envelopeConfig model.EnvelopeConfig) *Client {
return &Client{
publisher: publisher,
logger: logger,
envelopeConfig: envelopeConfig,
}
}
func (c *Client) GetLow() message.Publisher {
return c.publisher
}
func (c *Client) OperationPublish(operation *model.Operation) error {
if operation == nil {
c.logger.Error("operation publish failed: operation is nil")
return errors.New("operation cannot be nil")
}
c.logger.Debug("publishing operation",
"opID", operation.OpID,
"opType", operation.OpType,
"doPrefix", operation.DoPrefix,
)
err := publish(operation, adapter.OperationTopic, c.publisher, c.envelopeConfig, c.logger)
if err != nil {
c.logger.Error("operation publish failed",
"opID", operation.OpID,
"error", err,
)
return err
}
c.logger.Info("operation published successfully",
"opID", operation.OpID,
"opType", operation.OpType,
)
return nil
}
func (c *Client) RecordPublish(record *model.Record) error {
if record == nil {
c.logger.Error("record publish failed: record is nil")
return errors.New("record cannot be nil")
}
c.logger.Debug("publishing record",
"recordID", record.ID,
"rcType", record.RCType,
"doPrefix", record.DoPrefix,
)
err := publish(record, adapter.RecordTopic, c.publisher, c.envelopeConfig, c.logger)
if err != nil {
c.logger.Error("record publish failed",
"recordID", record.ID,
"error", err,
)
return err
}
c.logger.Info("record published successfully",
"recordID", record.ID,
"rcType", record.RCType,
)
return nil
}
func (c *Client) Close() error {
c.logger.Info("closing high client")
err := c.publisher.Close()
if err != nil {
c.logger.Error("failed to close publisher", "error", err)
return err
}
c.logger.Info("high client closed successfully")
return nil
}
// publish 通用的发布函数,支持任何实现了 Trustlog 接口的类型。
// 使用 Envelope 格式序列化并发布到指定 topic。
func publish(
data model.Trustlog,
topic string,
publisher message.Publisher,
config model.EnvelopeConfig,
logger logger.Logger,
) error {
messageKey := data.Key()
logger.Debug("starting envelope serialization",
"messageKey", messageKey,
"topic", topic,
)
// 使用 Envelope 序列化MarshalTrustlog 会自动提取 producerID
envelopeData, err := model.MarshalTrustlog(data, config)
if err != nil {
logger.Error("envelope serialization failed",
"messageKey", messageKey,
"error", err,
)
return fmt.Errorf("failed to marshal envelope: %w", err)
}
logger.Debug("envelope serialized successfully",
"messageKey", messageKey,
"envelopeSize", len(envelopeData),
)
msg := message.NewMessage(messageKey, envelopeData)
logger.Debug("publishing message to topic",
"messageKey", messageKey,
"topic", topic,
)
if publishErr := publisher.Publish(topic, msg); publishErr != nil {
logger.Error("failed to publish to topic",
"messageKey", messageKey,
"topic", topic,
"error", publishErr,
)
return fmt.Errorf("failed to publish message to topic %s: %w", topic, publishErr)
}
logger.Debug("message published to topic successfully",
"messageKey", messageKey,
"topic", topic,
)
return nil
}