Files
go-trustlog/api/model/converter.go
ryan 4b72a37120 feat: 完善数据库持久化与存证功能
主要更新:

1. 数据库持久化功能
   - 支持三种策略:仅落库、既落库又存证、仅存证
   - 实现 Cursor Worker 异步扫描和存证机制
   - 实现 Retry Worker 失败重试机制
   - 支持 PostgreSQL、MySQL、SQLite 等多种数据库
   - 添加 ClientIP 和 ServerIP 字段(可空,仅落库)

2. 集群并发安全
   - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理
   - 实现 CAS (Compare-And-Set) 原子状态更新
   - 添加 updated_at 字段支持并发控制

3. Cursor 初始化优化
   - 自动基于历史数据初始化 cursor
   - 确保不遗漏任何历史记录
   - 修复 UPSERT 逻辑

4. 测试完善
   - 添加 E2E 集成测试(含 Pulsar 消费者验证)
   - 添加 PostgreSQL 集成测试
   - 添加 Pulsar 集成测试
   - 添加集群并发安全测试
   - 添加 Cursor 初始化验证测试
   - 补充大量单元测试,提升覆盖率

5. 工具脚本
   - 添加数据库迁移脚本
   - 添加 Cursor 状态检查工具
   - 添加 Cursor 初始化工具
   - 添加 Pulsar 消息验证工具

6. 文档清理
   - 删除冗余文档,只保留根目录 README

测试结果:
- 所有 E2E 测试通过(100%)
- 数据库持久化与异步存证流程验证通过
- 集群环境下的并发安全性验证通过
- Cursor 自动初始化和历史数据处理验证通过
2025-12-24 15:31:11 +08:00

201 lines
5.4 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package model
import (
"errors"
"fmt"
"google.golang.org/protobuf/types/known/timestamppb"
"go.yandata.net/iod/iod/go-trustlog/api/grpc/pb"
)
// FromProtobuf 将protobuf的OperationData转换为model.Operation.
func FromProtobuf(pbOp *pb.OperationData) (*Operation, error) {
if pbOp == nil {
return nil, errors.New("protobuf operation data is nil")
}
// 转换时间戳
if pbOp.GetTimestamp() == nil {
return nil, errors.New("timestamp is required")
}
timestamp := pbOp.GetTimestamp().AsTime()
// 构建Operation
operation := &Operation{
OpID: pbOp.GetOpId(),
Timestamp: timestamp,
OpSource: Source(pbOp.GetOpSource()),
OpType: Type(pbOp.GetOpType()),
DoPrefix: pbOp.GetDoPrefix(),
DoRepository: pbOp.GetDoRepository(),
Doid: pbOp.GetDoid(),
ProducerID: pbOp.GetProducerId(),
OpActor: pbOp.GetOpActor(),
// OpAlgorithm和OpMetaHash字段已移除固定使用Sha256Simd哈希值由Envelope的OriginalHash提供
}
// 处理可选的哈希字段
if reqHash := pbOp.GetRequestBodyHash(); reqHash != "" {
operation.RequestBodyHash = &reqHash
}
if respHash := pbOp.GetResponseBodyHash(); respHash != "" {
operation.ResponseBodyHash = &respHash
}
return operation, nil
}
// ToProtobuf 将model.Operation转换为protobuf的OperationData.
func ToProtobuf(op *Operation) (*pb.OperationData, error) {
if op == nil {
return nil, errors.New("operation is nil")
}
// 转换时间戳
timestamp := timestamppb.New(op.Timestamp)
pbOp := &pb.OperationData{
OpId: op.OpID,
Timestamp: timestamp,
OpSource: string(op.OpSource),
OpType: string(op.OpType),
DoPrefix: op.DoPrefix,
DoRepository: op.DoRepository,
Doid: op.Doid,
ProducerId: op.ProducerID,
OpActor: op.OpActor,
// OpAlgorithm、OpMetaHash和OpHash字段已移除固定使用Sha256Simd哈希值由Envelope的OriginalHash提供
}
// 处理可选的哈希字段
if op.RequestBodyHash != nil {
pbOp.RequestBodyHash = *op.RequestBodyHash
}
if op.ResponseBodyHash != nil {
pbOp.ResponseBodyHash = *op.ResponseBodyHash
}
return pbOp, nil
}
// FromProtobufValidationResult 将protobuf的ValidationStreamRes转换为model.ValidationResult.
func FromProtobufValidationResult(pbRes *pb.ValidationStreamRes) (*ValidationResult, error) {
if pbRes == nil {
return nil, errors.New("protobuf validation result is nil")
}
result := &ValidationResult{
Code: pbRes.GetCode(),
Msg: pbRes.GetMsg(),
Progress: pbRes.GetProgress(),
Proof: ProofFromProtobuf(pbRes.GetProof()), // 取证证明
}
// 如果有操作数据,则转换
if pbRes.GetData() != nil {
op, err := FromProtobuf(pbRes.GetData())
if err != nil {
return nil, fmt.Errorf("failed to convert operation data: %w", err)
}
result.Data = op
}
return result, nil
}
// RecordFromProtobuf 将protobuf的RecordData转换为model.Record.
func RecordFromProtobuf(pbRec *pb.RecordData) (*Record, error) {
if pbRec == nil {
return nil, errors.New("protobuf record data is nil")
}
// 构建Record
record := &Record{
ID: pbRec.GetId(),
DoPrefix: pbRec.GetDoPrefix(),
ProducerID: pbRec.GetProducerId(),
Operator: pbRec.GetOperator(),
Extra: pbRec.GetExtra(),
RCType: pbRec.GetRcType(),
}
// 转换时间戳
if pbRec.GetTimestamp() != nil {
record.Timestamp = pbRec.GetTimestamp().AsTime()
}
return record, nil
}
// RecordToProtobuf 将model.Record转换为protobuf的RecordData.
func RecordToProtobuf(rec *Record) (*pb.RecordData, error) {
if rec == nil {
return nil, errors.New("record is nil")
}
// 转换时间戳
timestamp := timestamppb.New(rec.Timestamp)
pbRec := &pb.RecordData{
Id: rec.ID,
DoPrefix: rec.DoPrefix,
ProducerId: rec.ProducerID,
Timestamp: timestamp,
Operator: rec.Operator,
Extra: rec.Extra,
RcType: rec.RCType,
}
return pbRec, nil
}
// RecordValidationResult 包装记录验证的流式响应结果.
type RecordValidationResult struct {
Code int32 // 状态码100处理中200完成500失败
Msg string // 消息描述
Progress string // 当前进度(比如 "50%"
Data *Record // 最终完成时返回的记录数据,过程中可为空
Proof *Proof // 取证证明(仅在完成时返回)
}
// IsProcessing 判断是否正在处理中.
func (r *RecordValidationResult) IsProcessing() bool {
return r.Code == ValidationCodeProcessing
}
// IsCompleted 判断是否已完成.
func (r *RecordValidationResult) IsCompleted() bool {
return r.Code == ValidationCodeCompleted
}
// IsFailed 判断是否失败.
func (r *RecordValidationResult) IsFailed() bool {
return r.Code >= ValidationCodeFailed
}
// RecordFromProtobufValidationResult 将protobuf的RecordValidationStreamRes转换为model.RecordValidationResult.
func RecordFromProtobufValidationResult(pbRes *pb.RecordValidationStreamRes) (*RecordValidationResult, error) {
if pbRes == nil {
return nil, errors.New("protobuf record validation result is nil")
}
result := &RecordValidationResult{
Code: pbRes.GetCode(),
Msg: pbRes.GetMsg(),
Progress: pbRes.GetProgress(),
Proof: ProofFromProtobuf(pbRes.GetProof()), // 取证证明
}
// 如果有记录数据,则转换
if pbRes.GetResult() != nil {
rec, err := RecordFromProtobuf(pbRes.GetResult())
if err != nil {
return nil, fmt.Errorf("failed to convert record data: %w", err)
}
result.Data = rec
}
return result, nil
}