主要更新: 1. 数据库持久化功能 - 支持三种策略:仅落库、既落库又存证、仅存证 - 实现 Cursor Worker 异步扫描和存证机制 - 实现 Retry Worker 失败重试机制 - 支持 PostgreSQL、MySQL、SQLite 等多种数据库 - 添加 ClientIP 和 ServerIP 字段(可空,仅落库) 2. 集群并发安全 - 使用 SELECT FOR UPDATE SKIP LOCKED 防止重复处理 - 实现 CAS (Compare-And-Set) 原子状态更新 - 添加 updated_at 字段支持并发控制 3. Cursor 初始化优化 - 自动基于历史数据初始化 cursor - 确保不遗漏任何历史记录 - 修复 UPSERT 逻辑 4. 测试完善 - 添加 E2E 集成测试(含 Pulsar 消费者验证) - 添加 PostgreSQL 集成测试 - 添加 Pulsar 集成测试 - 添加集群并发安全测试 - 添加 Cursor 初始化验证测试 - 补充大量单元测试,提升覆盖率 5. 工具脚本 - 添加数据库迁移脚本 - 添加 Cursor 状态检查工具 - 添加 Cursor 初始化工具 - 添加 Pulsar 消息验证工具 6. 文档清理 - 删除冗余文档,只保留根目录 README 测试结果: - 所有 E2E 测试通过(100%) - 数据库持久化与异步存证流程验证通过 - 集群环境下的并发安全性验证通过 - Cursor 自动初始化和历史数据处理验证通过
546 lines
11 KiB
Go
546 lines
11 KiB
Go
package model_test
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"go.yandata.net/iod/iod/go-trustlog/api/model"
|
|
)
|
|
|
|
func TestGetHashTool(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
hashType model.HashType
|
|
}{
|
|
{
|
|
name: "SHA256",
|
|
hashType: model.SHA256,
|
|
},
|
|
{
|
|
name: "SHA256Simd",
|
|
hashType: model.Sha256Simd,
|
|
},
|
|
{
|
|
name: "MD5",
|
|
hashType: model.MD5,
|
|
},
|
|
{
|
|
name: "SHA1",
|
|
hashType: model.SHA1,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
tool := model.GetHashTool(tt.hashType)
|
|
assert.NotNil(t, tool)
|
|
// Verify it works
|
|
_, err := tool.HashString("test")
|
|
require.NoError(t, err)
|
|
// Verify hash type
|
|
assert.Equal(t, tt.hashType, tool.GetHashType())
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestNewHashTool(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
assert.NotNil(t, tool)
|
|
// Verify it works
|
|
_, err := tool.HashString("test")
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func TestHashTool_HashString(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
hashType model.HashType
|
|
input string
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "SHA256",
|
|
hashType: model.SHA256,
|
|
input: "test",
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "SHA256Simd",
|
|
hashType: model.Sha256Simd,
|
|
input: "test",
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "MD5",
|
|
hashType: model.MD5,
|
|
input: "test",
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "SHA1",
|
|
hashType: model.SHA1,
|
|
input: "test",
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "SHA512",
|
|
hashType: model.SHA512,
|
|
input: "test",
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "empty string",
|
|
hashType: model.SHA256,
|
|
input: "",
|
|
wantErr: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
tool := model.NewHashTool(tt.hashType)
|
|
result, err := tool.HashString(tt.input)
|
|
if tt.wantErr {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
assert.NotEmpty(t, result)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHashTool_HashBytes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
hashType model.HashType
|
|
input []byte
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "SHA256",
|
|
hashType: model.SHA256,
|
|
input: []byte("test"),
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "SHA256Simd",
|
|
hashType: model.Sha256Simd,
|
|
input: []byte("test"),
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "empty bytes",
|
|
hashType: model.SHA256,
|
|
input: []byte{},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "large input",
|
|
hashType: model.SHA256,
|
|
input: make([]byte, 1000),
|
|
wantErr: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
tool := model.NewHashTool(tt.hashType)
|
|
result, err := tool.HashBytes(tt.input)
|
|
if tt.wantErr {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
assert.NotEmpty(t, result)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHashTool_Deterministic(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
input := "test string"
|
|
|
|
result1, err1 := tool.HashString(input)
|
|
require.NoError(t, err1)
|
|
|
|
result2, err2 := tool.HashString(input)
|
|
require.NoError(t, err2)
|
|
|
|
// Same input should produce same hash
|
|
assert.Equal(t, result1, result2)
|
|
}
|
|
|
|
func TestHashTool_DifferentInputs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
|
|
result1, err1 := tool.HashString("input1")
|
|
require.NoError(t, err1)
|
|
|
|
result2, err2 := tool.HashString("input2")
|
|
require.NoError(t, err2)
|
|
|
|
// Different inputs should produce different hashes
|
|
assert.NotEqual(t, result1, result2)
|
|
}
|
|
|
|
func TestHashTool_StringVsBytes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
input := "test"
|
|
|
|
stringHash, err1 := tool.HashString(input)
|
|
require.NoError(t, err1)
|
|
|
|
bytesHash, err2 := tool.HashBytes([]byte(input))
|
|
require.NoError(t, err2)
|
|
|
|
// Same data in different formats should produce same hash
|
|
assert.Equal(t, stringHash, bytesHash)
|
|
}
|
|
|
|
func TestHashTool_MultipleTypes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
input := "test"
|
|
hashTypes := []model.HashType{
|
|
model.MD5,
|
|
model.SHA1,
|
|
model.SHA256,
|
|
model.SHA512,
|
|
model.Sha256Simd,
|
|
}
|
|
|
|
results := make(map[model.HashType]string)
|
|
for _, hashType := range hashTypes {
|
|
tool := model.NewHashTool(hashType)
|
|
result, err := tool.HashString(input)
|
|
require.NoError(t, err)
|
|
results[hashType] = result
|
|
}
|
|
|
|
// All should produce different hashes (except possibly some edge cases)
|
|
// At minimum, verify they all produced valid hashes
|
|
for hashType, result := range results {
|
|
assert.NotEmpty(t, result, "HashType: %v", hashType)
|
|
}
|
|
}
|
|
|
|
func TestHashTool_GetHashTool_Caching(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hashType := model.SHA256
|
|
tool1 := model.GetHashTool(hashType)
|
|
tool2 := model.GetHashTool(hashType)
|
|
|
|
// Should return the same instance (cached)
|
|
assert.Equal(t, tool1, tool2)
|
|
}
|
|
|
|
func TestHashTool_HashFile(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Create a temporary file
|
|
tmpFile := t.TempDir() + "/test.txt"
|
|
err := os.WriteFile(tmpFile, []byte("test content"), 0o600)
|
|
require.NoError(t, err)
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
ctx := context.Background()
|
|
result, err := tool.HashFile(ctx, tmpFile)
|
|
require.NoError(t, err)
|
|
assert.NotEmpty(t, result)
|
|
}
|
|
|
|
func TestHashTool_HashFile_NotExists(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
ctx := context.Background()
|
|
_, err := tool.HashFile(ctx, "/nonexistent/file")
|
|
require.Error(t, err)
|
|
}
|
|
|
|
func TestHashTool_HashStream(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
reader := bytes.NewReader([]byte("test content"))
|
|
|
|
result, err := tool.HashStream(reader)
|
|
require.NoError(t, err)
|
|
assert.NotEmpty(t, result)
|
|
}
|
|
|
|
func TestHashTool_HashStream_Empty(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
reader := bytes.NewReader([]byte{})
|
|
|
|
result, err := tool.HashStream(reader)
|
|
require.NoError(t, err)
|
|
assert.NotEmpty(t, result) // Even empty input produces a hash
|
|
}
|
|
|
|
func TestGetSupportedAlgorithms(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
algorithms := model.GetSupportedAlgorithms()
|
|
assert.NotEmpty(t, algorithms)
|
|
assert.Contains(t, algorithms, string(model.SHA256))
|
|
assert.Contains(t, algorithms, string(model.Sha256Simd))
|
|
// Verify case-insensitive check
|
|
assert.True(t, model.IsAlgorithmSupported("SHA256"))
|
|
assert.True(t, model.IsAlgorithmSupported("sha256"))
|
|
assert.True(t, model.IsAlgorithmSupported("sha256-simd"))
|
|
}
|
|
|
|
func TestIsAlgorithmSupported(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
algorithm string
|
|
expected bool
|
|
}{
|
|
{
|
|
name: "SHA256",
|
|
algorithm: "SHA256",
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "SHA256 lowercase",
|
|
algorithm: "sha256",
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "Sha256Simd",
|
|
algorithm: "sha256-simd",
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "Sha256Simd mixed case",
|
|
algorithm: "Sha256-Simd",
|
|
expected: true,
|
|
},
|
|
{
|
|
name: "unsupported",
|
|
algorithm: "UNSUPPORTED",
|
|
expected: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
result := model.IsAlgorithmSupported(tt.algorithm)
|
|
assert.Equal(t, tt.expected, result)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHashTool_GetHashType(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA512)
|
|
assert.Equal(t, model.SHA512, tool.GetHashType())
|
|
}
|
|
|
|
func TestHashTool_AllHashTypes(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hashTypes := []model.HashType{
|
|
model.MD5,
|
|
model.SHA1,
|
|
model.SHA224,
|
|
model.SHA256,
|
|
model.SHA384,
|
|
model.SHA512,
|
|
model.Sha256Simd,
|
|
model.BLAKE3,
|
|
}
|
|
|
|
for _, hashType := range hashTypes {
|
|
tool := model.NewHashTool(hashType)
|
|
result, err := tool.HashString("test")
|
|
require.NoError(t, err, "HashType: %v", hashType)
|
|
assert.NotEmpty(t, result, "HashType: %v", hashType)
|
|
assert.Equal(t, hashType, tool.GetHashType())
|
|
}
|
|
}
|
|
|
|
func TestHashTool_CompareHash(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
data := "test data"
|
|
|
|
// Generate hash
|
|
hash, err := tool.HashString(data)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
data string
|
|
expectedHash string
|
|
shouldMatch bool
|
|
}{
|
|
{
|
|
name: "匹配的哈希值",
|
|
data: data,
|
|
expectedHash: hash,
|
|
shouldMatch: true,
|
|
},
|
|
{
|
|
name: "大小写不同但内容相同",
|
|
data: data,
|
|
expectedHash: strings.ToUpper(hash),
|
|
shouldMatch: true,
|
|
},
|
|
{
|
|
name: "不匹配的哈希值",
|
|
data: data,
|
|
expectedHash: "invalid_hash",
|
|
shouldMatch: false,
|
|
},
|
|
{
|
|
name: "不同的数据",
|
|
data: "different data",
|
|
expectedHash: hash,
|
|
shouldMatch: false,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
match, err := tool.CompareHash(tt.data, tt.expectedHash)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, tt.shouldMatch, match)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHashTool_CompareFileHash(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Create a temporary file
|
|
tmpFile := t.TempDir() + "/test.txt"
|
|
content := []byte("test file content")
|
|
err := os.WriteFile(tmpFile, content, 0o600)
|
|
require.NoError(t, err)
|
|
|
|
tool := model.NewHashTool(model.SHA256)
|
|
ctx := context.Background()
|
|
|
|
// Generate expected hash
|
|
expectedHash, err := tool.HashFile(ctx, tmpFile)
|
|
require.NoError(t, err)
|
|
|
|
tests := []struct {
|
|
name string
|
|
filePath string
|
|
expectedHash string
|
|
shouldMatch bool
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "匹配的文件哈希",
|
|
filePath: tmpFile,
|
|
expectedHash: expectedHash,
|
|
shouldMatch: true,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "大小写不同但内容相同",
|
|
filePath: tmpFile,
|
|
expectedHash: strings.ToUpper(expectedHash),
|
|
shouldMatch: true,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "不匹配的文件哈希",
|
|
filePath: tmpFile,
|
|
expectedHash: "invalid_hash",
|
|
shouldMatch: false,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "文件不存在",
|
|
filePath: "/nonexistent/file",
|
|
expectedHash: expectedHash,
|
|
shouldMatch: false,
|
|
wantErr: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
match, err := tool.CompareFileHash(ctx, tt.filePath, tt.expectedHash)
|
|
if tt.wantErr {
|
|
require.Error(t, err)
|
|
} else {
|
|
require.NoError(t, err)
|
|
assert.Equal(t, tt.shouldMatch, match)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestHashList_GetHashType(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Create mock hash data
|
|
mockHash := &mockHashData{
|
|
key: "test-key",
|
|
hash: "test-hash",
|
|
hashType: model.SHA256,
|
|
}
|
|
|
|
hashList := model.HashList{mockHash}
|
|
assert.Equal(t, model.SHA256, hashList.GetHashType())
|
|
}
|
|
|
|
// mockHashData implements HashData interface for testing.
|
|
type mockHashData struct {
|
|
key string
|
|
hash string
|
|
hashType model.HashType
|
|
}
|
|
|
|
func (m *mockHashData) Key() string {
|
|
return m.key
|
|
}
|
|
|
|
func (m *mockHashData) Hash() string {
|
|
return m.hash
|
|
}
|
|
|
|
func (m *mockHashData) Type() model.HashType {
|
|
return m.hashType
|
|
}
|