bdware/common
2
0
Fork 0
mirror of https://gitee.com/BDWare/common synced 2025-01-09 17:34:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: binarySearch bugs in TimeSerialIndex

Browse Source 
feat: support multi tag time index util
This commit is contained in:
CaiHQ 2022-01-06 20:33:46 +08:00
parent 7032dd3b28
commit 2ce57c111e
7 changed files with 157 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,4 +1,5 @@
/build/
/testoutput/
*/build/*
# Compiled class file
*.class

1
src/main/base/org/bdware/sc/node/Permission.java
View File

@ -10,6 +10,7 @@ public enum Permission {
RocksDB,
MongoDB,
BDWareTimeSeriesDB,
MultiTagIndexDB,
SM2,
AES,
Ledger,

65
src/main/entry/org/bdware/sc/db/MultiIndexTimeRocksDBUtil.java
View File

@ -18,6 +18,7 @@ import java.util.*;
public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
private static final Logger LOGGER = LogManager.getLogger(MultiIndexTimeRocksDBUtil.class);
private final Map<String, TimeSerialIndex> secondaryIndex;
static String primaryKey = "_DB_primary_";
public String dbPath;
public String tableName;
Random random = new Random();
@ -42,7 +43,7 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
Options options = new Options();
options.setCreateIfMissing(true);
File lockFile = new File(file, "LOCK");
File timeIndex = new File(dbPath + "/" + tableName + "/DB.primary.timeindex");
File timeIndex = new File(dbPath + "/" + tableName + "/" + primaryIndex + ".timeindex");
LOGGER.trace("create directory " + file.getAbsolutePath() + ": " + file.mkdirs());
LOGGER.trace("delete file" + lockFile.getAbsolutePath() + ": " + lockFile.delete());
try {
@ -51,8 +52,16 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
} catch (RocksDBException e) {
e.printStackTrace();
}
primaryIndex = new TimeSerialIndex(timeIndex.getAbsolutePath());
secondaryIndex.put(primaryKey, primaryIndex);
for (File f : timeIndex.getParentFile().listFiles()) {
if (!f.getName().endsWith(".timeindex")) continue;
if (f.getName().equals(timeIndex.getName())) continue;
TimeSerialIndex index =
new TimeSerialIndex(f.getAbsolutePath());
secondaryIndex.putIfAbsent(f.getName().substring(0,
f.getName().length() - ".timeindex".length()), index);
}
}
public synchronized void put(String label, String val) {
@ -141,8 +150,13 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
public long size(String label) {
try {
TimeSerialIndex index = getIndex(label);
if (index != null) return index.size();
if (label == null || label.length() == 0) {
return primaryIndex.size();
}
if (secondaryIndex.containsKey(label)) {
TimeSerialIndex index = getIndex(label);
return index.size();
}
} catch (Exception e) {
e.printStackTrace();
}
@ -152,17 +166,23 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
public List<JsonObject> queryByOffset(String label, long offset, int count) {
List<JsonObject> ret = new ArrayList<>();
TimeSerialIndex index = getIndex(label);
List<Long> data = index.request(offset, count);
for (Long l : data) {
List<TimeSerialIndex.IndexEntry> data = index.requestIndexEntry(offset, count);
for (TimeSerialIndex.IndexEntry entry : data) {
try {
String t = new String(db.get(longToByte(l)));
String t = new String(db.get(longToByte(entry.value)));
JsonObject jo;
if (!t.isEmpty()) {
jo = JsonUtil.parseStringAsJsonObject(t);
try {
jo = JsonUtil.parseStringAsJsonObject(t);
} catch (Exception e) {
jo = new JsonObject();
jo.addProperty("data", t);
}
} else {
jo = new JsonObject();
}
jo.addProperty("key", l.toString());
jo.addProperty("key", entry.value);
jo.addProperty("timestamp", entry.key);
ret.add(jo);
} catch (Exception e) {
e.printStackTrace();
@ -181,7 +201,7 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
}
private TimeSerialIndex getIndex(String label) {
if (null == label) {
if (null == label || label.length() == 0) {
return primaryIndex;
}
if (secondaryIndex.containsKey(label)) {
@ -275,6 +295,31 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
// manuellyIndex可参考LenVarTimeIndex写法
}
public void close() {
db.close();
}
public List<String> getIndexStartWith(String prefix) {
List<String> ret = new ArrayList<>();
if (prefix == null) {
prefix = "";
}
for (String key : secondaryIndex.keySet()) {
if (key.startsWith(prefix))
ret.add(key);
}
ret.remove(primaryKey);
return ret;
}
public List<String> getAllIndexKey() {
Set<String> data = secondaryIndex.keySet();
List<String> ret = new ArrayList<>();
ret.addAll(data);
ret.remove(primaryKey);
return ret;
}
// TODO
static class BytesPair {
byte[] key, value;

44
src/main/entry/org/bdware/sc/index/TimeSerialIndex.java
View File

@ -85,6 +85,39 @@ public class TimeSerialIndex {
} else return new ArrayList<>();
}
public static class IndexEntry {
public long key, value;
}
public synchronized List<IndexEntry> requestIndexEntry(long offset, int len) {
List<IndexEntry> ret = new ArrayList<>();
if (offset < 0) offset = 0;
if (offset < fileSize) {
long pos = 0;
try {
pos = file.getFilePointer();
file.seek(2L * offset * 8L);
for (; offset < fileSize && len > 0; len--) {
IndexEntry entry = new IndexEntry();
entry.key = file.readLong();
entry.value = file.readLong();
ret.add(entry);
offset++;
}
file.seek(pos);
} catch (IOException e) {
e.printStackTrace();
} finally {
try {
file.seek(pos);
} catch (IOException e) {
e.printStackTrace();
}
}
return ret;
} else return new ArrayList<>();
}
private long getIndex(long offset) {
if (offset < fileSize) {
try {
@ -109,7 +142,7 @@ public class TimeSerialIndex {
public synchronized long findNearest(long timeStamp) {
try {
long pos = file.getFilePointer();
long ret = binarySearch(0L, fileSize + 1, timeStamp);
long ret = binarySearch(0L, fileSize, timeStamp);
file.seek(pos);
return ret;
} catch (IOException e) {
@ -120,9 +153,14 @@ public class TimeSerialIndex {
}
private long binarySearch(long start, long end, long timeStamp) {
if (start >= end - 1) return start;
if (end <= 0) return end;
if (start >= end - 1) {
long key = getIndex(end - 1);
if (key >= timeStamp) {
return end - 1;
} else return end;
}
long mid = (start + end) / 2;
if (mid >= end - 1) return mid;
long key = getIndex(mid);
if (key >= timeStamp) {
return binarySearch(start, mid, timeStamp);

51
src/test/java/org/bdware/MultiIndextTimeRocksDBUtilTest.java Normal file
View File

@ -0,0 +1,51 @@
package org.bdware;
import com.google.gson.JsonObject;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bdware.sc.db.MultiIndexTimeRocksDBUtil;
import org.junit.Before;
import org.junit.Test;
import java.util.List;
public class MultiIndextTimeRocksDBUtilTest {
static Logger LOGGER = LogManager.getLogger(MultiIndextTimeRocksDBUtilTest.class);
MultiIndexTimeRocksDBUtil util;
@Before
public void init() {
util = new MultiIndexTimeRocksDBUtil("testoutput/timeIndexDB", "defaultTable");
}
@Test
public void put() {
for (int i = 0; i < 100; i++) {
try {
Thread.sleep(10);
} catch (InterruptedException e) {
e.printStackTrace();
}
util.put("abc_123", "ddd " + System.currentTimeMillis());
}
}
@Test
public void listAll() {
List<JsonObject> tt = util.queryByOffset(null, 0, 100);
for (int i = 0; i < tt.size(); i++) {
long val = tt.get(i).get("timestamp").getAsLong();
LOGGER.info(String.format("pre %d cu %d suc %d", util.queryOffset("abc_123", val - 1L),
util.queryOffset("abc_123", val), util.queryOffset("abc_123", val + 1L), util.queryOffset("abc_123", val - 1L)));
}
}
@Test
public void run() {
LOGGER.info(util.queryOffset("abc_123", 1641454140657L));
LOGGER.info(util.queryOffset("abc_123", 1641454140669L));
LOGGER.info(util.queryOffset("abc_123", 1641450028482L));
LOGGER.info(util.queryOffset("abc_123", 1641450028483L));
}
}

16
src/test/java/org/bdware/sc/LedgerUtilTest.java
View File

@ -1,21 +1,17 @@
package org.bdware.sc;
import org.bdware.bdledger.api.grpc.Client;
//import org.bdware.bdledger.api.grpc.pb.QueryOuterClass;
import org.bdware.bdledger.api.grpc.pb.QueryProto;
import org.junit.Test;
public class LedgerUtilTest {
@Test
public void query(){
String str = "0828c62576606e27ef8d079dea60cadd6eb03351";
Client c = new Client("022.node.internetapi.cn",21121);
QueryProto.GetTransactionByHashResponse syncResult = c.getTransactionByHashSync("default", "5c79fb8d71771615a5c8173b8089cd4841c3adfb");
System.out.println(new String(syncResult.getTransaction().getData().toByteArray()));
System.out.println(syncResult.getTransaction().getData().toByteArray().length);
// 092：47.107.94.135
// String str = "0828c62576606e27ef8d079dea60cadd6eb03351";
// Client c = new Client("022.node.internetapi.cn",21121);
// QueryProto.GetTransactionByHashResponse syncResult = c.getTransactionByHashSync("default", "5c79fb8d71771615a5c8173b8089cd4841c3adfb");
// System.out.println(new String(syncResult.getTransaction().getData().toByteArray()));
// System.out.println(syncResult.getTransaction().getData().toByteArray().length);
// 092：47.107.94.135
// 093：120.79.215.187
// QueryOuterClass.GetTransactionByHashResponse hash = c.getTransactionByHashSync("test", str);
// String str2 = hash.getTransaction().getFrom().toStringUtf8();

5
src/test/java/org/bdware/sc/boundary/utils/SM2UtilTest.java
View File

@ -10,7 +10,6 @@ import org.junit.Test;
import org.zz.gmhelper.BCECUtil;
import org.zz.gmhelper.SM2KeyPair;
import org.zz.gmhelper.SM2Util;
import sun.misc.BASE64Encoder;
import java.math.BigInteger;
import java.security.KeyPair;
@ -25,8 +24,8 @@ public class SM2UtilTest {
String gstr =
"09146332716e7767327923056946406e49570e5909146332716e7767327923056946406e49570e59";
byte[] bytes = ByteUtils.fromHexString(gstr);
String str = new BASE64Encoder().encode(bytes);
System.out.println(str);
// String str = new BASE64Encoder().encode(bytes);
// System.out.println(str);
}
@Test