From 2ce57c111eb453aac62ebd497d6d8af2d0adaf96 Mon Sep 17 00:00:00 2001 From: CaiHQ Date: Thu, 6 Jan 2022 20:33:46 +0800 Subject: [PATCH] fix: binarySearch bugs in TimeSerialIndex feat: support multi tag time index util --- .gitignore | 1 + .../base/org/bdware/sc/node/Permission.java | 1 + .../sc/db/MultiIndexTimeRocksDBUtil.java | 65 ++++++++++++++++--- .../org/bdware/sc/index/TimeSerialIndex.java | 44 ++++++++++++- .../MultiIndextTimeRocksDBUtilTest.java | 51 +++++++++++++++ .../java/org/bdware/sc/LedgerUtilTest.java | 16 ++--- .../bdware/sc/boundary/utils/SM2UtilTest.java | 5 +- 7 files changed, 157 insertions(+), 26 deletions(-) create mode 100644 src/test/java/org/bdware/MultiIndextTimeRocksDBUtilTest.java diff --git a/.gitignore b/.gitignore index d3e6078..e9e941c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ /build/ +/testoutput/ */build/* # Compiled class file *.class diff --git a/src/main/base/org/bdware/sc/node/Permission.java b/src/main/base/org/bdware/sc/node/Permission.java index 8b45851..2e59e4e 100644 --- a/src/main/base/org/bdware/sc/node/Permission.java +++ b/src/main/base/org/bdware/sc/node/Permission.java @@ -10,6 +10,7 @@ public enum Permission { RocksDB, MongoDB, BDWareTimeSeriesDB, + MultiTagIndexDB, SM2, AES, Ledger, diff --git a/src/main/entry/org/bdware/sc/db/MultiIndexTimeRocksDBUtil.java b/src/main/entry/org/bdware/sc/db/MultiIndexTimeRocksDBUtil.java index 0f9f50b..a486b23 100644 --- a/src/main/entry/org/bdware/sc/db/MultiIndexTimeRocksDBUtil.java +++ b/src/main/entry/org/bdware/sc/db/MultiIndexTimeRocksDBUtil.java @@ -18,6 +18,7 @@ import java.util.*; public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf { private static final Logger LOGGER = LogManager.getLogger(MultiIndexTimeRocksDBUtil.class); private final Map secondaryIndex; + static String primaryKey = "_DB_primary_"; public String dbPath; public String tableName; Random random = new Random(); @@ -42,7 +43,7 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf { Options options = new Options(); options.setCreateIfMissing(true); File lockFile = new File(file, "LOCK"); - File timeIndex = new File(dbPath + "/" + tableName + "/DB.primary.timeindex"); + File timeIndex = new File(dbPath + "/" + tableName + "/" + primaryIndex + ".timeindex"); LOGGER.trace("create directory " + file.getAbsolutePath() + ": " + file.mkdirs()); LOGGER.trace("delete file" + lockFile.getAbsolutePath() + ": " + lockFile.delete()); try { @@ -51,8 +52,16 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf { } catch (RocksDBException e) { e.printStackTrace(); } - primaryIndex = new TimeSerialIndex(timeIndex.getAbsolutePath()); + secondaryIndex.put(primaryKey, primaryIndex); + for (File f : timeIndex.getParentFile().listFiles()) { + if (!f.getName().endsWith(".timeindex")) continue; + if (f.getName().equals(timeIndex.getName())) continue; + TimeSerialIndex index = + new TimeSerialIndex(f.getAbsolutePath()); + secondaryIndex.putIfAbsent(f.getName().substring(0, + f.getName().length() - ".timeindex".length()), index); + } } public synchronized void put(String label, String val) { @@ -141,8 +150,13 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf { public long size(String label) { try { - TimeSerialIndex index = getIndex(label); - if (index != null) return index.size(); + if (label == null || label.length() == 0) { + return primaryIndex.size(); + } + if (secondaryIndex.containsKey(label)) { + TimeSerialIndex index = getIndex(label); + return index.size(); + } } catch (Exception e) { e.printStackTrace(); } @@ -152,17 +166,23 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf { public List queryByOffset(String label, long offset, int count) { List ret = new ArrayList<>(); TimeSerialIndex index = getIndex(label); - List data = index.request(offset, count); - for (Long l : data) { + List data = index.requestIndexEntry(offset, count); + for (TimeSerialIndex.IndexEntry entry : data) { try { - String t = new String(db.get(longToByte(l))); + String t = new String(db.get(longToByte(entry.value))); JsonObject jo; if (!t.isEmpty()) { - jo = JsonUtil.parseStringAsJsonObject(t); + try { + jo = JsonUtil.parseStringAsJsonObject(t); + } catch (Exception e) { + jo = new JsonObject(); + jo.addProperty("data", t); + } } else { jo = new JsonObject(); } - jo.addProperty("key", l.toString()); + jo.addProperty("key", entry.value); + jo.addProperty("timestamp", entry.key); ret.add(jo); } catch (Exception e) { e.printStackTrace(); @@ -181,7 +201,7 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf { } private TimeSerialIndex getIndex(String label) { - if (null == label) { + if (null == label || label.length() == 0) { return primaryIndex; } if (secondaryIndex.containsKey(label)) { @@ -275,6 +295,31 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf { // manuellyIndex可参考LenVarTimeIndex写法 } + public void close() { + db.close(); + } + + public List getIndexStartWith(String prefix) { + List ret = new ArrayList<>(); + if (prefix == null) { + prefix = ""; + } + for (String key : secondaryIndex.keySet()) { + if (key.startsWith(prefix)) + ret.add(key); + } + ret.remove(primaryKey); + return ret; + } + + public List getAllIndexKey() { + Set data = secondaryIndex.keySet(); + List ret = new ArrayList<>(); + ret.addAll(data); + ret.remove(primaryKey); + return ret; + } + // TODO static class BytesPair { byte[] key, value; diff --git a/src/main/entry/org/bdware/sc/index/TimeSerialIndex.java b/src/main/entry/org/bdware/sc/index/TimeSerialIndex.java index dd41800..dcda8dd 100644 --- a/src/main/entry/org/bdware/sc/index/TimeSerialIndex.java +++ b/src/main/entry/org/bdware/sc/index/TimeSerialIndex.java @@ -85,6 +85,39 @@ public class TimeSerialIndex { } else return new ArrayList<>(); } + public static class IndexEntry { + public long key, value; + } + + public synchronized List requestIndexEntry(long offset, int len) { + List ret = new ArrayList<>(); + if (offset < 0) offset = 0; + if (offset < fileSize) { + long pos = 0; + try { + pos = file.getFilePointer(); + file.seek(2L * offset * 8L); + for (; offset < fileSize && len > 0; len--) { + IndexEntry entry = new IndexEntry(); + entry.key = file.readLong(); + entry.value = file.readLong(); + ret.add(entry); + offset++; + } + file.seek(pos); + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + file.seek(pos); + } catch (IOException e) { + e.printStackTrace(); + } + } + return ret; + } else return new ArrayList<>(); + } + private long getIndex(long offset) { if (offset < fileSize) { try { @@ -109,7 +142,7 @@ public class TimeSerialIndex { public synchronized long findNearest(long timeStamp) { try { long pos = file.getFilePointer(); - long ret = binarySearch(0L, fileSize + 1, timeStamp); + long ret = binarySearch(0L, fileSize, timeStamp); file.seek(pos); return ret; } catch (IOException e) { @@ -120,9 +153,14 @@ public class TimeSerialIndex { } private long binarySearch(long start, long end, long timeStamp) { - if (start >= end - 1) return start; + if (end <= 0) return end; + if (start >= end - 1) { + long key = getIndex(end - 1); + if (key >= timeStamp) { + return end - 1; + } else return end; + } long mid = (start + end) / 2; - if (mid >= end - 1) return mid; long key = getIndex(mid); if (key >= timeStamp) { return binarySearch(start, mid, timeStamp); diff --git a/src/test/java/org/bdware/MultiIndextTimeRocksDBUtilTest.java b/src/test/java/org/bdware/MultiIndextTimeRocksDBUtilTest.java new file mode 100644 index 0000000..ffd46d5 --- /dev/null +++ b/src/test/java/org/bdware/MultiIndextTimeRocksDBUtilTest.java @@ -0,0 +1,51 @@ +package org.bdware; + +import com.google.gson.JsonObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.bdware.sc.db.MultiIndexTimeRocksDBUtil; +import org.junit.Before; +import org.junit.Test; + +import java.util.List; + +public class MultiIndextTimeRocksDBUtilTest { + static Logger LOGGER = LogManager.getLogger(MultiIndextTimeRocksDBUtilTest.class); + MultiIndexTimeRocksDBUtil util; + + @Before + public void init() { + util = new MultiIndexTimeRocksDBUtil("testoutput/timeIndexDB", "defaultTable"); + } + + @Test + public void put() { + for (int i = 0; i < 100; i++) { + try { + Thread.sleep(10); + } catch (InterruptedException e) { + e.printStackTrace(); + } + util.put("abc_123", "ddd " + System.currentTimeMillis()); + } + } + + @Test + public void listAll() { + List tt = util.queryByOffset(null, 0, 100); + for (int i = 0; i < tt.size(); i++) { + long val = tt.get(i).get("timestamp").getAsLong(); + LOGGER.info(String.format("pre %d cu %d suc %d", util.queryOffset("abc_123", val - 1L), + util.queryOffset("abc_123", val), util.queryOffset("abc_123", val + 1L), util.queryOffset("abc_123", val - 1L))); + } + + } + + @Test + public void run() { + LOGGER.info(util.queryOffset("abc_123", 1641454140657L)); + LOGGER.info(util.queryOffset("abc_123", 1641454140669L)); + LOGGER.info(util.queryOffset("abc_123", 1641450028482L)); + LOGGER.info(util.queryOffset("abc_123", 1641450028483L)); + } +} diff --git a/src/test/java/org/bdware/sc/LedgerUtilTest.java b/src/test/java/org/bdware/sc/LedgerUtilTest.java index d82fdb3..f8129a5 100644 --- a/src/test/java/org/bdware/sc/LedgerUtilTest.java +++ b/src/test/java/org/bdware/sc/LedgerUtilTest.java @@ -1,21 +1,17 @@ package org.bdware.sc; -import org.bdware.bdledger.api.grpc.Client; -//import org.bdware.bdledger.api.grpc.pb.QueryOuterClass; -import org.bdware.bdledger.api.grpc.pb.QueryProto; import org.junit.Test; public class LedgerUtilTest { @Test public void query(){ - String str = "0828c62576606e27ef8d079dea60cadd6eb03351"; - Client c = new Client("022.node.internetapi.cn",21121); - QueryProto.GetTransactionByHashResponse syncResult = c.getTransactionByHashSync("default", "5c79fb8d71771615a5c8173b8089cd4841c3adfb"); - System.out.println(new String(syncResult.getTransaction().getData().toByteArray())); - System.out.println(syncResult.getTransaction().getData().toByteArray().length); - - // 092：47.107.94.135 +// String str = "0828c62576606e27ef8d079dea60cadd6eb03351"; +// Client c = new Client("022.node.internetapi.cn",21121); +// QueryProto.GetTransactionByHashResponse syncResult = c.getTransactionByHashSync("default", "5c79fb8d71771615a5c8173b8089cd4841c3adfb"); +// System.out.println(new String(syncResult.getTransaction().getData().toByteArray())); +// System.out.println(syncResult.getTransaction().getData().toByteArray().length); + // 092：47.107.94.135 // 093：120.79.215.187 // QueryOuterClass.GetTransactionByHashResponse hash = c.getTransactionByHashSync("test", str); // String str2 = hash.getTransaction().getFrom().toStringUtf8(); diff --git a/src/test/java/org/bdware/sc/boundary/utils/SM2UtilTest.java b/src/test/java/org/bdware/sc/boundary/utils/SM2UtilTest.java index 0d579e5..d3bc552 100644 --- a/src/test/java/org/bdware/sc/boundary/utils/SM2UtilTest.java +++ b/src/test/java/org/bdware/sc/boundary/utils/SM2UtilTest.java @@ -10,7 +10,6 @@ import org.junit.Test; import org.zz.gmhelper.BCECUtil; import org.zz.gmhelper.SM2KeyPair; import org.zz.gmhelper.SM2Util; -import sun.misc.BASE64Encoder; import java.math.BigInteger; import java.security.KeyPair; @@ -25,8 +24,8 @@ public class SM2UtilTest { String gstr = "09146332716e7767327923056946406e49570e5909146332716e7767327923056946406e49570e59"; byte[] bytes = ByteUtils.fromHexString(gstr); - String str = new BASE64Encoder().encode(bytes); - System.out.println(str); + // String str = new BASE64Encoder().encode(bytes); + // System.out.println(str); } @Test