fix: binarySearch bugs in TimeSerialIndex

feat: support multi tag time index util

.gitignore

@@ -1,4 +1,5 @@
 /build/
+/testoutput/
 */build/*
 # Compiled class file
 *.class

src/main/base/org/bdware/sc/node/Permission.java

@@ -10,6 +10,7 @@ public enum Permission {
     RocksDB,
     MongoDB,
     BDWareTimeSeriesDB,
+    MultiTagIndexDB,
     SM2,
     AES,
     Ledger,

src/main/entry/org/bdware/sc/db/MultiIndexTimeRocksDBUtil.java

@@ -18,6 +18,7 @@ import java.util.*;
 public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
     private static final Logger LOGGER = LogManager.getLogger(MultiIndexTimeRocksDBUtil.class);
     private final Map<String, TimeSerialIndex> secondaryIndex;
+    static String primaryKey = "_DB_primary_";
     public String dbPath;
     public String tableName;
     Random random = new Random();
@@ -42,7 +43,7 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
         Options options = new Options();
         options.setCreateIfMissing(true);
         File lockFile = new File(file, "LOCK");
-        File timeIndex = new File(dbPath + "/" + tableName + "/DB.primary.timeindex");
+        File timeIndex = new File(dbPath + "/" + tableName + "/" + primaryIndex + ".timeindex");
         LOGGER.trace("create directory " + file.getAbsolutePath() + ": " + file.mkdirs());
         LOGGER.trace("delete file" + lockFile.getAbsolutePath() + ": " + lockFile.delete());
         try {
@@ -51,8 +52,16 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
         } catch (RocksDBException e) {
             e.printStackTrace();
         }
-
         primaryIndex = new TimeSerialIndex(timeIndex.getAbsolutePath());
+        secondaryIndex.put(primaryKey, primaryIndex);
+        for (File f : timeIndex.getParentFile().listFiles()) {
+            if (!f.getName().endsWith(".timeindex")) continue;
+            if (f.getName().equals(timeIndex.getName())) continue;
+            TimeSerialIndex index =
+                    new TimeSerialIndex(f.getAbsolutePath());
+            secondaryIndex.putIfAbsent(f.getName().substring(0,
+                    f.getName().length() - ".timeindex".length()), index);
+        }
     }
 
     public synchronized void put(String label, String val) {
@@ -141,8 +150,13 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
 
     public long size(String label) {
         try {
-            TimeSerialIndex index = getIndex(label);
+            if (label == null || label.length() == 0) {
-            if (index != null) return index.size();
+                return primaryIndex.size();
+            }
+            if (secondaryIndex.containsKey(label)) {
+                TimeSerialIndex index = getIndex(label);
+                return index.size();
+            }
         } catch (Exception e) {
             e.printStackTrace();
         }
@@ -152,17 +166,23 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
     public List<JsonObject> queryByOffset(String label, long offset, int count) {
         List<JsonObject> ret = new ArrayList<>();
         TimeSerialIndex index = getIndex(label);
-        List<Long> data = index.request(offset, count);
+        List<TimeSerialIndex.IndexEntry> data = index.requestIndexEntry(offset, count);
-        for (Long l : data) {
+        for (TimeSerialIndex.IndexEntry entry : data) {
             try {
-                String t = new String(db.get(longToByte(l)));
+                String t = new String(db.get(longToByte(entry.value)));
                 JsonObject jo;
                 if (!t.isEmpty()) {
-                    jo = JsonUtil.parseStringAsJsonObject(t);
+                    try {
+                        jo = JsonUtil.parseStringAsJsonObject(t);
+                    } catch (Exception e) {
+                        jo = new JsonObject();
+                        jo.addProperty("data", t);
+                    }
                 } else {
                     jo = new JsonObject();
                 }
-                jo.addProperty("key", l.toString());
+                jo.addProperty("key", entry.value);
+                jo.addProperty("timestamp", entry.key);
                 ret.add(jo);
             } catch (Exception e) {
                 e.printStackTrace();
@@ -181,7 +201,7 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
     }
 
     private TimeSerialIndex getIndex(String label) {
-        if (null == label) {
+        if (null == label || label.length() == 0) {
             return primaryIndex;
         }
         if (secondaryIndex.containsKey(label)) {
@@ -275,6 +295,31 @@ public class MultiIndexTimeRocksDBUtil implements MultiIndexTimeDBUtilIntf {
         // manuellyIndex可参考LenVarTimeIndex写法
     }
 
+    public void close() {
+        db.close();
+    }
+
+    public List<String> getIndexStartWith(String prefix) {
+        List<String> ret = new ArrayList<>();
+        if (prefix == null) {
+            prefix = "";
+        }
+        for (String key : secondaryIndex.keySet()) {
+            if (key.startsWith(prefix))
+                ret.add(key);
+        }
+        ret.remove(primaryKey);
+        return ret;
+    }
+
+    public List<String> getAllIndexKey() {
+        Set<String> data = secondaryIndex.keySet();
+        List<String> ret = new ArrayList<>();
+        ret.addAll(data);
+        ret.remove(primaryKey);
+        return ret;
+    }
+
     // TODO
     static class BytesPair {
         byte[] key, value;

src/main/entry/org/bdware/sc/index/TimeSerialIndex.java

@@ -85,6 +85,39 @@ public class TimeSerialIndex {
         } else return new ArrayList<>();
     }
 
+    public static class IndexEntry {
+        public long key, value;
+    }
+
+    public synchronized List<IndexEntry> requestIndexEntry(long offset, int len) {
+        List<IndexEntry> ret = new ArrayList<>();
+        if (offset < 0) offset = 0;
+        if (offset < fileSize) {
+            long pos = 0;
+            try {
+                pos = file.getFilePointer();
+                file.seek(2L * offset * 8L);
+                for (; offset < fileSize && len > 0; len--) {
+                    IndexEntry entry = new IndexEntry();
+                    entry.key = file.readLong();
+                    entry.value = file.readLong();
+                    ret.add(entry);
+                    offset++;
+                }
+                file.seek(pos);
+            } catch (IOException e) {
+                e.printStackTrace();
+            } finally {
+                try {
+                    file.seek(pos);
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }
+            }
+            return ret;
+        } else return new ArrayList<>();
+    }
+
     private long getIndex(long offset) {
         if (offset < fileSize) {
             try {
@@ -109,7 +142,7 @@ public class TimeSerialIndex {
     public synchronized long findNearest(long timeStamp) {
         try {
             long pos = file.getFilePointer();
-            long ret = binarySearch(0L, fileSize + 1, timeStamp);
+            long ret = binarySearch(0L, fileSize, timeStamp);
             file.seek(pos);
             return ret;
         } catch (IOException e) {
@@ -120,9 +153,14 @@ public class TimeSerialIndex {
     }
 
     private long binarySearch(long start, long end, long timeStamp) {
-        if (start >= end - 1) return start;
+        if (end <= 0) return end;
+        if (start >= end - 1) {
+            long key = getIndex(end - 1);
+            if (key >= timeStamp) {
+                return end - 1;
+            } else return end;
+        }
         long mid = (start + end) / 2;
-        if (mid >= end - 1) return mid;
         long key = getIndex(mid);
         if (key >= timeStamp) {
             return binarySearch(start, mid, timeStamp);

src/test/java/org/bdware/MultiIndextTimeRocksDBUtilTest.java

@@ -0,0 +1,51 @@
+package org.bdware;
+
+import com.google.gson.JsonObject;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.bdware.sc.db.MultiIndexTimeRocksDBUtil;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+public class MultiIndextTimeRocksDBUtilTest {
+    static Logger LOGGER = LogManager.getLogger(MultiIndextTimeRocksDBUtilTest.class);
+    MultiIndexTimeRocksDBUtil util;
+
+    @Before
+    public void init() {
+        util = new MultiIndexTimeRocksDBUtil("testoutput/timeIndexDB", "defaultTable");
+    }
+
+    @Test
+    public void put() {
+        for (int i = 0; i < 100; i++) {
+            try {
+                Thread.sleep(10);
+            } catch (InterruptedException e) {
+                e.printStackTrace();
+            }
+            util.put("abc_123", "ddd " + System.currentTimeMillis());
+        }
+    }
+
+    @Test
+    public void listAll() {
+        List<JsonObject> tt = util.queryByOffset(null, 0, 100);
+        for (int i = 0; i < tt.size(); i++) {
+            long val = tt.get(i).get("timestamp").getAsLong();
+            LOGGER.info(String.format("pre %d cu %d suc %d", util.queryOffset("abc_123", val - 1L),
+                    util.queryOffset("abc_123", val), util.queryOffset("abc_123", val + 1L), util.queryOffset("abc_123", val - 1L)));
+        }
+
+    }
+
+    @Test
+    public void run() {
+        LOGGER.info(util.queryOffset("abc_123", 1641454140657L));
+        LOGGER.info(util.queryOffset("abc_123", 1641454140669L));
+        LOGGER.info(util.queryOffset("abc_123", 1641450028482L));
+        LOGGER.info(util.queryOffset("abc_123", 1641450028483L));
+    }
+}

src/test/java/org/bdware/sc/LedgerUtilTest.java

@@ -1,21 +1,17 @@
 package org.bdware.sc;
 
-import org.bdware.bdledger.api.grpc.Client;
-//import org.bdware.bdledger.api.grpc.pb.QueryOuterClass;
-import org.bdware.bdledger.api.grpc.pb.QueryProto;
 import org.junit.Test;
 
 public class LedgerUtilTest {
 
     @Test
     public void query(){
-        String str = "0828c62576606e27ef8d079dea60cadd6eb03351";
+//        String str = "0828c62576606e27ef8d079dea60cadd6eb03351";
-        Client c = new Client("022.node.internetapi.cn",21121);
+//        Client c = new Client("022.node.internetapi.cn",21121);
-        QueryProto.GetTransactionByHashResponse syncResult = c.getTransactionByHashSync("default", "5c79fb8d71771615a5c8173b8089cd4841c3adfb");
+//        QueryProto.GetTransactionByHashResponse syncResult = c.getTransactionByHashSync("default", "5c79fb8d71771615a5c8173b8089cd4841c3adfb");
-        System.out.println(new String(syncResult.getTransaction().getData().toByteArray()));
+//        System.out.println(new String(syncResult.getTransaction().getData().toByteArray()));
-        System.out.println(syncResult.getTransaction().getData().toByteArray().length);
+//        System.out.println(syncResult.getTransaction().getData().toByteArray().length);
-
+       // 092：47.107.94.135
-        // 092：47.107.94.135
        // 093：120.79.215.187
 //        QueryOuterClass.GetTransactionByHashResponse hash = c.getTransactionByHashSync("test", str);
 //        String str2 =  hash.getTransaction().getFrom().toStringUtf8();

src/test/java/org/bdware/sc/boundary/utils/SM2UtilTest.java

@@ -10,7 +10,6 @@ import org.junit.Test;
 import org.zz.gmhelper.BCECUtil;
 import org.zz.gmhelper.SM2KeyPair;
 import org.zz.gmhelper.SM2Util;
-import sun.misc.BASE64Encoder;
 
 import java.math.BigInteger;
 import java.security.KeyPair;
@@ -25,8 +24,8 @@ public class SM2UtilTest {
         String gstr =
                 "09146332716e7767327923056946406e49570e5909146332716e7767327923056946406e49570e59";
         byte[] bytes = ByteUtils.fromHexString(gstr);
-        String str = new BASE64Encoder().encode(bytes);
+       // String str = new BASE64Encoder().encode(bytes);
-        System.out.println(str);
+     //   System.out.println(str);
     }
 
     @Test