add repo topology

add repo statistics
fix concurrent bugs in getAllTopology

backend/build.gradle

@@ -4,7 +4,7 @@ buildscript {
         mavenLocal()
     }
     dependencies {
-        classpath "org.bdware.bdcontract:simple-ypk-packer:0.5.0"
+        classpath "org.bdware.bdcontract:simple-ypk-packer:0.5.3"
         classpath "org.bdware.bdcontract:ypk-deploy-tool:0.5.2"
 
     }
@@ -14,7 +14,7 @@ plugins {
     id 'java'
     id 'java-library'
 }
-version = "1.2.2"
+version = "1.2.5"
 group = 'org.bdware.sc.controlproxy'
 
 tasks.withType(JavaCompile) {
@@ -82,7 +82,7 @@ task copyJar(type: Copy, dependsOn: [":backend:jar", ":backend:copyLibs"]) {
 }
 task grepCP(dependsOn: [":backend:copyJar"]) {
     doLast {
-        org.bdware.datanet.YPKPacker.grepJarByCPVersion("./backend/build/output/libs", org.bdware.datanet.CPVersion.cp_1_5_3)
+        org.bdware.datanet.YPKPacker.grepJarByCPVersion("./backend/build/output/libs", org.bdware.datanet.CPVersion.cp_1_5_7)
         //      org.bdware.datanet.YPKPacker.grepCPLibWithFilter("./backend/cplibs/libs/", "./backend/build/output/libs", "./backend/grepcp.list")
     }
 }

backend/debugconf.json

@@ -1,15 +1,15 @@
 {
-  "agentHttpAddr": "127.0.0.1:18010",
+  "agentAddress": "127.0.0.1:18000",
   "createParam": {
     "privateKey": "4616ff0e2a4f982364914f9be30b51c6bc6ccb6602114a9ee8792f2ccf67465b",
     "publicKey": "04f9b9b8f324908464f78a6235e2dd93e4dfdaf045e9b1b5cfd57374516cc61a79a86fc2b42d3321a5b49a0f25381a7bed61901b40b729f72354e716673d551e98",
-    "prefix": "bdtest",
+    "prefix": "shanxi",
     "router": "GlobalRouter",
-    "routerURI": "tcp://127.0.0.1:18041",
+    "routerURI": "tcp://39.104.201.40:18041",
     "auditType": "OnlyHash"
   },
-
+  "killBeforeStart": "ControlProxy",
-  "script": "/Users/huaqiancai/BDWare/datanet/ControlProxy/backend/build/ControlProxy-0.9.8.ypk",
+  "ypkPath": "/Users/huaqiancai/BDWare/datanet/ControlProxy/backend/build/ControlProxy-1.2.5.ypk",
-  "pubKey": "04d1924329f72ced148f6f333fb985ccbaa31b1e3aacf10be5f43d4a4ff5ad88899a005e79e37fc06993e1d66ada8cf8b711cb36f59538bb7d3e39e70fa9360ddd",
+  "publicKey": "04d1924329f72ced148f6f333fb985ccbaa31b1e3aacf10be5f43d4a4ff5ad88899a005e79e37fc06993e1d66ada8cf8b711cb36f59538bb7d3e39e70fa9360ddd",
-  "privKey": "589d94ee5688358a1c5c18430dd9c75097ddddebf769f139da36a807911d20f8"
+  "privateKey": "589d94ee5688358a1c5c18430dd9c75097ddddebf769f139da36a807911d20f8"
 }

backend/src/main/java/org/bdware/sc/controlproxy/RepoProxy.java

@@ -3,6 +3,7 @@ package org.bdware.sc.controlproxy;
 import com.google.gson.*;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.bdware.doip.audit.AuditDoaClient;
 import org.bdware.doip.audit.EndpointConfig;
 import org.bdware.doip.audit.SM2Signer;
 import org.bdware.doip.audit.client.AuditDoipClient;
@@ -13,6 +14,7 @@ import org.bdware.doip.audit.writer.AuditType;
 import org.bdware.doip.auditrepo.AutoAuditDO;
 import org.bdware.doip.codec.doipMessage.DoipMessage;
 import org.bdware.doip.endpoint.client.ClientConfig;
+import org.bdware.irp.exception.IrpClientException;
 import org.bdware.irp.stateinfo.StateInfoBase;
 import org.bdware.sc.engine.JSONTool;
 
@@ -46,7 +48,7 @@ public class RepoProxy {
             endpointConfig = storage.loadAsEndpointConfig();
             prefix = storage.load().get("prefix").getAsString();
 
-           router = storage.load().get("router").getAsString();
+            router = storage.load().get("router").getAsString();
 
         }
     }
@@ -59,6 +61,77 @@ public class RepoProxy {
         return "hello..dafdskf";
     }
 
+    public static Object getRepoStaticsMock(String repoDoid) {
+        JsonObject result = new JsonObject();
+        result.addProperty("doCount", 1234);
+        result.addProperty("invokeCount", 5678);
+        result.addProperty("doIndexCount", 1200);
+        return JSONTool.convertJsonElementToMirror(result);
+    }
+
+    public static Object getRepoStatistics(String repoDoid) {
+        JsonObject result = new JsonObject();
+        try {
+            DoipMessage helloMsg = getHelloMessage(repoDoid);
+            JsonObject helloAttrs = helloMsg.header.parameters.attributes;
+            result.add("doCount", getPropWithDefault(helloAttrs, "doCount", new JsonPrimitive(-1)));
+            result.add("invokeCount", getPropWithDefault(helloAttrs, "invokeCount", new JsonPrimitive(-1)));
+            result.add("doIndexCount", getPropWithDefault(helloAttrs, "doIndexCount", new JsonPrimitive(-1)));
+            LOGGER.info(helloMsg);
+            return JSONTool.convertJsonElementToMirror(result);
+        } catch (Exception e) {
+            result.addProperty("msg", e.getMessage());
+        }
+        result.addProperty("status", "offline");
+        return JSONTool.convertJsonElementToMirror(result);
+    }
+
+    private static JsonElement getPropWithDefault(JsonObject jo, String attr, JsonPrimitive defaultValue) {
+        if (jo != null && jo.has(attr))
+            return jo.get(attr);
+        return defaultValue;
+    }
+
+    private static DoipMessage getHelloMessage(String repoDoid) {
+        try {
+            AuditIrpClient irpClient = new AuditIrpClient(endpointConfig);
+            AuditConfig auditConfig = irpClient.getAuditConfig();
+            AuditDoipClient client = new AuditDoipClient(auditConfig, new SM2Signer(null));
+            StateInfoBase stateInfoBase = irpClient.resolve(repoDoid);
+            if (isDOIPRepo(stateInfoBase)) {
+                ClientConfig config = new ClientConfig(stateInfoBase.getValues("address"));
+                client.connect(config);
+                return client.helloSync(repoDoid);
+            }
+        } catch (Exception e) {
+        }
+        return null;
+    }
+
+    //用于新增repo时判断它是否在线。
+    //使用doip协议，发一个hello请求。
+    public static Object pingRepo(String repoDoid) {
+        JsonObject result = new JsonObject();
+        try {
+            if (getHelloMessage(repoDoid) != null) {
+                result.addProperty("status", "online");
+                return JSONTool.convertJsonElementToMirror(result);
+            }
+        } catch (Exception e) {
+            result.addProperty("msg", e.getMessage());
+        }
+        result.addProperty("status", "offline");
+        return JSONTool.convertJsonElementToMirror(result);
+    }
+
+    private static boolean isDOIPRepo(StateInfoBase base) {
+        JsonObject handleValue = base.getHandleValues();
+        return handleValue != null &&
+                handleValue.has("protocol") &&
+                handleValue.get("protocol").getAsString().equalsIgnoreCase("DOIP");
+
+    }
+
     public static JsonElement getDoAuditRule(Object obj) {
         JsonObject jo = JSONTool.convertMirrorToJson(obj).getAsJsonObject();
         String doId = jo.get("doId").getAsString();

backend/src/main/java/org/bdware/sc/controlproxy/TopologyCollector.java

@@ -19,6 +19,7 @@ public class TopologyCollector {
 
     static Logger LOGGER = LogManager.getLogger(TopologyCollector.class);
 
+
     public static List<JsonObject> getCurrentTopology() {
         TopologyGraph graph = new TopologyGraph();
         //通过SearchEngine，可拿到本级的Repo（或下组的SE)与SearchEngine的关系。
@@ -105,11 +106,31 @@ public class TopologyCollector {
         ContractResult contractResult = new Gson().fromJson(je, ContractResult.class);
         return contractResult;
     }
+
     //{
     //       "name":"小店区仓库","doId":"bdware.ss.ab/Repo1","type":"Repo","previousNode":["bdware.ss/Gateway1","bdware.ss/Gateway2","bdware.ss.ab/Repo2","bdware.ss.zz/SearchEngine"]}
     //
+    public static List<JsonObject> getLevelTopologyByDoId(String componentDoid, int level) {
+        TopologyGraph graph = new TopologyGraph();
+        graph.addGraph(getLevelTopology(2));
+        graph.grepByDoId(componentDoid);
+        return graph.ret;
+    }
 
-    public static List<JsonObject> getLevelTopology(int arg) {
+    static long lastUpdate = 0;
+    static TopologyGraph cachedGraph;
+
+    public static boolean isCacheValid() {
+        if (System.currentTimeMillis() - lastUpdate < 10000) {
+            return true;
+        }
+        return false;
+    }
+
+    public synchronized static List<JsonObject> getLevelTopology(int arg) {
+        if (isCacheValid()) {
+            return cachedGraph.ret;
+        }
         List<JsonObject> points = getCurrentTopology();
         TopologyGraph graph = new TopologyGraph();
         graph.addGraph(points);
@@ -135,6 +156,8 @@ public class TopologyCollector {
                 }
             }
         }
+        lastUpdate = System.currentTimeMillis();
+        cachedGraph = graph;
         return graph.ret;
     }
 }

backend/src/main/java/org/bdware/sc/controlproxy/TopologyGraph.java

@@ -3,6 +3,7 @@ package org.bdware.sc.controlproxy;
 import com.google.gson.JsonArray;
 import com.google.gson.JsonElement;
 import com.google.gson.JsonObject;
+import com.google.gson.JsonPrimitive;
 
 import java.util.*;
 
@@ -85,5 +86,34 @@ public class TopologyGraph {
         return doIdToNode.get(from);
     }
 
+    private boolean hasPreviousNode(JsonObject currentNode, String previousNode) {
+        if (currentNode.has("previousNode")) {
+            JsonElement pn = currentNode.get("previousNode");
+            if (pn.isJsonArray()) {
+                JsonArray previousNodes = pn.getAsJsonArray();
+                return previousNodes.contains(new JsonPrimitive(previousNode));
+            }
+        }
+        return false;
+    }
 
+    public List<JsonObject> grepByDoId(String componentDoid) {
+        TopologyGraph newGraph = new TopologyGraph();
+        JsonObject currentNode = this.getOrCreateNode(componentDoid);
+        newGraph.addNode(currentNode);
+        //只添加上一级的节点
+        //无需处理
+        // JsonArray previousNodes = currentNode.get("previousNode").getAsJsonArray();
+        //只添加下一级的节点
+        for (JsonObject jo : doIdToNode.values()) {
+            if (hasPreviousNode(jo, componentDoid)) {
+                JsonObject jo2 = jo.deepCopy();
+                JsonArray arr = new JsonArray();
+                arr.add(componentDoid);
+                jo2.add("previousNodes", arr);
+                newGraph.addNode(jo2);
+            }
+        }
+        return newGraph.ret;
+    }
 }

backend/yjs/ControlProxy.yjs

@@ -40,7 +40,7 @@ contract ControlProxy {
     export function isOwner() {
         return checkPermission(requester);
     }
-    @Descripton("参数为，{\"doId\":\"bdware.ss/Repox\"}")
+    @Description("参数为，{\"doId\":\"bdware.ss/Repox\"}")
     export function addRepo(arg) {
         arg = convertArgToJson(arg);
         if (arg.doId ==undefined ){
@@ -54,7 +54,7 @@ contract ControlProxy {
         };
     }
 //网关需新增的接口
-    @Descripton("参数为，{\"doId\":\"bdware.ss/Repox\"}")
+    @Description("参数为，{\"doId\":\"bdware.ss/Repox\"}")
     export function pingRepo(arg) {
         arg = convertArgToJson(arg);
         if (arg.doId ==undefined ){
@@ -71,9 +71,17 @@ contract ControlProxy {
         "msg":"repo is unconnected", "code":1
         };
     }
+    @Description("shanxi/TestLocal shanxi/Repository")
+    export function testPingRepo(arg) {
+        return org.bdware.sc.controlproxy.RepoProxy.pingRepo(arg);
+    }
 
+    @Description("shanxi/TestLocal shanxi/Repository")
+    export function testRepoStatistics(arg) {
+        return org.bdware.sc.controlproxy.RepoProxy.getRepoStatistics(arg);
+    }
 //网关的接口
-    @Descripton("无参数")
+    @Description("无参数")
     export function getRepoList(arg) {
     //use configed  arg.routerId;
         ret = executeContract(Global.router, "listRepo", "");
@@ -82,7 +90,7 @@ contract ControlProxy {
         return ret.result;
     }
 //网关的接口
-    @Descripton("参数为 {\"doId\":\"bdware.ss/Repox\"}")
+    @Description("参数为 {\"doId\":\"bdware.ss/Repox\"}")
     export function deleteRepo(arg) {
         arg = convertArgToJson(arg);
         if (arg.doId != null && arg.doId.indexOf(Global.prefix) != -1){
@@ -96,7 +104,7 @@ contract ControlProxy {
         };
     }
 //网关的接口 需新增
-    @Descripton("参数为 [{\"doId\":\"bdware.ss/Repox\"},{\"doId\":\"bdware.ss/Repoy\"}]")
+    @Description("参数为 [{\"doId\":\"bdware.ss/Repox\"},{\"doId\":\"bdware.ss/Repoy\"}]")
     export function deleteRepoList(arg) {
         arg = convertArgToJson(arg);
         var ret = [];
@@ -108,7 +116,7 @@ contract ControlProxy {
         return ret;
     }
 //网关的接口
-    @Descripton("参数为 {\"doId\":..., \"address\":..., \"owner\":..., }")
+    @Description("参数为 {\"doId\":..., \"address\":..., \"owner\":..., }")
     export function updateRepo(arg) {
         arg = convertArgToJson(arg);
         if (arg.doId!=null){
@@ -134,7 +142,7 @@ contract ControlProxy {
 //总体拓扑图
 //@范博
     export function getAllTopology2() {
-            return getAllTopology(2);
+        return getAllTopology(2);
         var nodeList = [];
         nodeList.push({
         "name":"小店区仓库", "doId":"bdware.ss.ab/Repo1", "type":"Repo", "previousNode":["bdware.ss/Gateway1", "bdware.ss/Gateway2", "bdware.ss.ab/Repo2", "bdware.ss.zz/SearchEngine"]

backend/yjs/repodetail.yjs

@@ -1,97 +1,87 @@
-module RepoDetail{
+module RepoDetail {
-  //宏伟
+//宏伟
-  //如果有指定repo，那就是指定repo的DOlist。
+//如果有指定repo，那就是指定repo的DOlist。
-  //如果没指定repo，那就是全部的。
+//如果没指定repo，那就是全部的。
-  //这是search engine的接口，为什么还没接上？？
+//这是search engine的接口，为什么还没接上？？
-  //TODO TODO
+//TODO TODO
-  export function getDoList(arg){
+    export function getDoList(arg) {
-    print("GetDOList===========");
+        print("GetDOList===========");
-    var ret = executeContract("SearchEngine","getDoList",arg);
+        var ret = executeContract("SearchEngine", "getDoList", arg);
-    return ret.result;
+        return ret.result;
-  }
-  export function getDoList2(arg){
-     arg =  convertArgToJson(arg);
-    //if (arg.doId == null) 
-    //  return {
-    //    "msg":"missing arguments doId", "code":1, params: arg
-    //  };
-    //if (arg.count>1000)
-    //  return {
-    //    "msg":"request to much!", "code":1
-    //  };
-    var ret = [];
-    if(arg.count==undefined) arg.count=10;
-    if(arg.offset==undefined) arg.offset=0;
-    for (var i=arg.offset; i<arg.offset/1.0+arg.count/1.0;i++){
-      var fakeDo = {
-      
-      };
-      fakeDo.doId="bdware.ss/Repo1/abc"+i;
-      fakeDo.originalData={doid:fakeDo.doId,info:"ddddd", tag: ["aaa", "bbb", "ccc"]}
-      fakeDo.enableIndex = true
-      ret.push(fakeDo);
     }
-    return {data: ret, total: 100};
+    export function getDoList2(arg) {
-  }
+        arg = convertArgToJson(arg);
-  //
+        //if (arg.doId == null)
-  export function getRepoTopology(arg){
+        //  return {
-    arg =  convertArgToJson(arg);
+        //    "msg":"missing arguments doId", "code":1, params: arg
-    if (arg.doId == null) 
+        //  };
-      return {
+        //if (arg.count>1000)
-        "msg":"missing arguments doId", "code":1};
+        //  return {
-    var nodeList = [];
+        //    "msg":"request to much!", "code":1
-    nodeList.push({
+        //  };
-      "name":"小店区仓库","doId":"bdware.ss.ab/Repo1","previousNode":["bdware.ss/Gateway1","bdware.ss/Gateway2"]}
+        var ret = [];
-                 );
+        if (arg.count==undefined) arg.count = 10;
-    nodeList.push({
+        if (arg.offset==undefined) arg.offset = 0;
-      "name":"太原内部共享网关","doId":"bdware.ss/Gateway1","previousNode":["bdware/Gateway1", "bdware/Gateway2"]}
+        for (var i = arg.offset;
-                 );
+        i<arg.offset/1.0+arg.count/1.0;
-    nodeList.push({
+        i++){
-      "name":"太原对外开放网关","doId":"bdware.ss/Gateway2", "previousNode":["bdware/Gateway1"]}
+            var fakeDo = {
-                 );
+            };
-    nodeList.push({
+            fakeDo.doId = "bdware.ss/Repo1/abc"+i;
-      "name":"山西内部共享网关","doId":"bdware/Gateway1"}
+            fakeDo.originalData = {
-                 );
+            doid:fakeDo.doId, info:"ddddd", tag: ["aaa", "bbb", "ccc"]
-    nodeList.push({
+            }
-      "name":"山西内部共享网关2","doId":"bdware/Gateway2"}
+                    fakeDo.enableIndex = true
-                 );
+                    ret.push(fakeDo);
-    return nodeList;
+        }
-  }
+        return {
-  //指标1 获取某个repo的DO数量。
+        data: ret, total: 100
-  //使用search.yjs里的getDoCount
+        };
-  //指标2 获取某个repo的操作次数。使用search.yjs中的
+    }
-  //countRecordByDate
+//TODO check arguments
-  //参数格式为：
+    export function getRepoTopology(arg) {
-  //{"repoId":"xxx.xxx/xxx","startTime":1641454745128, "endTime":1641454746128,"interval":5000, "eventPrefix":["abc"]}"
+        return org.bdware.sc.controlproxy.TopologyCollector.getLevelTopologyByDoId(arg, 2);
-  //比如，查询总搜索次数，参数格式为：eventPrefix:["doip_search"]
+    }
-  //查询调用统计，参数格式为eventPrefix:[""]，然后interval/startTime/fromTime传对了就行。
+//指标1 获取某个repo的DO数量。
-  //索引统计，暂时还没有。
+//使用search.yjs里的getDoCount
-  export function getStaticIndex(arg){
+//指标2 获取某个repo的操作次数。使用search.yjs中的
-    arg =  convertArgToJson(arg);
+//countRecordByDate
-    if (arg.from==undefined)
+//参数格式为：
-      return {
+//{"repoId":"xxx.xxx/xxx","startTime":1641454745128, "endTime":1641454746128,"interval":5000, "eventPrefix":["abc"]}"
-        "msg":"missing arguments: from", "code":1};
+//比如，查询总搜索次数，参数格式为：eventPrefix:["doip_search"]
-    if (arg.to==undefined)
+//查询调用统计，参数格式为eventPrefix:[""]，然后interval/startTime/fromTime传对了就行。
-      return {
+//索引统计，暂时还没有。
-        "msg":"missing arguments: to", "code":1};
+    export function getStaticIndex(arg) {
-    if (arg.interval==undefined)
+        arg = convertArgToJson(arg);
-      return {
+        if (arg.from==undefined)
-        "msg":"missing arguments: interval", "code":1};
+            return {
-    switch(arg.tag){
+            "msg":"missing arguments: from", "code":1
-      case "search":
+            };
-      case "index":
+        if (arg.to==undefined)
-      case "call":
+            return {
+            "msg":"missing arguments: to", "code":1
+            };
+        if (arg.interval==undefined)
+            return {
+            "msg":"missing arguments: interval", "code":1
+            };
+        switch(arg.tag){
+        case "search":
+        case "index":
+        case "call":
         return fakeList(from, to, interval);
+        }
+        return {
+        "msg":"unreconized tag", "code":1
+        };
     }
-    return {
+    function fakeList(from, to, interval) {
-      "msg":"unreconized tag", "code":1};
+        var ret = [];
-  }
+        for (i = from;
-  function fakeList(from, to, interval){
+        i<to;
-    var ret =[];
+        i++){
-    for (i=from;i<to;i++){
+            ret.push(Math.floor(Math.random()*200+20));
-      ret.push(Math.floor(Math.random()*200+20));
+        }
+        return ret;
     }
-    return ret;
-  }
-
 }