From 410a133adaa482ec8c21ed28ec1ddf9ad2f575a8 Mon Sep 17 00:00:00 2001 From: CaiHQ Date: Tue, 28 Jun 2022 15:39:02 +0800 Subject: [PATCH] add repo topology add repo statistics fix concurrent bugs in getAllTopology --- backend/build.gradle | 6 +- backend/debugconf.json | 14 +- .../org/bdware/sc/controlproxy/RepoProxy.java | 75 +++++++- .../sc/controlproxy/TopologyCollector.java | 25 ++- .../bdware/sc/controlproxy/TopologyGraph.java | 30 +++ backend/yjs/ControlProxy.yjs | 22 ++- backend/yjs/repodetail.yjs | 174 +++++++++--------- 7 files changed, 235 insertions(+), 111 deletions(-) diff --git a/backend/build.gradle b/backend/build.gradle index 94bef4b..15bdb2b 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -4,7 +4,7 @@ buildscript { mavenLocal() } dependencies { - classpath "org.bdware.bdcontract:simple-ypk-packer:0.5.0" + classpath "org.bdware.bdcontract:simple-ypk-packer:0.5.3" classpath "org.bdware.bdcontract:ypk-deploy-tool:0.5.2" } @@ -14,7 +14,7 @@ plugins { id 'java' id 'java-library' } -version = "1.2.2" +version = "1.2.5" group = 'org.bdware.sc.controlproxy' tasks.withType(JavaCompile) { @@ -82,7 +82,7 @@ task copyJar(type: Copy, dependsOn: [":backend:jar", ":backend:copyLibs"]) { } task grepCP(dependsOn: [":backend:copyJar"]) { doLast { - org.bdware.datanet.YPKPacker.grepJarByCPVersion("./backend/build/output/libs", org.bdware.datanet.CPVersion.cp_1_5_3) + org.bdware.datanet.YPKPacker.grepJarByCPVersion("./backend/build/output/libs", org.bdware.datanet.CPVersion.cp_1_5_7) // org.bdware.datanet.YPKPacker.grepCPLibWithFilter("./backend/cplibs/libs/", "./backend/build/output/libs", "./backend/grepcp.list") } } diff --git a/backend/debugconf.json b/backend/debugconf.json index 6592100..153ed3f 100644 --- a/backend/debugconf.json +++ b/backend/debugconf.json @@ -1,15 +1,15 @@ { - "agentHttpAddr": "127.0.0.1:18010", + "agentAddress": "127.0.0.1:18000", "createParam": { "privateKey": "4616ff0e2a4f982364914f9be30b51c6bc6ccb6602114a9ee8792f2ccf67465b", "publicKey": "04f9b9b8f324908464f78a6235e2dd93e4dfdaf045e9b1b5cfd57374516cc61a79a86fc2b42d3321a5b49a0f25381a7bed61901b40b729f72354e716673d551e98", - "prefix": "bdtest", + "prefix": "shanxi", "router": "GlobalRouter", - "routerURI": "tcp://127.0.0.1:18041", + "routerURI": "tcp://39.104.201.40:18041", "auditType": "OnlyHash" }, - - "script": "/Users/huaqiancai/BDWare/datanet/ControlProxy/backend/build/ControlProxy-0.9.8.ypk", - "pubKey": "04d1924329f72ced148f6f333fb985ccbaa31b1e3aacf10be5f43d4a4ff5ad88899a005e79e37fc06993e1d66ada8cf8b711cb36f59538bb7d3e39e70fa9360ddd", - "privKey": "589d94ee5688358a1c5c18430dd9c75097ddddebf769f139da36a807911d20f8" + "killBeforeStart": "ControlProxy", + "ypkPath": "/Users/huaqiancai/BDWare/datanet/ControlProxy/backend/build/ControlProxy-1.2.5.ypk", + "publicKey": "04d1924329f72ced148f6f333fb985ccbaa31b1e3aacf10be5f43d4a4ff5ad88899a005e79e37fc06993e1d66ada8cf8b711cb36f59538bb7d3e39e70fa9360ddd", + "privateKey": "589d94ee5688358a1c5c18430dd9c75097ddddebf769f139da36a807911d20f8" } \ No newline at end of file diff --git a/backend/src/main/java/org/bdware/sc/controlproxy/RepoProxy.java b/backend/src/main/java/org/bdware/sc/controlproxy/RepoProxy.java index ccf3ec1..22b3379 100644 --- a/backend/src/main/java/org/bdware/sc/controlproxy/RepoProxy.java +++ b/backend/src/main/java/org/bdware/sc/controlproxy/RepoProxy.java @@ -3,6 +3,7 @@ package org.bdware.sc.controlproxy; import com.google.gson.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.bdware.doip.audit.AuditDoaClient; import org.bdware.doip.audit.EndpointConfig; import org.bdware.doip.audit.SM2Signer; import org.bdware.doip.audit.client.AuditDoipClient; @@ -13,6 +14,7 @@ import org.bdware.doip.audit.writer.AuditType; import org.bdware.doip.auditrepo.AutoAuditDO; import org.bdware.doip.codec.doipMessage.DoipMessage; import org.bdware.doip.endpoint.client.ClientConfig; +import org.bdware.irp.exception.IrpClientException; import org.bdware.irp.stateinfo.StateInfoBase; import org.bdware.sc.engine.JSONTool; @@ -46,7 +48,7 @@ public class RepoProxy { endpointConfig = storage.loadAsEndpointConfig(); prefix = storage.load().get("prefix").getAsString(); - router = storage.load().get("router").getAsString(); + router = storage.load().get("router").getAsString(); } } @@ -59,6 +61,77 @@ public class RepoProxy { return "hello..dafdskf"; } + public static Object getRepoStaticsMock(String repoDoid) { + JsonObject result = new JsonObject(); + result.addProperty("doCount", 1234); + result.addProperty("invokeCount", 5678); + result.addProperty("doIndexCount", 1200); + return JSONTool.convertJsonElementToMirror(result); + } + + public static Object getRepoStatistics(String repoDoid) { + JsonObject result = new JsonObject(); + try { + DoipMessage helloMsg = getHelloMessage(repoDoid); + JsonObject helloAttrs = helloMsg.header.parameters.attributes; + result.add("doCount", getPropWithDefault(helloAttrs, "doCount", new JsonPrimitive(-1))); + result.add("invokeCount", getPropWithDefault(helloAttrs, "invokeCount", new JsonPrimitive(-1))); + result.add("doIndexCount", getPropWithDefault(helloAttrs, "doIndexCount", new JsonPrimitive(-1))); + LOGGER.info(helloMsg); + return JSONTool.convertJsonElementToMirror(result); + } catch (Exception e) { + result.addProperty("msg", e.getMessage()); + } + result.addProperty("status", "offline"); + return JSONTool.convertJsonElementToMirror(result); + } + + private static JsonElement getPropWithDefault(JsonObject jo, String attr, JsonPrimitive defaultValue) { + if (jo != null && jo.has(attr)) + return jo.get(attr); + return defaultValue; + } + + private static DoipMessage getHelloMessage(String repoDoid) { + try { + AuditIrpClient irpClient = new AuditIrpClient(endpointConfig); + AuditConfig auditConfig = irpClient.getAuditConfig(); + AuditDoipClient client = new AuditDoipClient(auditConfig, new SM2Signer(null)); + StateInfoBase stateInfoBase = irpClient.resolve(repoDoid); + if (isDOIPRepo(stateInfoBase)) { + ClientConfig config = new ClientConfig(stateInfoBase.getValues("address")); + client.connect(config); + return client.helloSync(repoDoid); + } + } catch (Exception e) { + } + return null; + } + + //用于新增repo时判断它是否在线。 + //使用doip协议，发一个hello请求。 + public static Object pingRepo(String repoDoid) { + JsonObject result = new JsonObject(); + try { + if (getHelloMessage(repoDoid) != null) { + result.addProperty("status", "online"); + return JSONTool.convertJsonElementToMirror(result); + } + } catch (Exception e) { + result.addProperty("msg", e.getMessage()); + } + result.addProperty("status", "offline"); + return JSONTool.convertJsonElementToMirror(result); + } + + private static boolean isDOIPRepo(StateInfoBase base) { + JsonObject handleValue = base.getHandleValues(); + return handleValue != null && + handleValue.has("protocol") && + handleValue.get("protocol").getAsString().equalsIgnoreCase("DOIP"); + + } + public static JsonElement getDoAuditRule(Object obj) { JsonObject jo = JSONTool.convertMirrorToJson(obj).getAsJsonObject(); String doId = jo.get("doId").getAsString(); diff --git a/backend/src/main/java/org/bdware/sc/controlproxy/TopologyCollector.java b/backend/src/main/java/org/bdware/sc/controlproxy/TopologyCollector.java index 3af1684..cf1236d 100644 --- a/backend/src/main/java/org/bdware/sc/controlproxy/TopologyCollector.java +++ b/backend/src/main/java/org/bdware/sc/controlproxy/TopologyCollector.java @@ -19,6 +19,7 @@ public class TopologyCollector { static Logger LOGGER = LogManager.getLogger(TopologyCollector.class); + public static List getCurrentTopology() { TopologyGraph graph = new TopologyGraph(); //通过SearchEngine，可拿到本级的Repo（或下组的SE)与SearchEngine的关系。 @@ -105,11 +106,31 @@ public class TopologyCollector { ContractResult contractResult = new Gson().fromJson(je, ContractResult.class); return contractResult; } + //{ // "name":"小店区仓库","doId":"bdware.ss.ab/Repo1","type":"Repo","previousNode":["bdware.ss/Gateway1","bdware.ss/Gateway2","bdware.ss.ab/Repo2","bdware.ss.zz/SearchEngine"]} // + public static List getLevelTopologyByDoId(String componentDoid, int level) { + TopologyGraph graph = new TopologyGraph(); + graph.addGraph(getLevelTopology(2)); + graph.grepByDoId(componentDoid); + return graph.ret; + } - public static List getLevelTopology(int arg) { + static long lastUpdate = 0; + static TopologyGraph cachedGraph; + + public static boolean isCacheValid() { + if (System.currentTimeMillis() - lastUpdate < 10000) { + return true; + } + return false; + } + + public synchronized static List getLevelTopology(int arg) { + if (isCacheValid()) { + return cachedGraph.ret; + } List points = getCurrentTopology(); TopologyGraph graph = new TopologyGraph(); graph.addGraph(points); @@ -135,6 +156,8 @@ public class TopologyCollector { } } } + lastUpdate = System.currentTimeMillis(); + cachedGraph = graph; return graph.ret; } } diff --git a/backend/src/main/java/org/bdware/sc/controlproxy/TopologyGraph.java b/backend/src/main/java/org/bdware/sc/controlproxy/TopologyGraph.java index f238d49..3956ee3 100644 --- a/backend/src/main/java/org/bdware/sc/controlproxy/TopologyGraph.java +++ b/backend/src/main/java/org/bdware/sc/controlproxy/TopologyGraph.java @@ -3,6 +3,7 @@ package org.bdware.sc.controlproxy; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; +import com.google.gson.JsonPrimitive; import java.util.*; @@ -85,5 +86,34 @@ public class TopologyGraph { return doIdToNode.get(from); } + private boolean hasPreviousNode(JsonObject currentNode, String previousNode) { + if (currentNode.has("previousNode")) { + JsonElement pn = currentNode.get("previousNode"); + if (pn.isJsonArray()) { + JsonArray previousNodes = pn.getAsJsonArray(); + return previousNodes.contains(new JsonPrimitive(previousNode)); + } + } + return false; + } + public List grepByDoId(String componentDoid) { + TopologyGraph newGraph = new TopologyGraph(); + JsonObject currentNode = this.getOrCreateNode(componentDoid); + newGraph.addNode(currentNode); + //只添加上一级的节点 + //无需处理 + // JsonArray previousNodes = currentNode.get("previousNode").getAsJsonArray(); + //只添加下一级的节点 + for (JsonObject jo : doIdToNode.values()) { + if (hasPreviousNode(jo, componentDoid)) { + JsonObject jo2 = jo.deepCopy(); + JsonArray arr = new JsonArray(); + arr.add(componentDoid); + jo2.add("previousNodes", arr); + newGraph.addNode(jo2); + } + } + return newGraph.ret; + } } \ No newline at end of file diff --git a/backend/yjs/ControlProxy.yjs b/backend/yjs/ControlProxy.yjs index b276d1b..5a81b83 100755 --- a/backend/yjs/ControlProxy.yjs +++ b/backend/yjs/ControlProxy.yjs @@ -40,7 +40,7 @@ contract ControlProxy { export function isOwner() { return checkPermission(requester); } - @Descripton("参数为，{\"doId\":\"bdware.ss/Repox\"}") + @Description("参数为，{\"doId\":\"bdware.ss/Repox\"}") export function addRepo(arg) { arg = convertArgToJson(arg); if (arg.doId ==undefined ){ @@ -54,7 +54,7 @@ contract ControlProxy { }; } //网关需新增的接口 - @Descripton("参数为，{\"doId\":\"bdware.ss/Repox\"}") + @Description("参数为，{\"doId\":\"bdware.ss/Repox\"}") export function pingRepo(arg) { arg = convertArgToJson(arg); if (arg.doId ==undefined ){ @@ -71,9 +71,17 @@ contract ControlProxy { "msg":"repo is unconnected", "code":1 }; } + @Description("shanxi/TestLocal shanxi/Repository") + export function testPingRepo(arg) { + return org.bdware.sc.controlproxy.RepoProxy.pingRepo(arg); + } + @Description("shanxi/TestLocal shanxi/Repository") + export function testRepoStatistics(arg) { + return org.bdware.sc.controlproxy.RepoProxy.getRepoStatistics(arg); + } //网关的接口 - @Descripton("无参数") + @Description("无参数") export function getRepoList(arg) { //use configed arg.routerId; ret = executeContract(Global.router, "listRepo", ""); @@ -82,7 +90,7 @@ contract ControlProxy { return ret.result; } //网关的接口 - @Descripton("参数为 {\"doId\":\"bdware.ss/Repox\"}") + @Description("参数为 {\"doId\":\"bdware.ss/Repox\"}") export function deleteRepo(arg) { arg = convertArgToJson(arg); if (arg.doId != null && arg.doId.indexOf(Global.prefix) != -1){ @@ -96,7 +104,7 @@ contract ControlProxy { }; } //网关的接口 需新增 - @Descripton("参数为 [{\"doId\":\"bdware.ss/Repox\"},{\"doId\":\"bdware.ss/Repoy\"}]") + @Description("参数为 [{\"doId\":\"bdware.ss/Repox\"},{\"doId\":\"bdware.ss/Repoy\"}]") export function deleteRepoList(arg) { arg = convertArgToJson(arg); var ret = []; @@ -108,7 +116,7 @@ contract ControlProxy { return ret; } //网关的接口 - @Descripton("参数为 {\"doId\":..., \"address\":..., \"owner\":..., }") + @Description("参数为 {\"doId\":..., \"address\":..., \"owner\":..., }") export function updateRepo(arg) { arg = convertArgToJson(arg); if (arg.doId!=null){ @@ -134,7 +142,7 @@ contract ControlProxy { //总体拓扑图 //@范博 export function getAllTopology2() { - return getAllTopology(2); + return getAllTopology(2); var nodeList = []; nodeList.push({ "name":"小店区仓库", "doId":"bdware.ss.ab/Repo1", "type":"Repo", "previousNode":["bdware.ss/Gateway1", "bdware.ss/Gateway2", "bdware.ss.ab/Repo2", "bdware.ss.zz/SearchEngine"] diff --git a/backend/yjs/repodetail.yjs b/backend/yjs/repodetail.yjs index e747ff1..0f153ba 100755 --- a/backend/yjs/repodetail.yjs +++ b/backend/yjs/repodetail.yjs @@ -1,97 +1,87 @@ -module RepoDetail{ - //宏伟 - //如果有指定repo，那就是指定repo的DOlist。 - //如果没指定repo，那就是全部的。 - //这是search engine的接口，为什么还没接上？？ - //TODO TODO - export function getDoList(arg){ - print("GetDOList==========="); - var ret = executeContract("SearchEngine","getDoList",arg); - return ret.result; - } - export function getDoList2(arg){ - arg = convertArgToJson(arg); - //if (arg.doId == null) - // return { - // "msg":"missing arguments doId", "code":1, params: arg - // }; - //if (arg.count>1000) - // return { - // "msg":"request to much!", "code":1 - // }; - var ret = []; - if(arg.count==undefined) arg.count=10; - if(arg.offset==undefined) arg.offset=0; - for (var i=arg.offset; i1000) + // return { + // "msg":"request to much!", "code":1 + // }; + var ret = []; + if (arg.count==undefined) arg.count = 10; + if (arg.offset==undefined) arg.offset = 0; + for (var i = arg.offset; + i