songchunqiao/ControlProxy
1
0
Fork 0
forked from iod/ControlProxy
Code Pull Requests Activity

add repo topology

Browse Source 
add repo statistics
fix concurrent bugs in getAllTopology
This commit is contained in:
CaiHQ 2022-06-28 15:39:02 +08:00
parent 1eb2950c41
commit 410a133ada
7 changed files with 235 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
backend/build.gradle
View File

@ -4,7 +4,7 @@ buildscript {
mavenLocal()
}
dependencies {
classpath "org.bdware.bdcontract:simple-ypk-packer:0.5.0"
classpath "org.bdware.bdcontract:simple-ypk-packer:0.5.3"
classpath "org.bdware.bdcontract:ypk-deploy-tool:0.5.2"
}
@ -14,7 +14,7 @@ plugins {
id 'java'
id 'java-library'
}
version = "1.2.2"
version = "1.2.5"
group = 'org.bdware.sc.controlproxy'
tasks.withType(JavaCompile) {
@ -82,7 +82,7 @@ task copyJar(type: Copy, dependsOn: [":backend:jar", ":backend:copyLibs"]) {
}
task grepCP(dependsOn: [":backend:copyJar"]) {
doLast {
org.bdware.datanet.YPKPacker.grepJarByCPVersion("./backend/build/output/libs", org.bdware.datanet.CPVersion.cp_1_5_3)
org.bdware.datanet.YPKPacker.grepJarByCPVersion("./backend/build/output/libs", org.bdware.datanet.CPVersion.cp_1_5_7)
// org.bdware.datanet.YPKPacker.grepCPLibWithFilter("./backend/cplibs/libs/", "./backend/build/output/libs", "./backend/grepcp.list")
}
}

14
backend/debugconf.json
View File

@ -1,15 +1,15 @@
{
"agentHttpAddr": "127.0.0.1:18010",
"agentAddress": "127.0.0.1:18000",
"createParam": {
"privateKey": "4616ff0e2a4f982364914f9be30b51c6bc6ccb6602114a9ee8792f2ccf67465b",
"publicKey": "04f9b9b8f324908464f78a6235e2dd93e4dfdaf045e9b1b5cfd57374516cc61a79a86fc2b42d3321a5b49a0f25381a7bed61901b40b729f72354e716673d551e98",
"prefix": "bdtest",
"prefix": "shanxi",
"router": "GlobalRouter",
"routerURI": "tcp://127.0.0.1:18041",
"routerURI": "tcp://39.104.201.40:18041",
"auditType": "OnlyHash"
},
"script": "/Users/huaqiancai/BDWare/datanet/ControlProxy/backend/build/ControlProxy-0.9.8.ypk",
"pubKey": "04d1924329f72ced148f6f333fb985ccbaa31b1e3aacf10be5f43d4a4ff5ad88899a005e79e37fc06993e1d66ada8cf8b711cb36f59538bb7d3e39e70fa9360ddd",
"privKey": "589d94ee5688358a1c5c18430dd9c75097ddddebf769f139da36a807911d20f8"
"killBeforeStart": "ControlProxy",
"ypkPath": "/Users/huaqiancai/BDWare/datanet/ControlProxy/backend/build/ControlProxy-1.2.5.ypk",
"publicKey": "04d1924329f72ced148f6f333fb985ccbaa31b1e3aacf10be5f43d4a4ff5ad88899a005e79e37fc06993e1d66ada8cf8b711cb36f59538bb7d3e39e70fa9360ddd",
"privateKey": "589d94ee5688358a1c5c18430dd9c75097ddddebf769f139da36a807911d20f8"
}

73
backend/src/main/java/org/bdware/sc/controlproxy/RepoProxy.java
View File

@ -3,6 +3,7 @@ package org.bdware.sc.controlproxy;
import com.google.gson.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.bdware.doip.audit.AuditDoaClient;
import org.bdware.doip.audit.EndpointConfig;
import org.bdware.doip.audit.SM2Signer;
import org.bdware.doip.audit.client.AuditDoipClient;
@ -13,6 +14,7 @@ import org.bdware.doip.audit.writer.AuditType;
import org.bdware.doip.auditrepo.AutoAuditDO;
import org.bdware.doip.codec.doipMessage.DoipMessage;
import org.bdware.doip.endpoint.client.ClientConfig;
import org.bdware.irp.exception.IrpClientException;
import org.bdware.irp.stateinfo.StateInfoBase;
import org.bdware.sc.engine.JSONTool;
@ -59,6 +61,77 @@ public class RepoProxy {
return "hello..dafdskf";
}
public static Object getRepoStaticsMock(String repoDoid) {
JsonObject result = new JsonObject();
result.addProperty("doCount", 1234);
result.addProperty("invokeCount", 5678);
result.addProperty("doIndexCount", 1200);
return JSONTool.convertJsonElementToMirror(result);
}
public static Object getRepoStatistics(String repoDoid) {
JsonObject result = new JsonObject();
try {
DoipMessage helloMsg = getHelloMessage(repoDoid);
JsonObject helloAttrs = helloMsg.header.parameters.attributes;
result.add("doCount", getPropWithDefault(helloAttrs, "doCount", new JsonPrimitive(-1)));
result.add("invokeCount", getPropWithDefault(helloAttrs, "invokeCount", new JsonPrimitive(-1)));
result.add("doIndexCount", getPropWithDefault(helloAttrs, "doIndexCount", new JsonPrimitive(-1)));
LOGGER.info(helloMsg);
return JSONTool.convertJsonElementToMirror(result);
} catch (Exception e) {
result.addProperty("msg", e.getMessage());
}
result.addProperty("status", "offline");
return JSONTool.convertJsonElementToMirror(result);
}
private static JsonElement getPropWithDefault(JsonObject jo, String attr, JsonPrimitive defaultValue) {
if (jo != null && jo.has(attr))
return jo.get(attr);
return defaultValue;
}
private static DoipMessage getHelloMessage(String repoDoid) {
try {
AuditIrpClient irpClient = new AuditIrpClient(endpointConfig);
AuditConfig auditConfig = irpClient.getAuditConfig();
AuditDoipClient client = new AuditDoipClient(auditConfig, new SM2Signer(null));
StateInfoBase stateInfoBase = irpClient.resolve(repoDoid);
if (isDOIPRepo(stateInfoBase)) {
ClientConfig config = new ClientConfig(stateInfoBase.getValues("address"));
client.connect(config);
return client.helloSync(repoDoid);
}
} catch (Exception e) {
}
return null;
}
//用于新增repo时判断它是否在线。
//使用doip协议，发一个hello请求。
public static Object pingRepo(String repoDoid) {
JsonObject result = new JsonObject();
try {
if (getHelloMessage(repoDoid) != null) {
result.addProperty("status", "online");
return JSONTool.convertJsonElementToMirror(result);
}
} catch (Exception e) {
result.addProperty("msg", e.getMessage());
}
result.addProperty("status", "offline");
return JSONTool.convertJsonElementToMirror(result);
}
private static boolean isDOIPRepo(StateInfoBase base) {
JsonObject handleValue = base.getHandleValues();
return handleValue != null &&
handleValue.has("protocol") &&
handleValue.get("protocol").getAsString().equalsIgnoreCase("DOIP");
}
public static JsonElement getDoAuditRule(Object obj) {
JsonObject jo = JSONTool.convertMirrorToJson(obj).getAsJsonObject();
String doId = jo.get("doId").getAsString();

25
backend/src/main/java/org/bdware/sc/controlproxy/TopologyCollector.java
View File

@ -19,6 +19,7 @@ public class TopologyCollector {
static Logger LOGGER = LogManager.getLogger(TopologyCollector.class);
public static List<JsonObject> getCurrentTopology() {
TopologyGraph graph = new TopologyGraph();
//通过SearchEngine，可拿到本级的Repo（或下组的SE)与SearchEngine的关系。
@ -105,11 +106,31 @@ public class TopologyCollector {
ContractResult contractResult = new Gson().fromJson(je, ContractResult.class);
return contractResult;
}
//{
// "name":"小店区仓库","doId":"bdware.ss.ab/Repo1","type":"Repo","previousNode":["bdware.ss/Gateway1","bdware.ss/Gateway2","bdware.ss.ab/Repo2","bdware.ss.zz/SearchEngine"]}
//
public static List<JsonObject> getLevelTopologyByDoId(String componentDoid, int level) {
TopologyGraph graph = new TopologyGraph();
graph.addGraph(getLevelTopology(2));
graph.grepByDoId(componentDoid);
return graph.ret;
}
public static List<JsonObject> getLevelTopology(int arg) {
static long lastUpdate = 0;
static TopologyGraph cachedGraph;
public static boolean isCacheValid() {
if (System.currentTimeMillis() - lastUpdate < 10000) {
return true;
}
return false;
}
public synchronized static List<JsonObject> getLevelTopology(int arg) {
if (isCacheValid()) {
return cachedGraph.ret;
}
List<JsonObject> points = getCurrentTopology();
TopologyGraph graph = new TopologyGraph();
graph.addGraph(points);
@ -135,6 +156,8 @@ public class TopologyCollector {
}
}
}
lastUpdate = System.currentTimeMillis();
cachedGraph = graph;
return graph.ret;
}
}

30
backend/src/main/java/org/bdware/sc/controlproxy/TopologyGraph.java
View File

@ -3,6 +3,7 @@ package org.bdware.sc.controlproxy;
import com.google.gson.JsonArray;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import java.util.*;
@ -85,5 +86,34 @@ public class TopologyGraph {
return doIdToNode.get(from);
}
private boolean hasPreviousNode(JsonObject currentNode, String previousNode) {
if (currentNode.has("previousNode")) {
JsonElement pn = currentNode.get("previousNode");
if (pn.isJsonArray()) {
JsonArray previousNodes = pn.getAsJsonArray();
return previousNodes.contains(new JsonPrimitive(previousNode));
}
}
return false;
}
public List<JsonObject> grepByDoId(String componentDoid) {
TopologyGraph newGraph = new TopologyGraph();
JsonObject currentNode = this.getOrCreateNode(componentDoid);
newGraph.addNode(currentNode);
//只添加上一级的节点
//无需处理
// JsonArray previousNodes = currentNode.get("previousNode").getAsJsonArray();
//只添加下一级的节点
for (JsonObject jo : doIdToNode.values()) {
if (hasPreviousNode(jo, componentDoid)) {
JsonObject jo2 = jo.deepCopy();
JsonArray arr = new JsonArray();
arr.add(componentDoid);
jo2.add("previousNodes", arr);
newGraph.addNode(jo2);
}
}
return newGraph.ret;
}
}

20
backend/yjs/ControlProxy.yjs
View File

@ -40,7 +40,7 @@ contract ControlProxy {
export function isOwner() {
return checkPermission(requester);
}
@Descripton("参数为，{\"doId\":\"bdware.ss/Repox\"}")
@Description("参数为，{\"doId\":\"bdware.ss/Repox\"}")
export function addRepo(arg) {
arg = convertArgToJson(arg);
if (arg.doId ==undefined ){
@ -54,7 +54,7 @@ contract ControlProxy {
};
}
//网关需新增的接口
@Descripton("参数为，{\"doId\":\"bdware.ss/Repox\"}")
@Description("参数为，{\"doId\":\"bdware.ss/Repox\"}")
export function pingRepo(arg) {
arg = convertArgToJson(arg);
if (arg.doId ==undefined ){
@ -71,9 +71,17 @@ contract ControlProxy {
"msg":"repo is unconnected", "code":1
};
}
@Description("shanxi/TestLocal shanxi/Repository")
export function testPingRepo(arg) {
return org.bdware.sc.controlproxy.RepoProxy.pingRepo(arg);
}
@Description("shanxi/TestLocal shanxi/Repository")
export function testRepoStatistics(arg) {
return org.bdware.sc.controlproxy.RepoProxy.getRepoStatistics(arg);
}
//网关的接口
@Descripton("无参数")
@Description("无参数")
export function getRepoList(arg) {
//use configed arg.routerId;
ret = executeContract(Global.router, "listRepo", "");
@ -82,7 +90,7 @@ contract ControlProxy {
return ret.result;
}
//网关的接口
@Descripton("参数为 {\"doId\":\"bdware.ss/Repox\"}")
@Description("参数为 {\"doId\":\"bdware.ss/Repox\"}")
export function deleteRepo(arg) {
arg = convertArgToJson(arg);
if (arg.doId != null && arg.doId.indexOf(Global.prefix) != -1){
@ -96,7 +104,7 @@ contract ControlProxy {
};
}
//网关的接口 需新增
@Descripton("参数为 [{\"doId\":\"bdware.ss/Repox\"},{\"doId\":\"bdware.ss/Repoy\"}]")
@Description("参数为 [{\"doId\":\"bdware.ss/Repox\"},{\"doId\":\"bdware.ss/Repoy\"}]")
export function deleteRepoList(arg) {
arg = convertArgToJson(arg);
var ret = [];
@ -108,7 +116,7 @@ contract ControlProxy {
return ret;
}
//网关的接口
@Descripton("参数为 {\"doId\":..., \"address\":..., \"owner\":..., }")
@Description("参数为 {\"doId\":..., \"address\":..., \"owner\":..., }")
export function updateRepo(arg) {
arg = convertArgToJson(arg);
if (arg.doId!=null){

104
backend/yjs/repodetail.yjs
View File

@ -1,15 +1,15 @@
module RepoDetail{
//宏伟
//如果有指定repo，那就是指定repo的DOlist。
//如果没指定repo，那就是全部的。
//这是search engine的接口，为什么还没接上？？
//TODO TODO
export function getDoList(arg){
module RepoDetail {
//宏伟
//如果有指定repo，那就是指定repo的DOlist。
//如果没指定repo，那就是全部的。
//这是search engine的接口，为什么还没接上？？
//TODO TODO
export function getDoList(arg) {
print("GetDOList===========");
var ret = executeContract("SearchEngine","getDoList",arg);
var ret = executeContract("SearchEngine", "getDoList", arg);
return ret.result;
}
export function getDoList2(arg){
export function getDoList2(arg) {
arg = convertArgToJson(arg);
//if (arg.doId == null)
// return {
@ -20,63 +20,51 @@ module RepoDetail{
// "msg":"request to much!", "code":1
// };
var ret = [];
if(arg.count==undefined) arg.count=10;
if(arg.offset==undefined) arg.offset=0;
for (var i=arg.offset; i<arg.offset/1.0+arg.count/1.0;i++){
if (arg.count==undefined) arg.count = 10;
if (arg.offset==undefined) arg.offset = 0;
for (var i = arg.offset;
i<arg.offset/1.0+arg.count/1.0;
i++){
var fakeDo = {
};
fakeDo.doId="bdware.ss/Repo1/abc"+i;
fakeDo.originalData={doid:fakeDo.doId,info:"ddddd", tag: ["aaa", "bbb", "ccc"]}
fakeDo.doId = "bdware.ss/Repo1/abc"+i;
fakeDo.originalData = {
doid:fakeDo.doId, info:"ddddd", tag: ["aaa", "bbb", "ccc"]
}
fakeDo.enableIndex = true
ret.push(fakeDo);
}
return {data: ret, total: 100};
}
//
export function getRepoTopology(arg){
arg = convertArgToJson(arg);
if (arg.doId == null)
return {
"msg":"missing arguments doId", "code":1};
var nodeList = [];
nodeList.push({
"name":"小店区仓库","doId":"bdware.ss.ab/Repo1","previousNode":["bdware.ss/Gateway1","bdware.ss/Gateway2"]}
);
nodeList.push({
"name":"太原内部共享网关","doId":"bdware.ss/Gateway1","previousNode":["bdware/Gateway1", "bdware/Gateway2"]}
);
nodeList.push({
"name":"太原对外开放网关","doId":"bdware.ss/Gateway2", "previousNode":["bdware/Gateway1"]}
);
nodeList.push({
"name":"山西内部共享网关","doId":"bdware/Gateway1"}
);
nodeList.push({
"name":"山西内部共享网关2","doId":"bdware/Gateway2"}
);
return nodeList;
data: ret, total: 100
};
}
//指标1 获取某个repo的DO数量。
//使用search.yjs里的getDoCount
//指标2 获取某个repo的操作次数。使用search.yjs中的
//countRecordByDate
//参数格式为：
//{"repoId":"xxx.xxx/xxx","startTime":1641454745128, "endTime":1641454746128,"interval":5000, "eventPrefix":["abc"]}"
//比如，查询总搜索次数，参数格式为：eventPrefix:["doip_search"]
//查询调用统计，参数格式为eventPrefix:[""]，然后interval/startTime/fromTime传对了就行。
//索引统计，暂时还没有。
export function getStaticIndex(arg){
//TODO check arguments
export function getRepoTopology(arg) {
return org.bdware.sc.controlproxy.TopologyCollector.getLevelTopologyByDoId(arg, 2);
}
//指标1 获取某个repo的DO数量。
//使用search.yjs里的getDoCount
//指标2 获取某个repo的操作次数。使用search.yjs中的
//countRecordByDate
//参数格式为：
//{"repoId":"xxx.xxx/xxx","startTime":1641454745128, "endTime":1641454746128,"interval":5000, "eventPrefix":["abc"]}"
//比如，查询总搜索次数，参数格式为：eventPrefix:["doip_search"]
//查询调用统计，参数格式为eventPrefix:[""]，然后interval/startTime/fromTime传对了就行。
//索引统计，暂时还没有。
export function getStaticIndex(arg) {
arg = convertArgToJson(arg);
if (arg.from==undefined)
return {
"msg":"missing arguments: from", "code":1};
"msg":"missing arguments: from", "code":1
};
if (arg.to==undefined)
return {
"msg":"missing arguments: to", "code":1};
"msg":"missing arguments: to", "code":1
};
if (arg.interval==undefined)
return {
"msg":"missing arguments: interval", "code":1};
"msg":"missing arguments: interval", "code":1
};
switch(arg.tag){
case "search":
case "index":
@ -84,14 +72,16 @@ module RepoDetail{
return fakeList(from, to, interval);
}
return {
"msg":"unreconized tag", "code":1};
"msg":"unreconized tag", "code":1
};
}
function fakeList(from, to, interval){
var ret =[];
for (i=from;i<to;i++){
function fakeList(from, to, interval) {
var ret = [];
for (i = from;
i<to;
i++){
ret.push(Math.floor(Math.random()*200+20));
}
return ret;
}
}