From 641d6cda650380fd8f4c8fd40a6fa0c5451add26 Mon Sep 17 00:00:00 2001 From: PhiTer9426 <793916584@qq.com> Date: Tue, 3 May 2022 21:07:23 +0800 Subject: [PATCH] init --- backend/build.gradle | 3 + backend/deployconfig.json | 2 +- build.gradle | 9 + datanet-search-engine-backend/.gitignore | 25 ++ .../ContractDB/SearchEngine.index | 0 .../SearchEngine/defaultLog/000029.sst | Bin 0 -> 1259 bytes .../SearchEngine/defaultLog/000035.sst | Bin 0 -> 1109 bytes .../SearchEngine/defaultLog/000041.sst | Bin 0 -> 1109 bytes .../SearchEngine/defaultLog/CURRENT | 1 + .../SearchEngine/defaultLog/IDENTITY | 1 + .../ContractDB/SearchEngine/defaultLog/LOCK | 0 .../ContractDB/SearchEngine/defaultLog/LOG | 302 +++++++++++++ .../defaultLog/LOG.old.1647409410560607 | 290 +++++++++++++ .../defaultLog/LOG.old.1647409489160227 | 297 +++++++++++++ .../defaultLog/LOG.old.1647409653934387 | 297 +++++++++++++ .../defaultLog/LOG.old.1647409943656846 | 297 +++++++++++++ .../defaultLog/LOG.old.1647410376106051 | 297 +++++++++++++ .../defaultLog/LOG.old.1647410721084139 | 302 +++++++++++++ .../defaultLog/LOG.old.1647410840955077 | 368 ++++++++++++++++ .../defaultLog/LOG.old.1647411637683065 | 105 +++++ .../SearchEngine/defaultLog/MANIFEST-000042 | Bin 0 -> 308 bytes .../SearchEngine/defaultLog/OPTIONS-000039 | 181 ++++++++ .../SearchEngine/defaultLog/OPTIONS-000045 | 181 ++++++++ datanet-search-engine-backend/LICENSE | 127 ++++++ datanet-search-engine-backend/README.en.md | 36 ++ datanet-search-engine-backend/README.md | 23 + datanet-search-engine-backend/build.gradle | 122 ++++++ datanet-search-engine-backend/dataNode.txt | 10 + datanet-search-engine-backend/debugconf.json | 6 + .../input/auditrepo.json | 10 + datanet-search-engine-backend/input/doi.txt | 1 + .../bdware/datanet/searchengine/DoMeta.java | 16 + .../datanet/searchengine/MetaGraph.java | 333 ++++++++++++++ .../datanet/searchengine/MetaIndex.java | 341 +++++++++++++++ .../datanet/searchengine/ResultModel.java | 48 ++ .../datanet/searchengine/SearchEngine.java | 182 ++++++++ .../src/test/java/DebugMain.java | 5 + .../src/test/java/GenerateDO.java | 54 +++ .../src/test/java/GraphDBTEst.java | 26 ++ .../src/test/java/luceneTest.java | 69 +++ .../src/test/java/searchEnginetest.java | 184 ++++++++ datanet-search-engine-backend/yjs/DAC.yjs | 49 +++ .../yjs/SearchEngine.yjs | 410 ++++++++++++++++++ .../yjs/manifest.json | 9 + .../yjs/mockConfig.json | 6 + settings.gradle | 3 +- 46 files changed, 5026 insertions(+), 2 deletions(-) create mode 100644 datanet-search-engine-backend/.gitignore create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine.index create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/000029.sst create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/000035.sst create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/000041.sst create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/CURRENT create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/IDENTITY create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOCK create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG.old.1647409410560607 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG.old.1647409489160227 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG.old.1647409653934387 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG.old.1647409943656846 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG.old.1647410376106051 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG.old.1647410721084139 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG.old.1647410840955077 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/LOG.old.1647411637683065 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/MANIFEST-000042 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/OPTIONS-000039 create mode 100644 datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/OPTIONS-000045 create mode 100644 datanet-search-engine-backend/LICENSE create mode 100644 datanet-search-engine-backend/README.en.md create mode 100644 datanet-search-engine-backend/README.md create mode 100644 datanet-search-engine-backend/build.gradle create mode 100644 datanet-search-engine-backend/dataNode.txt create mode 100644 datanet-search-engine-backend/debugconf.json create mode 100644 datanet-search-engine-backend/input/auditrepo.json create mode 100644 datanet-search-engine-backend/input/doi.txt create mode 100644 datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/DoMeta.java create mode 100644 datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/MetaGraph.java create mode 100644 datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/MetaIndex.java create mode 100644 datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/ResultModel.java create mode 100644 datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/SearchEngine.java create mode 100644 datanet-search-engine-backend/src/test/java/DebugMain.java create mode 100644 datanet-search-engine-backend/src/test/java/GenerateDO.java create mode 100644 datanet-search-engine-backend/src/test/java/GraphDBTEst.java create mode 100644 datanet-search-engine-backend/src/test/java/luceneTest.java create mode 100644 datanet-search-engine-backend/src/test/java/searchEnginetest.java create mode 100644 datanet-search-engine-backend/yjs/DAC.yjs create mode 100644 datanet-search-engine-backend/yjs/SearchEngine.yjs create mode 100644 datanet-search-engine-backend/yjs/manifest.json create mode 100644 datanet-search-engine-backend/yjs/mockConfig.json diff --git a/backend/build.gradle b/backend/build.gradle index 4eb5014..3e89505 100644 --- a/backend/build.gradle +++ b/backend/build.gradle @@ -48,6 +48,9 @@ dependencies { implementation 'org.postgresql:postgresql:42.3.1' } +tasks.withType(JavaCompile) { + options.encoding = "UTF-8" +} //task copyLibs(type: Copy) { // from configurations.runtimeClasspath // into "./build/output/libs/" diff --git a/backend/deployconfig.json b/backend/deployconfig.json index 2664fd9..fbe965b 100644 --- a/backend/deployconfig.json +++ b/backend/deployconfig.json @@ -2,7 +2,7 @@ "host": "023.node.internetapi.cn", "privateKey": "e85ce2f4d8882ff343d32ce42adde91d09e29c321452dd4ef9f07ebe76d1c6a5", "publicKey": "04da01345770b7e09d4774bf6c0395399b18814aa0b7b158f64b634b8f3d628d9964af6523835225af11e467271f4969d67bf90b32eaa09f517c79b2d1f9b8a926", - "ypkPath": "./backend/build/registry0.0.1.ypk", + "ypkPath": "./backend/build/registry-0.0.5.ypk", "killBeforeStart": "Registry", "createParam": { "privateKey": "4616ff0e2a4f982364914f9be30b51c6bc6ccb6602114a9ee8792f2ccf67465b", diff --git a/build.gradle b/build.gradle index 66ac7ba..e84b118 100644 --- a/build.gradle +++ b/build.gradle @@ -18,5 +18,14 @@ project(':backend') { version = projectIds.version repositories { maven { url 'https://maven.aliyun.com/repository/public' } + mavenCentral() } } +project(':datanet-search-engine-backend') { + group = "org.bdware.sc.example" + version = projectIds.version + repositories { + maven { url 'https://maven.aliyun.com/repository/public' } + mavenCentral() + } +} \ No newline at end of file diff --git a/datanet-search-engine-backend/.gitignore b/datanet-search-engine-backend/.gitignore new file mode 100644 index 0000000..2ea3022 --- /dev/null +++ b/datanet-search-engine-backend/.gitignore @@ -0,0 +1,25 @@ +/build/ +MetaIndex +# Compiled class file +*.class + +# Log file +*.log + +# BlueJ files +*.ctxt + +# Mobile Tools for Java (J2ME) +.mtj.tmp/ + +# Package Files # +*.jar +*.war +*.nar +*.ear +*.zip +*.tar.gz +*.rar + +# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml +hs_err_pid* diff --git a/datanet-search-engine-backend/ContractDB/SearchEngine.index b/datanet-search-engine-backend/ContractDB/SearchEngine.index new file mode 100644 index 0000000..e69de29 diff --git a/datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/000029.sst b/datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/000029.sst new file mode 100644 index 0000000000000000000000000000000000000000..fd7e647c21f135abdc2cef891c0da0212e39ef6f GIT binary patch literal 1259 zcmaJ=L2nyH6y9;u*iKz1X`lyR2v&~uYG-%t-EEXZ3RQ{})Y4Rm6ojmqowv@av%6+z zHnxj6B5~jV2QEl(=!t*8F97vYA@LLV2S`YW8*gma7D8e#JM-qv_rCXiZ>-MC8Xfq7 zPaW3_T$c`g&v6N{=HF-C96+4Gr``JrlM3#2m03IZ*GyNcd%t^D#W5(=z1J;4RZ?T< z3ZyLdFN#tM$4_X`kGbpeSOhT)827<<9EigtgwPv!gP1x&7}Ch~IXGSvIsqgRO(Gr% z*NJ@(sO$Tl=Y|1IV#f=4D1wlBETH{3>C=GWRyYwPoDV!K2LXprBtDI(%L9hQQuRTu z^3ya+k=?QnU1%mx7C{eA zsi92sk<-~DldA$#pTle&ri~pE7qZH7J7HNWXLc&AyBqbKWgsA7=*w24lZ{!)G};2! zAhG^~nTAQK;JxC_NvpnzE-F;wENA2KY_BmL7GtA~njp7gGUTaNM?LBv?$1$mf&cWg;(>7DE%ptTbQ8Avax%e_yJS1j=U!QlK+yMD9CZJlQD>hGVw*=#gxf@x-( zp?FPN%X;iivZk@f~LB@Wtm$u*M|j)t-YNURKbTnG@P(Iy#`Be07xlq?s@+PhIlck@chu`V0i zB(E}iPIf+MinQtk)hkoLWqZkVxQtk>ZGtEx=wTI@J8mIFAqAj>hMkcB`PP>s2~#vP z#(K6HvSrb&-I`&eXEsiw1aquD{-n`D=*b+!^weo>)fPRO-OKg!IQZqfY28^pk8wH2 pLb`v{zS~?qq8~o|HXvU&-+f(s$He~syNSnt{rcw@KOO)6hp58P9IQbr$yOBOP^S&9z-^o#B!m!@*_|hOn%UXy z%&dMS^jPS%*WPl8slB6Tzl!Q&+8<5 zVt7bHi3b`o1DskGN}xh<7*jh*!QCmZl2k)qjA>s;3tFf_y}U+7hY5y0fyt>k@K;Hc z$ShX0&tf4bRB-S7d`__#IP@8|zvP$Wlxb!YEnx@}>tCN(7zqQPCT~22+#-@?khVLD zrRikNANP_ejb?<{l2I=d*6h@R)^=r@WvAI_Z?(58205}YnC#TID={1QxQJ{o#Qert zo*K)0&Wuq)o!UXUuKC-SvVI@*X-s}?T?4b&J4MKXBmKfog3YiA!DdurEeLBt08y>e z=AGJ=db6?F48y1%c1p`Dhl#N#clLJ=?%q78eReOnv3GrMzqIg(Q*#Qs~e*3J`DZSduWtVMbVwI*}b0T`J8vG=i?|*n_Pu3&V(_9GmXY#}1$rAnc_r*=}Z|?1= S|16;UpVxnVd$jxOcmD$_jZ}gF literal 0 HcmV?d00001 diff --git a/datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/000041.sst b/datanet-search-engine-backend/ContractDB/SearchEngine/defaultLog/000041.sst new file mode 100644 index 0000000000000000000000000000000000000000..f34654cb8929363d2bafcbd40a0bd21406fe8253 GIT binary patch literal 1109 zcmaJ=&ube;6y9-cYvstY6(@lzhp58P9-=`j$&nr7K$03;;3TeNl0XQ`?9P)s$?WWQ zW>!B^dMYWs_S%0Sr(6QXryko|=^xNrq4yq213mSP zYZ6> zL25;!s@JL-OeQTFpR)m@lBqtumr1#wpih{NGHOr6!d|*tv594JhnXrlScVZq8R>jf z@~vj2#exwpA^*vz15ow=B(uO5JC(0Ie29C0eEdz_yYN0@@2zuBV!W%Bn?HH0vpR`h z7#`A4;(>CP5~xrdM%0c|aCgefB-PLpBia+vf)=V@FRzf{QG%f_V0>l{{ACg) zGK&@Mu~^7472LZ#n^Prwz=>ZZG3Y+fBn8bBH{Qy*98OoEv{S47C8DyJPQXrthAWaI$etE}$Knan?uo@K zED_AohZa3&5=kTnq|l*r6d>N*i6mi!!;G*Vbs{?{x=@;NXzZAx8YMWldGgdR zVCvx{iR-CcSeTji details.file.getAbsolutePath().contains("cplib") + } + into "./build/output/libs/" +} + +//task copyLibs(type: Copy) { +// +// from configurations.runtimeClasspath +// into "./build/output/libs/" +//} +task copyYJS(type: Copy) { + from "./yjs" + into "./build/output/" +} + + + +task sourcesJar(type:Jar) { + from sourceSets.main.allJava + archiveClassifier = 'sources' +} + + +task copyJar(type: Copy, dependsOn: [":datanet-search-engine-backend:jar", ":datanet-search-engine-backend:copyLibs"]) { +//task copyJar(type: Copy, dependsOn: ["mergeJar"]) { + from "./build/libs/$project.name-${project.version}.jar" + into "./build/output" + rename { String fileName -> "SearchEngine.jar" } + doFirst { + println "copyJar start" + } +} +task copyFront(type: Copy){ + from "../datanet-search-engine-front/dist" + into "./build/output/assets/" + doFirst { + println "copyFront start" + } +} + +def reltivePath = "./datanet-search-engine-backend" +//reltivePath="." +task grepCP (dependsOn:["copyJar"]){ + doLast { + org.bdware.datanet.YPKPacker.grepCPLib("${reltivePath}/cplib","${reltivePath}/build/output/libs") + } +} + +task buildZip(type: Zip, dependsOn: ["copyJar", "copyYJS","copyFront","grepCP"]) { + from './build/output/' + duplicatesStrategy = DuplicatesStrategy.INCLUDE + archiveFileName = 'SearchEngine.zip' + destinationDirectory = file('build/') +} + + +task buildYPK (dependsOn:["buildZip"]){ + doLast { + org.bdware.datanet.YPKPacker.staticPack("${reltivePath}/build/SearchEngine.zip","${reltivePath}/build/SearchEngine-${project.version}.ypk") + } +} diff --git a/datanet-search-engine-backend/dataNode.txt b/datanet-search-engine-backend/dataNode.txt new file mode 100644 index 0000000..d9e2de7 --- /dev/null +++ b/datanet-search-engine-backend/dataNode.txt @@ -0,0 +1,10 @@ +ADDRESS +streetName +city +valid +Gender +IdNumber +Internet +emailAddress +domainName +url \ No newline at end of file diff --git a/datanet-search-engine-backend/debugconf.json b/datanet-search-engine-backend/debugconf.json new file mode 100644 index 0000000..69d7a4e --- /dev/null +++ b/datanet-search-engine-backend/debugconf.json @@ -0,0 +1,6 @@ +{ + "script": "D:\\00PKUOS\\datanet-search-engine-bundle\\datanet-search-engine-backend\\build\\SearchEngine.ypk", + "agentHttpAddr": "127.0.0.1:21030", + "pubKey": "04f175cb22bcd7ce6ab4e400ec8f8e83636c107881924be039807ef91f9b9d20c43e1244332871fc038b8d7813a715cf517ad78d09efbb634f1e7be8df41912b7a", + "privKey": "0429b8fa0760ac28d5472ac9fc1a35ef94f1428a62145701eb94aae845ba2e44" +} \ No newline at end of file diff --git a/datanet-search-engine-backend/input/auditrepo.json b/datanet-search-engine-backend/input/auditrepo.json new file mode 100644 index 0000000..a5f5782 --- /dev/null +++ b/datanet-search-engine-backend/input/auditrepo.json @@ -0,0 +1,10 @@ +{ + "routerURI": "tcp://127.0.0.1:2042", + "useRoot": false, + "repoName": "repofile", + "publicKey": "04f9b9b8f324908464f78a6235e2dd93e4dfdaf045e9b1b5cfd57374516cc61a79a86fc2b42d3321a5b49a0f25381a7bed61901b40b729f72354e716673d551e98", + "privateKey": "4616ff0e2a4f982364914f9be30b51c6bc6ccb6602114a9ee8792f2ccf67465b", + "auditType": "OnlyHash", + "targetRepo": "macjw.ab/repofile", + "seURI":"127.0.0.1:43030" +} \ No newline at end of file diff --git a/datanet-search-engine-backend/input/doi.txt b/datanet-search-engine-backend/input/doi.txt new file mode 100644 index 0000000..a07f189 --- /dev/null +++ b/datanet-search-engine-backend/input/doi.txt @@ -0,0 +1 @@ +macjw.ab/repofile/7b67ab5b-0bd6-4385-9520-c46d7ad3bed0 diff --git a/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/DoMeta.java b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/DoMeta.java new file mode 100644 index 0000000..476b3cc --- /dev/null +++ b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/DoMeta.java @@ -0,0 +1,16 @@ +package org.bdware.datanet.searchengine; + +import com.google.gson.JsonObject; + +//RepoMeta要求是一个jsonObject，要求必须要有一个doi +public class DoMeta { + JsonObject metadata; + DoMeta(JsonObject jo){ + metadata=jo.deepCopy(); + } + + @Override + public String toString() { + return metadata.toString(); + } +} diff --git a/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/MetaGraph.java b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/MetaGraph.java new file mode 100644 index 0000000..8af8ef5 --- /dev/null +++ b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/MetaGraph.java @@ -0,0 +1,333 @@ +package org.bdware.datanet.searchengine; + + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.bdware.sc.engine.JSONTool; +import org.junit.Test; +import org.neo4j.graphdb.*; +import org.neo4j.graphdb.factory.GraphDatabaseFactory; +import org.neo4j.graphdb.index.Index; +import org.neo4j.graphdb.traversal.Evaluators; +import org.neo4j.graphdb.traversal.TraversalDescription; + +import java.io.File; +import java.util.Iterator; +import java.util.Map; + +public class MetaGraph { + + // private static final String DB_PATH = "testgraph.db"; + + private static String DB_PATH="./MetaGraph.db"; +// private static enum LABEL { +// DO, REAL, STANDARD +// } + + private GraphDatabaseService graphDB; + public MetaGraph(String name){ + DB_PATH="./"+name+".db"; + graphDB = new GraphDatabaseFactory().newEmbeddedDatabase(new File(DB_PATH)); + } + //DO类型 + + + /* + { + "doid":"xxx", + "maskinfo":{ + "stu_no":"@int(5,10)", + "name":"cname" + } + } + */ + public void createDoNode(String doid,JsonObject jo) { + Transaction tx = graphDB.beginTx(); + Node Do = graphDB.createNode(new NodeLabel("DO"));//命名规则 + Do.setProperty("doid",doid); + Iterator iter = jo.entrySet().iterator(); + while (iter.hasNext()) { + Map.Entry entry = (Map.Entry) iter.next(); + Do.setProperty(entry.getKey().toString(),entry.getValue().toString()); + } + tx.success(); + tx.close(); + //配置do的mask + } + public void iterDO() { + + Transaction tx = graphDB.beginTx(); + ResourceIterator iterator = graphDB.findNodes(new NodeLabel("DO")); + for (ResourceIterator it = iterator; it.hasNext(); ) { + Node node = it.next(); + String doid=(String)node.getProperty("doid"); + String id=(String)node.getProperty("id"); + System.out.println(doid + " " + id+ " "); + } + + tx.success(); + tx.close(); + } + //param:{"doid1":"xxx","doid2":"yyy","key1":"studentid","key2":"studentid"} + public void createRelationDo2Do(JsonObject jo){ + Transaction tx = graphDB.beginTx(); + String doid1=jo.get("id1").getAsString(); + String doid2=jo.get("id2").getAsString(); + String key1=jo.get("key1").getAsString(); + String key2=jo.get("key2").getAsString(); + + Node node1=graphDB.findNode(new NodeLabel("DO"),"doid",doid1); + Node node2=graphDB.findNode(new NodeLabel("DO"),"doid",doid2); + System.out.println(jo); + + node1.createRelationshipTo(node2, new RelationType(jo.toString())); + tx.success(); + tx.close(); + } + + + + //真实标签类型 + public void createRealNode(String key,String value) { + Transaction tx = graphDB.beginTx(); + Node realField = graphDB.createNode(new NodeLabel("realField"));//命名规则 + realField.setProperty("key",key); + realField.setProperty("value",value); + tx.success(); + tx.close(); + + } + + //标准标签 + + public void createStandardNode(String key,String value) { + Transaction tx = graphDB.beginTx(); + Node standardField = graphDB.createNode(new NodeLabel("standardField"));//命名规则 + standardField.setProperty("key",key); + standardField.setProperty("value",value); + tx.success(); + tx.close(); + } + public void iterRealField() { + + Transaction tx = graphDB.beginTx(); + ResourceIterator iterator = graphDB.findNodes(new NodeLabel("DO")); + for (ResourceIterator it = iterator; it.hasNext(); ) { + Node node = it.next(); + String doid=(String)node.getProperty("doid"); + String id=(String)node.getProperty("id"); + System.out.println(doid + " " + id+ " "); + } + + tx.success(); + tx.close(); + } + + public void iterStandardField() { + + Transaction tx = graphDB.beginTx(); + ResourceIterator iterator = graphDB.findNodes(new NodeLabel("DO")); + for (ResourceIterator it = iterator; it.hasNext(); ) { + Node node = it.next(); + String doid=(String)node.getProperty("doid"); + String id=(String)node.getProperty("id"); + System.out.println(doid + " " + id+ " "); + } + + tx.success(); + tx.close(); + } +// public void init() { +// graphDB = new GraphDatabaseFactory().newEmbeddedDatabase(new File(DB_PATH)); +// } + +// /**创建数据*/ +// public void create() { +// Transaction tx = graphDB.beginTx(); +// +// Node student = graphDB.createNode(new CaseLabel("STUDENT"));//命名规则 +// student.setProperty("name", "@student");//数据类型命名规则 +// student.setProperty("id", "@int(2001210000,2001210999)"); +// +// Node course = graphDB.createNode(new CaseLabel("COURSE"));//命名规则 +// course.setProperty("name", "[枚举or读文件]");//数据类型命名规则 +// course.setProperty("id", "@int(10000,10999)"); +// +// student.createRelationshipTo(course, new RelTypes("CRIME")); +// +// +// +// Node case1 = graphDB.createNode(new CaseLabel("CASEINFO")); +// case1.setProperty("name", "案件1"); +// case1.setProperty("address", "南山"); +// +// Node case2 = graphDB.createNode(new CaseLabel("CASEINFO")); +// case2.setProperty("name", "案件2"); +// case2.setProperty("address", "福田"); +// +// Node case3 = graphDB.createNode(new CaseLabel("CASEINFO")); +// case3.setProperty("name", "案件3"); +// case3.setProperty("address", "龙华"); +// +// Node userA = graphDB.createNode(new CaseLabel("PERSON")); +// userA.setProperty("name", "A"); +// userA.setProperty("idcard", "150302198012228239"); +// userA.setProperty("tel", "13685246639"); +// +// Node userB = graphDB.createNode(new CaseLabel("PERSON")); +// userB.setProperty("name", "B"); +// userB.setProperty("idcard", "370634199208304929"); +// userB.setProperty("tel", "13885246670"); +// +// Node userC = graphDB.createNode(new CaseLabel("PERSON")); +// userC.setProperty("name", "C"); +// userC.setProperty("idcard", "430721198802065735"); +// userC.setProperty("tel", "13966704782"); +// +// Node userD = graphDB.createNode(new CaseLabel("PERSON")); +// userD.setProperty("name", "D"); +// userD.setProperty("idcard", "522730198707118747"); +// userD.setProperty("tel", "13670478962"); +// +// Node userE = graphDB.createNode(new CaseLabel("PERSON")); +// userE.setProperty("name", "E"); +// userE.setProperty("idcard", "622926198609158032"); +// userE.setProperty("tel", "13047829667"); +// +// Node userF = graphDB.createNode(new CaseLabel("PERSON")); +// userF.setProperty("name", "F"); +// userF.setProperty("idcard", "500114197706138305"); +// userF.setProperty("tel", "13478296670"); +// +// Node userG = graphDB.createNode(new CaseLabel("PERSON")); +// userG.setProperty("name", "G"); +// userG.setProperty("idcard", "500114106138305152"); +// userG.setProperty("tel", "13476670156"); +// +// Node userH = graphDB.createNode(new CaseLabel("PERSON")); +// userH.setProperty("name", "H"); +// userH.setProperty("idcard", "500114197704751236"); +// userH.setProperty("tel", "13296156670"); +// +// case1.createRelationshipTo(userA, new RelTypes("CRIME")); +// case1.createRelationshipTo(userB, new RelTypes("CRIME")); +// case1.createRelationshipTo(userC, new RelTypes("CRIME")); +// case1.createRelationshipTo(userD, new RelTypes("CRIME")); +// +// case2.createRelationshipTo(userB, new RelTypes("CRIME")); +// case2.createRelationshipTo(userC, new RelTypes("CRIME")); +// case2.createRelationshipTo(userE, new RelTypes("CRIME")); +//// case2.createRelationshipTo(userD, RelTypes.CRIME); +// +// +// case3.createRelationshipTo(userF, new RelTypes("CRIME")); +// case3.createRelationshipTo(userG, new RelTypes("CRIME")); +// case3.createRelationshipTo(userH, new RelTypes("CRIME")); +// +// tx.success(); +// tx.close(); +// +// } +// +// public void search(String username) { +// Transaction tx = graphDB.beginTx(); +// System.out.println(graphDB.getAllNodes().stream().findFirst()); +// Node startNode = graphDB.findNode( "name", username); +// +// Iterable iterable = startNode +// .getRelationships(new RelTypes("CRIME"), Direction.INCOMING); +// for (Relationship r : iterable) { +// +// Node node = r.getStartNode(); +// long id = node.getId(); +// String name = (String)node.getProperty("name"); +//// String idcard = (String)node.getProperty("idcard"); +//// String tel = (String)node.getProperty("tel"); +// +// System.out.println(id + " " + name+ " "); +// } +// +// tx.success(); +// tx.close(); +// } +// +// /**遍历查找**/ +// public void searchUser(String username) { +// Transaction tx = graphDB.beginTx(); +// +// Node startNode = graphDB.findNode(new CaseLabel("PERSON"), "name", username); +//// Node startNode = graphDB.findNode(new CaseLabel("PERSON"), "name", "B"); +// +// TraversalDescription td = graphDB.traversalDescription(); +// td.relationships(new RelTypes("CRIME"), Direction.OUTGOING);//沿着关系 TONG +// td.depthFirst();//设置深度优先 +//// td.evaluator(Evaluators.excludeStartPosition()); +// +// //从开始节点开始 +// +// Iterable it = td.traverse(startNode).nodes(); +// for (Node node : it) { +// long id = node.getId(); +// if( node.hasLabel( new CaseLabel("PERSON") )) { +// String name = (String)node.getProperty("name"); +// String idcard = (String)node.getProperty("idcard"); +// String tel = (String)node.getProperty("tel"); +// +// System.out.println(id + " " + name+ " " + idcard + " " + tel); +// } else { +//// String name = (String)node.getProperty("name"); +//// String address = (String)node.getProperty("address"); +//// System.out.println(id + " " + name+ " " + address); +// } +// } +// +// tx.success(); +// tx.close(); +// } +// + + +// +// /**查询所有作案人员信息*/ +// public void searchAllPerson() { +// +// +// +// Transaction tx = graphDB.beginTx(); +// +// ResourceIterator iterator = graphDB.findNodes(new CaseLabel("PERSON")); +// for (ResourceIterator it = iterator; it.hasNext(); ) { +// Node node = it.next(); +// long id = node.getId(); +// String name = (String)node.getProperty("name"); +// String idcard = (String)node.getProperty("idcard"); +// String tel = (String)node.getProperty("tel"); +// +// System.out.println(id + " " + name+ " " + idcard + " " + tel); +// } +// +// tx.success(); +// tx.close(); +// } +} + +class RelationType implements RelationshipType { + private String name; + public RelationType(String name) { + this.name = name; + } + + public String name(){ + return name; + } +} +class NodeLabel implements Label { + private String name; + public NodeLabel(String name) { + this.name = name; + } + + public String name(){ + return name; + } +} diff --git a/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/MetaIndex.java b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/MetaIndex.java new file mode 100644 index 0000000..677cc5b --- /dev/null +++ b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/MetaIndex.java @@ -0,0 +1,341 @@ +package org.bdware.datanet.searchengine; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.document.*; +import org.apache.lucene.index.*; +import org.apache.lucene.queryparser.classic.MultiFieldQueryParser; +import org.apache.lucene.queryparser.classic.ParseException; +import org.apache.lucene.queryparser.classic.QueryParser; +import org.apache.lucene.search.*; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.Version; +import org.bdware.sc.boundry.utils.RocksDBUtil; +import org.rocksdb.RocksIterator; +import org.wltea.analyzer.lucene.IKAnalyzer; +import java.io.*; +import java.nio.file.Paths; +import java.util.*; +public class MetaIndex { + private final Logger LOGGER = LogManager.getLogger(MetaIndex.class); + public FSDirectory indexDir; + public boolean isEmpty = false;//描述当前repo的文件是否存在 + private IndexWriter indexWriter; + private static final int COUNT=20; + private static final int OFFSET=0; + MetaIndex(String name){ + initIndex(name); + } + MetaIndex(){ + initIndex("SearchEngineIndex"+System.currentTimeMillis()); + } + + public void initIndex(String Dirname) { + try { + File dir = new File(Dirname); + if (!dir.exists()) { + LOGGER.info("make metaIndex dir "); + dir.mkdirs(); + isEmpty = true; + } + indexDir = FSDirectory.open(Paths.get(dir.toURI())); + Analyzer analyzer = new IKAnalyzer(); + indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(analyzer)); + indexWriter.commit(); + + + } catch (Exception e) { + e.printStackTrace(); + } + } + public void indexRepo(DoMeta meta) throws IOException { + try { + //IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig()); + Document document = new Document(); + // System.out.println(meta.metadata); + for (Map.Entry entry : meta.metadata.entrySet()) { + String key=entry.getKey().toString(); + System.out.println(entry.getValue()); + String val=entry.getValue().toString().substring(1,entry.getValue().toString().length()-1); + System.out.println(key+" "+val); + //if(key.equals("doid")||key.equals("owner")) + // document.add(new StringField(key,val, Field.Store.YES)); + //范围类型 + if(key.equals("createTime")||key.equals("createDate")){ + document.add(new StringField(key,val,Field.Store.YES)); + } + + else if(key.equals("doId")){ + document.add(new StringField(key,val,Field.Store.YES)); + //document.add(new LongPoint(key,Long.valueOf(val)),Field.Store.YES); + } + + else{ + document.add(new TextField(key,val, Field.Store.YES)); + } + + //System.out.println("eee"+entry.getKey().toString()+" "+entry.getValue().toString().substring(1,entry.getValue().toString().length()-1)); + } + indexWriter.addDocument(document); + indexWriter.commit(); + isEmpty = false; + } + catch (Exception e) { + e.printStackTrace(); + } + + } + + public void updateRepoList(List DoRepos) + throws IOException, ParseException { + IndexReader indexReader = null; + for (DoMeta r : DoRepos) { + String doId=r.metadata.get("doId").toString(); + if(isEmpty){ + indexRepo(r); + LOGGER.info("--> index when dir is not empty"); + LOGGER.info("contract " + doId + " --> actually to index"); + } + else{ + try { + indexReader = DirectoryReader.open(indexDir); + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + LOGGER.debug( doId+ "--> try to index"); + //Query query = new TermQuery(new Term("doid", "AIBD/doip.Repo1001")); + Query query = new TermQuery(new Term("doId", doId)); + TopDocs docs = indexSearcher.search(query, 10); + + + LOGGER.debug(docs.scoreDocs); + + if(docs.scoreDocs == null || docs.scoreDocs.length == 0) { + indexRepo(r); + LOGGER.info("--> index when dir is not empty"); + LOGGER.info("contract " + doId + " --> actually to index"); + } + docs = indexSearcher.search(query, 10); + } catch (Exception e) { + LOGGER.warn("getting index failed! " + e.getMessage()); + } + } + + } + if (null != indexReader) { + indexReader.close(); + } + } + public void updateRepo(DoMeta DoRepo) + throws IOException, ParseException { + IndexReader indexReader = null; + + String doId=DoRepo.metadata.get("doId").getAsString(); + if(isEmpty){ + indexRepo(DoRepo); + LOGGER.info("--> index when dir is not empty"); + LOGGER.info("contract " + doId + " --> actually to index"); + } + else{ + try { + indexReader = DirectoryReader.open(indexDir); + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + LOGGER.debug( doId+ "--> try to index"); + Query query = new TermQuery(new Term("doId", doId)); + TopDocs docs = indexSearcher.search(query, 10); + LOGGER.debug(docs.scoreDocs); + System.out.println("bb"+docs.scoreDocs.length); + if(docs.scoreDocs == null || docs.scoreDocs.length == 0) { + indexRepo(DoRepo); + LOGGER.info("--> index when dir is not empty"); + LOGGER.info("--> index when dir is not empty"); + LOGGER.info("contract " + doId + " --> actually to index"); + } + docs = indexSearcher.search(query, 10); + } catch (Exception e) { + LOGGER.warn("getting index failed! " + e.getMessage()); + } + } + + + if (null != indexReader) { + indexReader.close(); + } + } + public void deleteRepo(String doId) { + + try { + IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig()); + Query query=new TermQuery( new Term("doId",doId)); + System.out.println("query"+query.toString()); + indexWriter.deleteDocuments(query); + indexWriter.commit(); + indexWriter.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + //模糊搜索 + public String search(JsonObject jo) { + try { + if (!jo.has("type")) { + return "missing arguments"; + } + String type=jo.get("type").getAsString(); + if (!jo.has("keyword")) return "missing arguments: keyword"; + int offset=OFFSET; + if (jo.has("offset")) offset= jo.get("offset").getAsInt(); + offset= Math.max(offset, 0); + int count=COUNT; + if (jo.has("count")) count= jo.get("count").getAsInt(); + count=count<0?COUNT:count; + String keyword = jo.get("keyword").getAsString(); + System.out.println(type+" "+keyword); +// Analyzer analyzer = new IKAnalyzer(); +// QueryParser queryParser = new QueryParser(type, analyzer); +// Query rmQuery = queryParser.parse(keyword); + + DirectoryReader indexReader = DirectoryReader.open(indexDir); + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + Query query = new TermQuery(new Term(type, keyword)); + TopDocs docs = indexSearcher.search(query, 10); + System.out.println("length "+docs.scoreDocs.length); + ResultModel resultModel = null; + if (docs.scoreDocs != null && docs.scoreDocs.length > 0) + resultModel = paginate(docs, indexReader,count,offset); + + if (resultModel != null) { + return resultModel.toString(); + } + ResultModel noDataResult=new ResultModel(); + return noDataResult.toString(); + } + catch (Exception e) { + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + e.printStackTrace(new PrintStream(bo)); + return bo.toString(); + } + } + //按照范围查询 + public String searchByRange(JsonObject jo) { + try { + if (!jo.has("type")) { + return "missing arguments"; + } + //if (!jo.has("keyword")) return "missing arguments: keyword"; + if (!jo.has("from")) return "missing arguments: from"; + if (!jo.has("to")) return "missing arguments: to"; + String type = jo.get("type").getAsString(); + //String keyword = jo.get("keyword").getAsString(); + String from = jo.get("from").getAsString(); + String to = jo.get("to").getAsString(); + + int offset = OFFSET; + if (jo.has("offset")) offset = jo.get("offset").getAsInt(); + offset = Math.max(offset, 0); + int count = COUNT; + if (jo.has("count")) count = jo.get("count").getAsInt(); + count = count < 0 ? COUNT : count; +// Analyzer analyzer = new IKAnalyzer(); +// QueryParser queryParser = new QueryParser(type, analyzer); +// Query rmQuery = queryParser.parse(keyword); + + DirectoryReader indexReader = DirectoryReader.open(indexDir); + IndexSearcher indexSearcher = new IndexSearcher(indexReader); + Query query = new TermRangeQuery(type, new BytesRef(from), new BytesRef(to), true, true); + //Query query = new TermQuery(new Term(type, keyword)); + TopDocs docs = indexSearcher.search(query, 10); + + ResultModel resultModel = null; + if (docs.scoreDocs != null && docs.scoreDocs.length > 0) { + resultModel = paginate(docs, indexReader, count,offset); + + if (resultModel != null) { + return resultModel.toString(); + } + ResultModel noDataResult = new ResultModel(); + return noDataResult.toString(); + } + } catch (Exception e) { + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + e.printStackTrace(new PrintStream(bo)); + return bo.toString(); + } + ResultModel noDataResult = new ResultModel(); + return noDataResult.toString(); + } + /* + {"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0} + */ + public String searchByMultiTag(JsonObject jo) { + try { + int offset = OFFSET; + if (jo.has("offset")) offset = jo.get("offset").getAsInt(); + offset = Math.max(offset, 0); + int count = COUNT; + if (jo.has("count")) count = jo.get("count").getAsInt(); + count = count < 0 ? COUNT : count; + + //System.out.println(count+" "+offset); + JsonArray parmas=jo.get("params").getAsJsonArray(); + parmas.get(0).getAsJsonObject().get("keyword"); + String[] type=new String[parmas.size()]; + String[] keyword=new String[parmas.size()]; + BooleanClause.Occur[] flags = new BooleanClause.Occur[parmas.size()]; + for(int i=0;i 0) { + resultModel = paginate(docs, indexReader, count,offset); + if (resultModel != null) { + return resultModel.toString(); + } + ResultModel noDataResult = new ResultModel(); + return noDataResult.toString(); + } + + }catch (Exception e){ + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + e.printStackTrace(new PrintStream(bo)); + return bo.toString(); + } + ResultModel noDataResult = new ResultModel(); + return noDataResult.toString(); + } + + public ResultModel paginate(TopDocs docs, IndexReader reader,int count,int offset ) throws IOException { + ResultModel resultModel = new ResultModel(); + resultModel.setContractCount(docs.totalHits.value); + ScoreDoc[] scoreDocs = docs.scoreDocs; + List doMetaList = new ArrayList<>(); + + if (scoreDocs != null) { + for (int i = offset; i < scoreDocs.length && i < offset+count; i++) { + Document document = reader.document(scoreDocs[i].doc); + JsonObject jo=new JsonObject(); + for (IndexableField in : document.getFields()) { + String key=in.name(); + String val=in.stringValue(); + jo.addProperty(key,val); + } + DoMeta repometa=new DoMeta(jo); + doMetaList.add(repometa); + } + } + resultModel.setDoMetaList(doMetaList); + return resultModel; + } + + +} diff --git a/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/ResultModel.java b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/ResultModel.java new file mode 100644 index 0000000..cb7463e --- /dev/null +++ b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/ResultModel.java @@ -0,0 +1,48 @@ +package org.bdware.datanet.searchengine; +import com.google.gson.JsonObject; + +import java.util.List; +public class ResultModel { + private List DoMetaList; + private Long DoCount; + public ResultModel() { + this.DoMetaList = null; + this.DoCount = 0L; + } + + public List getRepoMetaList() { + return DoMetaList; + } + + public void setDoMetaList(List metalist) { + this.DoMetaList = metalist; + Integer i= metalist.size(); + DoCount=i.longValue(); + } + + public Long getRepoCount() { + return DoCount; + } + + public void setContractCount(Long DoCount) { + this.DoCount = DoCount; + } + public JsonObject toJson(){ + JsonObject jo=new JsonObject(); + //jo.addProperty("repoCount",repoCount); + return jo; + } + + + @Override + public String toString() { + if(DoMetaList==null) + return "{\"data\":[],\"total\":0}"; + else + return "{" + + "\"data\":" + DoMetaList.toString() + + ", \"total\":" + DoCount + + '}'; + } +} + diff --git a/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/SearchEngine.java b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/SearchEngine.java new file mode 100644 index 0000000..e364e00 --- /dev/null +++ b/datanet-search-engine-backend/src/main/java/org/bdware/datanet/searchengine/SearchEngine.java @@ -0,0 +1,182 @@ +package org.bdware.datanet.searchengine; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import org.bdware.sc.ContractProcess; +import org.bdware.sc.engine.JSONTool; + +import wrp.jdk.nashorn.api.scripting.ScriptObjectMirror; +import wrp.jdk.nashorn.api.scripting.ScriptUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.lucene.queryparser.classic.ParseException; + +import javax.script.Invocable; +import javax.script.ScriptException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + + +public class SearchEngine{ + + private final Logger LOGGER = LogManager.getLogger(SearchEngine.class); + public MetaIndex metaIndex; + public SearchEngine(String Dirname){ + metaIndex=new MetaIndex(Dirname); + } + public SearchEngine(){ + metaIndex=new MetaIndex(); + } + + + public static SearchEngine createObj(String Dirname){ + return new SearchEngine(Dirname); + } + public static SearchEngine createObj(){ + return new SearchEngine(); + } + + public String test(){return "success";} + + //params: + //String type,String keyword,String count,String offset + //String type,String keyword + public String search(ScriptObjectMirror so){ + + if(JSONTool.convertMirrorToJson(so).isJsonObject()){ + JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject(); + System.out.println("search: "+jo); + if(jo.get("count")==null) jo.addProperty("count",10); + if(jo.get("offset")==null) jo.addProperty("offset",0); + return metaIndex.search(jo); + } + return "failed"; + } + + + + //根据时间范围查询 from ：to + public String searchByRange(ScriptObjectMirror so){ + + if(JSONTool.convertMirrorToJson(so).isJsonObject()){ + JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject(); + if(jo.get("count")==null) jo.addProperty("count",10); + if(jo.get("offset")==null) jo.addProperty("offset",0); + return metaIndex.searchByRange(jo); + } + return "failed"; + } + public String search(String type,String keyword,int count,int offset){ + JsonObject jo =new JsonObject(); + jo.addProperty("type",type); + jo.addProperty("keyword",keyword); + jo.addProperty("offset",offset); + jo.addProperty("count",count); + return metaIndex.search(jo); + } + public String search(String type,String keyword,String count,String offset){ + JsonObject jo =new JsonObject(); + jo.addProperty("type",type); + jo.addProperty("keyword",keyword); + jo.addProperty("offset",offset); + jo.addProperty("count",count); + return metaIndex.search(jo); + } + + /* + {"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0} + */ + public String searchByMultiTag(ScriptObjectMirror so){ + JsonElement je=JSONTool.convertMirrorToJson(so); + JsonObject jo =je.getAsJsonObject(); + return metaIndex.searchByMultiTag(jo); + } + + + + //{"doid":"xxx"} + public String delete(ScriptObjectMirror so){ + + if(JSONTool.convertMirrorToJson(so).isJsonObject()){ + JsonObject jo=JSONTool.convertMirrorToJson(so).getAsJsonObject(); + if(jo.get("doId")==null) return "failed"; + else{ + metaIndex.deleteRepo(jo.get("doId").getAsString()); + return "success"; + } + } + return "failed"; + } + public String delete(String doId){ + metaIndex.deleteRepo(doId); + return "success"; + } + + public String search(String type, String keyword){ + JsonObject jo =new JsonObject(); + jo.addProperty("type",type); + jo.addProperty("keyword",keyword); + jo.addProperty("offset",0); + jo.addProperty("count",10); + return metaIndex.search(jo); + } + + + + public int count(){ + return 0; + } + + //一个do或者一个do数组 + public String update(ScriptObjectMirror so) throws IOException, ParseException { + JsonElement je=JSONTool.convertMirrorToJson(so); +// if(je.isJsonArray()){ +// List list=new ArrayList<>(); +// for(JsonElement s : je.getAsJsonArray()) { +// DoMeta r = new DoMeta(s.getAsJsonObject()); +// list.add(r); +// } +// metaIndex.updateRepoList(list); +// return "success"; +// } + if(je.isJsonObject()){ + + DoMeta r=new DoMeta(je.getAsJsonObject()); + metaIndex.updateRepo(r); + return "success"; + } + else return "failed"; + } + public String testScriptObjectMirror(ScriptObjectMirror so) throws IOException, ParseException { + //JsonObject jo = (JsonObject) ScriptUtils.convert(so, JsonObject.class); + JsonElement je=JSONTool.convertMirrorToJson(so); + System.out.println(so.toString()); + System.out.println(so.getClass()); + System.out.println(je.getClass()); + System.out.println(je.toString()); + return "success"; + } + public void addFunction(String function){ + try{ + ContractProcess.instance.engine.compileFunction(function,false); + //ContractProcess.instance.engine.invokeFunction(function,args); + + //ContractProcess.instance.engine + }catch (ScriptException e){ + e.printStackTrace(); + } + } + + +// public String updateList(JsonArray ja) throws IOException, ParseException { +// List list=new ArrayList<>(); +// for(JsonElement s : ja.getAsJsonArray()){ +// RepoMeta r=new RepoMeta(s.getAsJsonObject()); +// list.add(r); +// } +// metaIndex.updateRepoList(list); +// return "success"; +// } +} \ No newline at end of file diff --git a/datanet-search-engine-backend/src/test/java/DebugMain.java b/datanet-search-engine-backend/src/test/java/DebugMain.java new file mode 100644 index 0000000..54643c0 --- /dev/null +++ b/datanet-search-engine-backend/src/test/java/DebugMain.java @@ -0,0 +1,5 @@ +public class DebugMain { + public static void main(String[]args){ + org.bdware.sc.debugger.DebugMain.main(null); + } +} diff --git a/datanet-search-engine-backend/src/test/java/GenerateDO.java b/datanet-search-engine-backend/src/test/java/GenerateDO.java new file mode 100644 index 0000000..1fc7996 --- /dev/null +++ b/datanet-search-engine-backend/src/test/java/GenerateDO.java @@ -0,0 +1,54 @@ +import com.google.gson.JsonObject; +import org.apache.log4j.Logger; +import org.bdware.doip.audit.EndpointConfig; +import org.bdware.doip.audit.client.AuditIrpClient; +import org.bdware.doip.audit.config.FileStorage; +import org.bdware.irp.exception.IrpClientException; +import org.bdware.irp.stateinfo.StateInfoBase; +import org.junit.Before; +import org.junit.Test; + +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.PrintStream; + +public class GenerateDO { + static Logger LOGGER = Logger.getLogger(GenerateDO.class); + AuditIrpClient client; + JsonObject otherConfig; + + @Before + public void init() throws FileNotFoundException { + FileStorage fs = new FileStorage("./input/auditrepo.json"); + EndpointConfig config = fs.loadAsEndpointConfig(); + config.repoName = null; + otherConfig = fs.load(); + client = new AuditIrpClient(config); + } + + @Test + public void test() throws Exception { + FileOutputStream fout = new FileOutputStream("./input/doi.txt",true); + PrintStream ps = new PrintStream(fout); + StateInfoBase base = new StateInfoBase(); + base.handleValues = new JsonObject(); + base.handleValues.addProperty("repoId", otherConfig.get("targetRepo").getAsString()); + base.handleValues.addProperty("timestamp", System.currentTimeMillis()); + try { + String doid = client.register(base); + if (doid != null) ps.println(doid); + String format = "http://%s/SCIDE/SCManager?action=executeContract&contractID=SearchEngine&operation=testAddDo&arg="; + String argStr = "{\"doId\":\"%s\",\"name\":\"repofile\",\"type\":\"text\",\"description\":\"这是一个模拟DO\",\"owner\":\"xx省yy市zz部门\",\"createTime\":%d}"; + format += argStr; + String seURI = otherConfig.get("seURI").getAsString(); + long createTime = System.currentTimeMillis(); + long l = (long) (Math.random() * 1000D * 3600D * 24D); + String data = String.format(format, seURI, doid, createTime - l); + System.out.println(data); + } catch (IrpClientException e) { + e.printStackTrace(); + } + fout.close(); + ps.close(); + } +} diff --git a/datanet-search-engine-backend/src/test/java/GraphDBTEst.java b/datanet-search-engine-backend/src/test/java/GraphDBTEst.java new file mode 100644 index 0000000..9b8c2eb --- /dev/null +++ b/datanet-search-engine-backend/src/test/java/GraphDBTEst.java @@ -0,0 +1,26 @@ +import com.google.gson.JsonObject; +import org.bdware.datanet.searchengine.MetaGraph; +import org.junit.Test; + +public class GraphDBTEst { + @Test + public void Test(){ + MetaGraph m=new MetaGraph("test"); + JsonObject do1=new JsonObject(); + do1.addProperty("id", "1"); + JsonObject do2=new JsonObject(); + do2.addProperty("id","2"); + m.createDoNode("001",do1); + m.createDoNode("002",do2); + JsonObject jo=new JsonObject(); + jo.addProperty("id1","001"); + jo.addProperty("id2","002"); + jo.addProperty("key1","stu_no"); + jo.addProperty("key2","student_no"); + m.createRelationDo2Do(jo); + m.iterDO(); + m.iterRealField(); + m.iterStandardField(); + + } +} diff --git a/datanet-search-engine-backend/src/test/java/luceneTest.java b/datanet-search-engine-backend/src/test/java/luceneTest.java new file mode 100644 index 0000000..0df52d7 --- /dev/null +++ b/datanet-search-engine-backend/src/test/java/luceneTest.java @@ -0,0 +1,69 @@ +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.junit.Test; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.standard.StandardAnalyzer; +//import org.wltea.analyzer.lucene.IKAnalyzer; + +import java.io.StringReader; + +public class luceneTest { + @Test + public void StandardAnalyzerTest() throws Exception{ + //英文测试 + String text="An IndexWriter creaters and maintains an index."; + Analyzer analyzer=new StandardAnalyzer(); + displayTokens(analyzer,text); + //中文测试 + String text1="我们这款产品底层逻辑是打通信息屏障，创建行业生态。顶层设计是聚焦用户感和赛道，通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能，体验度量作为闭环的评判标准。亮点是载体，优势是链路。思考整个生命周期，完善逻辑，考虑资源倾斜。方法论是组合拳达到平台化标准。"; + displayTokens(analyzer,text1); + } + /** + 58 * IKAnalyzerTest分词法测试,对中文支持很好，词库分词 + 59 * @throws Exception + 60 */ + @Test + public void IKAnalyzerTest() throws Exception{ + //英文测试 + String text="An IndexWriter creaters and maintains an index."; +// Analyzer analyzer=new IKAnalyzer(); + Analyzer analyzer=new StandardAnalyzer(); + displayTokens(analyzer,text); + //中文测试 + String text1="我们这款产品底层逻辑是打通信息屏障，创建行业生态。顶层设计是聚焦用户感和赛道，通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能，体验度量作为闭环的评判标准。亮点是载体，优势是链路。思考整个生命周期，完善逻辑，考虑资源倾斜。方法论是组合拳达到平台化标准。"; + displayTokens(analyzer,text1); + } + public static void displayTokens(Analyzer analyzer, String text) throws Exception { + System.out.println("当前使用的分词器：" + analyzer.getClass().getName()); + //分词流，即将对象分词后所得的Token在内存中以流的方式存在，也说是说如果在取得Token必须从TokenStream中获取，而分词对象可以是文档文本，也可以是查询文本。 + TokenStream tokenStream = analyzer.tokenStream("content", new StringReader(text)); + //表示token的首字母和尾字母在原文本中的位置。比如I'm的位置信息就是(0,3)，需要注意的是startOffset与endOffset的差值并不一定就是termText.length()， + //因为可能term已经用stemmer或者其他过滤器处理过； + OffsetAttribute offsetAttribute = tokenStream.addAttribute(OffsetAttribute.class); + //这个有点特殊，它表示tokenStream中的当前token与前一个token在实际的原文本中相隔的词语数量，用于短语查询。比如： 在tokenStream中[2:a]的前一个token是[1:I'm ]， + //它们在原文本中相隔的词语数是1，则token="a"的PositionIncrementAttribute值为1； + PositionIncrementAttribute positionIncrementAttribute = tokenStream.addAttribute(PositionIncrementAttribute.class); + + CharTermAttribute charTermAttribute= tokenStream.addAttribute(CharTermAttribute.class); + + //表示token词典类别信息，默认为“Word”，比如I'm就属于，有撇号的类型； + TypeAttribute typeAttribute = tokenStream.addAttribute(TypeAttribute.class); + tokenStream.reset(); + + int position = 0; + while (tokenStream.incrementToken()) { + int increment = positionIncrementAttribute.getPositionIncrement(); + if(increment > 0) { + position = position + increment; + } + int startOffset = offsetAttribute.startOffset(); + int endOffset = offsetAttribute.endOffset(); + String term ="输出结果为："+ charTermAttribute.toString(); + System.out.println("第"+position+"个分词，分词内容是:[" + term + "]" + "，分词内容的开始结束位置为：(" + startOffset + "-->" + endOffset + ")，类型是：" + typeAttribute.type()); + } + tokenStream.close(); + } +} diff --git a/datanet-search-engine-backend/src/test/java/searchEnginetest.java b/datanet-search-engine-backend/src/test/java/searchEnginetest.java new file mode 100644 index 0000000..374f1de --- /dev/null +++ b/datanet-search-engine-backend/src/test/java/searchEnginetest.java @@ -0,0 +1,184 @@ + +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngine; +import wrp.jdk.nashorn.api.scripting.ScriptUtils; +import org.apache.lucene.queryparser.classic.ParseException; +import org.bdware.sc.boundry.JavaScriptEntry; +import org.bdware.sc.engine.JSONTool; +import org.junit.Test; +import org.bdware.datanet.searchengine.SearchEngine; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngine; +import wrp.jdk.nashorn.api.scripting.NashornScriptEngineFactory; +import wrp.jdk.nashorn.api.scripting.ScriptObjectMirror; +import wrp.jdk.nashorn.internal.objects.Global; +import wrp.jdk.nashorn.internal.runtime.*; + +import java.io.IOException; +import java.util.ArrayList; + +public class searchEnginetest { + public static String dbname="MetaIndex01"; + @Test + public void test2() throws IOException, ParseException{ + System.out.println("hello"); + //1642696178922 + System.out.println(System.currentTimeMillis()); + } + + @Test + public void addDo() { + NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6"); + SearchEngine se=SearchEngine.createObj(dbname); + //SearchEngine se=new SearchEngine(); + try{ + engine.eval("var r1 = {\"url\":\"39.106.6.6:3306\",\"user\":\"loushuai\",\"password\":\"loushuai\",\"dbname\":\"newele\",\"tablename\":\"data\",\"createTime\":\"1641363811328\",\"owner\":\"zzz\",\"description\":\"这是一段说明\",\"schema\":\"{}\",\"prefix\":\"aibd.govdata.tj\",\"type\":\"SQL\",\"doId\":\"aibd.govdata.tj/DBRepo/do.newele.data\"};"); + se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r1")); + System.out.println(se.update((ScriptObjectMirror) engine.eval("r1"))); + //System.out.println(se.search("info","产品")); + + }catch(Exception e){ + e.printStackTrace(); + } + + } + + @Test + public void addDo2() { + NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6"); + SearchEngine se=SearchEngine.createObj(dbname); + //SearchEngine se=new SearchEngine(); + try{ + engine.eval("var r1 = {doId:\"AIBD/doip.Repo10101\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障，创建行业生态。顶层设计是聚焦用户感和赛道，通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能，体验度量作为闭环的评判标准。亮点是载体，优势是链路。思考整个生命周期，完善逻辑，考虑资源倾斜。方法论是组合拳达到平台化标准。\"};"); + se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r1")); + System.out.println(se.update((ScriptObjectMirror) engine.eval("r1"))); + + engine.eval("var r2 = {doId:\"AIBD/doip.Repo10102\", type:\"readme\", createTime:\"1842696178922\",info:\"我们这款产品底层逻辑是打通信息屏障，创建行业生态。顶层设计是聚焦用户感和赛道，通过差异化和颗粒度达到引爆点。交付价值是在垂直领域采用复用大法达成持久收益。抽离透传归因分析作为抓手为产品赋能，体验度量作为闭环的评判标准。亮点是载体，优势是链路。思考整个生命周期，完善逻辑，考虑资源倾斜。方法论是组合拳达到平台化标准。\"};"); + se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("r2")); + System.out.println(se.update((ScriptObjectMirror) engine.eval("r2"))); + //System.out.println(se.search("info","产品")); + }catch(Exception e){ + e.printStackTrace(); + } + } + @Test + public void addDoList() + { + NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6"); + SearchEngine se=SearchEngine.createObj(dbname); + try{ + engine.eval("var j1 = [{doId:\"AIBD/doip.Repo10101\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障……\"}," + + "{doId:\"AIBD/doip.Repo10102\", type:\"readme\", createTime:\"1642696198922\",info:\"你们这款产品底层逻辑是打通信息屏障……\"}];"); + se.update((ScriptObjectMirror) engine.eval("j1")); + System.out.println(se.search("info","产品")); + }catch(Exception e){ + e.printStackTrace(); + } + + } + @Test + public void searchRepoTest() throws IOException, ParseException { + NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6"); + SearchEngine se=SearchEngine.createObj(dbname); + try{ + engine.eval("var arg ={type:\"doId\",keyword:\"AIBD/doip.Repo\"};"); + System.out.println(se.search((ScriptObjectMirror) engine.eval("arg"))); + engine.eval("var arg1 ={type:\"owner\",keyword:\"zzz\"};"); + System.out.println(se.search((ScriptObjectMirror) engine.eval("arg1"))); + //System.out.println(se.search("info","产品",5,0)); + }catch(Exception e){ + e.printStackTrace(); + } + + + } + @Test + public void searchByRangeTest() throws IOException, ParseException { + NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6"); + SearchEngine se=SearchEngine.createObj(dbname); + try{ + engine.eval("var arg ={type:\"createTime\",from:\"1642696178922\",to:\"1942696178922\"};"); + //engine.eval("var arg ={type:\"createTime\",keyword:\"createTime\",from:\"1642696178922\",to:\"1642696178922\"};"); + System.out.println(se.searchByRange ((ScriptObjectMirror) engine.eval("arg"))); + }catch(Exception e){ + e.printStackTrace(); + } + + } + + //{"params":[{"type":"doid","ketword":"DBRepo"},{"type":"name","ketword":"xxx"}],"count":10,"offset":0} + + @Test + public void searchByMultiTagTest() { + NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6"); + SearchEngine se=SearchEngine.createObj(dbname); + try{ +// engine.eval("var r1 = {doId:\"repo1\", type:\"readme\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障，创建行业生态。\"};"); +// System.out.println(se.update((ScriptObjectMirror) engine.eval("r1"))); +// +// engine.eval("var r2 = {doId:\"repo2\", type:\"sql\", createTime:\"1642696178922\",info:\"我们这款产品底层逻辑是打通信息屏障，创建行业生态。\"};"); +// System.out.println(se.update((ScriptObjectMirror) engine.eval("r2"))); + + engine.eval("var arg2 ={type:\"info\",keyword:\"我们\",count:10,offset:0};"); + System.out.println(se.search ((ScriptObjectMirror) engine.eval("arg2"))); + + + engine.eval("var arg1 ={params:[{type:\"doId\",keyword:\"AIBDdoip.Repo1001\"}],count:10,offset:0};"); + System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg1"))); + + engine.eval("var arg ={params:[{type:\"info\",keyword:\"我们\"},{type:\"doId\",keyword:\"repo\"}],count:10,offset:0};"); + System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg"))); + + engine.eval("var arg ={\n" + + " params: [{ type: 'doId', keyword: 'd' }],\n" + + " count: 100,\n" + + " offset: 0,\n" + + " };"); + System.out.println(se.searchByMultiTag ((ScriptObjectMirror) engine.eval("arg"))); + + + + }catch(Exception e){ + e.printStackTrace(); + } + + } + + @Test + public void deleteTest(){ + NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6"); + SearchEngine se=SearchEngine.createObj(dbname); + try{ + + engine.eval("var r1 = {doId:\"AIBD/doip.Repo1001\"};"); + System.out.println(se.delete((ScriptObjectMirror)engine.eval("r1"))); + //se.delele((ScriptObjectMirror)engine.eval("doid")); + System.out.println(se.search("info","产品")); + System.out.println(se.search("doId","AIBD/doip.Repo1002")); + }catch(Exception e){ + e.printStackTrace(); + } + + } + @Test + public void jsTest(){ + NashornScriptEngineFactory factory = new NashornScriptEngineFactory(); + NashornScriptEngine engine = (NashornScriptEngine)factory.getScriptEngine("--language=es6"); + SearchEngine se=SearchEngine.createObj(dbname); + try{ + System.out.println(engine.eval("var car = {name:\"Fiat\", model:500, color:\"white\"};")); + System.out.println(engine.eval("car").getClass()); + se.testScriptObjectMirror((ScriptObjectMirror) engine.eval("car")); + }catch(Exception e){ + e.printStackTrace(); + } + } + +} diff --git a/datanet-search-engine-backend/yjs/DAC.yjs b/datanet-search-engine-backend/yjs/DAC.yjs new file mode 100644 index 0000000..6499de3 --- /dev/null +++ b/datanet-search-engine-backend/yjs/DAC.yjs @@ -0,0 +1,49 @@ +module NaiveDAC{ + function initDAC(req){ + Global.acceptList = { + }; + Global.applyList = { + }; + Global.owner = req; + } + function checkPermission(req){ + return (Global.acceptList[req]!=undefined); + } + export function hasPermission(arg){ + return (Global.acceptList[requester]!=undefined); + } + export function accept(req){ + if (requester==Global.owner){ + Global.acceptList[req] = Global.applyList[req]; + if (Global.applyList[req] != undefined) + Global.applyList[req] = undefined; + return true; + } + return false; + } + export function cancel(req){ + if (requester==Global.owner){ + Global.applyList[req] = Global.acceptList[req]; + if (Global.acceptList[req] != undefined) + Global.acceptList[req] = undefined; + return true; + } + return false; + } + export function apply(remark){ + if (remark != undefined && requester !=undefined){ + Global.applyList[requester] = remark; + return "success"; + } + return "failed" + } + export function isOwner(arg){ + return Global.owner==requester; + } + export function getApplyList(arg){ + return JSON.stringify(Global.applyList); + } + export function getAcceptList(arg){ + return JSON.stringify(Global.acceptList); + } +} diff --git a/datanet-search-engine-backend/yjs/SearchEngine.yjs b/datanet-search-engine-backend/yjs/SearchEngine.yjs new file mode 100644 index 0000000..282d995 --- /dev/null +++ b/datanet-search-engine-backend/yjs/SearchEngine.yjs @@ -0,0 +1,410 @@ +import "DAC.yjs" +@Permission("RocksDB") +//onCreate时初始化rocksdb +oracle SearchEngine { + function onCreate(){ + Global.message = ""; + Global.events = []; + Global.allSearchCount=0;//搜索次数 + Global.allIndexCount=0;//检索次数 + Global.allUpdateCount=0;//更新次数 + Global.allInvokeCount=0//调用次数 + Global.allRepoCount=0;//Repo数量 + //SearchEngine监管的Repo数据 + Global.allUserCount=0;//用户数量 + Global.AllDoCount=0;//Do数量 + //SearchEngine监管的Do数据 + initDAC(requester); + Global.se= org.bdware.datanet.searchengine.SearchEngine.createObj("SearchEngineDoMetaIndex"); + } + //onRecover初始化索引 + function onRecover(){ + // Global.se= org.bdware.datanet.searchengine.SearchEngine.createObj("SearchEngineDoMetaIndex"); + // Global.message = ""; + // Global.events = []; + // Global.allSearchCount=0;//搜索次数 + // Global.allIndexCount=0;//检索次数 + // Global.allUpdateCount=0;//更新次数 + // Global.allInvokeCount=0//调用次数 + // Global.allRepoCount=0;//Repo数量 + // //SearchEngine监管的Repo数据 + // Global.allUserCount=0;//用户数量 + // Global.AllDoCount=0;//Do数量 + // initDAC(requester); + } + function convertArgToJson(arg){ + if (typeof(arg)=='string') return JSON.parse(arg); + return arg; + } + function updateRepoHandler(e) { + var ret = "receive event, topic: " + e.topic + " content: " + e.content; + var params=JSON.parse(e.content); + //print("update"+e.content); + //print(params.contractID); + var res=executeContract(params.contractID+"","getMetaByOffset",e.content); + print("update res "+res.result); + jo=JSON.parse(res.result); + //print("zzz"+JSON.stringify(jo)); + //var jo=res.result; + for(var i=0;i0) ret.push({ + "doId":obj.key,"count":obj.value/1} + ); + } + return { + "data":ret}; + } + + export function testIndex(arg) { + return Global.se.test(); + } + //全量同步 + //{"doId":"aibd.govdata.tj/RepoDemo","repoName":"repoDemo","contractID":"repoDemo"} + export function fullSync(arg) { + //getCount + arg=JSON.parse(arg); + count=executeContract(arg.contractID+"","getCount","").result; + doCountDB = RocksDBUtil.loadDB("./SearchEngineDoCount/","false"); + doCountDB.put(arg.doId,count); + doCountDB.close(); + //print(count); + //getMetaByOffset + var param={ + }; + param.count=count/1; + param.offset=0; + var ret=executeContract(arg.contractID+"","getMetaByOffset",JSON.stringify(param)); + var dolist = JSON.parse(ret.result); + for(var i=0;i