update iod search

src/hooks/chat-helper/index.ts

@@ -2,7 +2,7 @@ import { saveHistory, saveMessage } from "@/db"
 import { setLastUsedChatModel, setLastUsedChatSystemPrompt } from "@/services/model-settings"
 import { generateTitle } from "@/services/title"
 import { ChatHistory } from "@/store/option"
-
+import { updateDialog } from "@/web/iod"
 export const saveMessageOnError = async ({
   e,
   history,
@@ -154,6 +154,7 @@ export const saveMessageOnSuccess = async ({
   prompt_content?: string
   reasoning_time_taken?: number
 }) => {
+  var  botMessage;
   if (historyId) {
     if (!isRegenerate) {
       await saveMessage(
@@ -170,7 +171,7 @@ export const saveMessageOnSuccess = async ({
         reasoning_time_taken
       )
     }
-    await saveMessage(
+    botMessage = await saveMessage(
       historyId,
       selectedModel!,
       "assistant",
@@ -183,6 +184,7 @@ export const saveMessageOnSuccess = async ({
       generationInfo,
       reasoning_time_taken
     )
+    updateDialog(historyId, botMessage)
     await setLastUsedChatModel(historyId, selectedModel!)
     if (prompt_id || prompt_content) {
       await setLastUsedChatSystemPrompt(historyId, { prompt_content, prompt_id })
@@ -203,7 +205,7 @@ export const saveMessageOnSuccess = async ({
       generationInfo,
       reasoning_time_taken
     )
-    await saveMessage(
+    botMessage = await saveMessage(
       newHistoryId.id,
       selectedModel!,
       "assistant",
@@ -216,6 +218,7 @@ export const saveMessageOnSuccess = async ({
       generationInfo,
       reasoning_time_taken
     )
+    updateDialog(newHistoryId.id, botMessage)
     setHistoryId(newHistoryId.id)
     await setLastUsedChatModel(newHistoryId.id, selectedModel!)
     if (prompt_id || prompt_content) {

src/hooks/useMessageOption.tsx

@@ -314,6 +314,9 @@ export const useMessageOption = () => {
         res = removeReasoning(res)
         keywords = res
           .replace(/^Keywords:/i, "")
+          .replace(/^关键词:/i, "")
+          .replace(/^：/i, "")
+          .replace(/^:/i, "")
           .split(", ")
           .map((k) => k.trim())
       }

src/web/iod.ts

@@ -8,17 +8,26 @@ import {
   totalSearchResults
 } from "@/services/search"
 import { getPageAssistTextSplitter } from "@/utils/text-splitter"
-import type { Document } from "@langchain/core/documents"
+import  { Document } from "@langchain/core/documents"
 import { MemoryVectorStore } from "langchain/vectorstores/memory"
 import type { IodRegistryEntry } from "~/types/iod"
 
-const makeRegSearchParams = (count: number, keyword: string) => ({
+
+import { PageAssitDatabase } from "@/db"
+//doipUrl = tcp://reg01.public.internetofdata.cn:21037
+export const iodConfig = {
+    "gatewayUrl": "tcp://127.0.0.1:21051",
+    "registry":"bdware/Registry",
+    "localRepository":"bdtest.local/myrepo1",
+    "doBrowser":"http://127.0.0.1:21030/SCIDE/SCManager"
+}
+export const makeRegSearchParams = (count: number, keyword: string) => ({
   action: "executeContract",
   contractID: "BDBrowser",
   operation: "sendRequestDirectly",
   arg: {
-    id: "670E241C9937B3537047C87053E3AA36",
+    id: iodConfig.registry,
-    doipUrl: "tcp://reg01.public.internetofdata.cn:21037",
+    doipUrl: iodConfig.gatewayUrl,
     op: "Search",
     attributes: {
       offset: 0,
@@ -46,6 +55,104 @@ const makeRegSearchParams = (count: number, keyword: string) => ({
   }
 })
 
+
+  export const makeDOIPParams = (doId:string, op:string, attributes:Object, requestBody: string) => ({
+    action: "executeContract",
+    contractID: "BDBrowser",
+    operation: "sendRequestDirectly",
+    arg: {
+      id: doId,
+      doipUrl: iodConfig.gatewayUrl,
+      op: op,
+      attributes: attributes,
+      body: requestBody
+    }
+  })  
+
+export const retrieveDoc = function(doId: string, traceId: string) : Promise<Document> {
+  console.log("retriveDoc:"+doId+" -> traceId:"+traceId)
+  const params = makeDOIPParams(doId,"Retrieve",{
+    "traceId": traceId,
+    bodyBase64Encoded: false
+  }, "");
+  const abortController = new AbortController()
+  setTimeout(() => abortController.abort(), 10000)
+  return fetch(iodConfig.doBrowser, {
+      method: "POST",
+      body: JSON.stringify(params),
+      signal: abortController.signal
+  }).then((response) => {
+    console.log("responseIn retrieveDoc:");
+    console.log(response);
+    return response.json()})
+  .then((res) => {
+       console.log("res:");
+       console.log(res.result.body);
+      return res.result.body
+  })
+}
+
+export const updateInLocalRepo = function(historyId: string, requestBody: Object) : Promise<string> {
+  const params = makeDOIPParams(iodConfig.localRepository,"Update",{
+    "aiDialogID": historyId,
+    bodyBase64Encoded: false
+  }, JSON.stringify(requestBody));
+  const abortController = new AbortController()
+  setTimeout(() => abortController.abort(), 10000)
+  return fetch(iodConfig.doBrowser, {
+    method: "POST",
+    body: JSON.stringify(params),
+    signal: abortController.signal
+  }).then((response) => response.json())
+  .then((res) => {
+    console.log("update dialog:"+JSON.stringify(res))
+    return res.body;
+  })
+}
+export const updateDialog = async function(histroyId : string, botMessage: any): Promise<string> {
+  //TODO @Nex confused by Message/MessageType in ./db/index.ts!
+  const db = new PageAssitDatabase()
+  const chatHistory = await db.getChatHistory(histroyId)
+  var userMessage = null;
+  for (var i=0;i<chatHistory.length;i++){
+    userMessage = chatHistory[i];
+    if (userMessage.role=='user') break;
+  }
+  let updateBody:any = {};
+  console.log(userMessage)
+  console.log(botMessage)
+  // !!!IMPORTANT!!!  traceId = histroyId+"/"+userMessage.id;
+  // Update traceId in retrieveDoc!
+  updateBody.traceId = histroyId+"/"+userMessage.id;
+  updateBody.question = {
+    "id": histroyId+"/"+userMessage.id,
+    "content": userMessage.content,
+    "tokenCount": userMessage.content.length
+  }
+  updateBody.answer = {
+    "id": histroyId+"/"+botMessage.id,
+    "content": botMessage.content,
+    "tokenCount": botMessage.content.length
+  }
+  //TODO set a correct model ID
+  updateBody.model = {"id":"bdware.ollama/" + userMessage.name}
+
+  //TODO incorrect tokenCount calculated!!
+  updateBody.webSources = botMessage.webSources?.map((r) => ({
+    url: r.url,
+    tokenCount: r.url.length,
+    content: r.url
+  })) ?? [];
+  updateBody.IoDSources = botMessage.iodSources?.map((r) => ({
+    id: r.doId,
+    tokenCount: r.description.length,
+    content: r.description
+  })) ?? [];
+  console.log("updateBody:");
+  console.log(updateBody)
+  return updateInLocalRepo(histroyId,updateBody)
+}
+
 export async function localIodSearch(
   query: string,
   keywords: string[]
@@ -57,10 +164,10 @@ export async function localIodSearch(
       keywords.map(async (keyword) => {
         const abortController = new AbortController()
         setTimeout(() => abortController.abort(), 10000)
+        //http://47.93.156.31:21033/SCIDE/SCManager
 
         const params = makeRegSearchParams(TOTAL_SEARCH_RESULTS, keyword)
-
+        return fetch(iodConfig.doBrowser, {
-        return fetch("http://47.93.156.31:21033/SCIDE/SCManager", {
           method: "POST",
           body: JSON.stringify(params),
           signal: abortController.signal
@@ -76,8 +183,7 @@ export async function localIodSearch(
               console.log(body)
               return []
             }
-            const results: IodRegistryEntry[] =
+            const results: IodRegistryEntry[] = body.data?.results || []
-              body.data?.results?.filter((r) => r.url || r.pdf_url) || []
             for (const r of results) {
               r.url = r.url || r.pdf_url
             }
@@ -96,7 +202,7 @@ export async function localIodSearch(
     for (const r of results) {
         map.set(r.doId, r)
     }
-
+  console.log("result from IoD:"+JSON.stringify(map)+"--> kw:"+JSON.stringify(keywords));
   return Array.from(map.values())
 }
 
@@ -107,6 +213,7 @@ export const searchIod = async (query: string, keywords: string[]) => {
   const searchResults = await localIodSearch(query, keywords)
 
   const isSimpleMode = await getIsSimpleInternetSearch()
+  console.log("searchMode:"+isSimpleMode+" ->searchResult:\n"+JSON.stringify(searchResults))
 
   if (isSimpleMode) {
     await getOllamaURL()
@@ -117,7 +224,18 @@ export const searchIod = async (query: string, keywords: string[]) => {
   const resMap = new Map<string, IodRegistryEntry>()
   for (const result of searchResults) {
     const url = result.url
-    if (!url) continue
+ 
+    if (result.doId){
+     //TODO !!!!@Nex traceId should be the id of history/question!
+     const traceId = new Date().getTime() + "";
+     let docFromRetrieve = await retrieveDoc(result.doId, traceId);
+     console.log("doc from Retrieve:"+result.doId+" -->"+JSON.stringify(docFromRetrieve))
+     docs.push(docFromRetrieve)
+     continue;
+    }
+    if (!url) {
+      continue;
+    }
 
     let htmlUrl = ""
     if (ARXIV_URL_PATTERN.test(url)) {