From 3fb66b4c364db1a3fd69c9634829af33584c6f0a Mon Sep 17 00:00:00 2001 From: CaiHQ Date: Mon, 24 Mar 2025 13:21:49 +0800 Subject: [PATCH] update iod search --- src/hooks/chat-helper/index.ts | 9 ++- src/hooks/useMessageOption.tsx | 3 + src/web/iod.ts | 138 ++++++++++++++++++++++++++++++--- 3 files changed, 137 insertions(+), 13 deletions(-) diff --git a/src/hooks/chat-helper/index.ts b/src/hooks/chat-helper/index.ts index 3e8be41..ae57dd6 100644 --- a/src/hooks/chat-helper/index.ts +++ b/src/hooks/chat-helper/index.ts @@ -2,7 +2,7 @@ import { saveHistory, saveMessage } from "@/db" import { setLastUsedChatModel, setLastUsedChatSystemPrompt } from "@/services/model-settings" import { generateTitle } from "@/services/title" import { ChatHistory } from "@/store/option" - +import { updateDialog } from "@/web/iod" export const saveMessageOnError = async ({ e, history, @@ -154,6 +154,7 @@ export const saveMessageOnSuccess = async ({ prompt_content?: string reasoning_time_taken?: number }) => { + var botMessage; if (historyId) { if (!isRegenerate) { await saveMessage( @@ -170,7 +171,7 @@ export const saveMessageOnSuccess = async ({ reasoning_time_taken ) } - await saveMessage( + botMessage = await saveMessage( historyId, selectedModel!, "assistant", @@ -183,6 +184,7 @@ export const saveMessageOnSuccess = async ({ generationInfo, reasoning_time_taken ) + updateDialog(historyId, botMessage) await setLastUsedChatModel(historyId, selectedModel!) if (prompt_id || prompt_content) { await setLastUsedChatSystemPrompt(historyId, { prompt_content, prompt_id }) @@ -203,7 +205,7 @@ export const saveMessageOnSuccess = async ({ generationInfo, reasoning_time_taken ) - await saveMessage( + botMessage = await saveMessage( newHistoryId.id, selectedModel!, "assistant", @@ -216,6 +218,7 @@ export const saveMessageOnSuccess = async ({ generationInfo, reasoning_time_taken ) + updateDialog(newHistoryId.id, botMessage) setHistoryId(newHistoryId.id) await setLastUsedChatModel(newHistoryId.id, selectedModel!) if (prompt_id || prompt_content) { diff --git a/src/hooks/useMessageOption.tsx b/src/hooks/useMessageOption.tsx index 9b850a0..bd03577 100644 --- a/src/hooks/useMessageOption.tsx +++ b/src/hooks/useMessageOption.tsx @@ -314,6 +314,9 @@ export const useMessageOption = () => { res = removeReasoning(res) keywords = res .replace(/^Keywords:/i, "") + .replace(/^关键词:/i, "") + .replace(/^：/i, "") + .replace(/^:/i, "") .split(", ") .map((k) => k.trim()) } diff --git a/src/web/iod.ts b/src/web/iod.ts index eabe9aa..69cc54b 100644 --- a/src/web/iod.ts +++ b/src/web/iod.ts @@ -8,17 +8,26 @@ import { totalSearchResults } from "@/services/search" import { getPageAssistTextSplitter } from "@/utils/text-splitter" -import type { Document } from "@langchain/core/documents" +import { Document } from "@langchain/core/documents" import { MemoryVectorStore } from "langchain/vectorstores/memory" import type { IodRegistryEntry } from "~/types/iod" -const makeRegSearchParams = (count: number, keyword: string) => ({ + +import { PageAssitDatabase } from "@/db" +//doipUrl = tcp://reg01.public.internetofdata.cn:21037 +export const iodConfig = { + "gatewayUrl": "tcp://127.0.0.1:21051", + "registry":"bdware/Registry", + "localRepository":"bdtest.local/myrepo1", + "doBrowser":"http://127.0.0.1:21030/SCIDE/SCManager" +} +export const makeRegSearchParams = (count: number, keyword: string) => ({ action: "executeContract", contractID: "BDBrowser", operation: "sendRequestDirectly", arg: { - id: "670E241C9937B3537047C87053E3AA36", - doipUrl: "tcp://reg01.public.internetofdata.cn:21037", + id: iodConfig.registry, + doipUrl: iodConfig.gatewayUrl, op: "Search", attributes: { offset: 0, @@ -46,6 +55,104 @@ const makeRegSearchParams = (count: number, keyword: string) => ({ } }) + + export const makeDOIPParams = (doId:string, op:string, attributes:Object, requestBody: string) => ({ + action: "executeContract", + contractID: "BDBrowser", + operation: "sendRequestDirectly", + arg: { + id: doId, + doipUrl: iodConfig.gatewayUrl, + op: op, + attributes: attributes, + body: requestBody + } + }) + +export const retrieveDoc = function(doId: string, traceId: string) : Promise { + console.log("retriveDoc:"+doId+" -> traceId:"+traceId) + const params = makeDOIPParams(doId,"Retrieve",{ + "traceId": traceId, + bodyBase64Encoded: false + }, ""); + const abortController = new AbortController() + setTimeout(() => abortController.abort(), 10000) + return fetch(iodConfig.doBrowser, { + method: "POST", + body: JSON.stringify(params), + signal: abortController.signal + }).then((response) => { + console.log("responseIn retrieveDoc:"); + console.log(response); + return response.json()}) + .then((res) => { + console.log("res:"); + console.log(res.result.body); + return res.result.body + }) +} + +export const updateInLocalRepo = function(historyId: string, requestBody: Object) : Promise { + const params = makeDOIPParams(iodConfig.localRepository,"Update",{ + "aiDialogID": historyId, + bodyBase64Encoded: false + }, JSON.stringify(requestBody)); + const abortController = new AbortController() + setTimeout(() => abortController.abort(), 10000) + return fetch(iodConfig.doBrowser, { + method: "POST", + body: JSON.stringify(params), + signal: abortController.signal + }).then((response) => response.json()) + .then((res) => { + console.log("update dialog:"+JSON.stringify(res)) + return res.body; + }) +} +export const updateDialog = async function(histroyId : string, botMessage: any): Promise { + //TODO @Nex confused by Message/MessageType in ./db/index.ts! + const db = new PageAssitDatabase() + const chatHistory = await db.getChatHistory(histroyId) + var userMessage = null; + for (var i=0;i ({ + url: r.url, + tokenCount: r.url.length, + content: r.url + })) ?? []; + updateBody.IoDSources = botMessage.iodSources?.map((r) => ({ + id: r.doId, + tokenCount: r.description.length, + content: r.description + })) ?? []; + console.log("updateBody:"); + console.log(updateBody) + return updateInLocalRepo(histroyId,updateBody) +} + export async function localIodSearch( query: string, keywords: string[] @@ -57,10 +164,10 @@ export async function localIodSearch( keywords.map(async (keyword) => { const abortController = new AbortController() setTimeout(() => abortController.abort(), 10000) + //http://47.93.156.31:21033/SCIDE/SCManager const params = makeRegSearchParams(TOTAL_SEARCH_RESULTS, keyword) - - return fetch("http://47.93.156.31:21033/SCIDE/SCManager", { + return fetch(iodConfig.doBrowser, { method: "POST", body: JSON.stringify(params), signal: abortController.signal @@ -76,8 +183,7 @@ export async function localIodSearch( console.log(body) return [] } - const results: IodRegistryEntry[] = - body.data?.results?.filter((r) => r.url || r.pdf_url) || [] + const results: IodRegistryEntry[] = body.data?.results || [] for (const r of results) { r.url = r.url || r.pdf_url } @@ -96,7 +202,7 @@ export async function localIodSearch( for (const r of results) { map.set(r.doId, r) } - + console.log("result from IoD:"+JSON.stringify(map)+"--> kw:"+JSON.stringify(keywords)); return Array.from(map.values()) } @@ -107,6 +213,7 @@ export const searchIod = async (query: string, keywords: string[]) => { const searchResults = await localIodSearch(query, keywords) const isSimpleMode = await getIsSimpleInternetSearch() + console.log("searchMode:"+isSimpleMode+" ->searchResult:\n"+JSON.stringify(searchResults)) if (isSimpleMode) { await getOllamaURL() @@ -117,7 +224,18 @@ export const searchIod = async (query: string, keywords: string[]) => { const resMap = new Map() for (const result of searchResults) { const url = result.url - if (!url) continue + + if (result.doId){ + //TODO !!!!@Nex traceId should be the id of history/question! + const traceId = new Date().getTime() + ""; + let docFromRetrieve = await retrieveDoc(result.doId, traceId); + console.log("doc from Retrieve:"+result.doId+" -->"+JSON.stringify(docFromRetrieve)) + docs.push(docFromRetrieve) + continue; + } + if (!url) { + continue; + } let htmlUrl = "" if (ARXIV_URL_PATTERN.test(url)) {