iod/page-assist
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update iod search

Browse Source
This commit is contained in:
CaiHQ 2025-03-24 13:21:49 +08:00
parent 3cbf4454da
commit 3fb66b4c36
3 changed files with 137 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
src/hooks/chat-helper/index.ts
View File

@ -2,7 +2,7 @@ import { saveHistory, saveMessage } from "@/db"
import { setLastUsedChatModel, setLastUsedChatSystemPrompt } from "@/services/model-settings"
import { generateTitle } from "@/services/title"
import { ChatHistory } from "@/store/option"
import { updateDialog } from "@/web/iod"
export const saveMessageOnError = async ({
e,
history,
@ -154,6 +154,7 @@ export const saveMessageOnSuccess = async ({
prompt_content?: string
reasoning_time_taken?: number
}) => {
var botMessage;
if (historyId) {
if (!isRegenerate) {
await saveMessage(
@ -170,7 +171,7 @@ export const saveMessageOnSuccess = async ({
reasoning_time_taken
)
}
await saveMessage(
botMessage = await saveMessage(
historyId,
selectedModel!,
"assistant",
@ -183,6 +184,7 @@ export const saveMessageOnSuccess = async ({
generationInfo,
reasoning_time_taken
)
updateDialog(historyId, botMessage)
await setLastUsedChatModel(historyId, selectedModel!)
if (prompt_id || prompt_content) {
await setLastUsedChatSystemPrompt(historyId, { prompt_content, prompt_id })
@ -203,7 +205,7 @@ export const saveMessageOnSuccess = async ({
generationInfo,
reasoning_time_taken
)
await saveMessage(
botMessage = await saveMessage(
newHistoryId.id,
selectedModel!,
"assistant",
@ -216,6 +218,7 @@ export const saveMessageOnSuccess = async ({
generationInfo,
reasoning_time_taken
)
updateDialog(newHistoryId.id, botMessage)
setHistoryId(newHistoryId.id)
await setLastUsedChatModel(newHistoryId.id, selectedModel!)
if (prompt_id || prompt_content) {

3
src/hooks/useMessageOption.tsx
View File

@ -314,6 +314,9 @@ export const useMessageOption = () => {
res = removeReasoning(res)
keywords = res
.replace(/^Keywords:/i, "")
.replace(/^关键词:/i, "")
.replace(/^：/i, "")
.replace(/^:/i, "")
.split(", ")
.map((k) => k.trim())
}

138
src/web/iod.ts
View File

@ -8,17 +8,26 @@ import {
totalSearchResults
} from "@/services/search"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
import type { Document } from "@langchain/core/documents"
import { Document } from "@langchain/core/documents"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
import type { IodRegistryEntry } from "~/types/iod"
const makeRegSearchParams = (count: number, keyword: string) => ({
import { PageAssitDatabase } from "@/db"
//doipUrl = tcp://reg01.public.internetofdata.cn:21037
export const iodConfig = {
"gatewayUrl": "tcp://127.0.0.1:21051",
"registry":"bdware/Registry",
"localRepository":"bdtest.local/myrepo1",
"doBrowser":"http://127.0.0.1:21030/SCIDE/SCManager"
}
export const makeRegSearchParams = (count: number, keyword: string) => ({
action: "executeContract",
contractID: "BDBrowser",
operation: "sendRequestDirectly",
arg: {
id: "670E241C9937B3537047C87053E3AA36",
doipUrl: "tcp://reg01.public.internetofdata.cn:21037",
id: iodConfig.registry,
doipUrl: iodConfig.gatewayUrl,
op: "Search",
attributes: {
offset: 0,
@ -46,6 +55,104 @@ const makeRegSearchParams = (count: number, keyword: string) => ({
}
})
export const makeDOIPParams = (doId:string, op:string, attributes:Object, requestBody: string) => ({
action: "executeContract",
contractID: "BDBrowser",
operation: "sendRequestDirectly",
arg: {
id: doId,
doipUrl: iodConfig.gatewayUrl,
op: op,
attributes: attributes,
body: requestBody
}
})
export const retrieveDoc = function(doId: string, traceId: string) : Promise<Document> {
console.log("retriveDoc:"+doId+" -> traceId:"+traceId)
const params = makeDOIPParams(doId,"Retrieve",{
"traceId": traceId,
bodyBase64Encoded: false
}, "");
const abortController = new AbortController()
setTimeout(() => abortController.abort(), 10000)
return fetch(iodConfig.doBrowser, {
method: "POST",
body: JSON.stringify(params),
signal: abortController.signal
}).then((response) => {
console.log("responseIn retrieveDoc:");
console.log(response);
return response.json()})
.then((res) => {
console.log("res:");
console.log(res.result.body);
return res.result.body
})
}
export const updateInLocalRepo = function(historyId: string, requestBody: Object) : Promise<string> {
const params = makeDOIPParams(iodConfig.localRepository,"Update",{
"aiDialogID": historyId,
bodyBase64Encoded: false
}, JSON.stringify(requestBody));
const abortController = new AbortController()
setTimeout(() => abortController.abort(), 10000)
return fetch(iodConfig.doBrowser, {
method: "POST",
body: JSON.stringify(params),
signal: abortController.signal
}).then((response) => response.json())
.then((res) => {
console.log("update dialog:"+JSON.stringify(res))
return res.body;
})
}
export const updateDialog = async function(histroyId : string, botMessage: any): Promise<string> {
//TODO @Nex confused by Message/MessageType in ./db/index.ts!
const db = new PageAssitDatabase()
const chatHistory = await db.getChatHistory(histroyId)
var userMessage = null;
for (var i=0;i<chatHistory.length;i++){
userMessage = chatHistory[i];
if (userMessage.role=='user') break;
}
let updateBody:any = {};
console.log(userMessage)
console.log(botMessage)
// !!!IMPORTANT!!! traceId = histroyId+"/"+userMessage.id;
// Update traceId in retrieveDoc!
updateBody.traceId = histroyId+"/"+userMessage.id;
updateBody.question = {
"id": histroyId+"/"+userMessage.id,
"content": userMessage.content,
"tokenCount": userMessage.content.length
}
updateBody.answer = {
"id": histroyId+"/"+botMessage.id,
"content": botMessage.content,
"tokenCount": botMessage.content.length
}
//TODO set a correct model ID
updateBody.model = {"id":"bdware.ollama/" + userMessage.name}
//TODO incorrect tokenCount calculated!!
updateBody.webSources = botMessage.webSources?.map((r) => ({
url: r.url,
tokenCount: r.url.length,
content: r.url
})) ?? [];
updateBody.IoDSources = botMessage.iodSources?.map((r) => ({
id: r.doId,
tokenCount: r.description.length,
content: r.description
})) ?? [];
console.log("updateBody:");
console.log(updateBody)
return updateInLocalRepo(histroyId,updateBody)
}
export async function localIodSearch(
query: string,
keywords: string[]
@ -57,10 +164,10 @@ export async function localIodSearch(
keywords.map(async (keyword) => {
const abortController = new AbortController()
setTimeout(() => abortController.abort(), 10000)
//http://47.93.156.31:21033/SCIDE/SCManager
const params = makeRegSearchParams(TOTAL_SEARCH_RESULTS, keyword)
return fetch("http://47.93.156.31:21033/SCIDE/SCManager", {
return fetch(iodConfig.doBrowser, {
method: "POST",
body: JSON.stringify(params),
signal: abortController.signal
@ -76,8 +183,7 @@ export async function localIodSearch(
console.log(body)
return []
}
const results: IodRegistryEntry[] =
body.data?.results?.filter((r) => r.url || r.pdf_url) || []
const results: IodRegistryEntry[] = body.data?.results || []
for (const r of results) {
r.url = r.url || r.pdf_url
}
@ -96,7 +202,7 @@ export async function localIodSearch(
for (const r of results) {
map.set(r.doId, r)
}
console.log("result from IoD:"+JSON.stringify(map)+"--> kw:"+JSON.stringify(keywords));
return Array.from(map.values())
}
@ -107,6 +213,7 @@ export const searchIod = async (query: string, keywords: string[]) => {
const searchResults = await localIodSearch(query, keywords)
const isSimpleMode = await getIsSimpleInternetSearch()
console.log("searchMode:"+isSimpleMode+" ->searchResult:\n"+JSON.stringify(searchResults))
if (isSimpleMode) {
await getOllamaURL()
@ -117,7 +224,18 @@ export const searchIod = async (query: string, keywords: string[]) => {
const resMap = new Map<string, IodRegistryEntry>()
for (const result of searchResults) {
const url = result.url
if (!url) continue
if (result.doId){
//TODO !!!!@Nex traceId should be the id of history/question!
const traceId = new Date().getTime() + "";
let docFromRetrieve = await retrieveDoc(result.doId, traceId);
console.log("doc from Retrieve:"+result.doId+" -->"+JSON.stringify(docFromRetrieve))
docs.push(docFromRetrieve)
continue;
}
if (!url) {
continue;
}
let htmlUrl = ""
if (ARXIV_URL_PATTERN.test(url)) {