iod/page-assist
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: upgrade registry

Browse Source
This commit is contained in:
CaiHQ 2025-08-17 22:39:12 +08:00
parent 3fb66b4c36
commit ef0e315bdc
18 changed files with 3288 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -49,7 +49,9 @@ cd page-assist
2. Install the dependencies
```bash
export PATH="/Users/huaqiancai/.bun/bin/:$PATH"
bun install
```
3. Build the extension (by default it will build for Chrome)

2500
bun.lock Normal file
View File

File diff suppressed because one or more lines are too long

BIN
bun.lockb
View File

Binary file not shown.

3
package.json
View File

@ -1,6 +1,6 @@
{
"name": "pageassist",
"displayName": "Page Assist - A Web UI for Local AI Models",
"displayName": "IoD Bot - A Web UI for Local AI Models",
"version": "1.0.9",
"description": "Use your locally running AI models to assist you in your web browsing.",
"author": "n4ze3m",
@ -58,6 +58,7 @@
"rehype-mathjax": "4.0.3",
"remark-gfm": "3.0.1",
"remark-math": "5.1.1",
"segmentit":"^2.0.3",
"tesseract.js": "^5.1.1",
"turndown": "^7.1.3",
"unist-util-visit": "^5.0.0",

4
src/assets/locale/zh/common.json
View File

@ -1,5 +1,5 @@
{
"pageAssist": "Page Assist",
"pageAssist": "IoD Bot",
"selectAModel": "选择一个模型",
"save": "保存",
"saved": "已保存",
@ -38,7 +38,7 @@
}
},
"copyToClipboard": "复制到剪贴板",
"webSearch": "搜索万维网",
"webSearch": "搜索中...",
"iodSearch": "搜索数联网",
"regenerate": "重新生成",
"edit": "编辑",

2
src/assets/locale/zh/playground.json
View File

@ -19,7 +19,7 @@
}
},
"tooltip": {
"searchInternet": "搜索万维网",
"searchInternet": "搜索中...",
"searchIod": "搜索数联网",
"speechToText": "语音到文本",
"uploadImage": "上传图片",

0
src/cs.js Normal file
View File

507
src/entries/auto-deeplink.content.ts Normal file
View File

@ -0,0 +1,507 @@
export default defineContentScript({
matches: ['<all_urls>'],
main(ctx) {
setTimeout(getDeepScript,1000)
},
});
async function getDeepScript(){
console.log("getDeepScript!!")
const href = document.location.href;
let id = "unknown";
if (href.startsWith("http://39.105.188.3:3838/topic3/missing/?autoexecute="))
id = "id1";
if (href.startsWith("http://39.105.188.3:3838/topic3/PKUCausalEfficacy/?autoexecute="))
id = "id2";
if (href.startsWith("http://39.105.188.3:3838/topic3/ADR23/?autoexecute="))
id = "id3";
if (idToScript[id]!=undefined){
idToScript[id]();
}
//sendMessageToServiceWorker({});
}
async function sendMessageToServiceWorker(message) {
chrome.runtime.sendMessage({ type: 'retrieveDeepScript', doId:"10.1002/2014JA019817" }, response => {
console.log(response);
});
return;
}
const idToScript = {
"id1":(function() {
// 等待函数
function wait(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
// 主函数
async function automate() {
try {
console.log("开始执行第二部分脚本...");
// 等待页面加载
await wait(2000);
// 点击"缺失数据填补"链接
const dataFillLink = Array.from(document.querySelectorAll('a')).find(a =>
a.textContent.includes("缺失数据填补")
);
if (dataFillLink) {
dataFillLink.click();
console.log("已点击'缺失数据填补'链接");
} else {
console.error("未找到'缺失数据填补'链接");
}
await wait(1000);
// 点击"选择数字对象"选项卡
const numObjTab = Array.from(document.querySelectorAll('[role="tab"]')).find(tab =>
tab.textContent.includes("选择数字对象")
);
if (numObjTab) {
numObjTab.click();
console.log("已点击'选择数字对象'选项卡");
} else {
console.error("未找到'选择数字对象'选项卡");
}
await wait(1000);
// 点击"多重填补数字对象"
const multipleNumObjs = Array.from(document.querySelectorAll('div')).filter(div =>
div.textContent.trim() === "多重填补数字对象"
);
if (multipleNumObjs.length > 1) {
multipleNumObjs[1].click();
console.log("已点击'多重填补数字对象'");
} else if (multipleNumObjs.length > 0) {
multipleNumObjs[0].click();
console.log("已点击'多重填补数字对象'");
} else {
console.error("未找到'多重填补数字对象'");
}
await wait(1000);
// 选择"围术期处理后"
const periOption = Array.from(document.querySelectorAll('[role="option"]')).find(option =>
option.textContent.includes("围术期处理后")
);
if (periOption) {
periOption.click();
console.log("已选择'围术期处理后'");
} else {
console.error("未找到'围术期处理后'选项");
}
await wait(1000);
// 勾选特定复选框
const checkbox = document.querySelector('#col_pro');
if (checkbox) {
checkbox.checked = true;
checkbox.dispatchEvent(new Event('change', { bubbles: true }));
console.log("已勾选复选框");
} else {
console.error("未找到指定复选框");
}
await wait(1000);
// 点击"开始填补"按钮
const startFillBtn = Array.from(document.querySelectorAll('button')).find(btn =>
btn.textContent.includes("开始填补")
);
if (startFillBtn) {
startFillBtn.click();
console.log("已点击'开始填补'按钮");
} else {
console.error("未找到'开始填补'按钮");
}
await wait(2000);
// 点击"进入数据分析"按钮
const enterAnalysisBtn = Array.from(document.querySelectorAll('button')).find(btn =>
btn.textContent.includes("进入数据分析")
);
if (enterAnalysisBtn) {
enterAnalysisBtn.click();
console.log("已点击'进入数据分析'按钮");
} else {
console.error("未找到'进入数据分析'按钮");
}
await wait(2000);
// 点击"中心序号"
const centerNumDivs = Array.from(document.querySelectorAll('[id^="tab-"][id$="-3"] div')).filter(div =>
div.textContent.includes("中心序号")
);
if (centerNumDivs.length > 1) {
centerNumDivs[1].click();
console.log("已点击'中心序号'");
} else if (centerNumDivs.length > 0) {
centerNumDivs[0].click();
console.log("已点击'中心序号'");
} else {
console.error("未找到'中心序号'");
}
await wait(1000);
// 选择"术后血红蛋白HB"
const hbOption = Array.from(document.querySelectorAll('[role="option"]')).find(option =>
option.textContent.includes("术后血红蛋白HB")
);
if (hbOption) {
hbOption.click();
console.log("已选择'术后血红蛋白HB'");
} else {
console.error("未找到'术后血红蛋白HB'选项");
}
await wait(1000);
// 点击X变量选择框
const xSelector = document.querySelector('[id="X"] > .form-group > div > .selectize-control > .selectize-input');
if (xSelector) {
xSelector.click();
console.log("已点击X变量选择框");
} else {
console.error("未找到X变量选择框");
}
await wait(1000);
// 选择"性别"
const genderOption = Array.from(document.querySelectorAll('[role="option"]')).find(option =>
option.textContent.includes("性别")
);
if (genderOption) {
genderOption.click();
console.log("已选择'性别'");
} else {
console.error("未找到'性别'选项");
}
await wait(1000);
// 点击变量选择区域
const varSelectors = Array.from(document.querySelectorAll('div')).filter(div =>
div.textContent.includes("1. 根据预览数据选择变量 *选择处理变量（Z")
);
if (varSelectors.length > 3) {
varSelectors[3].click();
console.log("已点击变量选择区域");
} else if (varSelectors.length > 0) {
varSelectors[varSelectors.length - 1].click();
console.log("已点击变量选择区域");
} else {
console.error("未找到变量选择区域");
}
await wait(1000);
// 点击"开始分析"按钮
const startAnalysisBtn = Array.from(document.querySelectorAll('button')).find(btn =>
btn.textContent.includes("开始分析")
);
if (startAnalysisBtn) {
startAnalysisBtn.click();
console.log("已点击'开始分析'按钮");
} else {
console.error("未找到'开始分析'按钮");
}
console.log("第二部分脚本执行完成");
} catch (error) {
console.error("执行过程中出错:", error);
}
}
// 执行自动化操作
automate();
}),
"id2":(function() {
function wait(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
// 主函数
async function automate() {
try {
console.log("开始执行 Case02 第二部分脚本...");
// 等待页面加载
await wait(2000);
// 点击"准备数据"链接
const prepDataLink = Array.from(document.querySelectorAll('a')).find(a =>
a.textContent.includes("准备数据")
);
if (prepDataLink) {
prepDataLink.click();
console.log("已点击'准备数据'链接");
} else {
console.error("未找到'准备数据'链接");
}
await wait(1000);
// 点击"选择数字对象"选项卡
const numObjTab = Array.from(document.querySelectorAll('[role="tab"]')).find(tab =>
tab.textContent.includes("选择数字对象")
);
if (numObjTab) {
numObjTab.click();
console.log("已点击'选择数字对象'选项卡");
} else {
console.error("未找到'选择数字对象'选项卡");
}
await wait(1000);
// 点击"RS"
const rsDivs = Array.from(document.querySelectorAll('div')).filter(div =>
div.textContent.trim() === "RS"
);
if (rsDivs.length > 1) {
rsDivs[1].click();
console.log("已点击'RS'");
} else if (rsDivs.length > 0) {
rsDivs[0].click();
console.log("已点击'RS'");
} else {
console.error("未找到'RS'");
}
await wait(1000);
// 选择"TQ-B2303-III-01_merged"
const mergedOption = Array.from(document.querySelectorAll('[role="option"]')).find(option =>
option.textContent.includes("TQ-B2303-III-01_merged")
);
if (mergedOption) {
mergedOption.click();
console.log("已选择'TQ-B2303-III-01_merged'");
} else {
console.error("未找到'TQ-B2303-III-01_merged'选项");
}
await wait(1000);
// 点击特定元素
const noncomplianceElement = document.querySelector('[id="data\\.goto\\.noncompliance1"]');
if (noncomplianceElement) {
noncomplianceElement.click();
console.log("已点击特定元素");
} else {
console.error("未找到特定元素");
}
await wait(1000);
// 点击"AGEGR1"下拉框
const agegr1Combobox = Array.from(document.querySelectorAll('[role="combobox"]')).find(box =>
box.getAttribute('name') === "AGEGR1"
);
if (agegr1Combobox) {
agegr1Combobox.click();
console.log("已点击'AGEGR1'下拉框");
await wait(500);
// 选择特定选项
const agegr1Option = document.querySelector('#bs-select-9-29');
if (agegr1Option) {
agegr1Option.click();
console.log("已选择AGEGR1选项");
} else {
console.error("未找到AGEGR1选项");
}
} else {
console.error("未找到'AGEGR1'下拉框");
}
await wait(1000);
// 点击"SUBJID"下拉框
const subjidCombobox = Array.from(document.querySelectorAll('[role="combobox"]')).find(box =>
box.getAttribute('name') === "SUBJID"
);
if (subjidCombobox) {
subjidCombobox.click();
console.log("已点击'SUBJID'下拉框");
await wait(500);
// 选择特定选项
const subjidOption = document.querySelector('#bs-select-14-42');
if (subjidOption) {
subjidOption.click();
console.log("已选择SUBJID选项");
} else {
console.error("未找到SUBJID选项");
}
} else {
console.error("未找到'SUBJID'下拉框");
}
await wait(1000);
// 点击"Nothing selected"下拉框
const nothingSelectedCombobox = Array.from(document.querySelectorAll('[role="combobox"]')).find(box =>
box.getAttribute('name') === "Nothing selected"
);
if (nothingSelectedCombobox) {
nothingSelectedCombobox.click();
console.log("已点击'Nothing selected'下拉框");
await wait(500);
// 选择特定选项
const nothingSelectedOption = document.querySelector('#bs-select-16-6');
if (nothingSelectedOption) {
nothingSelectedOption.click();
console.log("已选择选项");
} else {
console.error("未找到选项");
}
} else {
console.error("未找到'Nothing selected'下拉框");
}
await wait(1000);
// 点击"IPI"下拉框
const ipiCombobox = Array.from(document.querySelectorAll('[role="combobox"]')).find(box =>
box.getAttribute('name') === "IPI"
);
if (ipiCombobox) {
ipiCombobox.click();
console.log("已点击'IPI'下拉框");
await wait(500);
// 取消勾选第四个复选框
const checkboxes = document.querySelectorAll('[role="checkbox"]');
if (checkboxes.length > 3) {
checkboxes[3].click(); // 取消勾选
console.log("已取消勾选复选框");
} else {
console.error("未找到足够的复选框");
}
} else {
console.error("未找到'IPI'下拉框");
}
await wait(1000);
// 点击"计算估计结果"按钮
const calculateBtn = Array.from(document.querySelectorAll('button')).find(btn =>
btn.textContent.includes("计算估计结果")
);
if (calculateBtn) {
calculateBtn.click();
console.log("已点击'计算估计结果'按钮");
} else {
console.error("未找到'计算估计结果'按钮");
}
console.log("Case02 第二部分脚本执行完成");
} catch (error) {
console.error("执行过程中出错:", error);
}
}
// 执行自动化操作
automate();
}),
"id3":(function(){
function wait(ms) {
return new Promise(resolve => setTimeout(resolve, ms));
}
// 主函数
async function automate() {
try {
console.log("开始执行 Case03 第二部分脚本...");
// 等待页面加载
await wait(2000);
// 点击"模型预测"链接
const modelPredictLink = Array.from(document.querySelectorAll('a')).find(a =>
a.textContent.includes("模型预测")
);
if (modelPredictLink) {
modelPredictLink.click();
console.log("已点击'模型预测'链接");
} else {
console.error("未找到'模型预测'链接");
}
await wait(1000);
// 点击"选择数字对象"选项卡
const numObjTab = Array.from(document.querySelectorAll('[role="tab"]')).find(tab =>
tab.textContent.includes("选择数字对象")
);
if (numObjTab) {
numObjTab.click();
console.log("已点击'选择数字对象'选项卡");
} else {
console.error("未找到'选择数字对象'选项卡");
}
await wait(1000);
// 点击"多重填补数字对象"
const multipleNumObjs = Array.from(document.querySelectorAll('div')).filter(div =>
div.textContent.trim() === "多重填补数字对象"
);
if (multipleNumObjs.length > 1) {
multipleNumObjs[1].click();
console.log("已点击'多重填补数字对象'");
} else if (multipleNumObjs.length > 0) {
multipleNumObjs[0].click();
console.log("已点击'多重填补数字对象'");
} else {
console.error("未找到'多重填补数字对象'");
}
await wait(1000);
// 选择"Clopidogrel"
const clopidogrelOption = Array.from(document.querySelectorAll('[role="option"]')).find(option =>
option.textContent.includes("Clopidogrel")
);
if (clopidogrelOption) {
clopidogrelOption.click();
console.log("已选择'Clopidogrel'");
} else {
console.error("未找到'Clopidogrel'选项");
}
await wait(1000);
// 勾选特定复选框
const checkbox = document.querySelector('#col_pro');
if (checkbox) {
checkbox.checked = true;
checkbox.dispatchEvent(new Event('change', { bubbles: true }));
console.log("已勾选复选框");
} else {
console.error("未找到指定复选框");
}
await wait(1000);
// 点击"查看上传的数据"按钮
const viewDataBtn = Array.from(document.querySelectorAll('button')).find(btn =>
btn.textContent.includes("查看上传的数据")
);
if (viewDataBtn) {
viewDataBtn.click();
console.log("已点击'查看上传的数据'按钮");
} else {
console.error("未找到'查看上传的数据'按钮");
}
await wait(2000);
// 点击"计算模型预测结果"按钮
const calculateBtn = Array.from(document.querySelectorAll('button')).find(btn =>
btn.textContent.includes("计算模型预测结果")
);
if (calculateBtn) {
calculateBtn.click();
console.log("已点击'计算模型预测结果'按钮");
} else {
console.error("未找到'计算模型预测结果'按钮");
}
console.log("Case03 第二部分脚本执行完成");
} catch (error) {
console.error("执行过程中出错:", error);
}
}
// 执行自动化操作
automate();
})
}

89
src/entries/background.ts
View File

@ -5,24 +5,30 @@ import { clearBadge, streamDownload } from "@/utils/pull-ollama"
export default defineBackground({
main() {
let isCopilotRunning: boolean = false
browser.runtime.onMessage.addListener(async (message) => {
if (message.type === "sidepanel") {
await browser.sidebarAction.open()
} else if (message.type === "pull_model") {
const ollamaURL = await getOllamaURL()
browser.runtime.onMessage.addListener(async (message,sender,sendResponse) => {
switch(message.type){
case "sidepanel":
await browser.sidebarAction.open()
break;
case "pull_model":
const ollamaURL = await getOllamaURL()
const isRunning = await isOllamaRunning()
if (!isRunning) {
setBadgeText({ text: "E" })
setBadgeBackgroundColor({ color: "#FF0000" })
setTitle({ title: "Ollama is not running" })
setTimeout(() => {
clearBadge()
}, 5000)
}
await streamDownload(ollamaURL, message.modelName)
const isRunning = await isOllamaRunning()
if (!isRunning) {
setBadgeText({ text: "E" })
setBadgeBackgroundColor({ color: "#FF0000" })
setTitle({ title: "Ollama is not running" })
setTimeout(() => {
clearBadge()
}, 5000)
}
await streamDownload(ollamaURL, message.modelName)
break;
case "retrieveDeepScript":
return retrieveDeepScript(message);
default:
break;
}
})
@ -180,3 +186,52 @@ export default defineBackground({
},
persistent: true
})
const iodConfig = {
"gatewayUrl": "tcp://127.0.0.1:21051",
"registry":"bdware/Registry",
"localRepository":"bdtest.local/myrepo1",
"doBrowser":"http://127.0.0.1:21030/SCIDE/SCManager"
}
const makeDOIPParams = (doId:string, op:string, attributes:Object, requestBody: string) => ({
action: "executeContract",
contractID: "BDBrowser",
operation: "sendRequestDirectly",
arg: {
id: doId,
doipUrl: iodConfig.gatewayUrl,
op: op,
attributes: attributes,
body: requestBody
}
})
const retrieveDeepScript = async function(message) {
console.log(message);
const doId = message.doId;
console.log("retriveDoc:"+doId)
const params = makeDOIPParams(doId,"Retrieve",{
bodyBase64Encoded: false
}, "");
const abortController = new AbortController()
setTimeout(() => abortController.abort(), 10000)
return await fetch(iodConfig.doBrowser, {
method: "POST",
body: JSON.stringify(params),
signal: abortController.signal
}).then((response) => {
console.log("responseIn retrieveDoc:");
console.log(response);
return response.json()})
.then((res) => {
console.log("res:");
console.log(res.result.body);
//TODO
return {
metadata:{traceId:res.result.header.attributes?.traceId},
pageContent:res.result.body
}
})
}

2
src/entries/options/index.html
View File

@ -1,7 +1,7 @@
<!doctype html>
<html>
<head>
<title>Page Assist - A Web UI for Local AI Models</title>
<title>IoD Bot - A Web UI for Local AI Models</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<meta name="manifest.type" content="browser_action" />
<meta name="manifest.browser_style" content="false" />

10
src/hooks/useMessageOption.tsx
View File

@ -21,6 +21,8 @@ import {
import { useNavigate } from "react-router-dom"
import { notification } from "antd"
import { getSystemPromptForWeb } from "~/web/web"
import { tokenizeInput } from "~/web/iod"
import { generateHistory } from "@/utils/generate-history"
import { useTranslation } from "react-i18next"
import {
@ -38,6 +40,7 @@ import { pageAssistModel } from "@/models"
import { getNoOfRetrievedDocs } from "@/services/app"
import { humanMessageFormatter } from "@/utils/human-message"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import {
isReasoningEnded,
isReasoningStarted,
@ -307,6 +310,9 @@ export const useMessageOption = () => {
// Currently only IoD search use keywords
if (iodSearch) {
// Extract keywords
console.log("query:"+query+" --> "+JSON.stringify(tokenizeInput(query)));
keywords = tokenizeInput(query)
/*
const questionPrompt = await geWebSearchKeywordsPrompt()
const promptForQuestion = questionPrompt.replaceAll("{query}", query)
const response = await ollama.invoke(promptForQuestion)
@ -317,8 +323,10 @@ export const useMessageOption = () => {
.replace(/^关键词:/i, "")
.replace(/^：/i, "")
.replace(/^:/i, "")
.split(", ")
.replaceAll(" ", "")
.split(",")
.map((k) => k.trim())
*/
}
const { prompt, webSources, iodSources, iodSearchResults: iodData, iodTokenCount } =

2
src/public/_locales/zh_CN/messages.json
View File

@ -1,6 +1,6 @@
{
"extName": {
"message": "Page Assist - 本地 AI 模型的 Web UI"
"message": "IoD Bot - 本地 AI 模型的 Web UI"
},
"extDescription": {
"message": "使用本地运行的 AI 模型来辅助您的网络浏览。"

43
src/services/ollama.ts
View File

@ -21,7 +21,37 @@ const DEFAULT_RAG_QUESTION_PROMPT =
const DEFAUTL_RAG_SYSTEM_PROMPT = `You are a helpful AI assistant. Use the following pieces of context to answer the question at the end. If you don't know the answer, just say you don't know. DO NOT try to make up an answer. If the question is not related to the context, politely respond that you are tuned to only answer questions that are related to the context. {context} Question: {question} Helpful answer:`
const DEFAULT_WEBSEARCH_PROMPT = `You are an AI model who is expert at searching the web and answering user's queries.
const DEFAULT_WEBSEARCH_PROMPT = `You are an AI assistant specialized in retrieving and analyzing academic papers from Neo4j graph database.
Generate a response that how can user achieve his request based on provided search results. The current date and time are {current_date_time}.
The \`iod-search-results\` block provides information retrieved from Internet of Data. Each search result has a format of:
\`<result doId="{doId}" name="{title}" authors="{authors}" dataType="{paper,dataset or algorithm}" year="{year}" url="{url}" id="{id}">{abstract}</result>\`
Please show the \`doId\` and \`name\` of the search result when you refer to search result in your response and chain of thought, in the following format, in English:
\`[IoD source [id] doId: {doId} "{name}"]({url})\`
Or in Chinese:
\`[数联网引用[id] doId: {doId} "{name}"]({url})\`
For example, in English:
\`[IoD source [1] doId: 10.48550/arXiv.1803.05591v2 "On the insufficiency of existing momentum schemes for Stochastic Optimization"](http://arxiv.org/pdf/1803.05591v2.pdf)\`
Or in Chinese:
\`[数联网引用[1] doId: 10.48550/arXiv.1803.05591v2 "On the insufficiency of existing momentum schemes for Stochastic Optimization"](http://arxiv.org/pdf/1803.05591v2.pdf)\`
Use this information to generate a meaningful response that includes:
0. 如果搜索结果看着和用户想做的事儿无关，那么直接忽略它，不需要在思维链和回答中体现。
1. 从搜索结果中，用户可以参考哪些论文来实现他的目标。
2. 从搜索结果中，用户可以使用哪些数据集(dataset)
3. 用户想干的这个事儿，如何结合这些数据来实现。
4. 请用中文回答这个问题。
<iod-search-results>
{iod_search_results}
</iod-search-results>
`
const DEFAULT_WEBSEARCH2_PROMPT = `You are an AI model who is expert at searching the web and answering user's queries.
Generate a response that is informative and relevant to the user's query based on provided search results. the current date and time are {current_date_time}.
@ -86,24 +116,25 @@ const DEFAULT_WEBSEARCH_KEYWORDS_PROMPT = `Extract the most important keywords f
The result format should be: keyword_1, keyword_2, ..., keyword_n
注意，以下关键词请不要输出："research", "研究", "data analysis", "data", "数据" 。
注意，英文单词的输出首字母应该小写，仅需输出Keywords部分，Query部分不用输出。以下是一些例子。
Example:
Query: What are the symptoms of a heart attack?
Keywords: symptoms, 症状, heart attack, 心臟病
你的输出: symptoms, 症状, heart attack, 心臟病
Query: 什么是物联网?
Keywords: Internet of Things, IoT, 物联网
你的输出: Internet of Things, IoT, 物联网
Query: 人工智能的发展趋势？
Keywords: Artificial Intelligence, AI, 人工智能, trend, 趋势
你的输出: Artificial Intelligence, AI, 人工智能, trend, 趋势
接下来，开始你的关键词提取吧。
Query: {query}
Keywords:
`
export const getOllamaURL = async () => {

2
src/types/iod.ts
View File

@ -6,4 +6,6 @@ export type IodRegistryEntry = {
description: string
content?: string
data_space?: string
data_type?:string
traceId?:string
}

8
src/types/segmentit.d.ts vendored Normal file
View File

@ -0,0 +1,8 @@
import { Segment, useDefault, cnPOSTag, enPOSTag } from 'segmentit';
declare module 'segmentit' {
export = Segment;
export = useDefault;
export = cnPOSTag;
export = enPOSTag;
}

188
src/web/iod.ts
View File

@ -3,6 +3,7 @@ import { PageAssistHtmlLoader } from "@/loader/html"
import { PageAssistPDFUrlLoader } from "@/loader/pdf-url"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import { defaultEmbeddingModelForRag, getOllamaURL } from "@/services/ollama"
import {
getIsSimpleInternetSearch,
totalSearchResults
@ -14,65 +15,84 @@ import type { IodRegistryEntry } from "~/types/iod"
import { PageAssitDatabase } from "@/db"
import exp from "constants"
import { Segment, useDefault, cnPOSTag, enPOSTag} from 'segmentit';
const segment = useDefault(new Segment());
export const tokenizeInput = function (input: string): string[] {
const words = segment.doSegment(input, { simple: false });
console.log(words.map(function(word){return {w:word.w, p:enPOSTag(word.p)}}) );
return words.filter(word =>( word.w.length > 1)).map(word=>word.w);
}
//doipUrl = tcp://reg01.public.internetofdata.cn:21037
export const iodConfig = {
"gatewayUrl": "tcp://127.0.0.1:21051",
"gatewayUrl": "tcp://127.0.0.1:21036",
"registry":"bdware/Registry",
"localRepository":"bdtest.local/myrepo1",
"doBrowser":"http://127.0.0.1:21030/SCIDE/SCManager"
}
export const makeRegSearchParams = (count: number, keyword: string) => ({
action: "executeContract",
contractID: "BDBrowser",
operation: "sendRequestDirectly",
arg: {
id: iodConfig.registry,
doipUrl: iodConfig.gatewayUrl,
op: "Search",
attributes: {
offset: 0,
count,
bodyBase64Encoded: false,
searchMode: [
{
key: "data_type",
type: "MUST",
value: "paper"
},
// {
// key: "title",
// type: "MUST",
// value: keyword,
// },
{
key: "description",
type: "MUST",
value: keyword
}
]
},
body: ""
function inGrepList(str: string){
return "什么|问题|需要|合适|设计|考虑|合作|精度|传感器|最新|研究|药物".indexOf(str)!=-1;
}
export const makeRegSearchParams = function(count: number, keyword: string| string[]){
const searchMode = [];
if (typeof keyword === 'string') {
// 如果 keyword 是字符串，则直接添加一个 searchMode 条目
searchMode.push({
key: "description",
type: "MUST",
value: keyword
});
} else if (Array.isArray(keyword)) {
// 如果 keyword 是数组，则为每个元素添加一个 searchMode 条目
keyword.forEach(str => {
if (!inGrepList(str))
searchMode.push({
key: "description",
type: "SHOULD",
value: str
});
});
}
})
export const makeDOIPParams = (doId:string, op:string, attributes:Object, requestBody: string) => ({
return {
action: "executeContract",
contractID: "BDBrowser",
operation: "sendRequestDirectly",
arg: {
id: doId,
id: iodConfig.registry,
//doipUrl:"tcp://127.0.0.1:21039",
doipUrl: iodConfig.gatewayUrl,
op: op,
attributes: attributes,
body: requestBody
op: "Search",
vars:{
timeout:15000
},
attributes: {
offset: 0,
count,
bodyBase64Encoded: false,
searchMode:searchMode
},
body: ""
}
})
}
}
export const retrieveDoc = function(doId: string, traceId: string) : Promise<Document> {
console.log("retriveDoc:"+doId+" -> traceId:"+traceId)
export const makeDOIPParams = (doId:string, op:string, attributes:Object, requestBody: string) => ({
action: "executeContract",
contractID: "BDBrowser",
operation: "sendRequestDirectly",
arg: {
id: doId,
doipUrl: iodConfig.gatewayUrl,
op: op,
attributes: attributes,
body: requestBody
}
})
export const retrieveDoc = function(doId: string) : Promise<Document> {
console.log("retriveDoc:"+doId)
const params = makeDOIPParams(doId,"Retrieve",{
"traceId": traceId,
bodyBase64Encoded: false
}, "");
const abortController = new AbortController()
@ -88,7 +108,11 @@ export const retrieveDoc = function(doId: string, traceId: string) : Promise<Doc
.then((res) => {
console.log("res:");
console.log(res.result.body);
return res.result.body
//TODO
return {
metadata:{traceId:res.result.header.attributes?.traceId},
pageContent:res.result.body
}
})
}
@ -119,8 +143,6 @@ export const updateDialog = async function(histroyId : string, botMessage: any):
if (userMessage.role=='user') break;
}
let updateBody:any = {};
console.log(userMessage)
console.log(botMessage)
// !!!IMPORTANT!!! traceId = histroyId+"/"+userMessage.id;
// Update traceId in retrieveDoc!
updateBody.traceId = histroyId+"/"+userMessage.id;
@ -141,12 +163,14 @@ export const updateDialog = async function(histroyId : string, botMessage: any):
updateBody.webSources = botMessage.webSources?.map((r) => ({
url: r.url,
tokenCount: r.url.length,
content: r.url
content: r.url,
traceId: r?.traceId
})) ?? [];
updateBody.IoDSources = botMessage.iodSources?.map((r) => ({
id: r.doId,
tokenCount: r.description.length,
content: r.description
tokenCount: (r.content || r.description)?calculateTokenCount((r.content || r.description)):0,
content: r.content || r.description,
traceId: r?.traceId
})) ?? [];
console.log("updateBody:");
console.log(updateBody)
@ -158,7 +182,47 @@ export async function localIodSearch(
keywords: string[]
): Promise<IodRegistryEntry[]> {
const TOTAL_SEARCH_RESULTS = await totalSearchResults()
const abortController = new AbortController();
setTimeout(() => abortController.abort(), 10000);
const params = makeRegSearchParams(TOTAL_SEARCH_RESULTS, keywords);
try {
const response = await fetch(iodConfig.doBrowser, {
method: "POST",
body: JSON.stringify(params),
signal: abortController.signal
});
const res = await response.json();
if (res.status !== "Success") {
console.log(res);
return [];
}
const body = JSON.parse(res.result.body);
if (body.code !== 0) {
console.log(body);
return [];
}
let results: IodRegistryEntry[] = body.data?.results || [];
for (const r of results) {
r.url = r.url || r.pdf_url;
}
for (const r of results) {
r.doId = r.doId || r.doid;
}
// results 根据 doId 去重
const map = new Map<string, IodRegistryEntry>();
for (const r of results) {
map.set(r.doId, r);
}
return Array.from(map.values());
} catch (e) {
console.log(e);
return [];
}
/*
const results = (
await Promise.all(
keywords.map(async (keyword) => {
@ -187,6 +251,9 @@ export async function localIodSearch(
for (const r of results) {
r.url = r.url || r.pdf_url
}
for (const r of results) {
r.doId = r.doId || r.doid
}
return results
})
.catch((e) => {
@ -202,8 +269,8 @@ export async function localIodSearch(
for (const r of results) {
map.set(r.doId, r)
}
console.log("result from IoD:"+JSON.stringify(map)+"--> kw:"+JSON.stringify(keywords));
return Array.from(map.values())
*/
}
const ARXIV_URL_PATTERN = /^https?:\/\/arxiv\.org\//
@ -213,8 +280,7 @@ export const searchIod = async (query: string, keywords: string[]) => {
const searchResults = await localIodSearch(query, keywords)
const isSimpleMode = await getIsSimpleInternetSearch()
console.log("searchMode:"+isSimpleMode+" ->searchResult:\n"+JSON.stringify(searchResults))
console.log("searchMode:"+isSimpleMode+"\n kw:"+JSON.stringify(keywords)+"\n"+" ->searchResult:\n"+JSON.stringify(searchResults))
if (isSimpleMode) {
await getOllamaURL()
return searchResults
@ -224,13 +290,13 @@ export const searchIod = async (query: string, keywords: string[]) => {
const resMap = new Map<string, IodRegistryEntry>()
for (const result of searchResults) {
const url = result.url
if (result.doId){
//TODO !!!!@Nex traceId should be the id of history/question!
const traceId = new Date().getTime() + "";
let docFromRetrieve = await retrieveDoc(result.doId, traceId);
let docFromRetrieve = await retrieveDoc(result.doId);
console.log("doc from Retrieve:"+result.doId+" -->"+JSON.stringify(docFromRetrieve))
docs.push(docFromRetrieve)
result.description = docFromRetrieve.pageContent;
result.traceId = docFromRetrieve.metadata?.traceId;
continue;
}
if (!url) {
@ -296,6 +362,9 @@ export const searchIod = async (query: string, keywords: string[]) => {
}
}
}
return searchResults
/*
const ollamaUrl = await getOllamaURL()
const embeddingModle = await defaultEmbeddingModelForRag()
@ -326,4 +395,11 @@ export const searchIod = async (query: string, keywords: string[]) => {
}).filter((r) => r)
return searchResult
*/
}
export const calculateTokenCount = function(str:string){
const byteArray = new TextEncoder().encode(str);
return byteArray.length;
}

16
src/web/web.ts
View File

@ -11,6 +11,7 @@ import { webBaiduSearch } from "./search-engines/baidu"
import { searchIod } from "./iod"
import type { WebSearchResult } from "~/types/web"
import type { IodRegistryEntry } from "~/types/iod"
import {calculateTokenCount} from "./iod"
const getHostName = (url: string) => {
try {
@ -100,18 +101,24 @@ export const getSystemPromptForWeb = async (
doId: res.doId,
name: res.name,
url: res.url,
data_type: res.data_type,
data_space: res.data_space,
content: res.content || res.description,
tokenCount: (res.content || res.description)?.length ?? 0,
tokenCount: (res.content || res.description)?calculateTokenCount((res.content || res.description)):0,
traceId:res?.traceId
}))
const iod_search_results = _iodSearchResults
.map(
(result, idx) =>
`<result doId="${result.doId}" name="${result.name}" source="${result.url}" id="${idx + 1}">${result.content}</result>`
(result, idx) =>{
const nameAttr = result.name ? ` name="${result.name}"` : '';
const sourceAttr = result.url ? ` source="${result.url}"` : '';
const dataTypeAttr = result.data_type ? ` dataType="${result.data_type}"` : '';
const dataSourceAttr = result.data_space ?` 数据来源="${result.data_space}"`:''
return `<result doId="${result.doId}"${nameAttr}${sourceAttr}${dataTypeAttr}${dataSourceAttr}" >${result.content}</result>`
}
)
.join("\n")
console.log("iod_search_result: " + iod_search_results)
const web_search_results = webSearchResults
.map(
@ -119,7 +126,6 @@ export const getSystemPromptForWeb = async (
`<result source="${result.url}" name="${result.name}" id="${idx + 1}">${result.content}</result>`
)
.join("\n")
console.log("web_search_result: " + web_search_results)
const current_date_time = new Date().toLocaleString()

2
wxt.config.ts
View File

@ -54,7 +54,7 @@ export default defineConfig({
version: "1.5.0",
name:
process.env.TARGET === "firefox"
? "Page Assist - A Web UI for Local AI Models"
? "IoD Bot - A Web UI for Local AI Models"
: "__MSG_extName__",
description: "__MSG_extDescription__",
default_locale: "en",