10 Commits

Author SHA1 Message Date
李芳
c50bb49b37 feat: metring and detail pages 2025-02-22 18:20:11 +08:00
李芳
970ffdac15 Merge branch 'feat/metering' of gitea.internetapi.cn:iod/page-assist into feat/page 2025-02-22 17:00:58 +08:00
zhaoweijie
da162be01d feat: add metering data 2025-02-22 16:57:19 +08:00
zhaoweijie
6d79d42925 feat: add metering data 2025-02-22 14:09:57 +08:00
Nex Zhu
f617a05483 feat: improve DEFAULT_WEBSEARCH_PROMPT for IoD and 3W citations 2025-02-17 19:03:38 +08:00
Nex Zhu
4c5d5cfe99 feat: IoD search process HTML/PDF content 2025-02-17 16:44:33 +08:00
CaiHQ
51188b1428 update search result in prompt 2025-02-15 13:02:24 +08:00
Nex Zhu
a56e46a98d feat: IoD search process HTML/PDF content 2025-02-14 23:24:27 +08:00
Nex Zhu
e8471f1802 feat: add IoD search 2025-02-14 18:17:12 +08:00
Muhammed Nazeem
691575e449 Update README.md 2025-02-12 17:06:23 +05:30
48 changed files with 1180 additions and 324 deletions

View File

@@ -10,6 +10,8 @@ Page Assist supports Chromium-based browsers like Chrome, Brave, and Edge, as we
[![Chrome Web Store](https://pub-35424b4473484be483c0afa08c69e7da.r2.dev/UV4C4ybeBTsZt43U4xis.png)](https://chrome.google.com/webstore/detail/page-assist/jfgfiigpkhlkbnfnbobbkinehhfdhndo)
[![Firefox Add-on](https://pub-35424b4473484be483c0afa08c69e7da.r2.dev/get-the-addon.png)](https://addons.mozilla.org/en-US/firefox/addon/page-assist/)
[![Edge Add-on](https://pub-35424b4473484be483c0afa08c69e7da.r2.dev/edge-addon.png)](https://microsoftedge.microsoft.com/addons/detail/page-assist-a-web-ui-fo/ogkogooadflifpmmidmhjedogicnhooa)
Checkout the Demo (v1.0.0):

View File

@@ -109,7 +109,7 @@
"translate": "ترجمة",
"custom": "مخصص"
},
"citations": "الاقتباسات",
"webCitations": "الاقتباسات",
"segmented": {
"ollama": "نماذج Ollama",
"custom": "نماذج مخصصة"

View File

@@ -106,7 +106,7 @@
"translate": "Oversæt",
"custom": "Brugerdefineret"
},
"citations": "Citater",
"webCitations": "Citater",
"downloadCode": "Download Kode",
"date": {
"pinned": "Fastgjort",

View File

@@ -106,7 +106,7 @@
"translate": "Übersetzen",
"custom": "Benutzerdefiniert"
},
"citations": "Zitate",
"webCitations": "Zitate",
"downloadCode": "Code herunterladen",
"date": {
"pinned": "Angepinnt",

View File

@@ -39,6 +39,7 @@
},
"copyToClipboard": "Copy to clipboard",
"webSearch": "Searching the web",
"iodSearch": "Searching the Internet of Data",
"regenerate": "Regenerate",
"edit": "Edit",
"delete": "Delete",
@@ -136,7 +137,8 @@
"translate": "Translate",
"custom": "Custom"
},
"citations": "Citations",
"webCitations": "Web Citations",
"iodCitations": "Internet of Data Citations",
"segmented": {
"ollama": "Ollama Models",
"custom": "Custom Models"

View File

@@ -1,29 +1,30 @@
{
"newChat": "New Chat",
"selectAPrompt": "Select a Prompt",
"githubRepository": "GitHub Repository",
"settings": "Settings",
"sidebarTitle": "Chat History",
"error": "Error",
"somethingWentWrong": "Something went wrong",
"validationSelectModel": "Please select a model to continue",
"deleteHistoryConfirmation": "Are you sure you want to delete this history?",
"editHistoryTitle": "Enter a new title",
"temporaryChat": "Temporary Chat",
"more": {
"copy": {
"group": "Copy",
"asText": "Copy as Text",
"asMarkdown": "Copy as Markdown",
"success": "Copied to clipboard!"
},
"download": {
"group": "Download",
"text": "Text File (.txt)",
"markdown": "Markdown (.md)",
"json": "JSON File (.json)",
"image": "Image (.png)"
},
"share": "Share"
}
}
"newChat": "New Chat",
"selectAPrompt": "Select a Prompt",
"githubRepository": "GitHub Repository",
"settings": "Settings",
"metering": "Metering",
"sidebarTitle": "Chat History",
"error": "Error",
"somethingWentWrong": "Something went wrong",
"validationSelectModel": "Please select a model to continue",
"deleteHistoryConfirmation": "Are you sure you want to delete this history?",
"editHistoryTitle": "Enter a new title",
"temporaryChat": "Temporary Chat",
"more": {
"copy": {
"group": "Copy",
"asText": "Copy as Text",
"asMarkdown": "Copy as Markdown",
"success": "Copied to clipboard!"
},
"download": {
"group": "Download",
"text": "Text File (.txt)",
"markdown": "Markdown (.md)",
"json": "JSON File (.json)",
"image": "Image (.png)"
},
"share": "Share"
}
}

View File

@@ -20,6 +20,7 @@
},
"tooltip": {
"searchInternet": "Search Internet",
"searchIod": "Search Internet of Data",
"speechToText": "Speech to Text",
"uploadImage": "Upload Image",
"stopStreaming": "Stop Streaming",

View File

@@ -105,7 +105,7 @@
"rephrase": "Reformular",
"translate": "Traducir"
},
"citations": "Citas",
"webCitations": "Citas",
"downloadCode": "Descargar Código",
"date": {
"pinned": "Fijado",

View File

@@ -99,7 +99,7 @@
},
"advanced": "تنظیمات بیشتر مدل"
},
"citations": "منابع",
"webCitations": "منابع",
"downloadCode": "دانلود کد",
"date": {
"pinned": "پین شده",

View File

@@ -105,7 +105,7 @@
"rephrase": "Reformuler",
"translate": "Traduire"
},
"citations": "Citations",
"webCitations": "Citations",
"downloadCode": "Télécharger le code",
"date": {
"pinned": "Épinglé",

View File

@@ -105,7 +105,7 @@
"rephrase": "Riformulare",
"translate": "Tradurre"
},
"citations": "Citazioni",
"webCitations": "Citazioni",
"downloadCode": "Scarica Codice",
"date": {
"pinned": "Fissato",

View File

@@ -105,8 +105,7 @@
"rephrase": "言い換え",
"translate": "翻訳"
},
"citations": "万维网引用",
"iodcitations":"数联网引用",
"webCitations": "引用",
"downloadCode": "コードをダウンロード",
"date": {
"pinned": "固定",

View File

@@ -105,7 +105,7 @@
"rephrase": "다르게 표현",
"translate": "번역"
},
"citations": "인용",
"webCitations": "인용",
"downloadCode": "코드 다운로드",
"date": {
"pinned": "고정됨",

View File

@@ -104,7 +104,7 @@
"rephrase": "പുനഃരൂപീകരിക്കുക",
"translate": "വിവർത്തനം ചെയ്യുക"
},
"citations": "ഉദ്ധരണികൾ",
"webCitations": "ഉദ്ധരണികൾ",
"downloadCode": "കോഡ് ഡൗൺലോഡ് ചെയ്യുക",
"date": {
"pinned": "പിൻ ചെയ്തത്",

View File

@@ -106,7 +106,7 @@
"translate": "Oversett",
"custom": "Egendefinert"
},
"citations": "Sitater",
"webCitations": "Sitater",
"downloadCode": "Last ned kode",
"date": {
"pinned": "Festet",

View File

@@ -105,7 +105,7 @@
"rephrase": "Reformular",
"translate": "Traduzir"
},
"citations": "Citações",
"webCitations": "Citações",
"downloadCode": "Baixar Código",
"date": {
"pinned": "Fixado",

View File

@@ -105,7 +105,7 @@
"rephrase": "Перефразировать",
"translate": "Перевести"
},
"citations": "Цитаты",
"webCitations": "Цитаты",
"downloadCode": "Скачать код",
"date": {
"pinned": "Закреплено",

View File

@@ -106,7 +106,7 @@
"translate": "Översätt",
"custom": "Custom"
},
"citations": "Citat",
"webCitations": "Citat",
"segmented": {
"ollama": "Ollama-modeller",
"custom": "Custom modeller"

View File

@@ -106,7 +106,7 @@
"translate": "Перекласти",
"custom": "Власне"
},
"citations": "Цитати",
"webCitations": "Цитати",
"segmented": {
"ollama": "Моделі Ollama",
"custom": "Власні моделі"

View File

@@ -38,7 +38,8 @@
}
},
"copyToClipboard": "复制到剪贴板",
"webSearch": "搜索网",
"webSearch": "搜索万维网",
"iodSearch": "搜索数联网",
"regenerate": "重新生成",
"edit": "编辑",
"delete": "删除",
@@ -105,8 +106,8 @@
"rephrase": "重述",
"translate": "翻译"
},
"citations": "万维网引用",
"iodcitations":"数联网引用",
"webCitations": "万维网引用",
"iodCitations": "数联网引用",
"downloadCode": "下载代码",
"date": {
"pinned": "已置顶",

View File

@@ -1,28 +1,29 @@
{
"newChat": "新聊天",
"selectAPrompt": "本地回答",
"githubRepository": "GitHub 仓库",
"settings": "设置",
"sidebarTitle": "聊天历史",
"error": "错误",
"somethingWentWrong": "出现了错误",
"validationSelectModel": "请选择一个模型以继续",
"deleteHistoryConfirmation": "你确定要删除这个历史记录吗?",
"editHistoryTitle": "输入一个新的标题",
"temporaryChat": "临时聊天",
"more": {
"copy": {
"group": "复制",
"asText": "复制为文本",
"asMarkdown": "复制为 Markdown",
"success": "复制到剪贴板!"
},
"download": {
"group": "下载",
"text": "文本文件 (.txt)",
"markdown": "Markdown 文件 (.md)",
"json": "JSON 文件 (.json)"
},
"share": "分享"
}
}
"newChat": "新聊天",
"selectAPrompt": "选择一个提示词",
"githubRepository": "GitHub 仓库",
"settings": "设置",
"metering": "计量",
"sidebarTitle": "聊天历史",
"error": "错误",
"somethingWentWrong": "出现了错误",
"validationSelectModel": "请选择一个模型以继续",
"deleteHistoryConfirmation": "你确定要删除这个历史记录吗?",
"editHistoryTitle": "输入一个新的标题",
"temporaryChat": "临时聊天",
"more": {
"copy": {
"group": "复制",
"asText": "复制为文本",
"asMarkdown": "复制为 Markdown",
"success": "已复制到剪贴板!"
},
"download": {
"group": "下载",
"text": "文本文件 (.txt)",
"markdown": "Markdown 文件 (.md)",
"json": "JSON 文件 (.json)"
},
"share": "分享"
}
}

View File

@@ -19,7 +19,8 @@
}
},
"tooltip": {
"searchInternet": "搜索互联网",
"searchInternet": "搜索万维网",
"searchIod": "搜索数联网",
"speechToText": "语音到文本",
"uploadImage": "上传图片",
"stopStreaming": "停止流媒体",

View File

@@ -18,7 +18,7 @@ import { useTTS } from "@/hooks/useTTS"
import { tagColors } from "@/utils/color"
import { removeModelSuffix } from "@/db/models"
import { GenerationInfo } from "./GenerationInfo"
import { parseReasoning, } from "@/libs/reasoning"
import { parseReasoning } from "@/libs/reasoning"
import { humanizeMilliseconds } from "@/utils/humanize-milliseconds"
type Props = {
message: string
@@ -36,8 +36,8 @@ type Props = {
isProcessing: boolean
webSearch?: {}
isSearchingInternet?: boolean
sources?: any[]
iodSources?:any[]
webSources?: any[]
iodSources?: any[]
hideEditAndRegenerate?: boolean
onSourceClick?: (source: any) => void
isTTSEnabled?: boolean
@@ -49,6 +49,7 @@ type Props = {
export const PlaygroundMessage = (props: Props) => {
const [isBtnPressed, setIsBtnPressed] = React.useState(false)
const [editMode, setEditMode] = React.useState(false)
const { t } = useTranslation("common")
const { cancel, isSpeaking, speak } = useTTS()
return (
@@ -166,6 +167,34 @@ export const PlaygroundMessage = (props: Props) => {
</div>
)}
{props.isBot && props?.webSources && props?.webSources.length > 0 && (
<Collapse
className="mt-6"
ghost
items={[
{
key: "1",
label: (
<div className="italic text-gray-500 dark:text-gray-400">
{t("webCitations")}
</div>
),
children: (
<div className="mb-3 flex flex-wrap gap-2">
{props?.webSources?.map((source, index) => (
<MessageSource
onSourceClick={props.onSourceClick}
key={index}
index={index}
source={source}
/>
))}
</div>
)
}
]}
/>
)}
{props.isBot && props?.iodSources && props?.iodSources.length > 0 && (
<Collapse
className="mt-6"
@@ -175,45 +204,17 @@ export const PlaygroundMessage = (props: Props) => {
key: "1",
label: (
<div className="italic text-gray-500 dark:text-gray-400">
{t("iodcitations")}
{t("iodCitations")}
</div>
),
children: (
<div className="block">
<div className="mb-3 flex flex-wrap gap-2">
{props?.iodSources?.map((source, index) => (
<MessageSource
onSourceClick={props.onSourceClick}
key={index}
index={index}
source={source}
index = {index}
/>
))}
</div>
)
}
]}
/>
)}
{props.isBot && props?.sources && props?.sources.length > 0 && (
<Collapse
className="mt-6"
ghost
items={[
{
key: "1",
label: (
<div className="italic text-gray-500 dark:text-gray-400">
{t("citations")}
</div>
),
children: (
<div className="block">
{props?.sources?.map((source, index) => (
<MessageSource
onSourceClick={props.onSourceClick}
key={index}
source={source}
index = {index}
/>
))}
</div>

View File

@@ -1,6 +1,9 @@
import { useState } from "react"
import type React from "react"
import { KnowledgeIcon } from "@/components/Option/Knowledge/KnowledgeIcon"
type Props = {
index: number
source: {
name?: string
url?: string
@@ -8,42 +11,72 @@ type Props = {
type?: string
pageContent?: string
content?: string
doId?: string
description?: string
}
key: number
onSourceClick?: (source: any) => void
index: number
}
export const MessageSource: React.FC<Props> = ({ source, key, onSourceClick, index}) => {
export const MessageSource: React.FC<Props> = ({
index,
source,
onSourceClick
}) => {
// Add state for tracking and content visibility
const [showContent, setShowContent] = useState(false)
if (source?.mode === "rag" || source?.mode === "chat") {
return (
<div className="block items-center gap-1 text-xs text-gray-800 dark:text-gray-100 mb-1">
<span className="text-xs font-medium">[{index + 1}]</span> {/* 显示序号 */}
<button
onClick={() => {
onSourceClick && onSourceClick(source)
}}
className="inline-flex gap-2 cursor-pointer transition-shadow duration-300 ease-in-out hover:shadow-lg items-center rounded-md bg-gray-100 p-1 text-xs text-gray-800 border border-gray-300 dark:bg-gray-800 dark:border-gray-700 dark:text-gray-100 opacity-80 hover:opacity-100">
<KnowledgeIcon type={source.type} className="h-3 w-3" />
<span className="text-xs">{source.name}</span>
<a
href={source?.url}
target="_blank"
className="text-xs text-blue-500 hover:underline"
onClick={(e) => {
e.preventDefault(); // 阻止默认的链接行为
onSourceClick && onSourceClick(source); // 调用自定义点击事件
}}
>
{source.url}
</a>
</div>
);
</button>
)
}
const onContextMenu = (e: React.MouseEvent) => {
e.preventDefault()
e.stopPropagation
setShowContent(true)
}
return (
<div className="block items-center gap-1 text-xs text-gray-800 dark:text-gray-100 mb-1">
<span className="text-xs font-medium">[{index + 1}]</span> {/* 显示序号 */}
<span className="text-xs font-medium"></span>{" "}
<a
href={source?.url}
target="_blank"
className="text-xs text-blue-500 hover:underline"
>
{source.name}
onContextMenu={onContextMenu}
className="inline-block cursor-pointer transition-shadow duration-300 ease-in-out hover:shadow-lg items-center rounded-md bg-gray-100 p-1 text-xs text-gray-800 border border-gray-300 dark:bg-gray-800 dark:border-gray-700 dark:text-gray-100 opacity-80 hover:opacity-100">
{source.doId ? (
<>
<span className="text-xs">
[{index + 1}] doid: {source.doId}
</span>
<br />
<span className="text-xs">{source.name}</span>
{showContent && (
<div className="mt-2 p-2 border-t border-gray-200 dark:border-gray-700">
{source.content || source.pageContent || source.description}
</div>
)}
</>
) : (
<>
<span className="text-xs">
[{index + 1}] {source.name}
</span>
{showContent && (
<div className="mt-2 p-2 border-t border-gray-200 dark:border-gray-700">
{source.content || source.pageContent}
</div>
)}
</>
)}
</a>
</div>
)

View File

@@ -5,6 +5,7 @@ import {
ChevronRight,
CogIcon,
ComputerIcon,
Slice,
GithubIcon,
PanelLeftIcon,
ZapIcon
@@ -240,7 +241,14 @@ export const Header: React.FC<Props> = ({
<CogIcon className="w-6 h-6" />
</NavLink>
</Tooltip>
</div>
<Tooltip title={t("metering")}>
<NavLink
to="/metering"
className="!text-gray-500 dark:text-gray-400 hover:text-gray-600 dark:hover:text-gray-300 transition-colors">
<Slice className="w-6 h-6" />
</NavLink>
</Tooltip>
</div>
</div>
</div>
</div>

View File

@@ -0,0 +1,192 @@
import {
Card,
List,
Table,
Tag,
Space,
TableProps,
Divider,
Typography,
Tooltip
} from "antd"
import { NavLink } from "react-router-dom"
const data = [
{
key: "输出token数",
value: 2
},
{
key: "输入token数",
value: 2
},
{
key: "模型",
value: "xxx"
}
]
const inputTokenData = [
{
key: "关键词提示",
value: "xxx"
},
{
key: "问题",
value: "xxx"
},
{
key: "数联网引用数据",
value: "xxx"
},
{
key: "提供方",
value: "xxx"
},
{
key: "token数量",
value: 2
},
{
key: "内容",
value: "xxx"
}
]
const outputTokenData = [
{
key: "类型",
value: "xxx"
},
{
key: "来源",
value: "xxx"
},
{
key: "token数量",
value: 2
},
{
key: "内容",
value: "xxx"
}
]
interface DataType {
key: string
name: string
age: number
address: string
tags: number
content: string
}
const columns: TableProps<DataType>["columns"] = [
{
title: "序号",
dataIndex: "key",
key: "name",
render: (text) => <a>{text}</a>
},
{
title: "标识",
dataIndex: "age",
key: "age"
},
{
title: "提供方",
dataIndex: "address",
key: "address"
},
{
title: "token数量",
key: "tags",
dataIndex: "tags"
},
{
title: "内容",
key: "content",
dataIndex: "content"
}
]
const data1: DataType[] = [
{
key: "1",
name: "John Brown",
age: 32,
address: "New York No. 1 Lake Park",
tags: 2,
content: "内容"
},
{
key: "2",
name: "Jim Green",
age: 42,
address: "London No. 1 Lake Park",
tags: 3,
content: "内容"
},
{
key: "3",
name: "Joe Black",
age: 32,
address: "Sydney No. 1 Lake Park",
tags: 3,
content: "内容"
}
]
export const ListDetail = () => {
return (
<div className="p-[1rem] pt-[4rem]">
<List
grid={{ gutter: 16, column: 3 }}
dataSource={data}
renderItem={(item) => (
<List.Item>
<Card title={item.key}>{item.value}</Card>
</List.Item>
)}
style={{ marginBottom: "2rem" }}
/>
<div>
<Divider orientation="left">token详情</Divider>
<List
bordered
dataSource={inputTokenData}
renderItem={(item) => (
<List.Item style={{ justifyContent: "flex-start" }}>
<Typography.Text mark className="mr-1">
{item.key}
</Typography.Text>
<Tooltip placement="topLeft" title={item.value}>
{item.value}
</Tooltip>
</List.Item>
)}
style={{ marginBottom: "1rem" }}
/>
<Table<DataType> columns={columns} dataSource={data1} />
</div>
<div>
<Divider orientation="left">token详情</Divider>
<List
bordered
dataSource={outputTokenData}
renderItem={(item) => (
<List.Item style={{ justifyContent: "flex-start" }}>
<Typography.Text mark className="mr-1">
{item.key}
</Typography.Text>
<Tooltip placement="topLeft" title={item.value}>
{item.value}
</Tooltip>
</List.Item>
)}
/>
</div>
</div>
)
}

View File

@@ -0,0 +1,139 @@
import React from "react"
import { ChatMessage, useStoreMessageOption } from "@/store/option"
import { Card, List, Table, Tag, Space, TableProps, Tooltip } from "antd"
import { NavLink } from "react-router-dom"
import { formatDate } from "@/utils/date"
const data = [
{
key: "对话数量",
value: 2
},
{
key: "输出token数",
value: 2
},
{
key: "输入token数",
value: 2
}
]
const columns: TableProps<ChatMessage>["columns"] = [
{
title: "id",
dataIndex: "id",
key: "id",
width: "13%"
},
{
title: "问题",
dataIndex: "queryContent",
key: "queryContent"
},
{
title: "提示词全文",
dataIndex: "prompt",
key: "prompt",
ellipsis: {
showTitle: false
},
render: (prompt) => (
<Tooltip placement="topLeft" title={prompt}>
{prompt}
</Tooltip>
),
width: "10%"
},
{
title: "思维链",
key: "thinkingChain",
dataIndex: "thinkingChain",
width: "10%"
},
{
title: "回答",
dataIndex: "responseContent",
key: "responseContent",
ellipsis: {
showTitle: false
},
render: (responseContent) => (
<Tooltip placement="topLeft" title={responseContent}>
{responseContent}
</Tooltip>
),
width: "10%"
},
{
title: "关联数据个数",
dataIndex: "relatedDataCount",
key: "relatedDataCount"
},
{
title: "数联网token",
dataIndex: "iodOutputToken",
key: "iodOutputToken",
render: (iodOutputToken) => <div>{iodOutputToken?.length}</div>
},
{
title: "大模型token",
key: "largeModelToken",
dataIndex: "largeModelToken",
render: (_, record) => {
return (
<div>
{record.iodInputToken?.length + record.iodOutputToken?.length}
</div>
)
}
},
{
title: "日期",
dataIndex: "date",
key: "date",
render: (date) => {
return <div>{formatDate(date)}</div>
}
},
{
title: "耗时",
key: "timeTaken",
dataIndex: "timeTaken"
},
{
title: "操作",
key: "action",
render: (_, record) => (
<Space size="middle">
{/* <a>Invite {record.name}</a> */}
<NavLink to={`/metering/list/${record.id}`}>
<a></a>
</NavLink>
</Space>
)
}
]
export const MeteringDetail = () => {
const { chatMessages } = useStoreMessageOption()
console.log(chatMessages, "opppp")
return (
<div className="pt-[4rem]">
<List
grid={{ gutter: 16, column: 3 }}
dataSource={data}
renderItem={(item) => (
<List.Item>
<Card title={item.key}>{item.value}</Card>
</List.Item>
)}
/>
<Table<ChatMessage> columns={columns} dataSource={chatMessages} />
</div>
)
}

View File

@@ -36,7 +36,7 @@ export const PlaygroundChat = () => {
onRengerate={regenerateLastMessage}
isProcessing={streaming}
isSearchingInternet={isSearchingInternet}
sources={message.sources}
webSources={message.webSources}
iodSources={message.iodSources}
onEditFormSubmit={(value, isSend) => {
editMessage(index, value, !message.isBot, isSend)

View File

@@ -13,7 +13,7 @@ import { getVariable } from "@/utils/select-variable"
import { useTranslation } from "react-i18next"
import { KnowledgeSelect } from "../Knowledge/KnowledgeSelect"
import { useSpeechRecognition } from "@/hooks/useSpeechRecognition"
import { PiGlobe } from "react-icons/pi"
import { PiGlobe, PiNetwork } from "react-icons/pi"
import { handleChatInputKeyDown } from "@/utils/key-down"
import { getIsSimpleInternetSearch } from "@/services/search"
@@ -34,6 +34,8 @@ export const PlaygroundForm = ({ dropedFile }: Props) => {
streaming: isSending,
webSearch,
setWebSearch,
iodSearch,
setIodSearch,
selectedQuickPrompt,
textareaRef,
setSelectedQuickPrompt,
@@ -126,7 +128,6 @@ export const PlaygroundForm = ({ dropedFile }: Props) => {
}
}, [transcript])
/*
React.useEffect(() => {
if (selectedQuickPrompt) {
const word = getVariable(selectedQuickPrompt)
@@ -143,7 +144,7 @@ export const PlaygroundForm = ({ dropedFile }: Props) => {
}
}
}, [selectedQuickPrompt])
*/
const queryClient = useQueryClient()
const { mutateAsync: sendMessage } = useMutation({
@@ -300,6 +301,38 @@ export const PlaygroundForm = ({ dropedFile }: Props) => {
{...form.getInputProps("message")}
/>
<div className="mt-2 flex justify-between items-center">
<div className="flex">
{!selectedKnowledge && (
<div>
<Tooltip title={t("tooltip.searchInternet")}>
<div className="inline-flex items-center gap-2">
<PiGlobe
className={`h-5 w-5 dark:text-gray-300 `}
/>
<Switch
value={webSearch}
onChange={(e) => setWebSearch(e)}
checkedChildren={t("form.webSearch.on")}
unCheckedChildren={t("form.webSearch.off")}
/>
</div>
</Tooltip>
<Tooltip title={t("tooltip.searchIod")} className="ml-3">
<div className="inline-flex items-center gap-2">
<PiNetwork
className={`h-5 w-5 dark:text-gray-300 `}
/>
<Switch
value={iodSearch}
onChange={(e) => setIodSearch(e)}
checkedChildren={t("form.webSearch.on")}
unCheckedChildren={t("form.webSearch.off")}
/>
</div>
</Tooltip>
</div>
)}
</div>
<div className="flex !justify-end gap-3">
{!selectedKnowledge && (
<Tooltip title={t("tooltip.uploadImage")}>

View File

@@ -1,13 +0,0 @@
<Tooltip title={t("tooltip.searchInternet")}>
<div className="inline-flex items-center gap-2">
<PiGlobe
className={`h-5 w-5 dark:text-gray-300 `}
/>
<Switch
value={webSearch}
onChange={(e) => setWebSearch(e)}
checkedChildren={t("form.webSearch.on")}
unCheckedChildren={t("form.webSearch.off")}
/>
</div>
</Tooltip>

View File

@@ -32,7 +32,6 @@ type Props = {
setHistoryId: (historyId: string) => void
setSelectedModel: (model: string) => void
setSelectedSystemPrompt: (prompt: string) => void
setSelectedQuickPrompt: (prompt: string | undefined) => void
setSystemPrompt: (prompt: string) => void
clearChat: () => void
temporaryChat: boolean
@@ -47,7 +46,6 @@ export const Sidebar = ({
setHistoryId,
setSelectedModel,
setSelectedSystemPrompt,
setSelectedQuickPrompt,
clearChat,
historyId,
setSystemPrompt,

View File

@@ -39,7 +39,7 @@ export const SidePanelBody = () => {
message_type={message.messageType}
isProcessing={streaming}
isSearchingInternet={isSearchingInternet}
sources={message.sources}
webSources={message.webSources}
iodSources={message.iodSources}
onEditFormSubmit={(value) => {
editMessage(index, value, !message.isBot)

View File

@@ -29,8 +29,8 @@ type Message = {
role: string
content: string
images?: string[]
sources?: string[]
iodSources?:string[]
webSources?: string[]
iodSources?: string[]
search?: WebSearch
createdAt: number
reasoning_time_taken?: number
@@ -239,7 +239,7 @@ export const generateID = () => {
export const saveHistory = async (
title: string,
is_rag?: boolean,
message_source?: "copilot" | "web-ui",
message_source?: "copilot" | "web-ui"
) => {
const id = generateID()
const createdAt = Date.now()
@@ -255,8 +255,8 @@ export const saveMessage = async (
role: string,
content: string,
images: string[],
source?: any[],
iodSource?:any[],
webSources?: any[],
iodSources?: any[],
time?: number,
message_type?: string,
generationInfo?: any,
@@ -275,8 +275,8 @@ export const saveMessage = async (
content,
images,
createdAt,
sources: source,
iodSources:iodSource,
webSources,
iodSources,
messageType: message_type,
generationInfo: generationInfo,
reasoning_time_taken
@@ -306,7 +306,7 @@ export const formatToMessage = (messages: MessageHistory): MessageType[] => {
isBot: message.role === "assistant",
message: message.content,
name: message.name,
sources: message?.sources || [],
webSources: message?.webSources || [],
iodSources: message?.iodSources || [],
images: message.images || [],
generationInfo: message?.generationInfo,

View File

@@ -130,8 +130,8 @@ export const saveMessageOnSuccess = async ({
message,
image,
fullText,
source,
iodSource,
webSources,
iodSources,
message_source = "web-ui",
message_type, generationInfo,
prompt_id,
@@ -145,8 +145,8 @@ export const saveMessageOnSuccess = async ({
message: string
image: string
fullText: string
source: any[]
iodSource: any[]
webSources: any[]
iodSources: any[]
message_source?: "copilot" | "web-ui",
message_type?: string
generationInfo?: any
@@ -176,8 +176,8 @@ export const saveMessageOnSuccess = async ({
"assistant",
fullText,
[],
source,
iodSource,
webSources,
iodSources,
2,
message_type,
generationInfo,
@@ -209,8 +209,8 @@ export const saveMessageOnSuccess = async ({
"assistant",
fullText,
[],
source,
iodSource,
webSources,
iodSources,
2,
message_type,
generationInfo,

View File

@@ -59,6 +59,8 @@ export const useMessage = () => {
setIsSearchingInternet,
webSearch,
setWebSearch,
iodSearch,
setIodSearch,
isSearchingInternet
} = useStoreMessageOption()
const [defaultInternetSearchOn] = useStorage("defaultInternetSearchOn", false)
@@ -185,16 +187,16 @@ export const useMessage = () => {
isBot: false,
name: "You",
message,
sources: [],
iodSources:[],
webSources: [],
iodSources: [],
images: []
},
{
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
iodSources:[],
webSources: [],
iodSources: [],
id: generateMessageId
}
]
@@ -205,8 +207,8 @@ export const useMessage = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
iodSources:[],
webSources: [],
iodSources: [],
id: generateMessageId
}
]
@@ -337,7 +339,16 @@ export const useMessage = () => {
}
let context: string = ""
let source: {
let webSources: {
name: any
type: any
mode: string
url: string
pageContent: string
metadata: Record<string, any>
}[] = []
// TODO: update type
let iodSources: {
name: any
type: any
mode: string
@@ -349,7 +360,7 @@ export const useMessage = () => {
if (chatWithWebsiteEmbedding) {
const docs = await vectorstore.similaritySearch(query, 4)
context = formatDocs(docs)
source = docs.map((doc) => {
webSources = docs.map((doc) => {
return {
...doc,
name: doc?.metadata?.source || "untitled",
@@ -368,7 +379,7 @@ export const useMessage = () => {
.slice(0, maxWebsiteContext)
}
source = [
webSources = [
{
name: embedURL,
type: type,
@@ -479,7 +490,8 @@ export const useMessage = () => {
return {
...message,
message: fullText,
sources: source,
webSources,
iodSources,
generationInfo,
reasoning_time_taken: timetaken
}
@@ -500,7 +512,7 @@ export const useMessage = () => {
content: fullText
}
])
const iodSource = []
await saveMessageOnSuccess({
historyId,
setHistoryId,
@@ -509,8 +521,8 @@ export const useMessage = () => {
message,
image,
fullText,
source,
iodSource,
webSources,
iodSources,
message_source: "copilot",
generationInfo,
reasoning_time_taken: timetaken
@@ -610,15 +622,15 @@ export const useMessage = () => {
isBot: false,
name: "You",
message,
sources: [],
iodSources:[],
webSources: [],
iodSources: [],
images: []
},
{
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -630,7 +642,7 @@ export const useMessage = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -794,8 +806,8 @@ export const useMessage = () => {
message,
image,
fullText,
source: [],
iodSource:[],
webSources: [],
iodSources: [],
message_source: "copilot",
generationInfo,
reasoning_time_taken: timetaken
@@ -899,7 +911,7 @@ export const useMessage = () => {
isBot: false,
name: "You",
message,
sources: [],
webSources: [],
iodSources: [],
images: [image]
},
@@ -907,7 +919,7 @@ export const useMessage = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -919,7 +931,7 @@ export const useMessage = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -1088,8 +1100,8 @@ export const useMessage = () => {
message,
image,
fullText,
source: [],
iodSource:[],
webSources: [],
iodSources: [],
message_source: "copilot",
generationInfo,
reasoning_time_taken: timetaken
@@ -1126,12 +1138,14 @@ export const useMessage = () => {
}
const searchChatMode = async (
webSearch: boolean,
iodSearch,
message: string,
image: string,
isRegenerate: boolean,
messages: Message[],
history: ChatHistory,
signal: AbortSignal
signal: AbortSignal,
) => {
const url = await getOllamaURL()
setStreaming(true)
@@ -1188,7 +1202,7 @@ export const useMessage = () => {
isBot: false,
name: "You",
message,
sources: [],
webSources: [],
iodSources: [],
images: [image]
},
@@ -1196,7 +1210,7 @@ export const useMessage = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -1208,7 +1222,7 @@ export const useMessage = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -1286,10 +1300,10 @@ export const useMessage = () => {
query = removeReasoning(query)
}
const { prompt, source, iodSource } = await getSystemPromptForWeb(query, selectedQuickPrompt)
const { prompt, webSources, iodSources } =
await getSystemPromptForWeb(query, [], webSearch, iodSearch)
setIsSearchingInternet(false)
console.log("iodSource:")
console.log(iodSource)
// message = message.trim().replaceAll("\n", " ")
let humanMessage = await humanMessageFormatter({
@@ -1410,8 +1424,8 @@ export const useMessage = () => {
return {
...message,
message: fullText,
sources: source,
iodSources: iodSource,
webSources,
iodSources,
generationInfo,
reasoning_time_taken: timetaken
}
@@ -1441,8 +1455,8 @@ export const useMessage = () => {
message,
image,
fullText,
source,
iodSource,
webSources,
iodSources,
generationInfo,
reasoning_time_taken: timetaken
})
@@ -1541,7 +1555,7 @@ export const useMessage = () => {
isBot: false,
name: "You",
message,
sources: [],
webSources: [],
iodSources: [],
images: [image],
messageType: messageType
@@ -1550,7 +1564,7 @@ export const useMessage = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -1562,7 +1576,7 @@ export const useMessage = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -1709,8 +1723,8 @@ export const useMessage = () => {
message,
image,
fullText,
source: [],
iodSource:[],
webSources: [],
iodSources: [],
message_source: "copilot",
message_type: messageType,
generationInfo,
@@ -1788,14 +1802,16 @@ export const useMessage = () => {
)
} else {
if (chatMode === "normal") {
if (webSearch) {
if (webSearch || iodSearch) {
await searchChatMode(
webSearch,
iodSearch,
message,
image,
isRegenerate || false,
messages,
memory || history,
signal
signal,
)
} else {
await normalChatMode(
@@ -1928,6 +1944,8 @@ export const useMessage = () => {
regenerateLastMessage,
webSearch,
setWebSearch,
iodSearch,
setIodSearch,
isSearchingInternet,
selectedQuickPrompt,
setSelectedQuickPrompt,

View File

@@ -3,11 +3,12 @@ import { cleanUrl } from "~/libs/clean-url"
import {
defaultEmbeddingModelForRag,
geWebSearchFollowUpPrompt,
geWebSearchKeywordsPrompt,
getOllamaURL,
promptForRag,
systemPromptForNonRagOption
} from "~/services/ollama"
import { type ChatHistory, type Message } from "~/store/option"
import { type ChatHistory, ChatMessage, type Message } from "~/store/option"
import { SystemMessage } from "@langchain/core/messages"
import { useStoreMessageOption } from "~/store/option"
import {
@@ -54,6 +55,8 @@ export const useMessageOption = () => {
const {
history,
setHistory,
chatMessages,
setChatMessages,
setStreaming,
streaming,
setIsFirstMessage,
@@ -67,6 +70,8 @@ export const useMessageOption = () => {
setChatMode,
webSearch,
setWebSearch,
iodSearch,
setIodSearch,
isSearchingInternet,
setIsSearchingInternet,
selectedQuickPrompt,
@@ -109,8 +114,30 @@ export const useMessageOption = () => {
setWebSearch(true)
}
}
// 从最后的结果中解析出 思维链 和 结果
const responseResolver = (msg: string) => {
const thinkStart = msg.indexOf("<think>")
const thinkEnd = msg.indexOf("</think>")
let think = ""
let content = ""
if (thinkStart > -1 && thinkEnd > -1) {
think = msg.substring(thinkStart + 7, thinkEnd)
content = msg.substring(thinkEnd + 8)
} else {
content = msg
}
// 去掉换行符
think = think.replace(/\n/g, "")
content = content.replace(/\n/g, "")
return {
think,
content
}
}
const searchChatMode = async (
webSearch: boolean,
iodSearch: boolean,
message: string,
image: string,
isRegenerate: boolean,
@@ -161,9 +188,12 @@ export const useMessageOption = () => {
useMlock:
currentChatModelSettings?.useMlock ?? userDefaultModelSettings?.useMlock
})
let newMessage: Message[] = []
let generateMessageId = generateID()
const chatMessage: ChatMessage = {
id: generateMessageId,
queryContent: message
} as ChatMessage
if (!isRegenerate) {
newMessage = [
@@ -172,7 +202,7 @@ export const useMessageOption = () => {
isBot: false,
name: "You",
message,
sources: [],
webSources: [],
iodSources: [],
images: [image]
},
@@ -180,7 +210,7 @@ export const useMessageOption = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -192,7 +222,7 @@ export const useMessageOption = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -207,7 +237,8 @@ export const useMessageOption = () => {
setIsSearchingInternet(true)
let query = message
/*
let keywords: string[] = []
if (newMessage.length > 2) {
let questionPrompt = await geWebSearchFollowUpPrompt()
const lastTenMessages = newMessage.slice(-10)
@@ -270,17 +301,31 @@ export const useMessageOption = () => {
query = response.content.toString()
query = removeReasoning(query)
}
*/
const quickPrompt = selectedQuickPrompt;
console.log("quick prompt:"+quickPrompt)
const { prompt, source, iodSource } = await getSystemPromptForWeb(query, quickPrompt)
// Currently only IoD search use keywords
if (iodSearch) {
// Extract keywords
const questionPrompt = await geWebSearchKeywordsPrompt()
const promptForQuestion = questionPrompt.replaceAll("{query}", query)
const response = await ollama.invoke(promptForQuestion)
let res = response.content.toString()
res = removeReasoning(res)
keywords = res
.replace(/^Keywords:/i, "")
.split(", ")
.map((k) => k.trim())
}
const { prompt, webSources, iodSources } = await getSystemPromptForWeb(
query,
keywords,
webSearch,
iodSearch
)
console.log("prompt:\n" + prompt)
setIsSearchingInternet(false)
console.log("iodSource from useMessageOption:")
console.log(iodSource)
console.log("prompt")
console.log(prompt)
console.log("query")
console.log(query)
chatMessage.prompt = prompt
// message = message.trim().replaceAll("\n", " ")
let humanMessage = await humanMessageFormatter({
@@ -400,8 +445,8 @@ export const useMessageOption = () => {
return {
...message,
message: fullText,
sources: source,
iodSources:iodSource,
webSources,
iodSources,
generationInfo,
reasoning_time_taken: timetaken
}
@@ -431,14 +476,22 @@ export const useMessageOption = () => {
message,
image,
fullText,
source,
iodSource,
webSources,
iodSources,
generationInfo,
reasoning_time_taken: timetaken
})
setIsProcessing(false)
setStreaming(false)
chatMessage.relatedDataCount = keywords.length
chatMessage.timeTaken = timetaken
chatMessage.date = reasoningStartTime
const { think, content } = responseResolver(fullText)
chatMessage.thinkingChain = think
chatMessage.responseContent = content
setChatMessages([...chatMessages, chatMessage])
} catch (e) {
const errorSave = await saveMessageOnError({
e,
@@ -564,7 +617,7 @@ export const useMessageOption = () => {
isBot: false,
name: "You",
message,
sources: [],
webSources: [],
iodSources: [],
images: [image]
},
@@ -572,7 +625,7 @@ export const useMessageOption = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -584,7 +637,7 @@ export const useMessageOption = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -772,7 +825,6 @@ export const useMessageOption = () => {
image,
fullText,
source: [],
iodSource:[],
generationInfo,
prompt_content: promptContent,
prompt_id: promptId,
@@ -871,7 +923,7 @@ export const useMessageOption = () => {
isBot: false,
name: "You",
message,
sources: [],
webSources: [],
iodSources: [],
images: []
},
@@ -879,7 +931,7 @@ export const useMessageOption = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -891,7 +943,7 @@ export const useMessageOption = () => {
isBot: true,
name: selectedModel,
message: "▋",
sources: [],
webSources: [],
iodSources: [],
id: generateMessageId
}
@@ -998,8 +1050,7 @@ export const useMessageOption = () => {
}
})
// message = message.trim().replaceAll("\n", " ")
const iodSource = []
//TODO not support iodSource in RAG
let humanMessage = await humanMessageFormatter({
content: [
{
@@ -1096,8 +1147,7 @@ export const useMessageOption = () => {
return {
...message,
message: fullText,
sources: source,
iodSources: iodSource,
webSources: source,
generationInfo,
reasoning_time_taken: timetaken
}
@@ -1118,7 +1168,7 @@ export const useMessageOption = () => {
content: fullText
}
])
await saveMessageOnSuccess({
historyId,
setHistoryId,
@@ -1128,7 +1178,6 @@ export const useMessageOption = () => {
image,
fullText,
source,
iodSource,
generationInfo,
reasoning_time_taken: timetaken
})
@@ -1197,8 +1246,10 @@ export const useMessageOption = () => {
signal
)
} else {
if (webSearch) {
if (webSearch || iodSearch) {
await searchChatMode(
webSearch,
iodSearch,
message,
image,
isRegenerate,
@@ -1333,6 +1384,8 @@ export const useMessageOption = () => {
regenerateLastMessage,
webSearch,
setWebSearch,
iodSearch,
setIodSearch,
isSearchingInternet,
setIsSearchingInternet,
selectedQuickPrompt,

View File

@@ -12,6 +12,8 @@ import SidepanelSettings from "./sidepanel-settings"
import OptionRagSettings from "./option-rag"
import OptionChrome from "./option-settings-chrome"
import OptionOpenAI from "./option-settings-openai"
import OptionMetering from "./option-metering"
import MeteringListDetail from "./metering-list-detail"
export const OptionRoutingChrome = () => {
return (
@@ -27,6 +29,8 @@ export const OptionRoutingChrome = () => {
<Route path="/settings/knowledge" element={<OptionKnowledgeBase />} />
<Route path="/settings/rag" element={<OptionRagSettings />} />
<Route path="/settings/about" element={<OptionAbout />} />
<Route path="/metering" element={<OptionMetering />} />
<Route path="/metering/list/:id" element={<MeteringListDetail />} />
</Routes>
)
}

View File

@@ -10,6 +10,8 @@ const OptionModal = lazy(() => import("./option-settings-model"))
const OptionPrompt = lazy(() => import("./option-settings-prompt"))
const OptionOllamaSettings = lazy(() => import("./options-settings-ollama"))
const OptionSettings = lazy(() => import("./option-settings"))
const OptionMetering = lazy(() => import("./option-metering"))
const MeteringListDetail = lazy(() => import("./metering-list-detail"))
const OptionShare = lazy(() => import("./option-settings-share"))
const OptionKnowledgeBase = lazy(() => import("./option-settings-knowledge"))
const OptionAbout = lazy(() => import("./option-settings-about"))
@@ -29,6 +31,8 @@ export const OptionRoutingFirefox = () => {
<Route path="/settings/knowledge" element={<OptionKnowledgeBase />} />
<Route path="/settings/about" element={<OptionAbout />} />
<Route path="/settings/rag" element={<OptionRagSettings />} />
<Route path="/metering" element={<OptionMetering />} />
<Route path="/metering/list/:id" element={<MeteringListDetail />} />
</Routes>
)
}

View File

@@ -0,0 +1,12 @@
import OptionLayout from "~/components/Layouts/Layout"
import { ListDetail } from "~/components/Option/Metering/listDetail"
const OptionSettings = () => {
return (
<OptionLayout>
<ListDetail />
</OptionLayout>
)
}
export default OptionSettings

View File

@@ -0,0 +1,12 @@
import OptionLayout from "~/components/Layouts/Layout"
import { MeteringDetail } from "~/components/Option/Metering/detail"
const OptionSettings = () => {
return (
<OptionLayout>
<MeteringDetail />
</OptionLayout>
)
}
export default OptionSettings

View File

@@ -21,15 +21,39 @@ const DEFAULT_RAG_QUESTION_PROMPT =
const DEFAUTL_RAG_SYSTEM_PROMPT = `You are a helpful AI assistant. Use the following pieces of context to answer the question at the end. If you don't know the answer, just say you don't know. DO NOT try to make up an answer. If the question is not related to the context, politely respond that you are tuned to only answer questions that are related to the context. {context} Question: {question} Helpful answer:`
const DEFAULT_WEBSEARCH_PROMP = `You are an AI model who is expert at searching the web and answering user's queries.
const DEFAULT_WEBSEARCH_PROMPT = `You are an AI model who is expert at searching the web and answering user's queries.
Generate a response that is informative and relevant to the user's query based on provided search results. the current date and time are {current_date_time}.
\`search-results\` block provides knowledge from the web search results. You can use this information to generate a meaningful response.
\`iod-search-results\` block provides knowledge from the Internet of Data (数联网) search results. Each search result has a format of:
\`<result doId="{doId}" name="{name}" url="{url}" id="{id}">{content}</result>\`
Please show the \`doId\` and \`name\` of the search result when you cite the Internet of Data search result, in the following format, in English:
\`[IoD source [id] doId: {doId} "{name}"]({url})\`
Or in Chinese:
\`[数联网引用[id] doId: {doId} "{name}"]({url})\`
For example, in English:
\`[IoD source [1] doId: 10.48550/arXiv.1803.05591v2 "On the insufficiency of existing momentum schemes for Stochastic Optimization"](http://arxiv.org/pdf/1803.05591v2.pdf)\`
Or in Chinese:
\`[数联网引用[1] doId: 10.48550/arXiv.1803.05591v2 "On the insufficiency of existing momentum schemes for Stochastic Optimization"](http://arxiv.org/pdf/1803.05591v2.pdf)\`
<search-results>
{search_results}
</search-results>
\`web-search-results\` block provides knowledge from the World Wide Web (万维网) search results.
Please show the \`doId\` and \`name\` of the search result when you cite the search result, in the following format, in English:
\`[3W source [id] "{name}"]({url})\`
Or in Chinese:
\`[万维网引用[id] "{name}"]({url})\`
For example, in English:
\`[3W source [1] On the insufficiency of existing momentum schemes for Stochastic Optimization](http://arxiv.org/pdf/1803.05591v2.pdf)\`
Or in Chinese:
\`[万维网引用[1] On the insufficiency of existing momentum schemes for Stochastic Optimization](http://arxiv.org/pdf/1803.05591v2.pdf)\`
You can use these information to generate a meaningful response.
<iod-search-results>
{iod_search_results}
</iod-search-results>
<web-search-results>
{web_search_results}
</web-search-results>
`
const DEFAULT_WEBSEARCH_FOLLOWUP_PROMPT = `You will give a follow-up question. You need to rephrase the follow-up question if needed so it is a standalone question that can be used by the AI model to search the internet.
@@ -58,6 +82,30 @@ Follow-up question: {question}
Rephrased question:
`
const DEFAULT_WEBSEARCH_KEYWORDS_PROMPT = `Extract the most important keywords from the query (at most 3), and give me English and Chinese versions of the keywords.
The result format should be: keyword_1, keyword_2, ..., keyword_n
Example:
Query: What are the symptoms of a heart attack?
Keywords: symptoms, 症状, heart attack, 心臟病
Query: 什么是物联网?
Keywords: Internet of Things, IoT, 物联网
Query: 人工智能的发展趋势?
Keywords: Artificial Intelligence, AI, 人工智能, trend, 趋势
Query: {query}
Keywords:
`
export const getOllamaURL = async () => {
const ollamaURL = await storage.get("ollamaURL")
if (!ollamaURL || ollamaURL.length === 0) {
@@ -385,7 +433,7 @@ export const saveForRag = async (
export const getWebSearchPrompt = async () => {
const prompt = await storage.get("webSearchPrompt")
if (!prompt || prompt.length === 0) {
return DEFAULT_WEBSEARCH_PROMP
return DEFAULT_WEBSEARCH_PROMPT
}
return prompt
}
@@ -411,6 +459,18 @@ export const setWebPrompts = async (prompt: string, followUpPrompt: string) => {
await setWebSearchFollowUpPrompt(followUpPrompt)
}
export const geWebSearchKeywordsPrompt = async () => {
const prompt = await storage.get("webSearchKeywordsPrompt")
if (!prompt || prompt.length === 0) {
return DEFAULT_WEBSEARCH_KEYWORDS_PROMPT
}
return prompt
}
export const setWebSearchKeywordsPrompt = async (prompt: string) => {
await storage.set("webSearchKeywordsPrompt", prompt)
}
export const getPageShareUrl = async () => {
const pageShareUrl = await storage.get("pageShareUrl")
if (!pageShareUrl || pageShareUrl.length === 0) {

View File

@@ -14,7 +14,7 @@ export type Message = {
isBot: boolean
name: string
message: string
sources: any[]
webSources: any[]
iodSources: any[]
images?: string[]
search?: WebSearch
@@ -26,15 +26,43 @@ export type Message = {
export type ChatHistory = {
role: "user" | "assistant" | "system"
content: string
image?: string,
image?: string
messageType?: string
}[]
export type ChatMessage = {
id: string
// 问题
queryContent: string
// 提示词全文
prompt: string
// 思维链(只有深度思考时有)
thinkingChain?: string
// 回答
responseContent: string
// 关联数据个数
relatedDataCount: number
// 数联网输入token
iodInputToken: string
// 数联网输出token
iodOutputToken: string
// 大模型输入token
modelInputToken: string
// 大模型输出token
modelOutputToken: string
// 日期
date: Date
// 耗时
timeTaken: number
}
type State = {
messages: Message[]
setMessages: (messages: Message[]) => void
history: ChatHistory
setHistory: (history: ChatHistory) => void
chatMessages: ChatMessage[]
setChatMessages: (chatMessages: ChatMessage[]) => void
streaming: boolean
setStreaming: (streaming: boolean) => void
isFirstMessage: boolean
@@ -53,6 +81,8 @@ type State = {
setIsEmbedding: (isEmbedding: boolean) => void
webSearch: boolean
setWebSearch: (webSearch: boolean) => void
iodSearch: boolean
setIodSearch: (iodSearch: boolean) => void
isSearchingInternet: boolean
setIsSearchingInternet: (isSearchingInternet: boolean) => void
@@ -80,6 +110,8 @@ export const useStoreMessageOption = create<State>((set) => ({
setMessages: (messages) => set({ messages }),
history: [],
setHistory: (history) => set({ history }),
chatMessages: [],
setChatMessages: (chatMessages) => set({ chatMessages }),
streaming: false,
setStreaming: (streaming) => set({ streaming }),
isFirstMessage: true,
@@ -101,6 +133,8 @@ export const useStoreMessageOption = create<State>((set) => ({
setIsEmbedding: (isEmbedding) => set({ isEmbedding }),
webSearch: false,
setWebSearch: (webSearch) => set({ webSearch }),
iodSearch: false,
setIodSearch: (iodSearch) => set({ iodSearch }),
isSearchingInternet: false,
setIsSearchingInternet: (isSearchingInternet) => set({ isSearchingInternet }),
selectedSystemPrompt: null,
@@ -116,5 +150,5 @@ export const useStoreMessageOption = create<State>((set) => ({
setTemporaryChat: (temporaryChat) => set({ temporaryChat }),
useOCR: false,
setUseOCR: (useOCR) => set({ useOCR }),
setUseOCR: (useOCR) => set({ useOCR })
}))

8
src/types/iod.ts Normal file
View File

@@ -0,0 +1,8 @@
export type IodRegistryEntry = {
doId: string
name: string
url?: string
pdf_url?: string
description: string
content?: string
}

View File

@@ -11,7 +11,7 @@ export type Message = {
isBot: boolean
name: string
message: string
sources: any[]
webSources: any[]
iodSources: any[]
images?: string[]
search?: WebSearch

5
src/types/web.ts Normal file
View File

@@ -0,0 +1,5 @@
export type WebSearchResult = {
url: string
name: string
content: string
}

22
src/utils/date.ts Normal file
View File

@@ -0,0 +1,22 @@
export function formatDate(date) {
// 获取年份
const year = date.getFullYear()
// 获取月份注意月份是从0开始计数的所以需要加1并且确保月份是两位数
const month = String(date.getMonth() + 1).padStart(2, "0")
// 获取日期,确保日期是两位数
const day = String(date.getDate()).padStart(2, "0")
// 获取小时24小时制并确保小时是两位数
const hours = String(date.getHours()).padStart(2, "0")
// 获取分钟,并确保分钟是两位数
const minutes = String(date.getMinutes()).padStart(2, "0")
// 组合成所需的格式
return `${year}-${month}-${day} ${hours}:${minutes}`
}
// 示例使用
const now = new Date()

205
src/web/iod.ts Normal file
View File

@@ -0,0 +1,205 @@
import { cleanUrl } from "@/libs/clean-url"
import { PageAssistHtmlLoader } from "@/loader/html"
import { PageAssistPDFUrlLoader } from "@/loader/pdf-url"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import { defaultEmbeddingModelForRag, getOllamaURL } from "@/services/ollama"
import {
getIsSimpleInternetSearch,
totalSearchResults
} from "@/services/search"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
import type { Document } from "@langchain/core/documents"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
import type { IodRegistryEntry } from "~/types/iod"
const makeRegSearchParams = (count: number, keyword: string) => ({
action: "executeContract",
contractID: "BDBrowser",
operation: "sendRequestDirectly",
arg: {
id: "670E241C9937B3537047C87053E3AA36",
doipUrl: "tcp://reg01.public.internetofdata.cn:21037",
op: "Search",
attributes: {
offset: 0,
count,
bodyBase64Encoded: false,
searchMode: [
{
key: "data_type",
type: "MUST",
value: "paper"
},
// {
// key: "title",
// type: "MUST",
// value: keyword,
// },
{
key: "description",
type: "MUST",
value: keyword
}
]
},
body: ""
}
})
export async function localIodSearch(
query: string,
keywords: string[]
): Promise<IodRegistryEntry[]> {
const TOTAL_SEARCH_RESULTS = await totalSearchResults()
const results = (
await Promise.all(
keywords.map(async (keyword) => {
const abortController = new AbortController()
setTimeout(() => abortController.abort(), 10000)
const params = makeRegSearchParams(TOTAL_SEARCH_RESULTS, keyword)
return fetch("http://47.93.156.31:21033/SCIDE/SCManager", {
method: "POST",
body: JSON.stringify(params),
signal: abortController.signal
})
.then((response) => response.json())
.then((res) => {
if (res.status !== "Success") {
console.log(res)
return []
}
const body = JSON.parse(res.result.body)
if (body.code !== 0) {
console.log(body)
return []
}
const results: IodRegistryEntry[] =
body.data?.results?.filter((r) => r.url || r.pdf_url) || []
for (const r of results) {
r.url = r.url || r.pdf_url
}
return results
})
.catch((e) => {
console.log(e)
return []
})
})
)
).flat()
return results
}
const ARXIV_URL_PATTERN = /^https?:\/\/arxiv\.org\//
const ARXIV_NO_HTM = "No HTML for"
export const searchIod = async (query: string, keywords: string[]) => {
const searchResults = await localIodSearch(query, keywords)
const isSimpleMode = await getIsSimpleInternetSearch()
if (isSimpleMode) {
await getOllamaURL()
return searchResults
}
const docs: Document<Record<string, any>>[] = []
const resMap = new Map<string, IodRegistryEntry>()
for (const result of searchResults) {
const url = result.url
if (!url) continue
let htmlUrl = ""
if (ARXIV_URL_PATTERN.test(url)) {
htmlUrl = url.replace("/pdf/", "/html/").replace(".pdf", "")
}
let noHtml = htmlUrl === ""
if (!noHtml) {
const loader = new PageAssistHtmlLoader({
html: "",
url: htmlUrl
})
try {
const documents = await loader.loadByURL()
for (const doc of documents) {
if (doc.pageContent.includes(ARXIV_NO_HTM)) {
noHtml = true
return
}
docs.push(doc)
}
} catch (e) {
console.log(e)
noHtml = true
}
}
if (noHtml) {
if (url.endsWith(".pdf")) {
const loader = new PageAssistPDFUrlLoader({
name: result.name,
url
})
try {
const documents = await loader.load()
for (const doc of documents) {
docs.push(doc)
}
} catch (e) {
console.log(e)
}
} else {
const loader = new PageAssistHtmlLoader({
html: "",
url
})
try {
const documents = await loader.loadByURL()
for (const doc of documents) {
docs.push(doc)
}
} catch (e) {
console.log(e)
}
}
}
}
const ollamaUrl = await getOllamaURL()
const embeddingModle = await defaultEmbeddingModelForRag()
const ollamaEmbedding = await pageAssistEmbeddingModel({
model: embeddingModle || "",
baseUrl: cleanUrl(ollamaUrl)
})
const textSplitter = await getPageAssistTextSplitter()
const chunks = await textSplitter.splitDocuments(docs)
const store = new MemoryVectorStore(ollamaEmbedding)
await store.addDocuments(chunks)
const resultsWithEmbeddings = await store.similaritySearch(query, 3)
const searchResult = resultsWithEmbeddings.map((result) => {
// `source` for PDF type
const key = result.metadata.url || result.metadata.source
if (!key) return null
const fullRes = resMap[key]
return {
...fullRes,
content: result.pageContent
}
}).filter((r) => r)
return searchResult
}

View File

@@ -8,7 +8,9 @@ import { getWebsiteFromQuery, processSingleWebsite } from "./website"
import { searxngSearch } from "./search-engines/searxng"
import { braveAPISearch } from "./search-engines/brave-api"
import { webBaiduSearch } from "./search-engines/baidu"
import { LucideToggleRight } from "lucide-react"
import { searchIod } from "./iod"
import type { WebSearchResult } from "~/types/web"
import type { IodRegistryEntry } from "~/types/iod"
const getHostName = (url: string) => {
try {
@@ -19,110 +21,128 @@ const getHostName = (url: string) => {
}
}
const searchWeb = (provider: string, query: string) => {
async function searchWeb(
provider: string,
query: string
): Promise<WebSearchResult[]> {
let results = []
switch (provider) {
case "duckduckgo":
return webDuckDuckGoSearch(query)
results = await webDuckDuckGoSearch(query)
break
case "sogou":
return webSogouSearch(query)
results = await webSogouSearch(query)
break
case "brave":
return webBraveSearch(query)
results = await webBraveSearch(query)
break
case "searxng":
return searxngSearch(query)
results = await searxngSearch(query)
break
case "brave-api":
return braveAPISearch(query)
results = await braveAPISearch(query)
break
case "baidu":
return webBaiduSearch(query)
results = await webBaiduSearch(query)
break
default:
return webGoogleSearch(query)
results = await webGoogleSearch(query)
break
}
return results.map((r) => ({ ...r, name: getHostName(r.url) }))
}
export const getSystemPromptForWeb = async (query: string, promptMode) => {
export const getSystemPromptForWeb = async (
query: string,
keywords: string[] = [],
webSearch = true,
iodSearch = false
) => {
try {
if (!promptMode){
return {
prompt: "",
source: [],
iodSource:[]
}
}
let iodsearch = []
if (promptMode.indexOf("iod_search_results")!=-1){
iodsearch = [
{
url:"http://bdware.cn/resolve?id=CSTR:432421111.1233.53323",
content:"数联网Internet Of Data):数据作为互联网上可独立管理的资源,在“物理/机器”互联网之上形成一个“虚拟/数据”网络,实现全网一体化的数据互联互通互操作。",
id:"CSTR:432421111.1233.53323,数联网定义"
}, {
url:"http://bdware.cn/resolve?id=CSTR:1121311.3423.7754",
content:"数据空间:面向具体的领域和业务场景,按照数据所对应的物理实体的结构、关系来对数据进行管理和组织,构成物理世界的数字孪生。",
id:"CSTR:1121311.3423.7754,数据空间定义"
}
]
}
const websiteVisit = getWebsiteFromQuery(query)
let search: {
url: any;
content: string;
}[] = []
const isVisitSpecificWebsite = await getIsVisitSpecificWebsite()
if (isVisitSpecificWebsite && websiteVisit.hasUrl) {
const url = websiteVisit.url
const queryWithoutUrl = websiteVisit.queryWithouUrls
search = await processSingleWebsite(url, queryWithoutUrl)
} else if (promptMode.indexOf("web_search_results")!=-1) {
const searchProvider = await getSearchProvider()
search = await searchWeb(searchProvider, query)
let webSearchResults: WebSearchResult[] = []
// let search_results_web = ""
if (webSearch) {
const isVisitSpecificWebsite = await getIsVisitSpecificWebsite()
if (isVisitSpecificWebsite && websiteVisit.hasUrl) {
const url = websiteVisit.url
const queryWithoutUrl = websiteVisit.queryWithouUrls
webSearchResults = await processSingleWebsite(url, queryWithoutUrl)
} else {
const searchProvider = await getSearchProvider()
webSearchResults = await searchWeb(searchProvider, query)
}
// search_results_web = webSearchResults
// .map(
// (result, idx) =>
// `<result source="${result.url}" id="${idx}">${result.content}</result>`
// )
// .join("\n")
}
const search_results = search
let iodSearchResults: IodRegistryEntry[] = []
// let search_results_iod = ""
if (iodSearch) {
iodSearchResults = await searchIod(query, keywords)
// search_results_iod = iodSearchResults
// .map(
// (result, idx) =>
// `<result source="${result.url}" id="${idx}">${result.content}</result>`
// )
// .join("\n")
}
const iod_search_results = iodSearchResults
.map((res) => ({
doId: res.doId,
name: res.name,
url: res.url,
content: res.content || res.description
}))
.map(
(result, idx) =>
`<result source="${result.url}" id="${idx+1}">${result.content}</result>`
`<result doId="${result.doId}" name="${result.name}" source="${result.url}" id="${idx + 1}">${result.content}</result>`
)
.join("\n")
console.log("iod_search_result: " + iod_search_results)
const web_search_results = webSearchResults
.map(
(result, idx) =>
`<result source="${result.url}" name="${result.name}" id="${idx + 1}">${result.content}</result>`
)
.join("\n")
console.log("web_search_result: " + web_search_results)
const current_date_time = new Date().toLocaleString()
const system = promptMode
const iod_search_results= iodsearch.map(
(result, idx) =>
`<result source="${result.url}" id="${idx+1}">${result.content}</result>`
)
.join("\n")
console.log("iod_search_xml in web.ts")
console.log(iod_search_results)
const system = await getWebSearchPrompt()
const prompt = system
.replace("{current_date_time}", current_date_time)
.replace("{web_search_results}", search_results)
.replace("{iod_search_results}",iod_search_results)
.replace("{iod_search_results}", iod_search_results)
.replace("{web_search_results}", web_search_results)
return {
prompt,
source: search.map((result) => {
webSources: webSearchResults.map((result) => {
return {
url: result.url,
name: getHostName(result.url),
name: result.name,
type: "url"
}
}),
iodSource: iodsearch.map((result) => {
return {
url: result.url,
name: result.id,
type: "url"
}
})
iodSources: iodSearchResults
}
} catch (e) {
console.error(e)
return {
prompt: "",
source: [],
iodSource:[]
webSources: [],
iodSources: []
}
}
}