Muhammed Nazeem
GitHub
commit
325913cc9c
@ -31,6 +31,16 @@
|
||||
"label": "Send Notification After Finishing Processing the Knowledge Base"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "Copilot Chat With Website Settings",
|
||||
"ragEnabled": {
|
||||
"label": "Chat with website using vector embeddings"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "Normal mode website content size",
|
||||
"placeholder": "Content size (default 4028)"
|
||||
}
|
||||
},
|
||||
"webSearch": {
|
||||
"heading": "Manage Web Search",
|
||||
"searchMode": {
|
||||
@ -288,6 +298,11 @@
|
||||
"label": "Chunk Overlap",
|
||||
"placeholder": "Enter Chunk Overlap",
|
||||
"required": "Please enter a chunk overlap"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "Knowledge Base Default File Limit",
|
||||
"placeholder": "Enter default file limit (e.g., 10)",
|
||||
"required": "Please enter the default file limit"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -31,6 +31,16 @@
|
||||
"label": "Enviar notificación después de terminar el procesamiento de la base de conocimientos"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "Configuración del Chat Copilot con Sitio Web",
|
||||
"ragEnabled": {
|
||||
"label": "Chatear con el sitio web usando incrustaciones vectoriales"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "Tamaño del contenido del sitio web en modo normal",
|
||||
"placeholder": "Tamaño del contenido (predeterminado 4028)"
|
||||
}
|
||||
},
|
||||
"webSearch": {
|
||||
"heading": "Manejo de la busqueda Web",
|
||||
"searchMode": {
|
||||
@ -288,6 +298,11 @@
|
||||
"label": "Solapamiento del Chunk",
|
||||
"placeholder": "Ingrese el solapamiento del chunk",
|
||||
"required": "Por favor, ingresar el solapamiento del chunk"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "Límite predeterminado de archivos de la base de conocimientos",
|
||||
"placeholder": "Ingrese el límite predeterminado de archivos (ej. 10)",
|
||||
"required": "Por favor, ingrese el límite predeterminado de archivos"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -31,6 +31,16 @@
|
||||
"label": "Envoyer une notification après avoir terminé le traitement de la base de connaissances"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "Paramètres du Chat Copilot avec le Site Web",
|
||||
"ragEnabled": {
|
||||
"label": "Discuter avec le site web en utilisant des embeddings vectoriels"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "Taille du contenu du site web en mode normal",
|
||||
"placeholder": "Taille du contenu (par défaut 4028)"
|
||||
}
|
||||
},
|
||||
"webSearch": {
|
||||
"heading": "Gérer la recherche Web",
|
||||
"searchMode": {
|
||||
@ -288,6 +298,11 @@
|
||||
"label": "Chevauchement",
|
||||
"placeholder": "Entrez le chevauchement des morceaux",
|
||||
"required": "Veuillez saisir un chevauchement"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "Limite par défaut de fichiers de la base de connaissances",
|
||||
"placeholder": "Entrez la limite par défaut de fichiers (ex. 10)",
|
||||
"required": "Veuillez entrer la limite par défaut de fichiers"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -31,6 +31,16 @@
|
||||
"label": "Inviare notifica dopo aver terminato l'elaborazione della base di conoscenza"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "Impostazioni Chat Copilot con Sito Web",
|
||||
"ragEnabled": {
|
||||
"label": "Chatta con il sito web utilizzando embedding vettoriali"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "Dimensione del contenuto del sito web in modalità normale",
|
||||
"placeholder": "Dimensione del contenuto (predefinito 4028)"
|
||||
}
|
||||
},
|
||||
"webSearch": {
|
||||
"heading": "Gestione ricerca Web",
|
||||
"searchMode": {
|
||||
@ -288,6 +298,11 @@
|
||||
"label": "Sovrapposizione del Blocco (Chunk Overlap)",
|
||||
"placeholder": "Inserisci la Sovrapposizione del Blocco (Chunk Overlap)",
|
||||
"required": "Inserisci la Sovrapposizione del Blocco"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "Limite predefinito di file della base di conoscenza",
|
||||
"placeholder": "Inserisci il limite predefinito di file (es. 10)",
|
||||
"required": "Inserisci il limite predefinito di file"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -34,6 +34,16 @@
|
||||
"label": "ナレッジベースの処理完了後に通知を送信"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "ウェブサイトとのCopilotチャット設定",
|
||||
"ragEnabled": {
|
||||
"label": "ベクトル埋め込みを使用してウェブサイトとチャットする"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "通常モードのウェブサイトコンテンツサイズ",
|
||||
"placeholder": "コンテンツサイズ（デフォルト4028）"
|
||||
}
|
||||
},
|
||||
"webSearch": {
|
||||
"heading": "ウェブ検索を管理する",
|
||||
"searchMode": {
|
||||
@ -291,6 +301,11 @@
|
||||
"label": "チャンクオーバーラップ",
|
||||
"placeholder": "チャンクオーバーラップを入力",
|
||||
"required": "チャンクオーバーラップを入力してください"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "ナレッジベースのデフォルトファイル制限",
|
||||
"placeholder": "デフォルトのファイル制限を入力してください（例：10）",
|
||||
"required": "デフォルトのファイル制限を入力してください"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -34,6 +34,16 @@
|
||||
"label": "അറിവ് ശേഖരം പ്രോസസ്സ് ചെയ്ത് കഴിഞ്ഞതിന് ശേഷം അറിയിപ്പ് അയയ്ക്കുക"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "വെബ്സൈറ്റുമായുള്ള കോപൈലറ്റ് ചാറ്റ് ക്രമീകരണങ്ങൾ",
|
||||
"ragEnabled": {
|
||||
"label": "വെക്ടർ എംബെഡിംഗുകൾ ഉപയോഗിച്ച് വെബ്സൈറ്റുമായി ചാറ്റ് ചെയ്യുക"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "സാധാരണ മോഡിലെ വെബ്സൈറ്റ് ഉള്ളടക്ക വലുപ്പം",
|
||||
"placeholder": "ഉള്ളടക്ക വലുപ്പം (സ്ഥിരസ്ഥിതി 4028)"
|
||||
}
|
||||
},
|
||||
"webSearch": {
|
||||
"heading": "വെബ്ബ് തിരച്ചിൽ നിയന്ത്രിക്കുക",
|
||||
"searchMode": {
|
||||
@ -291,6 +301,11 @@
|
||||
"label": "ചങ്ക് ഓവര്ലാപ്പ്",
|
||||
"placeholder": "ചങ്ക് ഓവര്ലാപ്പ് നല്കുക",
|
||||
"required": "ദയവായി ചങ്ക് ഓവര്ലാപ്പ് നല്കുക"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "Limite padrão de arquivos da base de conhecimento",
|
||||
"placeholder": "Digite o limite padrão de arquivos (ex. 10)",
|
||||
"required": "Por favor, digite o limite padrão de arquivos"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -31,6 +31,16 @@
|
||||
"label": "Enviar notificação após concluir o processamento da base de conhecimento"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "Configurações de Chat Copilot com o Site",
|
||||
"ragEnabled": {
|
||||
"label": "Conversar com o site usando embeddings vetoriais"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "Tamanho do conteúdo do site no modo normal",
|
||||
"placeholder": "Tamanho do conteúdo (padrão 4028)"
|
||||
}
|
||||
},
|
||||
"webSearch": {
|
||||
"heading": "Gerenciar Pesquisa na Web",
|
||||
"searchMode": {
|
||||
@ -288,6 +298,11 @@
|
||||
"label": "Sobreposição do Pedaço",
|
||||
"placeholder": "Digite a Sobreposição do Pedaço",
|
||||
"required": "Por favor, insira uma sobreposição de pedaço"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "Limite padrão de arquivos da base de conhecimento",
|
||||
"placeholder": "Digite o limite padrão de arquivos (ex. 10)",
|
||||
"required": "Por favor, digite o limite padrão de arquivos"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -31,6 +31,17 @@
|
||||
"label": "Отправить уведомление после завершения обработки базы знаний"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "Настройки чата Copilot с веб-сайтом",
|
||||
"ragEnabled": {
|
||||
"label": "Общаться с веб-сайтом, используя векторные вложения"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "Размер содержимого веб-сайта в обычном режиме",
|
||||
"placeholder": "Размер содержимого (по умолчанию 4028)"
|
||||
}
|
||||
},
|
||||
|
||||
"webSearch": {
|
||||
"heading": "Управление веб-поиском",
|
||||
"searchMode": {
|
||||
@ -289,6 +300,11 @@
|
||||
"label": "Перекрытие фрагментов",
|
||||
"placeholder": "Введите перекрытие фрагментов",
|
||||
"required": "Пожалуйста, введите перекрытие фрагментов"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "Стандартный лимит файлов базы знаний",
|
||||
"placeholder": "Введите стандартный лимит файлов (напр. 10)",
|
||||
"required": "Пожалуйста, введите стандартный лимит файлов"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -34,6 +34,16 @@
|
||||
"label": "完成知识库处理后发送通知"
|
||||
}
|
||||
},
|
||||
"sidepanelRag": {
|
||||
"heading": "与网站对话的Copilot聊天设置",
|
||||
"ragEnabled": {
|
||||
"label": "使用向量嵌入与网站聊天"
|
||||
},
|
||||
"maxWebsiteContext": {
|
||||
"label": "普通模式下的网站内容大小",
|
||||
"placeholder": "内容大小（默认4028）"
|
||||
}
|
||||
},
|
||||
"webSearch": {
|
||||
"heading": "管理网络搜索",
|
||||
"searchMode": {
|
||||
@ -293,6 +303,11 @@
|
||||
"label": "嵌入重叠",
|
||||
"placeholder": "256-∞",
|
||||
"required": "请输入嵌入重叠"
|
||||
},
|
||||
"totalFilePerKB": {
|
||||
"label": "知识库默认文件限制",
|
||||
"placeholder": "输入默认文件限制（例如：10）",
|
||||
"required": "请输入默认文件限制"
|
||||
}
|
||||
},
|
||||
"prompt": {
|
||||
|
||||
@ -7,6 +7,7 @@ import { InboxIcon } from "lucide-react"
|
||||
import { useTranslation } from "react-i18next"
|
||||
import PubSub from "pubsub-js"
|
||||
import { KNOWLEDGE_QUEUE } from "@/queue"
|
||||
import { useStorage } from "@plasmohq/storage/hook"
|
||||
|
||||
type Props = {
|
||||
open: boolean
|
||||
@ -16,6 +17,7 @@ type Props = {
|
||||
export const AddKnowledge = ({ open, setOpen }: Props) => {
|
||||
const { t } = useTranslation(["knowledge", "common"])
|
||||
const [form] = Form.useForm()
|
||||
const [totalFilePerKB] = useStorage("totalFilePerKB", 10)
|
||||
|
||||
const onUploadHandler = async (data: {
|
||||
title: string
|
||||
@ -92,7 +94,7 @@ export const AddKnowledge = ({ open, setOpen }: Props) => {
|
||||
<Upload.Dragger
|
||||
accept={".pdf, .csv, .txt, .md, .docx"}
|
||||
multiple={true}
|
||||
maxCount={10}
|
||||
maxCount={totalFilePerKB}
|
||||
beforeUpload={(file) => {
|
||||
const allowedTypes = [
|
||||
"application/pdf",
|
||||
|
||||
@ -17,9 +17,9 @@ export const SelectedKnowledge = () => {
|
||||
<Tooltip
|
||||
title={knowledge.title}
|
||||
>
|
||||
<div className="inline-flex truncate items-center gap-2">
|
||||
<Blocks className="h-5 w-5 text-gray-400" />
|
||||
<span className="text-xs hidden lg:inline-block font-semibold dark:text-gray-100">
|
||||
<div className="inline-flex items-center gap-2 max-w-[150px]">
|
||||
<Blocks className="h-5 w-5 text-gray-400 flex-shrink-0" />
|
||||
<span className="text-xs hidden lg:inline-block font-semibold dark:text-gray-100 truncate">
|
||||
{knowledge.title}
|
||||
</span>
|
||||
</div>
|
||||
@ -34,4 +34,4 @@ export const SelectedKnowledge = () => {
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
}
|
||||
@ -126,30 +126,31 @@ export const PromptBody = () => {
|
||||
{
|
||||
title: t("managePrompts.columns.title"),
|
||||
dataIndex: "title",
|
||||
key: "title"
|
||||
key: "title",
|
||||
render: (content) => (<span className="line-clamp-1">{content}</span>)
|
||||
},
|
||||
{
|
||||
title: t("managePrompts.columns.prompt"),
|
||||
dataIndex: "content",
|
||||
key: "content"
|
||||
key: "content",
|
||||
render: (content) => (<span className="line-clamp-1">{content}</span>)
|
||||
},
|
||||
{
|
||||
title: t("managePrompts.columns.type"),
|
||||
dataIndex: "is_system",
|
||||
key: "is_system",
|
||||
render: (is_system) =>
|
||||
is_system ? (
|
||||
<span className="flex items-center gap-2">
|
||||
<Computer className="w-5 h-5 " />
|
||||
{t("managePrompts.systemPrompt")}
|
||||
</span>
|
||||
) : (
|
||||
<span className="flex items-center gap-2">
|
||||
<Zap className="w-5 h-5" />
|
||||
{t("managePrompts.quickPrompt")}
|
||||
</span>
|
||||
)
|
||||
},
|
||||
<span className="flex items-center gap-2 text-xs w-32">
|
||||
{is_system ? (
|
||||
<>
|
||||
<Computer className="size-4" /> {t("managePrompts.systemPrompt")}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<Zap className="size-4" /> {t("managePrompts.quickPrompt")}
|
||||
</>
|
||||
)}
|
||||
</span> },
|
||||
{
|
||||
title: t("managePrompts.columns.actions"),
|
||||
render: (_, record) => (
|
||||
@ -164,7 +165,7 @@ export const PromptBody = () => {
|
||||
}
|
||||
}}
|
||||
className="text-red-500 dark:text-red-400">
|
||||
<Trash2 className="w-5 h-5" />
|
||||
<Trash2 className="size-4" />
|
||||
</button>
|
||||
</Tooltip>
|
||||
<Tooltip title={t("managePrompts.tooltip.edit")}>
|
||||
@ -175,7 +176,7 @@ export const PromptBody = () => {
|
||||
setOpenEdit(true)
|
||||
}}
|
||||
className="text-gray-500 dark:text-gray-400">
|
||||
<Pen className="w-5 h-5" />
|
||||
<Pen className="size-4" />
|
||||
</button>
|
||||
</Tooltip>
|
||||
</div>
|
||||
|
||||
@ -1,39 +0,0 @@
|
||||
import { Tabs } from "antd"
|
||||
import { SettingsOllama } from "./Settings/ollama"
|
||||
import { SettingPrompt } from "./Settings/prompt"
|
||||
import { SettingOther } from "./Settings/general-settings"
|
||||
|
||||
type Props = {
|
||||
setClose: (close: boolean) => void
|
||||
}
|
||||
|
||||
export const Settings = ({ setClose }: Props) => {
|
||||
return (
|
||||
<div className="my-6 max-h-[80vh] overflow-y-auto">
|
||||
<Tabs
|
||||
tabPosition="left"
|
||||
defaultActiveKey="1"
|
||||
items={[
|
||||
{
|
||||
id: "1",
|
||||
key: "1",
|
||||
label: "Prompt",
|
||||
children: <SettingPrompt />
|
||||
},
|
||||
{
|
||||
id: "2",
|
||||
key: "2",
|
||||
label: "Web UI Settings",
|
||||
children: <SettingOther />
|
||||
},
|
||||
{
|
||||
id: "3",
|
||||
key: "3",
|
||||
label: "Ollama Settings",
|
||||
children: <SettingsOllama />
|
||||
}
|
||||
]}
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -10,6 +10,8 @@ import {
|
||||
} from "~/services/ollama"
|
||||
import { SettingPrompt } from "./prompt"
|
||||
import { useTranslation } from "react-i18next"
|
||||
import { getTotalFilePerKB } from "@/services/app"
|
||||
import { SidepanelRag } from "./sidepanel-rag"
|
||||
|
||||
export const RagSettings = () => {
|
||||
const { t } = useTranslation("settings")
|
||||
@ -19,19 +21,20 @@ export const RagSettings = () => {
|
||||
const { data: ollamaInfo, status } = useQuery({
|
||||
queryKey: ["fetchRAGSettings"],
|
||||
queryFn: async () => {
|
||||
const [allModels, chunkOverlap, chunkSize, defaultEM] = await Promise.all(
|
||||
[
|
||||
const [allModels, chunkOverlap, chunkSize, defaultEM, totalFilePerKB] =
|
||||
await Promise.all([
|
||||
getAllModels({ returnEmpty: true }),
|
||||
defaultEmbeddingChunkOverlap(),
|
||||
defaultEmbeddingChunkSize(),
|
||||
defaultEmbeddingModelForRag()
|
||||
]
|
||||
)
|
||||
defaultEmbeddingModelForRag(),
|
||||
getTotalFilePerKB()
|
||||
])
|
||||
return {
|
||||
models: allModels,
|
||||
chunkOverlap,
|
||||
chunkSize,
|
||||
defaultEM
|
||||
defaultEM,
|
||||
totalFilePerKB
|
||||
}
|
||||
}
|
||||
})
|
||||
@ -41,8 +44,9 @@ export const RagSettings = () => {
|
||||
model: string
|
||||
chunkSize: number
|
||||
overlap: number
|
||||
totalFilePerKB: number
|
||||
}) => {
|
||||
await saveForRag(data.model, data.chunkSize, data.overlap)
|
||||
await saveForRag(data.model, data.chunkSize, data.overlap, data.totalFilePerKB)
|
||||
return true
|
||||
},
|
||||
onSuccess: () => {
|
||||
@ -70,13 +74,15 @@ export const RagSettings = () => {
|
||||
saveRAG({
|
||||
model: data.defaultEM,
|
||||
chunkSize: data.chunkSize,
|
||||
overlap: data.chunkOverlap
|
||||
overlap: data.chunkOverlap,
|
||||
totalFilePerKB: data.totalFilePerKB
|
||||
})
|
||||
}}
|
||||
initialValues={{
|
||||
chunkSize: ollamaInfo?.chunkSize,
|
||||
chunkOverlap: ollamaInfo?.chunkOverlap,
|
||||
defaultEM: ollamaInfo?.defaultEM
|
||||
defaultEM: ollamaInfo?.defaultEM,
|
||||
totalFilePerKB: ollamaInfo?.totalFilePerKB
|
||||
}}>
|
||||
<Form.Item
|
||||
name="defaultEM"
|
||||
@ -85,9 +91,7 @@ export const RagSettings = () => {
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: t(
|
||||
"rag.ragSettings.model.required"
|
||||
)
|
||||
message: t("rag.ragSettings.model.required")
|
||||
}
|
||||
]}>
|
||||
<Select
|
||||
@ -99,9 +103,7 @@ export const RagSettings = () => {
|
||||
0
|
||||
}
|
||||
showSearch
|
||||
placeholder={t(
|
||||
"rag.ragSettings.model.placeholder"
|
||||
)}
|
||||
placeholder={t("rag.ragSettings.model.placeholder")}
|
||||
style={{ width: "100%" }}
|
||||
className="mt-4"
|
||||
options={ollamaInfo.models?.map((model) => ({
|
||||
@ -117,45 +119,53 @@ export const RagSettings = () => {
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: t(
|
||||
"rag.ragSettings.chunkSize.required"
|
||||
)
|
||||
message: t("rag.ragSettings.chunkSize.required")
|
||||
}
|
||||
]}>
|
||||
<InputNumber
|
||||
style={{ width: "100%" }}
|
||||
placeholder={t(
|
||||
"rag.ragSettings.chunkSize.placeholder"
|
||||
)}
|
||||
placeholder={t("rag.ragSettings.chunkSize.placeholder")}
|
||||
/>
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
name="chunkOverlap"
|
||||
label={t(
|
||||
"rag.ragSettings.chunkOverlap.label"
|
||||
)}
|
||||
label={t("rag.ragSettings.chunkOverlap.label")}
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: t(
|
||||
"rag.ragSettings.chunkOverlap.required"
|
||||
)
|
||||
message: t("rag.ragSettings.chunkOverlap.required")
|
||||
}
|
||||
]}>
|
||||
<InputNumber
|
||||
style={{ width: "100%" }}
|
||||
placeholder={t(
|
||||
"rag.ragSettings.chunkOverlap.placeholder"
|
||||
)}
|
||||
placeholder={t("rag.ragSettings.chunkOverlap.placeholder")}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
name="totalFilePerKB"
|
||||
label={t("rag.ragSettings.totalFilePerKB.label")}
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: t("rag.ragSettings.totalFilePerKB.required")
|
||||
}
|
||||
]}>
|
||||
<InputNumber
|
||||
style={{ width: "100%" }}
|
||||
placeholder={t("rag.ragSettings.totalFilePerKB.placeholder")}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
|
||||
<div className="flex justify-end">
|
||||
<SaveButton disabled={isSaveRAGPending} btnType="submit" />
|
||||
</div>
|
||||
</Form>
|
||||
</div>
|
||||
|
||||
<SidepanelRag />
|
||||
|
||||
<div>
|
||||
<div>
|
||||
<h2 className="text-base font-semibold leading-7 text-gray-900 dark:text-white">
|
||||
|
||||
63
src/components/Option/Settings/sidepanel-rag.tsx
Normal file
63
src/components/Option/Settings/sidepanel-rag.tsx
Normal file
@ -0,0 +1,63 @@
|
||||
import { useStorage } from "@plasmohq/storage/hook"
|
||||
import { InputNumber, Switch } from "antd"
|
||||
import { useTranslation } from "react-i18next"
|
||||
|
||||
export const SidepanelRag = ({ hideBorder }: { hideBorder?: boolean }) => {
|
||||
const { t } = useTranslation("settings")
|
||||
const [chatWithWebsiteEmbedding, setChatWithWebsiteEmbedding] = useStorage(
|
||||
"chatWithWebsiteEmbedding",
|
||||
true
|
||||
)
|
||||
const [maxWebsiteContext, setMaxWebsiteContext] = useStorage(
|
||||
"maxWebsiteContext",
|
||||
4028
|
||||
)
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="mb-5">
|
||||
<h2
|
||||
className={`${
|
||||
!hideBorder ? "text-base font-semibold leading-7" : "text-md"
|
||||
} text-gray-900 dark:text-white`}>
|
||||
{t("generalSettings.sidepanelRag.heading")}
|
||||
</h2>
|
||||
{!hideBorder && (
|
||||
<div className="border border-b border-gray-200 dark:border-gray-600 mt-3"></div>
|
||||
)}
|
||||
</div>
|
||||
<div className={`${
|
||||
!hideBorder ? "text-sm" : ""
|
||||
} space-y-4`}>
|
||||
<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between space-y-4 sm:space-y-0">
|
||||
<span className="text-gray-700 truncate dark:text-neutral-50">
|
||||
{t("generalSettings.sidepanelRag.ragEnabled.label")}
|
||||
</span>
|
||||
<div>
|
||||
<Switch
|
||||
className="mt-4 sm:mt-0"
|
||||
checked={chatWithWebsiteEmbedding}
|
||||
onChange={(checked) => setChatWithWebsiteEmbedding(checked)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex flex-col sm:flex-row sm:items-center sm:justify-between space-y-4 sm:space-y-0">
|
||||
<span className="text-gray-700 truncate dark:text-neutral-50">
|
||||
{t("generalSettings.sidepanelRag.maxWebsiteContext.label")}
|
||||
</span>
|
||||
<div>
|
||||
<InputNumber
|
||||
disabled={chatWithWebsiteEmbedding}
|
||||
className="mt-4 sm:mt-0"
|
||||
value={maxWebsiteContext}
|
||||
onChange={(value) => setMaxWebsiteContext(value)}
|
||||
placeholder={t(
|
||||
"generalSettings.sidepanelRag.maxWebsiteContext.placeholder"
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@ -35,7 +35,10 @@ export const TTSModeSettings = ({ hideBorder }: { hideBorder?: boolean }) => {
|
||||
return (
|
||||
<div>
|
||||
<div className="mb-5">
|
||||
<h2 className="text-base font-semibold leading-7 text-gray-900 dark:text-white">
|
||||
<h2
|
||||
className={`${
|
||||
!hideBorder ? "text-base font-semibold leading-7" : "text-md"
|
||||
} text-gray-900 dark:text-white`}>
|
||||
{t("generalSettings.tts.heading")}
|
||||
</h2>
|
||||
{!hideBorder && (
|
||||
|
||||
@ -33,6 +33,8 @@ import { useI18n } from "@/hooks/useI18n"
|
||||
import { TTSModeSettings } from "@/components/Option/Settings/tts-mode"
|
||||
import { AdvanceOllamaSettings } from "@/components/Common/Settings/AdvanceOllamaSettings"
|
||||
import { useStorage } from "@plasmohq/storage/hook"
|
||||
import { getTotalFilePerKB } from "@/services/app"
|
||||
import { SidepanelRag } from "@/components/Option/Settings/sidepanel-rag"
|
||||
|
||||
export const SettingsBody = () => {
|
||||
const { t } = useTranslation("settings")
|
||||
@ -66,7 +68,8 @@ export const SettingsBody = () => {
|
||||
allModels,
|
||||
chunkOverlap,
|
||||
chunkSize,
|
||||
defaultEM
|
||||
defaultEM,
|
||||
totalFilePerKB
|
||||
] = await Promise.all([
|
||||
getOllamaURL(),
|
||||
systemPromptForNonRag(),
|
||||
@ -74,7 +77,8 @@ export const SettingsBody = () => {
|
||||
getAllModels({ returnEmpty: true }),
|
||||
defaultEmbeddingChunkOverlap(),
|
||||
defaultEmbeddingChunkSize(),
|
||||
defaultEmbeddingModelForRag()
|
||||
defaultEmbeddingModelForRag(),
|
||||
getTotalFilePerKB()
|
||||
])
|
||||
|
||||
return {
|
||||
@ -85,18 +89,19 @@ export const SettingsBody = () => {
|
||||
models: allModels,
|
||||
chunkOverlap,
|
||||
chunkSize,
|
||||
defaultEM
|
||||
defaultEM,
|
||||
totalFilePerKB
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const { mutate: saveRAG, isPending: isSaveRAGPending } = useMutation({
|
||||
mutationFn: async (data: {
|
||||
mutationFn: async (f: {
|
||||
model: string
|
||||
chunkSize: number
|
||||
overlap: number
|
||||
}) => {
|
||||
await saveForRag(data.model, data.chunkSize, data.overlap)
|
||||
await saveForRag(f.model, f.chunkSize, f.overlap, data.totalFilePerKB)
|
||||
}
|
||||
})
|
||||
|
||||
@ -196,7 +201,9 @@ export const SettingsBody = () => {
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="border border-gray-300 dark:border-gray-700 rounded p-4 bg-white dark:bg-[#171717]">
|
||||
<SidepanelRag hideBorder />
|
||||
</div>
|
||||
<div className="border flex flex-col gap-4 border-gray-300 dark:border-gray-700 rounded p-4 bg-white dark:bg-[#171717]">
|
||||
<h2 className="text-md font-semibold dark:text-white">
|
||||
{t("ollamaSettings.heading")}
|
||||
@ -247,7 +254,6 @@ export const SettingsBody = () => {
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="border border-gray-300 dark:border-gray-700 rounded p-4 bg-white dark:bg-[#171717]">
|
||||
<h2 className="text-md mb-4 font-semibold dark:text-white">
|
||||
{t("rag.ragSettings.label")}
|
||||
@ -298,16 +304,12 @@ export const SettingsBody = () => {
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: t(
|
||||
"rag.ragSettings.chunkSize.required"
|
||||
)
|
||||
message: t("rag.ragSettings.chunkSize.required")
|
||||
}
|
||||
]}>
|
||||
<InputNumber
|
||||
style={{ width: "100%" }}
|
||||
placeholder={t(
|
||||
"rag.ragSettings.chunkSize.placeholder"
|
||||
)}
|
||||
placeholder={t("rag.ragSettings.chunkSize.placeholder")}
|
||||
/>
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
@ -316,16 +318,12 @@ export const SettingsBody = () => {
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: t(
|
||||
"rag.ragSettings.chunkOverlap.required"
|
||||
)
|
||||
message: t("rag.ragSettings.chunkOverlap.required")
|
||||
}
|
||||
]}>
|
||||
<InputNumber
|
||||
style={{ width: "100%" }}
|
||||
placeholder={t(
|
||||
"rag.ragSettings.chunkOverlap.placeholder"
|
||||
)}
|
||||
placeholder={t("rag.ragSettings.chunkOverlap.placeholder")}
|
||||
/>
|
||||
</Form.Item>
|
||||
|
||||
|
||||
@ -44,8 +44,16 @@ export const useMessage = () => {
|
||||
const { t } = useTranslation("option")
|
||||
const [selectedModel, setSelectedModel] = useStorage("selectedModel")
|
||||
const currentChatModelSettings = useStoreChatModelSettings()
|
||||
const { setIsSearchingInternet, webSearch, setWebSearch, isSearchingInternet } =
|
||||
useStoreMessageOption()
|
||||
const {
|
||||
setIsSearchingInternet,
|
||||
webSearch,
|
||||
setWebSearch,
|
||||
isSearchingInternet
|
||||
} = useStoreMessageOption()
|
||||
|
||||
|
||||
const [chatWithWebsiteEmbedding] = useStorage("chatWithWebsiteEmbedding", true)
|
||||
const [maxWebsiteContext] = useStorage("maxWebsiteContext", 4028)
|
||||
|
||||
const {
|
||||
history,
|
||||
@ -150,35 +158,32 @@ export const useMessage = () => {
|
||||
setMessages(newMessage)
|
||||
let fullText = ""
|
||||
let contentToSave = ""
|
||||
let isAlreadyExistEmbedding: MemoryVectorStore
|
||||
let embedURL: string, embedHTML: string, embedType: string
|
||||
let embedPDF: { content: string; page: number }[] = []
|
||||
|
||||
let isAlreadyExistEmbedding: MemoryVectorStore
|
||||
const {
|
||||
content: html,
|
||||
url: websiteUrl,
|
||||
type,
|
||||
pdf
|
||||
} = await getDataFromCurrentTab()
|
||||
|
||||
embedHTML = html
|
||||
embedURL = websiteUrl
|
||||
embedType = type
|
||||
embedPDF = pdf
|
||||
if (messages.length === 0) {
|
||||
const { content: html, url, type, pdf } = await getDataFromCurrentTab()
|
||||
embedHTML = html
|
||||
embedURL = url
|
||||
embedType = type
|
||||
embedPDF = pdf
|
||||
setCurrentURL(url)
|
||||
setCurrentURL(websiteUrl)
|
||||
isAlreadyExistEmbedding = keepTrackOfEmbedding[currentURL]
|
||||
} else {
|
||||
const { content: html, url, type, pdf } = await getDataFromCurrentTab()
|
||||
if (currentURL !== url) {
|
||||
embedHTML = html
|
||||
embedURL = url
|
||||
embedType = type
|
||||
embedPDF = pdf
|
||||
setCurrentURL(url)
|
||||
if (currentURL !== websiteUrl) {
|
||||
setCurrentURL(websiteUrl)
|
||||
} else {
|
||||
embedHTML = html
|
||||
embedURL = currentURL
|
||||
embedType = type
|
||||
embedPDF = pdf
|
||||
}
|
||||
isAlreadyExistEmbedding = keepTrackOfEmbedding[url]
|
||||
isAlreadyExistEmbedding = keepTrackOfEmbedding[websiteUrl]
|
||||
}
|
||||
|
||||
setMessages(newMessage)
|
||||
const ollamaUrl = await getOllamaURL()
|
||||
const embeddingModle = await defaultEmbeddingModelForRag()
|
||||
@ -196,20 +201,19 @@ export const useMessage = () => {
|
||||
try {
|
||||
if (isAlreadyExistEmbedding) {
|
||||
vectorstore = isAlreadyExistEmbedding
|
||||
console.log("Embedding already exist")
|
||||
} else {
|
||||
vectorstore = await memoryEmbedding({
|
||||
html: embedHTML,
|
||||
keepTrackOfEmbedding: keepTrackOfEmbedding,
|
||||
ollamaEmbedding: ollamaEmbedding,
|
||||
pdf: embedPDF,
|
||||
setIsEmbedding: setIsEmbedding,
|
||||
setKeepTrackOfEmbedding: setKeepTrackOfEmbedding,
|
||||
type: embedType,
|
||||
url: embedURL
|
||||
})
|
||||
|
||||
console.log("Embedding created")
|
||||
if (chatWithWebsiteEmbedding) {
|
||||
vectorstore = await memoryEmbedding({
|
||||
html: embedHTML,
|
||||
keepTrackOfEmbedding: keepTrackOfEmbedding,
|
||||
ollamaEmbedding: ollamaEmbedding,
|
||||
pdf: embedPDF,
|
||||
setIsEmbedding: setIsEmbedding,
|
||||
setKeepTrackOfEmbedding: setKeepTrackOfEmbedding,
|
||||
type: embedType,
|
||||
url: embedURL
|
||||
})
|
||||
}
|
||||
}
|
||||
let query = message
|
||||
const { ragPrompt: systemPrompt, ragQuestionPrompt: questionPrompt } =
|
||||
@ -247,25 +251,59 @@ export const useMessage = () => {
|
||||
query = response.content.toString()
|
||||
}
|
||||
|
||||
const docs = await vectorstore.similaritySearch(query, 4)
|
||||
const context = formatDocs(docs)
|
||||
const source = docs.map((doc) => {
|
||||
return {
|
||||
...doc,
|
||||
name: doc?.metadata?.source || "untitled",
|
||||
type: doc?.metadata?.type || "unknown",
|
||||
mode: "chat",
|
||||
url: ""
|
||||
let context: string = ""
|
||||
let source: {
|
||||
name: any
|
||||
type: any
|
||||
mode: string
|
||||
url: string
|
||||
pageContent: string
|
||||
metadata: Record<string, any>
|
||||
}[] = []
|
||||
|
||||
if (chatWithWebsiteEmbedding) {
|
||||
const docs = await vectorstore.similaritySearch(query, 4)
|
||||
context = formatDocs(docs)
|
||||
source = docs.map((doc) => {
|
||||
return {
|
||||
...doc,
|
||||
name: doc?.metadata?.source || "untitled",
|
||||
type: doc?.metadata?.type || "unknown",
|
||||
mode: "chat",
|
||||
url: ""
|
||||
}
|
||||
})
|
||||
} else {
|
||||
if (type === "html") {
|
||||
context = embedHTML.slice(0, maxWebsiteContext)
|
||||
} else {
|
||||
context = embedPDF
|
||||
.map((pdf) => pdf.content)
|
||||
.join(" ")
|
||||
.slice(0, maxWebsiteContext)
|
||||
}
|
||||
})
|
||||
// message = message.trim().replaceAll("\n", " ")
|
||||
|
||||
source = [
|
||||
{
|
||||
name: embedURL,
|
||||
type: type,
|
||||
mode: "chat",
|
||||
url: embedURL,
|
||||
pageContent: context,
|
||||
metadata: {
|
||||
source: embedURL,
|
||||
url: embedURL
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
let humanMessage = new HumanMessage({
|
||||
content: [
|
||||
{
|
||||
text: systemPrompt
|
||||
.replace("{context}", context)
|
||||
.replace("{question}", message),
|
||||
.replace("{question}", query),
|
||||
type: "text"
|
||||
}
|
||||
]
|
||||
@ -299,7 +337,7 @@ export const useMessage = () => {
|
||||
})
|
||||
count++
|
||||
}
|
||||
// update the message with the full text
|
||||
|
||||
setMessages((prev) => {
|
||||
return prev.map((message) => {
|
||||
if (message.id === generateMessageId) {
|
||||
@ -976,6 +1014,6 @@ export const useMessage = () => {
|
||||
regenerateLastMessage,
|
||||
webSearch,
|
||||
setWebSearch,
|
||||
isSearchingInternet,
|
||||
isSearchingInternet
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,10 +1,43 @@
|
||||
import * as cheerio from "cheerio"
|
||||
import TurndownService from "turndown"
|
||||
let turndownService = new TurndownService()
|
||||
import { Readability, isProbablyReaderable } from "@mozilla/readability"
|
||||
|
||||
export const defaultExtractContent = (html: string) => {
|
||||
|
||||
|
||||
const doc = new DOMParser().parseFromString(html, "text/html")
|
||||
if (isProbablyReaderable(doc)) {
|
||||
const reader = new Readability(doc)
|
||||
const article = reader.parse()
|
||||
const turndownService = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
codeBlockStyle: 'fenced'
|
||||
})
|
||||
return turndownService.turndown(article.content).trim()
|
||||
}
|
||||
|
||||
const $ = cheerio.load(html)
|
||||
const mainContent = $('[role="main"]').html() || $("main").html() || $.html()
|
||||
|
||||
$('script, style, link, svg, [src^="data:image/"]').remove()
|
||||
|
||||
$('*').each((_, element) => {
|
||||
if ('attribs' in element) {
|
||||
const attributes = element.attribs
|
||||
for (const attr in attributes) {
|
||||
if (attr !== 'href' && attr !== 'src') {
|
||||
$(element).removeAttr(attr)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
const mainContent = $('[role="main"]').html() || $("main").html() || $("body").html() || ""
|
||||
|
||||
const turndownService = new TurndownService({
|
||||
headingStyle: 'atx',
|
||||
codeBlockStyle: 'fenced'
|
||||
})
|
||||
const markdown = turndownService.turndown(mainContent)
|
||||
return markdown
|
||||
|
||||
return markdown.trim()
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { Readability } from "@mozilla/readability"
|
||||
import { Readability, } from "@mozilla/readability"
|
||||
import { defaultExtractContent } from "./default"
|
||||
export const extractReadabilityContent = async (url: string) => {
|
||||
const response = await fetch(url)
|
||||
|
||||
@ -98,3 +98,14 @@ export const getOpenOnRightClick = async (): Promise<string> => {
|
||||
export const setOpenOnRightClick = async (option: "webUI" | "sidePanel"): Promise<void> => {
|
||||
await storage.set("openOnRightClick", option);
|
||||
};
|
||||
|
||||
|
||||
export const getTotalFilePerKB = async (): Promise<number> => {
|
||||
const totalFilePerKB = await storage.get<number>("totalFilePerKB");
|
||||
return totalFilePerKB || 10;
|
||||
}
|
||||
|
||||
|
||||
export const setTotalFilePerKB = async (totalFilePerKB: number): Promise<void> => {
|
||||
await storage.set("totalFilePerKB", totalFilePerKB);
|
||||
};
|
||||
@ -2,6 +2,7 @@ import { Storage } from "@plasmohq/storage"
|
||||
import { cleanUrl } from "../libs/clean-url"
|
||||
import { urlRewriteRuntime } from "../libs/runtime"
|
||||
import { getChromeAIModel } from "./chrome"
|
||||
import { setTotalFilePerKB } from "./app"
|
||||
|
||||
const storage = new Storage()
|
||||
|
||||
@ -142,7 +143,7 @@ export const deleteModel = async (model: string) => {
|
||||
if (!response.ok) {
|
||||
throw new Error(response.statusText)
|
||||
}
|
||||
return response.json()
|
||||
return "ok"
|
||||
}
|
||||
|
||||
|
||||
@ -324,11 +325,13 @@ export const setDefaultEmbeddingChunkOverlap = async (overlap: number) => {
|
||||
export const saveForRag = async (
|
||||
model: string,
|
||||
chunkSize: number,
|
||||
overlap: number
|
||||
overlap: number,
|
||||
totalFilePerKB: number
|
||||
) => {
|
||||
await setDefaultEmbeddingModelForRag(model)
|
||||
await setDefaultEmbeddingChunkSize(chunkSize)
|
||||
await setDefaultEmbeddingChunkOverlap(overlap)
|
||||
await setTotalFilePerKB(totalFilePerKB)
|
||||
}
|
||||
|
||||
export const getWebSearchPrompt = async () => {
|
||||
|
||||
@ -50,7 +50,7 @@ export default defineConfig({
|
||||
outDir: "build",
|
||||
|
||||
manifest: {
|
||||
version: "1.1.15",
|
||||
version: "1.1.16",
|
||||
name:
|
||||
process.env.TARGET === "firefox"
|
||||
? "Page Assist - A Web UI for Local AI Models"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user