Merge pull request #153 from n4ze3m/next

v1.1.16

src/assets/locale/en/settings.json

@@ -31,6 +31,16 @@
         "label": "Send Notification After Finishing Processing the Knowledge Base"
       }
     },
+    "sidepanelRag": {
+      "heading": "Copilot Chat With Website Settings",
+      "ragEnabled": {
+        "label": "Chat with website using vector embeddings"
+      },
+      "maxWebsiteContext": {
+        "label": "Normal mode website content size",
+        "placeholder": "Content size (default 4028)"
+      }
+    },
     "webSearch": {
       "heading": "Manage Web Search",
       "searchMode": {
@@ -288,6 +298,11 @@
         "label": "Chunk Overlap",
         "placeholder": "Enter Chunk Overlap",
         "required": "Please enter a chunk overlap"
+      },
+      "totalFilePerKB": {
+        "label": "Knowledge Base Default File Limit",
+        "placeholder": "Enter default file limit (e.g., 10)",
+        "required": "Please enter the default file limit"
       }
     },
     "prompt": {

src/assets/locale/es/settings.json

@@ -31,6 +31,16 @@
         "label": "Enviar notificación después de terminar el procesamiento de la base de conocimientos"
       }
     },
+    "sidepanelRag": {
+      "heading": "Configuración del Chat Copilot con Sitio Web",
+      "ragEnabled": {
+        "label": "Chatear con el sitio web usando incrustaciones vectoriales"
+      },
+      "maxWebsiteContext": {
+        "label": "Tamaño del contenido del sitio web en modo normal",
+        "placeholder": "Tamaño del contenido (predeterminado 4028)"
+      }
+    },
     "webSearch": {
       "heading": "Manejo de la busqueda Web",
       "searchMode": {
@@ -288,6 +298,11 @@
         "label": "Solapamiento del Chunk",
         "placeholder": "Ingrese el solapamiento del chunk",
         "required": "Por favor, ingresar el solapamiento del chunk"
+      },
+      "totalFilePerKB": {
+        "label": "Límite predeterminado de archivos de la base de conocimientos",
+        "placeholder": "Ingrese el límite predeterminado de archivos (ej. 10)",
+        "required": "Por favor, ingrese el límite predeterminado de archivos"
       }
     },
     "prompt": {

src/assets/locale/fr/settings.json

@@ -31,6 +31,16 @@
         "label": "Envoyer une notification après avoir terminé le traitement de la base de connaissances"
       }
     },
+    "sidepanelRag": {
+      "heading": "Paramètres du Chat Copilot avec le Site Web",
+      "ragEnabled": {
+        "label": "Discuter avec le site web en utilisant des embeddings vectoriels"
+      },
+      "maxWebsiteContext": {
+        "label": "Taille du contenu du site web en mode normal",
+        "placeholder": "Taille du contenu (par défaut 4028)"
+      }
+    },
     "webSearch": {
       "heading": "Gérer la recherche Web",
       "searchMode": {
@@ -288,6 +298,11 @@
         "label": "Chevauchement",
         "placeholder": "Entrez le chevauchement des morceaux",
         "required": "Veuillez saisir un chevauchement"
+      },
+      "totalFilePerKB": {
+        "label": "Limite par défaut de fichiers de la base de connaissances",
+        "placeholder": "Entrez la limite par défaut de fichiers (ex. 10)",
+        "required": "Veuillez entrer la limite par défaut de fichiers"
       }
     },
     "prompt": {

src/assets/locale/it/settings.json

@@ -31,6 +31,16 @@
         "label": "Inviare notifica dopo aver terminato l'elaborazione della base di conoscenza"
       }
     },
+    "sidepanelRag": {
+      "heading": "Impostazioni Chat Copilot con Sito Web",
+      "ragEnabled": {
+        "label": "Chatta con il sito web utilizzando embedding vettoriali"
+      },
+      "maxWebsiteContext": {
+        "label": "Dimensione del contenuto del sito web in modalità normale",
+        "placeholder": "Dimensione del contenuto (predefinito 4028)"
+      }
+    },
     "webSearch": {
       "heading": "Gestione ricerca Web",
       "searchMode": {
@@ -288,6 +298,11 @@
         "label": "Sovrapposizione del Blocco (Chunk Overlap)",
         "placeholder": "Inserisci la Sovrapposizione del Blocco (Chunk Overlap)",
         "required": "Inserisci la Sovrapposizione del Blocco"
+      },
+      "totalFilePerKB": {
+        "label": "Limite predefinito di file della base di conoscenza",
+        "placeholder": "Inserisci il limite predefinito di file (es. 10)",
+        "required": "Inserisci il limite predefinito di file"
       }
     },
     "prompt": {

src/assets/locale/ja-JP/settings.json

@@ -34,6 +34,16 @@
         "label": "ナレッジベースの処理完了後に通知を送信"
       }
     },
+    "sidepanelRag": {
+      "heading": "ウェブサイトとのCopilotチャット設定",
+      "ragEnabled": {
+        "label": "ベクトル埋め込みを使用してウェブサイトとチャットする"
+      },
+      "maxWebsiteContext": {
+        "label": "通常モードのウェブサイトコンテンツサイズ",
+        "placeholder": "コンテンツサイズ（デフォルト4028）"
+      }
+    },
     "webSearch": {
       "heading": "ウェブ検索を管理する",
       "searchMode": {
@@ -291,6 +301,11 @@
         "label": "チャンクオーバーラップ",
         "placeholder": "チャンクオーバーラップを入力",
         "required": "チャンクオーバーラップを入力してください"
+      },
+      "totalFilePerKB": {
+        "label": "ナレッジベースのデフォルトファイル制限",
+        "placeholder": "デフォルトのファイル制限を入力してください（例：10）",
+        "required": "デフォルトのファイル制限を入力してください"
       }
     },
     "prompt": {

src/assets/locale/ml/settings.json

@@ -34,6 +34,16 @@
         "label": "അറിവ് ശേഖരം പ്രോസസ്സ് ചെയ്ത് കഴിഞ്ഞതിന് ശേഷം അറിയിപ്പ് അയയ്ക്കുക"
       }
     },
+    "sidepanelRag": {
+      "heading": "വെബ്‌സൈറ്റുമായുള്ള കോപൈലറ്റ് ചാറ്റ് ക്രമീകരണങ്ങൾ",
+      "ragEnabled": {
+        "label": "വെക്ടർ എംബെഡിംഗുകൾ ഉപയോഗിച്ച് വെബ്‌സൈറ്റുമായി ചാറ്റ് ചെയ്യുക"
+      },
+      "maxWebsiteContext": {
+        "label": "സാധാരണ മോഡിലെ വെബ്‌സൈറ്റ് ഉള്ളടക്ക വലുപ്പം",
+        "placeholder": "ഉള്ളടക്ക വലുപ്പം (സ്ഥിരസ്ഥിതി 4028)"
+      }
+    },
     "webSearch": {
       "heading": "വെബ്ബ് തിരച്ചിൽ നിയന്ത്രിക്കുക",
       "searchMode": {
@@ -291,6 +301,11 @@
         "label": "ചങ്ക് ഓവര്‍ലാപ്പ്",
         "placeholder": "ചങ്ക് ഓവര്‍ലാപ്പ് നല്കുക",
         "required": "ദയവായി ചങ്ക് ഓവര്‍ലാപ്പ് നല്കുക"
+      },
+      "totalFilePerKB": {
+        "label": "Limite padrão de arquivos da base de conhecimento",
+        "placeholder": "Digite o limite padrão de arquivos (ex. 10)",
+        "required": "Por favor, digite o limite padrão de arquivos"
       }
     },
     "prompt": {

src/assets/locale/pt-BR/settings.json

@@ -31,6 +31,16 @@
         "label": "Enviar notificação após concluir o processamento da base de conhecimento"
       }
     },
+    "sidepanelRag": {
+      "heading": "Configurações de Chat Copilot com o Site",
+      "ragEnabled": {
+        "label": "Conversar com o site usando embeddings vetoriais"
+      },
+      "maxWebsiteContext": {
+        "label": "Tamanho do conteúdo do site no modo normal",
+        "placeholder": "Tamanho do conteúdo (padrão 4028)"
+      }
+    },
     "webSearch": {
       "heading": "Gerenciar Pesquisa na Web",
       "searchMode": {
@@ -288,6 +298,11 @@
         "label": "Sobreposição do Pedaço",
         "placeholder": "Digite a Sobreposição do Pedaço",
         "required": "Por favor, insira uma sobreposição de pedaço"
+      },
+      "totalFilePerKB": {
+        "label": "Limite padrão de arquivos da base de conhecimento",
+        "placeholder": "Digite o limite padrão de arquivos (ex. 10)",
+        "required": "Por favor, digite o limite padrão de arquivos"
       }
     },
     "prompt": {

src/assets/locale/ru/settings.json

@@ -31,6 +31,17 @@
         "label": "Отправить уведомление после завершения обработки базы знаний"
       }
     },
+    "sidepanelRag": {
+      "heading": "Настройки чата Copilot с веб-сайтом",
+      "ragEnabled": {
+        "label": "Общаться с веб-сайтом, используя векторные вложения"
+      },
+      "maxWebsiteContext": {
+        "label": "Размер содержимого веб-сайта в обычном режиме",
+        "placeholder": "Размер содержимого (по умолчанию 4028)"
+      }
+    },
+
     "webSearch": {
       "heading": "Управление веб-поиском",
       "searchMode": {
@@ -289,6 +300,11 @@
         "label": "Перекрытие фрагментов",
         "placeholder": "Введите перекрытие фрагментов",
         "required": "Пожалуйста, введите перекрытие фрагментов"
+      },
+      "totalFilePerKB": {
+        "label": "Стандартный лимит файлов базы знаний",
+        "placeholder": "Введите стандартный лимит файлов (напр. 10)",
+        "required": "Пожалуйста, введите стандартный лимит файлов"
       }
     },
     "prompt": {

src/assets/locale/zh/settings.json

@@ -34,6 +34,16 @@
         "label": "完成知识库处理后发送通知"
       }
     },
+    "sidepanelRag": {
+      "heading": "与网站对话的Copilot聊天设置",
+      "ragEnabled": {
+        "label": "使用向量嵌入与网站聊天"
+      },
+      "maxWebsiteContext": {
+        "label": "普通模式下的网站内容大小",
+        "placeholder": "内容大小（默认4028）"
+      }
+    },
     "webSearch": {
       "heading": "管理网络搜索",
       "searchMode": {
@@ -293,6 +303,11 @@
         "label": "嵌入重叠",
         "placeholder": "256-∞",
         "required": "请输入嵌入重叠"
+      },
+      "totalFilePerKB": {
+        "label": "知识库默认文件限制",
+        "placeholder": "输入默认文件限制（例如：10）",
+        "required": "请输入默认文件限制"
       }
     },
     "prompt": {

src/components/Option/Knowledge/AddKnowledge.tsx

@@ -7,6 +7,7 @@ import { InboxIcon } from "lucide-react"
 import { useTranslation } from "react-i18next"
 import PubSub from "pubsub-js"
 import { KNOWLEDGE_QUEUE } from "@/queue"
+import { useStorage } from "@plasmohq/storage/hook"
 
 type Props = {
   open: boolean
@@ -16,6 +17,7 @@ type Props = {
 export const AddKnowledge = ({ open, setOpen }: Props) => {
   const { t } = useTranslation(["knowledge", "common"])
   const [form] = Form.useForm()
+  const [totalFilePerKB] = useStorage("totalFilePerKB", 10)
 
   const onUploadHandler = async (data: {
     title: string
@@ -92,7 +94,7 @@ export const AddKnowledge = ({ open, setOpen }: Props) => {
           <Upload.Dragger
             accept={".pdf, .csv, .txt, .md, .docx"}
             multiple={true}
-            maxCount={10}
+            maxCount={totalFilePerKB}
             beforeUpload={(file) => {
               const allowedTypes = [
                 "application/pdf",

src/components/Option/Knowledge/SelectedKnwledge.tsx

@@ -17,9 +17,9 @@ export const SelectedKnowledge = () => {
         <Tooltip
         title={knowledge.title}
         >
-        <div className="inline-flex truncate items-center gap-2">
+        <div className="inline-flex items-center gap-2 max-w-[150px]">
-          <Blocks className="h-5  w-5 text-gray-400" />
+          <Blocks className="h-5  w-5 text-gray-400 flex-shrink-0" />
-          <span className="text-xs hidden lg:inline-block  font-semibold dark:text-gray-100">
+          <span className="text-xs hidden lg:inline-block font-semibold dark:text-gray-100 truncate">
             {knowledge.title}
           </span>
         </div>

src/components/Option/Prompt/index.tsx

@@ -126,30 +126,31 @@ export const PromptBody = () => {
               {
                 title: t("managePrompts.columns.title"),
                 dataIndex: "title",
-                key: "title"
+                key: "title",
+                render: (content) => (<span className="line-clamp-1">{content}</span>)
               },
               {
                 title: t("managePrompts.columns.prompt"),
                 dataIndex: "content",
-                key: "content"
+                key: "content",
+                render: (content) => (<span className="line-clamp-1">{content}</span>)
               },
               {
                 title: t("managePrompts.columns.type"),
                 dataIndex: "is_system",
                 key: "is_system",
                 render: (is_system) =>
-                  is_system ? (
+                  <span className="flex items-center gap-2 text-xs w-32">
-                    <span className="flex items-center gap-2">
+                    {is_system ? (
-                      <Computer className="w-5 h-5 " />
+                      <>
-                      {t("managePrompts.systemPrompt")}
+                        <Computer className="size-4" /> {t("managePrompts.systemPrompt")}
-                    </span>
+                      </>
                     ) : (
-                    <span className="flex items-center gap-2">
+                      <>
-                      <Zap className="w-5 h-5" />
+                        <Zap className="size-4" /> {t("managePrompts.quickPrompt")}
-                      {t("managePrompts.quickPrompt")}
+                      </>
-                    </span>
+                    )}
-                  )
+                  </span>              },
-              },
               {
                 title: t("managePrompts.columns.actions"),
                 render: (_, record) => (
@@ -164,7 +165,7 @@ export const PromptBody = () => {
                           }
                         }}
                         className="text-red-500 dark:text-red-400">
-                        <Trash2 className="w-5 h-5" />
+                        <Trash2 className="size-4" />
                       </button>
                     </Tooltip>
                     <Tooltip title={t("managePrompts.tooltip.edit")}>
@@ -175,7 +176,7 @@ export const PromptBody = () => {
                           setOpenEdit(true)
                         }}
                         className="text-gray-500 dark:text-gray-400">
-                        <Pen className="w-5 h-5" />
+                        <Pen className="size-4" />
                       </button>
                     </Tooltip>
                   </div>

src/components/Option/Settings.tsx

@@ -1,39 +0,0 @@
-import { Tabs } from "antd"
-import { SettingsOllama } from "./Settings/ollama"
-import { SettingPrompt } from "./Settings/prompt"
-import { SettingOther } from "./Settings/general-settings"
-
-type Props = {
-  setClose: (close: boolean) => void
-}
-
-export const Settings = ({ setClose }: Props) => {
-  return (
-    <div className="my-6 max-h-[80vh] overflow-y-auto">
-      <Tabs
-        tabPosition="left"
-        defaultActiveKey="1"
-        items={[
-          {
-            id: "1",
-            key: "1",
-            label: "Prompt",
-            children: <SettingPrompt />
-          },
-          {
-            id: "2",
-            key: "2",
-            label: "Web UI Settings",
-            children: <SettingOther />
-          },
-          {
-            id: "3",
-            key: "3",
-            label: "Ollama Settings",
-            children: <SettingsOllama />
-          }
-        ]}
-      />
-    </div>
-  )
-}

src/components/Option/Settings/rag.tsx

@@ -10,6 +10,8 @@ import {
 } from "~/services/ollama"
 import { SettingPrompt } from "./prompt"
 import { useTranslation } from "react-i18next"
+import { getTotalFilePerKB } from "@/services/app"
+import { SidepanelRag } from "./sidepanel-rag"
 
 export const RagSettings = () => {
   const { t } = useTranslation("settings")
@@ -19,19 +21,20 @@ export const RagSettings = () => {
   const { data: ollamaInfo, status } = useQuery({
     queryKey: ["fetchRAGSettings"],
     queryFn: async () => {
-      const [allModels, chunkOverlap, chunkSize, defaultEM] = await Promise.all(
+      const [allModels, chunkOverlap, chunkSize, defaultEM, totalFilePerKB] =
-        [
+        await Promise.all([
           getAllModels({ returnEmpty: true }),
           defaultEmbeddingChunkOverlap(),
           defaultEmbeddingChunkSize(),
-          defaultEmbeddingModelForRag()
+          defaultEmbeddingModelForRag(),
-        ]
+          getTotalFilePerKB()
-      )
+        ])
       return {
         models: allModels,
         chunkOverlap,
         chunkSize,
-        defaultEM
+        defaultEM,
+        totalFilePerKB
       }
     }
   })
@@ -41,8 +44,9 @@ export const RagSettings = () => {
       model: string
       chunkSize: number
       overlap: number
+      totalFilePerKB: number
     }) => {
-      await saveForRag(data.model, data.chunkSize, data.overlap)
+      await saveForRag(data.model, data.chunkSize, data.overlap,  data.totalFilePerKB)
       return true
     },
     onSuccess: () => {
@@ -70,13 +74,15 @@ export const RagSettings = () => {
                 saveRAG({
                   model: data.defaultEM,
                   chunkSize: data.chunkSize,
-                  overlap: data.chunkOverlap
+                  overlap: data.chunkOverlap,
+                  totalFilePerKB: data.totalFilePerKB
                 })
               }}
               initialValues={{
                 chunkSize: ollamaInfo?.chunkSize,
                 chunkOverlap: ollamaInfo?.chunkOverlap,
-                defaultEM: ollamaInfo?.defaultEM
+                defaultEM: ollamaInfo?.defaultEM,
+                totalFilePerKB: ollamaInfo?.totalFilePerKB
               }}>
               <Form.Item
                 name="defaultEM"
@@ -85,9 +91,7 @@ export const RagSettings = () => {
                 rules={[
                   {
                     required: true,
-                    message: t(
+                    message: t("rag.ragSettings.model.required")
-                      "rag.ragSettings.model.required"
-                    )
                   }
                 ]}>
                 <Select
@@ -99,9 +103,7 @@ export const RagSettings = () => {
                       0
                   }
                   showSearch
-                  placeholder={t(
+                  placeholder={t("rag.ragSettings.model.placeholder")}
-                    "rag.ragSettings.model.placeholder"
-                  )}
                   style={{ width: "100%" }}
                   className="mt-4"
                   options={ollamaInfo.models?.map((model) => ({
@@ -117,45 +119,53 @@ export const RagSettings = () => {
                 rules={[
                   {
                     required: true,
-                    message: t(
+                    message: t("rag.ragSettings.chunkSize.required")
-                      "rag.ragSettings.chunkSize.required"
-                    )
                   }
                 ]}>
                 <InputNumber
                   style={{ width: "100%" }}
-                  placeholder={t(
+                  placeholder={t("rag.ragSettings.chunkSize.placeholder")}
-                    "rag.ragSettings.chunkSize.placeholder"
-                  )}
                 />
               </Form.Item>
               <Form.Item
                 name="chunkOverlap"
-                label={t(
+                label={t("rag.ragSettings.chunkOverlap.label")}
-                  "rag.ragSettings.chunkOverlap.label"
-                )}
                 rules={[
                   {
                     required: true,
-                    message: t(
+                    message: t("rag.ragSettings.chunkOverlap.required")
-                      "rag.ragSettings.chunkOverlap.required"
-                    )
                   }
                 ]}>
                 <InputNumber
                   style={{ width: "100%" }}
-                  placeholder={t(
+                  placeholder={t("rag.ragSettings.chunkOverlap.placeholder")}
-                    "rag.ragSettings.chunkOverlap.placeholder"
-                  )}
                 />
               </Form.Item>
 
+              <Form.Item
+                name="totalFilePerKB"
+                label={t("rag.ragSettings.totalFilePerKB.label")}
+                rules={[
+                  {
+                    required: true,
+                    message: t("rag.ragSettings.totalFilePerKB.required")
+                  }
+                ]}>
+                <InputNumber
+                  style={{ width: "100%" }}
+                  placeholder={t("rag.ragSettings.totalFilePerKB.placeholder")}
+                />
+              </Form.Item>
+
+
               <div className="flex justify-end">
                 <SaveButton disabled={isSaveRAGPending} btnType="submit" />
               </div>
             </Form>
           </div>
 
+          <SidepanelRag  />
+
           <div>
             <div>
               <h2 className="text-base font-semibold leading-7 text-gray-900 dark:text-white">

src/components/Option/Settings/sidepanel-rag.tsx

@@ -0,0 +1,63 @@
+import { useStorage } from "@plasmohq/storage/hook"
+import { InputNumber, Switch } from "antd"
+import { useTranslation } from "react-i18next"
+
+export const SidepanelRag = ({ hideBorder }: { hideBorder?: boolean }) => {
+  const { t } = useTranslation("settings")
+  const [chatWithWebsiteEmbedding, setChatWithWebsiteEmbedding] = useStorage(
+    "chatWithWebsiteEmbedding",
+    true
+  )
+  const [maxWebsiteContext, setMaxWebsiteContext] = useStorage(
+    "maxWebsiteContext",
+    4028
+  )
+
+  return (
+    <div>
+      <div className="mb-5">
+        <h2
+          className={`${
+            !hideBorder ? "text-base font-semibold leading-7" : "text-md"
+          } text-gray-900 dark:text-white`}>
+          {t("generalSettings.sidepanelRag.heading")}
+        </h2>
+        {!hideBorder && (
+          <div className="border border-b border-gray-200 dark:border-gray-600 mt-3"></div>
+        )}
+      </div>
+      <div className={`${
+        !hideBorder ? "text-sm" : ""
+      } space-y-4`}>
+        <div className="flex flex-col sm:flex-row sm:items-center sm:justify-between space-y-4 sm:space-y-0">
+          <span className="text-gray-700  truncate dark:text-neutral-50">
+            {t("generalSettings.sidepanelRag.ragEnabled.label")}
+          </span>
+          <div>
+            <Switch
+              className="mt-4 sm:mt-0"
+              checked={chatWithWebsiteEmbedding}
+              onChange={(checked) => setChatWithWebsiteEmbedding(checked)}
+            />
+          </div>
+        </div>
+        <div className="flex flex-col sm:flex-row sm:items-center sm:justify-between space-y-4 sm:space-y-0">
+          <span className="text-gray-700 truncate dark:text-neutral-50">
+            {t("generalSettings.sidepanelRag.maxWebsiteContext.label")}
+          </span>
+          <div>
+            <InputNumber
+              disabled={chatWithWebsiteEmbedding}
+              className="mt-4 sm:mt-0"
+              value={maxWebsiteContext}
+              onChange={(value) => setMaxWebsiteContext(value)}
+              placeholder={t(
+                "generalSettings.sidepanelRag.maxWebsiteContext.placeholder"
+              )}
+            />
+          </div>
+        </div>
+      </div>
+    </div>
+  )
+}

src/components/Option/Settings/tts-mode.tsx

@@ -35,7 +35,10 @@ export const TTSModeSettings = ({ hideBorder }: { hideBorder?: boolean }) => {
   return (
     <div>
       <div className="mb-5">
-        <h2 className="text-base font-semibold leading-7 text-gray-900 dark:text-white">
+        <h2
+          className={`${
+            !hideBorder ? "text-base font-semibold leading-7" : "text-md"
+          } text-gray-900 dark:text-white`}>
           {t("generalSettings.tts.heading")}
         </h2>
         {!hideBorder && (

src/components/Sidepanel/Settings/body.tsx

@@ -33,6 +33,8 @@ import { useI18n } from "@/hooks/useI18n"
 import { TTSModeSettings } from "@/components/Option/Settings/tts-mode"
 import { AdvanceOllamaSettings } from "@/components/Common/Settings/AdvanceOllamaSettings"
 import { useStorage } from "@plasmohq/storage/hook"
+import { getTotalFilePerKB } from "@/services/app"
+import { SidepanelRag } from "@/components/Option/Settings/sidepanel-rag"
 
 export const SettingsBody = () => {
   const { t } = useTranslation("settings")
@@ -66,7 +68,8 @@ export const SettingsBody = () => {
         allModels,
         chunkOverlap,
         chunkSize,
-        defaultEM
+        defaultEM,
+        totalFilePerKB
       ] = await Promise.all([
         getOllamaURL(),
         systemPromptForNonRag(),
@@ -74,7 +77,8 @@ export const SettingsBody = () => {
         getAllModels({ returnEmpty: true }),
         defaultEmbeddingChunkOverlap(),
         defaultEmbeddingChunkSize(),
-        defaultEmbeddingModelForRag()
+        defaultEmbeddingModelForRag(),
+        getTotalFilePerKB()
       ])
 
       return {
@@ -85,18 +89,19 @@ export const SettingsBody = () => {
         models: allModels,
         chunkOverlap,
         chunkSize,
-        defaultEM
+        defaultEM,
+        totalFilePerKB
       }
     }
   })
 
   const { mutate: saveRAG, isPending: isSaveRAGPending } = useMutation({
-    mutationFn: async (data: {
+    mutationFn: async (f: {
       model: string
       chunkSize: number
       overlap: number
     }) => {
-      await saveForRag(data.model, data.chunkSize, data.overlap)
+      await saveForRag(f.model, f.chunkSize, f.overlap, data.totalFilePerKB)
     }
   })
 
@@ -196,7 +201,9 @@ export const SettingsBody = () => {
           </div>
         )}
       </div>
-
+      <div className="border border-gray-300 dark:border-gray-700 rounded p-4 bg-white dark:bg-[#171717]">
+        <SidepanelRag hideBorder />
+      </div>
       <div className="border flex flex-col gap-4 border-gray-300 dark:border-gray-700 rounded p-4 bg-white dark:bg-[#171717]">
         <h2 className="text-md font-semibold dark:text-white">
           {t("ollamaSettings.heading")}
@@ -247,7 +254,6 @@ export const SettingsBody = () => {
           />
         </div>
       </div>
-
       <div className="border border-gray-300 dark:border-gray-700 rounded p-4 bg-white dark:bg-[#171717]">
         <h2 className="text-md mb-4 font-semibold dark:text-white">
           {t("rag.ragSettings.label")}
@@ -298,16 +304,12 @@ export const SettingsBody = () => {
             rules={[
               {
                 required: true,
-                message: t(
+                message: t("rag.ragSettings.chunkSize.required")
-                  "rag.ragSettings.chunkSize.required"
-                )
               }
             ]}>
             <InputNumber
               style={{ width: "100%" }}
-              placeholder={t(
+              placeholder={t("rag.ragSettings.chunkSize.placeholder")}
-                "rag.ragSettings.chunkSize.placeholder"
-              )}
             />
           </Form.Item>
           <Form.Item
@@ -316,16 +318,12 @@ export const SettingsBody = () => {
             rules={[
               {
                 required: true,
-                message: t(
+                message: t("rag.ragSettings.chunkOverlap.required")
-                  "rag.ragSettings.chunkOverlap.required"
-                )
               }
             ]}>
             <InputNumber
               style={{ width: "100%" }}
-              placeholder={t(
+              placeholder={t("rag.ragSettings.chunkOverlap.placeholder")}
-                "rag.ragSettings.chunkOverlap.placeholder"
-              )}
             />
           </Form.Item>
 

src/hooks/useMessage.tsx

@@ -44,8 +44,16 @@ export const useMessage = () => {
   const { t } = useTranslation("option")
   const [selectedModel, setSelectedModel] = useStorage("selectedModel")
   const currentChatModelSettings = useStoreChatModelSettings()
-  const { setIsSearchingInternet, webSearch, setWebSearch, isSearchingInternet } =
+  const {
-    useStoreMessageOption()
+    setIsSearchingInternet,
+    webSearch,
+    setWebSearch,
+    isSearchingInternet
+  } = useStoreMessageOption()
+
+
+  const [chatWithWebsiteEmbedding] = useStorage("chatWithWebsiteEmbedding", true)
+  const [maxWebsiteContext] = useStorage("maxWebsiteContext", 4028)
 
   const {
     history,
@@ -150,35 +158,32 @@ export const useMessage = () => {
     setMessages(newMessage)
     let fullText = ""
     let contentToSave = ""
-    let isAlreadyExistEmbedding: MemoryVectorStore
     let embedURL: string, embedHTML: string, embedType: string
     let embedPDF: { content: string; page: number }[] = []
 
-    if (messages.length === 0) {
+    let isAlreadyExistEmbedding: MemoryVectorStore
-      const { content: html, url, type, pdf } = await getDataFromCurrentTab()
+    const {
+      content: html,
+      url: websiteUrl,
+      type,
+      pdf
+    } = await getDataFromCurrentTab()
+
     embedHTML = html
-      embedURL = url
+    embedURL = websiteUrl
     embedType = type
     embedPDF = pdf
-      setCurrentURL(url)
+    if (messages.length === 0) {
+      setCurrentURL(websiteUrl)
       isAlreadyExistEmbedding = keepTrackOfEmbedding[currentURL]
     } else {
-      const { content: html, url, type, pdf } = await getDataFromCurrentTab()
+      if (currentURL !== websiteUrl) {
-      if (currentURL !== url) {
+        setCurrentURL(websiteUrl)
-        embedHTML = html
-        embedURL = url
-        embedType = type
-        embedPDF = pdf
-        setCurrentURL(url)
       } else {
-        embedHTML = html
         embedURL = currentURL
-        embedType = type
-        embedPDF = pdf
       }
-      isAlreadyExistEmbedding = keepTrackOfEmbedding[url]
+      isAlreadyExistEmbedding = keepTrackOfEmbedding[websiteUrl]
     }
-
     setMessages(newMessage)
     const ollamaUrl = await getOllamaURL()
     const embeddingModle = await defaultEmbeddingModelForRag()
@@ -196,8 +201,8 @@ export const useMessage = () => {
     try {
       if (isAlreadyExistEmbedding) {
         vectorstore = isAlreadyExistEmbedding
-        console.log("Embedding already exist")
       } else {
+        if (chatWithWebsiteEmbedding) {
           vectorstore = await memoryEmbedding({
             html: embedHTML,
             keepTrackOfEmbedding: keepTrackOfEmbedding,
@@ -208,8 +213,7 @@ export const useMessage = () => {
             type: embedType,
             url: embedURL
           })
-
+        }
-        console.log("Embedding created")
       }
       let query = message
       const { ragPrompt: systemPrompt, ragQuestionPrompt: questionPrompt } =
@@ -247,9 +251,20 @@ export const useMessage = () => {
         query = response.content.toString()
       }
 
+      let context: string = ""
+      let source: {
+        name: any
+        type: any
+        mode: string
+        url: string
+        pageContent: string
+        metadata: Record<string, any>
+      }[] = []
+
+      if (chatWithWebsiteEmbedding) {
         const docs = await vectorstore.similaritySearch(query, 4)
-      const context = formatDocs(docs)
+        context = formatDocs(docs)
-      const source = docs.map((doc) => {
+        source = docs.map((doc) => {
           return {
             ...doc,
             name: doc?.metadata?.source || "untitled",
@@ -258,14 +273,37 @@ export const useMessage = () => {
             url: ""
           }
         })
-      // message = message.trim().replaceAll("\n", " ")
+      } else {
+        if (type === "html") {
+          context = embedHTML.slice(0, maxWebsiteContext)
+        } else {
+          context = embedPDF
+            .map((pdf) => pdf.content)
+            .join(" ")
+            .slice(0, maxWebsiteContext)
+        }
+
+        source = [
+          {
+            name: embedURL,
+            type: type,
+            mode: "chat",
+            url: embedURL,
+            pageContent: context,
+            metadata: {
+              source: embedURL,
+              url: embedURL
+            }
+          }
+        ]
+      }
 
       let humanMessage = new HumanMessage({
         content: [
           {
             text: systemPrompt
               .replace("{context}", context)
-              .replace("{question}", message),
+              .replace("{question}", query),
             type: "text"
           }
         ]
@@ -299,7 +337,7 @@ export const useMessage = () => {
         })
         count++
       }
-      // update the message with the full text
+
       setMessages((prev) => {
         return prev.map((message) => {
           if (message.id === generateMessageId) {
@@ -976,6 +1014,6 @@ export const useMessage = () => {
     regenerateLastMessage,
     webSearch,
     setWebSearch,
-    isSearchingInternet,
+    isSearchingInternet
   }
 }

src/parser/default.ts

@@ -1,10 +1,43 @@
 import * as cheerio from "cheerio"
 import TurndownService from "turndown"
-let turndownService = new TurndownService()
+import { Readability, isProbablyReaderable } from "@mozilla/readability"
 
 export const defaultExtractContent = (html: string) => {
-  const $ = cheerio.load(html)
+
-  const mainContent = $('[role="main"]').html() || $("main").html() || $.html()
+
-  const markdown = turndownService.turndown(mainContent)
+  const doc = new DOMParser().parseFromString(html, "text/html")
-  return markdown
+  if (isProbablyReaderable(doc)) {
+    const reader = new Readability(doc)
+    const article = reader.parse()
+    const turndownService = new TurndownService({
+      headingStyle: 'atx',
+      codeBlockStyle: 'fenced'
+    })
+    return turndownService.turndown(article.content).trim()
+  }
+
+  const $ = cheerio.load(html)
+
+  $('script, style, link, svg, [src^="data:image/"]').remove()
+
+  $('*').each((_, element) => {
+    if ('attribs' in element) {
+      const attributes = element.attribs
+      for (const attr in attributes) {
+        if (attr !== 'href' && attr !== 'src') {
+          $(element).removeAttr(attr)
+        }
+      }
+    }
+  })
+
+  const mainContent = $('[role="main"]').html() || $("main").html() || $("body").html() || ""
+
+  const turndownService = new TurndownService({
+    headingStyle: 'atx',
+    codeBlockStyle: 'fenced'
+  })
+  const markdown = turndownService.turndown(mainContent)
+
+  return markdown.trim()
 }

src/parser/reader.ts

@@ -1,4 +1,4 @@
-import { Readability } from "@mozilla/readability"
+import { Readability, } from "@mozilla/readability"
 import { defaultExtractContent } from "./default"
 export const extractReadabilityContent = async (url: string) => {
     const response = await fetch(url)

src/services/app.ts

@@ -98,3 +98,14 @@ export const getOpenOnRightClick = async (): Promise<string> => {
 export const setOpenOnRightClick = async (option: "webUI" | "sidePanel"): Promise<void> => {
   await storage.set("openOnRightClick", option);
 };
+
+
+export const getTotalFilePerKB = async (): Promise<number> => {
+  const totalFilePerKB = await storage.get<number>("totalFilePerKB");
+  return totalFilePerKB || 10;
+}
+
+
+export const setTotalFilePerKB = async (totalFilePerKB: number): Promise<void> => {
+  await storage.set("totalFilePerKB", totalFilePerKB);
+};

src/services/ollama.ts

@@ -2,6 +2,7 @@ import { Storage } from "@plasmohq/storage"
 import { cleanUrl } from "../libs/clean-url"
 import { urlRewriteRuntime } from "../libs/runtime"
 import { getChromeAIModel } from "./chrome"
+import { setTotalFilePerKB } from "./app"
 
 const storage = new Storage()
 
@@ -142,7 +143,7 @@ export const deleteModel = async (model: string) => {
   if (!response.ok) {
     throw new Error(response.statusText)
   }
-  return response.json()
+  return "ok"
 }
 
 
@@ -324,11 +325,13 @@ export const setDefaultEmbeddingChunkOverlap = async (overlap: number) => {
 export const saveForRag = async (
   model: string,
   chunkSize: number,
-  overlap: number
+  overlap: number,
+  totalFilePerKB: number
 ) => {
   await setDefaultEmbeddingModelForRag(model)
   await setDefaultEmbeddingChunkSize(chunkSize)
   await setDefaultEmbeddingChunkOverlap(overlap)
+  await setTotalFilePerKB(totalFilePerKB)
 }
 
 export const getWebSearchPrompt = async () => {

wxt.config.ts

@@ -50,7 +50,7 @@ export default defineConfig({
   outDir: "build",
 
   manifest: {
-    version: "1.1.15",
+    version: "1.1.16",
     name:
       process.env.TARGET === "firefox"
         ? "Page Assist - A Web UI for Local AI Models"