iod/page-assist
7
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ability to update knowledge base default file limit

Browse Source
This commit is contained in:
n4ze3m 2024-07-21 20:13:37 +05:30
parent 943813f759
commit af09c8eed6
14 changed files with 101 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/assets/locale/en/settings.json
View File

@ -288,6 +288,11 @@
"label": "Chunk Overlap",
"placeholder": "Enter Chunk Overlap",
"required": "Please enter a chunk overlap"
},
"totalFilePerKB": {
"label": "Knowledge Base Default File Limit",
"placeholder": "Enter default file limit (e.g., 10)",
"required": "Please enter the default file limit"
}
},
"prompt": {

5
src/assets/locale/es/settings.json
View File

@ -288,6 +288,11 @@
"label": "Solapamiento del Chunk",
"placeholder": "Ingrese el solapamiento del chunk",
"required": "Por favor, ingresar el solapamiento del chunk"
},
"totalFilePerKB": {
"label": "Límite predeterminado de archivos de la base de conocimientos",
"placeholder": "Ingrese el límite predeterminado de archivos (ej. 10)",
"required": "Por favor, ingrese el límite predeterminado de archivos"
}
},
"prompt": {

5
src/assets/locale/fr/settings.json
View File

@ -288,6 +288,11 @@
"label": "Chevauchement",
"placeholder": "Entrez le chevauchement des morceaux",
"required": "Veuillez saisir un chevauchement"
},
"totalFilePerKB": {
"label": "Limite par défaut de fichiers de la base de connaissances",
"placeholder": "Entrez la limite par défaut de fichiers (ex. 10)",
"required": "Veuillez entrer la limite par défaut de fichiers"
}
},
"prompt": {

5
src/assets/locale/it/settings.json
View File

@ -288,6 +288,11 @@
"label": "Sovrapposizione del Blocco (Chunk Overlap)",
"placeholder": "Inserisci la Sovrapposizione del Blocco (Chunk Overlap)",
"required": "Inserisci la Sovrapposizione del Blocco"
},
"totalFilePerKB": {
"label": "Limite predefinito di file della base di conoscenza",
"placeholder": "Inserisci il limite predefinito di file (es. 10)",
"required": "Inserisci il limite predefinito di file"
}
},
"prompt": {

5
src/assets/locale/ja-JP/settings.json
View File

@ -291,6 +291,11 @@
"label": "チャンクオーバーラップ",
"placeholder": "チャンクオーバーラップを入力",
"required": "チャンクオーバーラップを入力してください"
},
"totalFilePerKB": {
"label": "ナレッジベースのデフォルトファイル制限",
"placeholder": "デフォルトのファイル制限を入力してください（例：10）",
"required": "デフォルトのファイル制限を入力してください"
}
},
"prompt": {

5
src/assets/locale/ml/settings.json
View File

@ -291,6 +291,11 @@
"label": "ചങ്ക് ഓവര്‍ലാപ്പ്",
"placeholder": "ചങ്ക് ഓവര്‍ലാപ്പ് നല്കുക",
"required": "ദയവായി ചങ്ക് ഓവര്‍ലാപ്പ് നല്കുക"
},
"totalFilePerKB": {
"label": "Limite padrão de arquivos da base de conhecimento",
"placeholder": "Digite o limite padrão de arquivos (ex. 10)",
"required": "Por favor, digite o limite padrão de arquivos"
}
},
"prompt": {

5
src/assets/locale/pt-BR/settings.json
View File

@ -288,6 +288,11 @@
"label": "Sobreposição do Pedaço",
"placeholder": "Digite a Sobreposição do Pedaço",
"required": "Por favor, insira uma sobreposição de pedaço"
},
"totalFilePerKB": {
"label": "Limite padrão de arquivos da base de conhecimento",
"placeholder": "Digite o limite padrão de arquivos (ex. 10)",
"required": "Por favor, digite o limite padrão de arquivos"
}
},
"prompt": {

5
src/assets/locale/ru/settings.json
View File

@ -289,6 +289,11 @@
"label": "Перекрытие фрагментов",
"placeholder": "Введите перекрытие фрагментов",
"required": "Пожалуйста, введите перекрытие фрагментов"
},
"totalFilePerKB": {
"label": "Стандартный лимит файлов базы знаний",
"placeholder": "Введите стандартный лимит файлов (напр. 10)",
"required": "Пожалуйста, введите стандартный лимит файлов"
}
},
"prompt": {

5
src/assets/locale/zh/settings.json
View File

@ -293,6 +293,11 @@
"label": "嵌入重叠",
"placeholder": "256-∞",
"required": "请输入嵌入重叠"
},
"totalFilePerKB": {
"label": "知识库默认文件限制",
"placeholder": "输入默认文件限制（例如：10）",
"required": "请输入默认文件限制"
}
},
"prompt": {

4
src/components/Option/Knowledge/AddKnowledge.tsx
View File

@ -7,6 +7,7 @@ import { InboxIcon } from "lucide-react"
import { useTranslation } from "react-i18next"
import PubSub from "pubsub-js"
import { KNOWLEDGE_QUEUE } from "@/queue"
import { useStorage } from "@plasmohq/storage/hook"
type Props = {
open: boolean
@ -16,6 +17,7 @@ type Props = {
export const AddKnowledge = ({ open, setOpen }: Props) => {
const { t } = useTranslation(["knowledge", "common"])
const [form] = Form.useForm()
const [totalFilePerKB] = useStorage("totalFilePerKB", 10)
const onUploadHandler = async (data: {
title: string
@ -92,7 +94,7 @@ export const AddKnowledge = ({ open, setOpen }: Props) => {
<Upload.Dragger
accept={".pdf, .csv, .txt, .md, .docx"}
multiple={true}
maxCount={10}
maxCount={totalFilePerKB}
beforeUpload={(file) => {
const allowedTypes = [
"application/pdf",

67
src/components/Option/Settings/rag.tsx
View File

@ -10,6 +10,7 @@ import {
} from "~/services/ollama"
import { SettingPrompt } from "./prompt"
import { useTranslation } from "react-i18next"
import { getTotalFilePerKB } from "@/services/app"
export const RagSettings = () => {
const { t } = useTranslation("settings")
@ -19,19 +20,20 @@ export const RagSettings = () => {
const { data: ollamaInfo, status } = useQuery({
queryKey: ["fetchRAGSettings"],
queryFn: async () => {
const [allModels, chunkOverlap, chunkSize, defaultEM] = await Promise.all(
[
const [allModels, chunkOverlap, chunkSize, defaultEM, totalFilePerKB] =
await Promise.all([
getAllModels({ returnEmpty: true }),
defaultEmbeddingChunkOverlap(),
defaultEmbeddingChunkSize(),
defaultEmbeddingModelForRag()
]
)
defaultEmbeddingModelForRag(),
getTotalFilePerKB()
])
return {
models: allModels,
chunkOverlap,
chunkSize,
defaultEM
defaultEM,
totalFilePerKB
}
}
})
@ -41,8 +43,9 @@ export const RagSettings = () => {
model: string
chunkSize: number
overlap: number
totalFilePerKB: number
}) => {
await saveForRag(data.model, data.chunkSize, data.overlap)
await saveForRag(data.model, data.chunkSize, data.overlap, data.totalFilePerKB)
return true
},
onSuccess: () => {
@ -70,13 +73,15 @@ export const RagSettings = () => {
saveRAG({
model: data.defaultEM,
chunkSize: data.chunkSize,
overlap: data.chunkOverlap
overlap: data.chunkOverlap,
totalFilePerKB: data.totalFilePerKB
})
}}
initialValues={{
chunkSize: ollamaInfo?.chunkSize,
chunkOverlap: ollamaInfo?.chunkOverlap,
defaultEM: ollamaInfo?.defaultEM
defaultEM: ollamaInfo?.defaultEM,
totalFilePerKB: ollamaInfo?.totalFilePerKB
}}>
<Form.Item
name="defaultEM"
@ -85,9 +90,7 @@ export const RagSettings = () => {
rules={[
{
required: true,
message: t(
"rag.ragSettings.model.required"
)
message: t("rag.ragSettings.model.required")
}
]}>
<Select
@ -99,9 +102,7 @@ export const RagSettings = () => {
0
}
showSearch
placeholder={t(
"rag.ragSettings.model.placeholder"
)}
placeholder={t("rag.ragSettings.model.placeholder")}
style={{ width: "100%" }}
className="mt-4"
options={ollamaInfo.models?.map((model) => ({
@ -117,39 +118,45 @@ export const RagSettings = () => {
rules={[
{
required: true,
message: t(
"rag.ragSettings.chunkSize.required"
)
message: t("rag.ragSettings.chunkSize.required")
}
]}>
<InputNumber
style={{ width: "100%" }}
placeholder={t(
"rag.ragSettings.chunkSize.placeholder"
)}
placeholder={t("rag.ragSettings.chunkSize.placeholder")}
/>
</Form.Item>
<Form.Item
name="chunkOverlap"
label={t(
"rag.ragSettings.chunkOverlap.label"
)}
label={t("rag.ragSettings.chunkOverlap.label")}
rules={[
{
required: true,
message: t(
"rag.ragSettings.chunkOverlap.required"
)
message: t("rag.ragSettings.chunkOverlap.required")
}
]}>
<InputNumber
style={{ width: "100%" }}
placeholder={t(
"rag.ragSettings.chunkOverlap.placeholder"
)}
placeholder={t("rag.ragSettings.chunkOverlap.placeholder")}
/>
</Form.Item>
<Form.Item
name="totalFilePerKB"
label={t("rag.ragSettings.totalFilePerKB.label")}
rules={[
{
required: true,
message: t("rag.ragSettings.totalFilePerKB.required")
}
]}>
<InputNumber
style={{ width: "100%" }}
placeholder={t("rag.ragSettings.totalFilePerKB.placeholder")}
/>
</Form.Item>
<div className="flex justify-end">
<SaveButton disabled={isSaveRAGPending} btnType="submit" />
</div>

11
src/services/app.ts
View File

@ -98,3 +98,14 @@ export const getOpenOnRightClick = async (): Promise<string> => {
export const setOpenOnRightClick = async (option: "webUI" | "sidePanel"): Promise<void> => {
await storage.set("openOnRightClick", option);
};
export const getTotalFilePerKB = async (): Promise<number> => {
const totalFilePerKB = await storage.get<number>("totalFilePerKB");
return totalFilePerKB || 10;
}
export const setTotalFilePerKB = async (totalFilePerKB: number): Promise<void> => {
await storage.set("totalFilePerKB", totalFilePerKB);
};

5
src/services/ollama.ts
View File

@ -2,6 +2,7 @@ import { Storage } from "@plasmohq/storage"
import { cleanUrl } from "../libs/clean-url"
import { urlRewriteRuntime } from "../libs/runtime"
import { getChromeAIModel } from "./chrome"
import { setTotalFilePerKB } from "./app"
const storage = new Storage()
@ -324,11 +325,13 @@ export const setDefaultEmbeddingChunkOverlap = async (overlap: number) => {
export const saveForRag = async (
model: string,
chunkSize: number,
overlap: number
overlap: number,
totalFilePerKB: number
) => {
await setDefaultEmbeddingModelForRag(model)
await setDefaultEmbeddingChunkSize(chunkSize)
await setDefaultEmbeddingChunkOverlap(overlap)
await setTotalFilePerKB(totalFilePerKB)
}
export const getWebSearchPrompt = async () => {

2
wxt.config.ts
View File

@ -50,7 +50,7 @@ export default defineConfig({
outDir: "build",
manifest: {
version: "1.1.15",
version: "1.1.16",
name:
process.env.TARGET === "firefox"
? "Page Assist - A Web UI for Local AI Models"