feat: ability to update knowledge base default file limit

This commit is contained in:
n4ze3m 2024-07-21 20:13:37 +05:30
parent 943813f759
commit af09c8eed6
14 changed files with 101 additions and 33 deletions

View File

@ -288,6 +288,11 @@
"label": "Chunk Overlap", "label": "Chunk Overlap",
"placeholder": "Enter Chunk Overlap", "placeholder": "Enter Chunk Overlap",
"required": "Please enter a chunk overlap" "required": "Please enter a chunk overlap"
},
"totalFilePerKB": {
"label": "Knowledge Base Default File Limit",
"placeholder": "Enter default file limit (e.g., 10)",
"required": "Please enter the default file limit"
} }
}, },
"prompt": { "prompt": {

View File

@ -288,6 +288,11 @@
"label": "Solapamiento del Chunk", "label": "Solapamiento del Chunk",
"placeholder": "Ingrese el solapamiento del chunk", "placeholder": "Ingrese el solapamiento del chunk",
"required": "Por favor, ingresar el solapamiento del chunk" "required": "Por favor, ingresar el solapamiento del chunk"
},
"totalFilePerKB": {
"label": "Límite predeterminado de archivos de la base de conocimientos",
"placeholder": "Ingrese el límite predeterminado de archivos (ej. 10)",
"required": "Por favor, ingrese el límite predeterminado de archivos"
} }
}, },
"prompt": { "prompt": {

View File

@ -288,6 +288,11 @@
"label": "Chevauchement", "label": "Chevauchement",
"placeholder": "Entrez le chevauchement des morceaux", "placeholder": "Entrez le chevauchement des morceaux",
"required": "Veuillez saisir un chevauchement" "required": "Veuillez saisir un chevauchement"
},
"totalFilePerKB": {
"label": "Limite par défaut de fichiers de la base de connaissances",
"placeholder": "Entrez la limite par défaut de fichiers (ex. 10)",
"required": "Veuillez entrer la limite par défaut de fichiers"
} }
}, },
"prompt": { "prompt": {

View File

@ -288,6 +288,11 @@
"label": "Sovrapposizione del Blocco (Chunk Overlap)", "label": "Sovrapposizione del Blocco (Chunk Overlap)",
"placeholder": "Inserisci la Sovrapposizione del Blocco (Chunk Overlap)", "placeholder": "Inserisci la Sovrapposizione del Blocco (Chunk Overlap)",
"required": "Inserisci la Sovrapposizione del Blocco" "required": "Inserisci la Sovrapposizione del Blocco"
},
"totalFilePerKB": {
"label": "Limite predefinito di file della base di conoscenza",
"placeholder": "Inserisci il limite predefinito di file (es. 10)",
"required": "Inserisci il limite predefinito di file"
} }
}, },
"prompt": { "prompt": {

View File

@ -291,6 +291,11 @@
"label": "チャンクオーバーラップ", "label": "チャンクオーバーラップ",
"placeholder": "チャンクオーバーラップを入力", "placeholder": "チャンクオーバーラップを入力",
"required": "チャンクオーバーラップを入力してください" "required": "チャンクオーバーラップを入力してください"
},
"totalFilePerKB": {
"label": "ナレッジベースのデフォルトファイル制限",
"placeholder": "デフォルトのファイル制限を入力してください10",
"required": "デフォルトのファイル制限を入力してください"
} }
}, },
"prompt": { "prompt": {

View File

@ -291,6 +291,11 @@
"label": "ചങ്ക് ഓവര്‍ലാപ്പ്", "label": "ചങ്ക് ഓവര്‍ലാപ്പ്",
"placeholder": "ചങ്ക് ഓവര്‍ലാപ്പ് നല്കുക", "placeholder": "ചങ്ക് ഓവര്‍ലാപ്പ് നല്കുക",
"required": "ദയവായി ചങ്ക് ഓവര്‍ലാപ്പ് നല്കുക" "required": "ദയവായി ചങ്ക് ഓവര്‍ലാപ്പ് നല്കുക"
},
"totalFilePerKB": {
"label": "Limite padrão de arquivos da base de conhecimento",
"placeholder": "Digite o limite padrão de arquivos (ex. 10)",
"required": "Por favor, digite o limite padrão de arquivos"
} }
}, },
"prompt": { "prompt": {

View File

@ -288,6 +288,11 @@
"label": "Sobreposição do Pedaço", "label": "Sobreposição do Pedaço",
"placeholder": "Digite a Sobreposição do Pedaço", "placeholder": "Digite a Sobreposição do Pedaço",
"required": "Por favor, insira uma sobreposição de pedaço" "required": "Por favor, insira uma sobreposição de pedaço"
},
"totalFilePerKB": {
"label": "Limite padrão de arquivos da base de conhecimento",
"placeholder": "Digite o limite padrão de arquivos (ex. 10)",
"required": "Por favor, digite o limite padrão de arquivos"
} }
}, },
"prompt": { "prompt": {

View File

@ -289,6 +289,11 @@
"label": "Перекрытие фрагментов", "label": "Перекрытие фрагментов",
"placeholder": "Введите перекрытие фрагментов", "placeholder": "Введите перекрытие фрагментов",
"required": "Пожалуйста, введите перекрытие фрагментов" "required": "Пожалуйста, введите перекрытие фрагментов"
},
"totalFilePerKB": {
"label": "Стандартный лимит файлов базы знаний",
"placeholder": "Введите стандартный лимит файлов (напр. 10)",
"required": "Пожалуйста, введите стандартный лимит файлов"
} }
}, },
"prompt": { "prompt": {

View File

@ -293,6 +293,11 @@
"label": "嵌入重叠", "label": "嵌入重叠",
"placeholder": "256-∞", "placeholder": "256-∞",
"required": "请输入嵌入重叠" "required": "请输入嵌入重叠"
},
"totalFilePerKB": {
"label": "知识库默认文件限制",
"placeholder": "输入默认文件限制例如10",
"required": "请输入默认文件限制"
} }
}, },
"prompt": { "prompt": {

View File

@ -7,6 +7,7 @@ import { InboxIcon } from "lucide-react"
import { useTranslation } from "react-i18next" import { useTranslation } from "react-i18next"
import PubSub from "pubsub-js" import PubSub from "pubsub-js"
import { KNOWLEDGE_QUEUE } from "@/queue" import { KNOWLEDGE_QUEUE } from "@/queue"
import { useStorage } from "@plasmohq/storage/hook"
type Props = { type Props = {
open: boolean open: boolean
@ -16,6 +17,7 @@ type Props = {
export const AddKnowledge = ({ open, setOpen }: Props) => { export const AddKnowledge = ({ open, setOpen }: Props) => {
const { t } = useTranslation(["knowledge", "common"]) const { t } = useTranslation(["knowledge", "common"])
const [form] = Form.useForm() const [form] = Form.useForm()
const [totalFilePerKB] = useStorage("totalFilePerKB", 10)
const onUploadHandler = async (data: { const onUploadHandler = async (data: {
title: string title: string
@ -92,7 +94,7 @@ export const AddKnowledge = ({ open, setOpen }: Props) => {
<Upload.Dragger <Upload.Dragger
accept={".pdf, .csv, .txt, .md, .docx"} accept={".pdf, .csv, .txt, .md, .docx"}
multiple={true} multiple={true}
maxCount={10} maxCount={totalFilePerKB}
beforeUpload={(file) => { beforeUpload={(file) => {
const allowedTypes = [ const allowedTypes = [
"application/pdf", "application/pdf",

View File

@ -10,6 +10,7 @@ import {
} from "~/services/ollama" } from "~/services/ollama"
import { SettingPrompt } from "./prompt" import { SettingPrompt } from "./prompt"
import { useTranslation } from "react-i18next" import { useTranslation } from "react-i18next"
import { getTotalFilePerKB } from "@/services/app"
export const RagSettings = () => { export const RagSettings = () => {
const { t } = useTranslation("settings") const { t } = useTranslation("settings")
@ -19,19 +20,20 @@ export const RagSettings = () => {
const { data: ollamaInfo, status } = useQuery({ const { data: ollamaInfo, status } = useQuery({
queryKey: ["fetchRAGSettings"], queryKey: ["fetchRAGSettings"],
queryFn: async () => { queryFn: async () => {
const [allModels, chunkOverlap, chunkSize, defaultEM] = await Promise.all( const [allModels, chunkOverlap, chunkSize, defaultEM, totalFilePerKB] =
[ await Promise.all([
getAllModels({ returnEmpty: true }), getAllModels({ returnEmpty: true }),
defaultEmbeddingChunkOverlap(), defaultEmbeddingChunkOverlap(),
defaultEmbeddingChunkSize(), defaultEmbeddingChunkSize(),
defaultEmbeddingModelForRag() defaultEmbeddingModelForRag(),
] getTotalFilePerKB()
) ])
return { return {
models: allModels, models: allModels,
chunkOverlap, chunkOverlap,
chunkSize, chunkSize,
defaultEM defaultEM,
totalFilePerKB
} }
} }
}) })
@ -41,8 +43,9 @@ export const RagSettings = () => {
model: string model: string
chunkSize: number chunkSize: number
overlap: number overlap: number
totalFilePerKB: number
}) => { }) => {
await saveForRag(data.model, data.chunkSize, data.overlap) await saveForRag(data.model, data.chunkSize, data.overlap, data.totalFilePerKB)
return true return true
}, },
onSuccess: () => { onSuccess: () => {
@ -70,13 +73,15 @@ export const RagSettings = () => {
saveRAG({ saveRAG({
model: data.defaultEM, model: data.defaultEM,
chunkSize: data.chunkSize, chunkSize: data.chunkSize,
overlap: data.chunkOverlap overlap: data.chunkOverlap,
totalFilePerKB: data.totalFilePerKB
}) })
}} }}
initialValues={{ initialValues={{
chunkSize: ollamaInfo?.chunkSize, chunkSize: ollamaInfo?.chunkSize,
chunkOverlap: ollamaInfo?.chunkOverlap, chunkOverlap: ollamaInfo?.chunkOverlap,
defaultEM: ollamaInfo?.defaultEM defaultEM: ollamaInfo?.defaultEM,
totalFilePerKB: ollamaInfo?.totalFilePerKB
}}> }}>
<Form.Item <Form.Item
name="defaultEM" name="defaultEM"
@ -85,9 +90,7 @@ export const RagSettings = () => {
rules={[ rules={[
{ {
required: true, required: true,
message: t( message: t("rag.ragSettings.model.required")
"rag.ragSettings.model.required"
)
} }
]}> ]}>
<Select <Select
@ -99,9 +102,7 @@ export const RagSettings = () => {
0 0
} }
showSearch showSearch
placeholder={t( placeholder={t("rag.ragSettings.model.placeholder")}
"rag.ragSettings.model.placeholder"
)}
style={{ width: "100%" }} style={{ width: "100%" }}
className="mt-4" className="mt-4"
options={ollamaInfo.models?.map((model) => ({ options={ollamaInfo.models?.map((model) => ({
@ -117,39 +118,45 @@ export const RagSettings = () => {
rules={[ rules={[
{ {
required: true, required: true,
message: t( message: t("rag.ragSettings.chunkSize.required")
"rag.ragSettings.chunkSize.required"
)
} }
]}> ]}>
<InputNumber <InputNumber
style={{ width: "100%" }} style={{ width: "100%" }}
placeholder={t( placeholder={t("rag.ragSettings.chunkSize.placeholder")}
"rag.ragSettings.chunkSize.placeholder"
)}
/> />
</Form.Item> </Form.Item>
<Form.Item <Form.Item
name="chunkOverlap" name="chunkOverlap"
label={t( label={t("rag.ragSettings.chunkOverlap.label")}
"rag.ragSettings.chunkOverlap.label"
)}
rules={[ rules={[
{ {
required: true, required: true,
message: t( message: t("rag.ragSettings.chunkOverlap.required")
"rag.ragSettings.chunkOverlap.required"
)
} }
]}> ]}>
<InputNumber <InputNumber
style={{ width: "100%" }} style={{ width: "100%" }}
placeholder={t( placeholder={t("rag.ragSettings.chunkOverlap.placeholder")}
"rag.ragSettings.chunkOverlap.placeholder"
)}
/> />
</Form.Item> </Form.Item>
<Form.Item
name="totalFilePerKB"
label={t("rag.ragSettings.totalFilePerKB.label")}
rules={[
{
required: true,
message: t("rag.ragSettings.totalFilePerKB.required")
}
]}>
<InputNumber
style={{ width: "100%" }}
placeholder={t("rag.ragSettings.totalFilePerKB.placeholder")}
/>
</Form.Item>
<div className="flex justify-end"> <div className="flex justify-end">
<SaveButton disabled={isSaveRAGPending} btnType="submit" /> <SaveButton disabled={isSaveRAGPending} btnType="submit" />
</div> </div>

View File

@ -98,3 +98,14 @@ export const getOpenOnRightClick = async (): Promise<string> => {
export const setOpenOnRightClick = async (option: "webUI" | "sidePanel"): Promise<void> => { export const setOpenOnRightClick = async (option: "webUI" | "sidePanel"): Promise<void> => {
await storage.set("openOnRightClick", option); await storage.set("openOnRightClick", option);
}; };
export const getTotalFilePerKB = async (): Promise<number> => {
const totalFilePerKB = await storage.get<number>("totalFilePerKB");
return totalFilePerKB || 10;
}
export const setTotalFilePerKB = async (totalFilePerKB: number): Promise<void> => {
await storage.set("totalFilePerKB", totalFilePerKB);
};

View File

@ -2,6 +2,7 @@ import { Storage } from "@plasmohq/storage"
import { cleanUrl } from "../libs/clean-url" import { cleanUrl } from "../libs/clean-url"
import { urlRewriteRuntime } from "../libs/runtime" import { urlRewriteRuntime } from "../libs/runtime"
import { getChromeAIModel } from "./chrome" import { getChromeAIModel } from "./chrome"
import { setTotalFilePerKB } from "./app"
const storage = new Storage() const storage = new Storage()
@ -324,11 +325,13 @@ export const setDefaultEmbeddingChunkOverlap = async (overlap: number) => {
export const saveForRag = async ( export const saveForRag = async (
model: string, model: string,
chunkSize: number, chunkSize: number,
overlap: number overlap: number,
totalFilePerKB: number
) => { ) => {
await setDefaultEmbeddingModelForRag(model) await setDefaultEmbeddingModelForRag(model)
await setDefaultEmbeddingChunkSize(chunkSize) await setDefaultEmbeddingChunkSize(chunkSize)
await setDefaultEmbeddingChunkOverlap(overlap) await setDefaultEmbeddingChunkOverlap(overlap)
await setTotalFilePerKB(totalFilePerKB)
} }
export const getWebSearchPrompt = async () => { export const getWebSearchPrompt = async () => {

View File

@ -50,7 +50,7 @@ export default defineConfig({
outDir: "build", outDir: "build",
manifest: { manifest: {
version: "1.1.15", version: "1.1.16",
name: name:
process.env.TARGET === "firefox" process.env.TARGET === "firefox"
? "Page Assist - A Web UI for Local AI Models" ? "Page Assist - A Web UI for Local AI Models"