iod/page-assist
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add text splitting configuration options

Browse Source
This commit is contained in:
n4ze3m 2025-01-04 23:24:23 +05:30
parent 1d9d704c76
commit 0af69a3be8
29 changed files with 315 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/assets/locale/ar/settings.json
View File

@ -334,6 +334,14 @@
"label": "عدد المستندات المسترجعة",
"placeholder": "أدخل عدد المستندات المسترجعة",
"required": "الرجاء إدخال عدد المستندات المسترجعة"
},
"splittingSeparator": {
"label": "الفاصل",
"placeholder": "أدخل الفاصل (مثال: \\n\\n)",
"required": "الرجاء إدخال الفاصل"
},
"splittingStrategy": {
"label": "مقسم النص"
}
},
"prompt": {
@ -355,4 +363,5 @@
},
"chromeAiSettings": {
"title": "إعدادات Chrome AI"
}}
}
}

8
src/assets/locale/da/settings.json
View File

@ -331,6 +331,14 @@
"label": "Antal Hentede Dokumenter",
"placeholder": "Indtast Number of Retrieved Documents",
"required": "Venligst indtast the number of retrieved documents"
},
"splittingSeparator": {
"label": "Separator",
"placeholder": "Indtast Separator (f.eks. \\n\\n)",
"required": "Indtast venligst en separator"
},
"splittingStrategy": {
"label": "Tekst Splitter"
}
},
"prompt": {

8
src/assets/locale/de/settings.json
View File

@ -331,6 +331,14 @@
"label": "Anzahl der abgerufenen Dokumente",
"placeholder": "Anzahl der abgerufenen Dokumente eingeben",
"required": "Bitte geben Sie die Anzahl der abgerufenen Dokumente ein"
},
"splittingSeparator": {
"label": "Separator",
"placeholder": "Separator eingeben (z.B. \\n\\n)",
"required": "Bitte geben Sie einen Separator ein"
},
"splittingStrategy": {
"label": "Text-Splitter"
}
},
"prompt": {

8
src/assets/locale/en/settings.json
View File

@ -337,6 +337,14 @@
"label": "Number of Retrieved Documents",
"placeholder": "Enter Number of Retrieved Documents",
"required": "Please enter the number of retrieved documents"
},
"splittingSeparator": {
"label": "Separator",
"placeholder": "Enter Separator (e.g., \\n\\n)",
"required": "Please enter a separator"
},
"splittingStrategy": {
"label": "Text Splitter"
}
},
"prompt": {

8
src/assets/locale/es/settings.json
View File

@ -331,6 +331,14 @@
"label": "Número de Documentos Recuperados",
"placeholder": "Ingrese el Número de Documentos Recuperados",
"required": "Por favor, ingrese el número de documentos recuperados"
},
"splittingSeparator": {
"label": "Separador",
"placeholder": "Ingrese el separador (ej., \\n\\n)",
"required": "Por favor, ingrese un separador"
},
"splittingStrategy": {
"label": "Divisor de Texto"
}
},
"prompt": {

8
src/assets/locale/fa/settings.json
View File

@ -327,6 +327,14 @@
"label": "تعداد اسناد بازیابی شده",
"placeholder": "تعداد اسناد بازیابی شده را وارد کنید",
"required": "لطفاً تعداد اسناد بازیابی شده را وارد کنید"
},
"splittingSeparator": {
"label": "جداکننده",
"placeholder": "جداکننده را وارد کنید (مثلاً \\n\\n)",
"required": "لطفاً یک جداکننده وارد کنید"
},
"splittingStrategy": {
"label": "تقسیم‌کننده متن"
}
},
"prompt": {

8
src/assets/locale/fr/settings.json
View File

@ -331,6 +331,14 @@
"label": "Nombre de documents récupérés",
"placeholder": "Entrez le nombre de documents récupérés",
"required": "Veuillez saisir le nombre de documents récupérés"
},
"splittingSeparator": {
"label": "Séparateur",
"placeholder": "Entrez le séparateur (par exemple, \\n\\n)",
"required": "Veuillez saisir un séparateur"
},
"splittingStrategy": {
"label": "Diviseur de texte"
}
},
"prompt": {

8
src/assets/locale/it/settings.json
View File

@ -331,6 +331,14 @@
"label": "Numero di Documenti Recuperati",
"placeholder": "Inserisci il Numero di Documenti Recuperati",
"required": "Inserisci il numero di documenti recuperati"
},
"splittingSeparator": {
"label": "Separatore",
"placeholder": "Inserisci il Separatore (es. \\n\\n)",
"required": "Inserisci un separatore"
},
"splittingStrategy": {
"label": "Divisore di Testo"
}
},
"prompt": {

8
src/assets/locale/ja-JP/settings.json
View File

@ -334,6 +334,14 @@
"label": "取得ドキュメント数",
"placeholder": "取得ドキュメント数を入力",
"required": "取得ドキュメント数を入力してください"
},
"splittingSeparator": {
"label": "セパレーター",
"placeholder": "セパレーターを入力（例：\\n\\n）",
"required": "セパレーターを入力してください"
},
"splittingStrategy": {
"label": "テキスト分割方式"
}
},
"prompt": {

8
src/assets/locale/ko/settings.json
View File

@ -334,6 +334,14 @@
"label": "검색 문서 수",
"placeholder": "검색 문서 수 입력",
"required": "검색 문서 수를 입력해주세요"
},
"splittingSeparator": {
"label": "구분자",
"placeholder": "구분자 입력 (예: \\n\\n)",
"required": "구분자를 입력해주세요"
},
"splittingStrategy": {
"label": "텍스트 분할기"
}
},
"prompt": {

8
src/assets/locale/ml/settings.json
View File

@ -334,6 +334,14 @@
"label": "വീണ്ടെടുത്ത രേഖകളുടെ എണ്ണം",
"placeholder": "വീണ്ടെടുത്ത രേഖകളുടെ എണ്ണം നൽകുക",
"required": "ദയവായി വീണ്ടെടുത്ത രേഖകളുടെ എണ്ണം നൽകുക"
},
"splittingSeparator": {
"label": "വിഭജന ചിഹ്നം",
"placeholder": "വിഭജന ചിഹ്നം നൽകുക (ഉദാ: \\n\\n)",
"required": "ദയവായി ഒരു വിഭജന ചിഹ്നം നൽകുക"
},
"splittingStrategy": {
"label": "ടെക്സ്റ്റ് സ്പ്ലിറ്റർ"
}
},
"prompt": {

8
src/assets/locale/no/settings.json
View File

@ -331,6 +331,14 @@
"label": "Antall hentede dokumenter",
"placeholder": "Skriv inn antall hentede dokumenter",
"required": "Vennligst skriv inn antall hentede dokumenter"
},
"splittingSeparator": {
"label": "Separator",
"placeholder": "Skriv inn separator (f.eks. \\n\\n)",
"required": "Vennligst skriv inn en separator"
},
"splittingStrategy": {
"label": "Tekstdeler"
}
},
"prompt": {

8
src/assets/locale/pt-BR/settings.json
View File

@ -331,6 +331,14 @@
"label": "Número de Documentos Recuperados",
"placeholder": "Digite o Número de Documentos Recuperados",
"required": "Por favor, insira o número de documentos recuperados"
},
"splittingSeparator": {
"label": "Separador",
"placeholder": "Digite o Separador (ex: \\n\\n)",
"required": "Por favor, insira um separador"
},
"splittingStrategy": {
"label": "Divisor de Texto"
}
},
"prompt": {

8
src/assets/locale/ru/settings.json
View File

@ -333,6 +333,14 @@
"label": "Количество извлеченных документов",
"placeholder": "Введите количество извлеченных документов",
"required": "Пожалуйста, введите количество извлеченных документов"
},
"splittingSeparator": {
"label": "Разделитель",
"placeholder": "Введите разделитель (например, \\n\\n)",
"required": "Пожалуйста, введите разделитель"
},
"splittingStrategy": {
"label": "Разделитель текста"
}
},
"prompt": {

8
src/assets/locale/sv/settings.json
View File

@ -331,6 +331,14 @@
"label": "Antal hämtade dokument",
"placeholder": "Ange antal hämtade dokument",
"required": "Vänligen ange antal hämtade dokument"
},
"splittingSeparator": {
"label": "Separator",
"placeholder": "Ange separator (t.ex. \\n\\n)",
"required": "Vänligen ange en separator"
},
"splittingStrategy": {
"label": "Textdelare"
}
},
"prompt": {

8
src/assets/locale/uk/settings.json
View File

@ -331,6 +331,14 @@
"label": "Кількість отриманих документів",
"placeholder": "Ввести кількість отриманих документів",
"required": "Будь ласка, введіть кількість документів"
},
"splittingSeparator": {
"label": "Роздільник",
"placeholder": "Введіть роздільник (напр., \\n\\n)",
"required": "Будь ласка, введіть роздільник"
},
"splittingStrategy": {
"label": "Розділювач тексту"
}
},
"prompt": {

8
src/assets/locale/zh/settings.json
View File

@ -336,6 +336,14 @@
"label": "检索文档数量",
"placeholder": "输入检索文档数量",
"required": "请输入检索文档数量"
},
"splittingSeparator": {
"label": "分隔符",
"placeholder": "输入分隔符（例如：\\n\\n）",
"required": "请输入分隔符"
},
"splittingStrategy": {
"label": "文本分割器"
}
},
"prompt": {

78
src/components/Option/Settings/rag.tsx
View File

@ -1,10 +1,12 @@
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"
import { Form, InputNumber, Select, Skeleton } from "antd"
import { Form, Input, InputNumber, Select, Skeleton } from "antd"
import { SaveButton } from "~/components/Common/SaveButton"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
defaultEmbeddingModelForRag,
defaultSplittingStrategy,
defaultSsplttingSeparator,
getEmbeddingModels,
saveForRag
} from "~/services/ollama"
@ -16,7 +18,8 @@ import { ProviderIcons } from "@/components/Common/ProviderIcon"
export const RagSettings = () => {
const { t } = useTranslation("settings")
const [form] = Form.useForm()
const splittingStrategy = Form.useWatch("splittingStrategy", form)
const queryClient = useQueryClient()
const { data: ollamaInfo, status } = useQuery({
@ -28,14 +31,18 @@ export const RagSettings = () => {
chunkSize,
defaultEM,
totalFilePerKB,
noOfRetrievedDocs
noOfRetrievedDocs,
splittingStrategy,
splittingSeparator
] = await Promise.all([
getEmbeddingModels({ returnEmpty: true }),
defaultEmbeddingChunkOverlap(),
defaultEmbeddingChunkSize(),
defaultEmbeddingModelForRag(),
getTotalFilePerKB(),
getNoOfRetrievedDocs()
getNoOfRetrievedDocs(),
defaultSplittingStrategy(),
defaultSsplttingSeparator()
])
return {
models: allModels,
@ -43,7 +50,9 @@ export const RagSettings = () => {
chunkSize,
defaultEM,
totalFilePerKB,
noOfRetrievedDocs
noOfRetrievedDocs,
splittingStrategy,
splittingSeparator
}
}
})
@ -55,13 +64,17 @@ export const RagSettings = () => {
overlap: number
totalFilePerKB: number
noOfRetrievedDocs: number
strategy: string
separator: string
}) => {
await saveForRag(
data.model,
data.chunkSize,
data.overlap,
data.totalFilePerKB,
data.noOfRetrievedDocs
data.noOfRetrievedDocs,
data.strategy,
data.separator
)
return true
},
@ -85,6 +98,7 @@ export const RagSettings = () => {
<div className="border border-b border-gray-200 dark:border-gray-600 mt-3 mb-6"></div>
</div>
<Form
form={form}
layout="vertical"
onFinish={(data) => {
saveRAG({
@ -92,7 +106,9 @@ export const RagSettings = () => {
chunkSize: data.chunkSize,
overlap: data.chunkOverlap,
totalFilePerKB: data.totalFilePerKB,
noOfRetrievedDocs: data.noOfRetrievedDocs
noOfRetrievedDocs: data.noOfRetrievedDocs,
separator: data.splittingSeparator,
strategy: data.splittingStrategy
})
}}
initialValues={{
@ -100,7 +116,9 @@ export const RagSettings = () => {
chunkOverlap: ollamaInfo?.chunkOverlap,
defaultEM: ollamaInfo?.defaultEM,
totalFilePerKB: ollamaInfo?.totalFilePerKB,
noOfRetrievedDocs: ollamaInfo?.noOfRetrievedDocs
noOfRetrievedDocs: ollamaInfo?.noOfRetrievedDocs,
splittingStrategy: ollamaInfo?.splittingStrategy,
splittingSeparator: ollamaInfo?.splittingSeparator
}}>
<Form.Item
name="defaultEM"
@ -140,6 +158,50 @@ export const RagSettings = () => {
/>
</Form.Item>
<Form.Item
name="splittingStrategy"
label={t("rag.ragSettings.splittingStrategy.label")}
rules={[
{
required: true,
message: t("rag.ragSettings.model.required")
}
]}>
<Select
size="large"
showSearch
style={{ width: "100%" }}
className="mt-4"
options={[
"RecursiveCharacterTextSplitter",
"CharacterTextSplitter"
].map((e) => ({
label: e,
value: e
}))}
/>
</Form.Item>
{splittingStrategy !== "RecursiveCharacterTextSplitter" && (
<Form.Item
name="splittingSeparator"
label={t("rag.ragSettings.splittingSeparator.label")}
rules={[
{
required: true,
message: t("rag.ragSettings.splittingSeparator.required")
}
]}>
<Input
size="large"
style={{ width: "100%" }}
placeholder={t(
"rag.ragSettings.splittingSeparator.placeholder"
)}
/>
</Form.Item>
)}
<Form.Item
name="chunkSize"
label={t("rag.ragSettings.chunkSize.label")}

27
src/libs/process-knowledge.ts
View File

@ -1,11 +1,6 @@
import { getKnowledgeById, updateKnowledgeStatus } from "@/db/knowledge"
import { PageAssistPDFUrlLoader } from "@/loader/pdf-url"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
getOllamaURL
} from "@/services/ollama"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { getOllamaURL } from "@/services/ollama"
import { PageAssistVectorStore } from "./PageAssistVectorStore"
import { PageAssisCSVUrlLoader } from "@/loader/csv"
import { PageAssisTXTUrlLoader } from "@/loader/txt"
@ -13,7 +8,7 @@ import { PageAssistDocxLoader } from "@/loader/docx"
import { cleanUrl } from "./clean-url"
import { sendEmbeddingCompleteNotification } from "./send-notification"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
export const processKnowledge = async (msg: any, id: string): Promise<void> => {
console.log(`Processing knowledge with id: ${id}`)
@ -32,12 +27,8 @@ export const processKnowledge = async (msg: any, id: string): Promise<void> => {
baseUrl: cleanUrl(ollamaUrl),
model: knowledge.embedding_model
})
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter()
for (const doc of knowledge.source) {
if (doc.type === "pdf" || doc.type === "application/pdf") {
@ -65,13 +56,15 @@ export const processKnowledge = async (msg: any, id: string): Promise<void> => {
knownledge_id: knowledge.id,
file_id: doc.source_id
})
} else if (doc.type === "docx" || doc.type === "application/vnd.openxmlformats-officedocument.wordprocessingml.document") {
} else if (
doc.type === "docx" ||
doc.type ===
"application/vnd.openxmlformats-officedocument.wordprocessingml.document"
) {
try {
const loader = new PageAssistDocxLoader({
fileName: doc.filename,
buffer: await toArrayBufferFromBase64(
doc.content
)
buffer: await toArrayBufferFromBase64(doc.content)
})
let docs = await loader.load()

37
src/services/ollama.ts
View File

@ -8,6 +8,9 @@ import { ollamaFormatAllCustomModels } from "@/db/models"
const storage = new Storage()
const storage2 = new Storage({
area: "local"
})
const DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434"
const DEFAULT_ASK_FOR_MODEL_SELECTION_EVERY_TIME = true
@ -310,6 +313,22 @@ export const defaultEmbeddingChunkSize = async () => {
return parseInt(embeddingChunkSize)
}
export const defaultSplittingStrategy = async () => {
const splittingStrategy = await storage.get("defaultSplittingStrategy")
if (!splittingStrategy || splittingStrategy.length === 0) {
return "RecursiveCharacterTextSplitter"
}
return splittingStrategy
}
export const defaultSsplttingSeparator = async () => {
const splittingSeparator = await storage.get("defaultSplittingSeparator")
if (!splittingSeparator || splittingSeparator.length === 0) {
return "\\n\\n"
}
return splittingSeparator
}
export const defaultEmbeddingChunkOverlap = async () => {
const embeddingChunkOverlap = await storage.get(
"defaultEmbeddingChunkOverlap"
@ -320,6 +339,14 @@ export const defaultEmbeddingChunkOverlap = async () => {
return parseInt(embeddingChunkOverlap)
}
export const setDefaultSplittingStrategy = async (strategy: string) => {
await storage.set("defaultSplittingStrategy", strategy)
}
export const setDefaultSplittingSeparator = async (separator: string) => {
await storage.set("defaultSplittingSeparator", separator)
}
export const setDefaultEmbeddingModelForRag = async (model: string) => {
await storage.set("defaultEmbeddingModel", model)
}
@ -337,7 +364,9 @@ export const saveForRag = async (
chunkSize: number,
overlap: number,
totalFilePerKB: number,
noOfRetrievedDocs?: number
noOfRetrievedDocs?: number,
strategy?: string,
separator?: string
) => {
await setDefaultEmbeddingModelForRag(model)
await setDefaultEmbeddingChunkSize(chunkSize)
@ -346,6 +375,12 @@ export const saveForRag = async (
if (noOfRetrievedDocs) {
await setNoOfRetrievedDocs(noOfRetrievedDocs)
}
if (strategy) {
await setDefaultSplittingStrategy(strategy)
}
if (separator) {
await setDefaultSplittingSeparator(separator)
}
}
export const getWebSearchPrompt = async () => {

13
src/utils/memory-embeddings.ts
View File

@ -1,12 +1,8 @@
import { PageAssistHtmlLoader } from "~/loader/html"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize
} from "@/services/ollama"
import { PageAssistPDFLoader } from "@/loader/pdf"
import { PAMemoryVectorStore } from "@/libs/PAMemoryVectorStore"
import { getPageAssistTextSplitter } from "./text-splitter"
export const getLoader = ({
html,
@ -54,12 +50,7 @@ export const memoryEmbedding = async ({
setIsEmbedding(true)
const loader = getLoader({ html, pdf, type, url })
const docs = await loader.load()
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter()
const chunks = await textSplitter.splitDocuments(docs)

37
src/utils/text-splitter.ts Normal file
View File

@ -0,0 +1,37 @@
import {
RecursiveCharacterTextSplitter,
CharacterTextSplitter
} from "langchain/text_splitter"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
defaultSsplttingSeparator,
defaultSplittingStrategy
} from "@/services/ollama"
export const getPageAssistTextSplitter = async () => {
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const splittingStrategy = await defaultSplittingStrategy()
switch (splittingStrategy) {
case "CharacterTextSplitter":
console.log("Using CharacterTextSplitter")
const splittingSeparator = await defaultSsplttingSeparator()
const processedSeparator = splittingSeparator
.replace(/\\n/g, "\n")
.replace(/\\t/g, "\t")
.replace(/\\r/g, "\r")
return new CharacterTextSplitter({
chunkSize,
chunkOverlap,
separator: processedSeparator
})
default:
return new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
}
}

11
src/web/search-engines/brave-api.ts
View File

@ -2,15 +2,13 @@ import { cleanUrl } from "~/libs/clean-url"
import { getIsSimpleInternetSearch, totalSearchResults, getBraveApiKey } from "@/services/search"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import type { Document } from "@langchain/core/documents"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
import { PageAssistHtmlLoader } from "~/loader/html"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
defaultEmbeddingModelForRag,
getOllamaURL
} from "~/services/ollama"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
interface BraveAPIResult {
title: string
@ -70,12 +68,7 @@ export const braveAPISearch = async (query: string) => {
baseUrl: cleanUrl(ollamaUrl)
})
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter()
const chunks = await textSplitter.splitDocuments(docs)
const store = new MemoryVectorStore(ollamaEmbedding)

12
src/web/search-engines/brave.ts
View File

@ -3,8 +3,6 @@ import { urlRewriteRuntime } from "@/libs/runtime"
import { PageAssistHtmlLoader } from "@/loader/html"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
defaultEmbeddingModelForRag,
getOllamaURL
} from "@/services/ollama"
@ -12,10 +10,10 @@ import {
getIsSimpleInternetSearch,
totalSearchResults
} from "@/services/search"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
import type { Document } from "@langchain/core/documents"
import * as cheerio from "cheerio"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
export const localBraveSearch = async (query: string) => {
@ -87,12 +85,8 @@ export const webBraveSearch = async (query: string) => {
baseUrl: cleanUrl(ollamaUrl)
})
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter();
const chunks = await textSplitter.splitDocuments(docs)

11
src/web/search-engines/duckduckgo.ts
View File

@ -3,8 +3,6 @@ import { urlRewriteRuntime } from "@/libs/runtime"
import { PageAssistHtmlLoader } from "@/loader/html"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
defaultEmbeddingModelForRag,
getOllamaURL
} from "@/services/ollama"
@ -12,9 +10,9 @@ import {
getIsSimpleInternetSearch,
totalSearchResults
} from "@/services/search"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
import type { Document } from "@langchain/core/documents"
import * as cheerio from "cheerio"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
export const localDuckDuckGoSearch = async (query: string) => {
@ -90,12 +88,7 @@ export const webDuckDuckGoSearch = async (query: string) => {
baseUrl: cleanUrl(ollamaUrl)
})
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter()
const chunks = await textSplitter.splitDocuments(docs)

12
src/web/search-engines/google.ts
View File

@ -4,15 +4,13 @@ import {
getIsSimpleInternetSearch,
totalSearchResults
} from "@/services/search"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
import type { Document } from "@langchain/core/documents"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
import { cleanUrl } from "~/libs/clean-url"
import { urlRewriteRuntime } from "~/libs/runtime"
import { PageAssistHtmlLoader } from "~/loader/html"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
defaultEmbeddingModelForRag,
getOllamaURL
} from "~/services/ollama"
@ -91,12 +89,8 @@ export const webGoogleSearch = async (query: string) => {
baseUrl: cleanUrl(ollamaUrl)
})
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter()
const chunks = await textSplitter.splitDocuments(docs)

12
src/web/search-engines/searxng.ts
View File

@ -3,15 +3,13 @@ import { cleanUrl } from "~/libs/clean-url"
import { getSearxngURL, isSearxngJSONMode, getIsSimpleInternetSearch, totalSearchResults } from "@/services/search"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import type { Document } from "@langchain/core/documents"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
import { PageAssistHtmlLoader } from "~/loader/html"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
defaultEmbeddingModelForRag,
getOllamaURL
} from "~/services/ollama"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
interface SearxNGJSONResult {
title: string
@ -73,12 +71,8 @@ export const searxngSearch = async (query: string) => {
baseUrl: cleanUrl(ollamaUrl)
})
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter();
const chunks = await textSplitter.splitDocuments(docs)
const store = new MemoryVectorStore(ollamaEmbedding)

11
src/web/search-engines/sogou.ts
View File

@ -3,8 +3,6 @@ import { urlRewriteRuntime } from "@/libs/runtime"
import { PageAssistHtmlLoader } from "@/loader/html"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import {
defaultEmbeddingChunkOverlap,
defaultEmbeddingChunkSize,
defaultEmbeddingModelForRag,
getOllamaURL
} from "@/services/ollama"
@ -12,9 +10,9 @@ import {
getIsSimpleInternetSearch,
totalSearchResults
} from "@/services/search"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
import type { Document } from "@langchain/core/documents"
import * as cheerio from "cheerio"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
const getCorrectTargeUrl = async (url: string) => {
if (!url) return ""
@ -104,12 +102,7 @@ export const webSogouSearch = async (query: string) => {
baseUrl: cleanUrl(ollamaUrl)
})
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter()
const chunks = await textSplitter.splitDocuments(docs)

13
src/web/website/index.ts
View File

@ -1,8 +1,9 @@
import { cleanUrl } from "@/libs/clean-url"
import { PageAssistHtmlLoader } from "@/loader/html"
import { pageAssistEmbeddingModel } from "@/models/embedding"
import { defaultEmbeddingChunkOverlap, defaultEmbeddingChunkSize, defaultEmbeddingModelForRag, getOllamaURL } from "@/services/ollama"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { defaultEmbeddingModelForRag, getOllamaURL } from "@/services/ollama"
import { getPageAssistTextSplitter } from "@/utils/text-splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
export const processSingleWebsite = async (url: string, query: string) => {
@ -20,12 +21,8 @@ export const processSingleWebsite = async (url: string, query: string) => {
baseUrl: cleanUrl(ollamaUrl)
})
const chunkSize = await defaultEmbeddingChunkSize()
const chunkOverlap = await defaultEmbeddingChunkOverlap()
const textSplitter = new RecursiveCharacterTextSplitter({
chunkSize,
chunkOverlap
})
const textSplitter = await getPageAssistTextSplitter()
const chunks = await textSplitter.splitDocuments(docs)