iod/page-assist
10
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Improve model selection and embedding

Browse Source 
Refactor embedding models and their handling to improve performance and simplify the process.
Add a new model selection mechanism,  and enhance the UI for model selection, offering clearer and more user-friendly options for embedding models.
Refactor embeddings to use a common model for page assist and RAG, further improving performance and streamlining the workflow.
This commit is contained in:
n4ze3m
2024-10-12 23:32:00 +05:30
parent ba071ffeb1
commit 768ff2e555
14 changed files with 98 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/libs/process-knowledge.ts
View File

@@ -5,7 +5,6 @@ import {
defaultEmbeddingChunkSize,
getOllamaURL
} from "@/services/ollama"
import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { PageAssistVectorStore } from "./PageAssistVectorStore"
import { PageAssisCSVUrlLoader } from "@/loader/csv"
@@ -13,6 +12,7 @@ import { PageAssisTXTUrlLoader } from "@/loader/txt"
import { PageAssistDocxLoader } from "@/loader/docx"
import { cleanUrl } from "./clean-url"
import { sendEmbeddingCompleteNotification } from "./send-notification"
import { pageAssistEmbeddingModel } from "@/models/embedding"
export const processKnowledge = async (msg: any, id: string): Promise<void> => {
@@ -28,7 +28,7 @@ export const processKnowledge = async (msg: any, id: string): Promise<void> => {
await updateKnowledgeStatus(id, "processing")
const ollamaEmbedding = new OllamaEmbeddings({
const ollamaEmbedding = await pageAssistEmbeddingModel({
baseUrl: cleanUrl(ollamaUrl),
model: knowledge.embedding_model
})