Update dependencies and fix import paths
This commit is contained in:
parent
d91d4c4761
commit
ac347a3970
@ -35,8 +35,10 @@
|
||||
"i18next-browser-languagedetector": "^7.2.0",
|
||||
"langchain": "^0.1.28",
|
||||
"lucide-react": "^0.350.0",
|
||||
"ml-distance": "^4.0.1",
|
||||
"pdfjs-dist": "^4.0.379",
|
||||
"property-information": "^6.4.1",
|
||||
"pubsub-js": "^1.9.4",
|
||||
"react": "18.2.0",
|
||||
"react-dom": "18.2.0",
|
||||
"react-i18next": "^14.1.0",
|
||||
@ -55,6 +57,7 @@
|
||||
"@types/chrome": "0.0.259",
|
||||
"@types/html-to-text": "^9.0.4",
|
||||
"@types/node": "20.11.9",
|
||||
"@types/pubsub-js": "^1.8.6",
|
||||
"@types/react": "18.2.48",
|
||||
"@types/react-dom": "18.2.18",
|
||||
"@types/react-syntax-highlighter": "^15.5.11",
|
||||
|
3
src/assets/locale/en/knownledge.json
Normal file
3
src/assets/locale/en/knownledge.json
Normal file
@ -0,0 +1,3 @@
|
||||
{
|
||||
"addBtn": "Add New Knowledge"
|
||||
}
|
@ -21,7 +21,8 @@
|
||||
"searchInternet": "Search Internet",
|
||||
"speechToText": "Speech to Text",
|
||||
"uploadImage": "Upload Image",
|
||||
"stopStreaming": "Stop Streaming"
|
||||
"stopStreaming": "Stop Streaming",
|
||||
"knowledge": "Knowledge"
|
||||
},
|
||||
"sendWhenEnter": "Send when Enter pressed"
|
||||
}
|
@ -242,5 +242,9 @@
|
||||
"koFi": "Support on Ko-fi",
|
||||
"githubSponsor": "Sponsor on GitHub",
|
||||
"githubRepo": "GitHub Repository"
|
||||
},
|
||||
"manageKnowledge": {
|
||||
"title": "Manage Knowledge",
|
||||
"heading": "Configure Knowledge Base"
|
||||
}
|
||||
}
|
1
src/assets/locale/ja-JP/knownledge.json
Normal file
1
src/assets/locale/ja-JP/knownledge.json
Normal file
@ -0,0 +1 @@
|
||||
{}
|
@ -21,7 +21,8 @@
|
||||
"searchInternet": "インターネットを検索",
|
||||
"speechToText": "音声入力",
|
||||
"uploadImage": "画像をアップロード",
|
||||
"stopStreaming": "ストリーミングを停止"
|
||||
"stopStreaming": "ストリーミングを停止",
|
||||
"knowledge": "知識"
|
||||
},
|
||||
"sendWhenEnter": "Enterキーを押すと送信"
|
||||
}
|
1
src/assets/locale/ml/knownledge.json
Normal file
1
src/assets/locale/ml/knownledge.json
Normal file
@ -0,0 +1 @@
|
||||
{}
|
@ -21,7 +21,8 @@
|
||||
"searchInternet": "ഇന്റര്നെറ്റ് തിരയുക",
|
||||
"speechToText": "സംഭാഷണം ടെക്സ്റ്റായി",
|
||||
"uploadImage": "ഇമേജ് അപ്ലോഡ് ചെയ്യുക",
|
||||
"stopStreaming": "സ്ട്രീമിംഗ് നിർത്തുക"
|
||||
"stopStreaming": "സ്ട്രീമിംഗ് നിർത്തുക",
|
||||
"knowledge": "അറിവ്"
|
||||
},
|
||||
"sendWhenEnter": "എന്റര് അമര്ത്തുമ്പോള് അയയ്ക്കുക"
|
||||
}
|
1
src/assets/locale/zh/knownledge.json
Normal file
1
src/assets/locale/zh/knownledge.json
Normal file
@ -0,0 +1 @@
|
||||
{}
|
@ -21,7 +21,8 @@
|
||||
"searchInternet": "搜索互联网",
|
||||
"speechToText": "语音到文本",
|
||||
"uploadImage": "上传图片",
|
||||
"stopStreaming": "停止流媒体"
|
||||
"stopStreaming": "停止流媒体",
|
||||
"knowledge": "知识"
|
||||
},
|
||||
"sendWhenEnter": "按Enter发送"
|
||||
}
|
@ -55,3 +55,13 @@
|
||||
background-position: 0% 50%;
|
||||
}
|
||||
}
|
||||
/* Hide scrollbar for Chrome, Safari and Opera */
|
||||
.no-scrollbar::-webkit-scrollbar {
|
||||
display: none;
|
||||
}
|
||||
|
||||
/* Hide scrollbar for IE, Edge and Firefox */
|
||||
.no-scrollbar {
|
||||
-ms-overflow-style: none; /* IE and Edge */
|
||||
scrollbar-width: none; /* Firefox */
|
||||
}
|
@ -7,7 +7,7 @@ import React from "react"
|
||||
import { useMutation } from "@tanstack/react-query"
|
||||
import { getPageShareUrl } from "~/services/ollama"
|
||||
import { cleanUrl } from "~/libs/clean-url"
|
||||
import { getUserId, saveWebshare } from "~/libs/db"
|
||||
import { getUserId, saveWebshare } from "@/db"
|
||||
import { useTranslation } from "react-i18next"
|
||||
|
||||
type Props = {
|
||||
|
@ -15,7 +15,7 @@ import {
|
||||
SquarePen,
|
||||
ZapIcon
|
||||
} from "lucide-react"
|
||||
import { getAllPrompts } from "~/libs/db"
|
||||
import { getAllPrompts } from "@/db"
|
||||
import { ShareBtn } from "~/components/Common/ShareBtn"
|
||||
import { useTranslation } from "react-i18next"
|
||||
import { OllamaIcon } from "../Icons/Ollama"
|
||||
|
@ -68,12 +68,12 @@ export const SettingsLayout = ({ children }: { children: React.ReactNode }) => {
|
||||
current={location.pathname}
|
||||
icon={BrainCircuit}
|
||||
/>
|
||||
{/* <LinkComponent
|
||||
<LinkComponent
|
||||
href="/settings/knowledge"
|
||||
name={t("manageKnowledge.title")}
|
||||
icon={BlocksIcon}
|
||||
current={location.pathname}
|
||||
/> */}
|
||||
/>
|
||||
<LinkComponent
|
||||
href="/settings/prompt"
|
||||
name={t("managePrompts.title")}
|
||||
|
139
src/components/Option/Knowledge/AddKnowledge.tsx
Normal file
139
src/components/Option/Knowledge/AddKnowledge.tsx
Normal file
@ -0,0 +1,139 @@
|
||||
import { Source, createKnowledge } from "@/db/knowledge"
|
||||
import { defaultEmbeddingModelForRag } from "@/services/ollama"
|
||||
import { convertToSource } from "@/utils/to-source"
|
||||
import { useMutation } from "@tanstack/react-query"
|
||||
import { Modal, Form, Input, Upload, message, UploadFile } from "antd"
|
||||
import { InboxIcon } from "lucide-react"
|
||||
import { useTranslation } from "react-i18next"
|
||||
import PubSub from "pubsub-js"
|
||||
import { KNOWLEDGE_QUEUE } from "@/queue"
|
||||
|
||||
type Props = {
|
||||
open: boolean
|
||||
setOpen: React.Dispatch<React.SetStateAction<boolean>>
|
||||
}
|
||||
|
||||
export const AddKnowledge = ({ open, setOpen }: Props) => {
|
||||
const { t } = useTranslation("knowledge")
|
||||
const [form] = Form.useForm()
|
||||
|
||||
const onUploadHandler = async (data: {
|
||||
title: string
|
||||
file: UploadFile[]
|
||||
}) => {
|
||||
const defaultEM = await defaultEmbeddingModelForRag()
|
||||
|
||||
if (!defaultEM) {
|
||||
throw new Error(t("noEmbeddingModel"))
|
||||
}
|
||||
|
||||
const source: Source[] = []
|
||||
|
||||
for (const file of data.file) {
|
||||
const data = await convertToSource(file)
|
||||
source.push(data)
|
||||
}
|
||||
|
||||
const knowledge = await createKnowledge({
|
||||
embedding_model: defaultEM,
|
||||
source,
|
||||
title: data.title
|
||||
})
|
||||
|
||||
return knowledge.id
|
||||
}
|
||||
|
||||
const { mutate: saveKnowledge, isPending: isSaving } = useMutation({
|
||||
mutationFn: onUploadHandler,
|
||||
onError: (error) => {
|
||||
message.error(error.message)
|
||||
},
|
||||
onSuccess: async (id) => {
|
||||
message.success(t("form.success"))
|
||||
PubSub.publish(KNOWLEDGE_QUEUE, id)
|
||||
setOpen(false)
|
||||
}
|
||||
})
|
||||
|
||||
return (
|
||||
<Modal
|
||||
title={t("addKnowledge")}
|
||||
open={open}
|
||||
footer={null}
|
||||
onCancel={() => setOpen(false)}>
|
||||
<Form onFinish={saveKnowledge} form={form} layout="vertical">
|
||||
<Form.Item
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: t("form.title.required")
|
||||
}
|
||||
]}
|
||||
name="title"
|
||||
label={t("form.title.label")}>
|
||||
<Input size="large" placeholder={t("form.title.placeholder")} />
|
||||
</Form.Item>
|
||||
<Form.Item
|
||||
name="file"
|
||||
label={t("form.uploadFile.label")}
|
||||
rules={[
|
||||
{
|
||||
required: true,
|
||||
message: t("form.uploadFile.required")
|
||||
}
|
||||
]}
|
||||
getValueFromEvent={(e) => {
|
||||
if (Array.isArray(e)) {
|
||||
return e
|
||||
}
|
||||
return e?.fileList
|
||||
}}>
|
||||
<Upload.Dragger
|
||||
accept={".pdf, .csv, .txt"}
|
||||
multiple={true}
|
||||
maxCount={10}
|
||||
beforeUpload={(file) => {
|
||||
const allowedTypes = [
|
||||
"application/pdf",
|
||||
// "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"text/csv",
|
||||
"text/plain"
|
||||
]
|
||||
.map((type) => type.toLowerCase())
|
||||
.join(", ")
|
||||
|
||||
if (!allowedTypes.includes(file.type.toLowerCase())) {
|
||||
message.error(
|
||||
t("form.uploadFile.uploadError", { allowedTypes })
|
||||
)
|
||||
return Upload.LIST_IGNORE
|
||||
}
|
||||
|
||||
return false
|
||||
}}>
|
||||
<div className="p-3">
|
||||
<p className="ant-upload-drag-icon justify-center flex">
|
||||
<InboxIcon className="h-10 w-10 text-gray-400" />
|
||||
</p>
|
||||
<p className="ant-upload-text">
|
||||
{t("form.uploadFile.uploadText")}
|
||||
</p>
|
||||
<p className="ant-upload-hint">
|
||||
{t("form.uploadFile.uploadHint")}
|
||||
</p>
|
||||
</div>
|
||||
</Upload.Dragger>
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={isSaving}
|
||||
className="inline-flex w-full text-center justify-center items-center rounded-md border border-transparent bg-black px-2 py-2 text-md font-medium leading-4 text-white shadow-sm hover:bg-gray-800 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:ring-offset-2 dark:bg-white dark:text-gray-800 dark:hover:bg-gray-100 dark:focus:ring-gray-500 dark:focus:ring-offset-gray-100 disabled:opacity-50">
|
||||
{t("form.submit")}
|
||||
</button>
|
||||
</Form.Item>
|
||||
</Form>
|
||||
</Modal>
|
||||
)
|
||||
}
|
51
src/components/Option/Knowledge/KnowledgeSelect.tsx
Normal file
51
src/components/Option/Knowledge/KnowledgeSelect.tsx
Normal file
@ -0,0 +1,51 @@
|
||||
import { getAllKnowledge } from "@/db/knowledge"
|
||||
import { useQuery } from "@tanstack/react-query"
|
||||
import { Dropdown, Tooltip } from "antd"
|
||||
import { Blocks } from "lucide-react"
|
||||
import React from "react"
|
||||
import { useTranslation } from "react-i18next"
|
||||
|
||||
export const KnowledgeSelect: React.FC = () => {
|
||||
const { t } = useTranslation("playground")
|
||||
const { data } = useQuery({
|
||||
queryKey: ["getAllKnowledge"],
|
||||
queryFn: async () => {
|
||||
const data = await getAllKnowledge("finished")
|
||||
return data
|
||||
},
|
||||
refetchInterval: 1000
|
||||
})
|
||||
|
||||
return (
|
||||
<Dropdown
|
||||
menu={{
|
||||
items:
|
||||
data?.map((d) => ({
|
||||
key: d.id,
|
||||
label: (
|
||||
<div className="w-52 gap-2 text-lg truncate inline-flex line-clamp-3 items-center dark:border-gray-700">
|
||||
<div>
|
||||
<Blocks className="h-6 w-6 text-gray-400" />
|
||||
</div>
|
||||
{d.title}
|
||||
</div>
|
||||
),
|
||||
onClick: () => {}
|
||||
})) || [],
|
||||
style: {
|
||||
maxHeight: 500,
|
||||
overflowY: "scroll"
|
||||
},
|
||||
// hidescrollbars: true
|
||||
className: "no-scrollbar"
|
||||
}}
|
||||
placement={"topLeft"}
|
||||
trigger={["click"]}>
|
||||
<Tooltip title={t("tooltip.knowledge")}>
|
||||
<button type="button" className="dark:text-gray-300">
|
||||
<Blocks className="h-6 w-6" />
|
||||
</button>
|
||||
</Tooltip>
|
||||
</Dropdown>
|
||||
)
|
||||
}
|
138
src/components/Option/Knowledge/index.tsx
Normal file
138
src/components/Option/Knowledge/index.tsx
Normal file
@ -0,0 +1,138 @@
|
||||
import { useState } from "react"
|
||||
import { useTranslation } from "react-i18next"
|
||||
import { AddKnowledge } from "./AddKnowledge"
|
||||
import {
|
||||
useMutation,
|
||||
useQuery,
|
||||
useQueryClient
|
||||
} from "@tanstack/react-query"
|
||||
import { deleteKnowledge, getAllKnowledge } from "@/db/knowledge"
|
||||
import { Skeleton, Table, Tag, Tooltip, message } from "antd"
|
||||
import { Trash2 } from "lucide-react"
|
||||
|
||||
export const KnowledgeSettings = () => {
|
||||
const { t } = useTranslation(["knownledge", "common"])
|
||||
const [open, setOpen] = useState(false)
|
||||
const queryClient = useQueryClient()
|
||||
|
||||
const { data, status } = useQuery({
|
||||
queryKey: ["fetchAllKnowledge"],
|
||||
queryFn: () => getAllKnowledge(),
|
||||
refetchInterval: 1000
|
||||
})
|
||||
|
||||
const { mutate: deleteKnowledgeMutation, isPending: isDeleting } =
|
||||
useMutation({
|
||||
mutationFn: deleteKnowledge,
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: ["fetchAllKnowledge"]
|
||||
})
|
||||
|
||||
message.success(t("deleteSuccess"))
|
||||
},
|
||||
onError: (error) => {
|
||||
message.error(error.message)
|
||||
}
|
||||
})
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div>
|
||||
{/* Add new model button */}
|
||||
<div className="mb-6">
|
||||
<div className="-ml-4 -mt-2 flex flex-wrap items-center justify-end sm:flex-nowrap">
|
||||
<div className="ml-4 mt-2 flex-shrink-0">
|
||||
<button
|
||||
onClick={() => setOpen(true)}
|
||||
className="inline-flex items-center rounded-md border border-transparent bg-black px-2 py-2 text-md font-medium leading-4 text-white shadow-sm hover:bg-gray-800 focus:outline-none focus:ring-2 focus:ring-indigo-500 focus:ring-offset-2 dark:bg-white dark:text-gray-800 dark:hover:bg-gray-100 dark:focus:ring-gray-500 dark:focus:ring-offset-gray-100 disabled:opacity-50">
|
||||
{t("addBtn")}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{status === "pending" && <Skeleton paragraph={{ rows: 8 }} />}
|
||||
|
||||
{status === "success" && (
|
||||
<Table
|
||||
columns={[
|
||||
{
|
||||
title: t("columns.title"),
|
||||
dataIndex: "title",
|
||||
key: "title"
|
||||
},
|
||||
{
|
||||
title: t("columns.status"),
|
||||
dataIndex: "status",
|
||||
key: "status",
|
||||
render: (text: string) => (
|
||||
<Tag color="blue">{t(`status.${text}`)}</Tag>
|
||||
)
|
||||
},
|
||||
{
|
||||
title: t("columns.embeddings"),
|
||||
dataIndex: "embedding_model",
|
||||
key: "embedding_model"
|
||||
},
|
||||
{
|
||||
title: t("columns.createdAt"),
|
||||
dataIndex: "createdAt",
|
||||
key: "createdAt",
|
||||
render: (text: number) => new Date(text).toLocaleString()
|
||||
},
|
||||
{
|
||||
title: t("columns.action"),
|
||||
key: "action",
|
||||
render: (text: string, record: any) => (
|
||||
<div className="flex gap-4">
|
||||
<Tooltip title={t("tooltip.delete")}>
|
||||
<button
|
||||
disabled={isDeleting}
|
||||
onClick={() => {
|
||||
if (window.confirm(t("confirm.delete"))) {
|
||||
deleteKnowledgeMutation(record.id)
|
||||
}
|
||||
}}
|
||||
className="text-red-500 dark:text-red-400">
|
||||
<Trash2 className="w-5 h-5" />
|
||||
</button>
|
||||
</Tooltip>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
]}
|
||||
expandable={{
|
||||
expandedRowRender: (record) => (
|
||||
<Table
|
||||
pagination={false}
|
||||
columns={[
|
||||
{
|
||||
title: t("expandedColumns.name"),
|
||||
key: "filename",
|
||||
dataIndex: "filename"
|
||||
},
|
||||
{
|
||||
title: t("expandedColumns.type"),
|
||||
key: "type",
|
||||
dataIndex: "type"
|
||||
}
|
||||
]}
|
||||
dataSource={record.source}
|
||||
locale={{
|
||||
emptyText: t("common:noData")
|
||||
}}
|
||||
/>
|
||||
),
|
||||
defaultExpandAllRows: false
|
||||
}}
|
||||
bordered
|
||||
dataSource={data}
|
||||
rowKey={(record) => `${record.name}-${record.id}`}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<AddKnowledge open={open} setOpen={setOpen} />
|
||||
</div>
|
||||
)
|
||||
}
|
@ -4,7 +4,7 @@ import React from "react"
|
||||
import useDynamicTextareaSize from "~/hooks/useDynamicTextareaSize"
|
||||
import { toBase64 } from "~/libs/to-base64"
|
||||
import { useMessageOption } from "~/hooks/useMessageOption"
|
||||
import { Checkbox, Dropdown, Switch, Tooltip } from "antd"
|
||||
import { Checkbox, Dropdown, Select, Switch, Tooltip } from "antd"
|
||||
import { Image } from "antd"
|
||||
import { useSpeechRecognition } from "~/hooks/useSpeechRecognition"
|
||||
import { useWebUI } from "~/store/webui"
|
||||
@ -12,6 +12,7 @@ import { defaultEmbeddingModelForRag } from "~/services/ollama"
|
||||
import { ImageIcon, MicIcon, StopCircleIcon, X } from "lucide-react"
|
||||
import { getVariable } from "~/utils/select-varaible"
|
||||
import { useTranslation } from "react-i18next"
|
||||
import { KnowledgeSelect } from "../Knowledge/KnowledgeSelect"
|
||||
|
||||
type Props = {
|
||||
dropedFile: File | undefined
|
||||
@ -249,6 +250,7 @@ export const PlaygroundForm = ({ dropedFile }: Props) => {
|
||||
</Tooltip>
|
||||
</div>
|
||||
<div className="flex !justify-end gap-3">
|
||||
<KnowledgeSelect />
|
||||
<Tooltip title={t("tooltip.speechToText")}>
|
||||
<button
|
||||
type="button"
|
||||
@ -273,6 +275,7 @@ export const PlaygroundForm = ({ dropedFile }: Props) => {
|
||||
)}
|
||||
</button>
|
||||
</Tooltip>
|
||||
|
||||
<Tooltip title={t("tooltip.uploadImage")}>
|
||||
<button
|
||||
type="button"
|
||||
@ -284,7 +287,7 @@ export const PlaygroundForm = ({ dropedFile }: Props) => {
|
||||
}`}>
|
||||
<ImageIcon className="h-5 w-5" />
|
||||
</button>
|
||||
</Tooltip>
|
||||
</Tooltip>
|
||||
{!isSending ? (
|
||||
<Dropdown.Button
|
||||
htmlType="submit"
|
||||
|
@ -18,7 +18,7 @@ import {
|
||||
getAllPrompts,
|
||||
savePrompt,
|
||||
updatePrompt
|
||||
} from "~/libs/db"
|
||||
} from "@/db"
|
||||
|
||||
export const PromptBody = () => {
|
||||
const queryClient = useQueryClient()
|
||||
|
@ -1,7 +1,7 @@
|
||||
import { useQueryClient } from "@tanstack/react-query"
|
||||
import { useDarkMode } from "~/hooks/useDarkmode"
|
||||
import { useMessageOption } from "~/hooks/useMessageOption"
|
||||
import { PageAssitDatabase } from "~/libs/db"
|
||||
import { PageAssitDatabase } from "@/db"
|
||||
import { Select } from "antd"
|
||||
import { SUPPORTED_LANGUAGES } from "~/utils/supporetd-languages"
|
||||
import { MoonIcon, SunIcon } from "lucide-react"
|
||||
|
@ -3,7 +3,7 @@ import { Form, Input, Skeleton, Table, Tooltip, message } from "antd"
|
||||
import { Trash2 } from "lucide-react"
|
||||
import { Trans, useTranslation } from "react-i18next"
|
||||
import { SaveButton } from "~/components/Common/SaveButton"
|
||||
import { deleteWebshare, getAllWebshares, getUserId } from "~/libs/db"
|
||||
import { deleteWebshare, getAllWebshares, getUserId } from "@/db"
|
||||
import { getPageShareUrl, setPageShareUrl } from "~/services/ollama"
|
||||
import { verifyPageShareURL } from "~/utils/verify-page-share"
|
||||
|
||||
|
@ -5,7 +5,7 @@ import {
|
||||
formatToMessage,
|
||||
deleteByHistoryId,
|
||||
updateHistory
|
||||
} from "~/libs/db"
|
||||
} from "@/db"
|
||||
import { Empty, Skeleton } from "antd"
|
||||
import { useMessageOption } from "~/hooks/useMessageOption"
|
||||
import { PencilIcon, Trash2 } from "lucide-react"
|
||||
|
192
src/db/knowledge.ts
Normal file
192
src/db/knowledge.ts
Normal file
@ -0,0 +1,192 @@
|
||||
import { deleteVector, deleteVectorByFileId } from "./vector"
|
||||
|
||||
export type Source = {
|
||||
source_id: string
|
||||
type: string
|
||||
filename?: string
|
||||
content: string
|
||||
}
|
||||
|
||||
export type Knowledge = {
|
||||
id: string
|
||||
db_type: string
|
||||
title: string
|
||||
status: string
|
||||
embedding_model: string
|
||||
source: Source[]
|
||||
knownledge: any
|
||||
createdAt: number
|
||||
}
|
||||
export const generateID = () => {
|
||||
return "pa_knowledge_xxxx-xxxx-xxx-xxxx".replace(/[x]/g, () => {
|
||||
const r = Math.floor(Math.random() * 16)
|
||||
return r.toString(16)
|
||||
})
|
||||
}
|
||||
export class PageAssistKnowledge {
|
||||
db: chrome.storage.StorageArea
|
||||
|
||||
constructor() {
|
||||
this.db = chrome.storage.local
|
||||
}
|
||||
|
||||
getAll = async (): Promise<Knowledge[]> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(null, (result) => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
const data = Object.keys(result).map((key) => result[key])
|
||||
resolve(data)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
getById = async (id: string): Promise<Knowledge> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(id, (result) => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve(result[id])
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
create = async (knowledge: Knowledge): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.set({ [knowledge.id]: knowledge }, () => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
update = async (knowledge: Knowledge): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.set({ [knowledge.id]: knowledge }, () => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
delete = async (id: string): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.remove(id, () => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
deleteSource = async (id: string, source_id: string): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(id, (result) => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
const data = result[id] as Knowledge
|
||||
data.source = data.source.filter((s) => s.source_id !== source_id)
|
||||
this.db.set({ [id]: data }, () => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const createKnowledge = async ({
|
||||
source,
|
||||
title,
|
||||
embedding_model
|
||||
}: {
|
||||
title: string
|
||||
source: Source[]
|
||||
embedding_model: string
|
||||
}) => {
|
||||
const db = new PageAssistKnowledge()
|
||||
const id = generateID()
|
||||
const knowledge: Knowledge = {
|
||||
id,
|
||||
title,
|
||||
db_type: "knowledge",
|
||||
source,
|
||||
status: "pending",
|
||||
knownledge: {},
|
||||
embedding_model,
|
||||
createdAt: Date.now()
|
||||
}
|
||||
await db.create(knowledge)
|
||||
return knowledge
|
||||
}
|
||||
|
||||
export const getKnowledgeById = async (id: string) => {
|
||||
const db = new PageAssistKnowledge()
|
||||
return db.getById(id)
|
||||
}
|
||||
|
||||
export const updateKnowledgeStatus = async (id: string, status: string) => {
|
||||
const db = new PageAssistKnowledge()
|
||||
const knowledge = await db.getById(id)
|
||||
await db.update({
|
||||
...knowledge,
|
||||
status
|
||||
})
|
||||
}
|
||||
|
||||
export const getAllKnowledge = async (status?: string) => {
|
||||
const db = new PageAssistKnowledge()
|
||||
const data = await db.getAll()
|
||||
|
||||
if (status) {
|
||||
return data
|
||||
.filter((d) => d.db_type === "knowledge")
|
||||
.filter((d) => d.status === status)
|
||||
.map((d) => {
|
||||
d.source.forEach((s) => {
|
||||
delete s.content
|
||||
})
|
||||
return d
|
||||
})
|
||||
.sort((a, b) => b.createdAt - a.createdAt)
|
||||
}
|
||||
|
||||
return data
|
||||
.filter((d) => d.db_type === "knowledge")
|
||||
.map((d) => {
|
||||
d.source.forEach((s) => {
|
||||
delete s.content
|
||||
})
|
||||
return d
|
||||
})
|
||||
.sort((a, b) => b.createdAt - a.createdAt)
|
||||
}
|
||||
|
||||
export const deleteKnowledge = async (id: string) => {
|
||||
const db = new PageAssistKnowledge()
|
||||
await db.delete(id)
|
||||
await deleteVector(`vector:${id}`)
|
||||
}
|
||||
|
||||
export const deleteSource = async (id: string, source_id: string) => {
|
||||
const db = new PageAssistKnowledge()
|
||||
await db.deleteSource(id, source_id)
|
||||
await deleteVectorByFileId(`vector:${id}`, source_id)
|
||||
}
|
131
src/db/vector.ts
Normal file
131
src/db/vector.ts
Normal file
@ -0,0 +1,131 @@
|
||||
interface PageAssistVector {
|
||||
file_id: string
|
||||
content: string
|
||||
embedding: number[]
|
||||
metadata: Record<string, any>
|
||||
}
|
||||
|
||||
export type VectorData = {
|
||||
id: string
|
||||
vectors: PageAssistVector[]
|
||||
}
|
||||
|
||||
export class PageAssistVectorDb {
|
||||
db: chrome.storage.StorageArea
|
||||
|
||||
constructor() {
|
||||
this.db = chrome.storage.local
|
||||
}
|
||||
|
||||
insertVector = async (
|
||||
id: string,
|
||||
vector: PageAssistVector[]
|
||||
): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(id, (result) => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
const data = result[id] as VectorData
|
||||
if (!data) {
|
||||
console.log("Creating new vector")
|
||||
this.db.set({ [id]: { id, vectors: [vector] } }, () => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
} else {
|
||||
console.log("Concatenating vectors")
|
||||
this.db.set(
|
||||
{
|
||||
[id]: {
|
||||
...data,
|
||||
vectors: data.vectors.concat(vector)
|
||||
}
|
||||
},
|
||||
() => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
deleteVector = async (id: string): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.remove(id, () => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
deleteVectorByFileId = async (id: string, file_id: string): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(id, (result) => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
const data = result[id] as VectorData
|
||||
data.vectors = data.vectors.filter((v) => v.file_id !== file_id)
|
||||
this.db.set({ [id]: data }, () => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve()
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
getVector = async (id: string): Promise<VectorData> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
this.db.get(id, (result) => {
|
||||
if (chrome.runtime.lastError) {
|
||||
reject(chrome.runtime.lastError)
|
||||
} else {
|
||||
resolve(result[id] as VectorData)
|
||||
}
|
||||
})
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const insertVector = async (
|
||||
id: string,
|
||||
vector: PageAssistVector[]
|
||||
): Promise<void> => {
|
||||
const db = new PageAssistVectorDb()
|
||||
return db.insertVector(id, vector)
|
||||
}
|
||||
|
||||
export const getVector = async (id: string): Promise<VectorData> => {
|
||||
const db = new PageAssistVectorDb()
|
||||
return db.getVector(id)
|
||||
}
|
||||
|
||||
export const deleteVector = async (id: string): Promise<void> => {
|
||||
const db = new PageAssistVectorDb()
|
||||
return db.deleteVector(id)
|
||||
}
|
||||
|
||||
export const deleteVectorByFileId = async (
|
||||
id: string,
|
||||
file_id: string
|
||||
): Promise<void> => {
|
||||
const db = new PageAssistVectorDb()
|
||||
return db.deleteVectorByFileId(id, file_id)
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
|
||||
import { processKnowledge } from "@/libs/process-knowledge"
|
||||
import { getOllamaURL, isOllamaRunning } from "../services/ollama"
|
||||
const progressHuman = (completed: number, total: number) => {
|
||||
return ((completed / total) * 100).toFixed(0) + "%"
|
||||
@ -78,13 +78,16 @@ export default defineBackground({
|
||||
main() {
|
||||
chrome.runtime.onMessage.addListener(async (message) => {
|
||||
if (message.type === "sidepanel") {
|
||||
chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => {
|
||||
const tab = tabs[0]
|
||||
chrome.sidePanel.open({
|
||||
// tabId: tab.id!,
|
||||
windowId: tab.windowId!,
|
||||
})
|
||||
})
|
||||
chrome.tabs.query(
|
||||
{ active: true, currentWindow: true },
|
||||
async (tabs) => {
|
||||
const tab = tabs[0]
|
||||
chrome.sidePanel.open({
|
||||
// tabId: tab.id!,
|
||||
windowId: tab.windowId!
|
||||
})
|
||||
}
|
||||
)
|
||||
} else if (message.type === "pull_model") {
|
||||
const ollamaURL = await getOllamaURL()
|
||||
|
||||
@ -93,8 +96,7 @@ export default defineBackground({
|
||||
if (!isRunning) {
|
||||
chrome.action.setBadgeText({ text: "E" })
|
||||
chrome.action.setBadgeBackgroundColor({ color: "#FF0000" })
|
||||
chrome.action.setTitle({ title: "Ollama is not running"
|
||||
})
|
||||
chrome.action.setTitle({ title: "Ollama is not running" })
|
||||
setTimeout(() => {
|
||||
clearBadge()
|
||||
}, 5000)
|
||||
@ -111,12 +113,15 @@ export default defineBackground({
|
||||
chrome.commands.onCommand.addListener((command) => {
|
||||
switch (command) {
|
||||
case "execute_side_panel":
|
||||
chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => {
|
||||
const tab = tabs[0]
|
||||
chrome.sidePanel.open({
|
||||
windowId: tab.windowId!
|
||||
})
|
||||
})
|
||||
chrome.tabs.query(
|
||||
{ active: true, currentWindow: true },
|
||||
async (tabs) => {
|
||||
const tab = tabs[0]
|
||||
chrome.sidePanel.open({
|
||||
windowId: tab.windowId!
|
||||
})
|
||||
}
|
||||
)
|
||||
break
|
||||
default:
|
||||
break
|
||||
@ -131,14 +136,17 @@ export default defineBackground({
|
||||
|
||||
chrome.contextMenus.onClicked.addListener((info, tab) => {
|
||||
if (info.menuItemId === "open-side-panel-pa") {
|
||||
chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => {
|
||||
const tab = tabs[0]
|
||||
await chrome.sidePanel.open({
|
||||
windowId: tab.windowId!,
|
||||
})
|
||||
})
|
||||
chrome.tabs.query(
|
||||
{ active: true, currentWindow: true },
|
||||
async (tabs) => {
|
||||
const tab = tabs[0]
|
||||
await chrome.sidePanel.open({
|
||||
windowId: tab.windowId!
|
||||
})
|
||||
}
|
||||
)
|
||||
}
|
||||
})
|
||||
},
|
||||
persistent: true
|
||||
})
|
||||
})
|
||||
|
@ -1,4 +1,4 @@
|
||||
import { saveHistory, saveMessage } from "@/libs/db"
|
||||
import { saveHistory, saveMessage } from "@/db"
|
||||
import { ChatHistory } from "@/store/option"
|
||||
|
||||
export const saveMessageOnError = async ({
|
||||
|
@ -15,7 +15,7 @@ import {
|
||||
getPromptById,
|
||||
removeMessageUsingHistoryId,
|
||||
updateMessageByIndex
|
||||
} from "~/libs/db"
|
||||
} from "@/db"
|
||||
import { useNavigate } from "react-router-dom"
|
||||
import { notification } from "antd"
|
||||
import { getSystemPromptForWeb } from "~/web/web"
|
||||
|
@ -3,12 +3,13 @@ import playground from "@/assets/locale/en/playground.json";
|
||||
import common from "@/assets/locale/en/common.json";
|
||||
import sidepanel from "@/assets/locale/en/sidepanel.json";
|
||||
import settings from "@/assets/locale/en/settings.json";
|
||||
|
||||
import knownledge from "@/assets/locale/en/knownledge.json";
|
||||
|
||||
export const en = {
|
||||
option,
|
||||
playground,
|
||||
common,
|
||||
sidepanel,
|
||||
settings
|
||||
settings,
|
||||
knownledge
|
||||
}
|
@ -3,6 +3,7 @@ import playground from "@/assets/locale/ja-JP/playground.json";
|
||||
import common from "@/assets/locale/ja-JP/common.json";
|
||||
import sidepanel from "@/assets/locale/ja-JP/sidepanel.json";
|
||||
import settings from "@/assets/locale/ja-JP/settings.json";
|
||||
import knownledge from "@/assets/locale/ja-JP/knownledge.json";
|
||||
|
||||
|
||||
export const ja = {
|
||||
@ -10,5 +11,6 @@ export const ja = {
|
||||
playground,
|
||||
common,
|
||||
sidepanel,
|
||||
settings
|
||||
settings,
|
||||
knownledge
|
||||
}
|
@ -3,12 +3,13 @@ import playground from "@/assets/locale/ml/playground.json";
|
||||
import common from "@/assets/locale/ml/common.json";
|
||||
import sidepanel from "@/assets/locale/ml/sidepanel.json";
|
||||
import settings from "@/assets/locale/ml/settings.json";
|
||||
|
||||
import knownledge from "@/assets/locale/ml/knownledge.json";
|
||||
|
||||
export const ml = {
|
||||
option,
|
||||
playground,
|
||||
common,
|
||||
sidepanel,
|
||||
settings
|
||||
settings,
|
||||
knownledge
|
||||
}
|
@ -3,6 +3,7 @@ import playground from "@/assets/locale/zh/playground.json";
|
||||
import common from "@/assets/locale/zh/common.json";
|
||||
import sidepanel from "@/assets/locale/zh/sidepanel.json";
|
||||
import settings from "@/assets/locale/zh/settings.json";
|
||||
import knownledge from "@/assets/locale/zh/knownledge.json";
|
||||
|
||||
|
||||
export const zh = {
|
||||
@ -10,5 +11,6 @@ export const zh = {
|
||||
playground,
|
||||
common,
|
||||
sidepanel,
|
||||
settings
|
||||
settings,
|
||||
knownledge
|
||||
}
|
201
src/libs/PageAssistVectorStore.ts
Normal file
201
src/libs/PageAssistVectorStore.ts
Normal file
@ -0,0 +1,201 @@
|
||||
import { similarity as ml_distance_similarity } from "ml-distance"
|
||||
import { VectorStore } from "@langchain/core/vectorstores"
|
||||
import type { EmbeddingsInterface } from "@langchain/core/embeddings"
|
||||
import { Document } from "@langchain/core/documents"
|
||||
import { getVector, insertVector } from "@/db/vector"
|
||||
|
||||
/**
|
||||
* Interface representing a vector in memory. It includes the content
|
||||
* (text), the corresponding embedding (vector), and any associated
|
||||
* metadata.
|
||||
*/
|
||||
interface PageAssistVector {
|
||||
content: string
|
||||
embedding: number[]
|
||||
metadata: Record<string, any>
|
||||
}
|
||||
|
||||
/**
|
||||
* Interface for the arguments that can be passed to the
|
||||
* `MemoryVectorStore` constructor. It includes an optional `similarity`
|
||||
* function.
|
||||
*/
|
||||
export interface MemoryVectorStoreArgs {
|
||||
knownledge_id: string
|
||||
file_id?: string
|
||||
similarity?: typeof ml_distance_similarity.cosine
|
||||
}
|
||||
|
||||
/**
|
||||
* Class that extends `VectorStore` to store vectors in memory. Provides
|
||||
* methods for adding documents, performing similarity searches, and
|
||||
* creating instances from texts, documents, or an existing index.
|
||||
*/
|
||||
export class PageAssistVectorStore extends VectorStore {
|
||||
declare FilterType: (doc: Document) => boolean
|
||||
|
||||
knownledge_id: string
|
||||
|
||||
file_id?: string
|
||||
|
||||
// memoryVectors: PageAssistVector[] = []
|
||||
|
||||
similarity: typeof ml_distance_similarity.cosine
|
||||
|
||||
_vectorstoreType(): string {
|
||||
return "memory"
|
||||
}
|
||||
|
||||
constructor(embeddings: EmbeddingsInterface, args: MemoryVectorStoreArgs) {
|
||||
super(embeddings, args)
|
||||
|
||||
this.similarity = args?.similarity ?? ml_distance_similarity.cosine
|
||||
|
||||
this.knownledge_id = args?.knownledge_id!
|
||||
|
||||
this.file_id = args?.file_id
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to add documents to the memory vector store. It extracts the
|
||||
* text from each document, generates embeddings for them, and adds the
|
||||
* resulting vectors to the store.
|
||||
* @param documents Array of `Document` instances to be added to the store.
|
||||
* @returns Promise that resolves when all documents have been added.
|
||||
*/
|
||||
async addDocuments(documents: Document[]): Promise<void> {
|
||||
const texts = documents.map(({ pageContent }) => pageContent)
|
||||
return this.addVectors(
|
||||
await this.embeddings.embedDocuments(texts),
|
||||
documents
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to add vectors to the memory vector store. It creates
|
||||
* `PageAssistVector` instances for each vector and document pair and adds
|
||||
* them to the store.
|
||||
* @param vectors Array of vectors to be added to the store.
|
||||
* @param documents Array of `Document` instances corresponding to the vectors.
|
||||
* @returns Promise that resolves when all vectors have been added.
|
||||
*/
|
||||
async addVectors(vectors: number[][], documents: Document[]): Promise<void> {
|
||||
const memoryVectors = vectors.map((embedding, idx) => ({
|
||||
content: documents[idx].pageContent,
|
||||
embedding,
|
||||
metadata: documents[idx].metadata,
|
||||
file_id: this.file_id
|
||||
}))
|
||||
console.log(`vector:${this.knownledge_id}`)
|
||||
await insertVector(`vector:${this.knownledge_id}`, memoryVectors)
|
||||
}
|
||||
|
||||
/**
|
||||
* Method to perform a similarity search in the memory vector store. It
|
||||
* calculates the similarity between the query vector and each vector in
|
||||
* the store, sorts the results by similarity, and returns the top `k`
|
||||
* results along with their scores.
|
||||
* @param query Query vector to compare against the vectors in the store.
|
||||
* @param k Number of top results to return.
|
||||
* @param filter Optional filter function to apply to the vectors before performing the search.
|
||||
* @returns Promise that resolves with an array of tuples, each containing a `Document` and its similarity score.
|
||||
*/
|
||||
async similaritySearchVectorWithScore(
|
||||
query: number[],
|
||||
k: number,
|
||||
filter?: this["FilterType"]
|
||||
): Promise<[Document, number][]> {
|
||||
const filterFunction = (memoryVector: PageAssistVector) => {
|
||||
if (!filter) {
|
||||
return true
|
||||
}
|
||||
|
||||
const doc = new Document({
|
||||
metadata: memoryVector.metadata,
|
||||
pageContent: memoryVector.content
|
||||
})
|
||||
return filter(doc)
|
||||
}
|
||||
const pgVector = await getVector(`vector:${this.knownledge_id}`)
|
||||
const filteredMemoryVectors = pgVector.vectors.filter(filterFunction)
|
||||
const searches = filteredMemoryVectors
|
||||
.map((vector, index) => ({
|
||||
similarity: this.similarity(query, vector.embedding),
|
||||
index
|
||||
}))
|
||||
.sort((a, b) => (a.similarity > b.similarity ? -1 : 0))
|
||||
.slice(0, k)
|
||||
|
||||
const result: [Document, number][] = searches.map((search) => [
|
||||
new Document({
|
||||
metadata: filteredMemoryVectors[search.index].metadata,
|
||||
pageContent: filteredMemoryVectors[search.index].content
|
||||
}),
|
||||
search.similarity
|
||||
])
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Static method to create a `MemoryVectorStore` instance from an array of
|
||||
* texts. It creates a `Document` for each text and metadata pair, and
|
||||
* adds them to the store.
|
||||
* @param texts Array of texts to be added to the store.
|
||||
* @param metadatas Array or single object of metadata corresponding to the texts.
|
||||
* @param embeddings `Embeddings` instance used to generate embeddings for the texts.
|
||||
* @param dbConfig Optional `MemoryVectorStoreArgs` to configure the `MemoryVectorStore` instance.
|
||||
* @returns Promise that resolves with a new `MemoryVectorStore` instance.
|
||||
*/
|
||||
static async fromTexts(
|
||||
texts: string[],
|
||||
metadatas: object[] | object,
|
||||
embeddings: EmbeddingsInterface,
|
||||
dbConfig?: MemoryVectorStoreArgs
|
||||
): Promise<PageAssistVectorStore> {
|
||||
const docs: Document[] = []
|
||||
for (let i = 0; i < texts.length; i += 1) {
|
||||
const metadata = Array.isArray(metadatas) ? metadatas[i] : metadatas
|
||||
const newDoc = new Document({
|
||||
pageContent: texts[i],
|
||||
metadata
|
||||
})
|
||||
docs.push(newDoc)
|
||||
}
|
||||
return PageAssistVectorStore.fromDocuments(docs, embeddings, dbConfig)
|
||||
}
|
||||
|
||||
/**
|
||||
* Static method to create a `MemoryVectorStore` instance from an array of
|
||||
* `Document` instances. It adds the documents to the store.
|
||||
* @param docs Array of `Document` instances to be added to the store.
|
||||
* @param embeddings `Embeddings` instance used to generate embeddings for the documents.
|
||||
* @param dbConfig Optional `MemoryVectorStoreArgs` to configure the `MemoryVectorStore` instance.
|
||||
* @returns Promise that resolves with a new `MemoryVectorStore` instance.
|
||||
*/
|
||||
static async fromDocuments(
|
||||
docs: Document[],
|
||||
embeddings: EmbeddingsInterface,
|
||||
dbConfig?: MemoryVectorStoreArgs
|
||||
): Promise<PageAssistVectorStore> {
|
||||
const instance = new this(embeddings, dbConfig)
|
||||
await instance.addDocuments(docs)
|
||||
return instance
|
||||
}
|
||||
|
||||
/**
|
||||
* Static method to create a `MemoryVectorStore` instance from an existing
|
||||
* index. It creates a new `MemoryVectorStore` instance without adding any
|
||||
* documents or vectors.
|
||||
* @param embeddings `Embeddings` instance used to generate embeddings for the documents.
|
||||
* @param dbConfig Optional `MemoryVectorStoreArgs` to configure the `MemoryVectorStore` instance.
|
||||
* @returns Promise that resolves with a new `MemoryVectorStore` instance.
|
||||
*/
|
||||
static async fromExistingIndex(
|
||||
embeddings: EmbeddingsInterface,
|
||||
dbConfig?: MemoryVectorStoreArgs
|
||||
): Promise<PageAssistVectorStore> {
|
||||
const instance = new this(embeddings, dbConfig)
|
||||
return instance
|
||||
}
|
||||
}
|
@ -1,28 +1,4 @@
|
||||
import { pdfDist } from "./pdfjs"
|
||||
|
||||
export const getPdf = async (data: ArrayBuffer) => {
|
||||
const pdf = pdfDist.getDocument({
|
||||
data,
|
||||
useWorkerFetch: false,
|
||||
isEvalSupported: false,
|
||||
useSystemFonts: true,
|
||||
});
|
||||
|
||||
pdf.onPassword = (callback: any) => {
|
||||
const password = prompt("Enter the password: ")
|
||||
if (!password) {
|
||||
throw new Error("Password required to open the PDF.");
|
||||
}
|
||||
callback(password);
|
||||
};
|
||||
|
||||
|
||||
const pdfDocument = await pdf.promise;
|
||||
|
||||
|
||||
return pdfDocument
|
||||
|
||||
}
|
||||
import { getPdf } from "./pdf"
|
||||
|
||||
const _getHtml = async () => {
|
||||
const url = window.location.href
|
||||
|
29
src/libs/pdf.ts
Normal file
29
src/libs/pdf.ts
Normal file
@ -0,0 +1,29 @@
|
||||
import { pdfDist } from "./pdfjs"
|
||||
|
||||
export const getPdf = async (data: ArrayBuffer) => {
|
||||
const pdf = pdfDist.getDocument({
|
||||
data,
|
||||
useWorkerFetch: false,
|
||||
isEvalSupported: false,
|
||||
useSystemFonts: true
|
||||
})
|
||||
|
||||
pdf.onPassword = (callback: any) => {
|
||||
const password = prompt("Enter the password: ")
|
||||
if (!password) {
|
||||
throw new Error("Password required to open the PDF.")
|
||||
}
|
||||
callback(password)
|
||||
}
|
||||
|
||||
const pdfDocument = await pdf.promise
|
||||
|
||||
return pdfDocument
|
||||
}
|
||||
|
||||
export const processPdf = async (base64: string) => {
|
||||
const res = await fetch(base64)
|
||||
const data = await res.arrayBuffer()
|
||||
const pdf = await getPdf(data)
|
||||
return pdf
|
||||
}
|
55
src/libs/process-knowledge.ts
Normal file
55
src/libs/process-knowledge.ts
Normal file
@ -0,0 +1,55 @@
|
||||
import { getKnowledgeById, updateKnowledgeStatus } from "@/db/knowledge"
|
||||
import { PageAssistPDFUrlLoader } from "@/loader/pdf-url"
|
||||
import {
|
||||
defaultEmbeddingChunkOverlap,
|
||||
defaultEmbeddingChunkSize
|
||||
} from "@/services/ollama"
|
||||
import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama"
|
||||
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
|
||||
import { PageAssistVectorStore } from "./PageAssistVectorStore"
|
||||
|
||||
export const processKnowledge = async (msg: any, id: string): Promise<void> => {
|
||||
console.log(`Processing knowledge with id: ${id}`)
|
||||
try {
|
||||
const knowledge = await getKnowledgeById(id)
|
||||
|
||||
if (!knowledge) {
|
||||
console.error(`Knowledge with id ${id} not found`)
|
||||
return
|
||||
}
|
||||
|
||||
await updateKnowledgeStatus(id, "processing")
|
||||
|
||||
const ollamaEmbedding = new OllamaEmbeddings({
|
||||
model: knowledge.embedding_model
|
||||
})
|
||||
const chunkSize = await defaultEmbeddingChunkSize()
|
||||
const chunkOverlap = await defaultEmbeddingChunkOverlap()
|
||||
const textSplitter = new RecursiveCharacterTextSplitter({
|
||||
chunkSize,
|
||||
chunkOverlap
|
||||
})
|
||||
|
||||
for (const doc of knowledge.source) {
|
||||
if (doc.type === "pdf" || doc.type === "application/pdf") {
|
||||
const loader = new PageAssistPDFUrlLoader({
|
||||
name: doc.filename,
|
||||
url: doc.content
|
||||
})
|
||||
let docs = await loader.load()
|
||||
const chunks = await textSplitter.splitDocuments(docs)
|
||||
await PageAssistVectorStore.fromDocuments(chunks, ollamaEmbedding, {
|
||||
knownledge_id: knowledge.id,
|
||||
file_id: doc.source_id
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
await updateKnowledgeStatus(id, "finished")
|
||||
} catch (error) {
|
||||
console.error(`Error processing knowledge with id: ${id}`, error)
|
||||
await updateKnowledgeStatus(id, "failed")
|
||||
} finally {
|
||||
console.log(`Finished processing knowledge with id: ${id}`)
|
||||
}
|
||||
}
|
49
src/loader/pdf-url.ts
Normal file
49
src/loader/pdf-url.ts
Normal file
@ -0,0 +1,49 @@
|
||||
import { BaseDocumentLoader } from "langchain/document_loaders/base"
|
||||
import { Document } from "@langchain/core/documents"
|
||||
import { processPdf } from "@/libs/pdf"
|
||||
export interface WebLoaderParams {
|
||||
url: string
|
||||
name: string
|
||||
}
|
||||
|
||||
export class PageAssistPDFUrlLoader
|
||||
extends BaseDocumentLoader
|
||||
implements WebLoaderParams
|
||||
{
|
||||
pdf: { content: string; page: number }[]
|
||||
url: string
|
||||
name: string
|
||||
|
||||
constructor({ url, name }: WebLoaderParams) {
|
||||
super()
|
||||
this.url = url
|
||||
this.name = name
|
||||
}
|
||||
|
||||
async load(): Promise<Document<Record<string, any>>[]> {
|
||||
const documents: Document[] = []
|
||||
|
||||
const data = await processPdf(this.url)
|
||||
|
||||
for (let i = 1; i <= data.numPages; i += 1) {
|
||||
const page = await data.getPage(i)
|
||||
const content = await page.getTextContent()
|
||||
|
||||
if (content?.items.length === 0) {
|
||||
continue
|
||||
}
|
||||
|
||||
const text = content?.items
|
||||
.map((item: any) => item.str)
|
||||
.join("\n")
|
||||
.replace(/\x00/g, "")
|
||||
.trim()
|
||||
documents.push({
|
||||
pageContent: text,
|
||||
metadata: { source: this.name, page: i }
|
||||
})
|
||||
}
|
||||
|
||||
return documents
|
||||
}
|
||||
}
|
@ -1,37 +1,36 @@
|
||||
import { BaseDocumentLoader } from "langchain/document_loaders/base"
|
||||
import { Document } from "@langchain/core/documents"
|
||||
export interface WebLoaderParams {
|
||||
pdf: { content: string, page: number }[]
|
||||
url: string
|
||||
pdf: { content: string; page: number }[]
|
||||
url: string
|
||||
}
|
||||
|
||||
export class PageAssistPDFLoader
|
||||
extends BaseDocumentLoader
|
||||
implements WebLoaderParams {
|
||||
pdf: { content: string, page: number }[]
|
||||
url: string
|
||||
extends BaseDocumentLoader
|
||||
implements WebLoaderParams
|
||||
{
|
||||
pdf: { content: string; page: number }[]
|
||||
url: string
|
||||
|
||||
constructor({ pdf, url }: WebLoaderParams) {
|
||||
super()
|
||||
this.pdf = pdf
|
||||
this.url = url
|
||||
constructor({ pdf, url }: WebLoaderParams) {
|
||||
super()
|
||||
this.pdf = pdf
|
||||
this.url = url
|
||||
}
|
||||
|
||||
async load(): Promise<Document<Record<string, any>>[]> {
|
||||
const documents: Document[] = []
|
||||
|
||||
for (const page of this.pdf) {
|
||||
const metadata = { source: this.url, page: page.page }
|
||||
documents.push(new Document({ pageContent: page.content, metadata }))
|
||||
}
|
||||
|
||||
async load(): Promise<Document<Record<string, any>>[]> {
|
||||
const documents: Document[] = [];
|
||||
|
||||
for (const page of this.pdf) {
|
||||
const metadata = { source: this.url, page: page.page }
|
||||
documents.push(new Document({ pageContent: page.content, metadata }))
|
||||
}
|
||||
|
||||
return [
|
||||
new Document({
|
||||
pageContent: documents.map((doc) => doc.pageContent).join("\n\n"),
|
||||
metadata: documents.map((doc) => doc.metadata),
|
||||
}),
|
||||
];
|
||||
|
||||
|
||||
}
|
||||
return [
|
||||
new Document({
|
||||
pageContent: documents.map((doc) => doc.pageContent).join("\n\n"),
|
||||
metadata: documents.map((doc) => doc.metadata)
|
||||
})
|
||||
]
|
||||
}
|
||||
}
|
||||
|
6
src/queue/index.ts
Normal file
6
src/queue/index.ts
Normal file
@ -0,0 +1,6 @@
|
||||
import { processKnowledge } from "@/libs/process-knowledge"
|
||||
import PubSub from "pubsub-js"
|
||||
|
||||
export const KNOWLEDGE_QUEUE = Symbol("queue")
|
||||
|
||||
PubSub.subscribe(KNOWLEDGE_QUEUE, processKnowledge)
|
@ -1,11 +1,12 @@
|
||||
import { SettingsLayout } from "~/components/Layouts/SettingsOptionLayout"
|
||||
import OptionLayout from "~/components/Layouts/Layout"
|
||||
import { KnowledgeSettings } from "@/components/Option/Knowledge"
|
||||
|
||||
export const OptionKnowledgeBase = () => {
|
||||
return (
|
||||
<OptionLayout>
|
||||
<SettingsLayout>
|
||||
hey
|
||||
<KnowledgeSettings />
|
||||
</SettingsLayout>
|
||||
</OptionLayout>
|
||||
)
|
||||
|
32
src/utils/to-source.ts
Normal file
32
src/utils/to-source.ts
Normal file
@ -0,0 +1,32 @@
|
||||
import { Source } from "@/db/knowledge"
|
||||
import { UploadFile } from "antd"
|
||||
|
||||
export const toBase64 = (file: File | Blob): Promise<string> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader()
|
||||
reader.readAsDataURL(file)
|
||||
reader.onload = () => resolve(reader.result as string)
|
||||
reader.onerror = (error) => reject(error)
|
||||
})
|
||||
}
|
||||
|
||||
export const toArrayBufferFromBase64 = async (base64: string) => {
|
||||
const res = await fetch(base64)
|
||||
const blob = await res.blob()
|
||||
return await blob.arrayBuffer()
|
||||
}
|
||||
|
||||
export const generateSourceId = () => {
|
||||
return "XXXXXXXX-XXXX-4XXX-YXXX-XXXXXXXXXXXX".replace(/[XY]/g, (c) => {
|
||||
const r = (Math.random() * 16) | 0
|
||||
const v = c === "X" ? r : (r & 0x3) | 0x8
|
||||
return v.toString(16)
|
||||
})
|
||||
}
|
||||
|
||||
export const convertToSource = async (file: UploadFile): Promise<Source> => {
|
||||
let type = file.type
|
||||
let filename = file.name
|
||||
const content = await toBase64(file.originFileObj)
|
||||
return { content, type, filename, source_id: generateSourceId() }
|
||||
}
|
@ -24,7 +24,7 @@ export default defineConfig({
|
||||
srcDir: "src",
|
||||
outDir: "build",
|
||||
manifest: {
|
||||
version: "1.1.1",
|
||||
version: "1.1.2",
|
||||
name: '__MSG_extName__',
|
||||
description: '__MSG_extDescription__',
|
||||
default_locale: 'en',
|
||||
|
Loading…
x
Reference in New Issue
Block a user