iod/page-assist
6
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add @langchain/core dependency and update imports***

Browse Source 
***Update SidepanelRouting to use dark mode***
***Add image support to PlaygroundMessage component
This commit is contained in:
n4ze3m 2024-02-03 17:51:11 +05:30
parent e6130f11da
commit be3a4ed256
13 changed files with 294 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -14,6 +14,7 @@
"@headlessui/react": "^1.7.18",
"@heroicons/react": "^2.1.1",
"@langchain/community": "^0.0.21",
"@langchain/core": "^0.1.22",
"@mantine/form": "^7.5.0",
"@plasmohq/storage": "^1.9.0",
"@tailwindcss/forms": "^0.5.7",

2
src/chain/chat-with-website.ts
View File

@ -1,5 +1,5 @@
import { BaseLanguageModel } from "langchain/base_language";
import { Document } from "langchain/document";
import { Document } from "@langchain/core/documents";
import {
ChatPromptTemplate,
MessagesPlaceholder,

25
src/components/Common/Playground/Message.tsx
View File

@ -1,7 +1,4 @@
import {
CheckIcon,
ClipboardIcon,
} from "@heroicons/react/24/outline"
import { CheckIcon, ClipboardIcon } from "@heroicons/react/24/outline"
import Markdown from "../../Common/Markdown"
import React from "react"
@ -12,6 +9,7 @@ type Props = {
userAvatar?: JSX.Element
isBot: boolean
name: string
images?: string[]
}
export const PlaygroundMessage = (props: Props) => {
@ -48,13 +46,11 @@ export const PlaygroundMessage = (props: Props) => {
</div>
</div>
<div className="relative flex w-[calc(100%-50px)] flex-col gap-1 md:gap-3 lg:w-[calc(100%-115px)]">
{
props.isBot && (
{props.isBot && (
<span className="absolute mb-8 -top-4 left-0 text-xs text-gray-400 dark:text-gray-500">
{props.name}
</span>
)
}
)}
<div className="flex flex-grow flex-col gap-3">
<Markdown message={props.message} />
</div>
@ -81,6 +77,19 @@ export const PlaygroundMessage = (props: Props) => {
)}
</div>
</div>
{props.images && (
<div className="flex md:max-w-2xl lg:max-w-xl xl:max-w-3xl p-3 m-auto w-full">
{props.images.map((image, index) => (
<div key={index} className="h-full rounded-md shadow relative">
<img
src={image}
alt="Uploaded"
className="h-full w-auto object-cover rounded-md min-w-[50px]"
/>
</div>
))}
</div>
)}
</div>
)
}

1
src/components/Sidepanel/Chat/body.tsx
View File

@ -20,6 +20,7 @@ export const SidePanelBody = () => {
isBot={message.isBot}
message={message.message}
name={message.name}
images={message.images || []}
/>
))}
<div className="w-full h-32 md:h-48 flex-shrink-0"></div>

91
src/components/Sidepanel/Chat/form.tsx
View File

@ -3,9 +3,17 @@ import { useMutation } from "@tanstack/react-query"
import React from "react"
import useDynamicTextareaSize from "~hooks/useDynamicTextareaSize"
import { useMessage } from "~hooks/useMessage"
import PhotoIcon from "@heroicons/react/24/outline/PhotoIcon"
import XMarkIcon from "@heroicons/react/24/outline/XMarkIcon"
import { toBase64 } from "~libs/to-base64"
export const SidepanelForm = () => {
type Props = {
dropedFile: File | undefined
}
export const SidepanelForm = ({ dropedFile }: Props) => {
const textareaRef = React.useRef<HTMLTextAreaElement>(null)
const inputRef = React.useRef<HTMLInputElement>(null)
const resetHeight = () => {
const textarea = textareaRef.current
@ -15,16 +23,34 @@ export const SidepanelForm = () => {
}
const form = useForm({
initialValues: {
message: ""
message: "",
image: ""
}
})
useDynamicTextareaSize(
textareaRef,
form.values.message,
)
const onInputChange = async (
e: React.ChangeEvent<HTMLInputElement> | File
) => {
if (e instanceof File) {
const base64 = await toBase64(e)
form.setFieldValue("image", base64)
} else {
if (e.target.files) {
const base64 = await toBase64(e.target.files[0])
form.setFieldValue("image", base64)
}
}
}
const { onSubmit, selectedModel } = useMessage()
React.useEffect(() => {
if (dropedFile) {
onInputChange(dropedFile)
}
}, [dropedFile])
useDynamicTextareaSize(textareaRef, form.values.message, 120)
const { onSubmit, selectedModel, chatMode } = useMessage()
const { mutateAsync: sendMessage, isPending: isSending } = useMutation({
mutationFn: onSubmit
@ -33,6 +59,24 @@ export const SidepanelForm = () => {
return (
<div className="p-3 md:p-6 md:bg-white dark:bg-[#0a0a0a] border rounded-t-xl border-black/10 dark:border-gray-900/50">
<div className="flex-grow space-y-6 ">
{chatMode === "normal" && form.values.image && (
<div className="h-full rounded-md shadow relative">
<div>
<img
src={form.values.image}
alt="Uploaded"
className="h-full w-auto object-cover rounded-md min-w-[50px]"
/>
<button
onClick={() => {
form.setFieldValue("image", "")
}}
className="absolute top-2 right-2 bg-white dark:bg-black p-1 rounded-full hover:bg-gray-100 dark:hover:bg-gray-800 text-black dark:text-gray-100">
<XMarkIcon className="h-5 w-5" />
</button>
</div>
</div>
)}
<div className="flex">
<form
onSubmit={form.onSubmit(async (value) => {
@ -42,10 +86,34 @@ export const SidepanelForm = () => {
}
form.reset()
resetHeight()
await sendMessage(value.message)
await sendMessage({
image: value.image,
message: value.message.trim()
})
})}
className="shrink-0 flex-grow flex items-center ">
<div className="flex items-center p-2 rounded-2xl border bg-gray-100 w-full dark:bg-black dark:border-gray-800">
<button
type="button"
onClick={() => {
inputRef.current?.click()
}}
className={`flex ml-3 items-center justify-center dark:text-gray-100 ${
chatMode === "rag" ? "hidden" : "block"
}`}>
<PhotoIcon className="h-5 w-5" />
</button>
<input
id="file-upload"
name="file-upload"
type="file"
className="sr-only"
ref={inputRef}
accept="image/*"
multiple={false}
onChange={onInputChange}
/>
<textarea
onKeyDown={(e) => {
if (e.key === "Enter" && !e.shiftKey && !isSending) {
@ -60,12 +128,15 @@ export const SidepanelForm = () => {
}
form.reset()
resetHeight()
await sendMessage(value.message)
await sendMessage({
image: value.image,
message: value.message.trim()
})
})()
}
}}
ref={textareaRef}
className="rounded-full pl-4 pr-2 py-2 w-full resize-none bg-transparent focus-within:outline-none sm:text-sm focus:ring-0 focus-visible:ring-0 ring-0 dark:ring-0 border-0 dark:text-gray-100"
className="pl-4 pr-2 py-2 w-full resize-none bg-transparent focus-within:outline-none sm:text-sm focus:ring-0 focus-visible:ring-0 ring-0 dark:ring-0 border-0 dark:text-gray-100"
required
rows={1}
tabIndex={0}

93
src/hooks/useMessage.tsx
View File

@ -3,12 +3,15 @@ import { cleanUrl } from "~libs/clean-url"
import { getOllamaURL, systemPromptForNonRag } from "~services/ollama"
import { useStoreMessage, type ChatHistory, type Message } from "~store"
import { ChatOllama } from "@langchain/community/chat_models/ollama"
import { HumanMessage, AIMessage } from "@langchain/core/messages"
import {
HumanMessage,
AIMessage,
type MessageContent
} from "@langchain/core/messages"
import { getHtmlOfCurrentTab } from "~libs/get-html"
import { PageAssistHtmlLoader } from "~loader/html"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { OllamaEmbeddings } from "langchain/embeddings/ollama"
import { Voy as VoyClient } from "voy-search"
import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama"
import { createChatWithWebsiteChain } from "~chain/chat-with-website"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
@ -25,19 +28,34 @@ const generateHistory = (
messages: {
role: "user" | "assistant" | "system"
content: string
image?: string
}[]
) => {
let history = []
for (const message of messages) {
if (message.role === "user") {
let content: MessageContent = [
{
type: "text",
text: message.content
}
]
if (message.image) {
content = [
{
type: "image_url",
image_url: message.image
},
{
type: "text",
text: message.content
}
]
}
history.push(
new HumanMessage({
content: [
{
type: "text",
text: message.content
}
]
content: content
})
)
} else if (message.role === "assistant") {
@ -239,9 +257,12 @@ export const useMessage = () => {
}
}
const normalChatMode = async (message: string) => {
const normalChatMode = async (message: string, image: string) => {
const url = await getOllamaURL()
if (image.length > 0) {
image = `data:image/jpeg;base64,${image.split(",")[1]}`
}
abortControllerRef.current = new AbortController()
const ollama = new ChatOllama({
@ -255,13 +276,14 @@ export const useMessage = () => {
isBot: false,
name: "You",
message,
sources: []
sources: [],
images: [image]
},
{
isBot: true,
name: selectedModel,
message: "▋",
sources: []
sources: [],
}
]
@ -271,17 +293,35 @@ export const useMessage = () => {
try {
const prompt = await systemPromptForNonRag()
let humanMessage = new HumanMessage({
content: [
{
text: message,
type: "text"
}
]
})
if (image.length > 0) {
humanMessage = new HumanMessage({
content: [
{
text: message,
type: "text"
},
{
image_url: image,
type: "image_url"
}
]
})
}
console.log("humanMessage", humanMessage)
const chunks = await ollama.stream(
[
...generateHistory(history),
new HumanMessage({
content: [
{
type: "text",
text: message
}
]
})
humanMessage
],
{
signal: abortControllerRef.current.signal
@ -312,7 +352,8 @@ export const useMessage = () => {
...history,
{
role: "user",
content: message
content: message,
image
},
{
role: "assistant",
@ -342,9 +383,15 @@ export const useMessage = () => {
}
}
const onSubmit = async (message: string) => {
const onSubmit = async ({
message,
image
}: {
message: string
image: string
}) => {
if (chatMode === "normal") {
await normalChatMode(message)
await normalChatMode(message, image)
} else {
await chatWithWebsiteMode(message)
}

7
src/libs/to-base64.ts Normal file
View File

@ -0,0 +1,7 @@
export const toBase64 = (file: File) =>
new Promise<string>((resolve, reject) => {
const reader = new FileReader()
reader.readAsDataURL(file)
reader.onload = () => resolve(reader.result as string)
reader.onerror = (error) => reject(error)
})

2
src/loader/html.ts
View File

@ -1,5 +1,5 @@
import { BaseDocumentLoader } from "langchain/document_loaders/base"
import { Document } from "langchain/document"
import { Document } from "@langchain/core/documents"
import { compile } from "html-to-text"
export interface WebLoaderParams {

21
src/routes/index.tsx
View File

@ -1,14 +1,19 @@
import { Route, Routes } from "react-router-dom"
import { SidepanelChat } from "./sidepanel-chat"
import { SidepanelSettingsHeader } from "~components/Sidepanel/Settings/header"
import { useDarkMode } from "~hooks/useDarkmode"
export const Routing = () => <Routes></Routes>
export const SidepanelRouting = () => (
<div className="dark">
<Routes>
<Route path="/" element={<SidepanelChat />} />
<Route path="/settings" element={<SidepanelSettingsHeader />} />
</Routes>
</div>
)
export const SidepanelRouting = () => {
const { mode } = useDarkMode()
return (
<div className={mode === "dark" ? "dark" : "light"}>
<Routes>
<Route path="/" element={<SidepanelChat />} />
<Route path="/settings" element={<SidepanelSettingsHeader />} />
</Routes>
</div>
)
}

74
src/routes/sidepanel-chat.tsx
View File

@ -1,10 +1,80 @@
import React from "react"
import { SidePanelBody } from "~components/Sidepanel/Chat/body"
import { SidepanelForm } from "~components/Sidepanel/Chat/form"
import { SidepanelHeader } from "~components/Sidepanel/Chat/header"
import { useMessage } from "~hooks/useMessage"
export const SidepanelChat = () => {
const drop = React.useRef<HTMLDivElement>(null)
const [dropedFile, setDropedFile] = React.useState<File | undefined>()
const [dropState, setDropState] = React.useState<
"idle" | "dragging" | "error"
>("idle")
const {chatMode} = useMessage()
React.useEffect(() => {
if (!drop.current) {
return
}
const handleDragOver = (e: DragEvent) => {
e.preventDefault()
e.stopPropagation()
}
const handleDrop = (e: DragEvent) => {
e.preventDefault()
e.stopPropagation()
setDropState("idle")
const files = Array.from(e.dataTransfer?.files || [])
const isImage = files.every((file) => file.type.startsWith("image/"))
if (!isImage) {
setDropState("error")
return
}
const newFiles = Array.from(e.dataTransfer?.files || []).slice(0, 1)
if (newFiles.length > 0) {
setDropedFile(newFiles[0])
}
}
const handleDragEnter = (e: DragEvent) => {
e.preventDefault()
e.stopPropagation()
setDropState("dragging")
}
const handleDragLeave = (e: DragEvent) => {
e.preventDefault()
e.stopPropagation()
setDropState("idle")
}
drop.current.addEventListener("dragover", handleDragOver)
drop.current.addEventListener("drop", handleDrop)
drop.current.addEventListener("dragenter", handleDragEnter)
drop.current.addEventListener("dragleave", handleDragLeave)
return () => {
if (drop.current) {
drop.current.removeEventListener("dragover", handleDragOver)
drop.current.removeEventListener("drop", handleDrop)
drop.current.removeEventListener("dragenter", handleDragEnter)
drop.current.removeEventListener("dragleave", handleDragLeave)
}
}
}, [])
return (
<div className="flex bg-white dark:bg-black flex-col min-h-screen mx-auto max-w-7xl">
<div
ref={drop}
className={`flex ${
dropState === "dragging" && chatMode === "normal"
? "bg-gray-100 dark:bg-gray-800 z-10"
: "bg-white dark:bg-black"
} flex-col min-h-screen mx-auto max-w-7xl`}>
<div className="sticky top-0 z-10">
<SidepanelHeader />
</div>
@ -13,7 +83,7 @@ export const SidepanelChat = () => {
<div className="bottom-0 w-full bg-transparent border-0 fixed pt-2">
<div className="stretch mx-2 flex flex-row gap-3 md:mx-4 lg:mx-auto lg:max-w-2xl xl:max-w-3xl">
<div className="relative flex flex-col h-full flex-1 items-stretch md:flex-col">
<SidepanelForm />
<SidepanelForm dropedFile={dropedFile} />
</div>
</div>
</div>

11
src/services/ollama.ts
View File

@ -68,8 +68,15 @@ export const fetchModels = async () => {
}
export const setOllamaURL = async (ollamaURL: string) => {
await chromeRunTime(cleanUrl(ollamaURL))
await storage.set("ollamaURL", cleanUrl(ollamaURL))
let formattedUrl = ollamaURL
if (formattedUrl.startsWith("http://localhost:")) {
formattedUrl = formattedUrl.replace(
"http://localhost:",
"http://127.0.0.1:"
)
}
await chromeRunTime(cleanUrl(formattedUrl))
await storage.set("ollamaURL", cleanUrl(formattedUrl))
}
export const systemPromptForNonRag = async () => {

4
src/store/index.tsx
View File

@ -5,11 +5,13 @@ export type Message = {
name: string
message: string
sources: any[]
images?: string[]
}
export type ChatHistory = {
role: "user" | "assistant" | "system"
content: string
content: string,
image?: string
}[]
type State = {

17
yarn.lock
View File

@ -537,6 +537,23 @@
uuid "^9.0.0"
zod "^3.22.3"
"@langchain/core@^0.1.22":
version "0.1.22"
resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.22.tgz#b558f4675f5ad4dea473899362469220a96b4144"
integrity sha512-I3KMv87D5AFeAvuJhzaGOYdppFL4h/bRm7LeJfwF2PspQIZwvDE9GP7hkw4n+7jwNaBxjU8ZTj6o3LZAh1R5LQ==
dependencies:
ansi-styles "^5.0.0"
camelcase "6"
decamelize "1.2.0"
js-tiktoken "^1.0.8"
langsmith "~0.0.48"
ml-distance "^4.0.0"
p-queue "^6.6.2"
p-retry "4"
uuid "^9.0.0"
zod "^3.22.4"
zod-to-json-schema "^3.22.3"
"@langchain/core@~0.1.13", "@langchain/core@~0.1.16":
version "0.1.20"
resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.20.tgz#599d3a8de1faa4692b03d3e8939d8e08e1dc2413"