Add @langchain/core dependency and update imports***

***Update SidepanelRouting to use dark mode***
***Add image support to PlaygroundMessage component

package.json

@@ -14,6 +14,7 @@
     "@headlessui/react": "^1.7.18",
     "@heroicons/react": "^2.1.1",
     "@langchain/community": "^0.0.21",
+    "@langchain/core": "^0.1.22",
     "@mantine/form": "^7.5.0",
     "@plasmohq/storage": "^1.9.0",
     "@tailwindcss/forms": "^0.5.7",

src/chain/chat-with-website.ts

@@ -1,5 +1,5 @@
 import { BaseLanguageModel } from "langchain/base_language";
-import { Document } from "langchain/document";
+import { Document } from "@langchain/core/documents";
 import {
   ChatPromptTemplate,
   MessagesPlaceholder,

src/components/Common/Playground/Message.tsx

@@ -1,7 +1,4 @@
-import {
+import { CheckIcon, ClipboardIcon } from "@heroicons/react/24/outline"
-  CheckIcon,
-  ClipboardIcon,
-} from "@heroicons/react/24/outline"
 import Markdown from "../../Common/Markdown"
 import React from "react"
 
@@ -12,6 +9,7 @@ type Props = {
   userAvatar?: JSX.Element
   isBot: boolean
   name: string
+  images?: string[]
 }
 
 export const PlaygroundMessage = (props: Props) => {
@@ -48,13 +46,11 @@ export const PlaygroundMessage = (props: Props) => {
             </div>
           </div>
           <div className="relative flex w-[calc(100%-50px)] flex-col gap-1 md:gap-3 lg:w-[calc(100%-115px)]">
-          {
+            {props.isBot && (
-            props.isBot && (
               <span className="absolute mb-8 -top-4 left-0 text-xs text-gray-400 dark:text-gray-500">
                 {props.name}
               </span>
-            )
+            )}
-          }
             <div className="flex flex-grow flex-col gap-3">
               <Markdown message={props.message} />
             </div>
@@ -81,6 +77,19 @@ export const PlaygroundMessage = (props: Props) => {
           )}
         </div>
       </div>
+      {props.images && (
+        <div className="flex md:max-w-2xl lg:max-w-xl xl:max-w-3xl p-3 m-auto w-full">
+          {props.images.map((image, index) => (
+            <div key={index} className="h-full rounded-md shadow relative">
+              <img
+                src={image}
+                alt="Uploaded"
+                className="h-full w-auto object-cover rounded-md min-w-[50px]"
+              />
+            </div>
+          ))}
+        </div>
+      )}
     </div>
   )
 }

src/components/Sidepanel/Chat/body.tsx

@@ -20,6 +20,7 @@ export const SidePanelBody = () => {
           isBot={message.isBot}
           message={message.message}
           name={message.name}
+          images={message.images || []}
         />
       ))}
       <div className="w-full h-32 md:h-48 flex-shrink-0"></div>

src/components/Sidepanel/Chat/form.tsx

@@ -3,9 +3,17 @@ import { useMutation } from "@tanstack/react-query"
 import React from "react"
 import useDynamicTextareaSize from "~hooks/useDynamicTextareaSize"
 import { useMessage } from "~hooks/useMessage"
+import PhotoIcon from "@heroicons/react/24/outline/PhotoIcon"
+import XMarkIcon from "@heroicons/react/24/outline/XMarkIcon"
+import { toBase64 } from "~libs/to-base64"
 
-export const SidepanelForm = () => {
+type Props = {
+  dropedFile: File | undefined
+}
+
+export const SidepanelForm = ({ dropedFile }: Props) => {
   const textareaRef = React.useRef<HTMLTextAreaElement>(null)
+  const inputRef = React.useRef<HTMLInputElement>(null)
 
   const resetHeight = () => {
     const textarea = textareaRef.current
@@ -15,16 +23,34 @@ export const SidepanelForm = () => {
   }
   const form = useForm({
     initialValues: {
-      message: ""
+      message: "",
+      image: ""
     }
   })
 
-  useDynamicTextareaSize(
+  const onInputChange = async (
-    textareaRef,
+    e: React.ChangeEvent<HTMLInputElement> | File
-    form.values.message,
+  ) => {
-  )
+    if (e instanceof File) {
+      const base64 = await toBase64(e)
+      form.setFieldValue("image", base64)
+    } else {
+      if (e.target.files) {
+        const base64 = await toBase64(e.target.files[0])
+        form.setFieldValue("image", base64)
+      }
+    }
+  }
 
-  const { onSubmit, selectedModel } = useMessage()
+  React.useEffect(() => {
+    if (dropedFile) {
+      onInputChange(dropedFile)
+    }
+  }, [dropedFile])
+
+  useDynamicTextareaSize(textareaRef, form.values.message, 120)
+
+  const { onSubmit, selectedModel, chatMode } = useMessage()
 
   const { mutateAsync: sendMessage, isPending: isSending } = useMutation({
     mutationFn: onSubmit
@@ -33,6 +59,24 @@ export const SidepanelForm = () => {
   return (
     <div className="p-3 md:p-6 md:bg-white dark:bg-[#0a0a0a] border rounded-t-xl   border-black/10 dark:border-gray-900/50">
       <div className="flex-grow space-y-6 ">
+        {chatMode === "normal" && form.values.image && (
+          <div className="h-full rounded-md shadow relative">
+            <div>
+              <img
+                src={form.values.image}
+                alt="Uploaded"
+                className="h-full w-auto object-cover rounded-md min-w-[50px]"
+              />
+              <button
+                onClick={() => {
+                  form.setFieldValue("image", "")
+                }}
+                className="absolute top-2 right-2 bg-white dark:bg-black p-1 rounded-full hover:bg-gray-100 dark:hover:bg-gray-800 text-black dark:text-gray-100">
+                <XMarkIcon className="h-5 w-5" />
+              </button>
+            </div>
+          </div>
+        )}
         <div className="flex">
           <form
             onSubmit={form.onSubmit(async (value) => {
@@ -42,10 +86,34 @@ export const SidepanelForm = () => {
               }
               form.reset()
               resetHeight()
-              await sendMessage(value.message)
+              await sendMessage({
+                image: value.image,
+                message: value.message.trim()
+              })
             })}
             className="shrink-0 flex-grow  flex items-center ">
             <div className="flex items-center p-2 rounded-2xl border  bg-gray-100 w-full dark:bg-black dark:border-gray-800">
+              <button
+                type="button"
+                onClick={() => {
+                  inputRef.current?.click()
+                }}
+                className={`flex ml-3 items-center justify-center dark:text-gray-100 ${
+                  chatMode === "rag" ? "hidden" : "block"
+                }`}>
+                <PhotoIcon className="h-5 w-5" />
+              </button>
+              <input
+                id="file-upload"
+                name="file-upload"
+                type="file"
+                className="sr-only"
+                ref={inputRef}
+                accept="image/*"
+                multiple={false}
+                onChange={onInputChange}
+              />
+
               <textarea
                 onKeyDown={(e) => {
                   if (e.key === "Enter" && !e.shiftKey && !isSending) {
@@ -60,12 +128,15 @@ export const SidepanelForm = () => {
                       }
                       form.reset()
                       resetHeight()
-                      await sendMessage(value.message)
+                      await sendMessage({
+                        image: value.image,
+                        message: value.message.trim()
+                      })
                     })()
                   }
                 }}
                 ref={textareaRef}
-                className="rounded-full pl-4 pr-2 py-2 w-full resize-none bg-transparent focus-within:outline-none sm:text-sm focus:ring-0 focus-visible:ring-0 ring-0 dark:ring-0 border-0 dark:text-gray-100"
+                className="pl-4 pr-2 py-2 w-full resize-none bg-transparent focus-within:outline-none sm:text-sm focus:ring-0 focus-visible:ring-0 ring-0 dark:ring-0 border-0 dark:text-gray-100"
                 required
                 rows={1}
                 tabIndex={0}

src/hooks/useMessage.tsx

@@ -3,12 +3,15 @@ import { cleanUrl } from "~libs/clean-url"
 import { getOllamaURL, systemPromptForNonRag } from "~services/ollama"
 import { useStoreMessage, type ChatHistory, type Message } from "~store"
 import { ChatOllama } from "@langchain/community/chat_models/ollama"
-import { HumanMessage, AIMessage } from "@langchain/core/messages"
+import {
+  HumanMessage,
+  AIMessage,
+  type MessageContent
+} from "@langchain/core/messages"
 import { getHtmlOfCurrentTab } from "~libs/get-html"
 import { PageAssistHtmlLoader } from "~loader/html"
 import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
-import { OllamaEmbeddings } from "langchain/embeddings/ollama"
+import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama"
-import { Voy as VoyClient } from "voy-search"
 import { createChatWithWebsiteChain } from "~chain/chat-with-website"
 import { MemoryVectorStore } from "langchain/vectorstores/memory"
 
@@ -25,19 +28,34 @@ const generateHistory = (
   messages: {
     role: "user" | "assistant" | "system"
     content: string
+    image?: string
   }[]
 ) => {
   let history = []
   for (const message of messages) {
     if (message.role === "user") {
+      let content: MessageContent = [
+        {
+          type: "text",
+          text: message.content
+        }
+      ]
+
+      if (message.image) {
+        content = [
+          {
+            type: "image_url",
+            image_url: message.image
+          },
+          {
+            type: "text",
+            text: message.content
+          }
+        ]
+      }
       history.push(
         new HumanMessage({
-          content: [
+          content: content
-            {
-              type: "text",
-              text: message.content
-            }
-          ]
         })
       )
     } else if (message.role === "assistant") {
@@ -239,9 +257,12 @@ export const useMessage = () => {
     }
   }
 
-  const normalChatMode = async (message: string) => {
+  const normalChatMode = async (message: string, image: string) => {
     const url = await getOllamaURL()
 
+    if (image.length > 0) {
+      image = `data:image/jpeg;base64,${image.split(",")[1]}`
+    }
     abortControllerRef.current = new AbortController()
 
     const ollama = new ChatOllama({
@@ -255,13 +276,14 @@ export const useMessage = () => {
         isBot: false,
         name: "You",
         message,
-        sources: []
+        sources: [],
+        images: [image]
       },
       {
         isBot: true,
         name: selectedModel,
         message: "▋",
-        sources: []
+        sources: [],
       }
     ]
 
@@ -271,17 +293,35 @@ export const useMessage = () => {
     try {
       const prompt = await systemPromptForNonRag()
 
+      let humanMessage =  new HumanMessage({
+        content: [
+          {
+            text: message,
+            type: "text"
+          }
+        ]
+      })
+      if (image.length > 0) {
+        humanMessage = new HumanMessage({
+          content: [
+            {
+              text: message,
+              type: "text"
+            },
+            {
+              image_url: image,
+              type: "image_url"
+            }
+          ]
+        }) 
+      }
+
+      console.log("humanMessage", humanMessage)
+
       const chunks = await ollama.stream(
         [
           ...generateHistory(history),
-          new HumanMessage({
+          humanMessage
-            content: [
-              {
-                type: "text",
-                text: message
-              }
-            ]
-          })
         ],
         {
           signal: abortControllerRef.current.signal
@@ -312,7 +352,8 @@ export const useMessage = () => {
         ...history,
         {
           role: "user",
-          content: message
+          content: message,
+          image
         },
         {
           role: "assistant",
@@ -342,9 +383,15 @@ export const useMessage = () => {
     }
   }
 
-  const onSubmit = async (message: string) => {
+  const onSubmit = async ({
+    message,
+    image
+  }: {
+    message: string
+    image: string
+  }) => {
     if (chatMode === "normal") {
-      await normalChatMode(message)
+      await normalChatMode(message, image)
     } else {
       await chatWithWebsiteMode(message)
     }

src/libs/to-base64.ts

@@ -0,0 +1,7 @@
+export const toBase64 = (file: File) =>
+  new Promise<string>((resolve, reject) => {
+    const reader = new FileReader()
+    reader.readAsDataURL(file)
+    reader.onload = () => resolve(reader.result as string)
+    reader.onerror = (error) => reject(error)
+  })

src/loader/html.ts

@@ -1,5 +1,5 @@
 import { BaseDocumentLoader } from "langchain/document_loaders/base"
-import { Document } from "langchain/document"
+import { Document } from "@langchain/core/documents"
 import { compile } from "html-to-text"
 
 export interface WebLoaderParams {

src/routes/index.tsx

@@ -1,14 +1,19 @@
 import { Route, Routes } from "react-router-dom"
 import { SidepanelChat } from "./sidepanel-chat"
 import { SidepanelSettingsHeader } from "~components/Sidepanel/Settings/header"
+import { useDarkMode } from "~hooks/useDarkmode"
 
 export const Routing = () => <Routes></Routes>
 
-export const SidepanelRouting = () => (
+export const SidepanelRouting = () => {
-  <div className="dark">
+  const { mode } = useDarkMode()
-    <Routes>
+
-      <Route path="/" element={<SidepanelChat />} />
+  return (
-      <Route path="/settings" element={<SidepanelSettingsHeader />} />  
+    <div className={mode === "dark" ? "dark" : "light"}>
-    </Routes>
+      <Routes>
-  </div>
+        <Route path="/" element={<SidepanelChat />} />
-)
+        <Route path="/settings" element={<SidepanelSettingsHeader />} />
+      </Routes>
+    </div>
+  )
+}

src/routes/sidepanel-chat.tsx

@@ -1,10 +1,80 @@
+import React from "react"
 import { SidePanelBody } from "~components/Sidepanel/Chat/body"
 import { SidepanelForm } from "~components/Sidepanel/Chat/form"
 import { SidepanelHeader } from "~components/Sidepanel/Chat/header"
+import { useMessage } from "~hooks/useMessage"
 
 export const SidepanelChat = () => {
+  const drop = React.useRef<HTMLDivElement>(null)
+  const [dropedFile, setDropedFile] = React.useState<File | undefined>()
+  const [dropState, setDropState] = React.useState<
+    "idle" | "dragging" | "error"
+  >("idle")
+  const {chatMode} = useMessage()
+  React.useEffect(() => {
+    if (!drop.current) {
+      return
+    }
+    const handleDragOver = (e: DragEvent) => {
+      e.preventDefault()
+      e.stopPropagation()
+    }
+
+    const handleDrop = (e: DragEvent) => {
+      e.preventDefault()
+      e.stopPropagation()
+
+      setDropState("idle")
+
+      const files = Array.from(e.dataTransfer?.files || [])
+
+      const isImage = files.every((file) => file.type.startsWith("image/"))
+
+      if (!isImage) {
+        setDropState("error")
+        return
+      }
+
+      const newFiles = Array.from(e.dataTransfer?.files || []).slice(0, 1)
+      if (newFiles.length > 0) {
+        setDropedFile(newFiles[0])
+      }
+    }
+
+    const handleDragEnter = (e: DragEvent) => {
+      e.preventDefault()
+      e.stopPropagation()
+      setDropState("dragging")
+    }
+
+    const handleDragLeave = (e: DragEvent) => {
+      e.preventDefault()
+      e.stopPropagation()
+      setDropState("idle")
+    }
+
+    drop.current.addEventListener("dragover", handleDragOver)
+    drop.current.addEventListener("drop", handleDrop)
+    drop.current.addEventListener("dragenter", handleDragEnter)
+    drop.current.addEventListener("dragleave", handleDragLeave)
+
+    return () => {
+      if (drop.current) {
+        drop.current.removeEventListener("dragover", handleDragOver)
+        drop.current.removeEventListener("drop", handleDrop)
+        drop.current.removeEventListener("dragenter", handleDragEnter)
+        drop.current.removeEventListener("dragleave", handleDragLeave)
+      }
+    }
+  }, [])
   return (
-    <div className="flex bg-white dark:bg-black flex-col min-h-screen mx-auto max-w-7xl">
+    <div
+      ref={drop}
+      className={`flex ${
+        dropState === "dragging" && chatMode === "normal"
+          ? "bg-gray-100 dark:bg-gray-800 z-10"
+          : "bg-white dark:bg-black"
+      } flex-col min-h-screen mx-auto max-w-7xl`}>
       <div className="sticky top-0 z-10">
         <SidepanelHeader />
       </div>
@@ -13,7 +83,7 @@ export const SidepanelChat = () => {
       <div className="bottom-0 w-full bg-transparent border-0 fixed pt-2">
         <div className="stretch mx-2 flex flex-row gap-3 md:mx-4 lg:mx-auto lg:max-w-2xl xl:max-w-3xl">
           <div className="relative flex flex-col h-full flex-1 items-stretch md:flex-col">
-            <SidepanelForm />
+            <SidepanelForm dropedFile={dropedFile} />
           </div>
         </div>
       </div>

src/services/ollama.ts

@@ -68,11 +68,18 @@ export const fetchModels = async () => {
 }
 
 export const setOllamaURL = async (ollamaURL: string) => {
-  await chromeRunTime(cleanUrl(ollamaURL))
+  let formattedUrl = ollamaURL
-  await storage.set("ollamaURL", cleanUrl(ollamaURL))
+  if (formattedUrl.startsWith("http://localhost:")) {
+    formattedUrl = formattedUrl.replace(
+      "http://localhost:",
+      "http://127.0.0.1:"
+    )
+  }
+  await chromeRunTime(cleanUrl(formattedUrl))
+  await storage.set("ollamaURL", cleanUrl(formattedUrl))
 }
 
 export const systemPromptForNonRag = async () => {
   const prompt = await storage.get("systemPromptForNonRag")
   return prompt
-}
+}

src/store/index.tsx

@@ -5,11 +5,13 @@ export type Message = {
   name: string
   message: string
   sources: any[]
+  images?: string[]
 }
 
 export type ChatHistory = {
   role: "user" | "assistant" | "system"
-  content: string
+  content: string,
+  image?: string
 }[]
 
 type State = {

yarn.lock

@@ -537,6 +537,23 @@
     uuid "^9.0.0"
     zod "^3.22.3"
 
+"@langchain/core@^0.1.22":
+  version "0.1.22"
+  resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.22.tgz#b558f4675f5ad4dea473899362469220a96b4144"
+  integrity sha512-I3KMv87D5AFeAvuJhzaGOYdppFL4h/bRm7LeJfwF2PspQIZwvDE9GP7hkw4n+7jwNaBxjU8ZTj6o3LZAh1R5LQ==
+  dependencies:
+    ansi-styles "^5.0.0"
+    camelcase "6"
+    decamelize "1.2.0"
+    js-tiktoken "^1.0.8"
+    langsmith "~0.0.48"
+    ml-distance "^4.0.0"
+    p-queue "^6.6.2"
+    p-retry "4"
+    uuid "^9.0.0"
+    zod "^3.22.4"
+    zod-to-json-schema "^3.22.3"
+
 "@langchain/core@~0.1.13", "@langchain/core@~0.1.16":
   version "0.1.20"
   resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.20.tgz#599d3a8de1faa4692b03d3e8939d8e08e1dc2413"