Update dependencies and refactor useMessage hook

package.json

@@ -18,8 +18,7 @@
     "@ant-design/cssinjs": "^1.18.4",
     "@headlessui/react": "^1.7.18",
     "@heroicons/react": "^2.1.1",
-    "@langchain/community": "^0.0.21",
-    "@langchain/core": "^0.1.22",
+    "@langchain/community": "^0.0.41",
     "@mantine/form": "^7.5.0",
     "@mantine/hooks": "^7.5.3",
     "@plasmohq/storage": "^1.9.0",
@@ -33,8 +32,9 @@
     "html-to-text": "^9.0.5",
     "i18next": "^23.10.1",
     "i18next-browser-languagedetector": "^7.2.0",
-    "langchain": "^0.1.9",
+    "langchain": "^0.1.28",
     "lucide-react": "^0.350.0",
+    "pdfjs-dist": "^4.0.379",
     "property-information": "^6.4.1",
     "react": "18.2.0",
     "react-dom": "18.2.0",
@@ -62,6 +62,10 @@
     "prettier": "3.2.4",
     "tailwindcss": "^3.4.1",
     "typescript": "5.3.3",
+    "vite-plugin-top-level-await": "^1.4.1",
     "wxt": "^0.17.7"
+  },
+  "resolutions": {
+    "@langchain/core": "0.1.45"
   }
 }

src/hooks/useMessage.tsx

@@ -16,7 +16,7 @@ import {
   type MessageContent,
   SystemMessage
 } from "@langchain/core/messages"
-import { getHtmlOfCurrentTab } from "~/libs/get-html"
+import { getDataFromCurrentTab } from "~/libs/get-html"
 import { PageAssistHtmlLoader } from "~/loader/html"
 import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
 import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama"
@@ -167,7 +167,7 @@ export const useMessage = () => {
       let isAlreadyExistEmbedding: MemoryVectorStore
       let embedURL: string, embedHTML: string
       if (messages.length === 0) {
-        const { html, url } = await getHtmlOfCurrentTab()
+        const { content: html, url, type } = await getDataFromCurrentTab()
         embedHTML = html
         embedURL = url
         setCurrentURL(url)

src/libs/get-html.ts

@@ -1,15 +1,46 @@
+import { pdfDist } from "./pdfjs"
 
-const _getHtml = () => {
+export const getPdf = async (data: ArrayBuffer) => {
+  const pdf = pdfDist.getDocument({
+    data,
+    useWorkerFetch: false,
+    isEvalSupported: false,
+    useSystemFonts: true,
+  });
+
+  pdf.onPassword = (callback: any) => {
+    const password = prompt("Enter the password: ")
+    if (!password) {
+      throw new Error("Password required to open the PDF.");
+    }
+    callback(password);
+  };
+
+
+  const pdfDocument = await pdf.promise;
+
+
+  return pdfDocument
+
+}
+
+const _getHtml = async () => {
   const url = window.location.href
+  // check the content type
+  if (document.contentType === "application/pdf") {
+
+
+    return { url, content: "", type: "pdf" }
+  }
   const html = Array.from(document.querySelectorAll("script")).reduce(
     (acc, script) => {
       return acc.replace(script.outerHTML, "")
     },
     document.documentElement.outerHTML
   )
-  return { url, html }
+  return { url, content: html, type: "html" }
 }
-export const getHtmlOfCurrentTab = async () => {
+export const getDataFromCurrentTab = async () => {
   const result = new Promise((resolve) => {
     chrome.tabs.query({ active: true, currentWindow: true }, async (tabs) => {
       const tab = tabs[0]
@@ -25,9 +56,41 @@ export const getHtmlOfCurrentTab = async () => {
     })
   }) as Promise<{
     url: string
-    html: string
+    content: string
+    type: string
   }>
 
-  return result
+
+  const { content, type, url } = await result
+
+  if (type === "pdf") {
+    const res = await fetch(url)
+    const data = await res.arrayBuffer()
+    let pdfHtml: string[] = []
+    const pdf = await getPdf(data)
+
+    for (let i = 1; i <= pdf.numPages; i += 1) {
+      const page = await pdf.getPage(i);
+      const content = await page.getTextContent();
+
+      if (content?.items.length === 0) {
+        continue;
+      }
+
+      const text = content?.items.map((item: any) => item.str).join("\n")
+        .replace(/\x00/g, "").trim();
+      pdfHtml.push(`<div class="pdf-page">${text}</div>`)
+    }
+
+
+    return {
+      url,
+      content: pdfHtml.join(""),
+      type: "html"
+    }
+
+  }
+
+  return { url, content, type }
 }
 

src/libs/pdfjs.ts

@@ -0,0 +1,8 @@
+import * as pdfDist from "pdfjs-dist"
+import * as pdfWorker from "pdfjs-dist/build/pdf.worker.mjs";
+
+pdfDist.GlobalWorkerOptions.workerSrc = pdfWorker
+
+export {
+    pdfDist
+}

wxt.config.ts

@@ -1,10 +1,24 @@
 import { defineConfig } from "wxt"
 import react from "@vitejs/plugin-react"
+import topLevelAwait from "vite-plugin-top-level-await"
 
 // See https://wxt.dev/api/config.html
 export default defineConfig({
   vite: () => ({
-    plugins: [react()],
+    plugins: [react(),
+      topLevelAwait({
+        promiseExportName: '__tla',
+        promiseImportName: i => `__tla_${i}`,
+      }),
+    ],
+    build: {
+      rollupOptions: {
+        external: [
+          "langchain",
+          "@langchain/community",
+        ]
+      }
+    }
   }),
   entrypointsDir: "entries",
   srcDir: "src",
@@ -16,7 +30,7 @@ export default defineConfig({
     default_locale: 'en',
     action: {},
     author: "n4ze3m",
-    host_permissions: ["http://*/*", "https://*/*"],
+    host_permissions: ["http://*/*", "https://*/*", "file://*/*"],
     commands: {
       _execute_action: {
         suggested_key: {