Add lucide-react package and remove unused icons

src/web/local-google.ts

@@ -1,12 +1,21 @@
+import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama"
+import type { Document } from "@langchain/core/documents"
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
+import { MemoryVectorStore } from "langchain/vectorstores/memory"
 import { cleanUrl } from "~libs/clean-url"
 import { chromeRunTime } from "~libs/runtime"
+import { PageAssistHtmlLoader } from "~loader/html"
+import { defaultEmbeddingChunkOverlap, defaultEmbeddingChunkSize, defaultEmbeddingModelForRag, getOllamaURL } from "~services/ollama"
 
 const BLOCKED_HOSTS = [
   "google.com",
   "youtube.com",
   "twitter.com",
+  "linkedin.com",
 ]
 
+const TOTAL_SEARCH_RESULTS = 2
+
 export const localGoogleSearch = async (query: string) => {
   await chromeRunTime(
     cleanUrl("https://www.google.com/search?hl=en&q=" + query)
@@ -43,3 +52,55 @@ export const localGoogleSearch = async (query: string) => {
     .filter((result) => result.title && result.link)
   return filteredSearchResults
 }
+
+
+export const webSearch = async (query: string) => {
+  const results = await localGoogleSearch(query)
+  const searchResults = results.slice(0, TOTAL_SEARCH_RESULTS)
+
+  const docs: Document<Record<string, any>>[] = [];
+  for (const result of searchResults) {
+    const loader = new PageAssistHtmlLoader({
+      html: "",
+      url: result.link
+    })
+
+    const documents = await loader.loadByURL()
+
+    documents.forEach((doc) => {
+      docs.push(doc)
+    })
+  }
+  const ollamaUrl = await getOllamaURL()
+
+  const embeddingModle = await defaultEmbeddingModelForRag()
+  const ollamaEmbedding = new OllamaEmbeddings({
+    model: embeddingModle || "",
+    baseUrl: cleanUrl(ollamaUrl),
+  })
+
+  const chunkSize = await defaultEmbeddingChunkSize();
+  const chunkOverlap = await defaultEmbeddingChunkOverlap();
+  const textSplitter = new RecursiveCharacterTextSplitter({
+    chunkSize,
+    chunkOverlap,
+  })
+
+  const chunks = await textSplitter.splitDocuments(docs)
+
+  const store = new MemoryVectorStore(ollamaEmbedding)
+
+  await store.addDocuments(chunks)
+
+
+  const resultsWithEmbeddings = await store.similaritySearch(query, 3)
+
+  const searchResult = resultsWithEmbeddings.map((result) => {
+    return {
+      url: result.metadata.url,
+      content: result.pageContent
+    }
+  })
+
+  return searchResult
+}

src/web/web.ts

@@ -0,0 +1,20 @@
+import { getWebSearchPrompt } from "~services/ollama"
+import { webSearch } from "./local-google"
+
+export const getSystemPromptForWeb = async (query: string) => {
+    try {
+        const search = await webSearch(query)
+
+        const search_results = search.map((result, idx) => `<result source="${result.url}" id="${idx}">${result.content}</result>`).join("\n")
+
+        const current_date_time = new Date().toLocaleString()
+
+        const system = await getWebSearchPrompt();
+
+        const prompt = system.replace("{current_date_time}", current_date_time).replace("{search_results}", search_results)
+
+        return prompt
+    } catch (e) {
+        return ''
+    }
+}