feat: Add SearXNG search provider and settings

src/assets/locale/da/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Besøg websitet nævnt i samtalen"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG URL"
+        }
       }
     },
     "system": {

src/assets/locale/de/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Die in der Nachricht erwähnte Website besuchen"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG-URL"
+        }
       }
     },
     "system": {

src/assets/locale/en/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Visit the website mentioned in the message"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG URL"
+        }
       }
     },
     "system": {

src/assets/locale/es/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Visita el sitio web mencionado en el mensaje"
+      },
+      "searxng": {
+        "url": {
+          "label": "URL de SearXNG"
+        }
       }
     },
     "system": {

src/assets/locale/fa/settings.json

@@ -62,6 +62,11 @@
       },
       "visitSpecificWebsite": {
         "label": "مراجعه به وب سایت ذکر شده در پیام"
+      },
+      "searxng": {
+        "url": {
+          "label": "آدرس SearXNG"
+        }
       }
     },
     "system": {

src/assets/locale/fr/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Visitez le site web mentionné dans le message"
+      },
+      "searxng": {
+        "url": {
+          "label": "URL SearXNG"
+        }
       }
     },
     "system": {

src/assets/locale/it/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Visita il sito web menzionato nel messaggio"
+      },
+      "searxng": {
+        "url": {
+          "label": "URL SearXNG"
+        }
       }
     },
     "system": {

src/assets/locale/ja-JP/settings.json

@@ -68,6 +68,11 @@
       },
       "visitSpecificWebsite": {
         "label": "メッセージに記載されたウェブサイトを訪問してください"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG URL"
+        }
       }
     },
     "system": {

src/assets/locale/ko/settings.json

@@ -68,6 +68,11 @@
       },
       "visitSpecificWebsite": {
         "label": "메시지에 언급된 웹사이트 방문"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG URL"
+        }
       }
     },
     "system": {

src/assets/locale/ml/settings.json

@@ -68,6 +68,11 @@
       },
       "visitSpecificWebsite": {
         "label": "സന്ദേശത്തിൽ പറയുന്ന വെബ്സൈറ്റ് സന്ദർശിക്കുക."
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG URL"
+        }
       }
     },
     "system": {

src/assets/locale/no/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Besøk nettstedet nevnt i samtalen"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG URL"
+        }
       }
     },
     "system": {

src/assets/locale/pt-BR/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Visitar o site mencionado na mensagem"
+      },
+      "searxng": {
+        "url": {
+          "label": "URL do SearXNG"
+        }
       }
     },
     "system": {

src/assets/locale/ru/settings.json

@@ -66,6 +66,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Посетите веб-сайт, указанный в сообщении."
+      },
+      "searxng": {
+        "url": {
+          "label": "URL-адрес SearXNG"
+        }
       }
     },
     "system": {

src/assets/locale/sv/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Besök webbplatsen som nämns i meddelandet"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG URL"
+        }
       }
     },
     "system": {

src/assets/locale/uk/settings.json

@@ -65,6 +65,11 @@
       },
       "visitSpecificWebsite": {
         "label": "Відвідати веб-сайт, згаданий у повідомленні"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG URL-адреса"
+        }
       }
     },
     "system": {

src/assets/locale/zh/settings.json

@@ -68,6 +68,11 @@
       },
       "visitSpecificWebsite": {
         "label": "访问消息中提到的网站。"
+      },
+      "searxng": {
+        "url": {
+          "label": "SearXNG 网址"
+        }
       }
     },
     "system": {

src/components/Option/Settings/search-mode.tsx

@@ -3,7 +3,7 @@ import { getSearchSettings, setSearchSettings } from "@/services/search"
 import { SUPPORTED_SERACH_PROVIDERS } from "@/utils/search-provider"
 import { useForm } from "@mantine/form"
 import { useQuery, useQueryClient } from "@tanstack/react-query"
-import { Select, Skeleton, Switch, InputNumber } from "antd"
+import { Select, Skeleton, Switch, InputNumber, Input } from "antd"
 import { useTranslation } from "react-i18next"
 
 export const SearchModeSettings = () => {
@@ -14,7 +14,9 @@ export const SearchModeSettings = () => {
       isSimpleInternetSearch: false,
       searchProvider: "",
       totalSearchResults: 0,
-      visitSpecificWebsite: false
+      visitSpecificWebsite: false,
+      searxngURL: "",
+      searxngJSONMode: false
     }
   })
 
@@ -62,6 +64,23 @@ export const SearchModeSettings = () => {
             />
           </div>
         </div>
+        {form.values.searchProvider === "searxng" && (
+          <>
+            <div className="flex sm:flex-row flex-col space-y-4 sm:space-y-0 sm:justify-between">
+              <span className="text-gray-700 dark:text-neutral-50">
+                {t("generalSettings.webSearch.searxng.url.label")}
+              </span>
+              <div>
+                <Input
+                  placeholder="https://searxng.example.com"
+                  className="w-full mt-4 sm:mt-0 sm:w-[200px]"
+                  required
+                  {...form.getInputProps("searxngURL")}
+                />
+              </div>
+            </div>
+          </>
+        )}
         <div className="flex sm:flex-row flex-col space-y-4 sm:space-y-0 sm:justify-between">
           <span className="text-gray-700 dark:text-neutral-50 ">
             {t("generalSettings.webSearch.searchMode.label")}

src/services/search.ts

@@ -62,20 +62,45 @@ export const setTotalSearchResults = async (totalSearchResults: number) => {
   await storage.set("totalSearchResults", totalSearchResults.toString())
 }
 
+export const getSearxngURL = async () => {
+  const searxngURL = await storage.get("searxngURL")
+  return searxngURL || ""
+}
+
+export const isSearxngJSONMode = async () => {
+  const searxngJSONMode = await storage.get<boolean>("searxngJSONMode")
+  return searxngJSONMode ?? false
+}
+
+export const setSearxngJSONMode = async (searxngJSONMode: boolean) => {
+  await storage.set("searxngJSONMode", searxngJSONMode)
+}
+
+export const setSearxngURL = async (searxngURL: string) => {
+  await storage.set("searxngURL", searxngURL)
+}
+
 export const getSearchSettings = async () => {
-  const [isSimpleInternetSearch, searchProvider, totalSearchResult, visitSpecificWebsite] =
+  const [isSimpleInternetSearch, searchProvider, totalSearchResult, visitSpecificWebsite,
+    searxngURL,
+    searxngJSONMode
+  ] =
     await Promise.all([
       getIsSimpleInternetSearch(),
       getSearchProvider(),
       totalSearchResults(),
-      getIsVisitSpecificWebsite()
+      getIsVisitSpecificWebsite(),
+      getSearxngURL(),
+      isSearxngJSONMode()
     ])
 
   return {
     isSimpleInternetSearch,
     searchProvider,
     totalSearchResults: totalSearchResult,
-    visitSpecificWebsite
+    visitSpecificWebsite,
+    searxngURL,
+    searxngJSONMode
   }
 }
 
@@ -83,17 +108,23 @@ export const setSearchSettings = async ({
   isSimpleInternetSearch,
   searchProvider,
   totalSearchResults,
-  visitSpecificWebsite
+  visitSpecificWebsite,
+  searxngJSONMode,
+  searxngURL
 }: {
   isSimpleInternetSearch: boolean
   searchProvider: string
   totalSearchResults: number
   visitSpecificWebsite: boolean
+  searxngURL: string
+  searxngJSONMode: boolean
 }) => {
   await Promise.all([
     setIsSimpleInternetSearch(isSimpleInternetSearch),
     setSearchProvider(searchProvider),
     setTotalSearchResults(totalSearchResults),
-    setIsVisitSpecificWebsite(visitSpecificWebsite)
+    setIsVisitSpecificWebsite(visitSpecificWebsite),
+    setSearxngJSONMode(searxngJSONMode),
+    setSearxngURL(searxngURL)
   ])
 }

src/utils/search-provider.ts

@@ -14,5 +14,9 @@ export const SUPPORTED_SERACH_PROVIDERS = [
     {
         label: "Brave",
         value: "brave"
+    },
+    {
+        label: "Searxng",
+        value: "searxng"
     }
 ]

src/web/search-engines/searxng.ts

@@ -0,0 +1,145 @@
+import { urlRewriteRuntime } from "~/libs/runtime"
+import { cleanUrl } from "~/libs/clean-url"
+import { getSearxngURL, isSearxngJSONMode, getIsSimpleInternetSearch, totalSearchResults } from "@/services/search"
+import { pageAssistEmbeddingModel } from "@/models/embedding"
+import type { Document } from "@langchain/core/documents"
+import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
+import { MemoryVectorStore } from "langchain/vectorstores/memory"
+import { PageAssistHtmlLoader } from "~/loader/html"
+import {
+  defaultEmbeddingChunkOverlap,
+  defaultEmbeddingChunkSize,
+  defaultEmbeddingModelForRag,
+  getOllamaURL
+} from "~/services/ollama"
+
+interface SearxNGJSONResult {
+  title: string
+  url: string
+  content: string
+}
+
+interface SearxNGJSONResponse {
+  results: SearxNGJSONResult[]
+}
+
+export const searxngSearch = async (query: string) => {
+  const searxngURL = await getSearxngURL()
+  if (!searxngURL) {
+    throw new Error("SearXNG URL not configured")
+  }
+
+  const isJSONMode = await isSearxngJSONMode()
+  const results = isJSONMode 
+    ? await searxngJSONSearch(searxngURL, query)
+    : await searxngWebSearch(searxngURL, query)
+
+  const TOTAL_SEARCH_RESULTS = await totalSearchResults()
+  const searchResults = results.slice(0, TOTAL_SEARCH_RESULTS)
+
+  const isSimpleMode = await getIsSimpleInternetSearch()
+
+  if (isSimpleMode) {
+    await getOllamaURL()
+    return searchResults.map((result) => {
+      return {
+        url: result.link,
+        content: result.content
+      }
+    })
+  }
+
+  const docs: Document<Record<string, any>>[] = []
+  for (const result of searchResults) {
+    const loader = new PageAssistHtmlLoader({
+      html: "",
+      url: result.link
+    })
+
+    const documents = await loader.loadByURL()
+    documents.forEach((doc) => {
+      docs.push(doc)
+    })
+  }
+
+  const ollamaUrl = await getOllamaURL()
+  const embeddingModel = await defaultEmbeddingModelForRag()
+  const ollamaEmbedding = await pageAssistEmbeddingModel({
+    model: embeddingModel || "",
+    baseUrl: cleanUrl(ollamaUrl)
+  })
+
+  const chunkSize = await defaultEmbeddingChunkSize()
+  const chunkOverlap = await defaultEmbeddingChunkOverlap()
+  const textSplitter = new RecursiveCharacterTextSplitter({
+    chunkSize,
+    chunkOverlap
+  })
+
+  const chunks = await textSplitter.splitDocuments(docs)
+  const store = new MemoryVectorStore(ollamaEmbedding)
+  await store.addDocuments(chunks)
+
+  return store
+}
+
+const searxngJSONSearch = async (baseURL: string, query: string) => {
+  const searchURL = `${cleanUrl(baseURL)}?q=${encodeURIComponent(query)}&format=json`
+
+  const abortController = new AbortController()
+  setTimeout(() => abortController.abort(), 20000)
+
+  try {
+    const response = await fetch(searchURL, {
+      signal: abortController.signal,
+      headers: {
+        'Accept': 'application/json'
+      }
+    })
+
+    if (!response.ok) {
+      throw new Error(`SearXNG search failed: ${response.statusText}`)
+    }
+
+    const data = await response.json() as SearxNGJSONResponse
+
+    return data.results.map(result => ({
+      title: result.title,
+      link: result.url,
+      content: result.content
+    }))
+  } catch (error) {
+    console.error('SearXNG JSON search failed:', error)
+    return []
+  }
+}
+
+const searxngWebSearch = async (baseURL: string, query: string) => {
+  const searchURL = `${cleanUrl(baseURL)}?q=${encodeURIComponent(query)}`
+
+  await urlRewriteRuntime(cleanUrl(searchURL), "searxng")
+
+  const abortController = new AbortController()
+  setTimeout(() => abortController.abort(), 10000)
+
+  try {
+    const htmlString = await fetch(searchURL, {
+      signal: abortController.signal
+    }).then(response => response.text())
+
+    const parser = new DOMParser()
+    const doc = parser.parseFromString(htmlString, "text/html")
+
+    const searchResults = Array.from(doc.querySelectorAll("article.result")).map(result => {
+      const title = result.querySelector("h3")?.textContent?.trim()
+      const link = result.querySelector("a.url_header")?.getAttribute("href")
+      const content = result.querySelector("p.content")?.textContent?.trim()
+      return { title, link, content }
+    }).filter(result => result.title && result.link && result.content)
+
+    return searchResults
+  } catch (error) {
+    console.error('SearXNG web search failed:', error)
+    return []
+  }
+}

src/web/web.ts

@@ -5,6 +5,7 @@ import { getIsVisitSpecificWebsite, getSearchProvider } from "@/services/search"
 import { webSogouSearch } from "./search-engines/sogou"
 import { webBraveSearch } from "./search-engines/brave"
 import { getWebsiteFromQuery, processSingleWebsite } from "./website"
+import { searxngSearch } from "./search-engines/searxng"
 
 const getHostName = (url: string) => {
   try {
@@ -23,6 +24,8 @@ const searchWeb = (provider: string, query: string) => {
       return webSogouSearch(query)
     case "brave":
       return webBraveSearch(query)
+    case "searxng":
+      return searxngSearch(query)
     default:
       return webGoogleSearch(query)
   }
@@ -47,11 +50,11 @@ export const getSystemPromptForWeb = async (query: string) => {
 
     } else {
       const searchProvider = await getSearchProvider()
+      //@ts-ignore
       search = await searchWeb(searchProvider, query)
     }
 
 
-
     const search_results = search
       .map(
         (result, idx) =>