diff --git a/src/assets/locale/da/settings.json b/src/assets/locale/da/settings.json index 9c16997..37168c4 100644 --- a/src/assets/locale/da/settings.json +++ b/src/assets/locale/da/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Besøg websitet nævnt i samtalen" + }, + "searxng": { + "url": { + "label": "SearXNG URL" + } } }, "system": { diff --git a/src/assets/locale/de/settings.json b/src/assets/locale/de/settings.json index b310307..6315f85 100644 --- a/src/assets/locale/de/settings.json +++ b/src/assets/locale/de/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Die in der Nachricht erwähnte Website besuchen" + }, + "searxng": { + "url": { + "label": "SearXNG-URL" + } } }, "system": { diff --git a/src/assets/locale/en/settings.json b/src/assets/locale/en/settings.json index 7a97966..f0498df 100644 --- a/src/assets/locale/en/settings.json +++ b/src/assets/locale/en/settings.json @@ -36,8 +36,8 @@ "generateTitle": { "label": "Generate Title using AI" }, - "ollamaStatus" :{ - "label":"Enable or disable Ollama connection status check" + "ollamaStatus": { + "label": "Enable or disable Ollama connection status check" } }, "sidepanelRag": { @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Visit the website mentioned in the message" + }, + "searxng": { + "url": { + "label": "SearXNG URL" + } } }, "system": { diff --git a/src/assets/locale/es/settings.json b/src/assets/locale/es/settings.json index 17db408..eb98d80 100644 --- a/src/assets/locale/es/settings.json +++ b/src/assets/locale/es/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Visita el sitio web mencionado en el mensaje" + }, + "searxng": { + "url": { + "label": "URL de SearXNG" + } } }, "system": { diff --git a/src/assets/locale/fa/settings.json b/src/assets/locale/fa/settings.json index 0a0cde8..985639a 100644 --- a/src/assets/locale/fa/settings.json +++ b/src/assets/locale/fa/settings.json @@ -62,6 +62,11 @@ }, "visitSpecificWebsite": { "label": "مراجعه به وب سایت ذکر شده در پیام" + }, + "searxng": { + "url": { + "label": "آدرس SearXNG" + } } }, "system": { diff --git a/src/assets/locale/fr/settings.json b/src/assets/locale/fr/settings.json index 18211df..3ef9b49 100644 --- a/src/assets/locale/fr/settings.json +++ b/src/assets/locale/fr/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Visitez le site web mentionné dans le message" + }, + "searxng": { + "url": { + "label": "URL SearXNG" + } } }, "system": { diff --git a/src/assets/locale/it/settings.json b/src/assets/locale/it/settings.json index a88f90a..4e07754 100644 --- a/src/assets/locale/it/settings.json +++ b/src/assets/locale/it/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Visita il sito web menzionato nel messaggio" + }, + "searxng": { + "url": { + "label": "URL SearXNG" + } } }, "system": { diff --git a/src/assets/locale/ja-JP/settings.json b/src/assets/locale/ja-JP/settings.json index bd5266e..d34b63b 100644 --- a/src/assets/locale/ja-JP/settings.json +++ b/src/assets/locale/ja-JP/settings.json @@ -68,6 +68,11 @@ }, "visitSpecificWebsite": { "label": "メッセージに記載されたウェブサイトを訪問してください" + }, + "searxng": { + "url": { + "label": "SearXNG URL" + } } }, "system": { diff --git a/src/assets/locale/ko/settings.json b/src/assets/locale/ko/settings.json index 4779bdc..73ac857 100644 --- a/src/assets/locale/ko/settings.json +++ b/src/assets/locale/ko/settings.json @@ -68,6 +68,11 @@ }, "visitSpecificWebsite": { "label": "메시지에 언급된 웹사이트 방문" + }, + "searxng": { + "url": { + "label": "SearXNG URL" + } } }, "system": { diff --git a/src/assets/locale/ml/settings.json b/src/assets/locale/ml/settings.json index 63f5dc8..b9ccb0d 100644 --- a/src/assets/locale/ml/settings.json +++ b/src/assets/locale/ml/settings.json @@ -68,6 +68,11 @@ }, "visitSpecificWebsite": { "label": "സന്ദേശത്തിൽ പറയുന്ന വെബ്സൈറ്റ് സന്ദർശിക്കുക." + }, + "searxng": { + "url": { + "label": "SearXNG URL" + } } }, "system": { diff --git a/src/assets/locale/no/settings.json b/src/assets/locale/no/settings.json index 10e4495..c73cbec 100644 --- a/src/assets/locale/no/settings.json +++ b/src/assets/locale/no/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Besøk nettstedet nevnt i samtalen" + }, + "searxng": { + "url": { + "label": "SearXNG URL" + } } }, "system": { diff --git a/src/assets/locale/pt-BR/settings.json b/src/assets/locale/pt-BR/settings.json index 8c812b8..019ad68 100644 --- a/src/assets/locale/pt-BR/settings.json +++ b/src/assets/locale/pt-BR/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Visitar o site mencionado na mensagem" + }, + "searxng": { + "url": { + "label": "URL do SearXNG" + } } }, "system": { diff --git a/src/assets/locale/ru/settings.json b/src/assets/locale/ru/settings.json index fe509c7..7e6b75b 100644 --- a/src/assets/locale/ru/settings.json +++ b/src/assets/locale/ru/settings.json @@ -66,6 +66,11 @@ }, "visitSpecificWebsite": { "label": "Посетите веб-сайт, указанный в сообщении." + }, + "searxng": { + "url": { + "label": "URL-адрес SearXNG" + } } }, "system": { diff --git a/src/assets/locale/sv/settings.json b/src/assets/locale/sv/settings.json index 7d782de..3edaa52 100644 --- a/src/assets/locale/sv/settings.json +++ b/src/assets/locale/sv/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Besök webbplatsen som nämns i meddelandet" + }, + "searxng": { + "url": { + "label": "SearXNG URL" + } } }, "system": { diff --git a/src/assets/locale/uk/settings.json b/src/assets/locale/uk/settings.json index bdcaa0b..91fe5de 100644 --- a/src/assets/locale/uk/settings.json +++ b/src/assets/locale/uk/settings.json @@ -65,6 +65,11 @@ }, "visitSpecificWebsite": { "label": "Відвідати веб-сайт, згаданий у повідомленні" + }, + "searxng": { + "url": { + "label": "SearXNG URL-адреса" + } } }, "system": { diff --git a/src/assets/locale/zh/settings.json b/src/assets/locale/zh/settings.json index 0ec0554..83c576d 100644 --- a/src/assets/locale/zh/settings.json +++ b/src/assets/locale/zh/settings.json @@ -68,6 +68,11 @@ }, "visitSpecificWebsite": { "label": "访问消息中提到的网站。" + }, + "searxng": { + "url": { + "label": "SearXNG 网址" + } } }, "system": { diff --git a/src/components/Option/Settings/search-mode.tsx b/src/components/Option/Settings/search-mode.tsx index 53f391b..69ae112 100644 --- a/src/components/Option/Settings/search-mode.tsx +++ b/src/components/Option/Settings/search-mode.tsx @@ -3,7 +3,7 @@ import { getSearchSettings, setSearchSettings } from "@/services/search" import { SUPPORTED_SERACH_PROVIDERS } from "@/utils/search-provider" import { useForm } from "@mantine/form" import { useQuery, useQueryClient } from "@tanstack/react-query" -import { Select, Skeleton, Switch, InputNumber } from "antd" +import { Select, Skeleton, Switch, InputNumber, Input } from "antd" import { useTranslation } from "react-i18next" export const SearchModeSettings = () => { @@ -14,7 +14,9 @@ export const SearchModeSettings = () => { isSimpleInternetSearch: false, searchProvider: "", totalSearchResults: 0, - visitSpecificWebsite: false + visitSpecificWebsite: false, + searxngURL: "", + searxngJSONMode: false } }) @@ -62,6 +64,23 @@ export const SearchModeSettings = () => { /> + {form.values.searchProvider === "searxng" && ( + <> +
+ + {t("generalSettings.webSearch.searxng.url.label")} + +
+ +
+
+ + )}
{t("generalSettings.webSearch.searchMode.label")} diff --git a/src/services/search.ts b/src/services/search.ts index f548cfe..66fb5cf 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -62,20 +62,45 @@ export const setTotalSearchResults = async (totalSearchResults: number) => { await storage.set("totalSearchResults", totalSearchResults.toString()) } +export const getSearxngURL = async () => { + const searxngURL = await storage.get("searxngURL") + return searxngURL || "" +} + +export const isSearxngJSONMode = async () => { + const searxngJSONMode = await storage.get("searxngJSONMode") + return searxngJSONMode ?? false +} + +export const setSearxngJSONMode = async (searxngJSONMode: boolean) => { + await storage.set("searxngJSONMode", searxngJSONMode) +} + +export const setSearxngURL = async (searxngURL: string) => { + await storage.set("searxngURL", searxngURL) +} + export const getSearchSettings = async () => { - const [isSimpleInternetSearch, searchProvider, totalSearchResult, visitSpecificWebsite] = + const [isSimpleInternetSearch, searchProvider, totalSearchResult, visitSpecificWebsite, + searxngURL, + searxngJSONMode + ] = await Promise.all([ getIsSimpleInternetSearch(), getSearchProvider(), totalSearchResults(), - getIsVisitSpecificWebsite() + getIsVisitSpecificWebsite(), + getSearxngURL(), + isSearxngJSONMode() ]) return { isSimpleInternetSearch, searchProvider, totalSearchResults: totalSearchResult, - visitSpecificWebsite + visitSpecificWebsite, + searxngURL, + searxngJSONMode } } @@ -83,17 +108,23 @@ export const setSearchSettings = async ({ isSimpleInternetSearch, searchProvider, totalSearchResults, - visitSpecificWebsite + visitSpecificWebsite, + searxngJSONMode, + searxngURL }: { isSimpleInternetSearch: boolean searchProvider: string totalSearchResults: number visitSpecificWebsite: boolean + searxngURL: string + searxngJSONMode: boolean }) => { await Promise.all([ setIsSimpleInternetSearch(isSimpleInternetSearch), setSearchProvider(searchProvider), setTotalSearchResults(totalSearchResults), - setIsVisitSpecificWebsite(visitSpecificWebsite) + setIsVisitSpecificWebsite(visitSpecificWebsite), + setSearxngJSONMode(searxngJSONMode), + setSearxngURL(searxngURL) ]) } diff --git a/src/utils/search-provider.ts b/src/utils/search-provider.ts index d623df2..404c13f 100644 --- a/src/utils/search-provider.ts +++ b/src/utils/search-provider.ts @@ -14,5 +14,9 @@ export const SUPPORTED_SERACH_PROVIDERS = [ { label: "Brave", value: "brave" + }, + { + label: "Searxng", + value: "searxng" } ] \ No newline at end of file diff --git a/src/web/search-engines/searxng.ts b/src/web/search-engines/searxng.ts new file mode 100644 index 0000000..e5ad6b8 --- /dev/null +++ b/src/web/search-engines/searxng.ts @@ -0,0 +1,145 @@ +import { urlRewriteRuntime } from "~/libs/runtime" +import { cleanUrl } from "~/libs/clean-url" +import { getSearxngURL, isSearxngJSONMode, getIsSimpleInternetSearch, totalSearchResults } from "@/services/search" +import { pageAssistEmbeddingModel } from "@/models/embedding" +import type { Document } from "@langchain/core/documents" +import { RecursiveCharacterTextSplitter } from "langchain/text_splitter" +import { MemoryVectorStore } from "langchain/vectorstores/memory" +import { PageAssistHtmlLoader } from "~/loader/html" +import { + defaultEmbeddingChunkOverlap, + defaultEmbeddingChunkSize, + defaultEmbeddingModelForRag, + getOllamaURL +} from "~/services/ollama" + +interface SearxNGJSONResult { + title: string + url: string + content: string +} + +interface SearxNGJSONResponse { + results: SearxNGJSONResult[] +} + +export const searxngSearch = async (query: string) => { + const searxngURL = await getSearxngURL() + if (!searxngURL) { + throw new Error("SearXNG URL not configured") + } + + const isJSONMode = await isSearxngJSONMode() + const results = isJSONMode + ? await searxngJSONSearch(searxngURL, query) + : await searxngWebSearch(searxngURL, query) + + const TOTAL_SEARCH_RESULTS = await totalSearchResults() + const searchResults = results.slice(0, TOTAL_SEARCH_RESULTS) + + const isSimpleMode = await getIsSimpleInternetSearch() + + if (isSimpleMode) { + await getOllamaURL() + return searchResults.map((result) => { + return { + url: result.link, + content: result.content + } + }) + } + + const docs: Document>[] = [] + for (const result of searchResults) { + const loader = new PageAssistHtmlLoader({ + html: "", + url: result.link + }) + + const documents = await loader.loadByURL() + documents.forEach((doc) => { + docs.push(doc) + }) + } + + const ollamaUrl = await getOllamaURL() + const embeddingModel = await defaultEmbeddingModelForRag() + const ollamaEmbedding = await pageAssistEmbeddingModel({ + model: embeddingModel || "", + baseUrl: cleanUrl(ollamaUrl) + }) + + const chunkSize = await defaultEmbeddingChunkSize() + const chunkOverlap = await defaultEmbeddingChunkOverlap() + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize, + chunkOverlap + }) + + const chunks = await textSplitter.splitDocuments(docs) + const store = new MemoryVectorStore(ollamaEmbedding) + await store.addDocuments(chunks) + + return store +} + +const searxngJSONSearch = async (baseURL: string, query: string) => { + const searchURL = `${cleanUrl(baseURL)}?q=${encodeURIComponent(query)}&format=json` + + const abortController = new AbortController() + setTimeout(() => abortController.abort(), 20000) + + try { + const response = await fetch(searchURL, { + signal: abortController.signal, + headers: { + 'Accept': 'application/json' + } + }) + + if (!response.ok) { + throw new Error(`SearXNG search failed: ${response.statusText}`) + } + + const data = await response.json() as SearxNGJSONResponse + + return data.results.map(result => ({ + title: result.title, + link: result.url, + content: result.content + })) + } catch (error) { + console.error('SearXNG JSON search failed:', error) + return [] + } +} + +const searxngWebSearch = async (baseURL: string, query: string) => { + const searchURL = `${cleanUrl(baseURL)}?q=${encodeURIComponent(query)}` + + await urlRewriteRuntime(cleanUrl(searchURL), "searxng") + + const abortController = new AbortController() + setTimeout(() => abortController.abort(), 10000) + + try { + const htmlString = await fetch(searchURL, { + signal: abortController.signal + }).then(response => response.text()) + + const parser = new DOMParser() + const doc = parser.parseFromString(htmlString, "text/html") + + const searchResults = Array.from(doc.querySelectorAll("article.result")).map(result => { + const title = result.querySelector("h3")?.textContent?.trim() + const link = result.querySelector("a.url_header")?.getAttribute("href") + const content = result.querySelector("p.content")?.textContent?.trim() + return { title, link, content } + }).filter(result => result.title && result.link && result.content) + + return searchResults + } catch (error) { + console.error('SearXNG web search failed:', error) + return [] + } +} diff --git a/src/web/web.ts b/src/web/web.ts index 88aabc4..0f678d9 100644 --- a/src/web/web.ts +++ b/src/web/web.ts @@ -5,6 +5,7 @@ import { getIsVisitSpecificWebsite, getSearchProvider } from "@/services/search" import { webSogouSearch } from "./search-engines/sogou" import { webBraveSearch } from "./search-engines/brave" import { getWebsiteFromQuery, processSingleWebsite } from "./website" +import { searxngSearch } from "./search-engines/searxng" const getHostName = (url: string) => { try { @@ -23,6 +24,8 @@ const searchWeb = (provider: string, query: string) => { return webSogouSearch(query) case "brave": return webBraveSearch(query) + case "searxng": + return searxngSearch(query) default: return webGoogleSearch(query) } @@ -47,11 +50,11 @@ export const getSystemPromptForWeb = async (query: string) => { } else { const searchProvider = await getSearchProvider() + //@ts-ignore search = await searchWeb(searchProvider, query) } - const search_results = search .map( (result, idx) =>