iod/page-assist
10
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: Update logic of runtime url rewrite

Browse Source
This commit is contained in:
n4ze3m
2024-05-12 21:14:27 +05:30
parent bd84b90e5f
commit 62ffe8346e
11 changed files with 148 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/web/search-engines/duckduckgo.ts
View File

@@ -1,5 +1,5 @@
import { cleanUrl } from "@/libs/clean-url"
import { chromeRunTime } from "@/libs/runtime"
import { urlRewriteRuntime } from "@/libs/runtime"
import { PageAssistHtmlLoader } from "@/loader/html"
import {
defaultEmbeddingChunkOverlap,
@@ -18,7 +18,7 @@ import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
export const localDuckDuckGoSearch = async (query: string) => {
await chromeRunTime(cleanUrl("https://html.duckduckgo.com/html/?q=" + query))
await urlRewriteRuntime(cleanUrl("https://html.duckduckgo.com/html/?q=" + query))
const abortController = new AbortController()
setTimeout(() => abortController.abort(), 10000)

4
src/web/search-engines/google.ts
View File

@@ -7,7 +7,7 @@ import type { Document } from "@langchain/core/documents"
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter"
import { MemoryVectorStore } from "langchain/vectorstores/memory"
import { cleanUrl } from "~/libs/clean-url"
import { chromeRunTime } from "~/libs/runtime"
import { urlRewriteRuntime } from "~/libs/runtime"
import { PageAssistHtmlLoader } from "~/loader/html"
import {
defaultEmbeddingChunkOverlap,
@@ -18,7 +18,7 @@ import {
export const localGoogleSearch = async (query: string) => {
await chromeRunTime(
await urlRewriteRuntime(
cleanUrl("https://www.google.com/search?hl=en&q=" + query)
)
const abortController = new AbortController()

4
src/web/search-engines/sogou.ts
View File

@@ -1,5 +1,5 @@
import { cleanUrl } from "@/libs/clean-url"
import { chromeRunTime } from "@/libs/runtime"
import { urlRewriteRuntime } from "@/libs/runtime"
import { PageAssistHtmlLoader } from "@/loader/html"
import {
defaultEmbeddingChunkOverlap,
@@ -25,7 +25,7 @@ const getCorrectTargeUrl = async (url: string) => {
return matches?.[1] || ""
}
export const localSogouSearch = async (query: string) => {
await chromeRunTime(cleanUrl("https://www.sogou.com/web?query=" + query))
await urlRewriteRuntime(cleanUrl("https://www.sogou.com/web?query=" + query))
const abortController = new AbortController()