diff --git a/bun.lockb b/bun.lockb index 1212713..c2f8f9b 100644 Binary files a/bun.lockb and b/bun.lockb differ diff --git a/package.json b/package.json index 5d64161..44daa54 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "@langchain/community": "^0.0.41", "@mantine/form": "^7.5.0", "@mantine/hooks": "^7.5.3", + "@mozilla/readability": "^0.5.0", "@plasmohq/storage": "^1.9.0", "@tailwindcss/forms": "^0.5.7", "@tailwindcss/typography": "^0.5.10", diff --git a/src/assets/locale/en/settings.json b/src/assets/locale/en/settings.json index 0396fda..7107a00 100644 --- a/src/assets/locale/en/settings.json +++ b/src/assets/locale/en/settings.json @@ -37,6 +37,9 @@ "totalSearchResults": { "label": "Total Search Results", "placeholder": "Enter Total Search Results" + }, + "visitSpecificWebsite": { + "label": "Visit the website mentioned in the message" } }, "system": { diff --git a/src/assets/locale/es/common.json b/src/assets/locale/es/common.json new file mode 100644 index 0000000..e316a0b --- /dev/null +++ b/src/assets/locale/es/common.json @@ -0,0 +1,88 @@ +{ + "pageAssist": "Page Assist", + "selectAModel": "Selecione un Modelo", + "save": "Guardar", + "saved": "Guardado", + "cancel": "Cancelar", + "retry": "Reintentar", + "share": { + "tooltip": { + "share": "Compartir" + }, + "modal": { + "title": "Compartir enlace para chat" + }, + "form": { + "defaultValue": { + "name": "Anónimo", + "title": "Chat sin título" + }, + "title": { + "label": "Título del Chat", + "placeholder": "Ingresar el título del Chat", + "required": "El título del Chat es obligatorio" + }, + "name": { + "label": "Tu nombre", + "placeholder": "Ingresar tu nombre", + "required": "Tu nombre es obligatorio" + }, + "btn": { + "save": "Generar enlace", + "saving": "Generando enlace..." + } + }, + "notification": { + "successGenerate": "Enlace copiado al Clipboard", + "failGenerate": "Fallo al generar el enlace" + } + }, + "copyToClipboard": "Copiar al clipboard", + "webSearch": "Buscando en la web", + "regenerate": "Regenerar", + "edit": "Editar", + "saveAndSubmit": "Guardar y Enviar", + "editMessage": { + "placeholder": "Ingresar un mensaje..." + }, + "submit": "Enviar", + "noData": "Sin datos", + "noHistory": "Chat sin histórico", + "chatWithCurrentPage": "Conversar con la página actual", + "beta": "Beta", + "tts": "Leer en voz alta", + "currentChatModelSettings": "Configuraciones del Modelo de Chat Actual", + "modelSettings": { + "label": "Configuraciones del Modelo", + "description": "Definir las opciones del modelo globalmente para todos los chats", + "form": { + "keepAlive": { + "label": "Mantener vivo", + "help": "controlar cuanto tiempo el modelo permanecera cargado en la memoria luego de su utilización (por defecto: 5m)", + "placeholder": "Ingresar duración para mantenerlo vivo (ej: 5m, 10m, 1h)" + }, + "temperature": { + "label": "Temperatura", + "placeholder": "Ingresar valor de la Temperatura (ej: 0.7, 1.0)" + }, + "numCtx": { + "label": "Cantidad de contextos", + "placeholder": "Ingresar el valor de tamaño de la ventana de contexto (por defecto: 2048)" + }, + "seed": { + "label": "Semilla", + "placeholder": "Ingresar el valor de la semilla (ej: 1234)", + "help": "Reproductibilidad de la salida del modelo" + }, + "topK": { + "label": "Top K", + "placeholder": "Ingresar el valor de Top K (ej: 40, 100)" + }, + "topP": { + "label": "Top P", + "placeholder": "Ingresar el valor de Top P (ej: 0.9, 0.95)" + } + }, + "advanced": "Más Configuraciones del Modelo" + } +} diff --git a/src/assets/locale/es/knowledge.json b/src/assets/locale/es/knowledge.json new file mode 100644 index 0000000..86ddbf3 --- /dev/null +++ b/src/assets/locale/es/knowledge.json @@ -0,0 +1,42 @@ +{ + "addBtn": "Agregar Nuevo Conocimiento", + "columns": { + "title": "Título", + "status": "Estado", + "embeddings": "Modelo de Embedding", + "createdAt": "Creado", + "action": "Acciones" + }, + "expandedColumns": { + "name": "Nombre" + }, + "tooltip": { + "delete": "Borrar" + }, + "confirm": { + "delete": "¿Esta seguro que desea borrar este conocimiento?" + }, + "deleteSuccess": "Conocimiento borrado", + "status": { + "pending": "Pendiente", + "finished": "Finalizado", + "processing": "Procesando" + }, + "addKnowledge": "Agregar Conocimiento", + "form": { + "title": { + "label": "Título del Conocimiento", + "placeholder": "Ingresar un título de conocimiento", + "required": "El Título de conocimiento es obligatorio" + }, + "uploadFile": { + "label": "Subir un Archivo", + "uploadText": "Arraste y suelte un archivo aquí o haga click para subirlo", + "uploadHint": "Tipos de archivo soportados: .pdf, .csv, .txt, .md, .docx", + "required": "El archivo es obligatorio" + }, + "submit": "Enviar", + "success": "Conocimiento agregado exitosamente" + }, + "noEmbeddingModel": "Por favor, agregue un modelo de embedding de la página de configuraciones de RAG primero" +} diff --git a/src/assets/locale/es/option.json b/src/assets/locale/es/option.json new file mode 100644 index 0000000..3c48761 --- /dev/null +++ b/src/assets/locale/es/option.json @@ -0,0 +1,12 @@ +{ + "newChat": "Nuevo Chat", + "selectAPrompt": "Selecione un Prompt", + "githubRepository": "Repositorio de GitHub", + "settings": "Configuraciones", + "sidebarTitle": "Histórico del Chat", + "error": "Error", + "somethingWentWrong": "Hubo un error", + "validationSelectModel": "Selecione un modelo para continuar", + "deleteHistoryConfirmation": "¿Esta seguro que quiere borrar éste histórico?", + "editHistoryTitle": "Ingrese un nuevo título" +} diff --git a/src/assets/locale/es/playground.json b/src/assets/locale/es/playground.json new file mode 100644 index 0000000..afc3771 --- /dev/null +++ b/src/assets/locale/es/playground.json @@ -0,0 +1,29 @@ +{ + "ollamaState": { + "searching": "Buscando tu Ollama 🦙", + "running": "Ollama está funcionando 🦙", + "notRunning": "No fue posible conectar con Ollama 🦙", + "connectionError": "Hubo un error de conexión. Por favor, consulte la documentación para solucionar el problema." + }, + "formError": { + "noModel": "Por favor, selecione un modelo", + "noEmbeddingModel": "Por favor, defina un modelo de embedding para la página de configuraciones > RAG" + }, + "form": { + "textarea": { + "placeholder": "Ingrese un mensaje..." + }, + "webSearch": { + "on": "On", + "off": "Off" + } + }, + "tooltip": { + "searchInternet": "Buscar en Internet", + "speechToText": "Voz a Texto", + "uploadImage": "Subir Imagén", + "stopStreaming": "Parar Transmisión", + "knowledge": "Conocimiento" + }, + "sendWhenEnter": "Enviar cuando presione Enter" +} diff --git a/src/assets/locale/es/settings.json b/src/assets/locale/es/settings.json new file mode 100644 index 0000000..d1f1785 --- /dev/null +++ b/src/assets/locale/es/settings.json @@ -0,0 +1,292 @@ +{ + "generalSettings": { + "title": "Configuraciones Generales", + "settings": { + "heading": "Configuraciones de la Interfaz Web", + "speechRecognitionLang": { + "label": "Idioma de Reconocimiento de Voz", + "placeholder": "Selecione un idioma" + }, + "language": { + "label": "Idioma", + "placeholder": "Selecione un idioma" + }, + "darkMode": { + "label": "Cambiar Tema", + "options": { + "light": "Claro", + "dark": "Oscuro" + } + }, + "copilotResumeLastChat": { + "label": "Retomar el último chat al abrir el Panel Lateral (Copilot)" + }, + "hideCurrentChatModelSettings": { + "label": "Ocultar Configuraciones del Modelo de Chat Actual" + } + }, + "webSearch": { + "heading": "Manejo de la busqueda Web", + "searchMode": { + "label": "Realizar busquedas Simples en Internet" + }, + "provider": { + "label": "Motor de Busqueda", + "placeholder": "Selecione un motor de busqueda" + }, + "totalSearchResults": { + "label": "Resultados totales de la busqueda", + "placeholder": "Ingresar el total de Resultados de la busqueda" + }, + "visitSpecificWebsite": { + "label": "Visita el sitio web mencionado en el mensaje" + } + }, + "system": { + "heading": "Configuraciones del Sistema", + "deleteChatHistory": { + "label": "Borrar Histórico del Chat", + "button": "Borrar", + "confirm": "¿Esta seguro que desea borrar su histórico del chat? Esta acción no podra ser desecha." + }, + "export": { + "label": "Exportar Histórico del Chat, Base de Conocimiento y Prompts", + "button": "Exportar Datos", + "success": "Exportación exitosa" + }, + "import": { + "label": "Importar Histórico del Chat, Base de Conocimiento y Prompts", + "button": "Importar Datos", + "success": "Importación existosa", + "error": "Error de importación" + } + }, + "tts": { + "heading": "Configuraciones de Text-to-speech", + "ttsEnabled": { + "label": "Habilitar Texto-a-Voz" + }, + "ttsProvider": { + "label": "Proveedor de Text-to-speech", + "placeholder": "Selecione un proveedor" + }, + "ttsVoice": { + "label": "Voz de Text-to-speech", + "placeholder": "Selecione una voz" + }, + "ssmlEnabled": { + "label": "Habilitar SSML (Speech Synthesis Markup Language)" + } + } + }, + "manageModels": { + "title": "Administar de Modelos", + "addBtn": "Agregar Nuevo Modelo", + "columns": { + "name": "Nombre", + "digest": "Resumen", + "modifiedAt": "Modificado", + "size": "Tamaño", + "actions": "Acciones" + }, + "expandedColumns": { + "parentModel": "Modelo Padre", + "format": "Formato", + "family": "Familia", + "parameterSize": "Tamaño de Parametros", + "quantizationLevel": "Nível de Quantización" + }, + "tooltip": { + "delete": "Borrar Modelo", + "repull": "Traer nuevamente el Modelo" + }, + "confirm": { + "delete": "¿Esta seguro que desea borrar este modelos?", + "repull": "¿Esta seguro que desea traer nuevamente este modelo?" + }, + "modal": { + "title": "Traer Nuevo Modelo", + "placeholder": "Ingresar el nombre del modelo", + "pull": "Traer Modelo" + }, + "notification": { + "pullModel": "Trayendo Modelo", + "pullModelDescription": "Trayendo modelo {{modelName}}. Para más detalles, verifique el ícono de la extensión.", + "success": "Exito", + "error": "Error", + "successDescription": "Modelo traido exitosamente", + "successDeleteDescription": "Modelo borrado exitosamente", + "someError": "Hubo un error. Intente nuevamente más tarde" + } + }, + "managePrompts": { + "title": "Administrar de Prompts", + "addBtn": "Agregar Nuevo Prompt", + "option1": "Normal", + "option2": "RAG", + "questionPrompt": "Prompt de Pregunta", + "columns": { + "title": "Título", + "prompt": "Prompt", + "type": "Tipo de Prompt", + "actions": "Acciones" + }, + "systemPrompt": "Prompt del Sistema", + "quickPrompt": "Prompt Rápido", + "tooltip": { + "delete": "Borrar Prompt", + "edit": "Editar Prompt" + }, + "confirm": { + "delete": "¿Esta seguro que desea borrar este prompt? Esta acción no tiene vuelta a atrás." + }, + "modal": { + "addTitle": "Agregar Nuevo Prompt", + "editTitle": "Editar Prompt" + }, + "form": { + "title": { + "label": "Título", + "placeholder": "Mi Prompt genial", + "required": "Por favor, ingrese un título" + }, + "prompt": { + "label": "Prompt", + "placeholder": "Ingrese un prompt", + "required": "Por favor, ingrese un prompt", + "help": "Puede usar {key} como variable en su prompt." + }, + "isSystem": { + "label": "Es un Prompt del Sistema" + }, + "btnSave": { + "saving": "Agregando un Prompt...", + "save": "Agregar Prompt" + }, + "btnEdit": { + "saving": "Actualizando Prompt...", + "save": "Actualizar Prompt" + } + }, + "notification": { + "addSuccess": "Prompt Agregado", + "addSuccessDesc": "Prompt agregado exitosamente", + "error": "Error", + "someError": "Hubo un error. Intente nuevamente más tarde", + "updatedSuccess": "Prompt Actualizado", + "updatedSuccessDesc": "Prompt actualizado exitosamente", + "deletedSuccess": "Prompt Borrado", + "deletedSuccessDesc": "Prompt borrado exitosamente" + } + }, + "manageShare": { + "title": "Administrar los recursos compartidos", + "heading": "Configurar URL de Página Compartida", + "form": { + "url": { + "label": "URL de Página compartida", + "placeholder": "Ingresar URL de Página compartida", + "required": "Por favor, ingrese URL de Página compartida", + "help": "Por motivos de privacidad, podes hacer self-host de la página compartida y proveer una URL aqui. Aprende más." + } + }, + "webshare": { + "heading": "Compartir una Web", + "columns": { + "title": "Título", + "url": "URL", + "actions": "Acciones" + }, + "tooltip": { + "delete": "Borrar lo compartido" + }, + "confirm": { + "delete": "¿Esta seguro de desear borrar esta web compartida? Esta acción no tiene vuelta a atrás." + }, + "label": "Administrar páginas compartidas", + "description": "Habilitar o deshabilitar el recurso de páginas compartidas" + }, + "notification": { + "pageShareSuccess": "URL compartida actualizada exitosamente", + "someError": "Hubo un error. Intente nuevamente más tarde", + "webShareDeleteSuccess": "Web compartida borrada exitosamente com sucesso" + } + }, + "ollamaSettings": { + "title": "Configuraciones de Ollama", + "heading": "Configurar Ollama", + "settings": { + "ollamaUrl": { + "label": "URL de Ollama", + "placeholder": "Ingrese la URL de Ollama" + }, + "advanced": { + "label": "Configuración avanzada de URL de Ollama", + "urlRewriteEnabled": { + "label": "Habilitar o Deshabilitar URL Personalizada" + }, + "rewriteUrl": { + "label": "URL Personalizada", + "placeholder": "Ingresar URL Personalizada" + }, + "help": "Si tenes problemas de conexión con Ollama en Page Assist, podes configurar una URL de personalizada. Para saber más sobre la configuración, click aqui." + } + } + }, + "manageSearch": { + "title": "Administrar Busqueda Web", + "heading": "Configurar Busqueda Web" + }, + "about": { + "title": "Sobre", + "heading": "Sobre", + "chromeVersion": "Versión de Page Assist", + "ollamaVersion": "Versión de Ollama", + "support": "Podes apoyar el proyecto Page Assist haciendo donaciones o patrocinarnos a través de las seguientes plataformas:", + "koFi": "Apoyar en Ko-fi", + "githubSponsor": "Patrocinarnos en GitHub", + "githubRepo": "Repositorio de GitHub" + }, + "manageKnowledge": { + "title": "Administrar Conocimiento", + "heading": "Configurar Bases de Conocimiento" + }, + "rag": { + "title": "Configuraciones de RAG", + "ragSettings": { + "label": "Configuraciones de RAG", + "model": { + "label": "Modelo de embeddings", + "required": "Por favor, selecione un modelo", + "help": "Es recomendable usar modelos de embeddings como `nomic-embed-text`.", + "placeholder": "Selecione un modelo" + }, + "chunkSize": { + "label": "Tamaño del Chunk", + "placeholder": "Ingresar el tamaño del chunk", + "required": "Por favor, ingrese el tamaño del chunk" + }, + "chunkOverlap": { + "label": "Solapamiento del Chunk", + "placeholder": "Ingrese el solapamiento del chunk", + "required": "Por favor, ingresar el solapamiento del chunk" + } + }, + "prompt": { + "label": "Configurar el Prompt del RAG", + "option1": "Normal", + "option2": "Web", + "alert": "Es obsoleto configurar aquí el prompt del sistema. Por favor, use la sección de Administrar Prompts para agregar o editar prompts. Esta sección se quitará en una versión futura", + "systemPrompt": "Prompt del Sistema", + "systemPromptPlaceholder": "Ingresar el prompt del sistema", + "webSearchPrompt": "Prompt de la busqueda Web", + "webSearchPromptHelp": "No borre `{search_results}` del prompt.", + "webSearchPromptError": "Por favor, ingresar un prompt de busqueda web", + "webSearchPromptPlaceholder": "Ingrese un prompt de busqueda web", + "webSearchFollowUpPrompt": "Prompt de Seguimiento de busqueda Web", + "webSearchFollowUpPromptHelp": "No borre `{chat_history}` y `{question}` del prompt.", + "webSearchFollowUpPromptError": "Por favor, ingrese el prompt de seguimiento de la busqueda web", + "webSearchFollowUpPromptPlaceholder": "Su prompt de seguimiento de busqueda web" + } + } +} \ No newline at end of file diff --git a/src/assets/locale/es/sidepanel.json b/src/assets/locale/es/sidepanel.json new file mode 100644 index 0000000..214c8b6 --- /dev/null +++ b/src/assets/locale/es/sidepanel.json @@ -0,0 +1,7 @@ +{ + "tooltip": { + "embed": "Puede demorar algunos minutos para incluir la página. Por favor, aguarde...", + "clear": "Borrar el histórico de conversación", + "history": "Histórico de la conversación" + } +} diff --git a/src/assets/locale/fr/settings.json b/src/assets/locale/fr/settings.json index e06e7ed..634358f 100644 --- a/src/assets/locale/fr/settings.json +++ b/src/assets/locale/fr/settings.json @@ -37,6 +37,9 @@ "totalSearchResults": { "label": "Résultats de la recherche totaux", "placeholder": "Entrez les résultats de la recherche totaux" + }, + "visitSpecificWebsite": { + "label": "Visitez le site web mentionné dans le message" } }, "system": { diff --git a/src/assets/locale/it/settings.json b/src/assets/locale/it/settings.json index c083e5f..41bef3d 100644 --- a/src/assets/locale/it/settings.json +++ b/src/assets/locale/it/settings.json @@ -37,6 +37,9 @@ "totalSearchResults": { "label": "Risultati della ricerca", "placeholder": "Inserisci il totale delle ricerche" + }, + "visitSpecificWebsite": { + "label": "Visita il sito web menzionato nel messaggio" } }, "system": { diff --git a/src/assets/locale/ja-JP/settings.json b/src/assets/locale/ja-JP/settings.json index cef567f..74c9567 100644 --- a/src/assets/locale/ja-JP/settings.json +++ b/src/assets/locale/ja-JP/settings.json @@ -40,6 +40,9 @@ "totalSearchResults": { "label": "合計検索結果", "placeholder": "合計検索結果を入力する" + }, + "visitSpecificWebsite": { + "label": "メッセージに記載されたウェブサイトを訪問してください" } }, "system": { diff --git a/src/assets/locale/ml/settings.json b/src/assets/locale/ml/settings.json index b83a07e..46780df 100644 --- a/src/assets/locale/ml/settings.json +++ b/src/assets/locale/ml/settings.json @@ -40,6 +40,9 @@ "totalSearchResults": { "label": "ആകെ തിരച്ചിൽ ഫലങ്ങൾ", "placeholder": "ആകെ തിരച്ചിൽ ഫലങ്ങളുടെ എണ്ണം നൽകുക" + }, + "visitSpecificWebsite": { + "label": "സന്ദേശത്തിൽ പറയുന്ന വെബ്സൈറ്റ് സന്ദർശിക്കുക." } }, "system": { diff --git a/src/assets/locale/pt-BR/settings.json b/src/assets/locale/pt-BR/settings.json index fb22698..21bcee4 100644 --- a/src/assets/locale/pt-BR/settings.json +++ b/src/assets/locale/pt-BR/settings.json @@ -37,6 +37,9 @@ "totalSearchResults": { "label": "Resultados de Pesquisa Totais", "placeholder": "Insira Resultados de Pesquisa Totais" + }, + "visitSpecificWebsite": { + "label": "Visite o site mencionado na mensagem." } }, "system": { diff --git a/src/assets/locale/ru/settings.json b/src/assets/locale/ru/settings.json index d745db8..6099b2a 100644 --- a/src/assets/locale/ru/settings.json +++ b/src/assets/locale/ru/settings.json @@ -37,6 +37,9 @@ "totalSearchResults": { "label": "Общее количество результатов поиска", "placeholder": "Введите общее количество результатов поиска" + }, + "visitSpecificWebsite": { + "label": "Посетите веб-сайт, указанный в сообщении." } }, "system": { diff --git a/src/assets/locale/zh/settings.json b/src/assets/locale/zh/settings.json index 9b27993..44213eb 100644 --- a/src/assets/locale/zh/settings.json +++ b/src/assets/locale/zh/settings.json @@ -40,7 +40,10 @@ "totalSearchResults": { "label": "总搜索结果", "placeholder": "输入总搜索结果" - } + }, + "visitSpecificWebsite": { + "label": "访问消息中提到的网站。" + } }, "system": { "heading": "系统设置", diff --git a/src/components/Option/Playground/PlaygroundForm.tsx b/src/components/Option/Playground/PlaygroundForm.tsx index c3e97ec..8d248f8 100644 --- a/src/components/Option/Playground/PlaygroundForm.tsx +++ b/src/components/Option/Playground/PlaygroundForm.tsx @@ -14,6 +14,7 @@ import { useTranslation } from "react-i18next" import { KnowledgeSelect } from "../Knowledge/KnowledgeSelect" import { useSpeechRecognition } from "@/hooks/useSpeechRecognition" import { PiGlobe } from "react-icons/pi" +import { extractReadabilityContent } from "@/parser/reader" type Props = { dropedFile: File | undefined diff --git a/src/components/Option/Settings/search-mode.tsx b/src/components/Option/Settings/search-mode.tsx index e7cc04d..53f391b 100644 --- a/src/components/Option/Settings/search-mode.tsx +++ b/src/components/Option/Settings/search-mode.tsx @@ -13,7 +13,8 @@ export const SearchModeSettings = () => { initialValues: { isSimpleInternetSearch: false, searchProvider: "", - totalSearchResults: 0 + totalSearchResults: 0, + visitSpecificWebsite: false } }) @@ -67,7 +68,7 @@ export const SearchModeSettings = () => {
{
+
+ + {t("generalSettings.webSearch.visitSpecificWebsite.label")} + +
+ +
+
+
diff --git a/src/hooks/useScrollAnchor.tsx b/src/hooks/useScrollAnchor.tsx index a8ec2ad..c2d72a2 100644 --- a/src/hooks/useScrollAnchor.tsx +++ b/src/hooks/useScrollAnchor.tsx @@ -1,87 +1,106 @@ import { useCallback, useEffect, useRef, useState } from "react" +import { useMessageOption } from "./useMessageOption" export const useScrollAnchor = () => { - const messagesRef = useRef(null) - const scrollRef = useRef(null) - const visibilityRef = useRef(null) + const { isProcessing, messages } = useMessageOption() + const [isAtTop, setIsAtTop] = useState(false) const [isAtBottom, setIsAtBottom] = useState(true) - const [isVisible, setIsVisible] = useState(false) + const [userScrolled, setUserScrolled] = useState(false) + const [isOverflowing, setIsOverflowing] = useState(false) + + const messagesStartRef = useRef(null) + const messagesEndRef = useRef(null) + const containerRef = useRef(null) + const isAutoScrolling = useRef(false) + + console.log(`isAtTop: ${isAtTop}, isAtBottom: ${isAtBottom}, userScrolled: ${userScrolled}, isOverflowing: ${isOverflowing}`) + + useEffect(() => { + if (!isProcessing && userScrolled) { + console.log("userScrolled") + setUserScrolled(false) + } + }, [isProcessing]) + + useEffect(() => { + if (isProcessing && !userScrolled) { + scrollToBottom() + } + }, [messages]) + + useEffect(() => { + const container = containerRef.current + if (!container) return + + const topObserver = new IntersectionObserver( + ([entry]) => { + setIsAtTop(entry.isIntersecting) + }, + { threshold: 1 } + ) + + const bottomObserver = new IntersectionObserver( + ([entry]) => { + setIsAtBottom(entry.isIntersecting) + if (entry.isIntersecting) { + setUserScrolled(false) + } else if (!isAutoScrolling.current) { + setUserScrolled(true) + } + }, + { threshold: 1 } + ) + + if (messagesStartRef.current) { + topObserver.observe(messagesStartRef.current) + } + + if (messagesEndRef.current) { + bottomObserver.observe(messagesEndRef.current) + } + + const resizeObserver = new ResizeObserver(() => { + setIsOverflowing(container.scrollHeight > container.clientHeight) + }) + + resizeObserver.observe(container) + + return () => { + topObserver.disconnect() + bottomObserver.disconnect() + resizeObserver.disconnect() + } + }, []) + + const scrollToTop = useCallback(() => { + if (messagesStartRef.current) { + messagesStartRef.current.scrollIntoView({ behavior: "smooth" }) + } + }, []) const scrollToBottom = useCallback(() => { - if (messagesRef.current) { - messagesRef.current.scrollIntoView({ - block: "end", - behavior: "smooth" - }) - } + isAutoScrolling.current = true + + setTimeout(() => { + if (messagesEndRef.current) { + messagesEndRef.current.scrollIntoView({ behavior: "smooth" }) + } + + isAutoScrolling.current = false + }, 100) }, []) - useEffect(() => { - if (messagesRef.current) { - if (isAtBottom && !isVisible) { - messagesRef.current.scrollIntoView({ - block: "end" - }) - } - } - }, [isAtBottom, isVisible]) - - useEffect(() => { - const { current } = scrollRef - - if (current) { - const handleScroll = (event: Event) => { - const target = event.target as HTMLDivElement - const offset = 25 - const isAtBottom = - target.scrollTop + target.clientHeight >= target.scrollHeight - offset - console.log(target.scrollTop, target.clientHeight, target.scrollHeight) - setIsAtBottom(isAtBottom) - } - - current.addEventListener("scroll", handleScroll, { - passive: true - }) - - return () => { - current.removeEventListener("scroll", handleScroll) - } - } - }, []) - - useEffect(() => { - if (visibilityRef.current) { - let observer = new IntersectionObserver( - (entries) => { - entries.forEach((entry) => { - console.log(entry.isIntersecting) - if (entry.isIntersecting) { - setIsVisible(true) - } else { - setIsVisible(false) - } - }) - }, - { - rootMargin: "0px 0px -100px 0px" - } - ) - - observer.observe(visibilityRef.current) - - return () => { - observer.disconnect() - } - } - }) - return { - messagesRef, - scrollRef, - visibilityRef, - scrollToBottom, + messagesStartRef, + messagesEndRef, + containerRef, + isAtTop, isAtBottom, - isVisible + userScrolled, + isOverflowing, + scrollToTop, + scrollToBottom, + setIsAtBottom } -} +} \ No newline at end of file diff --git a/src/i18n/index.ts b/src/i18n/index.ts index 525ada9..6c5915f 100644 --- a/src/i18n/index.ts +++ b/src/i18n/index.ts @@ -8,6 +8,7 @@ import { ml } from "./lang/ml"; import { zh } from "./lang/zh"; import { ja } from "./lang/ja"; import { it } from "./lang/it"; +import { es } from "./lang/es"; import LanguageDetector from 'i18next-browser-languagedetector'; i18n @@ -16,6 +17,7 @@ i18n .init({ resources: { en: en, + es: es, fr: fr, "it": it, ml: ml, @@ -31,4 +33,4 @@ i18n lng: localStorage.getItem("i18nextLng") || "en", }) -export default i18n; \ No newline at end of file +export default i18n; diff --git a/src/i18n/lang/es.ts b/src/i18n/lang/es.ts new file mode 100644 index 0000000..c666286 --- /dev/null +++ b/src/i18n/lang/es.ts @@ -0,0 +1,15 @@ +import option from "@/assets/locale/es/option.json"; +import playground from "@/assets/locale/es/playground.json"; +import common from "@/assets/locale/es/common.json"; +import sidepanel from "@/assets/locale/es/sidepanel.json"; +import settings from "@/assets/locale/es/settings.json"; +import knowledge from "@/assets/locale/es/knowledge.json"; + +export const es = { + option, + playground, + common, + sidepanel, + settings, + knowledge +} diff --git a/src/i18n/support-language.ts b/src/i18n/support-language.ts index 89b0d97..7cd56a0 100644 --- a/src/i18n/support-language.ts +++ b/src/i18n/support-language.ts @@ -4,6 +4,10 @@ export const supportLanguage = [ label: "English", value: "en" }, + { + label: "Español", + value: "es" + }, { label: "Français", value: "fr" @@ -19,7 +23,7 @@ export const supportLanguage = [ { label: "Português (Brasil)", value: "pt-BR" - }, + }, { label: "മലയാളം", value: "ml" @@ -32,4 +36,4 @@ export const supportLanguage = [ label: "日本語", value: "ja-JP" } -] \ No newline at end of file +] diff --git a/src/loader/html.ts b/src/loader/html.ts index 786c60e..5eeb168 100644 --- a/src/loader/html.ts +++ b/src/loader/html.ts @@ -1,9 +1,9 @@ import { BaseDocumentLoader } from "langchain/document_loaders/base" import { Document } from "@langchain/core/documents" -import { compile } from "html-to-text" import { urlRewriteRuntime } from "~/libs/runtime" import { YtTranscript } from "yt-transcript" import { isWikipedia, parseWikipedia } from "@/parser/wiki" +import { extractReadabilityContent } from "@/parser/reader" const YT_REGEX = /(?:https?:\/\/)?(?:www\.)?(?:youtube\.com|youtu\.be)\/(?:watch\?v=)?([a-zA-Z0-9_-]+)/ @@ -24,8 +24,7 @@ export interface WebLoaderParams { export class PageAssistHtmlLoader extends BaseDocumentLoader - implements WebLoaderParams -{ + implements WebLoaderParams { html: string url: string @@ -52,30 +51,14 @@ export class PageAssistHtmlLoader { metadata: { source: this.url, + url: this.url, audio: { chunks: transcript } }, pageContent: text } ] } - - // let html = this.html - - // if (isWikipedia(this.url)) { - // console.log("Wikipedia URL detected") - // html = parseWikipedia(html) - // } - - // // else if (isTwitter(this.url)) { - // // console.log("Twitter URL detected") - // // html = parseTweet(html, this.url) - // // } - - // const htmlCompiler = compile({ - // wordwrap: false - // }) - // const text = htmlCompiler(html) - const metadata = { source: this.url } + const metadata = { source: this.url, url: this.url, } return [new Document({ pageContent: this.html, metadata })] } @@ -95,6 +78,7 @@ export class PageAssistHtmlLoader return [ { metadata: { + url: this.url, source: this.url, audio: { chunks: transcript } }, @@ -103,22 +87,15 @@ export class PageAssistHtmlLoader ] } await urlRewriteRuntime(this.url, "web") - const fetchHTML = await fetch(this.url) - let html = await fetchHTML.text() - + let text = ""; if (isWikipedia(this.url)) { console.log("Wikipedia URL detected") - html = parseWikipedia(await fetchHTML.text()) + const fetchHTML = await fetch(this.url) + text = parseWikipedia(await fetchHTML.text()) + } else { + text = await extractReadabilityContent(this.url) } - const htmlCompiler = compile({ - wordwrap: false, - selectors: [ - { selector: "img", format: "skip" }, - { selector: "script", format: "skip" } - ] - }) - const text = htmlCompiler(html) const metadata = { url: this.url } return [new Document({ pageContent: text, metadata })] } diff --git a/src/parser/reader.ts b/src/parser/reader.ts new file mode 100644 index 0000000..23b22bf --- /dev/null +++ b/src/parser/reader.ts @@ -0,0 +1,19 @@ +import { Readability } from "@mozilla/readability" +import { defaultExtractContent } from "./default" +export const extractReadabilityContent = async (url: string) => { + const response = await fetch(url) + if (!response.ok) { + throw new Error(`Failed to fetch ${url}`) + } + + const html = await response.text() + + // create a fake dom for Readability + const doc = new DOMParser().parseFromString(html, "text/html") + const reader = new Readability(doc) + const article = reader.parse() + + // convert the article to markdown + const markdown = defaultExtractContent(article.content) + return markdown +} \ No newline at end of file diff --git a/src/parser/wiki.ts b/src/parser/wiki.ts index 36f567c..2dbe88c 100644 --- a/src/parser/wiki.ts +++ b/src/parser/wiki.ts @@ -1,4 +1,5 @@ import * as cheerio from "cheerio" +import { defaultExtractContent } from "./default" export const isWikipedia = (url: string) => { const WIKI_REGEX = /wikipedia\.org\/wiki\//g @@ -24,5 +25,5 @@ export const parseWikipedia = (html: string) => { content?.find("div.toc")?.remove() const newHtml = content?.html() - return `
TITLE: ${title?.text()}
${newHtml}
` + return defaultExtractContent(`
TITLE: ${title?.text()}
${newHtml}
`) } diff --git a/src/services/app.ts b/src/services/app.ts index e7d871a..9fb2ccd 100644 --- a/src/services/app.ts +++ b/src/services/app.ts @@ -38,4 +38,18 @@ export const getAdvancedOllamaSettings = async () => { export const copilotResumeLastChat = async () => { return await storage.get("copilotResumeLastChat") +} + + +export const defaultSidebarOpen = async () => { + const sidebarOpen = await storage.get("sidebarOpen") + if (!sidebarOpen || sidebarOpen === "") { + return "right_clk" + } + return sidebarOpen +} + + +export const setSidebarOpen = async (sidebarOpen: string) => { + await storage.set("sidebarOpen", sidebarOpen) } \ No newline at end of file diff --git a/src/services/search.ts b/src/services/search.ts index f3851b9..f548cfe 100644 --- a/src/services/search.ts +++ b/src/services/search.ts @@ -15,6 +15,21 @@ export const getIsSimpleInternetSearch = async () => { return isSimpleInternetSearch === "true" } +export const getIsVisitSpecificWebsite = async () => { + const isVisitSpecificWebsite = await storage.get("isVisitSpecificWebsite") + if (!isVisitSpecificWebsite || isVisitSpecificWebsite.length === 0) { + return true + } + return isVisitSpecificWebsite === "true" +} + + +export const setIsVisitSpecificWebsite = async ( + isVisitSpecificWebsite: boolean +) => { + await storage.set("isVisitSpecificWebsite", isVisitSpecificWebsite.toString()) +} + export const setIsSimpleInternetSearch = async ( isSimpleInternetSearch: boolean ) => { @@ -48,32 +63,37 @@ export const setTotalSearchResults = async (totalSearchResults: number) => { } export const getSearchSettings = async () => { - const [isSimpleInternetSearch, searchProvider, totalSearchResult] = + const [isSimpleInternetSearch, searchProvider, totalSearchResult, visitSpecificWebsite] = await Promise.all([ getIsSimpleInternetSearch(), getSearchProvider(), - totalSearchResults() + totalSearchResults(), + getIsVisitSpecificWebsite() ]) return { isSimpleInternetSearch, searchProvider, - totalSearchResults: totalSearchResult + totalSearchResults: totalSearchResult, + visitSpecificWebsite } } export const setSearchSettings = async ({ isSimpleInternetSearch, searchProvider, - totalSearchResults + totalSearchResults, + visitSpecificWebsite }: { isSimpleInternetSearch: boolean searchProvider: string totalSearchResults: number + visitSpecificWebsite: boolean }) => { await Promise.all([ setIsSimpleInternetSearch(isSimpleInternetSearch), setSearchProvider(searchProvider), - setTotalSearchResults(totalSearchResults) + setTotalSearchResults(totalSearchResults), + setIsVisitSpecificWebsite(visitSpecificWebsite) ]) } diff --git a/src/utils/rerank.ts b/src/utils/rerank.ts new file mode 100644 index 0000000..fcf30cf --- /dev/null +++ b/src/utils/rerank.ts @@ -0,0 +1,44 @@ +import type { Embeddings } from "@langchain/core/embeddings" +import type { Document } from "@langchain/core/documents" +import * as ml_distance from "ml-distance" + +export const rerankDocs = async ({ + query, + docs, + embedding +}: { + query: string + docs: Document[] + embedding: Embeddings +}) => { + if (docs.length === 0) { + return docs + } + + const docsWithContent = docs.filter( + (doc) => doc.pageContent && doc.pageContent.length > 0 + ) + + const [docEmbeddings, queryEmbedding] = await Promise.all([ + embedding.embedDocuments(docsWithContent.map((doc) => doc.pageContent)), + embedding.embedQuery(query) + ]) + + const similarity = docEmbeddings.map((docEmbedding, i) => { + // perform cosine similarity between query and document + const sim = ml_distance.similarity.cosine(queryEmbedding, docEmbedding) + + return { + index: i, + similarity: sim + } + }) + + const sortedDocs = similarity + .sort((a, b) => b.similarity - a.similarity) + .filter((sim) => sim.similarity > 0.5) + .slice(0, 15) + .map((sim) => docsWithContent[sim.index]) + + return sortedDocs +} diff --git a/src/web/web.ts b/src/web/web.ts index e9c1765..88aabc4 100644 --- a/src/web/web.ts +++ b/src/web/web.ts @@ -1,9 +1,10 @@ import { getWebSearchPrompt } from "~/services/ollama" import { webGoogleSearch } from "./search-engines/google" import { webDuckDuckGoSearch } from "./search-engines/duckduckgo" -import { getSearchProvider } from "@/services/search" +import { getIsVisitSpecificWebsite, getSearchProvider } from "@/services/search" import { webSogouSearch } from "./search-engines/sogou" import { webBraveSearch } from "./search-engines/brave" +import { getWebsiteFromQuery, processSingleWebsite } from "./website" const getHostName = (url: string) => { try { @@ -29,8 +30,27 @@ const searchWeb = (provider: string, query: string) => { export const getSystemPromptForWeb = async (query: string) => { try { - const searchProvider = await getSearchProvider() - const search = await searchWeb(searchProvider, query) + + const websiteVisit = getWebsiteFromQuery(query) + let search: { + url: any; + content: string; + }[] = [] + + const isVisitSpecificWebsite = await getIsVisitSpecificWebsite() + + if (isVisitSpecificWebsite && websiteVisit.hasUrl) { + + const url = websiteVisit.url + const queryWithoutUrl = websiteVisit.queryWithouUrls + search = await processSingleWebsite(url, queryWithoutUrl) + + } else { + const searchProvider = await getSearchProvider() + search = await searchWeb(searchProvider, query) + } + + const search_results = search .map( diff --git a/src/web/website/index.ts b/src/web/website/index.ts new file mode 100644 index 0000000..31ed5f9 --- /dev/null +++ b/src/web/website/index.ts @@ -0,0 +1,76 @@ +import { cleanUrl } from "@/libs/clean-url" +import { PageAssistHtmlLoader } from "@/loader/html" +import { defaultEmbeddingChunkOverlap, defaultEmbeddingChunkSize, defaultEmbeddingModelForRag, getOllamaURL } from "@/services/ollama" +import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama" +import { RecursiveCharacterTextSplitter } from "langchain/text_splitter" +import { MemoryVectorStore } from "langchain/vectorstores/memory" + +export const processSingleWebsite = async (url: string, query: string) => { + const loader = new PageAssistHtmlLoader({ + html: "", + url + }) + const docs = await loader.loadByURL() + + const ollamaUrl = await getOllamaURL() + + const embeddingModle = await defaultEmbeddingModelForRag() + const ollamaEmbedding = new OllamaEmbeddings({ + model: embeddingModle || "", + baseUrl: cleanUrl(ollamaUrl) + }) + + const chunkSize = await defaultEmbeddingChunkSize() + const chunkOverlap = await defaultEmbeddingChunkOverlap() + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize, + chunkOverlap + }) + + const chunks = await textSplitter.splitDocuments(docs) + + const store = new MemoryVectorStore(ollamaEmbedding) + + await store.addDocuments(chunks) + + const resultsWithEmbeddings = await store.similaritySearch(query, 4) + + const searchResult = resultsWithEmbeddings.map((result) => { + return { + url: result.metadata.url, + content: result.pageContent + } + }) + + return searchResult +} + + +export const getWebsiteFromQuery = (query: string): { + queryWithouUrls: string, + url: string, + hasUrl: boolean +} => { + + const urlRegex = /https?:\/\/[^\s]+/g + + const urls = query.match(urlRegex) + + if (!urls) { + return { + queryWithouUrls: query, + url: "", + hasUrl: false + } + } + + const url = urls[0] + + const queryWithouUrls = query.replace(url, "") + + return { + queryWithouUrls, + url, + hasUrl: true + } +} \ No newline at end of file diff --git a/wxt.config.ts b/wxt.config.ts index 8d94b9f..be6459d 100644 --- a/wxt.config.ts +++ b/wxt.config.ts @@ -48,7 +48,7 @@ export default defineConfig({ outDir: "build", manifest: { - version: "1.1.12", + version: "1.1.13", name: process.env.TARGET === "firefox" ? "Page Assist - A Web UI for Local AI Models"