diff --git a/package.json b/package.json index a21fede..8cf6e16 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "pageassist", "displayName": "Page Assist - A Web UI for Local AI Models", - "version": "1.0.7", + "version": "1.0.8", "description": "Use your locally running AI models to assist you in your web browsing.", "author": "n4ze3m", "scripts": { @@ -26,7 +26,7 @@ "dayjs": "^1.11.10", "html-to-text": "^9.0.5", "langchain": "^0.1.9", - "lucide-react": "^0.340.0", + "lucide-react": "^0.350.0", "plasmo": "0.84.1", "property-information": "^6.4.1", "react": "18.2.0", @@ -38,6 +38,7 @@ "rehype-mathjax": "4.0.3", "remark-gfm": "3.0.1", "remark-math": "5.1.1", + "yt-transcript": "^0.0.2", "zustand": "^4.5.0" }, "devDependencies": { diff --git a/src/components/Sidepanel/Chat/empty.tsx b/src/components/Sidepanel/Chat/empty.tsx index 9258385..4042d86 100644 --- a/src/components/Sidepanel/Chat/empty.tsx +++ b/src/components/Sidepanel/Chat/empty.tsx @@ -38,7 +38,7 @@ export const EmptySidePanel = () => { } }, [ollamaInfo]) - const { setSelectedModel, selectedModel, chatMode, setChatMode } = + const { setSelectedModel, selectedModel, chatMode, setChatMode, } = useMessage() return ( diff --git a/src/hooks/useMessage.tsx b/src/hooks/useMessage.tsx index f24259c..7e93de1 100644 --- a/src/hooks/useMessage.tsx +++ b/src/hooks/useMessage.tsx @@ -20,9 +20,12 @@ import { getHtmlOfCurrentTab } from "~libs/get-html" import { PageAssistHtmlLoader } from "~loader/html" import { RecursiveCharacterTextSplitter } from "langchain/text_splitter" import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama" -import { createChatWithWebsiteChain, groupMessagesByConversation } from "~chain/chat-with-website" +import { + createChatWithWebsiteChain, + groupMessagesByConversation +} from "~chain/chat-with-website" import { MemoryVectorStore } from "langchain/vectorstores/memory" - +import { chromeRunTime } from "~libs/runtime" export type BotResponse = { bot: { text: string @@ -134,11 +137,11 @@ export const useMessage = () => { url }) const docs = await loader.load() - const chunkSize = await defaultEmbeddingChunkSize(); - const chunkOverlap = await defaultEmbeddingChunkOverlap(); + const chunkSize = await defaultEmbeddingChunkSize() + const chunkOverlap = await defaultEmbeddingChunkOverlap() const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize, - chunkOverlap, + chunkOverlap }) const chunks = await textSplitter.splitDocuments(docs) @@ -158,64 +161,65 @@ export const useMessage = () => { } const chatWithWebsiteMode = async (message: string) => { - const ollamaUrl = await getOllamaURL() - const { html, url } = await getHtmlOfCurrentTab() - const isAlreadyExistEmbedding = keepTrackOfEmbedding[url] - let newMessage: Message[] = [ - ...messages, - { - isBot: false, - name: "You", - message, - sources: [] - }, - { - isBot: true, - name: selectedModel, - message: "▋", - sources: [] - } - ] - - const appendingIndex = newMessage.length - 1 - setMessages(newMessage) - const embeddingModle = await defaultEmbeddingModelForRag() - const ollamaEmbedding = new OllamaEmbeddings({ - model: embeddingModle || selectedModel, - baseUrl: cleanUrl(ollamaUrl) - }) - - - const ollamaChat = new ChatOllama({ - model: selectedModel, - baseUrl: cleanUrl(ollamaUrl) - }) - - let vectorstore: MemoryVectorStore - - if (isAlreadyExistEmbedding) { - vectorstore = isAlreadyExistEmbedding - } else { - vectorstore = await memoryEmbedding(url, html, ollamaEmbedding) - } - - const { ragPrompt: systemPrompt, ragQuestionPrompt: questionPrompt } = - await promptForRag() - - const sanitizedQuestion = message.trim().replaceAll("\n", " ") - - const chain = createChatWithWebsiteChain({ - llm: ollamaChat, - question_llm: ollamaChat, - question_template: questionPrompt, - response_template: systemPrompt, - retriever: vectorstore.asRetriever() - }) - try { + let isAlreadyExistEmbedding: MemoryVectorStore + const { html, url } = await getHtmlOfCurrentTab() + isAlreadyExistEmbedding = keepTrackOfEmbedding[url] + let newMessage: Message[] = [ + ...messages, + { + isBot: false, + name: "You", + message, + sources: [] + }, + { + isBot: true, + name: selectedModel, + message: "▋", + sources: [] + } + ] + + const appendingIndex = newMessage.length - 1 + setMessages(newMessage) + const ollamaUrl = await getOllamaURL() + const embeddingModle = await defaultEmbeddingModelForRag() + + const ollamaEmbedding = new OllamaEmbeddings({ + model: embeddingModle || selectedModel, + baseUrl: cleanUrl(ollamaUrl) + }) + + const ollamaChat = new ChatOllama({ + model: selectedModel, + baseUrl: cleanUrl(ollamaUrl) + }) + + let vectorstore: MemoryVectorStore + + if (isAlreadyExistEmbedding) { + vectorstore = isAlreadyExistEmbedding + } else { + vectorstore = await memoryEmbedding(url, html, ollamaEmbedding) + } + + const { ragPrompt: systemPrompt, ragQuestionPrompt: questionPrompt } = + await promptForRag() + + const sanitizedQuestion = message.trim().replaceAll("\n", " ") + + const chain = createChatWithWebsiteChain({ + llm: ollamaChat, + question_llm: ollamaChat, + question_template: questionPrompt, + response_template: systemPrompt, + retriever: vectorstore.asRetriever() + }) + const chunks = await chain.stream({ question: sanitizedQuestion, - chat_history: groupMessagesByConversation(history), + chat_history: groupMessagesByConversation(history) }) let count = 0 for await (const chunk of chunks) { @@ -258,7 +262,8 @@ export const useMessage = () => { { isBot: true, name: selectedModel, - message: `Something went wrong. Check out the following logs: + message: `Error in chat with website mode. Check out the following logs: + ~~~ ${e?.message} ~~~ diff --git a/src/libs/get-html.ts b/src/libs/get-html.ts index 91f4b95..6b92384 100644 --- a/src/libs/get-html.ts +++ b/src/libs/get-html.ts @@ -1,3 +1,4 @@ + const _getHtml = () => { const url = window.location.href const html = Array.from(document.querySelectorAll("script")).reduce( @@ -29,3 +30,4 @@ export const getHtmlOfCurrentTab = async () => { return result } + diff --git a/src/loader/html.ts b/src/loader/html.ts index 94206f4..6cac46a 100644 --- a/src/loader/html.ts +++ b/src/loader/html.ts @@ -2,6 +2,20 @@ import { BaseDocumentLoader } from "langchain/document_loaders/base" import { Document } from "@langchain/core/documents" import { compile } from "html-to-text" import { chromeRunTime } from "~libs/runtime" +import { YtTranscript } from "yt-transcript" + +const YT_REGEX = + /(?:https?:\/\/)?(?:www\.)?(?:youtube\.com|youtu\.be)\/(?:watch\?v=)?([a-zA-Z0-9_-]+)/ + +const isYoutubeLink = (url: string) => { + return YT_REGEX.test(url) +} + +const getTranscript = async (url: string) => { + const ytTranscript = new YtTranscript({ url }) + return await ytTranscript.getTranscript() +} + export interface WebLoaderParams { html: string @@ -21,6 +35,29 @@ export class PageAssistHtmlLoader } async load(): Promise>[]> { + if (isYoutubeLink(this.url)) { + const transcript = await getTranscript(this.url) + if (!transcript) { + throw new Error("Transcript not found for this video.") + } + + let text = "" + + transcript.forEach((item) => { + text += item.text + " " + }) + + + return [ + { + metadata: { + source: this.url, + audio: { chunks: transcript } + }, + pageContent: text + } + ] + } const htmlCompiler = compile({ wordwrap: false }) @@ -30,6 +67,29 @@ export class PageAssistHtmlLoader } async loadByURL(): Promise>[]> { + if (isYoutubeLink(this.url)) { + const transcript = await getTranscript(this.url) + if (!transcript) { + throw new Error("Transcript not found for this video.") + } + + let text = "" + + transcript.forEach((item) => { + text += item.text + " " + }) + + + return [ + { + metadata: { + source: this.url, + audio: { chunks: transcript } + }, + pageContent: text + } + ] + } await chromeRunTime(this.url) const fetchHTML = await fetch(this.url) const html = await fetchHTML.text() diff --git a/yarn.lock b/yarn.lock index ca7ad13..9d97196 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4967,10 +4967,10 @@ lru-cache@^6.0.0: resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.2.0.tgz#0bd445ca57363465900f4d1f9bd8db343a4d95c3" integrity sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q== -lucide-react@^0.340.0: - version "0.340.0" - resolved "https://registry.yarnpkg.com/lucide-react/-/lucide-react-0.340.0.tgz#67a6fac6a5e257f2036dffae0dd94d6ccb28ce8e" - integrity sha512-mWzYhbyy2d+qKuKHh+GWElPwa+kIquTnKbmSLGWOuZy+bjfZCkYD8DQWVFlqI4mQwc4HNxcqcOvtQ7ZS2PwURg== +lucide-react@^0.350.0: + version "0.350.0" + resolved "https://registry.yarnpkg.com/lucide-react/-/lucide-react-0.350.0.tgz#78b45342f4daff4535290e37b1ea7eb0961a3dab" + integrity sha512-5IZVKsxxG8Nn81gpsz4XLNgCAXkppCh0Y0P0GLO39h5iVD2WEaB9of6cPkLtzys1GuSfxJxmwsDh487y7LAf/g== magic-string@^0.30.0: version "0.30.6" @@ -7772,6 +7772,13 @@ ws@^8.11.0: resolved "https://registry.yarnpkg.com/ws/-/ws-8.16.0.tgz#d1cd774f36fbc07165066a60e40323eab6446fd4" integrity sha512-HS0c//TP7Ina87TfiPUz1rQzMhHrl/SG2guqRcTOIUYD2q8uhUdNHZYJUaQ8aTGPzCh+c6oawMKW35nFl1dxyQ== +xml-js@^1.6.11: + version "1.6.11" + resolved "https://registry.yarnpkg.com/xml-js/-/xml-js-1.6.11.tgz#927d2f6947f7f1c19a316dd8eea3614e8b18f8e9" + integrity sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g== + dependencies: + sax "^1.2.4" + xml-name-validator@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835" @@ -7817,6 +7824,14 @@ yaml@^2.2.1, yaml@^2.3.4: resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.4.tgz#53fc1d514be80aabf386dc6001eb29bf3b7523b2" integrity sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA== +yt-transcript@^0.0.2: + version "0.0.2" + resolved "https://registry.yarnpkg.com/yt-transcript/-/yt-transcript-0.0.2.tgz#1c54aede89bb8a03bbca3ba58520dbbd9c828571" + integrity sha512-+cNRqW6tSQNDkQDVrWNT6hc6X2TnaQLvUJIepzn9r7XdEvPtUDkfsyhptW5+j0EPIEpnlsKyA/epCUrE4QKn2g== + dependencies: + axios "^1.6.7" + xml-js "^1.6.11" + zod-to-json-schema@^3.22.3: version "3.22.4" resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.22.4.tgz#f8cc691f6043e9084375e85fb1f76ebafe253d70"