From 9eaa0c9d664b34af244b8a0f586854e8bbbb3d38 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sun, 14 Apr 2024 18:16:47 +0530 Subject: [PATCH] Update localization messages for Chinese, English, and Japanese languages --- src/assets/locale/en/sidepanel.json | 4 +- src/components/Option/Settings/tts-mode.tsx | 22 +++-- src/components/Sidepanel/Chat/body.tsx | 3 + src/components/Sidepanel/Chat/empty.tsx | 5 +- src/components/Sidepanel/Chat/header.tsx | 27 ++++--- src/components/Sidepanel/Settings/body.tsx | 4 + src/entries/background.ts | 8 +- src/loader/html.ts | 35 ++++++-- src/parser/twitter.ts | 90 +++++++++++++++++++++ src/parser/wiki.ts | 28 +++++++ 10 files changed, 194 insertions(+), 32 deletions(-) create mode 100644 src/parser/twitter.ts create mode 100644 src/parser/wiki.ts diff --git a/src/assets/locale/en/sidepanel.json b/src/assets/locale/en/sidepanel.json index d46e213..4e24fdb 100644 --- a/src/assets/locale/en/sidepanel.json +++ b/src/assets/locale/en/sidepanel.json @@ -1,5 +1,7 @@ { "tooltip": { - "embed": "It may take a few minutes to embed the page. Please wait..." + "embed": "It may take a few minutes to embed the page. Please wait...", + "clear": "Erase chat history", + "history": "Chat history" } } \ No newline at end of file diff --git a/src/components/Option/Settings/tts-mode.tsx b/src/components/Option/Settings/tts-mode.tsx index 7a316de..bf5ad36 100644 --- a/src/components/Option/Settings/tts-mode.tsx +++ b/src/components/Option/Settings/tts-mode.tsx @@ -1,14 +1,12 @@ import { SaveButton } from "@/components/Common/SaveButton" -import { getSearchSettings, setSearchSettings } from "@/services/search" import { getTTSSettings, setTTSSettings } from "@/services/tts" import { useWebUI } from "@/store/webui" -import { SUPPORTED_SERACH_PROVIDERS } from "@/utils/search-provider" import { useForm } from "@mantine/form" -import { useQuery, useQueryClient } from "@tanstack/react-query" -import { Select, Skeleton, Switch, InputNumber } from "antd" +import { useQuery } from "@tanstack/react-query" +import { Select, Skeleton, Switch } from "antd" import { useTranslation } from "react-i18next" -export const TTSModeSettings = ({ hideTitle }: { hideTitle?: boolean }) => { +export const TTSModeSettings = ({ hideBorder }: { hideBorder?: boolean }) => { const { t } = useTranslation("settings") const { setTTSEnabled } = useWebUI() @@ -36,14 +34,14 @@ export const TTSModeSettings = ({ hideTitle }: { hideTitle?: boolean }) => { return (
- {!hideTitle && ( -
-

- {t("generalSettings.tts.heading")} -

+
+

+ {t("generalSettings.tts.heading")} +

+ {!hideBorder && (
-
- )} + )} +
{ await setTTSSettings(values) diff --git a/src/components/Sidepanel/Chat/body.tsx b/src/components/Sidepanel/Chat/body.tsx index 1b62bd8..fc4bedb 100644 --- a/src/components/Sidepanel/Chat/body.tsx +++ b/src/components/Sidepanel/Chat/body.tsx @@ -2,10 +2,12 @@ import React from "react" import { PlaygroundMessage } from "~/components/Common/Playground/Message" import { useMessage } from "~/hooks/useMessage" import { EmptySidePanel } from "../Chat/empty" +import { useWebUI } from "@/store/webui" export const SidePanelBody = () => { const { messages, streaming } = useMessage() const divRef = React.useRef(null) + const {ttsEnabled} = useWebUI() React.useEffect(() => { if (divRef.current) { divRef.current.scrollIntoView({ behavior: "smooth" }) @@ -27,6 +29,7 @@ export const SidePanelBody = () => { onRengerate={() => {}} isProcessing={streaming} hideEditAndRegenerate + isTTSEnabled={ttsEnabled} /> ))}
diff --git a/src/components/Sidepanel/Chat/empty.tsx b/src/components/Sidepanel/Chat/empty.tsx index d6c168c..99f9d43 100644 --- a/src/components/Sidepanel/Chat/empty.tsx +++ b/src/components/Sidepanel/Chat/empty.tsx @@ -8,7 +8,8 @@ import { getAllModels, getOllamaURL, isOllamaRunning, - setOllamaURL as saveOllamaURL + setOllamaURL as saveOllamaURL, + fetchChatModels } from "~/services/ollama" export const EmptySidePanel = () => { @@ -24,7 +25,7 @@ export const EmptySidePanel = () => { queryFn: async () => { const ollamaURL = await getOllamaURL() const isOk = await isOllamaRunning() - const models = await getAllModels({ returnEmpty: false }) + const models = await fetchChatModels({ returnEmpty: false }) return { isOk, diff --git a/src/components/Sidepanel/Chat/header.tsx b/src/components/Sidepanel/Chat/header.tsx index 69ae12f..0d52747 100644 --- a/src/components/Sidepanel/Chat/header.tsx +++ b/src/components/Sidepanel/Chat/header.tsx @@ -2,10 +2,10 @@ import logoImage from "~/assets/icon.png" import { useMessage } from "~/hooks/useMessage" import { Link } from "react-router-dom" import { Tooltip } from "antd" -import { BoxesIcon, CogIcon, RefreshCcw } from "lucide-react" +import { BoxesIcon, CogIcon, EraserIcon, HistoryIcon } from "lucide-react" import { useTranslation } from "react-i18next" export const SidepanelHeader = () => { - const { clearChat, isEmbedding } = useMessage() + const { clearChat, isEmbedding, messages } = useMessage() const { t } = useTranslation(["sidepanel", "common"]) return ( @@ -25,13 +25,22 @@ export const SidepanelHeader = () => { ) : null} - + {messages.length > 0 && ( + + + + )} + {/* + + + + */} diff --git a/src/components/Sidepanel/Settings/body.tsx b/src/components/Sidepanel/Settings/body.tsx index 77cfb97..db695b2 100644 --- a/src/components/Sidepanel/Settings/body.tsx +++ b/src/components/Sidepanel/Settings/body.tsx @@ -22,6 +22,7 @@ import { useMessage } from "~/hooks/useMessage" import { MoonIcon, SunIcon } from "lucide-react" import { useTranslation } from "react-i18next" import { useI18n } from "@/hooks/useI18n" +import { TTSModeSettings } from "@/components/Option/Settings/tts-mode" export const SettingsBody = () => { const { t } = useTranslation("settings") @@ -285,6 +286,9 @@ export const SettingsBody = () => {
+
+ +

{t("generalSettings.settings.language.label")}{" "} diff --git a/src/entries/background.ts b/src/entries/background.ts index 85fb678..f671467 100644 --- a/src/entries/background.ts +++ b/src/entries/background.ts @@ -1,4 +1,6 @@ import { getOllamaURL, isOllamaRunning } from "../services/ollama" +import { Storage } from "@plasmohq/storage" + const progressHuman = (completed: number, total: number) => { return ((completed / total) * 100).toFixed(0) + "%" } @@ -75,6 +77,8 @@ const streamDownload = async (url: string, model: string) => { } export default defineBackground({ main() { + const storage = new Storage() + chrome.runtime.onMessage.addListener(async (message) => { if (message.type === "sidepanel") { chrome.tabs.query( @@ -139,8 +143,8 @@ export default defineBackground({ { active: true, currentWindow: true }, async (tabs) => { const tab = tabs[0] - await chrome.sidePanel.open({ - windowId: tab.windowId! + chrome.sidePanel.open({ + tabId: tab.id! }) } ) diff --git a/src/loader/html.ts b/src/loader/html.ts index 367b02a..de567f0 100644 --- a/src/loader/html.ts +++ b/src/loader/html.ts @@ -3,6 +3,7 @@ import { Document } from "@langchain/core/documents" import { compile } from "html-to-text" import { chromeRunTime } from "~/libs/runtime" import { YtTranscript } from "yt-transcript" +import { isWikipedia, parseWikipedia } from "@/parser/wiki" const YT_REGEX = /(?:https?:\/\/)?(?:www\.)?(?:youtube\.com|youtu\.be)\/(?:watch\?v=)?([a-zA-Z0-9_-]+)/ @@ -16,7 +17,6 @@ const getTranscript = async (url: string) => { return await ytTranscript.getTranscript() } - export interface WebLoaderParams { html: string url: string @@ -24,7 +24,8 @@ export interface WebLoaderParams { export class PageAssistHtmlLoader extends BaseDocumentLoader - implements WebLoaderParams { + implements WebLoaderParams +{ html: string url: string @@ -47,7 +48,6 @@ export class PageAssistHtmlLoader text += item.text + " " }) - return [ { metadata: { @@ -58,10 +58,23 @@ export class PageAssistHtmlLoader } ] } + + let html = this.html + + if (isWikipedia(this.url)) { + console.log("Wikipedia URL detected") + html = parseWikipedia(html) + } + + // else if (isTwitter(this.url)) { + // console.log("Twitter URL detected") + // html = parseTweet(html, this.url) + // } + const htmlCompiler = compile({ wordwrap: false }) - const text = htmlCompiler(this.html) + const text = htmlCompiler(html) const metadata = { source: this.url } return [new Document({ pageContent: text, metadata })] } @@ -79,7 +92,6 @@ export class PageAssistHtmlLoader text += item.text + " " }) - return [ { metadata: { @@ -92,7 +104,18 @@ export class PageAssistHtmlLoader } await chromeRunTime(this.url) const fetchHTML = await fetch(this.url) - const html = await fetchHTML.text() + let html = await fetchHTML.text() + + if (isWikipedia(this.url)) { + console.log("Wikipedia URL detected") + html = parseWikipedia(await fetchHTML.text()) + } + + // else if (isTwitter(this.url)) { + // console.log("Twitter URL detected") + // html = parseTweet(await fetchHTML.text(), this.url) + // } + const htmlCompiler = compile({ wordwrap: false, selectors: [ diff --git a/src/parser/twitter.ts b/src/parser/twitter.ts new file mode 100644 index 0000000..a998945 --- /dev/null +++ b/src/parser/twitter.ts @@ -0,0 +1,90 @@ +import * as cheerio from "cheerio" + +export const isTweet = (url: string) => { + const TWEET_REGEX = /twitter\.com\/[a-zA-Z0-9_]+\/status\/[0-9]+/g + return TWEET_REGEX.test(url) +} + +export const isTwitterProfile = (url: string) => { + const PROFILE_REGEX = /twitter\.com\/[a-zA-Z0-9_]+/g + return PROFILE_REGEX.test(url) +} + +export const isTwitterTimeline = (url: string) => { + const TIMELINE_REGEX = /twitter\.com\/home/g + return TIMELINE_REGEX.test(url) +} + +export const isTwitter = (url: string) => { + return isTweet(url) || isTwitterProfile(url) || isTwitterTimeline(url) +} + +export const isTwitterNotification = (url: string) => { + const NOTIFICATION_REGEX = /twitter\.com\/notifications/g + return NOTIFICATION_REGEX.test(url) +} + +export const parseTweet = (html: string, url: string) => { + if (!html) { + return "" + } + + const $ = cheerio.load(html) + + if (isTweet(url)) { + console.log("tweet") + const tweet = $("div[data-testid='tweet']") + const tweetContent = tweet.find("div[lang]") + const tweetMedia = tweet.find("div[role='group']") + const author = tweet.find("a[role='link']").text() + const date = tweet.find("time").text() + return `
${author} ${tweetContent.text()} ${tweetMedia.html()} ${date}
` + } + + if (isTwitterTimeline(url)) { + console.log("timeline") + const timeline = $("div[data-testid='primaryColumn']") + const timelineContent = timeline.find("div[data-testid='tweet']") + console.log(timelineContent.html()) + const tweet = timelineContent + .map((i, el) => { + const author = $(el).find("a[role='link']").text() + const content = $(el).find("div[lang]").text() + const media = $(el).find("div[role='group']").html() + const date = $(el).find("time").text() + return `
${author} ${content} ${media} ${date}
` + }) + .get() + .join("") + console.log(tweet) + return `
${tweet}
` + } + + if (isTwitterNotification(url)) { + console.log("notification") + const notification = $("div[data-testid='primaryColumn']") + const notificationContent = notification.find("div[data-testid='tweet']") + return `
${notificationContent.html()}
` + } + if (isTwitterProfile(url)) { + console.log("profile") + const profile = $("div[data-testid='primaryColumn']") + const profileContent = profile.find( + "div[data-testid='UserProfileHeader_Items']" + ) + const profileTweets = profile.find("div[data-testid='tweet']") + return `
${profileContent.html()}
${profileTweets.html()}
` + } + console.log("no match") + const timeline = $("div[data-testid='primaryColumn']") + const timelineContent = timeline.find("div[data-testid='tweet']") + const tweet = timelineContent.map((i, el) => { + const author = $(el).find("a[role='link']").text() + const content = $(el).find("div[lang]").text() + const media = $(el).find("div[role='group']").html() + const date = $(el).find("time").text() + return `
${author} ${content} ${media} ${date}
` + }) + + return `
${tweet}
` +} diff --git a/src/parser/wiki.ts b/src/parser/wiki.ts new file mode 100644 index 0000000..36f567c --- /dev/null +++ b/src/parser/wiki.ts @@ -0,0 +1,28 @@ +import * as cheerio from "cheerio" + +export const isWikipedia = (url: string) => { + const WIKI_REGEX = /wikipedia\.org\/wiki\//g + return WIKI_REGEX.test(url) +} + +export const parseWikipedia = (html: string) => { + if (!html) { + return "" + } + const $ = cheerio.load(html) + const title = $("h1#firstHeading") + const content = $("#mw-content-text") + content?.find("sup.reference")?.remove() + content?.find("div.thumb")?.remove() + content?.find("div.reflist")?.remove() + content?.find("div.navbox")?.remove() + content?.find("table.infobox")?.remove() + content?.find("div.sister-wikipedia")?.remove() + content?.find("div.sister-projects")?.remove() + content?.find("div.metadata")?.remove() + content?.find("div.vertical-navbox")?.remove() + content?.find("div.toc")?.remove() + const newHtml = content?.html() + + return `
TITLE: ${title?.text()}
${newHtml}
` +}