From a3810cd534543c34f9ea01e9b28f853415f1342e Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Thu, 11 Apr 2024 00:08:20 +0530 Subject: [PATCH] Update import statement in local-duckduckgo.ts, prompt.tsx, wxt.config.ts, webui.tsx, PlaygroundChat.tsx, other.tsx, Markdown.tsx, and useMessageOption.tsx --- src/components/Common/Markdown.tsx | 2 - src/components/Common/Playground/Message.tsx | 48 ++++++-- .../Option/Playground/PlaygroundChat.tsx | 4 +- src/components/Option/Settings/other.tsx | 2 + src/components/Option/Settings/prompt.tsx | 2 - .../Option/Settings/search-mode.tsx | 59 +++++---- src/components/Option/Settings/tts-mode.tsx | 116 ++++++++++++++++++ src/hooks/useMessageOption.tsx | 18 ++- src/hooks/useTTS.tsx | 54 ++++++++ src/services/tts.ts | 91 ++++++++++++++ src/store/webui.tsx | 8 +- src/utils/markdown-to-ssml.ts | 20 +++ src/web/local-duckduckgo.ts | 2 +- wxt.config.ts | 5 +- 14 files changed, 385 insertions(+), 46 deletions(-) create mode 100644 src/components/Option/Settings/tts-mode.tsx create mode 100644 src/hooks/useTTS.tsx create mode 100644 src/services/tts.ts create mode 100644 src/utils/markdown-to-ssml.ts diff --git a/src/components/Common/Markdown.tsx b/src/components/Common/Markdown.tsx index c4090e2..9d1bf13 100644 --- a/src/components/Common/Markdown.tsx +++ b/src/components/Common/Markdown.tsx @@ -1,7 +1,6 @@ import { Prism as SyntaxHighlighter } from "react-syntax-highlighter" import remarkGfm from "remark-gfm" import { nightOwl } from "react-syntax-highlighter/dist/cjs/styles/prism" -import rehypeMathjax from "rehype-mathjax" import remarkMath from "remark-math" import ReactMarkdown from "react-markdown" import "property-information" @@ -19,7 +18,6 @@ export default function Markdown({ message }: { message: string }) { void + isTTSEnabled?: boolean } export const PlaygroundMessage = (props: Props) => { @@ -32,6 +41,7 @@ export const PlaygroundMessage = (props: Props) => { const [editMode, setEditMode] = React.useState(false) const { t } = useTranslation("common") + const { cancel, isSpeaking, speak } = useTTS() return (
@@ -98,8 +108,10 @@ export const PlaygroundMessage = (props: Props) => {
{props?.sources?.map((source, index) => ( + onSourceClick={props.onSourceClick} + key={index} + source={source} + /> ))}
)} @@ -110,11 +122,31 @@ export const PlaygroundMessage = (props: Props) => { ? "hidden group-hover:flex" : "flex" }`}> + {props.isTTSEnabled && ( + + + + )} {props.isBot && ( <> {!props.hideCopy && ( - +
+

diff --git a/src/components/Option/Settings/prompt.tsx b/src/components/Option/Settings/prompt.tsx index 499011c..fc7efee 100644 --- a/src/components/Option/Settings/prompt.tsx +++ b/src/components/Option/Settings/prompt.tsx @@ -5,8 +5,6 @@ import { useTranslation } from "react-i18next" import { SaveButton } from "~/components/Common/SaveButton" import { getWebSearchPrompt, - setSystemPromptForNonRagOption, - systemPromptForNonRagOption, geWebSearchFollowUpPrompt, setWebPrompts, promptForRag, diff --git a/src/components/Option/Settings/search-mode.tsx b/src/components/Option/Settings/search-mode.tsx index ff17714..cf09183 100644 --- a/src/components/Option/Settings/search-mode.tsx +++ b/src/components/Option/Settings/search-mode.tsx @@ -44,43 +44,50 @@ export const SearchModeSettings = () => { await setSearchSettings(values) })} className="space-y-4"> -
+
{t("generalSettings.webSearch.provider.label")} - + option!.label.toLowerCase().indexOf(input.toLowerCase()) >= 0 || + option!.value.toLowerCase().indexOf(input.toLowerCase()) >= 0 + } + {...form.getInputProps("searchProvider")} + /> +
-
+
{t("generalSettings.webSearch.searchMode.label")} - +
+ +
-
+
{t("generalSettings.webSearch.totalSearchResults.label")} - +
+ +
diff --git a/src/components/Option/Settings/tts-mode.tsx b/src/components/Option/Settings/tts-mode.tsx new file mode 100644 index 0000000..8630ae4 --- /dev/null +++ b/src/components/Option/Settings/tts-mode.tsx @@ -0,0 +1,116 @@ +import { SaveButton } from "@/components/Common/SaveButton" +import { getSearchSettings, setSearchSettings } from "@/services/search" +import { getTTSSettings, setTTSSettings } from "@/services/tts" +import { SUPPORTED_SERACH_PROVIDERS } from "@/utils/search-provider" +import { useForm } from "@mantine/form" +import { useQuery, useQueryClient } from "@tanstack/react-query" +import { Select, Skeleton, Switch, InputNumber } from "antd" +import { useTranslation } from "react-i18next" + +export const TTSModeSettings = ({ hideTitle }: { hideTitle?: boolean }) => { + const { t } = useTranslation("settings") + const queryClient = useQueryClient() + + const form = useForm({ + initialValues: { + ttsEnabled: false, + ttsProvider: "", + voice: "", + ssmlEnabled: false + } + }) + + const { status, data } = useQuery({ + queryKey: ["fetchTTSSettings"], + queryFn: async () => { + const data = await getTTSSettings() + form.setValues(data) + return data + } + }) + + if (status === "pending" || status === "error") { + return + } + + return ( +
+ {!hideTitle && ( +
+

+ {t("generalSettings.tts.heading")} +

+
+
+ )} +
{ + await setTTSSettings(values) + })} + className="space-y-4"> +
+ + {t("generalSettings.tts.ttsEnabled.label")} + +
+ +
+
+
+ + {t("generalSettings.tts.ttsProvider.label")} + +
+ + ({ + label: `${voice.voiceName} - ${voice.lang}`.trim(), + value: voice.voiceName + }) || [] + )} + {...form.getInputProps("voice")} + /> +
+
+
+ + {t("generalSettings.tts.ssmlEnabled.label")} + +
+ +
+
+ +
+ +
+
+
+ ) +} diff --git a/src/hooks/useMessageOption.tsx b/src/hooks/useMessageOption.tsx index 38a5308..4320ea9 100644 --- a/src/hooks/useMessageOption.tsx +++ b/src/hooks/useMessageOption.tsx @@ -28,6 +28,8 @@ import { usePageAssist } from "@/context" import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama" import { PageAssistVectorStore } from "@/libs/PageAssistVectorStore" import { formatDocs } from "@/chain/chat-with-x" +import { useWebUI } from "@/store/webui" +import { isTTSEnabled } from "@/services/tts" export const useMessageOption = () => { const { @@ -66,11 +68,22 @@ export const useMessageOption = () => { setSelectedKnowledge } = useStoreMessageOption() + const { ttsEnabled, setTTSEnabled } = useWebUI() + const { t } = useTranslation("option") const navigate = useNavigate() const textareaRef = React.useRef(null) + React.useEffect(() => { + const checkTTSEnabled = async () => { + const tts = await isTTSEnabled() + setTTSEnabled(tts) + } + + checkTTSEnabled() + }, []) + const clearChat = () => { navigate("/") setMessages([]) @@ -835,7 +848,7 @@ export const useMessageOption = () => { } const currentHumanMessage = newMessages[index] - + newMessages[index].message = message const previousMessages = newMessages.slice(0, index + 1) setMessages(previousMessages) const previousHistory = newHistory.slice(0, index) @@ -893,6 +906,7 @@ export const useMessageOption = () => { setSelectedSystemPrompt, textareaRef, selectedKnowledge, - setSelectedKnowledge + setSelectedKnowledge, + ttsEnabled } } diff --git a/src/hooks/useTTS.tsx b/src/hooks/useTTS.tsx new file mode 100644 index 0000000..075f005 --- /dev/null +++ b/src/hooks/useTTS.tsx @@ -0,0 +1,54 @@ +import { useEffect, useState } from "react" +import { notification } from "antd" +import { getVoice, isSSMLEnabled } from "@/services/tts" +import { markdownToSSML } from "@/utils/markdown-to-ssml" + +type VoiceOptions = { + utterance: string +} + +export const useTTS = () => { + const [isSpeaking, setIsSpeaking] = useState(false) + + const speak = async ({ utterance }: VoiceOptions) => { + try { + const voice = await getVoice() + const isSSML = await isSSMLEnabled() + if (isSSML) { + utterance = markdownToSSML(utterance) + } + chrome.tts.speak(utterance, { + voiceName: voice, + onEvent(event) { + if (event.type === "start") { + setIsSpeaking(true) + } else if (event.type === "end") { + setIsSpeaking(false) + } + } + }) + } catch (error) { + notification.error({ + message: "Error", + description: "Something went wrong while trying to play the audio" + }) + } + } + + const cancel = () => { + chrome.tts.stop() + setIsSpeaking(false) + } + + useEffect(() => { + return () => { + cancel() + } + }, []) + + return { + speak, + cancel, + isSpeaking + } +} diff --git a/src/services/tts.ts b/src/services/tts.ts new file mode 100644 index 0000000..fbfe4b7 --- /dev/null +++ b/src/services/tts.ts @@ -0,0 +1,91 @@ +import { Storage } from "@plasmohq/storage" + +const storage = new Storage() + +const DEFAULT_TTS_PROVIDER = "browser" + +const AVAILABLE_TTS_PROVIDERS = ["browser"] as const + +export const getTTSProvider = async (): Promise< + (typeof AVAILABLE_TTS_PROVIDERS)[number] +> => { + const ttsProvider = await storage.get("ttsProvider") + if (!ttsProvider || ttsProvider.length === 0) { + return DEFAULT_TTS_PROVIDER + } + return ttsProvider as (typeof AVAILABLE_TTS_PROVIDERS)[number] +} + +export const setTTSProvider = async (ttsProvider: string) => { + await storage.set("ttsProvider", ttsProvider) +} + +export const getBrowserTTSVoices = async () => { + const tts = await chrome.tts.getVoices() + return tts +} + +export const getVoice = async () => { + const voice = await storage.get("voice") + return voice +} + +export const setVoice = async (voice: string) => { + await storage.set("voice", voice) +} + +export const isTTSEnabled = async () => { + const data = await storage.get("isTTSEnabled") + return data === "true" +} + +export const setTTSEnabled = async (isTTSEnabled: boolean) => { + await storage.set("isTTSEnabled", isTTSEnabled.toString()) +} + +export const isSSMLEnabled = async () => { + const data = await storage.get("isSSMLEnabled") + return data === "true" +} + +export const setSSMLEnabled = async (isSSMLEnabled: boolean) => { + await storage.set("isSSMLEnabled", isSSMLEnabled.toString()) +} + +export const getTTSSettings = async () => { + const [ttsEnabled, ttsProvider, browserTTSVoices, voice, ssmlEnabled] = + await Promise.all([ + isTTSEnabled(), + getTTSProvider(), + getBrowserTTSVoices(), + getVoice(), + isSSMLEnabled() + ]) + + return { + ttsEnabled, + ttsProvider, + browserTTSVoices, + voice, + ssmlEnabled + } +} + +export const setTTSSettings = async ({ + ttsEnabled, + ttsProvider, + voice, + ssmlEnabled +}: { + ttsEnabled: boolean + ttsProvider: string + voice: string + ssmlEnabled: boolean +}) => { + await Promise.all([ + setTTSEnabled(ttsEnabled), + setTTSProvider(ttsProvider), + setVoice(voice), + setSSMLEnabled(ssmlEnabled) + ]) +} diff --git a/src/store/webui.tsx b/src/store/webui.tsx index ad11be6..59fefb7 100644 --- a/src/store/webui.tsx +++ b/src/store/webui.tsx @@ -3,9 +3,15 @@ import { create } from "zustand" type State = { sendWhenEnter: boolean setSendWhenEnter: (sendWhenEnter: boolean) => void + + ttsEnabled: boolean + setTTSEnabled: (isTTSEnabled: boolean) => void } export const useWebUI = create((set) => ({ sendWhenEnter: true, - setSendWhenEnter: (sendWhenEnter) => set({ sendWhenEnter }) + setSendWhenEnter: (sendWhenEnter) => set({ sendWhenEnter }), + + ttsEnabled: false, + setTTSEnabled: (ttsEnabled) => set({ ttsEnabled }) })) diff --git a/src/utils/markdown-to-ssml.ts b/src/utils/markdown-to-ssml.ts new file mode 100644 index 0000000..e620747 --- /dev/null +++ b/src/utils/markdown-to-ssml.ts @@ -0,0 +1,20 @@ +export function markdownToSSML(markdown: string): string { + let ssml = markdown.replace(/\\n/g, "") + + ssml = ssml.replace( + /^(#{1,6}) (.*?)(?=\r?\n\s*?(?:\r?\n|$))/gm, + (match, hashes, heading) => { + const level = hashes.length + const rate = (level - 1) * 10 + 100 + return `${heading}` + } + ) + + ssml = ssml.replace(/\\\*\\\*(.\*?)\\\*\\\*/g, "$1") + ssml = ssml.replace( + /\\\*(.\*?)\\\*/g, + '$1' + ) + ssml = `${ssml}` + return `${ssml}` +} diff --git a/src/web/local-duckduckgo.ts b/src/web/local-duckduckgo.ts index 51045d7..f43e929 100644 --- a/src/web/local-duckduckgo.ts +++ b/src/web/local-duckduckgo.ts @@ -93,7 +93,7 @@ export const webDuckDuckGoSearch = async (query: string) => { const textSplitter = new RecursiveCharacterTextSplitter({ chunkSize, chunkOverlap - }) + }) const chunks = await textSplitter.splitDocuments(docs) diff --git a/wxt.config.ts b/wxt.config.ts index 954dfb9..e286089 100644 --- a/wxt.config.ts +++ b/wxt.config.ts @@ -24,7 +24,7 @@ export default defineConfig({ srcDir: "src", outDir: "build", manifest: { - version: "1.1.2", + version: "1.1.3", name: '__MSG_extName__', description: '__MSG_extDescription__', default_locale: 'en', @@ -52,7 +52,8 @@ export default defineConfig({ "declarativeNetRequest", "action", "unlimitedStorage", - "contextMenus" + "contextMenus", + "tts" ] } })