feat(vision): add vision chat mode

- Add new "vision" chat mode to the application
- Implement the `visionChatMode` function to handle vision-based chat interactions
- Update the UI to include a new button to toggle the vision chat mode
- Add new translations for the "vision" chat mode tooltip
- Disable certain UI elements when the vision chat mode is active
This commit is contained in:
n4ze3m
2024-11-23 14:04:57 +05:30
parent edc5380a76
commit 2c12b17dda
5 changed files with 368 additions and 47 deletions

View File

@@ -7,7 +7,14 @@ import { toBase64 } from "~/libs/to-base64"
import { Checkbox, Dropdown, Image, Switch, Tooltip } from "antd"
import { useWebUI } from "~/store/webui"
import { defaultEmbeddingModelForRag } from "~/services/ollama"
import { ImageIcon, MicIcon, StopCircleIcon, X } from "lucide-react"
import {
ImageIcon,
MicIcon,
StopCircleIcon,
X,
EyeIcon,
EyeOffIcon
} from "lucide-react"
import { useTranslation } from "react-i18next"
import { ModelSelect } from "@/components/Common/ModelSelect"
import { useSpeechRecognition } from "@/hooks/useSpeechRecognition"
@@ -36,7 +43,7 @@ export const SidepanelForm = ({ dropedFile }: Props) => {
resetTranscript,
start: startListening,
stop: stopSpeechRecognition,
supported: browserSupportsSpeechRecognition,
supported: browserSupportsSpeechRecognition
} = useSpeechRecognition()
const stopListening = async () => {
@@ -237,7 +244,10 @@ export const SidepanelForm = ({ dropedFile }: Props) => {
}
}
await stopListening()
if (value.message.trim().length === 0 && value.image.length === 0) {
if (
value.message.trim().length === 0 &&
value.image.length === 0
) {
return
}
form.reset()
@@ -281,20 +291,22 @@ export const SidepanelForm = ({ dropedFile }: Props) => {
{...form.getInputProps("message")}
/>
<div className="flex mt-4 justify-end gap-3">
<Tooltip title={t("tooltip.searchInternet")}>
<button
type="button"
onClick={() => setWebSearch(!webSearch)}
className={`inline-flex items-center gap-2 ${
chatMode === "rag" ? "hidden" : "block"
}`}>
{webSearch ? (
<PiGlobe className="h-5 w-5 dark:text-gray-300" />
) : (
<PiGlobeX className="h-5 w-5 text-gray-600 dark:text-gray-400" />
)}
</button>
</Tooltip>
{chatMode !== "vision" && (
<Tooltip title={t("tooltip.searchInternet")}>
<button
type="button"
onClick={() => setWebSearch(!webSearch)}
className={`inline-flex items-center gap-2 ${
chatMode === "rag" ? "hidden" : "block"
}`}>
{webSearch ? (
<PiGlobe className="h-5 w-5 dark:text-gray-300" />
) : (
<PiGlobeX className="h-5 w-5 text-gray-600 dark:text-gray-400" />
)}
</button>
</Tooltip>
)}
<ModelSelect />
{browserSupportsSpeechRecognition && (
<Tooltip title={t("tooltip.speechToText")}>
@@ -323,13 +335,35 @@ export const SidepanelForm = ({ dropedFile }: Props) => {
</button>
</Tooltip>
)}
<Tooltip title={t("tooltip.vision")}>
<button
type="button"
onClick={() => {
if (chatMode === "vision") {
setChatMode("normal")
} else {
setChatMode("vision")
}
}}
disabled={chatMode === "rag"}
className={`flex items-center justify-center dark:text-gray-300 ${
chatMode === "rag" ? "hidden" : "block"
} disabled:opacity-50`}>
{chatMode === "vision" ? (
<EyeIcon className="h-5 w-5" />
) : (
<EyeOffIcon className="h-5 w-5" />
)}
</button>
</Tooltip>
<Tooltip title={t("tooltip.uploadImage")}>
<button
type="button"
onClick={() => {
inputRef.current?.click()
}}
className={`flex items-center justify-center dark:text-gray-300 ${
disabled={chatMode === "vision"}
className={`flex items-center justify-center disabled:opacity-50 dark:text-gray-300 ${
chatMode === "rag" ? "hidden" : "block"
}`}>
<ImageIcon className="h-5 w-5" />