iod/page-assist
10
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Add OCR support to message formatting

Browse Source
This commit is contained in:
n4ze3m
2024-12-28 16:27:49 +05:30
parent eed1a5a0d1
commit 18bc597dc7
16 changed files with 431 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/hooks/useMessage.tsx
View File

@@ -84,7 +84,9 @@ export const useMessage = () => {
selectedQuickPrompt,
setSelectedQuickPrompt,
selectedSystemPrompt,
setSelectedSystemPrompt
setSelectedSystemPrompt,
useOCR,
setUseOCR
} = useStoreMessage()
const [speechToTextLanguage, setSpeechToTextLanguage] = useStorage(
@@ -329,7 +331,7 @@ export const useMessage = () => {
]
}
let humanMessage = humanMessageFormatter({
let humanMessage = await humanMessageFormatter({
content: [
{
text: systemPrompt
@@ -338,7 +340,8 @@ export const useMessage = () => {
type: "text"
}
],
model: selectedModel
model: selectedModel,
useOCR
})
const applicationChatHistory = generateHistory(history, selectedModel)
@@ -564,7 +567,7 @@ export const useMessage = () => {
)
}
let humanMessage = humanMessageFormatter({
let humanMessage = await humanMessageFormatter({
content: [
{
text: message,
@@ -575,7 +578,8 @@ export const useMessage = () => {
type: "image_url"
}
],
model: selectedModel
model: selectedModel,
useOCR
})
let generationInfo: any | undefined = undefined
@@ -769,17 +773,18 @@ export const useMessage = () => {
const prompt = await systemPromptForNonRag()
const selectedPrompt = await getPromptById(selectedSystemPrompt)
let humanMessage = humanMessageFormatter({
let humanMessage = await humanMessageFormatter({
content: [
{
text: message,
type: "text"
}
],
model: selectedModel
model: selectedModel,
useOCR
})
if (image.length > 0) {
humanMessage = humanMessageFormatter({
humanMessage = await humanMessageFormatter({
content: [
{
text: message,
@@ -790,7 +795,8 @@ export const useMessage = () => {
type: "image_url"
}
],
model: selectedModel
model: selectedModel,
useOCR
})
}
@@ -1048,17 +1054,18 @@ export const useMessage = () => {
// message = message.trim().replaceAll("\n", " ")
let humanMessage = humanMessageFormatter({
let humanMessage = await humanMessageFormatter({
content: [
{
text: message,
type: "text"
}
],
model: selectedModel
model: selectedModel,
useOCR
})
if (image.length > 0) {
humanMessage = humanMessageFormatter({
humanMessage = await humanMessageFormatter({
content: [
{
text: message,
@@ -1069,7 +1076,8 @@ export const useMessage = () => {
type: "image_url"
}
],
model: selectedModel
model: selectedModel,
useOCR
})
}
@@ -1270,17 +1278,18 @@ export const useMessage = () => {
try {
const prompt = await getPrompt(messageType)
let humanMessage = humanMessageFormatter({
let humanMessage = await humanMessageFormatter({
content: [
{
text: prompt.replace("{text}", message),
type: "text"
}
],
model: selectedModel
model: selectedModel,
useOCR
})
if (image.length > 0) {
humanMessage = humanMessageFormatter({
humanMessage = await humanMessageFormatter({
content: [
{
text: prompt.replace("{text}", message),
@@ -1291,7 +1300,8 @@ export const useMessage = () => {
type: "image_url"
}
],
model: selectedModel
model: selectedModel,
useOCR
})
}
@@ -1590,6 +1600,8 @@ export const useMessage = () => {
selectedSystemPrompt,
setSelectedSystemPrompt,
speechToTextLanguage,
setSpeechToTextLanguage
setSpeechToTextLanguage,
useOCR,
setUseOCR
}
}

33
src/hooks/useMessageOption.tsx
View File

@@ -70,7 +70,9 @@ export const useMessageOption = () => {
selectedKnowledge,
setSelectedKnowledge,
temporaryChat,
setTemporaryChat
setTemporaryChat,
useOCR,
setUseOCR
} = useStoreMessageOption()
const currentChatModelSettings = useStoreChatModelSettings()
const [selectedModel, setSelectedModel] = useStorage("selectedModel")
@@ -225,17 +227,18 @@ export const useMessageOption = () => {
// message = message.trim().replaceAll("\n", " ")
let humanMessage = humanMessageFormatter({
let humanMessage = await humanMessageFormatter({
content: [
{
text: message,
type: "text"
}
],
model: selectedModel
model: selectedModel,
useOCR: useOCR
})
if (image.length > 0) {
humanMessage = humanMessageFormatter({
humanMessage = await humanMessageFormatter({
content: [
{
text: message,
@@ -246,7 +249,8 @@ export const useMessageOption = () => {
type: "image_url"
}
],
model: selectedModel
model: selectedModel,
useOCR: useOCR
})
}
@@ -482,17 +486,18 @@ export const useMessageOption = () => {
const prompt = await systemPromptForNonRagOption()
const selectedPrompt = await getPromptById(selectedSystemPrompt)
let humanMessage = humanMessageFormatter({
let humanMessage = await humanMessageFormatter({
content: [
{
text: message,
type: "text"
}
],
model: selectedModel
model: selectedModel,
useOCR: useOCR
})
if (image.length > 0) {
humanMessage = humanMessageFormatter({
humanMessage = await humanMessageFormatter({
content: [
{
text: message,
@@ -503,7 +508,8 @@ export const useMessageOption = () => {
type: "image_url"
}
],
model: selectedModel
model: selectedModel,
useOCR: useOCR
})
}
@@ -802,7 +808,7 @@ export const useMessageOption = () => {
})
// message = message.trim().replaceAll("\n", " ")
let humanMessage = humanMessageFormatter({
let humanMessage = await humanMessageFormatter({
content: [
{
text: systemPrompt
@@ -811,7 +817,8 @@ export const useMessageOption = () => {
type: "text"
}
],
model: selectedModel
model: selectedModel,
useOCR: useOCR
})
const applicationChatHistory = generateHistory(history, selectedModel)
@@ -1106,6 +1113,8 @@ export const useMessageOption = () => {
setSelectedKnowledge,
ttsEnabled,
temporaryChat,
setTemporaryChat
setTemporaryChat,
useOCR,
setUseOCR
}
}