From 4ef17ff479d50bf123275f392e78b7fe4f97f999 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Tue, 20 Aug 2024 16:11:50 +0530 Subject: [PATCH] feat: support for GPU layer --- src/assets/locale/en/common.json | 4 +++ src/assets/locale/es/common.json | 4 +++ src/assets/locale/fa/common.json | 6 ++++- src/assets/locale/fr/common.json | 4 +++ src/assets/locale/it/common.json | 5 +++- src/assets/locale/ja-JP/common.json | 4 +++ src/assets/locale/ml/common.json | 4 +++ src/assets/locale/pt-BR/common.json | 4 +++ src/assets/locale/ru/common.json | 5 +++- src/assets/locale/zh/common.json | 4 +++ .../Settings/CurrentChatModelSettings.tsx | 13 +++++++++- .../Option/Settings/model-settings.tsx | 21 +++++++++++---- src/hooks/useMessage.tsx | 26 +++++++++++++------ src/hooks/useMessageOption.tsx | 20 ++++++++++---- src/models/ChatOllama.ts | 2 +- src/models/index.ts | 7 +++-- 16 files changed, 108 insertions(+), 25 deletions(-) diff --git a/src/assets/locale/en/common.json b/src/assets/locale/en/common.json index 05ef169..47bcd67 100644 --- a/src/assets/locale/en/common.json +++ b/src/assets/locale/en/common.json @@ -81,6 +81,10 @@ "topP": { "label": "Top P", "placeholder": "Enter Top P value (e.g. 0.9, 0.95)" + }, + "numGpu": { + "label": "Num GPU", + "placeholder": "Enter number of layers to send to GPU(s)" } }, "advanced": "More Model Settings" diff --git a/src/assets/locale/es/common.json b/src/assets/locale/es/common.json index 06d0a00..9f1950c 100644 --- a/src/assets/locale/es/common.json +++ b/src/assets/locale/es/common.json @@ -81,6 +81,10 @@ "topP": { "label": "Top P", "placeholder": "Ingresar el valor de Top P (ej: 0.9, 0.95)" + }, + "numGpu": { + "label": "Num GPU", + "placeholder": "Ingrese el número de capas para enviar a la(s) GPU(s)" } }, "advanced": "Más Configuraciones del Modelo" diff --git a/src/assets/locale/fa/common.json b/src/assets/locale/fa/common.json index 3c02c21..3e9635b 100644 --- a/src/assets/locale/fa/common.json +++ b/src/assets/locale/fa/common.json @@ -81,8 +81,12 @@ "topP": { "label": "Top P", "placeholder": "مقدار Top P را وارد کنید (مثلا 0.9, 0.95)" + }, + "numGpu": { + "label": "Num GPU", + "placeholder": "تعداد لایه‌هایی که به GPU(ها) ارسال می‌شود را وارد کنید" } }, "advanced": "تنظیمات بیشتر مدل" } -} +} \ No newline at end of file diff --git a/src/assets/locale/fr/common.json b/src/assets/locale/fr/common.json index cf601db..4833892 100644 --- a/src/assets/locale/fr/common.json +++ b/src/assets/locale/fr/common.json @@ -81,6 +81,10 @@ "topP": { "label": "Top P", "placeholder": "Entrez la valeur Top P (par exemple 0,9, 0,95)" + }, + "numGpu": { + "label": "Num GPU", + "placeholder": "Entrez le nombre de couches à envoyer au(x) GPU(s)" } }, "advanced": "Plus de paramètres du modèle" diff --git a/src/assets/locale/it/common.json b/src/assets/locale/it/common.json index 0b0708e..e56b203 100644 --- a/src/assets/locale/it/common.json +++ b/src/assets/locale/it/common.json @@ -81,7 +81,10 @@ "topP": { "label": "Top P", "placeholder": "Inserisci il Valore Top P (e.g. 0.9, 0.95)" - } + }, + "numGpu": { + "label": "Num GPU", + "placeholder": "Inserisci il numero di layer da inviare alla/e GPU" } }, "advanced": "Altre Impostazioni del Modello" }, diff --git a/src/assets/locale/ja-JP/common.json b/src/assets/locale/ja-JP/common.json index 4fcde04..d65ca6d 100644 --- a/src/assets/locale/ja-JP/common.json +++ b/src/assets/locale/ja-JP/common.json @@ -81,6 +81,10 @@ "topP": { "label": "Top P", "placeholder": "Top P値を入力してください（例：0.9、0.95）" + }, + "numGpu": { + "label": "Num GPU", + "placeholder": "GPU(s)に送信するレイヤー数を入力してください" } }, "advanced": "その他のモデル設定" diff --git a/src/assets/locale/ml/common.json b/src/assets/locale/ml/common.json index 4d923c0..5f03cbc 100644 --- a/src/assets/locale/ml/common.json +++ b/src/assets/locale/ml/common.json @@ -80,6 +80,10 @@ "topP": { "label": "ടോപ് P", "placeholder": "ടോപ് P മൂല്യം നൽകുക (ഉദാ: 0.9, 0.95)" + }, + "numGpu": { + "label": "ജിപിയു എണ്ണം", + "placeholder": "ജിപിയു(കൾ)ക്ക് അയക്കേണ്ട ലേയറുകളുടെ എണ്ണം നൽകുക" } }, "advanced": "കൂടുതൽ മോഡൽ ക്രമീകരണങ്ങൾ" diff --git a/src/assets/locale/pt-BR/common.json b/src/assets/locale/pt-BR/common.json index a322b7c..f9b3609 100644 --- a/src/assets/locale/pt-BR/common.json +++ b/src/assets/locale/pt-BR/common.json @@ -81,6 +81,10 @@ "topP": { "label": "Top P", "placeholder": "Digite o valor do Top P (ex: 0.9, 0.95)" + }, + "numGpu": { + "label": "Num GPUs", + "placeholder": "Digite o número de camadas para enviar para a(s) GPU(s)" } }, "advanced": "Mais Configurações do Modelo" diff --git a/src/assets/locale/ru/common.json b/src/assets/locale/ru/common.json index b14c855..a4262fa 100644 --- a/src/assets/locale/ru/common.json +++ b/src/assets/locale/ru/common.json @@ -81,7 +81,10 @@ "topP": { "label": "Top P", "placeholder": "Введите значение Top P (например, 0.9, 0.95)" - } + }, + "numGpu": { + "label": "Num GPU", + "placeholder": "Введите количество слоев для отправки на GPU" } }, "advanced": "Больше настроек модели" }, diff --git a/src/assets/locale/zh/common.json b/src/assets/locale/zh/common.json index 9145e48..badcec4 100644 --- a/src/assets/locale/zh/common.json +++ b/src/assets/locale/zh/common.json @@ -81,6 +81,10 @@ "topP": { "label": "Top P", "placeholder": "输入Top P值（例如：0.9、0.95）" + }, + "numGpu": { + "label": "Num GPU", + "placeholder": "输入要发送到 GPU 的层数" } }, "advanced": "更多模型设置" diff --git a/src/components/Common/Settings/CurrentChatModelSettings.tsx b/src/components/Common/Settings/CurrentChatModelSettings.tsx index b1d1c07..147b232 100644 --- a/src/components/Common/Settings/CurrentChatModelSettings.tsx +++ b/src/components/Common/Settings/CurrentChatModelSettings.tsx @@ -24,7 +24,8 @@ export const CurrentChatModelSettings = ({ open, setOpen }: Props) => { topP: cUserSettings.topP ?? data.topP, keepAlive: cUserSettings.keepAlive ?? data.keepAlive, numCtx: cUserSettings.numCtx ?? data.numCtx, - seed: cUserSettings.seed + seed: cUserSettings.seed, + numGpu: cUserSettings.numGpu ?? data.numGpu, }) return data }, @@ -118,6 +119,16 @@ export const CurrentChatModelSettings = ({ open, setOpen }: Props) => { placeholder={t("modelSettings.form.topP.placeholder")} /> + + + + ) } diff --git a/src/components/Option/Settings/model-settings.tsx b/src/components/Option/Settings/model-settings.tsx index b7a5b71..b91dda5 100644 --- a/src/components/Option/Settings/model-settings.tsx +++ b/src/components/Option/Settings/model-settings.tsx @@ -6,7 +6,6 @@ import { Form, Skeleton, Input, InputNumber, Collapse } from "antd" import React from "react" import { useTranslation } from "react-i18next" - export const ModelSettings = () => { const { t } = useTranslation("common") const [form] = Form.useForm() @@ -24,10 +23,10 @@ export const ModelSettings = () => {
- -

- {t("modelSettings.label")} -

+ +

+ {t("modelSettings.label")} +

{t("modelSettings.description")} @@ -41,6 +40,7 @@ export const ModelSettings = () => { temperature: number topK: number topP: number + numGpu: number }) => { Object.entries(values).forEach(([key, value]) => { setModelSetting(key, value) @@ -106,6 +106,17 @@ export const ModelSettings = () => { placeholder={t("modelSettings.form.topP.placeholder")} /> + + + ) } diff --git a/src/hooks/useMessage.tsx b/src/hooks/useMessage.tsx index 3d61528..abf9a6b 100644 --- a/src/hooks/useMessage.tsx +++ b/src/hooks/useMessage.tsx @@ -122,7 +122,9 @@ export const useMessage = () => { topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP, numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) let newMessage: Message[] = [] @@ -248,7 +250,9 @@ export const useMessage = () => { numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) const response = await questionOllama.invoke(promptForQuestion) query = response.content.toString() @@ -442,7 +446,9 @@ export const useMessage = () => { topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP, numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) let newMessage: Message[] = [] @@ -645,7 +651,9 @@ export const useMessage = () => { topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP, numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) let newMessage: Message[] = [] @@ -718,7 +726,9 @@ export const useMessage = () => { numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) const response = await questionOllama.invoke(promptForQuestion) query = response.content.toString() @@ -890,7 +900,9 @@ export const useMessage = () => { topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP, numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) let newMessage: Message[] = [] @@ -932,8 +944,6 @@ export const useMessage = () => { let contentToSave = "" try { - - const prompt = await getPrompt(messageType) let humanMessage = new HumanMessage({ content: [ diff --git a/src/hooks/useMessageOption.tsx b/src/hooks/useMessageOption.tsx index 2b28aa7..a362f69 100644 --- a/src/hooks/useMessageOption.tsx +++ b/src/hooks/useMessageOption.tsx @@ -117,7 +117,9 @@ export const useMessageOption = () => { topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP, numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) let newMessage: Message[] = [] @@ -190,7 +192,9 @@ export const useMessageOption = () => { numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) const response = await questionOllama.invoke(promptForQuestion) query = response.content.toString() @@ -360,7 +364,9 @@ export const useMessageOption = () => { topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP, numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) let newMessage: Message[] = [] @@ -576,7 +582,9 @@ export const useMessageOption = () => { topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP, numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) let newMessage: Message[] = [] @@ -665,7 +673,9 @@ export const useMessageOption = () => { numCtx: currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx, - seed: currentChatModelSettings?.seed + seed: currentChatModelSettings?.seed, + numGpu: + currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu }) const response = await questionOllama.invoke(promptForQuestion) query = response.content.toString() diff --git a/src/models/ChatOllama.ts b/src/models/ChatOllama.ts index 2a4c28b..1bc7ddb 100644 --- a/src/models/ChatOllama.ts +++ b/src/models/ChatOllama.ts @@ -126,7 +126,7 @@ export class ChatOllama this.mirostatTau = fields.mirostatTau; this.numBatch = fields.numBatch; this.numCtx = fields.numCtx; - this.numGpu = fields.numGpu; + this.numGpu = fields.numGpu === null ? undefined : fields.numGpu; this.numGqa = fields.numGqa; this.numKeep = fields.numKeep; this.numPredict = fields.numPredict; diff --git a/src/models/index.ts b/src/models/index.ts index 4048cb3..ce3ab39 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -9,7 +9,8 @@ export const pageAssistModel = async ({ topK, topP, numCtx, - seed + seed, + numGpu }: { model: string baseUrl: string @@ -19,6 +20,7 @@ export const pageAssistModel = async ({ topP?: number numCtx?: number seed?: number + numGpu?: number }) => { switch (model) { case "chrome::gemini-nano::page-assist": @@ -35,7 +37,8 @@ export const pageAssistModel = async ({ topP, numCtx, seed, - model + model, + numGpu }) } }