feat: support for GPU layer

src/assets/locale/en/common.json

@@ -81,6 +81,10 @@
             "topP": {
                 "label": "Top P",
                 "placeholder": "Enter Top P value (e.g. 0.9, 0.95)"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Enter number of layers to send to GPU(s)"
             }
         },
         "advanced": "More Model Settings"

src/assets/locale/es/common.json

@@ -81,6 +81,10 @@
             "topP": {
                 "label": "Top P",
                 "placeholder": "Ingresar el valor de Top P (ej: 0.9, 0.95)"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Ingrese el número de capas para enviar a la(s) GPU(s)"
             }
         },
         "advanced": "Más Configuraciones del Modelo"

src/assets/locale/fa/common.json

@@ -81,6 +81,10 @@
             "topP": {
                 "label": "Top P",
                 "placeholder": "مقدار Top P را وارد کنید (مثلا 0.9, 0.95)"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "تعداد لایه‌هایی که به GPU(ها) ارسال می‌شود را وارد کنید"
             }
         },
         "advanced": "تنظیمات بیشتر مدل"

src/assets/locale/fr/common.json

@@ -81,6 +81,10 @@
             "topP": {
                 "label": "Top P",
                 "placeholder": "Entrez la valeur Top P (par exemple 0,9, 0,95)"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Entrez le nombre de couches à envoyer au(x) GPU(s)"
             }
         },
         "advanced": "Plus de paramètres du modèle"

src/assets/locale/it/common.json

@@ -81,7 +81,10 @@
             "topP": {
                 "label": "Top P",
                 "placeholder": "Inserisci il Valore Top P (e.g. 0.9, 0.95)"
-            }
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Inserisci il numero di layer da inviare alla/e GPU"            }
         },
         "advanced": "Altre Impostazioni del Modello"
     },

src/assets/locale/ja-JP/common.json

@@ -81,6 +81,10 @@
       "topP": {
         "label": "Top P",
         "placeholder": "Top P値を入力してください（例：0.9、0.95）"
+      },
+      "numGpu": {
+        "label": "Num GPU",
+        "placeholder": "GPU(s)に送信するレイヤー数を入力してください"
       }
     },
     "advanced": "その他のモデル設定"

src/assets/locale/ml/common.json

@@ -80,6 +80,10 @@
             "topP": {
                 "label": "ടോപ് P",
                 "placeholder": "ടോപ് P മൂല്യം നൽകുക (ഉദാ: 0.9, 0.95)"
+            },
+            "numGpu": {
+                "label": "ജിപിയു എണ്ണം",
+                "placeholder": "ജിപിയു(കൾ)ക്ക് അയക്കേണ്ട ലേയറുകളുടെ എണ്ണം നൽകുക"
             }
         },
         "advanced": "കൂടുതൽ മോഡൽ ക്രമീകരണങ്ങൾ"

src/assets/locale/pt-BR/common.json

@@ -81,6 +81,10 @@
       "topP": {
         "label": "Top P",
         "placeholder": "Digite o valor do Top P (ex: 0.9, 0.95)"
+      },
+      "numGpu": {
+          "label": "Num GPUs",
+          "placeholder": "Digite o número de camadas para enviar para a(s) GPU(s)"
       }
     },
     "advanced": "Mais Configurações do Modelo"

src/assets/locale/ru/common.json

@@ -81,7 +81,10 @@
             "topP": {
                 "label": "Top P",
                 "placeholder": "Введите значение Top P (например, 0.9, 0.95)"
-            }
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Введите количество слоев для отправки на GPU"            }
         },
         "advanced": "Больше настроек модели"
     },

src/assets/locale/zh/common.json

@@ -81,6 +81,10 @@
             "topP": {
                 "label": "Top P",
                 "placeholder": "输入Top P值（例如：0.9、0.95）"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "输入要发送到 GPU 的层数"
             }
         },
         "advanced": "更多模型设置"

src/components/Common/Settings/CurrentChatModelSettings.tsx

@@ -24,7 +24,8 @@ export const CurrentChatModelSettings = ({ open, setOpen }: Props) => {
         topP: cUserSettings.topP ?? data.topP,
         keepAlive: cUserSettings.keepAlive ?? data.keepAlive,
         numCtx: cUserSettings.numCtx ?? data.numCtx,
-        seed: cUserSettings.seed
+        seed: cUserSettings.seed,
+        numGpu: cUserSettings.numGpu ?? data.numGpu,
       })
       return data
     },
@@ -118,6 +119,16 @@ export const CurrentChatModelSettings = ({ open, setOpen }: Props) => {
                         placeholder={t("modelSettings.form.topP.placeholder")}
                       />
                     </Form.Item>
+
+                    <Form.Item
+                      name="numGpu"
+                      label={t("modelSettings.form.numGpu.label")}>
+                      <InputNumber
+                        style={{ width: "100%" }}
+                        size="large"
+                        placeholder={t("modelSettings.form.numGpu.placeholder")}
+                      />
+                    </Form.Item>
                   </React.Fragment>
                 )
               }

src/components/Option/Settings/model-settings.tsx

@@ -6,7 +6,6 @@ import { Form, Skeleton, Input, InputNumber, Collapse } from "antd"
 import React from "react"
 import { useTranslation } from "react-i18next"
 
-
 export const ModelSettings = () => {
   const { t } = useTranslation("common")
   const [form] = Form.useForm()
@@ -41,6 +40,7 @@ export const ModelSettings = () => {
             temperature: number
             topK: number
             topP: number
+            numGpu: number
           }) => {
             Object.entries(values).forEach(([key, value]) => {
               setModelSetting(key, value)
@@ -106,6 +106,17 @@ export const ModelSettings = () => {
                         placeholder={t("modelSettings.form.topP.placeholder")}
                       />
                     </Form.Item>
+                    <Form.Item
+                      name="numGpu"
+                      label={t("modelSettings.form.numGpu.label")}>
+                      <InputNumber
+                        style={{ width: "100%" }}
+                        size="large"
+                        placeholder={t(
+                          "modelSettings.form.numGpu.placeholder"
+                        )}
+                      />
+                    </Form.Item>
                   </React.Fragment>
                 )
               }

src/hooks/useMessage.tsx

@@ -122,7 +122,9 @@ export const useMessage = () => {
       topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
       numCtx:
         currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
     })
 
     let newMessage: Message[] = []
@@ -248,7 +250,9 @@ export const useMessage = () => {
           numCtx:
             currentChatModelSettings?.numCtx ??
             userDefaultModelSettings?.numCtx,
-          seed: currentChatModelSettings?.seed
+          seed: currentChatModelSettings?.seed,
+          numGpu:
+            currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
         })
         const response = await questionOllama.invoke(promptForQuestion)
         query = response.content.toString()
@@ -442,7 +446,9 @@ export const useMessage = () => {
       topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
       numCtx:
         currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
     })
 
     let newMessage: Message[] = []
@@ -645,7 +651,9 @@ export const useMessage = () => {
       topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
       numCtx:
         currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
     })
 
     let newMessage: Message[] = []
@@ -718,7 +726,9 @@ export const useMessage = () => {
           numCtx:
             currentChatModelSettings?.numCtx ??
             userDefaultModelSettings?.numCtx,
-          seed: currentChatModelSettings?.seed
+          seed: currentChatModelSettings?.seed,
+          numGpu:
+            currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
         })
         const response = await questionOllama.invoke(promptForQuestion)
         query = response.content.toString()
@@ -890,7 +900,9 @@ export const useMessage = () => {
       topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
       numCtx:
         currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
     })
 
     let newMessage: Message[] = []
@@ -932,8 +944,6 @@ export const useMessage = () => {
     let contentToSave = ""
 
     try {
-
-      
       const prompt = await getPrompt(messageType)
       let humanMessage = new HumanMessage({
         content: [

src/hooks/useMessageOption.tsx

@@ -117,7 +117,9 @@ export const useMessageOption = () => {
       topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
       numCtx:
         currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
     })
 
     let newMessage: Message[] = []
@@ -190,7 +192,9 @@ export const useMessageOption = () => {
           numCtx:
             currentChatModelSettings?.numCtx ??
             userDefaultModelSettings?.numCtx,
-          seed: currentChatModelSettings?.seed
+          seed: currentChatModelSettings?.seed,
+          numGpu:
+            currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
         })
         const response = await questionOllama.invoke(promptForQuestion)
         query = response.content.toString()
@@ -360,7 +364,9 @@ export const useMessageOption = () => {
       topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
       numCtx:
         currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
     })
 
     let newMessage: Message[] = []
@@ -576,7 +582,9 @@ export const useMessageOption = () => {
       topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
       numCtx:
         currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
     })
 
     let newMessage: Message[] = []
@@ -665,7 +673,9 @@ export const useMessageOption = () => {
           numCtx:
             currentChatModelSettings?.numCtx ??
             userDefaultModelSettings?.numCtx,
-          seed: currentChatModelSettings?.seed
+          seed: currentChatModelSettings?.seed,
+          numGpu:
+            currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
         })
         const response = await questionOllama.invoke(promptForQuestion)
         query = response.content.toString()

src/models/ChatOllama.ts

@@ -126,7 +126,7 @@ export class ChatOllama
         this.mirostatTau = fields.mirostatTau;
         this.numBatch = fields.numBatch;
         this.numCtx = fields.numCtx;
-        this.numGpu = fields.numGpu;
+        this.numGpu = fields.numGpu === null ? undefined : fields.numGpu;
         this.numGqa = fields.numGqa;
         this.numKeep = fields.numKeep;
         this.numPredict = fields.numPredict;

src/models/index.ts

@@ -9,7 +9,8 @@ export const pageAssistModel = async ({
   topK,
   topP,
   numCtx,
-  seed
+  seed,
+  numGpu
 }: {
   model: string
   baseUrl: string
@@ -19,6 +20,7 @@ export const pageAssistModel = async ({
   topP?: number
   numCtx?: number
   seed?: number
+  numGpu?: number
 }) => {
   switch (model) {
     case "chrome::gemini-nano::page-assist":
@@ -35,7 +37,8 @@ export const pageAssistModel = async ({
         topP,
         numCtx,
         seed,
-        model
+        model,
+        numGpu
       })
   }
 }