feat: support for GPU layer

2024-08-20 16:11:50 +05:30 · 2024-08-20 16:11:50 +05:30 · 4ef17ff479
commit 4ef17ff479
parent 00735cddad
16 changed files with 108 additions and 25 deletions
--- a/src/assets/locale/en/common.json
+++ b/src/assets/locale/en/common.json
@ -81,6 +81,10 @@
            "topP": {
                "label": "Top P",
                "placeholder": "Enter Top P value (e.g. 0.9, 0.95)"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Enter number of layers to send to GPU(s)"
            }
        },
        "advanced": "More Model Settings"
--- a/src/assets/locale/es/common.json
+++ b/src/assets/locale/es/common.json
@ -81,6 +81,10 @@
            "topP": {
                "label": "Top P",
                "placeholder": "Ingresar el valor de Top P (ej: 0.9, 0.95)"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Ingrese el número de capas para enviar a la(s) GPU(s)"
            }
        },
        "advanced": "Más Configuraciones del Modelo"
--- a/src/assets/locale/fa/common.json
+++ b/src/assets/locale/fa/common.json
@ -81,6 +81,10 @@
            "topP": {
                "label": "Top P",
                "placeholder": "مقدار Top P را وارد کنید (مثلا 0.9, 0.95)"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "تعداد لایه‌هایی که به GPU(ها) ارسال می‌شود را وارد کنید"
            }
        },
        "advanced": "تنظیمات بیشتر مدل"
--- a/src/assets/locale/fr/common.json
+++ b/src/assets/locale/fr/common.json
@ -81,6 +81,10 @@
            "topP": {
                "label": "Top P",
                "placeholder": "Entrez la valeur Top P (par exemple 0,9, 0,95)"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Entrez le nombre de couches à envoyer au(x) GPU(s)"
            }
        },
        "advanced": "Plus de paramètres du modèle"
--- a/src/assets/locale/it/common.json
+++ b/src/assets/locale/it/common.json
@ -81,7 +81,10 @@
            "topP": {
                "label": "Top P",
                "placeholder": "Inserisci il Valore Top P (e.g. 0.9, 0.95)"
-            }
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Inserisci il numero di layer da inviare alla/e GPU"            }
        },
        "advanced": "Altre Impostazioni del Modello"
    },
--- a/src/assets/locale/ja-JP/common.json
+++ b/src/assets/locale/ja-JP/common.json
@ -81,6 +81,10 @@
      "topP": {
        "label": "Top P",
        "placeholder": "Top P値を入力してください（例：0.9、0.95）"
+      },
+      "numGpu": {
+        "label": "Num GPU",
+        "placeholder": "GPU(s)に送信するレイヤー数を入力してください"
      }
    },
    "advanced": "その他のモデル設定"
--- a/src/assets/locale/ml/common.json
+++ b/src/assets/locale/ml/common.json
@ -80,6 +80,10 @@
            "topP": {
                "label": "ടോപ് P",
                "placeholder": "ടോപ് P മൂല്യം നൽകുക (ഉദാ: 0.9, 0.95)"
+            },
+            "numGpu": {
+                "label": "ജിപിയു എണ്ണം",
+                "placeholder": "ജിപിയു(കൾ)ക്ക് അയക്കേണ്ട ലേയറുകളുടെ എണ്ണം നൽകുക"
            }
        },
        "advanced": "കൂടുതൽ മോഡൽ ക്രമീകരണങ്ങൾ"
--- a/src/assets/locale/pt-BR/common.json
+++ b/src/assets/locale/pt-BR/common.json
@ -81,6 +81,10 @@
      "topP": {
        "label": "Top P",
        "placeholder": "Digite o valor do Top P (ex: 0.9, 0.95)"
+      },
+      "numGpu": {
+          "label": "Num GPUs",
+          "placeholder": "Digite o número de camadas para enviar para a(s) GPU(s)"
      }
    },
    "advanced": "Mais Configurações do Modelo"
--- a/src/assets/locale/ru/common.json
+++ b/src/assets/locale/ru/common.json
@ -81,7 +81,10 @@
            "topP": {
                "label": "Top P",
                "placeholder": "Введите значение Top P (например, 0.9, 0.95)"
-            }
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "Введите количество слоев для отправки на GPU"            }
        },
        "advanced": "Больше настроек модели"
    },
--- a/src/assets/locale/zh/common.json
+++ b/src/assets/locale/zh/common.json
@ -81,6 +81,10 @@
            "topP": {
                "label": "Top P",
                "placeholder": "输入Top P值（例如：0.9、0.95）"
+            },
+            "numGpu": {
+                "label": "Num GPU",
+                "placeholder": "输入要发送到 GPU 的层数"
            }
        },
        "advanced": "更多模型设置"
--- a/src/components/Common/Settings/CurrentChatModelSettings.tsx
+++ b/src/components/Common/Settings/CurrentChatModelSettings.tsx
@ -24,7 +24,8 @@ export const CurrentChatModelSettings = ({ open, setOpen }: Props) => {
        topP: cUserSettings.topP ?? data.topP,
        keepAlive: cUserSettings.keepAlive ?? data.keepAlive,
        numCtx: cUserSettings.numCtx ?? data.numCtx,
-        seed: cUserSettings.seed
+        seed: cUserSettings.seed,
+        numGpu: cUserSettings.numGpu ?? data.numGpu,
      })
      return data
    },
@ -118,6 +119,16 @@ export const CurrentChatModelSettings = ({ open, setOpen }: Props) => {
                        placeholder={t("modelSettings.form.topP.placeholder")}
                      />
                    </Form.Item>
+
+                    <Form.Item
+                      name="numGpu"
+                      label={t("modelSettings.form.numGpu.label")}>
+                      <InputNumber
+                        style={{ width: "100%" }}
+                        size="large"
+                        placeholder={t("modelSettings.form.numGpu.placeholder")}
+                      />
+                    </Form.Item>
                  </React.Fragment>
                )
              }
--- a/src/components/Option/Settings/model-settings.tsx
+++ b/src/components/Option/Settings/model-settings.tsx
@ -6,7 +6,6 @@ import { Form, Skeleton, Input, InputNumber, Collapse } from "antd"
 import React from "react"
 import { useTranslation } from "react-i18next"

-
 export const ModelSettings = () => {
  const { t } = useTranslation("common")
  const [form] = Form.useForm()
@ -24,10 +23,10 @@ export const ModelSettings = () => {
    <div>
      <div>
        <div className="inline-flex items-center gap-2">
-        <BetaTag />
-        <h2 className="text-base font-semibold leading-7 text-gray-900 dark:text-white">
-          {t("modelSettings.label")} 
-        </h2>
+          <BetaTag />
+          <h2 className="text-base font-semibold leading-7 text-gray-900 dark:text-white">
+            {t("modelSettings.label")}
+          </h2>
        </div>
        <p className="text-sm text-gray-700 dark:text-neutral-400 mt-1">
          {t("modelSettings.description")}
@ -41,6 +40,7 @@ export const ModelSettings = () => {
            temperature: number
            topK: number
            topP: number
+            numGpu: number
          }) => {
            Object.entries(values).forEach(([key, value]) => {
              setModelSetting(key, value)
@ -106,6 +106,17 @@ export const ModelSettings = () => {
                        placeholder={t("modelSettings.form.topP.placeholder")}
                      />
                    </Form.Item>
+                    <Form.Item
+                      name="numGpu"
+                      label={t("modelSettings.form.numGpu.label")}>
+                      <InputNumber
+                        style={{ width: "100%" }}
+                        size="large"
+                        placeholder={t(
+                          "modelSettings.form.numGpu.placeholder"
+                        )}
+                      />
+                    </Form.Item>
                  </React.Fragment>
                )
              }
--- a/src/hooks/useMessage.tsx
+++ b/src/hooks/useMessage.tsx
@ -122,7 +122,9 @@ export const useMessage = () => {
      topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
      numCtx:
        currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
    })

    let newMessage: Message[] = []
@ -248,7 +250,9 @@ export const useMessage = () => {
          numCtx:
            currentChatModelSettings?.numCtx ??
            userDefaultModelSettings?.numCtx,
-          seed: currentChatModelSettings?.seed
+          seed: currentChatModelSettings?.seed,
+          numGpu:
+            currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
        })
        const response = await questionOllama.invoke(promptForQuestion)
        query = response.content.toString()
@ -442,7 +446,9 @@ export const useMessage = () => {
      topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
      numCtx:
        currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
    })

    let newMessage: Message[] = []
@ -645,7 +651,9 @@ export const useMessage = () => {
      topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
      numCtx:
        currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
    })

    let newMessage: Message[] = []
@ -718,7 +726,9 @@ export const useMessage = () => {
          numCtx:
            currentChatModelSettings?.numCtx ??
            userDefaultModelSettings?.numCtx,
-          seed: currentChatModelSettings?.seed
+          seed: currentChatModelSettings?.seed,
+          numGpu:
+            currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
        })
        const response = await questionOllama.invoke(promptForQuestion)
        query = response.content.toString()
@ -890,7 +900,9 @@ export const useMessage = () => {
      topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
      numCtx:
        currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
    })

    let newMessage: Message[] = []
@ -932,8 +944,6 @@ export const useMessage = () => {
    let contentToSave = ""

    try {
-
-      
      const prompt = await getPrompt(messageType)
      let humanMessage = new HumanMessage({
        content: [
--- a/src/hooks/useMessageOption.tsx
+++ b/src/hooks/useMessageOption.tsx
@ -117,7 +117,9 @@ export const useMessageOption = () => {
      topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
      numCtx:
        currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
    })

    let newMessage: Message[] = []
@ -190,7 +192,9 @@ export const useMessageOption = () => {
          numCtx:
            currentChatModelSettings?.numCtx ??
            userDefaultModelSettings?.numCtx,
-          seed: currentChatModelSettings?.seed
+          seed: currentChatModelSettings?.seed,
+          numGpu:
+            currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
        })
        const response = await questionOllama.invoke(promptForQuestion)
        query = response.content.toString()
@ -360,7 +364,9 @@ export const useMessageOption = () => {
      topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
      numCtx:
        currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
    })

    let newMessage: Message[] = []
@ -576,7 +582,9 @@ export const useMessageOption = () => {
      topP: currentChatModelSettings?.topP ?? userDefaultModelSettings?.topP,
      numCtx:
        currentChatModelSettings?.numCtx ?? userDefaultModelSettings?.numCtx,
-      seed: currentChatModelSettings?.seed
+      seed: currentChatModelSettings?.seed,
+      numGpu:
+        currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
    })

    let newMessage: Message[] = []
@ -665,7 +673,9 @@ export const useMessageOption = () => {
          numCtx:
            currentChatModelSettings?.numCtx ??
            userDefaultModelSettings?.numCtx,
-          seed: currentChatModelSettings?.seed
+          seed: currentChatModelSettings?.seed,
+          numGpu:
+            currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu
        })
        const response = await questionOllama.invoke(promptForQuestion)
        query = response.content.toString()
--- a/src/models/ChatOllama.ts
+++ b/src/models/ChatOllama.ts
@ -126,7 +126,7 @@ export class ChatOllama
        this.mirostatTau = fields.mirostatTau;
        this.numBatch = fields.numBatch;
        this.numCtx = fields.numCtx;
-        this.numGpu = fields.numGpu;
+        this.numGpu = fields.numGpu === null ? undefined : fields.numGpu;
        this.numGqa = fields.numGqa;
        this.numKeep = fields.numKeep;
        this.numPredict = fields.numPredict;
--- a/src/models/index.ts
+++ b/src/models/index.ts
@ -9,7 +9,8 @@ export const pageAssistModel = async ({
  topK,
  topP,
  numCtx,
-  seed
+  seed,
+  numGpu
 }: {
  model: string
  baseUrl: string
@ -19,6 +20,7 @@ export const pageAssistModel = async ({
  topP?: number
  numCtx?: number
  seed?: number
+  numGpu?: number
 }) => {
  switch (model) {
    case "chrome::gemini-nano::page-assist":
@ -35,7 +37,8 @@ export const pageAssistModel = async ({
        topP,
        numCtx,
        seed,
-        model
+        model,
+        numGpu
      })
  }
 }