From 6d80798da9ecc5384bc2377c0769fd2ee676af92 Mon Sep 17 00:00:00 2001 From: n4ze3m Date: Sat, 30 Nov 2024 20:17:03 +0530 Subject: [PATCH] feat: Add useMMap option to model settings --- src/assets/locale/en/common.json | 3 ++ .../Settings/CurrentChatModelSettings.tsx | 15 +++++++--- .../Option/Settings/model-settings.tsx | 25 +++++++++------- src/hooks/useMessage.tsx | 30 ++++++++++++++----- src/hooks/useMessageOption.tsx | 22 ++++++++++---- src/models/index.ts | 5 +++- 6 files changed, 72 insertions(+), 28 deletions(-) diff --git a/src/assets/locale/en/common.json b/src/assets/locale/en/common.json index d033144..88c1be5 100644 --- a/src/assets/locale/en/common.json +++ b/src/assets/locale/en/common.json @@ -87,6 +87,9 @@ "label": "Top P", "placeholder": "Enter Top P value (e.g. 0.9, 0.95)" }, + "useMMap": { + "label": "useMmap" + }, "numGpu": { "label": "Num GPU", "placeholder": "Enter number of layers to send to GPU(s)" diff --git a/src/components/Common/Settings/CurrentChatModelSettings.tsx b/src/components/Common/Settings/CurrentChatModelSettings.tsx index e950bd9..2914266 100644 --- a/src/components/Common/Settings/CurrentChatModelSettings.tsx +++ b/src/components/Common/Settings/CurrentChatModelSettings.tsx @@ -10,7 +10,8 @@ import { Input, InputNumber, Modal, - Skeleton + Skeleton, + Switch } from "antd" import React from "react" import { useTranslation } from "react-i18next" @@ -35,7 +36,7 @@ export const CurrentChatModelSettings = ({ queryFn: async () => { const data = await getAllModelSettings() - let tempSystemPrompt = ""; + let tempSystemPrompt = "" // i hate this method but i need this feature so badly that i need to do this if (selectedSystemPrompt) { @@ -52,7 +53,8 @@ export const CurrentChatModelSettings = ({ seed: cUserSettings.seed, numGpu: cUserSettings.numGpu ?? data.numGpu, numPredict: cUserSettings.numPredict ?? data.numPredict, - systemPrompt: cUserSettings.systemPrompt ?? tempSystemPrompt + systemPrompt: cUserSettings.systemPrompt ?? tempSystemPrompt, + useMMap: cUserSettings.useMMap ?? data.useMMap }) return data }, @@ -61,7 +63,6 @@ export const CurrentChatModelSettings = ({ refetchOnWindowFocus: false }) - const renderBody = () => { return ( <> @@ -176,6 +177,12 @@ export const CurrentChatModelSettings = ({ )} /> + + + + ) } diff --git a/src/components/Option/Settings/model-settings.tsx b/src/components/Option/Settings/model-settings.tsx index 40523f7..18015ba 100644 --- a/src/components/Option/Settings/model-settings.tsx +++ b/src/components/Option/Settings/model-settings.tsx @@ -2,7 +2,7 @@ import { BetaTag } from "@/components/Common/Beta" import { SaveButton } from "@/components/Common/SaveButton" import { getAllModelSettings, setModelSetting } from "@/services/model-settings" import { useQuery, useQueryClient } from "@tanstack/react-query" -import { Form, Skeleton, Input, InputNumber, Collapse } from "antd" +import { Form, Skeleton, Input, InputNumber, Collapse, Switch } from "antd" import React from "react" import { useTranslation } from "react-i18next" @@ -78,13 +78,13 @@ export const ModelSettings = () => { /> - - + name="numPredict" + label={t("modelSettings.form.numPredict.label")}> + + { + + + ) } diff --git a/src/hooks/useMessage.tsx b/src/hooks/useMessage.tsx index eb17f3e..806aa94 100644 --- a/src/hooks/useMessage.tsx +++ b/src/hooks/useMessage.tsx @@ -139,7 +139,9 @@ export const useMessage = () => { currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? userDefaultModelSettings?.useMMap }) let newMessage: Message[] = [] @@ -271,7 +273,10 @@ export const useMessage = () => { userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? + userDefaultModelSettings?.useMMap }) const response = await questionOllama.invoke(promptForQuestion) query = response.content.toString() @@ -482,7 +487,9 @@ export const useMessage = () => { currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? userDefaultModelSettings?.useMMap }) let newMessage: Message[] = [] @@ -716,7 +723,9 @@ export const useMessage = () => { currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? userDefaultModelSettings?.useMMap }) let newMessage: Message[] = [] @@ -944,7 +953,9 @@ export const useMessage = () => { currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? userDefaultModelSettings?.useMMap }) let newMessage: Message[] = [] @@ -1023,7 +1034,10 @@ export const useMessage = () => { userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? + userDefaultModelSettings?.useMMap }) const response = await questionOllama.invoke(promptForQuestion) query = response.content.toString() @@ -1211,7 +1225,9 @@ export const useMessage = () => { currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? userDefaultModelSettings?.useMMap }) let newMessage: Message[] = [] diff --git a/src/hooks/useMessageOption.tsx b/src/hooks/useMessageOption.tsx index 322287c..d687344 100644 --- a/src/hooks/useMessageOption.tsx +++ b/src/hooks/useMessageOption.tsx @@ -130,7 +130,9 @@ export const useMessageOption = () => { currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? userDefaultModelSettings?.useMMap }) let newMessage: Message[] = [] @@ -209,7 +211,10 @@ export const useMessageOption = () => { userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? + userDefaultModelSettings?.useMMap }) const response = await questionOllama.invoke(promptForQuestion) query = response.content.toString() @@ -429,7 +434,9 @@ export const useMessageOption = () => { currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? userDefaultModelSettings?.useMMap }) let newMessage: Message[] = [] @@ -667,7 +674,9 @@ export const useMessageOption = () => { currentChatModelSettings?.numGpu ?? userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? userDefaultModelSettings?.useMMap }) let newMessage: Message[] = [] @@ -762,7 +771,10 @@ export const useMessageOption = () => { userDefaultModelSettings?.numGpu, numPredict: currentChatModelSettings?.numPredict ?? - userDefaultModelSettings?.numPredict + userDefaultModelSettings?.numPredict, + useMMap: + currentChatModelSettings?.useMMap ?? + userDefaultModelSettings?.useMMap }) const response = await questionOllama.invoke(promptForQuestion) query = response.content.toString() diff --git a/src/models/index.ts b/src/models/index.ts index 135025f..3481d52 100644 --- a/src/models/index.ts +++ b/src/models/index.ts @@ -16,6 +16,7 @@ export const pageAssistModel = async ({ seed, numGpu, numPredict, + useMMap }: { model: string baseUrl: string @@ -27,6 +28,7 @@ export const pageAssistModel = async ({ seed?: number numGpu?: number numPredict?: number + useMMap?: boolean }) => { if (model === "chrome::gemini-nano::page-assist") { @@ -73,7 +75,8 @@ export const pageAssistModel = async ({ seed, model, numGpu, - numPredict + numPredict, + useMMap, })