feat: add metering data

src/web/iod.ts

@@ -91,7 +91,13 @@ export async function localIodSearch(
     )
   ).flat()
 
-  return results
+  // results 根据 doId 去重
+    const map = new Map<string, IodRegistryEntry>()
+    for (const r of results) {
+        map.set(r.doId, r)
+    }
+
+  return Array.from(map.values())
 }
 
 const ARXIV_URL_PATTERN = /^https?:\/\/arxiv\.org\//

src/web/web.ts

@@ -95,13 +95,17 @@ export const getSystemPromptForWeb = async (
       //   )
       //   .join("\n")
     }
-    const iod_search_results = iodSearchResults
-      .map((res) => ({
-        doId: res.doId,
-        name: res.name,
-        url: res.url,
-        content: res.content || res.description
-      }))
+    const _iodSearchResults = iodSearchResults
+        .map((res) => ({
+          doId: res.doId,
+          name: res.name,
+          url: res.url,
+          data_space: res.data_space,
+          tokenCount: (res.content || res.description)?.length ?? 0,
+          content: res.content || res.description
+        }))
+
+    const iod_search_results = _iodSearchResults
       .map(
         (result, idx) =>
           `<result doId="${result.doId}" name="${result.name}" source="${result.url}" id="${idx + 1}">${result.content}</result>`
@@ -135,7 +139,9 @@ export const getSystemPromptForWeb = async (
           type: "url"
         }
       }),
-      iodSources: iodSearchResults
+      iodSources: iodSearchResults,
+      iodData: _iodSearchResults,
+      iodDataTokenCount: _iodSearchResults.reduce((acc, cur) => (acc + cur.content.length), 0)
     }
   } catch (e) {
     console.error(e)