{t("generalSettings.settings.language.label")}{" "}
diff --git a/src/entries/background.ts b/src/entries/background.ts
index 85fb678..f671467 100644
--- a/src/entries/background.ts
+++ b/src/entries/background.ts
@@ -1,4 +1,6 @@
import { getOllamaURL, isOllamaRunning } from "../services/ollama"
+import { Storage } from "@plasmohq/storage"
+
const progressHuman = (completed: number, total: number) => {
return ((completed / total) * 100).toFixed(0) + "%"
}
@@ -75,6 +77,8 @@ const streamDownload = async (url: string, model: string) => {
}
export default defineBackground({
main() {
+ const storage = new Storage()
+
chrome.runtime.onMessage.addListener(async (message) => {
if (message.type === "sidepanel") {
chrome.tabs.query(
@@ -139,8 +143,8 @@ export default defineBackground({
{ active: true, currentWindow: true },
async (tabs) => {
const tab = tabs[0]
- await chrome.sidePanel.open({
- windowId: tab.windowId!
+ chrome.sidePanel.open({
+ tabId: tab.id!
})
}
)
diff --git a/src/loader/html.ts b/src/loader/html.ts
index 367b02a..de567f0 100644
--- a/src/loader/html.ts
+++ b/src/loader/html.ts
@@ -3,6 +3,7 @@ import { Document } from "@langchain/core/documents"
import { compile } from "html-to-text"
import { chromeRunTime } from "~/libs/runtime"
import { YtTranscript } from "yt-transcript"
+import { isWikipedia, parseWikipedia } from "@/parser/wiki"
const YT_REGEX =
/(?:https?:\/\/)?(?:www\.)?(?:youtube\.com|youtu\.be)\/(?:watch\?v=)?([a-zA-Z0-9_-]+)/
@@ -16,7 +17,6 @@ const getTranscript = async (url: string) => {
return await ytTranscript.getTranscript()
}
-
export interface WebLoaderParams {
html: string
url: string
@@ -24,7 +24,8 @@ export interface WebLoaderParams {
export class PageAssistHtmlLoader
extends BaseDocumentLoader
- implements WebLoaderParams {
+ implements WebLoaderParams
+{
html: string
url: string
@@ -47,7 +48,6 @@ export class PageAssistHtmlLoader
text += item.text + " "
})
-
return [
{
metadata: {
@@ -58,10 +58,23 @@ export class PageAssistHtmlLoader
}
]
}
+
+ let html = this.html
+
+ if (isWikipedia(this.url)) {
+ console.log("Wikipedia URL detected")
+ html = parseWikipedia(html)
+ }
+
+ // else if (isTwitter(this.url)) {
+ // console.log("Twitter URL detected")
+ // html = parseTweet(html, this.url)
+ // }
+
const htmlCompiler = compile({
wordwrap: false
})
- const text = htmlCompiler(this.html)
+ const text = htmlCompiler(html)
const metadata = { source: this.url }
return [new Document({ pageContent: text, metadata })]
}
@@ -79,7 +92,6 @@ export class PageAssistHtmlLoader
text += item.text + " "
})
-
return [
{
metadata: {
@@ -92,7 +104,18 @@ export class PageAssistHtmlLoader
}
await chromeRunTime(this.url)
const fetchHTML = await fetch(this.url)
- const html = await fetchHTML.text()
+ let html = await fetchHTML.text()
+
+ if (isWikipedia(this.url)) {
+ console.log("Wikipedia URL detected")
+ html = parseWikipedia(await fetchHTML.text())
+ }
+
+ // else if (isTwitter(this.url)) {
+ // console.log("Twitter URL detected")
+ // html = parseTweet(await fetchHTML.text(), this.url)
+ // }
+
const htmlCompiler = compile({
wordwrap: false,
selectors: [
diff --git a/src/parser/twitter.ts b/src/parser/twitter.ts
new file mode 100644
index 0000000..a998945
--- /dev/null
+++ b/src/parser/twitter.ts
@@ -0,0 +1,90 @@
+import * as cheerio from "cheerio"
+
+export const isTweet = (url: string) => {
+ const TWEET_REGEX = /twitter\.com\/[a-zA-Z0-9_]+\/status\/[0-9]+/g
+ return TWEET_REGEX.test(url)
+}
+
+export const isTwitterProfile = (url: string) => {
+ const PROFILE_REGEX = /twitter\.com\/[a-zA-Z0-9_]+/g
+ return PROFILE_REGEX.test(url)
+}
+
+export const isTwitterTimeline = (url: string) => {
+ const TIMELINE_REGEX = /twitter\.com\/home/g
+ return TIMELINE_REGEX.test(url)
+}
+
+export const isTwitter = (url: string) => {
+ return isTweet(url) || isTwitterProfile(url) || isTwitterTimeline(url)
+}
+
+export const isTwitterNotification = (url: string) => {
+ const NOTIFICATION_REGEX = /twitter\.com\/notifications/g
+ return NOTIFICATION_REGEX.test(url)
+}
+
+export const parseTweet = (html: string, url: string) => {
+ if (!html) {
+ return ""
+ }
+
+ const $ = cheerio.load(html)
+
+ if (isTweet(url)) {
+ console.log("tweet")
+ const tweet = $("div[data-testid='tweet']")
+ const tweetContent = tweet.find("div[lang]")
+ const tweetMedia = tweet.find("div[role='group']")
+ const author = tweet.find("a[role='link']").text()
+ const date = tweet.find("time").text()
+ return `
${author} ${tweetContent.text()} ${tweetMedia.html()} ${date}
`
+ }
+
+ if (isTwitterTimeline(url)) {
+ console.log("timeline")
+ const timeline = $("div[data-testid='primaryColumn']")
+ const timelineContent = timeline.find("div[data-testid='tweet']")
+ console.log(timelineContent.html())
+ const tweet = timelineContent
+ .map((i, el) => {
+ const author = $(el).find("a[role='link']").text()
+ const content = $(el).find("div[lang]").text()
+ const media = $(el).find("div[role='group']").html()
+ const date = $(el).find("time").text()
+ return `${author} ${content} ${media} ${date}
`
+ })
+ .get()
+ .join("")
+ console.log(tweet)
+ return `${tweet}
`
+ }
+
+ if (isTwitterNotification(url)) {
+ console.log("notification")
+ const notification = $("div[data-testid='primaryColumn']")
+ const notificationContent = notification.find("div[data-testid='tweet']")
+ return `${notificationContent.html()}
`
+ }
+ if (isTwitterProfile(url)) {
+ console.log("profile")
+ const profile = $("div[data-testid='primaryColumn']")
+ const profileContent = profile.find(
+ "div[data-testid='UserProfileHeader_Items']"
+ )
+ const profileTweets = profile.find("div[data-testid='tweet']")
+ return `${profileContent.html()}
${profileTweets.html()}
`
+ }
+ console.log("no match")
+ const timeline = $("div[data-testid='primaryColumn']")
+ const timelineContent = timeline.find("div[data-testid='tweet']")
+ const tweet = timelineContent.map((i, el) => {
+ const author = $(el).find("a[role='link']").text()
+ const content = $(el).find("div[lang]").text()
+ const media = $(el).find("div[role='group']").html()
+ const date = $(el).find("time").text()
+ return `${author} ${content} ${media} ${date}
`
+ })
+
+ return `${tweet}
`
+}
diff --git a/src/parser/wiki.ts b/src/parser/wiki.ts
new file mode 100644
index 0000000..36f567c
--- /dev/null
+++ b/src/parser/wiki.ts
@@ -0,0 +1,28 @@
+import * as cheerio from "cheerio"
+
+export const isWikipedia = (url: string) => {
+ const WIKI_REGEX = /wikipedia\.org\/wiki\//g
+ return WIKI_REGEX.test(url)
+}
+
+export const parseWikipedia = (html: string) => {
+ if (!html) {
+ return ""
+ }
+ const $ = cheerio.load(html)
+ const title = $("h1#firstHeading")
+ const content = $("#mw-content-text")
+ content?.find("sup.reference")?.remove()
+ content?.find("div.thumb")?.remove()
+ content?.find("div.reflist")?.remove()
+ content?.find("div.navbox")?.remove()
+ content?.find("table.infobox")?.remove()
+ content?.find("div.sister-wikipedia")?.remove()
+ content?.find("div.sister-projects")?.remove()
+ content?.find("div.metadata")?.remove()
+ content?.find("div.vertical-navbox")?.remove()
+ content?.find("div.toc")?.remove()
+ const newHtml = content?.html()
+
+ return `TITLE: ${title?.text()}
${newHtml}
`
+}