getCoreContentText for any websites using https://github.com/mozilla/…

…readability

package.json

@@ -19,6 +19,7 @@
     "lint"
   ],
   "dependencies": {
+    "@mozilla/readability": "^0.5.0",
     "@nem035/gpt-3-encoder": "^1.1.7",
     "@picocss/pico": "^1.5.9",
     "@primer/octicons-react": "^18.3.0",

src/utils/get-core-content-text.mjs

@@ -1,9 +1,5 @@
 import { getPossibleElementByQuerySelector } from './get-possible-element-by-query-selector.mjs'
-
-function getArea(e) {
-  const rect = e.getBoundingClientRect()
-  return rect.width * rect.height
-}
+import { Readability } from "@mozilla/readability"
 
 const adapters = {
   'scholar.google': ['#gs_res_ccl_mid'],
@@ -17,31 +13,6 @@ const adapters = {
   'new.qq.com': ['.content-article'],
 }
 
-function findLargestElement(e) {
-  if (!e) {
-    return null
-  }
-  let maxArea = 0
-  let largestElement = null
-  const limitedArea = 0.8 * getArea(e)
-
-  function traverseDOM(node) {
-    if (node.nodeType === Node.ELEMENT_NODE) {
-      const area = getArea(node)
-
-      if (area > maxArea && area < limitedArea) {
-        maxArea = area
-        largestElement = node
-      }
-
-      Array.from(node.children).forEach(traverseDOM)
-    }
-  }
-
-  traverseDOM(e)
-  return largestElement
-}
-
 export function getCoreContentText() {
   function getTextFrom(e) {
     return e.innerText || e.textContent
@@ -60,24 +31,10 @@ export function getCoreContentText() {
     return getTextFrom(element)
   }
 
-  const largestElement = findLargestElement(document.body)
-  const secondLargestElement = findLargestElement(largestElement)
-  console.log(largestElement)
-  console.log(secondLargestElement)
-
-  let ret
-  if (!largestElement) {
-    ret = getTextFrom(document.body)
-    console.log('use document.body')
-  } else if (
-    secondLargestElement &&
-    getArea(secondLargestElement) > 0.5 * getArea(largestElement)
-  ) {
-    ret = getTextFrom(secondLargestElement)
-    console.log('use second')
-  } else {
-    ret = getTextFrom(largestElement)
-    console.log('use first')
-  }
-  return ret.trim().replaceAll('  ', '').replaceAll('\n\n', '').replaceAll(',,', '')
+  let article = new Readability(document.cloneNode(true), {
+    keepClasses: true
+  }).parse()
+  let content = article.textContent.trim().replaceAll('  ', '').replaceAll('\t', '').replaceAll('\n\n', '').replaceAll(',,', '')
+  console.log(content)
+  return content
 }