web_reader_tool.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339
  1. import hashlib
  2. import json
  3. import mimetypes
  4. import os
  5. import re
  6. import site
  7. import subprocess
  8. import tempfile
  9. import unicodedata
  10. from contextlib import contextmanager
  11. from urllib.parse import unquote
  12. import cloudscraper
  13. from bs4 import BeautifulSoup, CData, Comment, NavigableString
  14. from regex import regex
  15. from core.helper import ssrf_proxy
  16. from core.rag.extractor import extract_processor
  17. from core.rag.extractor.extract_processor import ExtractProcessor
  18. FULL_TEMPLATE = """
  19. TITLE: {title}
  20. AUTHORS: {authors}
  21. PUBLISH DATE: {publish_date}
  22. TOP_IMAGE_URL: {top_image}
  23. TEXT:
  24. {text}
  25. """
  26. def page_result(text: str, cursor: int, max_length: int) -> str:
  27. """Page through `text` and return a substring of `max_length` characters starting from `cursor`."""
  28. return text[cursor: cursor + max_length]
  29. def get_url(url: str, user_agent: str = None) -> str:
  30. """Fetch URL and return the contents as a string."""
  31. headers = {
  32. "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
  33. }
  34. if user_agent:
  35. headers["User-Agent"] = user_agent
  36. main_content_type = None
  37. supported_content_types = extract_processor.SUPPORT_URL_CONTENT_TYPES + ["text/html"]
  38. response = ssrf_proxy.head(url, headers=headers, follow_redirects=True, timeout=(5, 10))
  39. if response.status_code == 200:
  40. # check content-type
  41. content_type = response.headers.get('Content-Type')
  42. if content_type:
  43. main_content_type = response.headers.get('Content-Type').split(';')[0].strip()
  44. else:
  45. content_disposition = response.headers.get('Content-Disposition', '')
  46. filename_match = re.search(r'filename="([^"]+)"', content_disposition)
  47. if filename_match:
  48. filename = unquote(filename_match.group(1))
  49. extension = re.search(r'\.(\w+)$', filename)
  50. if extension:
  51. main_content_type = mimetypes.guess_type(filename)[0]
  52. if main_content_type not in supported_content_types:
  53. return "Unsupported content-type [{}] of URL.".format(main_content_type)
  54. if main_content_type in extract_processor.SUPPORT_URL_CONTENT_TYPES:
  55. return ExtractProcessor.load_from_url(url, return_text=True)
  56. response = ssrf_proxy.get(url, headers=headers, follow_redirects=True, timeout=(120, 300))
  57. elif response.status_code == 403:
  58. scraper = cloudscraper.create_scraper()
  59. scraper.perform_request = ssrf_proxy.make_request
  60. response = scraper.get(url, headers=headers, follow_redirects=True, timeout=(120, 300))
  61. if response.status_code != 200:
  62. return "URL returned status code {}.".format(response.status_code)
  63. a = extract_using_readabilipy(response.text)
  64. if not a['plain_text'] or not a['plain_text'].strip():
  65. return ''
  66. res = FULL_TEMPLATE.format(
  67. title=a['title'],
  68. authors=a['byline'],
  69. publish_date=a['date'],
  70. top_image="",
  71. text=a['plain_text'] if a['plain_text'] else "",
  72. )
  73. return res
  74. def extract_using_readabilipy(html):
  75. with tempfile.NamedTemporaryFile(delete=False, mode='w+') as f_html:
  76. f_html.write(html)
  77. f_html.close()
  78. html_path = f_html.name
  79. # Call Mozilla's Readability.js Readability.parse() function via node, writing output to a temporary file
  80. article_json_path = html_path + ".json"
  81. jsdir = os.path.join(find_module_path('readabilipy'), 'javascript')
  82. with chdir(jsdir):
  83. subprocess.check_call(["node", "ExtractArticle.js", "-i", html_path, "-o", article_json_path])
  84. # Read output of call to Readability.parse() from JSON file and return as Python dictionary
  85. with open(article_json_path, encoding="utf-8") as json_file:
  86. input_json = json.loads(json_file.read())
  87. # Deleting files after processing
  88. os.unlink(article_json_path)
  89. os.unlink(html_path)
  90. article_json = {
  91. "title": None,
  92. "byline": None,
  93. "date": None,
  94. "content": None,
  95. "plain_content": None,
  96. "plain_text": None
  97. }
  98. # Populate article fields from readability fields where present
  99. if input_json:
  100. if input_json.get("title"):
  101. article_json["title"] = input_json["title"]
  102. if input_json.get("byline"):
  103. article_json["byline"] = input_json["byline"]
  104. if input_json.get("date"):
  105. article_json["date"] = input_json["date"]
  106. if input_json.get("content"):
  107. article_json["content"] = input_json["content"]
  108. article_json["plain_content"] = plain_content(article_json["content"], False, False)
  109. article_json["plain_text"] = extract_text_blocks_as_plain_text(article_json["plain_content"])
  110. if input_json.get("textContent"):
  111. article_json["plain_text"] = input_json["textContent"]
  112. article_json["plain_text"] = re.sub(r'\n\s*\n', '\n', article_json["plain_text"])
  113. return article_json
  114. def find_module_path(module_name):
  115. for package_path in site.getsitepackages():
  116. potential_path = os.path.join(package_path, module_name)
  117. if os.path.exists(potential_path):
  118. return potential_path
  119. return None
  120. @contextmanager
  121. def chdir(path):
  122. """Change directory in context and return to original on exit"""
  123. # From https://stackoverflow.com/a/37996581, couldn't find a built-in
  124. original_path = os.getcwd()
  125. os.chdir(path)
  126. try:
  127. yield
  128. finally:
  129. os.chdir(original_path)
  130. def extract_text_blocks_as_plain_text(paragraph_html):
  131. # Load article as DOM
  132. soup = BeautifulSoup(paragraph_html, 'html.parser')
  133. # Select all lists
  134. list_elements = soup.find_all(['ul', 'ol'])
  135. # Prefix text in all list items with "* " and make lists paragraphs
  136. for list_element in list_elements:
  137. plain_items = "".join(list(filter(None, [plain_text_leaf_node(li)["text"] for li in list_element.find_all('li')])))
  138. list_element.string = plain_items
  139. list_element.name = "p"
  140. # Select all text blocks
  141. text_blocks = [s.parent for s in soup.find_all(string=True)]
  142. text_blocks = [plain_text_leaf_node(block) for block in text_blocks]
  143. # Drop empty paragraphs
  144. text_blocks = list(filter(lambda p: p["text"] is not None, text_blocks))
  145. return text_blocks
  146. def plain_text_leaf_node(element):
  147. # Extract all text, stripped of any child HTML elements and normalise it
  148. plain_text = normalise_text(element.get_text())
  149. if plain_text != "" and element.name == "li":
  150. plain_text = "* {}, ".format(plain_text)
  151. if plain_text == "":
  152. plain_text = None
  153. if "data-node-index" in element.attrs:
  154. plain = {"node_index": element["data-node-index"], "text": plain_text}
  155. else:
  156. plain = {"text": plain_text}
  157. return plain
  158. def plain_content(readability_content, content_digests, node_indexes):
  159. # Load article as DOM
  160. soup = BeautifulSoup(readability_content, 'html.parser')
  161. # Make all elements plain
  162. elements = plain_elements(soup.contents, content_digests, node_indexes)
  163. if node_indexes:
  164. # Add node index attributes to nodes
  165. elements = [add_node_indexes(element) for element in elements]
  166. # Replace article contents with plain elements
  167. soup.contents = elements
  168. return str(soup)
  169. def plain_elements(elements, content_digests, node_indexes):
  170. # Get plain content versions of all elements
  171. elements = [plain_element(element, content_digests, node_indexes)
  172. for element in elements]
  173. if content_digests:
  174. # Add content digest attribute to nodes
  175. elements = [add_content_digest(element) for element in elements]
  176. return elements
  177. def plain_element(element, content_digests, node_indexes):
  178. # For lists, we make each item plain text
  179. if is_leaf(element):
  180. # For leaf node elements, extract the text content, discarding any HTML tags
  181. # 1. Get element contents as text
  182. plain_text = element.get_text()
  183. # 2. Normalise the extracted text string to a canonical representation
  184. plain_text = normalise_text(plain_text)
  185. # 3. Update element content to be plain text
  186. element.string = plain_text
  187. elif is_text(element):
  188. if is_non_printing(element):
  189. # The simplified HTML may have come from Readability.js so might
  190. # have non-printing text (e.g. Comment or CData). In this case, we
  191. # keep the structure, but ensure that the string is empty.
  192. element = type(element)("")
  193. else:
  194. plain_text = element.string
  195. plain_text = normalise_text(plain_text)
  196. element = type(element)(plain_text)
  197. else:
  198. # If not a leaf node or leaf type call recursively on child nodes, replacing
  199. element.contents = plain_elements(element.contents, content_digests, node_indexes)
  200. return element
  201. def add_node_indexes(element, node_index="0"):
  202. # Can't add attributes to string types
  203. if is_text(element):
  204. return element
  205. # Add index to current element
  206. element["data-node-index"] = node_index
  207. # Add index to child elements
  208. for local_idx, child in enumerate(
  209. [c for c in element.contents if not is_text(c)], start=1):
  210. # Can't add attributes to leaf string types
  211. child_index = "{stem}.{local}".format(
  212. stem=node_index, local=local_idx)
  213. add_node_indexes(child, node_index=child_index)
  214. return element
  215. def normalise_text(text):
  216. """Normalise unicode and whitespace."""
  217. # Normalise unicode first to try and standardise whitespace characters as much as possible before normalising them
  218. text = strip_control_characters(text)
  219. text = normalise_unicode(text)
  220. text = normalise_whitespace(text)
  221. return text
  222. def strip_control_characters(text):
  223. """Strip out unicode control characters which might break the parsing."""
  224. # Unicode control characters
  225. # [Cc]: Other, Control [includes new lines]
  226. # [Cf]: Other, Format
  227. # [Cn]: Other, Not Assigned
  228. # [Co]: Other, Private Use
  229. # [Cs]: Other, Surrogate
  230. control_chars = {'Cc', 'Cf', 'Cn', 'Co', 'Cs'}
  231. retained_chars = ['\t', '\n', '\r', '\f']
  232. # Remove non-printing control characters
  233. return "".join(["" if (unicodedata.category(char) in control_chars) and (char not in retained_chars) else char for char in text])
  234. def normalise_unicode(text):
  235. """Normalise unicode such that things that are visually equivalent map to the same unicode string where possible."""
  236. normal_form = "NFKC"
  237. text = unicodedata.normalize(normal_form, text)
  238. return text
  239. def normalise_whitespace(text):
  240. """Replace runs of whitespace characters with a single space as this is what happens when HTML text is displayed."""
  241. text = regex.sub(r"\s+", " ", text)
  242. # Remove leading and trailing whitespace
  243. text = text.strip()
  244. return text
  245. def is_leaf(element):
  246. return (element.name in ['p', 'li'])
  247. def is_text(element):
  248. return isinstance(element, NavigableString)
  249. def is_non_printing(element):
  250. return any(isinstance(element, _e) for _e in [Comment, CData])
  251. def add_content_digest(element):
  252. if not is_text(element):
  253. element["data-content-digest"] = content_digest(element)
  254. return element
  255. def content_digest(element):
  256. if is_text(element):
  257. # Hash
  258. trimmed_string = element.string.strip()
  259. if trimmed_string == "":
  260. digest = ""
  261. else:
  262. digest = hashlib.sha256(trimmed_string.encode('utf-8')).hexdigest()
  263. else:
  264. contents = element.contents
  265. num_contents = len(contents)
  266. if num_contents == 0:
  267. # No hash when no child elements exist
  268. digest = ""
  269. elif num_contents == 1:
  270. # If single child, use digest of child
  271. digest = content_digest(contents[0])
  272. else:
  273. # Build content digest from the "non-empty" digests of child nodes
  274. digest = hashlib.sha256()
  275. child_digests = list(
  276. filter(lambda x: x != "", [content_digest(content) for content in contents]))
  277. for child in child_digests:
  278. digest.update(child.encode('utf-8'))
  279. digest = digest.hexdigest()
  280. return digest