:recycle: [Refactor] Move hardcoded consts to network_configs

documents/webpage_content_extractor.py

@@ -5,6 +5,7 @@ from bs4 import BeautifulSoup, Comment, NavigableString, Tag
 from tiktoken import get_encoding as tiktoken_get_encoding
 from utils.logger import logger
 from markdownify import markdownify
+from networks.network_configs import IGNORE_CLASSES
 
 # from trafilatura import extract as extract_text_from_html
 # from inscriptis import get_text as extract_text_from_html
@@ -26,17 +27,7 @@ class WebpageContentExtractor:
 
         ignore_tags = ["script", "style", "button"]
 
-        ignore_classes = [
-            "sidebar",
-            "footer",
-            "related",
-            "comment",
-            "topbar",
-            "menu",
-            "offcanvas",
-            "navbar",
-        ]
-        ignore_classes_pattern = f'{"|".join(ignore_classes)}'
+        ignore_classes_pattern = f'{"|".join(IGNORE_CLASSES)}'
         removed_element_counts = 0
         for element in soup.find_all():
             class_str = ""
@@ -61,10 +52,12 @@ class WebpageContentExtractor:
                 or (re.search(ignore_classes_pattern, class_str, flags=re.IGNORECASE))
                 or (re.search(ignore_classes_pattern, id_str, flags=re.IGNORECASE))
             ):
-                # try:
-                #     logger.note(f"Removing:\n{element}")
-                # except:
-                #     logger.note(f"Removing unknown element")
+                try:
+                    logger.note(f"Removing:\n{element}")
+                    logger.warn(class_str)
+                except:
+                    # logger.note(f"Removing unknown element")
+                    pass
                 element.decompose()
                 removed_element_counts += 1
 
@@ -76,9 +69,14 @@ class WebpageContentExtractor:
         return html_str
 
     def extract(self, html_path):
-        logger.note(f"Extracing content from:{html_path}")
-        with open(html_path, "r", encoding="utf-8") as f:
-            html_str = f.read()
+        logger.note(f"Extracting content from: {html_path}")
+
+        if not Path(html_path).exists():
+            logger.warn(f"File not found: {html_path}")
+            return ""
+
+        with open(html_path, "r", encoding="utf-8") as rf:
+            html_str = rf.read()
 
         html_str = self.filter_html_str(html_str)
 
@@ -108,8 +106,9 @@ if __name__ == "__main__":
         / "files"
         / "urls"
         # / "stackoverflow.com_questions_295135_turn-a-string-into-a-valid-filename.html"
-        / "www.liaoxuefeng.com_wiki_1016959663602400_1017495723838528.html"
+        # / "www.liaoxuefeng.com_wiki_1016959663602400_1017495723838528.html"
         # / "docs.python.org_zh-cn_3_tutorial_interpreter.html"
+        / "zh.wikipedia.org_zh-hans_%E7%94%B0%E4%B8%AD%E6%9F%A0%E6%AA%AC.html"
     )
     extractor = WebpageContentExtractor()
     main_content = extractor.extract(html_path)

networks/google_searcher.py

@@ -3,6 +3,7 @@ from pathlib import Path
 from utils.enver import enver
 from utils.logger import logger
 from networks.filepath_converter import QueryToFilepathConverter
+from networks.network_configs import REQUESTS_HEADERS
 
 
 class GoogleSearcher:
@@ -16,9 +17,7 @@ class GoogleSearcher:
     def send_request(self, result_num=10, safe=False):
         self.request_response = requests.get(
             url=self.url,
-            headers={
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62",
-            },
+            headers=REQUESTS_HEADERS,
             params={
                 "q": self.query,
                 "num": result_num,

networks/html_fetcher.py

@@ -4,8 +4,7 @@ from pathlib import Path
 from utils.enver import enver
 from utils.logger import logger
 from networks.filepath_converter import UrlToFilepathConverter
-
-IGNORE_HOSTS = ["weibo.com"]
+from networks.network_configs import IGNORE_HOSTS, REQUESTS_HEADERS
 
 
 class HTMLFetcher:
@@ -24,9 +23,7 @@ class HTMLFetcher:
     def send_request(self):
         self.request_response = requests.get(
             url=self.url,
-            headers={
-                "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62",
-            },
+            headers=REQUESTS_HEADERS,
             proxies=self.enver.requests_proxies,
         )
 

networks/network_configs.py

@@ -0,0 +1,20 @@
+IGNORE_CLASSES = [
+    "sidebar",
+    "footer",
+    "related",
+    "comment",
+    "topbar",
+    # "menu",
+    "offcanvas",
+    "navbar",
+    "post_side",
+]
+
+IGNORE_HOSTS = [
+    "weibo.com",
+    "hymson.com",
+]
+
+REQUESTS_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/111.0.0.0 Safari/537.36 Edg/111.0.1661.62",
+}