Spaces:
Paused
Paused
:recycle: [Refactor] HTMLFetcher: replace save_path with output_path
Browse files- networks/html_fetcher.py +9 -6
networks/html_fetcher.py
CHANGED
|
@@ -21,24 +21,27 @@ class HTMLFetcher:
|
|
| 21 |
)
|
| 22 |
|
| 23 |
def save_response(self):
|
| 24 |
-
self.
|
| 25 |
-
if not self.
|
| 26 |
-
self.
|
| 27 |
|
| 28 |
-
logger.success(f"Saving to: [{self.
|
| 29 |
|
| 30 |
-
with open(self.
|
| 31 |
wf.write(self.request_response.content)
|
| 32 |
|
| 33 |
def fetch(self, url):
|
| 34 |
self.url = url
|
| 35 |
self.send_request()
|
| 36 |
self.save_response()
|
|
|
|
| 37 |
|
| 38 |
|
| 39 |
if __name__ == "__main__":
|
| 40 |
url = (
|
| 41 |
-
"https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename"
|
|
|
|
|
|
|
| 42 |
)
|
| 43 |
fetcher = HTMLFetcher()
|
| 44 |
fetcher.fetch(url)
|
|
|
|
| 21 |
)
|
| 22 |
|
| 23 |
def save_response(self):
|
| 24 |
+
self.output_path = UrlToFilepathConverter().convert(self.url)
|
| 25 |
+
if not self.output_path.exists():
|
| 26 |
+
self.output_path.parent.mkdir(parents=True, exist_ok=True)
|
| 27 |
|
| 28 |
+
logger.success(f"Saving to: [{self.output_path}]")
|
| 29 |
|
| 30 |
+
with open(self.output_path, "wb") as wf:
|
| 31 |
wf.write(self.request_response.content)
|
| 32 |
|
| 33 |
def fetch(self, url):
|
| 34 |
self.url = url
|
| 35 |
self.send_request()
|
| 36 |
self.save_response()
|
| 37 |
+
return self.output_path
|
| 38 |
|
| 39 |
|
| 40 |
if __name__ == "__main__":
|
| 41 |
url = (
|
| 42 |
+
# "https://stackoverflow.com/questions/295135/turn-a-string-into-a-valid-filename"
|
| 43 |
+
# "https://www.liaoxuefeng.com/wiki/1016959663602400/1017495723838528"
|
| 44 |
+
"https://docs.python.org/zh-cn/3/tutorial/interpreter.html"
|
| 45 |
)
|
| 46 |
fetcher = HTMLFetcher()
|
| 47 |
fetcher.fetch(url)
|