Upload folder using huggingface_hub

models.py

@@ -8,7 +8,7 @@ import tldextract
 from urllib.parse import urlparse, urljoin, urlunparse
 from datetime import datetime
 from typing import Dict, List, Any, Optional, Set, Tuple
-from pydantic import BaseModel, Field, HttpUrl, validator
+from pydantic import BaseModel, Field, HttpUrl, field_validator
 from enum import Enum
 import logging
 
@@ -50,14 +50,14 @@ class URL(BaseModel):
     error: Optional[str] = None  # Error message if failed
     metadata: Dict[str, Any] = Field(default_factory=dict)  # Additional metadata
 
-    @validator("normalized_url", pre=True, always=True)
+    @field_validator("normalized_url", mode="before")
     def set_normalized_url(cls, v, values):
         """Normalize the URL if not already set"""
         if not v and "url" in values:
             return normalize_url(values["url"])
         return v
 
-    @validator("domain", pre=True, always=True)
+    @field_validator("domain", mode="before")
     def set_domain(cls, v, values):
         """Extract domain from URL if not already set"""
         if not v and "url" in values:
@@ -65,6 +65,9 @@ class URL(BaseModel):
             return f"{parsed.domain}.{parsed.suffix}" if parsed.suffix else parsed.domain
         return v
 
+    class Config:
+        arbitrary_types_allowed = True
+
 
 class RobotsInfo(BaseModel):
     """Information from robots.txt for a domain"""
@@ -75,6 +78,9 @@ class RobotsInfo(BaseModel):
     user_agents: Dict[str, Dict[str, Any]] = Field(default_factory=dict)  # Info per user agent
     status_code: Optional[int] = None  # HTTP status code when fetching robots.txt
 
+    class Config:
+        arbitrary_types_allowed = True
+
 
 class Page(BaseModel):
     """Web page model with content and metadata"""
@@ -92,6 +98,9 @@ class Page(BaseModel):
     is_duplicate: bool = False  # Whether this is duplicate content
     metadata: Dict[str, Any] = Field(default_factory=dict)  # Additional metadata
 
+    class Config:
+        arbitrary_types_allowed = True
+
 
 class DomainStats(BaseModel):
     """Statistics for a domain"""
@@ -104,6 +113,9 @@ class DomainStats(BaseModel):
     crawl_times: List[float] = Field(default_factory=list)  # Recent crawl times
     errors: Dict[int, int] = Field(default_factory=dict)  # Status code counts for errors
 
+    class Config:
+        arbitrary_types_allowed = True
+
 
 def normalize_url(url: str) -> str:
     """

requirements.txt

@@ -1,24 +1,24 @@
 # Core dependencies
-requests==2.31.0
-beautifulsoup4==4.12.3
-aiohttp==3.9.3
+requests>=2.31.0
+beautifulsoup4>=4.12.0
+aiohttp>=3.9.0
 lxml==4.9.2
 html5lib==1.1
-pydantic==1.10.7
-pymongo==4.6.1
-redis==5.0.1
+pydantic>=2.0,<3.0
+pymongo>=4.6.0
+redis>=5.0.0
 boto3==1.26.123
 docopt==0.6.2
 
 # URL and DNS handling
 dnspython==2.3.0
-tldextract==5.1.1
+tldextract>=5.1.1
 validators==0.20.0
 robotexclusionrulesparser==1.7.1
 urllib3==1.26.15
 
 # Monitoring and metrics
-prometheus-client==0.16.0
+prometheus-client>=0.19.0
 
 # HTML processing
 html2text==2020.1.16
@@ -28,16 +28,20 @@ anyio==3.6.2
 asyncio==3.4.3
 
 # Utilities
-python-dateutil==2.8.2
+python-dateutil>=2.8.2
 pytz==2023.3
 retry==0.9.2
 cryptography==40.0.1
 cachetools==5.3.0
 
 # Added from the code block
-openai==1.12.0
-gradio==4.16.0
+openai>=1.12.0
+gradio>=4.16.0
 chardet==5.2.0 
 
 # Dotenv
-python-dotenv
+python-dotenv>=1.0.0
+
+# New dependencies
+mmh3>=4.0.0
+httpx>=0.26.0