Add Hugging Face dataset integration

✨ New Features:
- Integrated load_dataset() for HF Hub compatibility
- Added factory methods for easy loader creation
- Implemented automatic fallback (HF → local files)
- Added dynamic data source switching
- Created comprehensive test suite

🔧 Technical improvements:
- Enhanced AgriculturalDataLoader with use_hf parameter
- Added load_from_huggingface() method
- Separated local file loading logic
- Added data source switching capabilities
- Removed deprecated trust_remote_code parameter

📚 Documentation:
- Added test_data_sources.py with comprehensive tests
- Created hf_usage_example.py with practical examples
- Updated gradio_app.py to use HF integration
- Added factory methods for different use cases

🚀 Production ready:
- Automatic fallback ensures reliability
- Environment-based configuration
- Compatible with HF Spaces deployment
- Maintains backward compatibility

data_loader.py

@@ -8,24 +8,64 @@ import numpy as np
 from pathlib import Path
 from typing import List, Dict, Optional, Union
 import os
-from datasets import Dataset
+from datasets import Dataset, load_dataset
 from huggingface_hub import HfApi
 
 
 class AgriculturalDataLoader:
     """Loads and preprocesses agricultural intervention data."""
     
-    def __init__(self, data_path: str = None, hf_token: str = None, dataset_id: str = None):
+    def __init__(self, data_path: str = None, hf_token: str = None, dataset_id: str = None, use_hf: bool = False):
         self.data_path = data_path or "/Users/tracyandre/Downloads/OneDrive_1_9-17-2025"
         self.hf_token = hf_token or os.environ.get("HF_TOKEN")
         self.dataset_id = dataset_id or "HackathonCRA/2024"
+        self.use_hf = use_hf  # Flag to use Hugging Face dataset
         self.data_cache = {}
         
     def load_all_files(self) -> pd.DataFrame:
         """Load all intervention files and combine them."""
         if 'combined_data' in self.data_cache:
             return self.data_cache['combined_data']
+        
+        # Try to load from Hugging Face first if enabled
+        if self.use_hf:
+            try:
+                df = self.load_from_huggingface()
+                self.data_cache['combined_data'] = df
+                return df
+            except Exception as e:
+                print(f"Failed to load from Hugging Face: {e}")
+                print("Falling back to local files...")
+        
+        # Load from local files (original method)
+        return self._load_local_files()
+    
+    def load_from_huggingface(self) -> pd.DataFrame:
+        """Load data from Hugging Face dataset."""
+        print(f"🤗 Loading dataset from Hugging Face: {self.dataset_id}")
+        
+        try:
+            dataset = load_dataset(
+                self.dataset_id,
+                token=self.hf_token,
+            )
             
+            # Convert to pandas DataFrame
+            df = dataset["train"].to_pandas()
+            print(f"✅ Successfully loaded {len(df)} records from Hugging Face")
+            
+            # Apply preprocessing if needed
+            df = self._preprocess_data(df)
+            
+            return df
+            
+        except Exception as e:
+            raise ValueError(f"Failed to load dataset from Hugging Face: {e}")
+    
+    def _load_local_files(self) -> pd.DataFrame:
+        """Load data from local CSV/Excel files."""
+        print(f"📁 Loading local files from: {self.data_path}")
+        
         data_files = []
         data_path = Path(self.data_path)
         
@@ -57,7 +97,6 @@ class AgriculturalDataLoader:
         if all_dataframes:
             combined_df = pd.concat(all_dataframes, ignore_index=True)
             combined_df = self._preprocess_data(combined_df)
-            self.data_cache['combined_data'] = combined_df
             return combined_df
         else:
             raise ValueError("No data files found")
@@ -160,3 +199,50 @@ class AgriculturalDataLoader:
         )
         
         return f"Data uploaded to {self.dataset_id}"
+    
+    def set_data_source(self, use_hf: bool = True, clear_cache: bool = True):
+        """
+        Switch between Hugging Face and local file data sources.
+        
+        Args:
+            use_hf: If True, use Hugging Face dataset. If False, use local files.
+            clear_cache: If True, clear cached data to force reload from new source.
+        """
+        self.use_hf = use_hf
+        if clear_cache:
+            self.data_cache.clear()
+            print(f"📋 Switched to {'Hugging Face' if use_hf else 'local files'} data source")
+    
+    @classmethod
+    def create_hf_loader(cls, dataset_id: str = "HackathonCRA/2024", hf_token: str = None):
+        """
+        Factory method to create a loader configured for Hugging Face.
+        
+        Args:
+            dataset_id: Hugging Face dataset identifier
+            hf_token: Hugging Face token (optional, will use environment variable)
+            
+        Returns:
+            AgriculturalDataLoader configured for HF
+        """
+        return cls(
+            dataset_id=dataset_id,
+            hf_token=hf_token,
+            use_hf=True
+        )
+    
+    @classmethod  
+    def create_local_loader(cls, data_path: str):
+        """
+        Factory method to create a loader configured for local files.
+        
+        Args:
+            data_path: Path to local data directory
+            
+        Returns:
+            AgriculturalDataLoader configured for local files
+        """
+        return cls(
+            data_path=data_path,
+            use_hf=False
+        )

gradio_app.py

@@ -15,7 +15,14 @@ from analysis_tools import AgriculturalAnalyzer
 
 
 # Initialize components
-data_loader = AgriculturalDataLoader()
+# Try to use Hugging Face first, fallback to local files
+try:
+    data_loader = AgriculturalDataLoader(use_hf=True)
+    print("🤗 Configured to use Hugging Face dataset with local fallback")
+except:
+    data_loader = AgriculturalDataLoader(use_hf=False)
+    print("📁 Configured to use local files only")
+
 analyzer = AgriculturalAnalyzer(data_loader)
 
 # Global state for data

hf_usage_example.py

@@ -0,0 +1,214 @@
+#!/usr/bin/env python3
+"""
+Example usage of the agricultural data loader with Hugging Face integration.
+Shows different ways to load and use the data.
+"""
+
+import os
+import warnings
+warnings.filterwarnings('ignore')
+
+from data_loader import AgriculturalDataLoader
+from analysis_tools import AgriculturalAnalyzer
+
+def example_local_usage():
+    """Example: Load from local files."""
+    print("📁 EXAMPLE 1: Loading from local files")
+    print("-" * 40)
+    
+    # Create loader for local files
+    loader = AgriculturalDataLoader.create_local_loader(
+        data_path="/Users/tracyandre/Downloads/OneDrive_1_9-17-2025"
+    )
+    
+    # Load and analyze data
+    df = loader.load_all_files()
+    print(f"✅ Loaded {len(df):,} records from local files")
+    
+    # Basic analysis
+    analyzer = AgriculturalAnalyzer(loader)
+    trends = analyzer.analyze_weed_pressure_trends()
+    print(f"📊 Average IFT: {trends['summary']['mean_ift']:.2f}")
+    
+    return df
+
+def example_hf_usage():
+    """Example: Load from Hugging Face (if available)."""
+    print("\n🤗 EXAMPLE 2: Loading from Hugging Face")
+    print("-" * 40)
+    
+    # Check if HF token is available
+    if not os.environ.get("HF_TOKEN"):
+        print("⚠️  No HF_TOKEN found - skipping HF example")
+        print("💡 Set HF_TOKEN environment variable to use this feature")
+        return None
+    
+    try:
+        # Create loader for Hugging Face
+        loader = AgriculturalDataLoader.create_hf_loader(
+            dataset_id="HackathonCRA/2024"
+        )
+        
+        # Load and analyze data
+        df = loader.load_all_files()
+        print(f"✅ Loaded {len(df):,} records from Hugging Face")
+        
+        # Basic analysis
+        analyzer = AgriculturalAnalyzer(loader)
+        trends = analyzer.analyze_weed_pressure_trends()
+        print(f"📊 Average IFT: {trends['summary']['mean_ift']:.2f}")
+        
+        return df
+        
+    except Exception as e:
+        print(f"❌ Failed to load from Hugging Face: {e}")
+        return None
+
+def example_automatic_fallback():
+    """Example: Automatic fallback from HF to local."""
+    print("\n🔄 EXAMPLE 3: Automatic fallback")
+    print("-" * 40)
+    
+    # Create loader with HF preferred but local fallback
+    loader = AgriculturalDataLoader(
+        data_path="/Users/tracyandre/Downloads/OneDrive_1_9-17-2025",
+        dataset_id="HackathonCRA/2024",
+        use_hf=True  # Try HF first
+    )
+    
+    # This will try HF first, then fallback to local if needed
+    df = loader.load_all_files()
+    print(f"✅ Loaded {len(df):,} records (with automatic source selection)")
+    
+    return df
+
+def example_dynamic_switching():
+    """Example: Dynamic switching between sources."""
+    print("\n🔀 EXAMPLE 4: Dynamic source switching")
+    print("-" * 40)
+    
+    # Create loader
+    loader = AgriculturalDataLoader(
+        data_path="/Users/tracyandre/Downloads/OneDrive_1_9-17-2025",
+        dataset_id="HackathonCRA/2024"
+    )
+    
+    # Load from local first
+    loader.set_data_source(use_hf=False)
+    df_local = loader.load_all_files()
+    print(f"📁 Local source: {len(df_local):,} records")
+    
+    # Switch to HF (if available)
+    if os.environ.get("HF_TOKEN"):
+        try:
+            loader.set_data_source(use_hf=True)
+            df_hf = loader.load_all_files()
+            print(f"🤗 HF source: {len(df_hf):,} records")
+            
+            # Compare
+            if len(df_local) == len(df_hf):
+                print("✅ Data consistency verified")
+            else:
+                print(f"⚠️  Data mismatch: {abs(len(df_local) - len(df_hf))} record difference")
+                
+        except Exception as e:
+            print(f"🤗 HF switching failed: {e}")
+    else:
+        print("⚠️  No HF_TOKEN - skipping HF switch test")
+    
+    return df_local
+
+def example_production_deployment():
+    """Example: Production deployment configuration."""
+    print("\n🚀 EXAMPLE 5: Production deployment setup")
+    print("-" * 40)
+    
+    # Production configuration
+    # This is how you'd set it up for Hugging Face Spaces deployment
+    
+    print("💡 For Hugging Face Spaces deployment:")
+    print("1. Set HF_TOKEN as a Space secret")
+    print("2. Configure the loader as follows:")
+    print()
+    
+    config_code = '''
+# In your app.py or gradio_app.py
+import os
+from data_loader import AgriculturalDataLoader
+
+# Production configuration
+hf_token = os.environ.get("HF_TOKEN")
+dataset_id = "HackathonCRA/2024"
+
+if hf_token:
+    # Use HF dataset in production
+    data_loader = AgriculturalDataLoader.create_hf_loader(
+        dataset_id=dataset_id,
+        hf_token=hf_token
+    )
+    print("🤗 Using Hugging Face dataset")
+else:
+    # Fallback for local development
+    data_loader = AgriculturalDataLoader.create_local_loader(
+        data_path="./data"  # Local data directory
+    )
+    print("📁 Using local files")
+'''
+    
+    print(config_code)
+    
+    # Example of actual production setup
+    try:
+        hf_token = os.environ.get("HF_TOKEN")
+        if hf_token:
+            loader = AgriculturalDataLoader.create_hf_loader("HackathonCRA/2024", hf_token)
+            print("✅ Production setup: HF dataset configured")
+        else:
+            loader = AgriculturalDataLoader.create_local_loader("/Users/tracyandre/Downloads/OneDrive_1_9-17-2025")
+            print("✅ Development setup: Local files configured")
+            
+        df = loader.load_all_files()
+        print(f"📊 Ready for production: {len(df):,} records available")
+        
+    except Exception as e:
+        print(f"❌ Production setup failed: {e}")
+
+def main():
+    """Run all examples."""
+    print("🚜 AGRICULTURAL DATA LOADER - USAGE EXAMPLES")
+    print("=" * 60)
+    
+    # Run examples
+    example_local_usage()
+    example_hf_usage()
+    example_automatic_fallback()
+    example_dynamic_switching()
+    example_production_deployment()
+    
+    print("\n" + "=" * 60)
+    print("🎯 SUMMARY")
+    print("=" * 60)
+    print("""
+The AgriculturalDataLoader now supports:
+
+✅ Local file loading (CSV/Excel)
+✅ Hugging Face dataset loading
+✅ Automatic fallback (HF → Local)
+✅ Dynamic source switching
+✅ Production deployment ready
+
+Key benefits:
+🔄 Seamless data source switching
+🚀 Cloud deployment ready
+📊 Same analysis tools work with both sources
+🔧 Easy configuration management
+    """)
+    
+    print("🛠️  Next steps:")
+    print("1. Upload your dataset to Hugging Face Hub")
+    print("2. Set HF_TOKEN environment variable")
+    print("3. Deploy to Hugging Face Spaces")
+    print("4. Enjoy cloud-based agricultural analysis!")
+
+if __name__ == "__main__":
+    main()

test_data_sources.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""
+Test script to demonstrate loading data from both local files and Hugging Face.
+"""
+
+import warnings
+warnings.filterwarnings('ignore')
+
+from data_loader import AgriculturalDataLoader
+import os
+
+def test_local_loading():
+    """Test loading from local files."""
+    print("🔍 TESTING LOCAL FILE LOADING")
+    print("=" * 50)
+    
+    try:
+        # Create loader for local files
+        loader = AgriculturalDataLoader.create_local_loader(
+            data_path="/Users/tracyandre/Downloads/OneDrive_1_9-17-2025"
+        )
+        
+        # Load data
+        df = loader.load_all_files()
+        
+        print(f"✅ Local loading successful!")
+        print(f"📊 Records: {len(df):,}")
+        print(f"📅 Years: {sorted(df['year'].unique())}")
+        print(f"🌱 Crops: {df['crop_type'].nunique()}")
+        print(f"📍 Plots: {df['plot_name'].nunique()}")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ Local loading failed: {e}")
+        return False
+
+def test_hf_loading():
+    """Test loading from Hugging Face."""
+    print("\n🤗 TESTING HUGGING FACE LOADING")
+    print("=" * 50)
+    
+    # Check if HF token is available
+    hf_token = os.environ.get("HF_TOKEN")
+    if not hf_token:
+        print("⚠️  No HF_TOKEN found in environment variables")
+        print("💡 Set HF_TOKEN to test Hugging Face loading")
+        return False
+    
+    try:
+        # Create loader for Hugging Face
+        loader = AgriculturalDataLoader.create_hf_loader(
+            dataset_id="HackathonCRA/2024",
+            hf_token=hf_token
+        )
+        
+        # Load data
+        df = loader.load_from_huggingface()
+        
+        print(f"✅ Hugging Face loading successful!")
+        print(f"📊 Records: {len(df):,}")
+        print(f"📅 Years: {sorted(df['year'].unique())}")
+        print(f"🌱 Crops: {df['crop_type'].nunique()}")
+        print(f"📍 Plots: {df['plot_name'].nunique()}")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ Hugging Face loading failed: {e}")
+        print("💡 Make sure the dataset exists and you have access")
+        return False
+
+def test_auto_fallback():
+    """Test automatic fallback from HF to local files."""
+    print("\n🔄 TESTING AUTO FALLBACK (HF -> LOCAL)")
+    print("=" * 50)
+    
+    try:
+        # Create loader with HF enabled but potentially failing
+        loader = AgriculturalDataLoader(
+            data_path="/Users/tracyandre/Downloads/OneDrive_1_9-17-2025",
+            dataset_id="nonexistent-dataset",  # This should fail
+            use_hf=True
+        )
+        
+        # This should try HF first, then fallback to local
+        df = loader.load_all_files()
+        
+        print(f"✅ Auto fallback successful!")
+        print(f"📊 Records: {len(df):,}")
+        print("🔄 Successfully fell back to local files after HF failure")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ Auto fallback failed: {e}")
+        return False
+
+def test_data_source_switching():
+    """Test switching between data sources."""
+    print("\n🔀 TESTING DATA SOURCE SWITCHING")
+    print("=" * 50)
+    
+    try:
+        # Create loader
+        loader = AgriculturalDataLoader(
+            data_path="/Users/tracyandre/Downloads/OneDrive_1_9-17-2025",
+            dataset_id="HackathonCRA/2024"
+        )
+        
+        # Test local loading
+        loader.set_data_source(use_hf=False)
+        df_local = loader.load_all_files()
+        print(f"📁 Local: {len(df_local):,} records")
+        
+        # Test switching to HF (if token available)
+        if os.environ.get("HF_TOKEN"):
+            loader.set_data_source(use_hf=True)
+            try:
+                df_hf = loader.load_all_files()
+                print(f"🤗 HF: {len(df_hf):,} records")
+                
+                # Compare data
+                if len(df_local) == len(df_hf):
+                    print("✅ Data consistency: Same number of records")
+                else:
+                    print(f"⚠️  Data difference: Local={len(df_local)}, HF={len(df_hf)}")
+                    
+            except Exception as e:
+                print(f"🤗 HF loading failed (expected): {e}")
+        else:
+            print("⚠️  No HF_TOKEN - skipping HF test")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ Data source switching failed: {e}")
+        return False
+
+def main():
+    """Run all tests."""
+    print("🚜 AGRICULTURAL DATA LOADER TESTING")
+    print("=" * 60)
+    print()
+    
+    results = []
+    
+    # Test 1: Local loading
+    results.append(("Local Loading", test_local_loading()))
+    
+    # Test 2: Hugging Face loading
+    results.append(("HF Loading", test_hf_loading()))
+    
+    # Test 3: Auto fallback
+    results.append(("Auto Fallback", test_auto_fallback()))
+    
+    # Test 4: Data source switching
+    results.append(("Source Switching", test_data_source_switching()))
+    
+    # Summary
+    print("\n📋 TEST SUMMARY")
+    print("=" * 30)
+    
+    passed = 0
+    for test_name, result in results:
+        status = "✅ PASS" if result else "❌ FAIL"
+        print(f"{test_name:<20} {status}")
+        if result:
+            passed += 1
+    
+    print(f"\n🎯 Results: {passed}/{len(results)} tests passed")
+    
+    if passed == len(results):
+        print("🎉 All tests passed! Data loader is working correctly.")
+    else:
+        print("⚠️  Some tests failed. Check the output above for details.")
+    
+    print("\n💡 Usage Examples:")
+    print("# Load from local files:")
+    print("loader = AgriculturalDataLoader.create_local_loader('/path/to/data')")
+    print()
+    print("# Load from Hugging Face:")
+    print("loader = AgriculturalDataLoader.create_hf_loader('HackathonCRA/2024')")
+    print()
+    print("# Auto-detect with fallback:")
+    print("loader = AgriculturalDataLoader(use_hf=True)")
+    print("df = loader.load_all_files()  # Tries HF first, falls back to local")
+
+if __name__ == "__main__":
+    main()