Remove local file loading - HF dataset only

🗑️ Removed Features:
- Removed local file loading capabilities
- Removed automatic fallback mechanism
- Removed factory methods for local loading
- Removed data_path parameter
- Removed use_hf parameter

✨ Simplified Architecture:
- AgriculturalDataLoader now loads exclusively from HF datasets
- Simplified constructor (only hf_token and dataset_id)
- Direct load_from_huggingface() call
- Clear error messages when HF loading fails

🔧 Updated Components:
- Updated gradio_app.py for HF-only usage
- Simplified data_loader.py interface
- Added test_hf_only.py validation script
- Removed unused imports (pathlib, etc.)

🎯 Benefits:
- Cleaner, simpler codebase
- Production-ready for HF Spaces
- No local file dependencies
- Clear separation of concerns
- Easier deployment and maintenance

data_loader.py

@@ -1,44 +1,33 @@
 """
 Data loader for agricultural intervention data.
-Handles loading and preprocessing of CSV and Excel files.
+Loads data exclusively from Hugging Face datasets.
 """
 
 import pandas as pd
 import numpy as np
-from pathlib import Path
-from typing import List, Dict, Optional, Union
+from typing import List, Optional
 import os
 from datasets import Dataset, load_dataset
 from huggingface_hub import HfApi
 
 
 class AgriculturalDataLoader:
-    """Loads and preprocesses agricultural intervention data."""
+    """Loads and preprocesses agricultural intervention data from Hugging Face datasets."""
     
-    def __init__(self, data_path: str = None, hf_token: str = None, dataset_id: str = None, use_hf: bool = False):
-        self.data_path = data_path or "/Users/tracyandre/Downloads/OneDrive_1_9-17-2025"
+    def __init__(self, hf_token: str = None, dataset_id: str = None):
         self.hf_token = hf_token or os.environ.get("HF_TOKEN")
         self.dataset_id = dataset_id or "HackathonCRA/2024"
-        self.use_hf = use_hf  # Flag to use Hugging Face dataset
         self.data_cache = {}
         
     def load_all_files(self) -> pd.DataFrame:
-        """Load all intervention files and combine them."""
+        """Load data from Hugging Face dataset."""
         if 'combined_data' in self.data_cache:
             return self.data_cache['combined_data']
         
-        # Try to load from Hugging Face first if enabled
-        if self.use_hf:
-            try:
-                df = self.load_from_huggingface()
-                self.data_cache['combined_data'] = df
-                return df
-            except Exception as e:
-                print(f"Failed to load from Hugging Face: {e}")
-                print("Falling back to local files...")
-        
-        # Load from local files (original method)
-        return self._load_local_files()
+        # Load from Hugging Face only
+        df = self.load_from_huggingface()
+        self.data_cache['combined_data'] = df
+        return df
     
     def load_from_huggingface(self) -> pd.DataFrame:
         """Load data from Hugging Face dataset."""
@@ -62,45 +51,6 @@ class AgriculturalDataLoader:
         except Exception as e:
             raise ValueError(f"Failed to load dataset from Hugging Face: {e}")
     
-    def _load_local_files(self) -> pd.DataFrame:
-        """Load data from local CSV/Excel files."""
-        print(f"📁 Loading local files from: {self.data_path}")
-        
-        data_files = []
-        data_path = Path(self.data_path)
-        
-        # Get all CSV and Excel files
-        csv_files = list(data_path.glob("Interventions-*.csv"))
-        xlsx_files = list(data_path.glob("Interventions-*.xlsx"))
-        
-        all_dataframes = []
-        
-        # Load CSV files
-        for file_path in csv_files:
-            try:
-                df = pd.read_csv(file_path, skiprows=1)  # Skip the first header row
-                all_dataframes.append(df)
-                print(f"Loaded {file_path.name}: {len(df)} rows")
-            except Exception as e:
-                print(f"Error loading {file_path}: {e}")
-        
-        # Load Excel files
-        for file_path in xlsx_files:
-            try:
-                df = pd.read_excel(file_path, skiprows=1)  # Skip the first header row
-                all_dataframes.append(df)
-                print(f"Loaded {file_path.name}: {len(df)} rows")
-            except Exception as e:
-                print(f"Error loading {file_path}: {e}")
-        
-        # Combine all dataframes
-        if all_dataframes:
-            combined_df = pd.concat(all_dataframes, ignore_index=True)
-            combined_df = self._preprocess_data(combined_df)
-            return combined_df
-        else:
-            raise ValueError("No data files found")
-    
     def _preprocess_data(self, df: pd.DataFrame) -> pd.DataFrame:
         """Preprocess the agricultural data."""
         # Convert date columns
@@ -200,49 +150,7 @@ class AgriculturalDataLoader:
         
         return f"Data uploaded to {self.dataset_id}"
     
-    def set_data_source(self, use_hf: bool = True, clear_cache: bool = True):
-        """
-        Switch between Hugging Face and local file data sources.
-        
-        Args:
-            use_hf: If True, use Hugging Face dataset. If False, use local files.
-            clear_cache: If True, clear cached data to force reload from new source.
-        """
-        self.use_hf = use_hf
-        if clear_cache:
-            self.data_cache.clear()
-            print(f"📋 Switched to {'Hugging Face' if use_hf else 'local files'} data source")
-    
-    @classmethod
-    def create_hf_loader(cls, dataset_id: str = "HackathonCRA/2024", hf_token: str = None):
-        """
-        Factory method to create a loader configured for Hugging Face.
-        
-        Args:
-            dataset_id: Hugging Face dataset identifier
-            hf_token: Hugging Face token (optional, will use environment variable)
-            
-        Returns:
-            AgriculturalDataLoader configured for HF
-        """
-        return cls(
-            dataset_id=dataset_id,
-            hf_token=hf_token,
-            use_hf=True
-        )
-    
-    @classmethod  
-    def create_local_loader(cls, data_path: str):
-        """
-        Factory method to create a loader configured for local files.
-        
-        Args:
-            data_path: Path to local data directory
-            
-        Returns:
-            AgriculturalDataLoader configured for local files
-        """
-        return cls(
-            data_path=data_path,
-            use_hf=False
-        )
+    def clear_cache(self):
+        """Clear cached data to force reload from Hugging Face."""
+        self.data_cache.clear()
+        print("📋 Cache cleared - will reload from Hugging Face on next access")

gradio_app.py

@@ -15,13 +15,9 @@ from analysis_tools import AgriculturalAnalyzer
 
 
 # Initialize components
-# Try to use Hugging Face first, fallback to local files
-try:
-    data_loader = AgriculturalDataLoader(use_hf=True)
-    print("🤗 Configured to use Hugging Face dataset with local fallback")
-except:
-    data_loader = AgriculturalDataLoader(use_hf=False)
-    print("📁 Configured to use local files only")
+# Use Hugging Face dataset exclusively
+data_loader = AgriculturalDataLoader()
+print("🤗 Configured to use Hugging Face dataset exclusively")
 
 analyzer = AgriculturalAnalyzer(data_loader)
 

test_hf_only.py

@@ -0,0 +1,155 @@
+#!/usr/bin/env python3
+"""
+Test script to validate Hugging Face only loading.
+"""
+
+import os
+import warnings
+warnings.filterwarnings('ignore')
+
+def test_hf_only_loading():
+    """Test that the loader only works with Hugging Face."""
+    print("🤗 TESTING HUGGING FACE ONLY LOADING")
+    print("=" * 50)
+    
+    from data_loader import AgriculturalDataLoader
+    
+    # Check if HF token is available
+    hf_token = os.environ.get("HF_TOKEN")
+    if not hf_token:
+        print("⚠️  No HF_TOKEN found in environment variables")
+        print("💡 Set HF_TOKEN to test Hugging Face loading")
+        print("🔧 For this test, we'll try without token (may fail)")
+    
+    try:
+        # Create loader (HF only)
+        loader = AgriculturalDataLoader(
+            dataset_id="HackathonCRA/2024",
+            hf_token=hf_token
+        )
+        
+        print(f"🤗 Attempting to load from dataset: {loader.dataset_id}")
+        
+        # Load data
+        df = loader.load_all_files()
+        
+        print(f"✅ Success! Loaded {len(df):,} records from Hugging Face")
+        print(f"📊 Years: {sorted(df['year'].unique())}")
+        print(f"🌱 Crops: {df['crop_type'].nunique()}")
+        print(f"📍 Plots: {df['plot_name'].nunique()}")
+        print(f"💊 Herbicide applications: {df['is_herbicide'].sum()}")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ Failed to load from Hugging Face: {e}")
+        print("💡 This is expected if the dataset doesn't exist yet")
+        print("🔧 Make sure to upload your dataset to HF Hub first")
+        return False
+
+def test_no_local_fallback():
+    """Test that there's no local fallback."""
+    print("\n🚫 TESTING NO LOCAL FALLBACK")
+    print("=" * 50)
+    
+    from data_loader import AgriculturalDataLoader
+    
+    try:
+        # Create loader with non-existent dataset
+        loader = AgriculturalDataLoader(
+            dataset_id="nonexistent/dataset"
+        )
+        
+        # This should fail without falling back to local
+        df = loader.load_all_files()
+        
+        print(f"❌ Unexpected success - loaded {len(df)} records")
+        print("⚠️  This suggests local fallback is still active")
+        return False
+        
+    except Exception as e:
+        print(f"✅ Expected failure: {e}")
+        print("✅ Confirmed: No local fallback, HF only")
+        return True
+
+def test_simple_usage():
+    """Test simple usage pattern."""
+    print("\n📝 SIMPLE USAGE EXAMPLE")
+    print("=" * 50)
+    
+    print("💡 Recommended usage pattern:")
+    print()
+    
+    usage_code = '''
+from data_loader import AgriculturalDataLoader
+
+# Simple HF-only loader
+loader = AgriculturalDataLoader(dataset_id="HackathonCRA/2024")
+
+# Load data (will use HF_TOKEN from environment)
+df = loader.load_all_files()
+
+# Analyze data
+print(f"Loaded {len(df)} records from Hugging Face")
+'''
+    
+    print(usage_code)
+    
+    try:
+        from data_loader import AgriculturalDataLoader
+        loader = AgriculturalDataLoader(dataset_id="HackathonCRA/2024")
+        print("✅ Loader created successfully")
+        print(f"🎯 Target dataset: {loader.dataset_id}")
+        print(f"🔑 Using token: {'Yes' if loader.hf_token else 'No (from env)'}")
+        
+        return True
+        
+    except Exception as e:
+        print(f"❌ Failed to create loader: {e}")
+        return False
+
+def main():
+    """Run all tests."""
+    print("🚜 HUGGING FACE ONLY - VALIDATION TESTS")
+    print("=" * 60)
+    print()
+    
+    results = []
+    
+    # Test 1: HF loading
+    results.append(("HF Only Loading", test_hf_only_loading()))
+    
+    # Test 2: No local fallback
+    results.append(("No Local Fallback", test_no_local_fallback()))
+    
+    # Test 3: Simple usage
+    results.append(("Simple Usage", test_simple_usage()))
+    
+    # Summary
+    print("\n📋 TEST SUMMARY")
+    print("=" * 30)
+    
+    passed = 0
+    for test_name, result in results:
+        status = "✅ PASS" if result else "❌ FAIL"
+        print(f"{test_name:<20} {status}")
+        if result:
+            passed += 1
+    
+    print(f"\n🎯 Results: {passed}/{len(results)} tests passed")
+    
+    if passed >= 2:  # Allow HF loading to fail if dataset doesn't exist
+        print("🎉 Validation successful! Loader is HF-only.")
+    else:
+        print("⚠️  Validation issues detected.")
+    
+    print("\n🚀 DEPLOYMENT CHECKLIST:")
+    print("✅ Remove local file dependencies")
+    print("✅ HF-only data loading")
+    print("✅ No fallback mechanisms")
+    print("🔲 Upload dataset to HF Hub")
+    print("🔲 Set HF_TOKEN in production")
+    print("🔲 Test with real HF dataset")
+
+if __name__ == "__main__":
+    main()