Upload app.py with huggingface_hub

app.py

@@ -136,8 +136,20 @@ async def collect_and_store_data():
             logger.info(f"Creating new dataset (existing not found): {e}")
             combined_df = new_df
         
+        # De-duplicate by monthly_requests_int, keeping earliest timestamp for each value
+        combined_df['timestamp'] = pd.to_datetime(combined_df['timestamp'])
+        combined_df = combined_df.sort_values('timestamp')
+        
+        # Group by provider and monthly_requests_int, keep first (earliest) occurrence
+        deduplicated_df = combined_df.groupby(['provider', 'monthly_requests_int']).first().reset_index()
+        
+        # Convert timestamp back to string format
+        deduplicated_df['timestamp'] = deduplicated_df['timestamp'].dt.strftime('%Y-%m-%dT%H:%M:%S.%f%z')
+        
+        logger.info(f"De-duplicated dataset: {len(combined_df)} -> {len(deduplicated_df)} records")
+        
         # Convert back to dataset and push
-        new_dataset = Dataset.from_pandas(combined_df)
+        new_dataset = Dataset.from_pandas(deduplicated_df)
         new_dataset.push_to_hub(DATASET_REPO_NAME, token=HF_TOKEN, private=False)
         
         logger.info(f"Successfully stored data for {len(results)} providers")