url ingest

app.py

@@ -3,6 +3,7 @@ import json
 import time
 import traceback
 from validation import validate_json, validate_croissant, validate_records
+import requests
 
 def process_file(file):
     results = []
@@ -31,24 +32,32 @@ def create_ui():
     with gr.Blocks(theme=gr.themes.Soft()) as app:
         gr.Markdown("# Croissant JSON-LD Validator for NeurIPS")
         gr.Markdown("""
-        Upload your Croissant JSON-LD file to validate if it meets the requirements for NeurIPS submission.
+        Upload your Croissant JSON-LD file or enter a URL to validate if it meets the requirements for NeurIPS submission.
         The validator will check:
         1. If the file is valid JSON
         2. If it passes Croissant schema validation
         3. If records can be generated within a reasonable time
         """)
         
-        with gr.Row():
-            file_input = gr.File(label="Upload Croissant JSON-LD File", file_types=[".json", ".jsonld"])
+        with gr.Tabs() as tabs:
+            with gr.TabItem("Upload File"):
+                file_input = gr.File(label="Upload Croissant JSON-LD File", file_types=[".json", ".jsonld"])
+            
+            with gr.TabItem("URL Input"):
+                url_input = gr.Textbox(
+                    label="Enter Croissant JSON-LD URL",
+                    placeholder="https://huggingface.co/api/datasets/facebook/natural_reasoning/croissant"
+                )
+                fetch_btn = gr.Button("Fetch and Validate", variant="primary")
         
         upload_progress = gr.HTML(
             """<div class="progress-container">
-                 <div class="progress-status">Ready for upload</div>
+                 <div class="progress-status">Ready for validation</div>
                </div>""", visible=True)
         
-        validate_btn = gr.Button("Validate", variant="primary")
+        validate_btn = gr.Button("Validate Uploaded File", variant="primary")
         
-        # Create containers for each validation step
+        # Create containers for validation results
         validation_results = gr.HTML(visible=False)
         
         # Define CSS for the validation UI
@@ -119,22 +128,68 @@ def create_ui():
                      <div class="progress-status">✅ File uploaded successfully</div>
                    </div>""", gr.update(visible=False)
         
-        def on_validate(file):
-            if file is None:
-                return gr.update(visible=False)
-            
-            # Process the file and get results
-            results = process_file(file)
+        def fetch_from_url(url):
+            if not url:
+                return """<div class="progress-container">
+                         <div class="progress-status">Please enter a URL</div>
+                       </div>""", gr.update(visible=False)
             
-            # Build the HTML for validation results
+            try:
+                # Fetch JSON from URL
+                response = requests.get(url, timeout=10)
+                response.raise_for_status()  # Raise exception for 4XX/5XX status codes
+                
+                # Try to parse as JSON
+                json_data = response.json()
+                
+                # Show success message
+                progress_html = """<div class="progress-container">
+                                 <div class="progress-status">✅ JSON fetched successfully from URL</div>
+                               </div>"""
+                
+                # Validate the fetched JSON
+                results = []
+                
+                # Check 1: JSON validation (already done by parsing)
+                results.append(("JSON Format Validation", True, "✅ The URL returned valid JSON."))
+                
+                # Check 2: Croissant validation
+                croissant_valid, croissant_message = validate_croissant(json_data)
+                results.append(("Croissant Schema Validation", croissant_valid, croissant_message))
+                
+                if not croissant_valid:
+                    return progress_html, build_results_html(results)
+                
+                # Check 3: Records validation
+                records_valid, records_message = validate_records(json_data)
+                results.append(("Records Generation Test", records_valid, records_message))
+                
+                return progress_html, build_results_html(results)
+                
+            except requests.exceptions.RequestException as e:
+                error_message = f"❌ Error fetching URL: {str(e)}"
+                return f"""<div class="progress-container">
+                         <div class="progress-status">{error_message}</div>
+                       </div>""", gr.update(visible=False)
+            except json.JSONDecodeError as e:
+                error_message = f"❌ URL did not return valid JSON: {str(e)}"
+                return f"""<div class="progress-container">
+                         <div class="progress-status">{error_message}</div>
+                       </div>""", gr.update(visible=False)
+            except Exception as e:
+                error_message = f"❌ Unexpected error: {str(e)}"
+                return f"""<div class="progress-container">
+                         <div class="progress-status">{error_message}</div>
+                       </div>""", gr.update(visible=False)
+        
+        def build_results_html(results):
+            # Extract the HTML generation logic to a reusable function
             html = '<div class="validation-results">'
             
             for i, (test_name, passed, message) in enumerate(results):
-                # Determine status class
                 status_class = "status-success" if passed else "status-error"
                 status_icon = "✓" if passed else "✗"
                 
-                # Use direct inline JavaScript manipulation instead of a function call
                 html += f'''
                 <div class="validation-step" id="step-{i}">
                     <div class="step-header" onclick="
@@ -160,19 +215,28 @@ def create_ui():
                 '''
             
             html += '</div>'
-            
-            # No separate JavaScript function needed
             return gr.update(value=html, visible=True)
         
+        def on_validate(file):
+            if file is None:
+                return gr.update(visible=False)
+            
+            # Process the file and get results
+            results = process_file(file)
+            return build_results_html(results)
+        
+        # Connect UI events to functions
         file_input.change(on_file_upload, inputs=file_input, outputs=[upload_progress, validation_results])
         validate_btn.click(on_validate, inputs=file_input, outputs=validation_results)
+        fetch_btn.click(fetch_from_url, inputs=url_input, outputs=[upload_progress, validation_results])
         
+        # Footer
         gr.HTML("""
         <div style="text-align: center; margin-top: 20px;">
             <p>Based on the <a href="https://github.com/mlcommons/croissant" target="_blank">Croissant format</a> from MLCommons.</p>
         </div>
         """)
-        
+    
     return app
 
 if __name__ == "__main__":

requirements.txt

@@ -1,3 +1,4 @@
 gradio>=3.50.2
 mlcroissant
-func_timeout 
+func_timeout
+requests