validate dataframe with tests

submit.py

@@ -2,12 +2,15 @@ from pathlib import Path
 import tempfile
 from typing import BinaryIO
 import json
+import pandas as pd
+import io
 
 import gradio as gr
 from datetime import datetime
 import uuid
 
-from about import API, submissions_repo
+from constants import API, SUBMISSIONS_REPO
+from validation import validate_csv_file
 
 def make_submission(
     submitted_file: BinaryIO,
@@ -17,17 +20,26 @@ def make_submission(
     if user_state is None:
         raise gr.Error("You must submit your username to submit a file.")
     
+    if submitted_file is None:
+        raise gr.Error("Please upload a CSV file before submitting.")
+    
     file_path = submitted_file.name
 
     if not file_path:
         raise gr.Error("Uploaded file object does not have a valid file path.")
     
     path_obj = Path(file_path)
+
+    if path_obj.suffix.lower() != '.csv':
+        raise gr.Error("File must be a CSV file. Please upload a .csv file.")
+    
     timestamp = datetime.utcnow().isoformat()
     submission_id = str(uuid.uuid4())
 
     with (path_obj.open("rb") as f_in):
         file_content = f_in.read().decode("utf-8")
+        
+        validate_csv_file(file_content)
 
         # write to dataset
         filename = f"{submission_id}.json"
@@ -49,7 +61,7 @@ def make_submission(
         API.upload_file(
             path_or_fileobj=tmp_name, 
             path_in_repo=filename,
-            repo_id=submissions_repo,
+            repo_id=SUBMISSIONS_REPO,
             repo_type="dataset",
             commit_message=f"Add submission for {user_state} at {timestamp}"
         )

test/conftest.py

@@ -0,0 +1,26 @@
+import pytest
+import pandas as pd
+from constants import MINIMAL_NUMBER_OF_ROWS, ASSAY_LIST
+
+
+@pytest.fixture
+def valid_csv_data():
+    """Fixture providing valid CSV data with all required columns"""
+    return {
+        "antibody_id": ["AB001"] * MINIMAL_NUMBER_OF_ROWS,
+        "antibody_name": ["AB001"] * MINIMAL_NUMBER_OF_ROWS,
+        "vh_protein_sequence": ["EVQLVESGGGLVQPGGSLRLSCAASGFTFSSYAMHWVRQAPGKGLEWVSAISGSGGSTYYADSVKGRFTISRDNSKNTLYLQMNSLRAEDTAVYYCARDYGDGYYFDYWGQGTLVTVSS"] * MINIMAL_NUMBER_OF_ROWS,
+        "vl_protein_sequence": ["DIQMTQSPSSLSASVGDRVTITCRASQSISSYLNWYQQKPGKAPKLLIYAASTLQSGVPSRFSGSGSGTDFTLTISSLQPEDFATYYCQQSYSTPFTFGQGTKVEIK"] * MINIMAL_NUMBER_OF_ROWS,
+        **{assay: [0.85] * MINIMAL_NUMBER_OF_ROWS for assay in ASSAY_LIST},
+    }
+
+
+@pytest.fixture
+def valid_input_dataframe(valid_csv_data):
+    """Fixture providing a valid input dataframe"""
+    return pd.DataFrame(valid_csv_data)
+
+@pytest.fixture
+def valid_csv_content(valid_input_dataframe):
+    """Fixture providing valid CSV content as string"""
+    return valid_input_dataframe.to_csv(index=False)

test/test_validation.py

@@ -0,0 +1,109 @@
+import pytest
+import pandas as pd
+import gradio as gr
+from validation import validate_csv_file, validate_csv_can_be_read, validate_dataframe
+from constants import REQUIRED_COLUMNS, MINIMAL_NUMBER_OF_ROWS
+
+
+class TestValidateCsvCanBeRead:
+    """Test cases for validate_csv_can_be_read function"""
+    
+    def test_valid_csv_can_be_read(self, valid_csv_content):
+        """Test that valid CSV content can be read"""
+        df = validate_csv_can_be_read(valid_csv_content)
+        assert isinstance(df, pd.DataFrame)
+        assert len(df) == MINIMAL_NUMBER_OF_ROWS
+        assert list(df.columns) == list(REQUIRED_COLUMNS)
+    
+    def test_empty_csv_raises_error(self):
+        """Test that empty CSV raises an error"""
+        empty_csv = ""
+        
+        with pytest.raises(gr.Error) as exc_info:
+            validate_csv_can_be_read(empty_csv)
+        
+        assert "empty or contains no valid data" in str(exc_info.value)
+    
+    def test_invalid_csv_format_raises_error(self):
+        """Test that invalid CSV format raises an error"""
+        # Create a CSV with malformed structure that pandas cannot parse
+        malformed_csv = "column1,column2\nvalue1,\"unclosed quote\nvalue4,value5"
+        
+        with pytest.raises(gr.Error) as exc_info:
+            validate_csv_can_be_read(malformed_csv)
+        
+        assert "Invalid CSV format" in str(exc_info.value)
+    
+    def test_csv_with_quoted_fields_can_be_read(self):
+        """Test that CSV with quoted fields can be read"""
+        # Create CSV with quoted fields and enough rows
+        base_row = 'AB001,"EVQLVESGGGLVQPGGSLRLSCAASGFTFSSYAMHWVRQAPGKGLEWVSAISGSGGSTYYADSVKGRFTISRDNSKNTLYLQMNSLRAEDTAVYYCARDYGDGYYFDYWGQGTLVTVSS","DIQMTQSPSSLSASVGDRVTITCRASQSISSYLNWYQQKPGKAPKLLIYAASTLQSGVPSRFSGSGSGTDFTLTISSLQPEDFATYYCQQSYSTPFTFGQGTKVEIK",95.2,0.85,0.92,0.78,0.81,72.5'
+        csv_content = "antibody_id,vh_protein_sequence,vl_protein_sequence,SEC %Monomer,HIC,PR_CHO,AC-SINS_pH6.0,AC-SINS_pH7.4,Tm\n"
+        csv_content += "\n".join([base_row] * MINIMAL_NUMBER_OF_ROWS)
+        
+        df = validate_csv_can_be_read(csv_content)
+        assert isinstance(df, pd.DataFrame)
+        assert len(df) == MINIMAL_NUMBER_OF_ROWS
+
+
+class TestValidateDataframe:
+    """Test cases for validate_dataframe function"""
+    
+    def test_valid_dataframe_passes(self, valid_input_dataframe):
+        """Test that valid DataFrame passes validation"""
+        validate_dataframe(valid_input_dataframe)
+    
+    def test_missing_columns_raises_error(self, valid_input_dataframe):
+        """Test that DataFrame with missing columns raises an error"""
+        missing_column = REQUIRED_COLUMNS[0]
+        df = valid_input_dataframe.copy()
+        df.drop(columns=[missing_column], inplace=True)
+        
+        with pytest.raises(gr.Error) as exc_info:
+            validate_dataframe(df)
+        
+        assert f"Missing required columns: {missing_column}" in str(exc_info.value)
+    
+    
+    def test_empty_dataframe_raises_error(self, valid_input_dataframe):
+        """Test that empty DataFrame raises an error"""
+        empty_df = valid_input_dataframe.head(0)
+        
+        with pytest.raises(gr.Error) as exc_info:
+            validate_dataframe(empty_df)
+        
+        assert "CSV file is empty" in str(exc_info.value)
+    
+    def test_insufficient_rows_raises_error(self, valid_input_dataframe):
+        """Test that DataFrame with insufficient rows raises an error"""
+        df = valid_input_dataframe.head(MINIMAL_NUMBER_OF_ROWS - 1)
+        with pytest.raises(gr.Error) as exc_info:
+            validate_dataframe(df)
+        
+        assert f"CSV should have at least {MINIMAL_NUMBER_OF_ROWS} rows" in str(exc_info.value)
+    
+    def test_missing_values_raises_error(self, valid_input_dataframe):
+        """Test that DataFrame with missing values raises an error"""
+        bad_column = REQUIRED_COLUMNS[0]
+        df = valid_input_dataframe.copy()
+        df[bad_column] = [None] * len(df)
+        with pytest.raises(gr.Error) as exc_info:
+            validate_dataframe(df)
+        
+        assert f"contains {len(df)} missing values" in str(exc_info.value)
+    
+    def test_csv_with_extra_columns_passes(self, valid_input_dataframe):
+        """Test that DataFrame with extra columns passes validation"""
+        extra_column = "extra_column_1"
+        df = valid_input_dataframe.copy()
+        df[extra_column] = ["extra1"] * len(df)
+        df[extra_column] = ["extra2"] * len(df)
+        validate_dataframe(df)
+    
+
+class TestValidateCsvFile:
+    """Test cases for the combined validate_csv_file function"""
+    
+    def test_valid_csv_passes(self, valid_csv_content):
+        """Test that a valid CSV with all required columns passes validation"""
+        validate_csv_file(valid_csv_content)

validation.py

@@ -0,0 +1,101 @@
+import pandas as pd
+import io
+import gradio as gr
+from constants import REQUIRED_COLUMNS, MINIMAL_NUMBER_OF_ROWS
+
+def validate_csv_can_be_read(file_content: str) -> pd.DataFrame:
+    """
+    Validate that the CSV file can be read and parsed.
+    
+    Parameters
+    ----------
+    file_content: str
+        The content of the uploaded CSV file.
+    
+    Returns
+    -------
+    pd.DataFrame
+        The parsed DataFrame if successful.
+    
+    Raises
+    ------
+    gr.Error: If CSV cannot be read or parsed
+    """
+    try:
+        # Read CSV content
+        df = pd.read_csv(io.StringIO(file_content))
+        return df
+        
+    except pd.errors.EmptyDataError:
+        raise gr.Error(
+            "❌ CSV file is empty or contains no valid data"
+        )
+    except pd.errors.ParserError as e:
+        raise gr.Error(
+            f"❌ Invalid CSV format<br><br>"
+            f"Error: {str(e)}"
+        )
+    except UnicodeDecodeError:
+        raise gr.Error(
+            "❌ File encoding error<br><br>"
+            "Your file appears to have an unsupported encoding.<br>"
+            "Please save your CSV file with UTF-8 encoding and try again."
+        )
+
+def validate_dataframe(df: pd.DataFrame) -> None:
+    """
+    Validate the DataFrame content and structure.
+    
+    Parameters
+    ----------
+    df: pd.DataFrame
+        The DataFrame to validate.
+    
+    Raises
+    ------
+    gr.Error: If validation fails
+    """
+    # Required columns should be present
+    missing_columns = set(REQUIRED_COLUMNS) - set(df.columns)
+    if missing_columns:
+        raise gr.Error(
+            f"❌ Missing required columns: {', '.join(missing_columns)}"
+        )
+    
+    # Data should not be empty
+    if df.empty:
+        raise gr.Error(
+            "❌ CSV file is empty"
+        )
+    
+    # Check for missing values in required columns
+    for col in REQUIRED_COLUMNS:
+        missing_count = df[col].isnull().sum()
+        if missing_count > 0:
+            raise gr.Error(
+                f"❌ Column '{col}' contains {missing_count} missing values"
+            )
+    
+    # Check for reasonable number of rows
+    if len(df) < MINIMAL_NUMBER_OF_ROWS:
+        raise gr.Error(
+            f"❌ CSV should have at least {MINIMAL_NUMBER_OF_ROWS} rows"
+        )
+    
+    print(f"✅ CSV validation passed! Found {len(df)} rows with columns: {', '.join(df.columns)}")
+
+def validate_csv_file(file_content: str) -> None:
+    """
+    Validate the uploaded CSV file.
+    
+    Parameters
+    ----------
+    file_content: str
+        The content of the uploaded CSV file.
+    
+    Raises
+    ------
+    gr.Error: If validation fails
+    """
+    df = validate_csv_can_be_read(file_content)
+    validate_dataframe(df)