Update logging.py

climateqa/logging.py

@@ -8,9 +8,9 @@ import pandas as pd
 import io
 from typing import TypedDict, List
 from climateqa.constants import DOCUMENT_METADATA_DEFAULT_VALUES
+from langchain_core.documents import Document
 
-
-def serialize_docs(docs:list)->list:
+def serialize_docs(docs:list[Document])->list:
     """Convert document objects to a simplified format compatible with Hugging Face datasets.
     
     This function processes document objects by extracting their page content and metadata,
@@ -27,13 +27,13 @@ def serialize_docs(docs:list)->list:
     for doc in docs:
         # Make sure we have a clean doc format
         new_doc = {
-            "page_content": doc.get("page_content", ""),
+            "page_content": doc.page_content,
             "metadata": {}
         }
         
         # Ensure all metadata fields exist with defaults if missing
         for field, default_value in DOCUMENT_METADATA_DEFAULT_VALUES.items():
-            new_value =  (doc.get("metadata", {}).get(field, default_value))
+            new_value =  doc.metadata.get(field, default_value)
             try:
                 new_doc["metadata"][field] = type(default_value)(new_value)
             except: