Update file_utils.py

file_utils.py

@@ -1,7 +1,22 @@
 import os
 from magika import Magika
 
-SUPPORTED_FILE_TYPES = ["txt", "shell", "python", "markdown", "yaml", "json", "csv", "tsv", "xml", "html", "ini", "jsonl", "ipynb"]
+def validate_file_types(directory):
+    m = Magika()
+    file_types = {}
+    for root, _, files in os.walk(directory):
+        if '.git' in root:
+            continue
+        for file_name in files:
+            file_path = os.path.join(root, file_name)
+            try:
+                with open(file_path, 'rb') as file:
+                    file_bytes = file.read()
+                result = m.identify_bytes(file_bytes)
+                file_types[file_path] = result.output.ct_label
+            except Exception as e:
+                file_types[file_path] = f"Error: {str(e)}"
+    return file_types
 
 def get_file_summary(file_path, file_type):
     size = os.path.getsize(file_path)
@@ -20,19 +35,6 @@ def read_file_content(file_path, max_size=32*1024):
         else:
             return file.read()
 
-def validate_file_types(directory):
-    m = Magika()
-    file_types = {}
-    for root, _, files in os.walk(directory):
-        if '.git' in root:
-            continue
-        for file_name in files:
-            file_path = os.path.join(root, file_name)
-            try:
-                with open(file_path, 'rb') as file:
-                    file_bytes = file.read()
-                result = m.identify_bytes(file_bytes)
-                file_types[file_path] = result.output.ct_label
-            except Exception as e:
-                file_types[file_path] = f"Error: {str(e)}"
-    return file_types
+def summarize_content(content):
+    # Implement your summarization logic here, potentially using transformers
+    pass