fix issue with scene splitting

vms/ui/app_ui.py

@@ -396,11 +396,14 @@ class AppUI:
                 model_version_val = available_model_versions[0]
                 logger.info(f"Using first available model version: {model_version_val}")
             
-            # IMPORTANT: Update the dropdown choices directly in the UI component
-            # This is essential to avoid the error when loading the UI
+            # IMPORTANT: Create a new list of simple strings for the dropdown choices
+            # This ensures each choice is a single string, not a tuple or other structure
+            simple_choices = [str(version) for version in available_model_versions]
+            
+            # Update the dropdown choices directly in the UI component
             try:
-                self.project_tabs["train_tab"].components["model_version"].choices = available_model_versions
-                logger.info(f"Updated model_version dropdown choices: {len(available_model_versions)} options")
+                self.project_tabs["train_tab"].components["model_version"].choices = simple_choices
+                logger.info(f"Updated model_version dropdown choices: {len(simple_choices)} options")
             except Exception as e:
                 logger.error(f"Error updating model_version dropdown: {str(e)}")
         else:
@@ -410,7 +413,7 @@ class AppUI:
                 self.project_tabs["train_tab"].components["model_version"].choices = []
             except Exception as e:
                 logger.error(f"Error setting empty model_version choices: {str(e)}")
-                    
+            
         # Ensure training_type is a valid display name
         training_type_val = ui_state.get("training_type", list(TRAINING_TYPES.keys())[0])
         if training_type_val not in TRAINING_TYPES:

vms/ui/project/services/importing/file_upload.py

@@ -22,20 +22,23 @@ logger = logging.getLogger(__name__)
 class FileUploadHandler:
     """Handles processing of uploaded files"""
     
-    def process_uploaded_files(self, file_paths: List[str]) -> str:
+    def process_uploaded_files(self, file_paths: List[str], enable_splitting: bool) -> str:
         """Process uploaded file (ZIP, TAR, MP4, or image)
         
         Args:
             file_paths: File paths to the uploaded files from Gradio
+            enable_splitting: Whether to enable automatic video splitting
                 
         Returns:
             Status message string
         """
+        print(f"process_uploaded_files called with enable_splitting={enable_splitting} and file_paths = {str(file_paths)}")
         if not file_paths or len(file_paths) == 0:
             logger.warning("No files provided to process_uploaded_files")
             return "No files provided"
-            
+        
         for file_path in file_paths:
+            print(f" - {str(file_path)}")
             file_path = Path(file_path)
             try:
                 original_name = file_path.name
@@ -45,11 +48,11 @@ class FileUploadHandler:
                 file_ext = file_path.suffix.lower()
 
                 if file_ext == '.zip':
-                    return self.process_zip_file(file_path)
+                    return self.process_zip_file(file_path, enable_splitting)
                 elif file_ext == '.tar':
-                    return self.process_tar_file(file_path)
+                    return self.process_tar_file(file_path, enable_splitting)
                 elif file_ext == '.mp4' or file_ext == '.webm':
-                    return self.process_mp4_file(file_path, original_name)
+                    return self.process_mp4_file(file_path, original_name, enable_splitting)
                 elif is_image_file(file_path):
                     return self.process_image_file(file_path, original_name)
                 else:
@@ -60,56 +63,12 @@ class FileUploadHandler:
                 logger.error(f"Error processing file {file_path}: {str(e)}", exc_info=True)
                 raise gr.Error(f"Error processing file: {str(e)}")
 
-    def process_image_file(self, file_path: Path, original_name: str) -> str:
-        """Process a single image file
-        
-        Args:
-            file_path: Path to the image
-            original_name: Original filename
-            
-        Returns:
-            Status message string
-        """
-        try:
-            # Create a unique filename with configured extension
-            stem = Path(original_name).stem
-            target_path = STAGING_PATH / f"{stem}.{NORMALIZE_IMAGES_TO}"
-            
-            # If file already exists, add number suffix
-            counter = 1
-            while target_path.exists():
-                target_path = STAGING_PATH / f"{stem}___{counter}.{NORMALIZE_IMAGES_TO}"
-                counter += 1
-
-            logger.info(f"Processing image file: {original_name} -> {target_path}")
-            
-            # Convert to normalized format and remove black bars
-            success = normalize_image(file_path, target_path)
-            
-            if not success:
-                logger.error(f"Failed to process image: {original_name}")
-                raise gr.Error(f"Failed to process image: {original_name}")
-
-            # Handle caption
-            src_caption_path = file_path.with_suffix('.txt')
-            if src_caption_path.exists():
-                caption = src_caption_path.read_text()
-                caption = add_prefix_to_caption(caption, DEFAULT_PROMPT_PREFIX)
-                target_path.with_suffix('.txt').write_text(caption)
-
-            logger.info(f"Successfully stored image: {target_path.name}")
-            gr.Info(f"Successfully stored image: {target_path.name}")
-            return f"Successfully stored image: {target_path.name}"
-
-        except Exception as e:
-            logger.error(f"Error processing image file: {str(e)}", exc_info=True)
-            raise gr.Error(f"Error processing image file: {str(e)}")
-
-    def process_zip_file(self, file_path: Path) -> str:
+    def process_zip_file(self, file_path: Path, enable_splitting: bool) -> str:
         """Process uploaded ZIP file containing media files or WebDataset tar files
         
         Args:
             file_path: Path to the uploaded ZIP file
+            enable_splitting: Whether to enable automatic video splitting
                 
         Returns:
             Status message string
@@ -143,17 +102,18 @@ class FileUploadHandler:
                                 logger.info(f"Processing WebDataset archive from ZIP: {file}")
                                 # Process WebDataset shard
                                 vid_count, img_count = webdataset_handler.process_webdataset_shard(
-                                    file_path, VIDEOS_TO_SPLIT_PATH, STAGING_PATH
+                                    file_path, VIDEOS_TO_SPLIT_PATH if enable_splitting else STAGING_PATH, STAGING_PATH
                                 )
                                 video_count += vid_count
                                 image_count += img_count
                                 tar_count += 1
                             elif is_video_file(file_path):
-                                # Copy video to videos_to_split
-                                target_path = VIDEOS_TO_SPLIT_PATH / file_path.name
+                                # Choose target directory based on auto-splitting setting
+                                target_dir = VIDEOS_TO_SPLIT_PATH if enable_splitting else STAGING_PATH
+                                target_path = target_dir / file_path.name
                                 counter = 1
                                 while target_path.exists():
-                                    target_path = VIDEOS_TO_SPLIT_PATH / f"{file_path.stem}___{counter}{file_path.suffix}"
+                                    target_path = target_dir / f"{file_path.stem}___{counter}{file_path.suffix}"
                                     counter += 1
                                 shutil.copy2(file_path, target_path)
                                 logger.info(f"Extracted video from ZIP: {file} -> {target_path.name}")
@@ -208,11 +168,12 @@ class FileUploadHandler:
             logger.error(f"Error processing ZIP: {str(e)}", exc_info=True)
             raise gr.Error(f"Error processing ZIP: {str(e)}")
 
-    def process_tar_file(self, file_path: Path) -> str:
+    def process_tar_file(self, file_path: Path, enable_splitting: bool) -> str:
         """Process a WebDataset tar file
         
         Args:
             file_path: Path to the uploaded tar file
+            enable_splitting: Whether to enable automatic video splitting
                 
         Returns:
             Status message string
@@ -220,7 +181,7 @@ class FileUploadHandler:
         try:
             logger.info(f"Processing WebDataset TAR file: {file_path}")
             video_count, image_count = webdataset_handler.process_webdataset_shard(
-                file_path, VIDEOS_TO_SPLIT_PATH, STAGING_PATH
+                file_path, VIDEOS_TO_SPLIT_PATH if enable_splitting else STAGING_PATH, STAGING_PATH
             )
             
             # Generate status message
@@ -243,25 +204,30 @@ class FileUploadHandler:
             logger.error(f"Error processing WebDataset tar file: {str(e)}", exc_info=True)
             raise gr.Error(f"Error processing WebDataset tar file: {str(e)}")
 
-    def process_mp4_file(self, file_path: Path, original_name: str) -> str:
+    def process_mp4_file(self, file_path: Path, original_name: str, enable_splitting: bool) -> str:
         """Process a single video file
         
         Args:
             file_path: Path to the file
             original_name: Original filename
+            enable_splitting: Whether to enable automatic video splitting
             
         Returns:
             Status message string
         """
+        print(f"process_mp4_file(self, file_path={str(file_path)}, original_name={str(original_name)}, enable_splitting={enable_splitting})")
         try:
+            # Choose target directory based on auto-splitting setting
+            target_dir = VIDEOS_TO_SPLIT_PATH if enable_splitting else STAGING_PATH
+            print(f"target_dir = {target_dir}")
             # Create a unique filename
-            target_path = VIDEOS_TO_SPLIT_PATH / original_name
+            target_path = target_dir / original_name
             
             # If file already exists, add number suffix
             counter = 1
             while target_path.exists():
                 stem = Path(original_name).stem
-                target_path = VIDEOS_TO_SPLIT_PATH / f"{stem}___{counter}.mp4"
+                target_path = target_dir / f"{stem}___{counter}.mp4"
                 counter += 1
 
             logger.info(f"Processing video file: {original_name} -> {target_path}")

vms/ui/project/services/importing/hub_dataset.py

@@ -168,7 +168,7 @@ class HubDatasetBrowser:
         self, 
         dataset_id: str, 
         file_type: str, 
-        enable_splitting: bool = True,
+        enable_splitting: bool,
         progress_callback: Optional[Callable] = None
     ) -> str:
         """Download all files of a specific type from the dataset
@@ -328,7 +328,7 @@ class HubDatasetBrowser:
     async def download_dataset(
         self, 
         dataset_id: str, 
-        enable_splitting: bool = True,
+        enable_splitting: bool,
         progress_callback: Optional[Callable] = None
     ) -> Tuple[str, str]:
         """Download a dataset and process its video/image content

vms/ui/project/services/importing/import_service.py

@@ -28,32 +28,37 @@ class ImportingService:
         self.youtube_handler = YouTubeDownloader()
         self.hub_browser = HubDatasetBrowser(self.hf_api)
     
-    def process_uploaded_files(self, file_paths: List[str]) -> str:
+    def process_uploaded_files(self, file_paths: List[str], enable_splitting: bool) -> str:
         """Process uploaded file (ZIP, TAR, MP4, or image)
         
         Args:
             file_paths: File paths to the uploaded files from Gradio
+            enable_splitting: Whether to enable automatic video splitting
                 
         Returns:
             Status message string
         """
+        print(f"process_uploaded_files(...,  enable_splitting = { enable_splitting})")
         if not file_paths or len(file_paths) == 0:
             logger.warning("No files provided to process_uploaded_files")
             return "No files provided"
         
-        return self.file_handler.process_uploaded_files(file_paths)
+        print(f"process_uploaded_files(..., enable_splitting = {enable_splitting:})")
+        print(f"process_uploaded_files: calling self.file_handler.process_uploaded_files")
+        return self.file_handler.process_uploaded_files(file_paths, enable_splitting)
     
-    def download_youtube_video(self, url: str, progress=None) -> str:
+    def download_youtube_video(self, url: str, enable_splitting: bool, progress=None) -> str:
         """Download a video from YouTube
         
         Args:
             url: YouTube video URL
+            enable_splitting: Whether to enable automatic video splitting
             progress: Optional Gradio progress indicator
             
         Returns:
             Status message string
         """
-        return self.youtube_handler.download_video(url, progress)
+        return self.youtube_handler.download_video(url, enable_splitting, progress)
     
     def search_datasets(self, query: str) -> List[List[str]]:
         """Search for datasets on the Hugging Face Hub
@@ -80,7 +85,7 @@ class ImportingService:
     async def download_dataset(
         self, 
         dataset_id: str, 
-        enable_splitting: bool = True,
+        enable_splitting: bool,
         progress_callback: Optional[Callable] = None
     ) -> Tuple[str, str]:
         """Download a dataset and process its video/image content
@@ -99,7 +104,7 @@ class ImportingService:
         self, 
         dataset_id: str, 
         file_type: str, 
-        enable_splitting: bool = True,
+        enable_splitting: bool,
         progress_callback: Optional[Callable] = None
     ) -> str:
         """Download a group of files (videos or WebDatasets)

vms/ui/project/services/importing/youtube.py

@@ -17,11 +17,12 @@ logger = logging.getLogger(__name__)
 class YouTubeDownloader:
     """Handles downloading videos from YouTube"""
     
-    def download_video(self, url: str, progress: Optional[Callable] = None) -> str:
+    def download_video(self, url: str, enable_splitting: bool, progress: Optional[Callable] = None) -> str:
         """Download a video from YouTube
         
         Args:
             url: YouTube video URL
+            enable_splitting: Whether to enable automatic video splitting
             progress: Optional Gradio progress indicator
             
         Returns:
@@ -40,7 +41,10 @@ class YouTubeDownloader:
                 if progress else None)
             
             video_id = yt.video_id
-            output_path = VIDEOS_TO_SPLIT_PATH / f"{video_id}.mp4"
+            
+            # Choose target directory based on auto-splitting setting
+            target_dir = VIDEOS_TO_SPLIT_PATH if enable_splitting else STAGING_PATH
+            output_path = target_dir / f"{video_id}.mp4"
             
             # Download highest quality progressive MP4
             if progress:
@@ -58,7 +62,7 @@ class YouTubeDownloader:
                 logger.info("Starting YouTube video download...")
                 progress(0, desc="Starting download...")
             
-            video.download(output_path=str(VIDEOS_TO_SPLIT_PATH), filename=f"{video_id}.mp4")
+            video.download(output_path=str(target_dir), filename=f"{video_id}.mp4")
             
             # Update UI
             if progress:

vms/ui/project/services/splitting.py

@@ -63,7 +63,7 @@ class SplittingService:
         """Process a single video file to detect and split scenes"""
         try:
             self._processing_status[video_path.name] = f'Processing video "{video_path.name}"...'
-            
+            print(f'Going to split scenes for video "{video_path.name}"...')
             parent_caption_path = video_path.with_suffix('.txt')
             # Create output path for split videos
             base_name, _ = extract_scene_info(video_path.name)
@@ -180,6 +180,7 @@ class SplittingService:
 
     async def start_processing(self, enable_splitting: bool) -> None:
         """Start background processing of unprocessed videos"""
+        #print(f"start_processing(enable_splitting={enable_splitting}), self.processing = {self.processing}")
         if self.processing:
             return
             
@@ -188,6 +189,8 @@ class SplittingService:
             # Process each video
             for video_file in VIDEOS_TO_SPLIT_PATH.glob("*.mp4"):
                 self._current_file = video_file.name
+                #print(f"calling await self.process_video(video_file, {enable_splitting})")
+        
                 await self.process_video(video_file, enable_splitting)
                     
         finally:

vms/ui/project/tabs/import_tab/import_tab.py

@@ -90,25 +90,37 @@ class ImportTab(BaseTab):
         self.youtube_tab.connect_events()
         self.hub_tab.connect_events()
     
-    def on_import_success(self, enable_splitting, enable_automatic_content_captioning, prompt_prefix):
+    def on_import_success(
+        self,
+        enable_splitting: bool,
+        enable_automatic_content_captioning: bool,
+        prompt_prefix: str
+    ):
         """Handle successful import of files"""
+        #print(f"on_import_success(self, enable_splitting={enable_splitting}, enable_automatic_content_captioning={enable_automatic_content_captioning}, prompt_prefix={prompt_prefix})")
         # If splitting is disabled, we need to directly move videos to staging
-        if not enable_splitting:
-            # Copy files without splitting
-            self._start_copy_to_staging_bg()
-            msg = "Copying videos to staging directory without splitting..."
-        else:
+        if enable_splitting:
+            #print("on_import_success: -> splitting enabled!")
             # Start scene detection if not already running and there are videos to process
             if not self.app.splitting.is_processing():
+                #print("on_import_success: -> calling self._start_scene_detection_bg(enable_splitting)")
                 # Start the scene detection in a separate thread
                 self._start_scene_detection_bg(enable_splitting)
                 msg = "Starting automatic scene detection..."
             else:
                 msg = "Scene detection already running..."
 
-        # Copy files to training directory
-        self.app.tabs["caption_tab"].copy_files_to_training_dir(prompt_prefix)
-        
+            # Copy files to training directory
+            self.app.tabs["caption_tab"].copy_files_to_training_dir(prompt_prefix)
+        else:
+            #print("on_import_success: -> splitting NOT enabled")
+            # Copy files without splitting
+            self._start_copy_to_staging_bg()
+            msg = "Copying videos to staging directory without splitting..."
+            
+            # Also immediately copy to training directory
+            self.app.tabs["caption_tab"].copy_files_to_training_dir(prompt_prefix)
+
         # Start auto-captioning if enabled
         if enable_automatic_content_captioning:
             self._start_captioning_bg(DEFAULT_CAPTIONING_BOT_INSTRUCTIONS, prompt_prefix)
@@ -122,8 +134,9 @@ class ImportTab(BaseTab):
             logger.warning("Cannot switch tabs - project_tabs_component not available")
             return None, msg
     
-    def _start_scene_detection_bg(self, enable_splitting):
+    def _start_scene_detection_bg(self, enable_splitting: bool):
         """Start scene detection in a background thread"""
+        print(f"_start_scene_detection_bg(enable_splitting={enable_splitting})")
         def run_async_in_thread():
             loop = asyncio.new_event_loop()
             asyncio.set_event_loop(loop)
@@ -207,11 +220,13 @@ class ImportTab(BaseTab):
         thread.daemon = True
         thread.start()
         
-    async def update_titles_after_import(self, enable_splitting, enable_automatic_content_captioning, prompt_prefix):
+    async def update_titles_after_import(self, enable_splitting: bool, enable_automatic_content_captioning: bool, prompt_prefix: str):
         """Handle post-import updates including titles"""
         # Call the non-async version since we need to return immediately for the UI
         tabs, status_msg = self.on_import_success(
-            enable_splitting, enable_automatic_content_captioning, prompt_prefix
+            enable_splitting,
+            enable_automatic_content_captioning,
+            prompt_prefix
         )
         
         # Get updated titles

vms/ui/project/tabs/import_tab/upload_tab.py

@@ -62,11 +62,22 @@ class UploadTab(BaseTab):
             logger.warning("import_status component is not set in UploadTab")
             return
         
-        # File upload event
+        # File upload event with enable_splitting parameter
         upload_event = self.components["files"].upload(
-            fn=lambda x: self.app.importing.process_uploaded_files(x),
-            inputs=[self.components["files"]],
+            fn=self.app.importing.process_uploaded_files,
+            inputs=[self.components["files"], self.components["enable_automatic_video_split"]],
             outputs=[self.components["import_status"]]
+        ).success(
+            fn=self.app.tabs["import_tab"].on_import_success,
+            inputs=[
+                self.components["enable_automatic_video_split"],
+                self.components["enable_automatic_content_captioning"],
+                self.app.tabs["caption_tab"].components["custom_prompt_prefix"]
+            ],
+            outputs=[
+                self.app.project_tabs_component,
+                self.components["import_status"]
+            ]
         )
         
         # Only add success handler if all required components exist
@@ -102,4 +113,4 @@ class UploadTab(BaseTab):
                 )
             except (AttributeError, KeyError) as e:
                 logger.error(f"Error connecting event handlers in UploadTab: {str(e)}")
-                # Continue without the success handler
+                # Continue without the success handler

vms/ui/project/tabs/import_tab/youtube_tab.py

@@ -83,8 +83,8 @@ class YouTubeTab(BaseTab):
             
         # YouTube download event
         download_event = self.components["youtube_download_btn"].click(
-            fn=self.app.importing.download_youtube_video,
-            inputs=[self.components["youtube_url"]],
+            fn=self.download_youtube_with_splitting,
+            inputs=[self.components["youtube_url"], self.components["enable_automatic_video_split"]],
             outputs=[self.components["import_status"]]
         )
         
@@ -106,4 +106,8 @@ class YouTubeTab(BaseTab):
                 )
             except (AttributeError, KeyError) as e:
                 logger.error(f"Error connecting success handler in YouTubeTab: {str(e)}")
-                # Continue without the success handler
+                # Continue without the success handler
+                
+    def download_youtube_with_splitting(self, url, enable_splitting):
+        """Download YouTube video with splitting option"""
+        return self.app.importing.download_youtube_video(url, enable_splitting, gr.Progress())

vms/ui/project/tabs/preview_tab.py

@@ -200,8 +200,10 @@ class PreviewTab(BaseTab):
         # Return just the model IDs as a list of simple strings
         version_ids = list(MODEL_VERSIONS.get(internal_type, {}).keys())
         logger.info(f"Found {len(version_ids)} versions for {model_type}: {version_ids}")
-        return version_ids
-            
+        
+        # Ensure they're all strings
+        return [str(version) for version in version_ids]
+    
     def get_default_model_version(self, model_type: str) -> str:
         """Get default model version for the given model type"""
         # Convert UI display name to internal name

vms/ui/project/tabs/train_tab.py

@@ -462,12 +462,15 @@ class TrainTab(BaseTab):
             # Update UI state with proper model_type first
             self.app.update_ui_state(model_type=model_type)
             
+            # Ensure model_versions is a simple list of strings
+            model_versions = [str(version) for version in model_versions]
+            
             # Create a new dropdown with the updated choices
             if not model_versions:
                 logger.warning(f"No model versions available for {model_type}, using empty list")
                 # Return empty dropdown to avoid errors
                 return gr.Dropdown(choices=[], value=None)
-                
+                    
             # Ensure default_version is in model_versions
             if default_version not in model_versions and model_versions:
                 default_version = model_versions[0]
@@ -481,8 +484,7 @@ class TrainTab(BaseTab):
             logger.error(f"Error in update_model_versions: {str(e)}")
             # Return empty dropdown to avoid errors
             return gr.Dropdown(choices=[], value=None)
-            
-    
+        
     def handle_training_start(
         self, preset, model_type, model_version, training_type, 
         lora_rank, lora_alpha, train_steps, batch_size, learning_rate, 
@@ -561,7 +563,9 @@ class TrainTab(BaseTab):
         # Return just the model IDs as a list of simple strings
         version_ids = list(MODEL_VERSIONS.get(internal_type, {}).keys())
         logger.info(f"Found {len(version_ids)} versions for {model_type}: {version_ids}")
-        return version_ids
+        
+        # Ensure they're all strings
+        return [str(version) for version in version_ids]
 
     def get_default_model_version(self, model_type: str) -> str:
         """Get default model version for the given model type"""
@@ -749,9 +753,6 @@ class TrainTab(BaseTab):
         model_versions = self.get_model_version_choices(model_display_name)
         default_model_version = self.get_default_model_version(model_display_name)
 
-        # Create the model version dropdown update
-        model_version_update = gr.Dropdown(choices=model_versions, value=default_model_version)
-
         # Ensure we have valid choices and values
         if not model_versions:
             logger.warning(f"No versions found for {model_display_name}, using empty list")
@@ -761,6 +762,12 @@ class TrainTab(BaseTab):
             default_model_version = model_versions[0]
             logger.info(f"Reset default version to first available: {default_model_version}")
 
+        # Ensure model_versions is a simple list of strings
+        model_versions = [str(version) for version in model_versions]
+
+        # Create the model version dropdown update
+        model_version_update = gr.Dropdown(choices=model_versions, value=default_model_version)
+
         # Return values in the same order as the output components
         return (
             model_display_name,

vms/utils/webdataset_handler.py

@@ -41,7 +41,9 @@ def process_webdataset_shard(
     """
     video_count = 0
     image_count = 0
-    
+
+    print(f"videos_output_dir = {videos_output_dir}")
+    print(f"staging_output_dir = {staging_output_dir}")
     try:
         # Dictionary to store grouped files by prefix
         grouped_files = {}