Adding num rows

app.py

@@ -15,7 +15,7 @@ from sentence_transformers import SentenceTransformer
 from dotenv import load_dotenv
 import os
 
-import spaces
+# import spaces
 import gradio as gr
 
 
@@ -81,7 +81,7 @@ def get_docs_from_parquet(parquet_urls, column, offset, limit):
     return df[column].tolist()
 
 
-@spaces.GPU
+# @spaces.GPU
 def calculate_embeddings(docs):
     return sentence_model.encode(docs, show_progress_bar=True, batch_size=32)
 
@@ -92,7 +92,7 @@ def calculate_n_neighbors_and_components(n_rows):
     return n_neighbors, n_components
 
 
-@spaces.GPU
+# @spaces.GPU
 def fit_model(docs, embeddings, n_neighbors, n_components):
     global global_topic_model
 
@@ -116,11 +116,11 @@ def fit_model(docs, embeddings, n_neighbors, n_components):
     new_model = BERTopic(
         language="english",
         # Sub-models
-        embedding_model=sentence_model,
-        umap_model=umap_model,
-        hdbscan_model=hdbscan_model,
-        representation_model=representation_model,
-        vectorizer_model=vectorizer_model,
+        embedding_model=sentence_model,  # Step 1 - Extract embeddings
+        umap_model=umap_model,  # Step 2 - UMAP model
+        hdbscan_model=hdbscan_model,  # Step 3 - Cluster reduced embeddings
+        vectorizer_model=vectorizer_model,  # Step 4 - Tokenize topics
+        representation_model=representation_model,  # Step 5 - Label topics
         # Hyperparameters
         top_n_words=10,
         verbose=True,
@@ -162,12 +162,16 @@ def generate_topics(dataset, config, split, column, nested_column):
     all_docs = []
     reduced_embeddings_list = []
     topics_info, topic_plot = None, None
+    full_processing = split_rows <= MAX_ROWS
+    message = (
+        f"⚙️ Processing full dataset: 0 of ({split_rows} rows)"
+        if full_processing
+        else f"⚙️ Processing partial dataset 0 of ({limit} rows)"
+    )
     yield (
         gr.DataFrame(value=[], interactive=False, visible=True),
         gr.Plot(value=None, visible=True),
-        gr.Label(
-            {f"⚙️ Generating topics {dataset}": rows_processed / limit}, visible=True
-        ),
+        gr.Label({message: rows_processed / limit}, visible=True),
     )
     while offset < limit:
         docs = get_docs_from_parquet(parquet_urls, column, offset, CHUNK_SIZE)
@@ -207,10 +211,16 @@ def generate_topics(dataset, config, split, column, nested_column):
         rows_processed += len(docs)
         progress = min(rows_processed / limit, 1.0)
         logging.info(f"Progress: {progress} % - {rows_processed} of {limit}")
+        message = (
+            f"⚙️ Processing full dataset: {rows_processed} of {limit}"
+            if full_processing
+            else f"⚙️ Processing partial dataset: {rows_processed} of {limit} rows"
+        )
+
         yield (
             topics_info,
             topic_plot,
-            gr.Label({f"⚙️ Generating topics {dataset}": progress}, visible=True),
+            gr.Label({message: progress}, visible=True),
         )
 
         offset += CHUNK_SIZE
@@ -219,7 +229,9 @@ def generate_topics(dataset, config, split, column, nested_column):
     yield (
         topics_info,
         topic_plot,
-        gr.Label({f"✅ Generating topics {dataset}": 1.0}, visible=True),
+        gr.Label(
+            {f"✅ Done: {rows_processed} rows have been processed": 1.0}, visible=True
+        ),
     )
     cuda.empty_cache()
 
@@ -260,7 +272,7 @@ with gr.Blocks() as demo:
 
         generate_button = gr.Button("Generate Topics", variant="primary")
 
-    gr.Markdown("## Datamap")
+    gr.Markdown("## Data map")
     full_topics_generation_label = gr.Label(visible=False, show_label=False)
     topics_plot = gr.Plot()
     with gr.Accordion("Topics Info", open=False):