Removing stop words but just for english

app.py

@@ -22,6 +22,7 @@ from transformers import (
 from prompts import system_prompt, example_prompt, main_prompt
 from umap import UMAP
 from hdbscan import HDBSCAN
+from sklearn.feature_extraction.text import CountVectorizer
 
 # from cuml.cluster import HDBSCAN
 # from cuml.manifold import UMAP
@@ -36,7 +37,7 @@ session = requests.Session()
 sentence_model = SentenceTransformer("all-MiniLM-L6-v2")
 keybert = KeyBERTInspired()
 mmr = MaximalMarginalRelevance(diversity=0.3)
-
+vectorizer_model = CountVectorizer(stop_words="english")
 
 model_id = "meta-llama/Llama-2-7b-chat-hf"
 device = f"cuda:{cuda.current_device()}" if cuda.is_available() else "cpu"
@@ -125,6 +126,7 @@ def fit_model(base_model, docs, embeddings):
         umap_model=umap_model,
         hdbscan_model=hdbscan_model,
         representation_model=representation_model,
+        vectorizer_model=vectorizer_model,
         # Hyperparameters
         top_n_words=10,
         verbose=True,