mvectors

pages/Multimodal_Conversational_Search.py

@@ -249,7 +249,6 @@ def render_answer(question,answer,index,res_img):
                                             unsafe_allow_html=True
                                         )
                                         st.image(res_img[i]['file'])
-                                        st.markdown("</div>", unsafe_allow_html=True)
                         else:
                             if(res_img[i]['file'].lower()!='none' and idx < 1):
                                 col3,col4,col5 = st.columns([33,33,33])

pages/Semantic_Search.py

@@ -671,7 +671,14 @@ if(search_all_type == True or 1==1):
         ########################## enable for query_rewrite ########################
         if rewrite_query:
             st.session_state.input_is_rewrite_query = 'enabled'
-       
+        st.subheader(':blue[Vector Search]')
+        
+        mvector_rerank = st.checkbox("Search and Re-rank with Token level vectors",key = 'mvector_rerank',help = "Enabling this option uses 'all-MiniLM-L6-v2' model's token level embeddings to retrieve documents and MaxSim to re-rank documents.\n\n Hugging Face Model: https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2")
+        
+        if(mvector_rerank):
+            st.session_state.input_mvector_rerank = True
+        else:
+            st.session_state.input_mvector_rerank = False
         st.subheader(':blue[Hybrid Search]')
         with st.expander("Set query Weightage:"):
             st.number_input("Keyword %", min_value=0, max_value=100, value=100, step=5,  key='input_Keyword-weight', help=None)

semantic_search/all_search_execute.py

@@ -18,10 +18,7 @@ from requests.auth import HTTPBasicAuth
 from datetime import datetime
 import boto3
 import streamlit as st
-
-
-
-
+import utilities.mvectors as cb
 current_date_time = (datetime.now()).isoformat()
 today_ = datetime.today().strftime('%Y-%m-%d')
 
@@ -213,13 +210,19 @@ def handler(input_,session_id):
                         }
                     }]            
             del keyword_payload['match']
-#         print("keyword_payload**************")   
-#         print(keyword_payload)
-                    
-        
+ 
         hybrid_payload["query"]["hybrid"]["queries"].append(keyword_payload)
         
     if('Vector Search' in search_types):
+        if(st.session_state.input_mvector_rerank):
+            query_vector = cb.vectorise(query,False)
+            vector_field = "description_vector"
+            print("-------------COLBERT-----1-------------------------------------------------")
+            vector_payload = {"knn": {}}
+            vector_payload["knn"][vector_field]= {
+                            "vector":query_vector,
+                            "k": k_
+                        }
         
 #         path3 =  "_plugins/_ml/models/"+BEDROCK_TEXT_MODEL_ID+"/_predict"
         
@@ -249,7 +252,8 @@ def handler(input_,session_id):
 #                     }
         
         #using neural query 
-        vector_payload = {
+        else:
+            vector_payload = {
                         "neural": {
                         "product_description_vector": {
                             "query_text": query,
@@ -373,15 +377,8 @@ def handler(input_,session_id):
             if(st.session_state.input_price!=(0,0)):
                 sparse_payload['bool']['filter'].append({"range": {"price": {"gte": st.session_state.input_price[0],"lte": st.session_state.input_price[1] }}})
         
-            
-#         print("sparse_payload**************")   
-#         print(sparse_payload)
-            
-        
+      
         ###### end of efficient filter applying #####
-        
-        
-        #print(sparse_payload)
             
         # sparse_payload = {
             
@@ -409,21 +406,34 @@ def handler(input_,session_id):
         r = requests.post(url, auth=awsauth, json=payload, headers=headers)
     
     if(len(hybrid_payload["query"]["hybrid"]["queries"])==1):
-        single_query = hybrid_payload["query"]["hybrid"]["queries"][0]
-        del hybrid_payload["query"]["hybrid"]
-        hybrid_payload["query"] = single_query
-        if(st.session_state.re_ranker == 'true' and st.session_state.input_reranker == 'Cohere Rerank'):
-            path = "demostore-search-index/_search?search_pipeline=rerank_pipeline" 
+        if(st.session_state.input_mvector_rerank and 'Vector Search' in search_types):
+            path = "retail-search-colbert-description/_search" 
             url = host + path
-            hybrid_payload["ext"] = {"rerank": {
-                                          "query_context": {
-                                             "query_text": query
-                                          }
-                                        }}
-            
-        r = requests.get(url, auth=awsauth, json=hybrid_payload, headers=headers)
-        response_ = json.loads(r.text)
-        docs = response_['hits']['hits']
+            r = requests.get(url, auth=awsauth, json=hybrid_payload, headers=headers)
+            print(r.status_code)
+            #print(r.text)
+            response_ = json.loads(r.text)
+            print("-------------colbert ---- 3-----------")
+            #print(response_)
+            docs = response_['hits']['hits']
+            docs = cb.search(docs)
+            print("-------------COLBERT------------5------------------------------------------")
+        else:
+            single_query = hybrid_payload["query"]["hybrid"]["queries"][0]
+            del hybrid_payload["query"]["hybrid"]
+            hybrid_payload["query"] = single_query
+            if(st.session_state.re_ranker == 'true' and st.session_state.input_reranker == 'Cohere Rerank'):
+                path = "demostore-search-index/_search?search_pipeline=rerank_pipeline" 
+                url = host + path
+                hybrid_payload["ext"] = {"rerank": {
+                                            "query_context": {
+                                                "query_text": query
+                                            }
+                                            }}
+                
+            r = requests.get(url, auth=awsauth, json=hybrid_payload, headers=headers)
+            response_ = json.loads(r.text)
+            docs = response_['hits']['hits']
     
     
     else:

utilities/mvectors.py

@@ -0,0 +1,105 @@
+from transformers import AutoTokenizer, AutoModel
+import torch
+import torch.nn.functional as F
+import numpy as np
+import streamlit as st
+import boto3
+import json
+
+runtime = boto3.client('sagemaker-runtime',region_name='us-east-1')
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+endpoint_name = 'huggingface-pytorch-inference-2025-05-21-16-31-07-967'
+
+
+def mean_pooling(token_embeddings, attention_mask):
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, dim=1) / \
+           torch.clamp(input_mask_expanded.sum(dim=1), min=1e-9)
+
+
+def vectorise(sentence,token_level_vectors):
+    print("-------------colbert ---- 2-----------")
+    encoded_input = tokenizer(sentence, padding=True, truncation=True, return_tensors='pt')
+    # Get input IDs (token IDs)
+    input_ids = encoded_input['input_ids'][0]  
+
+    # Convert IDs to tokens
+    tokens = tokenizer.convert_ids_to_tokens(input_ids)
+    model_output = runtime.invoke_endpoint(
+    EndpointName=endpoint_name,
+    ContentType="application/json",
+    Body=json.dumps({"inputs": sentence})
+    )
+    token_vectors = json.loads(model_output['Body'].read().decode())
+    if(token_level_vectors):
+        return tokens,token_vectors
+    
+    token_vectors_tensor = torch.tensor(token_vectors) 
+    attention_mask = encoded_input['attention_mask']  
+    
+    # Perform pooling
+    sentence_embeddings = mean_pooling(token_vectors_tensor, attention_mask)
+
+    # Normalize embeddings
+    sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
+
+    return sentence_embeddings[0].tolist()
+
+def search(hits):
+    print("-------------COLBERT------------4------------------------------------------")
+    tokens,token_vectors = vectorise(st.session_state.input_text,True)
+    final_docs = []
+    for ind,j in enumerate(hits):
+        max_score_dict_list = []
+        doc={"_source":
+            {
+            "description":j["_source"]["description"],"caption":j["_source"]["title"],
+            "image_s3_url":j["_source"]["image_s3_url"],"price":j["_source"]["price"],
+            "style":j["_source"]["style"],"category":j["_source"]["category"]},"_id":j["_id"],"_score":j["_score"]}
+            
+        if("gender_affinity" in j["_source"]):
+            doc["_source"]["gender_affinity"] = j["_source"]["gender_affinity"]
+        else:
+            doc["_source"]["gender_affinity"] = ""
+        #print(j["_source"]["title"])
+        source_doc_token_keys = list(j["_source"].keys())
+        with_s = [x for x in source_doc_token_keys if x.startswith("description-token-")]
+        add_score = 0
+        
+        for index,i in enumerate(token_vectors[0]):
+            token = tokens[index]
+            if(token!='[SEP]' and token!='[CLS]'):
+                query_token_vector = np.array(i)
+                print("query token: "+token)
+                print("-----------------")
+                scores = []
+                for m in with_s:
+                    m_arr = m.split("-")
+                    if(m_arr[-1]!='[SEP]' and m_arr[-1]!='[CLS]'):
+                        #print("document token: "+m_arr[3])
+                        doc_token_vector = np.array(j["_source"][m])
+                        score = np.dot(query_token_vector,doc_token_vector)
+                        scores.append({"doc_token":m_arr[3],"score":score})
+                        #print({"doc_token":m_arr[3],"score":score})
+                    
+                newlist = sorted(scores, key=lambda d: d['score'], reverse=True)
+                max_score = newlist[0]['score']
+                add_score+=max_score
+                max_score_dict_list.append(newlist[0])
+                print(newlist[0])
+        max_score_dict_list_sorted = sorted(max_score_dict_list, key=lambda d: d['score'], reverse=True)
+        print(max_score_dict_list_sorted)
+        # print(add_score)
+        doc["total_score"] = add_score
+        doc['max_score_dict_list_sorted'] = max_score_dict_list_sorted
+        final_docs.append(doc)
+    final_docs_sorted = sorted(final_docs, key=lambda d: d['total_score'], reverse=True)
+    print("-------------COLBERT-----final--------")
+    print(final_docs_sorted)
+    return final_docs_sorted
+        
+                
+        
+                
+