Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+DOCUMENTS/Flykite_Airlines_HR_Policy.pdf filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,29 @@
+.env
+__pycache__/
+.python
+env/
+venv/
+*.egg-info
+dist/
+build/
+*.pyd
+*.pyo
+
+
+.vscode/
+*.code-workspace
+
+DATA/chroma_temp_db_eval/
+DATA/faiss_temp_db_eval/
+DATA/pinecone_temp_db_eval/
+DATA/
+
+*.log
+*.key
+*.pem
+
+#LLM
+*.bin
+*.h5
+*.pt
+

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/Fly-Kite-AirLine-RAG-Project.iml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Python 3.12" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="Black">
+    <option name="sdkName" value="Python 3.12" />
+  </component>
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.12" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/Fly-Kite-AirLine-RAG-Project.iml" filepath="$PROJECT_DIR$/.idea/Fly-Kite-AirLine-RAG-Project.iml" />
+    </modules>
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,160 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="SELECTIVE" />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="43d0b45c-4ac8-44fd-8129-96b2dd008826" name="Changes" comment="" />
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="FileTemplateManagerImpl">
+    <option name="RECENT_TEMPLATES">
+      <list>
+        <option value="Python Script" />
+      </list>
+    </option>
+  </component>
+  <component name="FlaskConsoleOptions" custom-start-script="import sys; print('Python %s on %s' % (sys.version, sys.platform)); sys.path.extend([WORKING_DIR_AND_PYTHON_PATHS])&#10;from flask.cli import ScriptInfo, NoAppException&#10;for module in [&quot;main.py&quot;, &quot;wsgi.py&quot;, &quot;app.py&quot;]:&#10;    try: locals().update(ScriptInfo(app_import_path=module, create_app=None).load_app().make_shell_context()); print(&quot;\nFlask App: %s&quot; % app.import_name); break&#10;    except NoAppException: pass">
+    <envs>
+      <env key="FLASK_APP" value="app" />
+    </envs>
+    <option name="myCustomStartScript" value="import sys; print('Python %s on %s' % (sys.version, sys.platform)); sys.path.extend([WORKING_DIR_AND_PYTHON_PATHS])&#10;from flask.cli import ScriptInfo, NoAppException&#10;for module in [&quot;main.py&quot;, &quot;wsgi.py&quot;, &quot;app.py&quot;]:&#10;    try: locals().update(ScriptInfo(app_import_path=module, create_app=None).load_app().make_shell_context()); print(&quot;\nFlask App: %s&quot; % app.import_name); break&#10;    except NoAppException: pass" />
+    <option name="myEnvs">
+      <map>
+        <entry key="FLASK_APP" value="app" />
+      </map>
+    </option>
+  </component>
+  <component name="ProjectColorInfo">{
+  &quot;associatedIndex&quot;: 1
+}</component>
+  <component name="ProjectId" id="33s38NvoyHdVLzaauzWfPb9TnxM" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent">{
+  &quot;keyToString&quot;: {
+    &quot;ModuleVcsDetector.initialDetectionPerformed&quot;: &quot;true&quot;,
+    &quot;Python.HRPolicyApp.executor&quot;: &quot;Run&quot;,
+    &quot;Python.create_user_db.executor&quot;: &quot;Run&quot;,
+    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
+    &quot;RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252&quot;: &quot;true&quot;,
+    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
+    &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
+    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
+    &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
+    &quot;nodejs_package_manager_path&quot;: &quot;npm&quot;,
+    &quot;settings.editor.selected.configurable&quot;: &quot;preferences.pluginManager&quot;,
+    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
+  }
+}</component>
+  <component name="SharedIndexes">
+    <attachedChunks>
+      <set>
+        <option value="bundled-js-predefined-d6986cc7102b-3aa1da707db6-JavaScript-PY-252.26830.99" />
+        <option value="bundled-python-sdk-164cda30dcd9-0af03a5fa574-com.jetbrains.pycharm.pro.sharedIndexes.bundled-PY-252.26830.99" />
+      </set>
+    </attachedChunks>
+  </component>
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="43d0b45c-4ac8-44fd-8129-96b2dd008826" name="Changes" comment="" />
+      <created>1760091804577</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1760091804577</updated>
+      <workItem from="1760091890212" duration="859000" />
+      <workItem from="1760093438967" duration="2452000" />
+      <workItem from="1760096239520" duration="21456000" />
+      <workItem from="1760158047614" duration="33908000" />
+      <workItem from="1760276025638" duration="8385000" />
+      <workItem from="1760361998945" duration="294000" />
+      <workItem from="1760696201916" duration="16790000" />
+      <workItem from="1760716047458" duration="13878000" />
+      <workItem from="1760764472535" duration="4802000" />
+      <workItem from="1760802622597" duration="4499000" />
+      <workItem from="1760844577587" duration="42402000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/rag_pipeline.py</url>
+          <option name="timeStamp" value="2" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/evaluation/evaluator.py</url>
+          <line>245</line>
+          <option name="timeStamp" value="3" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/documents_processing/chunking.py</url>
+          <line>49</line>
+          <option name="timeStamp" value="4" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/evaluation/evaluator.py</url>
+          <line>396</line>
+          <option name="timeStamp" value="6" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/evaluation/evaluator.py</url>
+          <line>163</line>
+          <option name="timeStamp" value="10" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/rag_pipeline.py</url>
+          <line>151</line>
+          <option name="timeStamp" value="11" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/llm/llmResponse.py</url>
+          <line>28</line>
+          <option name="timeStamp" value="12" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/main.py</url>
+          <line>237</line>
+          <option name="timeStamp" value="13" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/main.py</url>
+          <line>296</line>
+          <option name="timeStamp" value="14" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/evaluation/evaluator.py</url>
+          <line>451</line>
+          <option name="timeStamp" value="15" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/llm/llmResponse.py</url>
+          <line>27</line>
+          <option name="timeStamp" value="16" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/rag_scripts/llm/llmResponse.py</url>
+          <option name="timeStamp" value="17" />
+        </line-breakpoint>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/main.py</url>
+          <line>248</line>
+          <option name="timeStamp" value="18" />
+        </line-breakpoint>
+      </breakpoints>
+    </breakpoint-manager>
+  </component>
+  <component name="com.intellij.coverage.CoverageDataManagerImpl">
+    <SUITE FILE_PATH="coverage/Fly_Kite_AirLine_RAG_Project$HRPolicyApp.coverage" NAME="HRPolicyApp Coverage Results" MODIFIED="1760718510975" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+    <SUITE FILE_PATH="coverage/Fly_Kite_AirLine_RAG_Project$create_user_db.coverage" NAME="create_user_db Coverage Results" MODIFIED="1760724265707" SOURCE_PROVIDER="com.intellij.coverage.DefaultCoverageFileProvider" RUNNER="coverage.py" COVERAGE_BY_TEST_ENABLED="false" COVERAGE_TRACING_ENABLED="false" WORKING_DIRECTORY="$PROJECT_DIR$" />
+  </component>
+</project>

DOCUMENTS/Flykite_Airlines_HR_Policy.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1a5cdeec566a672375800654a4d004e4c3b7907578bd8c177ac5755e1cdb6cd0
+size 262937

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim-bookworm
+LABEL authors="karthikeyan"
+
+RUN apt-get update && apt-get install -y \
+    build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY . .
+
+#COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+RUN mkdir -p data/TunedDB data/ChromaDB data/FIASS_DB
+
+EXPOSE 7860
+
+CMD ["streamlit", "run", "app.py", "--server.port=7860", "--server.address=0.0.0.0", "--server.enableXsrfProtection=false"]

HostingIntoHuggingFace.py

@@ -0,0 +1,73 @@
+import os
+import inspect
+import traceback
+from huggingface_hub import HfApi, create_repo, login,hf_hub_download
+from huggingface_hub.utils import RepositoryNotFoundError
+from configuration import Configuration
+class HostingInHuggingFace:
+  def __init__(self):
+    self.base_path = Configuration.PROJECT_ROOT
+    self.hf_token = Configuration.HF_TOKEN
+    self.repo_id = 'jpkarthikeyan/FlyKiteAirlines'
+
+  def CreatingSpaceInHF(self):
+    print(f"Function Name {inspect.currentframe().f_code.co_name}")
+    api = HfApi()
+    try:
+      print(f"Checking for {self.repo_id} is correct or not")
+      api.repo_info(repo_id = self.repo_id,
+                    repo_type='space',
+                    token = self.hf_token)
+      print(f"Space {self.repo_id} already exists")
+    except RepositoryNotFoundError:
+      create_repo(repo_id=self.repo_id,
+                  repo_type='space',
+                  space_sdk='docker',
+                  private=False,
+                  token=self.hf_token)
+      print(f"Space created in {self.repo_id}")
+    except Exception as ex:
+      print(f"Exception in creating space {ex}")
+      traceback.print_exc()
+    finally:
+      print('-'*50)
+
+
+  def UploadDeploymentFile(self):
+    print(f"Function Name {inspect.currentframe().f_code.co_name}")
+    try:
+      api = HfApi(token=self.hf_token)
+      #directory_to_upload = os.path.join(self.base_path,'Deployment')
+      directory_to_upload = self.base_path
+      print(f"Directory to upload {directory_to_upload} into HF Space {self.repo_id}")
+      api.upload_folder(repo_id=self.repo_id, folder_path=directory_to_upload,
+                        repo_type='space')
+      print(f"Successfully upload {directory_to_upload} into {self.repo_id}")
+      return True
+    except Exception as ex:
+      print(f"Exception occured {ex}")
+      print(traceback.print_exc())
+      return False
+    finally:
+      print('-'*50)
+
+  def ToRunPipeline(self):
+    try:
+      self.CreatingSpaceInHF()
+      if self.UploadDeploymentFile():
+        print('Deployment pipeline completed')
+        return True
+      else:
+        print('Deployment pipeline failed')
+        return False
+    except Exception as ex:
+      print(f"Exception occured {ex}")
+      print(traceback.print_exc())
+      return False
+    finally:
+      print('-'*50)
+
+
+if __name__ == '__main__':
+    hosting = HostingInHuggingFace()
+    hosting.ToRunPipeline()

README.md

@@ -1,10 +1,13 @@
----
-title: FlyKiteAirlines
-emoji: 🌍
-colorFrom: gray
-colorTo: yellow
-sdk: docker
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: FlyKite Airlines HR Policy
+emoji: ✈️ 🤗
+colorFrom: blue
+colorTo: green
+sdk: docker
+sdk_version: 3.9
+app_file: app.py
+app_type: streamlit
+pinned: false
+license: mit
+---
+The streamlit app is frontend for RAG App for FlyKite Airlines HR Policy

app.py

@@ -0,0 +1,272 @@
+import streamlit as st
+import subprocess
+import os
+import json
+import sqlite3
+import hashlib
+from typing import Optional, Dict, Any
+
+# Assuming Configuration.py and other rag_scripts are in the same directory or accessible via sys.path
+# You might need to adjust sys.path if your project structure is different
+import sys
+# Make sure this path is correct for your setup
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), 'src')))
+from configuration import Configuration
+
+# --- Helper functions (from your main.py or adapted) ---
+
+# Removed create_user_db() as per your request
+
+def authenticate_user(username, password) -> Optional[Dict[str,str]]:
+    hashed_password = hashlib.sha256(password.encode()).hexdigest()
+    conn = sqlite3.connect('users.db') # Ensure 'users.db' is accessible
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT username, jobrole, department, location FROM users WHERE username = ? AND password = ?",
+        (username, hashed_password)
+    )
+    user = cursor.fetchone()
+    conn.close()
+    if user:
+        return {"username": user[0], "role": user[1], "department": user[2], "location": user[3]}
+    return None
+
+def run_rag_job(job_type: str, **kwargs) -> Dict[str, Any]:
+    """
+    Executes a RAG job using the main.py script as a subprocess.
+    Returns the output as a dictionary (assuming main.py prints JSON).
+    """
+    cmd = [sys.executable, 'main.py', '--job', job_type]
+    for key, value in kwargs.items():
+        if value is not None and value != '':
+            if isinstance(value, bool):
+                if value:
+                    cmd.append(f'--{key.replace("_", "-")}')
+            else:
+                cmd.extend([f'--{key.replace("_", "-")}', str(value)])
+
+    # Inject user context if available
+    if 'user_context' in st.session_state and st.session_state.user_context:
+        user_ctx_str = json.dumps(st.session_state.user_context)
+        cmd.extend(['--user-context', user_ctx_str])
+
+    st.write(f"Running command: `{' '.join(cmd)}`") # For debugging
+    try:
+        process = subprocess.run(cmd, capture_output=True, text=True, check=True)
+        # st.success(f"Job '{job_type}' completed successfully.")
+        # Attempt to parse JSON output. main.py currently prints to stdout directly.
+        # We need to capture the last JSON block.
+        output_lines = process.stdout.strip().split('\n')
+        json_output = {}
+        for line in reversed(output_lines):
+            try:
+                json_output = json.loads(line)
+                break
+            except json.JSONDecodeError:
+                continue
+        return json_output
+    except subprocess.CalledProcessError as e:
+        st.error(f"Error running RAG job '{job_type}': {e.stderr}")
+        return {"error": e.stderr}
+    except Exception as e:
+        st.error(f"An unexpected error occurred: {e}")
+        return {"error": str(e)}
+
+# --- Streamlit UI Components ---
+
+def home_page():
+    st.title("Welcome to Flykite RAG System")
+
+    if 'logged_in' not in st.session_state:
+        st.session_state.logged_in = False
+    if 'user_info' not in st.session_state:
+        st.session_state.user_info = None
+
+    if not st.session_state.logged_in:
+        st.subheader("Login")
+        with st.form("login_form"):
+            username = st.text_input("Username")
+            password = st.text_input("Password", type="password")
+            login_button = st.form_submit_button("Login")
+
+            if login_button:
+                user_data = authenticate_user(username, password)
+                if user_data:
+                    st.session_state.logged_in = True
+                    st.session_state.user_info = user_data
+                    st.session_state.user_context = {
+                        "role": user_data['role'],
+                        "department": user_data['department'],
+                        "location": user_data['location']
+                    }
+                    st.success(f"Logged in as {user_data['username']} ({user_data['role']})")
+                    st.rerun() # Changed from st.experimental_rerun()
+                else:
+                    st.error("Invalid username or password.")
+    else:
+        st.write(f"You are logged in as **{st.session_state.user_info['username']}** (Role: **{st.session_state.user_info['role']}**)")
+        if st.button("Logout"):
+            st.session_state.logged_in = False
+            st.session_state.user_info = None
+            st.session_state.user_context = None
+            st.rerun() # Changed from st.experimental_rerun()
+
+def admin_page():
+    st.title("Admin Dashboard")
+    st.write(f"Logged in as: {st.session_state.user_info['username']} (Role: {st.session_state.user_info['role']})")
+
+    if st.session_state.user_info and st.session_state.user_info['role'] == 'admin':
+        st.header("RAG Hypertuning")
+        st.info("Run hyperparameter tuning to find the best RAG configuration and build a tuned index.")
+
+        with st.form("hypertune_form"):
+            st.write("Hypertuning parameters (default values from main.py if not overridden):")
+            llm_model_ht = st.selectbox("LLM Model for Hypertuning Evaluation",
+                                        options=["mixtral-8x7b-32768", "llama2-70b-4096", "gemma-7b-it"],
+                                        key="llm_model_ht_select")
+            n_iter = st.number_input("Number of Hyper-tuning Iterations (n_iter)", min_value=1, value=1, step=1, key="n_iter_ht")
+
+            hypertune_button = st.form_submit_button("Run Hypertune Job")
+
+            if hypertune_button:
+                st.write("Starting RAG Hypertuning. This may take a while...")
+                with st.spinner("Running hypertuning..."):
+                    result = run_rag_job('eval-hypertune', llm_model=llm_model_ht, n_iter=n_iter)
+                    if result and "error" not in result:
+                        st.success("Hypertuning completed and tuned index built!")
+                        st.json(result)
+                    else:
+                        st.error("Hypertuning failed.")
+
+        st.header("RAG Testing")
+        st.info("Test the RAG pipeline with a specific query, optionally using the tuned database.")
+
+        with st.form("rag_test_form"):
+            test_query = st.text_area("Enter a test query for the RAG system:",
+                                      value="What is the policy on annual leave?",
+                                      key="test_query_input")
+            use_tuned_db = st.checkbox("Use Tuned RAG Database (if hypertuned previously)", value=True, key="use_tuned_db_checkbox")
+            display_raw = st.checkbox("Display Raw Retrieved Documents only (no LLM)", key="display_raw_docs_checkbox")
+            k_value = st.slider("Number of documents to retrieve (k)", min_value=1, max_value=10, value=5, key="k_value_slider")
+
+            test_rag_button = st.form_submit_button("Run RAG Test Query")
+
+            if test_rag_button:
+                st.write("Running RAG test query...")
+                with st.spinner("Getting RAG response..."):
+                    # Pass the LLM model configured in the backend's Configuration if not specifically chosen for testing
+                    # Or, make a selectbox for it in the admin testing section too.
+                    llm_for_search = st.session_state.get('llm_model_ht_select', Configuration.DEFAULT_GROQ_LLM_MODEL)
+                    result = run_rag_job('search',
+                                         query=test_query,
+                                         use_tuned=use_tuned_db,
+                                         raw=display_raw,
+                                         k=k_value,
+                                         llm_model=llm_for_search) # Ensure LLM model is passed
+
+                    if result and "error" not in result:
+                        st.success("RAG Test Query Completed!")
+                        st.subheader("RAG Response:")
+                        if display_raw:
+                            st.json(result)
+                        else:
+                            if 'response' in result and 'summary' in result['response']:
+                                st.write(result['response']['summary'])
+                                if 'sources' in result['response'] and result['response']['sources']:
+                                    st.subheader("Sources:")
+                                    for source in result['response']['sources']:
+                                        st.markdown(f"- **Document ID:** {source.get('document_id', 'N/A')}, **Page:** {source.get('page', 'N/A')}, **Section:** {source.get('section', 'N/A')}, **Clause:** {source.get('clause', 'N/A')}")
+                            else:
+                                st.json(result)
+                        if 'evaluation' in result:
+                            st.subheader("Evaluation Results:")
+                            st.json(result['evaluation'])
+                    else:
+                        st.error("RAG test query failed.")
+    else:
+        st.warning("You do not have administrative privileges to view this page.")
+        if st.button("Go to User Page"):
+            st.session_state.page = "User"
+            st.rerun() # Changed from st.experimental_rerun()
+
+def user_page():
+    st.title("Flykite HR Policy Query")
+    st.write(f"Logged in as: {st.session_state.user_info['username']} (Role: {st.session_state.user_info['role']})")
+
+    st.info("Ask any question about the Flykite Airlines HR policy document.")
+
+    with st.form("user_query_form"):
+        user_query = st.text_area("Your Query:", height=100, key="user_query_input")
+        response_type = st.radio("Choose Response Type:",
+                                 options=["LLM Tuned Response (RAG + LLM)", "RAG Raw Response (Retrieved Docs Only)"],
+                                 index=0, key="response_type_radio")
+        k_value_user = st.slider("Number of documents to consider (k)", min_value=1, max_value=10, value=5, key="k_value_user_slider")
+
+        submit_query_button = st.form_submit_button("Get Answer")
+
+        if submit_query_button and user_query:
+            st.subheader("Response:")
+            with st.spinner("Fetching answer..."):
+                display_raw = (response_type == "RAG Raw Response (Retrieved Docs Only)")
+                # Assuming the user page should always use the best available LLM model from config or tuned params
+                llm_for_user_query = Configuration.DEFAULT_GROQ_LLM_MODEL # Default from config
+                # If a tuned model was selected during hypertuning, we might want to use that for users
+                # This requires main.py to save and provide which LLM was part of the best_params
+                # For now, we'll stick to a default or what's explicitly passed.
+                result = run_rag_job('search',
+                                     query=user_query,
+                                     raw=display_raw,
+                                     k=k_value_user,
+                                     use_tuned=True, # User page always uses tuned if available
+                                     llm_model=llm_for_user_query) # Pass LLM model for search
+
+                if result and "error" not in result:
+                    if display_raw:
+                        st.json(result)
+                    else:
+                        if 'response' in result and 'summary' in result['response']:
+                            st.markdown(result['response']['summary'])
+                            if 'sources' in result['response'] and result['response']['sources']:
+                                st.subheader("Sources:")
+                                for source in result['response']['sources']:
+                                    st.markdown(f"- **Document ID:** {source.get('document_id', 'N/A')}, **Page:** {source.get('page', 'N/A')}, **Section:** {source.get('section', 'N/A')}, **Clause:** {source.get('clause', 'N/A')}")
+                        else:
+                            st.json(result)
+                else:
+                    st.error("Failed to get a response. Please try again.")
+        elif submit_query_button and not user_query:
+            st.warning("Please enter a query.")
+
+# --- Main Application Logic ---
+def main_app():
+    # create_user_db() # Removed this call as per your request
+
+    st.sidebar.title("Navigation")
+    if 'logged_in' not in st.session_state:
+        st.session_state.logged_in = False
+    if 'page' not in st.session_state:
+        st.session_state.page = "Home"
+
+    if not st.session_state.logged_in:
+        st.session_state.page = "Home" # Force to home/login if not logged in
+        home_page()
+    else:
+        # Navigation for logged-in users
+        if st.session_state.user_info and st.session_state.user_info['role'] == 'admin':
+            st.sidebar.button("Home", on_click=lambda: st.session_state.update(page="Home"))
+            st.sidebar.button("Admin Dashboard", on_click=lambda: st.session_state.update(page="Admin"))
+            st.sidebar.button("User Query", on_click=lambda: st.session_state.update(page="User"))
+        else: # Regular user
+            st.sidebar.button("Home", on_click=lambda: st.session_state.update(page="Home"))
+            st.sidebar.button("User Query", on_click=lambda: st.session_state.update(page="User"))
+
+        # Display the selected page
+        if st.session_state.page == "Home":
+            home_page()
+        elif st.session_state.page == "Admin":
+            admin_page()
+        elif st.session_state.page == "User":
+            user_page()
+
+if __name__ == "__main__":
+    main_app()

configuration.py

@@ -0,0 +1,62 @@
+import os
+from dotenv import load_dotenv
+
+load_dotenv()
+
+class Configuration:
+    GROQ_API_KEY = os.getenv('GROQ_API_KEY','default_groq_key')
+    OPEN_API_KEY = os.getenv('OPENAI_API_KEY','default_openai_key')
+    HF_TOKEN = os.getenv('HF_TOKEN','default_hf_token')
+
+    DEFAULT_CHUNK_SIZE = int(os.getenv('DEFAULT_CHUNK_SIZE'))
+    DEFAULT_CHUNK_OVERLAP = int(os.getenv('DEFAULT_CHUNK_OVERLAP'))
+    DEFAULT_SENTENCE_TRANSFORMER_MODEL = os.getenv('DEFAULT_SENTENCE_TRANSFORMER_MODEL','all-MiniLM-L6-v2')
+    DEFAULT_GROQ_LLM_MODEL = os.getenv('DEFAULT_GROQ_LLM_MODEL','llama-3.3-70b-versatile')
+    DEFAULT_RERANKER = os.getenv('DEFAULT_RERANKER')
+
+    PROJECT_ROOT_BASE = os.path.abspath(os.path.dirname(__file__))
+    print(f"PROJECT_ROOT {PROJECT_ROOT_BASE}")
+
+    if os.path.basename(PROJECT_ROOT_BASE) == "src":
+        ##PROJECT_ROOT = os.path.abspath(os.path.join(PROJECT_ROOT_BASE,os.pardir))
+        PROJECT_ROOT = os.path.abspath(os.path.join(PROJECT_ROOT_BASE, '..'))
+    else:
+        PROJECT_ROOT = PROJECT_ROOT_BASE
+
+
+    DOCUMENTS_DIR = os.path.join(PROJECT_ROOT,'DOCUMENTS')
+    print(f"DOCUMENTS_DIR {DOCUMENTS_DIR}")
+    DATA_DIR = os.path.join(PROJECT_ROOT,'DATA')
+
+
+    PDF_FILE_NAME = os.getenv('PDF_FILE_NAME')
+    FULL_PDF_PATH = os.path.join(DOCUMENTS_DIR,PDF_FILE_NAME)
+    print(f"FULL_PDF_PATH: {FULL_PDF_PATH} ")
+
+
+    CHROMA_DB_PATH = os.path.join(DATA_DIR,os.getenv('CHROMA_DB_PATH'))
+    FAISS_DB_PATH = os.path.join(DATA_DIR,os.getenv('FAISS_DB_PATH'))
+
+    COLLECTION_NAME = os.getenv('COLLECTION_NAME')
+    EVAL_DATA_PATH = os.path.join(PROJECT_ROOT, os.getenv('EVAL_DATA_PATH'))
+    
+    PINECONE_API_KEY=os.getenv('PINECONE_API_KEY')
+    PINECONE_CLOUD=os.getenv('PINECONE_CLOUD','aws')
+    PINECONE_REGION=os.getenv('PINECONE_REGION','us-east-1')
+
+    os.makedirs(DATA_DIR, exist_ok=True)
+    os.makedirs(DOCUMENTS_DIR,exist_ok=True)
+    os.makedirs(CHROMA_DB_PATH, exist_ok=True)
+    os.makedirs(FAISS_DB_PATH,exist_ok=True)
+
+    if not os.path.exists(FULL_PDF_PATH):
+        print(f"PDF not found in {FULL_PDF_PATH}")
+    else:
+        print(f"PDF file found in {FULL_PDF_PATH}")
+
+    if not os.path.exists(EVAL_DATA_PATH):
+        print(f"eval json file not found in {EVAL_DATA_PATH}")
+    else:
+        print(f"eval file found in {EVAL_DATA_PATH}")
+
+

create_user_db.py

@@ -0,0 +1,47 @@
+import sqlite3
+import hashlib
+import traceback
+
+from argon2 import hash_password
+
+
+class CreateUserDB():
+    def encrypttion_pwd(self,password):
+        return hashlib.sha256(password.encode()).hexdigest()
+
+    def setup_database(self):
+        try:
+            conn = sqlite3.connect('users.db')
+            cursor = conn.cursor()
+            cursor.execute('''
+                    CREATE TABLE IF NOT EXISTS users (
+                    username TEXT PRIMARY KEY,
+                    password TEXT,                
+                    jobrole TEXT,
+                    department TEXT,
+                    location TEXT)
+                    ''')
+            cursor.execute("SELECT COUNT(*) FROM users")
+            if cursor.fetchone()[0] == 0:
+                users = [('admin', self.encrypttion_pwd('admin'), 'admin', 'admin', 'chennai'),
+                    ('user1', self.encrypttion_pwd('user'), 'user', 'manager', 'chennai'),
+                    ('user2', self.encrypttion_pwd('user'), 'user', 'pilot', 'chennai'),
+                    ('user3', self.encrypttion_pwd('user'), 'user', 'engineer', 'chennai'),
+                    ('user4', self.encrypttion_pwd('user'), 'user', 'cabin crew', 'chennai'),
+                    ('user5', self.encrypttion_pwd('user'), 'user', 'ground staff', 'chennai')]
+                cursor.executemany("INSERT INTO users VALUES(?,?,?,?,?)",users)
+                conn.commit()
+            else:
+                print("Databases alerady contains users")
+
+        except Exception as ex:
+            traceback.print_exc()
+            print(f"Exception occurred: {ex}")
+        finally:
+            if conn:
+                conn.close()
+
+
+if __name__ == "__main__":
+    db_obj = CreateUserDB()
+    db_obj.setup_database()

eval_data.json

@@ -0,0 +1,33 @@
+[
+    {
+        "query":"what are the Criteria for Probation Extension?",
+        "expected_keywords":["60%", "PIP", "5 working days", "7 calendar days","measurable target"],
+        "expected_answer_snippet":"Extensions are granted only if: a. The employee has achieved at least 60% of their probationary objectives. b. A written PIP with measurable targets is issued within 5 working days of the original probation end date. c. The Department Head and HR Manager both sign off on the extension request. Employees will be notified of extensions in writing at least 7 calendar days before the probation end date."
+    },
+    {
+        "query":"Explain Exit Procedure & Timeline of FlyKite airline",
+        "expected_keywords":["3 working days", "within 7 working days", "Exit interview","no later than 5 working days"],
+        "expected_answer_snippet":"Company property (ID badge, uniforms, devices) must be returned on or before the last working day. Clearance forms must be signed by Finance, IT, and Admin within 3 working days post-termination. Final salary settlement is processed within 7 working days of clearance completion. Exit interview: Mandatory and to be scheduled no later than 5 working days after last working day. Failure to attend delays issuance of relieving letter."
+
+    },
+    {
+        "query":"Elaborate Allowable Expenses and Reimbursement Procedures",
+        "expected_keywords":["Eligibility","Exclusions", "Submission Deadlines", "Appeals","directly work-related","itemized","within 15 calendar days","7 working days"],
+        "expected_answer_snippet":"1. Eligibility: Expenses must be directly work-related and supported by itemized receipts. Per diem limits: ₹1,200/day domestic, ₹4,000/day international. Exclusions: Alcohol, entertainment unrelated to work, and non-economy travel (unless pre-approved) are not reimbursable. Submission Deadlines: Claims must be fi led within 15 calendar days of incurring expense. Appeals: Appeal must be submitted within 7 working days of claim denial with"
+    },
+    {
+        "query": "What are the documents required for special leave approval ?",
+        "expected_keywords": ["Death certificate", "funeral", "notice","obituary","court summons", "jury duty letter"],
+        "expected_answer_snippet": "Special leave is approved only upon submission of: 1. Death certificate, funeral notice, or obituary (bereavement). 2. Official court summons or jury duty letter (legal obligations). 3. Medical certificate from a registered practitioner (emergency care). 4. Government-issued disaster report or evacuation notice (natural disasters). 5. All documents must be submitted within 5 working days of returning to duty."
+    },
+    {
+        "query":"explain in detail about the Attendance and Absence Management of the flykite airlne hr policy",
+        "expected_keywords":["Core Hours","Notification","Consequences"],
+        "expected_answer_snippet": "1. Core Hours: 9:30 AM – 6:00 PM IST. 2. Notification: Planned absence: Email supervisor at least 1 day before. Emergency absence: Call within 1 hour of shift start. 3. Consequences:  3 unreported absences in 60 days → Written warning. 5 unreported absences in 90 days → Termination review."
+    },
+    {
+        "query":"Explain the compensation and termination policy of the flykite airline hr policy",
+        "expected_keywords":["Salary review cycle", "Mid-year adjustments", "Benefits eligiblity","Termination Impact"],
+        "expected_answer_snippet":"1. Salary Review Cycle: April annually. 2. Mid-Year Adjustments: Only on written approval from CEO & CFO. 3. Benefits Eligibility Health insurance starts after 30 days service (full-time). Retirement plan enrollment: Within 60 days of confirmation. 4. Termination Impact: Benefits end on last working day unless law mandates otherwise."
+    }
+]

main.py

@@ -0,0 +1,541 @@
+import os
+import sys
+import json
+import argparse
+import warnings
+import traceback
+import logging
+import chromadb
+import hashlib
+import sqlite3
+import regex as re
+from pinecone import Pinecone
+from typing import Optional, Dict, Any
+from sentence_transformers import SentenceTransformer, util
+
+
+os.environ["TF_CPP_MIN_LOG_LEVEL"]="3"
+warnings.filterwarnings("ignore")
+
+sys.path.insert(0,os.path.abspath(os.path.join(os.path.dirname(__file__),'src')))
+from sentence_transformers import SentenceTransformer
+from configuration import Configuration
+from rag_scripts.rag_pipeline import RAGPipeline
+from rag_scripts.documents_processing.chunking import PyMuPDFChunker
+from rag_scripts.embedding.embedder import SentenceTransformerEmbedder
+from rag_scripts.embedding.vector_db.chroma_db import chromaDBVectorDB
+from rag_scripts.embedding.vector_db.faiss_db import FAISSVectorDB
+from rag_scripts.embedding.vector_db.pinecone_db import PineconeVectorDB
+from rag_scripts.llm.llmResponse import GROQLLM
+from rag_scripts.evaluation.evaluator import RAGEvaluator
+
+class RAGOperations:
+    VALID_VECTOR_DB = {'chroma','faiss','pinecone'}
+
+    @staticmethod
+    def check_db(vector_db_type: str, db_path: str, collection_name: str) -> bool:
+        try:
+            if vector_db_type not in RAGOperations.VALID_VECTOR_DB:
+                print(f"Invalid Vector DB: {vector_db_type}")
+                raise
+            if vector_db_type.lower() == 'pinecone':
+                pc = Pinecone(api_key=Configuration.PINECONE_API_KEY)
+                return collection_name in pc.list_indexes().names()
+            elif vector_db_type.lower() == 'chroma':
+                return os.path.exists(db_path) and os.listdir(db_path)
+            elif vector_db_type.lower() == "faiss":
+                faiss_index_file = os.path.join(db_path,f"{collection_name}.faiss")
+                faiss_doc_store_file = os.path.join(db_path,f"{collection_name}_docs.pkl")
+                return os.path.exists(faiss_index_file) and os.path.exists(faiss_doc_store_file)
+        except Exception as ex:
+            print(f"Exception in checking {vector_db_type} existence")
+            traceback.print_exc()
+            return False
+
+    @staticmethod
+    def get_pipeline_params(args: argparse.Namespace, use_tuned: bool = False) -> Dict[str,Any]:
+        try:
+            best_param_path = os.path.join(Configuration.DATA_DIR,'best_params.json')
+            params = {
+                'document_path':Configuration.FULL_PDF_PATH,
+                'chunk_size':args.chunk_size,
+                'chunk_overlap':args.chunk_overlap,
+                'embedding_model_name':args.embedding_model,
+                'vector_db_type':args.vector_db_type,
+                'llm_model_name':args.llm_model,
+                'db_path': None,
+                'collection_name': Configuration.COLLECTION_NAME,
+                'vector_db': None,
+                'temperature': args.temperature,
+                'top_p':args.top_p,
+                'max_tokens':args.max_tokens,
+                're_ranker_model':args.re_ranker_model
+            }
+
+            if os.path.exists(best_param_path):
+                with open(best_param_path,'rb') as f:
+                    best_params = json.load(f)
+                print(f"Best params: {best_params} from the file {best_param_path}")
+
+                params.update({
+                    'vector_db_type': best_params.get('vector_db_type',params['vector_db_type']),
+                    'embedding_model_name': best_params.get('embedding_model',params['embedding_model_name']),
+                    'chunk_overlap': best_params.get('chunk_overlap',params['chunk_overlap']),
+                    'chunk_size': best_params.get('chunk_size',params['chunk_size']) ,
+                    're_ranker_model': best_params.get('re_ranker_model',params['re_ranker_model']) })
+                use_tuned = True
+
+            if use_tuned:
+                tuned_db_type = params['vector_db_type']
+                params['db_path'] = os.path.join(Configuration.DATA_DIR,'TunedDB',tuned_db_type) if tuned_db_type != 'pinecone' else ""
+                params['collection_name'] = 'tuned-'+Configuration.COLLECTION_NAME
+                if tuned_db_type in  ['chroma','faiss']:
+                    os.makedirs(params['db_path'],exist_ok=True)
+                print(f"Tuned db path: {params['db_path']}")
+            else:
+                params['db_path'] = ( Configuration.CHROMA_DB_PATH if params['vector_db_type'] == 'chroma'
+                                      else Configuration.FAISS_DB_PATH if params['vector_db_type'] == 'faiss'
+                                      else "")
+                if params['vector_db_type'] in ['chroma', 'faiss']:
+                    os.makedirs(params['db_path'],exist_ok=True)
+                    print(f"Created directory for {params['vector_db_type']} at {params['db_path']}")
+
+            return params
+        except Exception as ex:
+            print(f"Exception in get_pipeline_params: {ex}")
+            traceback.print_exc()
+            sys.exit(1)
+
+
+    @staticmethod
+    def check_embedding_dimension(vector_db_type: str,db_path: str,
+                                  collection_name: str, embedding_model: str) -> bool:
+        if vector_db_type !='chroma':
+            return True
+        try:
+            client = chromadb.PersistentClient(path=db_path)
+            collection = client.get_collection(collection_name)
+            model = SentenceTransformer(embedding_model)
+            sample_embedding = model.encode(["test"])[0]
+            try:
+                expected_dim = collection._embedding_function.dim
+            except AttributeError:
+                peek_result = collection.peek(limit=1)
+                if 'embedding' in peek_result and peek_result['embedding']:
+                    expected_dim = len(peek_result['embedding'][0])
+                else:
+                    return False
+            actual_dim = len(sample_embedding)
+            print(f"Expected dimension: {expected_dim} Actual dimension: {actual_dim}")
+            return expected_dim == actual_dim
+        except Exception as ex:
+            print(f"Error checking embedding dimension: {ex}")
+            return False
+
+
+    @staticmethod
+    def initialize_pipeline(params: dict[str,Any]) -> RAGPipeline:
+        try:
+            embedder = SentenceTransformerEmbedder(model_name=params['embedding_model_name'])
+            chunkerObj = PyMuPDFChunker(
+                pdf_path=params['document_path'],
+                chunk_size=params['chunk_size'],
+                chunk_overlap=params['chunk_overlap'])
+            llm_model = params['llm_model_name']
+            vector_db = None
+            if params['vector_db_type'] == 'chroma':
+                vector_db = chromaDBVectorDB(embedder=embedder,
+                                db_path=params['db_path'],
+                                collection_name=params['collection_name'])
+            elif params['vector_db_type'] == 'faiss':
+                vector_db = FAISSVectorDB(embedder=embedder,
+                                db_path=params['db_path'],
+                                collection_name=params['collection_name'] )
+            elif params['vector_db_type'] == 'pinecone':
+                vector_db = PineconeVectorDB(embedder=embedder,
+                                db_path=params['db_path'],
+                                collection_name=params['collection_name'])
+            else:
+                raise ValueError(f"Unknown vector_db_type: {params['vector_db_type']}")
+
+            return RAGPipeline( document_path=params['document_path'],
+                chunker=chunkerObj, embedder=embedder,
+                vector_db=vector_db,
+                llm=GROQLLM(model_name= llm_model),
+                re_ranker_model_name=params['re_ranker_model'] if params['re_ranker_model'] else Configuration.DEFAULT_RERANKER,)
+        except Exception as ex:
+            print(f"Exception in pipeline initialize: {ex}")
+            traceback.print_exc()
+            sys.exit(1)
+
+    @staticmethod
+    def run_build_job(args: argparse.Namespace) -> None:
+        try:
+            params = RAGOperations.get_pipeline_params(args)
+            pipeline = RAGOperations.initialize_pipeline(params)
+            pipeline.build_index()
+            print(f"RAG Build JOB completed")
+        except Exception as ex:
+            print(f"Exception in run build job: {ex}")
+            traceback.print_exc()
+            sys.exit(1)
+
+
+    @staticmethod
+    def run_search_job(args: argparse.Namespace,user_info: Dict[str,str]) -> None:
+        try:
+            params = RAGOperations.get_pipeline_params(args, use_tuned=args.use_tuned)
+            vector_db_type = params['vector_db_type']
+            db_path = params['db_path']
+            collection_name = params['collection_name']
+
+            pipeline = RAGOperations.initialize_pipeline(params)
+            db_exists = RAGOperations.check_db(vector_db_type,db_path,collection_name)
+
+            if args.use_rag:
+                if not db_exists:
+                    pipeline.build_index()
+                elif pipeline.vector_db.count_documents() == 0:
+                    pipeline.build_index()
+                elif not RAGOperations.check_embedding_dimension(vector_db_type,db_path,
+                                                                 collection_name,params['embedding_model_name'] ):
+                    print(f"Embedding dimension mismatch. rebuilding the index")
+                    pipeline.vector_db.delete_collection(collection_name)
+                    pipeline.build_index()
+
+                else:
+                    print(f"Using existing {vector_db_type} database with collection: {collection_name}")
+
+                if pipeline.vector_db.count_documents() == 0:
+                    print(f"No Documents found in vector database after re-build")
+                    sys.exit(1)
+
+            evaluator = RAGEvaluator(eval_data_path=Configuration.EVAL_DATA_PATH,
+                                     pdf_path=Configuration.FULL_PDF_PATH)
+
+            user_query = args.query if args.query else  (
+                            input("Enter your Query: "))
+            if user_query.lower() == 'exit':
+                return
+            user_context = {"role": user_info['role'],
+                            "location": user_info['location'],
+                            "department":user_info['department'] }
+
+            expected_answers = None
+            expected_keywords = []
+            query_found = False
+            try:
+                with open(Configuration.EVAL_DATA_PATH, 'r') as f:
+                    eval_data = json.load(f)
+                for item in eval_data:
+                    if item.get('query').strip().lower() == user_query.strip().lower():
+                        expected_keywords = item.get('expected_keywords',[])
+                        expected_answers = item.get('expected_answer_snippet',"")
+                        query_found = True
+                        break
+                if not expected_keywords and not expected_answers:
+                    print(f"No evaluation data found for query in json")
+            except Exception as ex:
+                print(f"No json file : {ex}")
+            retrieved_documents = []
+            if args.raw:
+                retrieved_documents = pipeline.retrieve_raw_documents(
+                                        user_query, k=args.k*2)
+                print("Raw documents retrieved")
+                print(json.dumps(retrieved_documents, indent=4))
+                if not retrieved_documents:
+                    response ={"summary":"No relevant documents found",
+                               "sources":[]}
+                else:
+                    #print("Similar documents retrieved")
+                    query_embedding = evaluator.embedder.encode(user_query,
+                                            convert_to_tensor=True,normalize_embeddings=True)
+                    similarities = [(doc, util.cos_sim(query_embedding,
+                                                       evaluator.embedder.encode(doc['content'],
+                                                                    convert_to_tensor=True,
+                                                                    normalize_embeddings=True)).item())
+                                    for doc in retrieved_documents]
+                    similarities.sort(key=lambda x: x[1], reverse=True)
+                    #print(f"Similar documents retrieved: {similarities}")
+                    top_docs = similarities[:min(3, len(similarities))]
+                    #print("Top documents retrieved")
+                    #print(json.dumps(top_docs, indent=4))
+                    truncated_content = []
+                    for doc, sim in top_docs:
+                        content_paragraphs = re.split(r'\n\s*\n', doc['content'].strip())
+                        para_sims = [(para, util.cos_sim(query_embedding,
+                                                         evaluator.embedder.encode(para.strip(), convert_to_tensor=True,
+                                                                                   normalize_embeddings=True)).item())
+                                     for para in content_paragraphs if para.strip()]
+                        para_sims.sort(key=lambda x: x[1], reverse=True)
+
+                        top_paras = [para for para, para_sim in para_sims[:2] if para_sim >= 0.3]
+                        if len(top_paras) < 1:  # Fallback to at least one paragraph
+                            top_paras = [para for para, _ in para_sims[:1]]
+                        truncated_content.append('\n\n'.join(top_paras))
+                    #print(f"Truncated content: {truncated_content}")
+                    response = {
+                    "summary": "\n".join(truncated_content),
+                    "sources":[{ "document_id":f"DOC {idx+1}",
+                        "page": str(doc['metadata'].get("page_number","NA")),
+                        "section": doc['metadata'].get("section","NA"),
+                        "clause": doc['metadata'].get("clause","NA")}
+                        for idx,(doc,_) in enumerate(top_docs)] }
+
+            else:
+                print("LLM+RAG")
+                response = pipeline.query(user_query, k=args.k,
+                                          include_metadata=True,
+                                          user_context=user_context
+                                          )
+                retrieved_documents = pipeline.retrieve_raw_documents(
+                    user_query, k=args.k)
+
+
+            final_expected_answer = expected_answers if expected_answers is not None else ""
+            additional_eval_metrices = {}
+            if not query_found:
+                print(f"No query found in eval_Data.json: {user_query}")
+                raw_reference_for_score = evaluator._syntesize_raw_reference(retrieved_documents)
+                if not final_expected_answer.strip():
+                    final_expected_answer = raw_reference_for_score
+
+                retrieved_documents_content = [doc.get('content','') for doc in retrieved_documents]
+                llm_as_judge = evaluator._evaluate_with_llm(user_query, response.get('summary',''),retrieved_documents_content)
+                if llm_as_judge:
+                    additional_eval_metrices.update(llm_as_judge)
+                    output = {"query": user_query, "response": response, "evaluation": additional_eval_metrices}
+                    print(json.dumps(output, indent=4))
+                    return json.dumps(output)
+                else:
+                    output = { "query": user_query, "response":response, "evaluation":llm_as_judge }
+                    print(json.dumps(output, indent=4))
+                    return json.dumps(output)
+
+            else:
+
+                eval_result = evaluator.evaluate_response(user_query, response, retrieved_documents,
+                                                expected_keywords, expected_answers)
+                output = { "query": user_query, "response":response, "evaluation":eval_result }
+                print(json.dumps(output,indent=2,ensure_ascii=False))
+
+                return json.dumps(output)
+
+
+        except Exception as ex:
+            print(f"Exception in run search job {ex}")
+            traceback.print_exc()
+
+    @staticmethod
+    def run_hypertune_job(args: argparse.Namespace) -> None:
+        try:
+            evaluator = RAGEvaluator(eval_data_path=Configuration.EVAL_DATA_PATH,
+                                     pdf_path=Configuration.FULL_PDF_PATH)
+
+            result = evaluator.evaluate_combined_params_grid(
+                chunk_size_to_test=[512,1024,2048],
+                chunk_overlap_to_test=[100,200,400],
+                embedding_models_to_test=["all-MiniLM-L6-v2",
+                                         "all-mpnet-base-v2",
+                                         "paraphrase-MiniLM-L3-v2",
+                                         "multi-qa-mpnet-base-dot-v1" ],
+                vector_db_types_to_test=['pinecone'],
+                llm_model_name=args.llm_model,
+                re_ranker_model = [ "cross-encoder/ms-marco-MiniLM-L-6-v2",
+                    "cross-encoder/ms-marco-TinyBERT-L-2"],
+                search_type='random', n_iter=1 )
+            # embedding_models_to_test = ["all-MiniLM-L6-v2",
+            #                             "all-mpnet-base-v2",
+            #                             "paraphrase-MiniLM-L3-v2",
+            #                             "multi-qa-mpnet-base-dot-v1"]
+            best_parameter = result['best_params']
+            best_score = result['best_score']
+            pkl_file = result['pkl_file']
+            best_metrics = result['best_metrics']
+
+            best_param_path = os.path.join(Configuration.DATA_DIR,'best_params.json')
+
+            with open(best_param_path, 'w') as f:
+                json.dump(best_parameter, f, indent=4)
+
+            tuned_db = best_parameter['vector_db_type']
+            tuned_path = os.path.join(Configuration.DATA_DIR,'TunedDB',tuned_db)
+            if tuned_db != 'pinecone':
+                os.makedirs(tuned_path, exist_ok=True)
+            tuned_collection_name = "tuned-"+Configuration.COLLECTION_NAME
+
+            tuned_params = {
+                'document_path': Configuration.FULL_PDF_PATH,
+                'chunk_size': best_parameter.get('chunk_size', Configuration.DEFAULT_CHUNK_SIZE),
+                'chunk_overlap': best_parameter.get('chunk_overlap',Configuration.DEFAULT_CHUNK_OVERLAP),
+                'embedding_model_name': best_parameter.get('embedding_model',Configuration.DEFAULT_SENTENCE_TRANSFORMER_MODEL),
+                'vector_db_type': tuned_db,
+                'llm_model_name':args.llm_model,
+                'db_path':tuned_path if tuned_db !='pinecone' else "",
+                'collection_name':tuned_collection_name,
+                'vector_db': None,
+                're_ranker_model':best_parameter.get('re_ranker', Configuration.DEFAULT_RERANKER)
+            }
+
+            if 're_ranker_model' in best_parameter:
+                tuned_params['re_ranker_model'] = best_parameter['re_ranker_model']
+            else:
+                tuned_params['re_ranker_model'] = Configuration.DEFAULT_RERANKER
+
+            tuned_pipeline = RAGOperations.initialize_pipeline(tuned_params)
+            tuned_pipeline.build_index()
+
+        except Exception as ex:
+            print(f"Exception in hypertune: {ex} ")
+            traceback.print_exc()
+
+
+    @staticmethod
+    def run_llm_with_prompt(args: argparse.Namespace,run_type: str) -> None:
+        try:
+            params = RAGOperations.get_pipeline_params(args,
+                                        use_tuned=args.use_tuned)
+            pipeline = RAGOperations.initialize_pipeline(params)
+
+
+            evaluator = RAGEvaluator(eval_data_path=Configuration.EVAL_DATA_PATH,
+                                     pdf_path=Configuration.FULL_PDF_PATH)
+
+            system_message = (
+                "You are an expert assistant for Flykite Airlines HR Policy Queries."
+                "Provide concise, accurate and policy-specific answers based solely on the the provided context."
+                "Structured your response clearly, using bullet points, newlines if applicable. "
+                "If the context lacks information, state that clearly and speculation."
+            ) if run_type == 'prompting' else None
+
+            user_query = input("Enter your query: ")
+            expected_answer = None
+            expected_keywords = []
+            try:
+                with open(Configuration.EVAL_DATA_PATH, 'r') as f:
+                    eval_data= json.load(f)
+                for item in eval_data:
+                    expected_answer = item.get('expected_answer_snippet',"")
+                    expected_keywords = item.get('expected_keywords',[])
+                    break
+            except Exception as ex:
+                print(f"Error loading eval_data.json for query {user_query}: {ex}")
+
+            if run_type == 'prompting':
+                prompt = (
+                    f"You are an expert assistant for Flykite Airlines HR Policy Queries."
+                    f"Answer the following question with a structured response, using bullet points or sections where applicable"
+                    f"Base your answer solely on the query and avoid hallucination"
+                    f"Question: \n {user_query} \n"
+                    f"Answer: ")
+
+            else:
+                prompt = user_query
+
+            response = pipeline.llm.generate_response(
+                    prompt=prompt,
+                    system_message=system_message,
+                    temperature = args.temperature,
+                    top_p = args.top_p,
+                    max_tokens = args.max_tokens
+                )
+            retreived_documents = []
+
+            eval_result = evaluator.evaluate_response(user_query,
+                                                response,
+                                                retreived_documents,
+                                                expected_keywords,
+                                                expected_answer)
+
+            output = {  "query":user_query,
+                        "response": {
+                             "summary: ":response.strip(),
+                            "source: ":["LLM Response Not RAG loaded"]},
+                        "evaluation": eval_result }
+
+
+            print(json.dumps(output, indent=2))
+
+        except Exception as ex:
+            print(f"Exception in LLm_prompting response: {ex}")
+            traceback.print_exc()
+            sys.exit(1)
+
+    @staticmethod
+    def login() -> Dict[str,str]:
+        username = input("Enter your username: ")
+        password = input("Enter your password: ")
+
+        hashed_password = hashlib.sha256(password.encode()).hexdigest()
+        try:
+            conn = sqlite3.connect('users.db')
+            cursor = conn.cursor()
+            cursor.execute(
+                "SELECT username,jobrole,department,location FROM users WHERE username = ? AND password = ?",
+                (username, hashed_password)
+            )
+            user = cursor.fetchone()
+            print(f"{user}")
+            conn.close()
+            if user:
+                return {"username": user[0], "role": user[1],"department": user[2],"location": user[3]}
+            else:
+                print("Invalid username or password")
+                sys.exit(1)
+
+        except sqlite3.Error as ex:
+
+            return False
+
+
+
+
+
+
+def main():
+
+    user_info = RAGOperations.login()
+    print(f"Logged in as {user_info['username']}"           
+          f"{user_info['department']} at {user_info['location']}")
+
+    parser = argparse.ArgumentParser(description='RAG PIPELINE OPERATIONS')
+
+    parser.add_argument('--job',type=str, required=True,
+                        choices=['rag-build','search','eval-hypertune','llm','prompting'],
+                        help='job to execute build RAG index, search and hypertune')
+
+    parser.add_argument('--raw', action='store_true',
+                        help='dispaly raw retrieved documents in json format')
+    parser.add_argument('--chunk_size', type=int, default=Configuration.DEFAULT_CHUNK_SIZE,help="Document chunking")
+    parser.add_argument('--chunk_overlap', type=int, default=Configuration.DEFAULT_CHUNK_OVERLAP,help="Document overlap")
+    parser.add_argument('--embedding_model', type=str, default=Configuration.DEFAULT_SENTENCE_TRANSFORMER_MODEL,help="Embedding model")
+    parser.add_argument('--vector_db_type', type=str,default="chroma",choices=['chroma','faiss','pinecone'],help='vector database type')
+    parser.add_argument('--llm_model',type=str, default=Configuration.DEFAULT_GROQ_LLM_MODEL)
+    parser.add_argument('--use-tuned',action='store_true',help='Use the tuned DB for search')
+    parser.add_argument('--k',type=int, default=5,help='number of doc to retrieve')
+    parser.add_argument('--query',type=str, default=None,help='query from user')
+    parser.add_argument('--temperature', type=float, default=0.1,help='LLM temperature for response generation')
+    parser.add_argument('--top_p', type=float, default=0.95, help='LLM top_p for response generation')
+    parser.add_argument('--max_tokens', type=int, default = 1000, help='LLM max tokens for response generation')
+    parser.add_argument('--use-rag', type=lambda x: x.lower() =='true',default=True, help ='llm or prompting')
+    parser.add_argument('--user-context', type=str, default=None, help='user context as JSON e.g., {"role": "admin", "location":"chennai", "designation":"engineer"}')
+    parser.add_argument('--fine-tune', action='store_true', help='use fine-tuned model')
+    parser.add_argument('--n-iter',type=int, default=10,help='number of iterations')
+    parser.add_argument('--re_ranker_model',type=str,default=Configuration.DEFAULT_RERANKER,help="ReRanking the retrieval documents")
+    args = parser.parse_args()
+
+    if args.job == 'eval-hypertune' and user_info['role']!= 'admin':
+        print(f"Access denied: Hypertune execution is forbidden")
+
+    if args.job == 'rag-build':
+        RAGOperations.run_build_job(args)
+    elif args.job == 'search':
+        RAGOperations.run_search_job(args,user_info)
+    elif args.job == 'eval-hypertune':
+        RAGOperations.run_hypertune_job(args)
+    elif args.job in ['llm','prompting']:
+        RAGOperations.run_llm_with_prompt(args, args.job)
+
+
+if __name__ == '__main__':
+    main()

rag_scripts/documents_processing/chunking.py

@@ -0,0 +1,134 @@
+import os
+import hashlib
+import traceback
+import pymupdf as fitz
+import regex as re
+from typing import List, Dict, Tuple, Any
+from langchain_text_splitters import RecursiveCharacterTextSplitter
+
+from configuration import Configuration
+from rag_scripts.interfaces import IDocumentChunker
+
+
+class PyMuPDFChunker(IDocumentChunker):
+    
+    def __init__(self, pdf_path: str, chunk_size: int = Configuration.DEFAULT_CHUNK_SIZE,
+                 chunk_overlap: int = Configuration.DEFAULT_CHUNK_OVERLAP):
+        
+        if not os.path.exists(pdf_path):
+            raise FileNotFoundError(f"PDF file not found at: {pdf_path}")
+        
+        self.pdf_path = pdf_path
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap        
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=self.chunk_size,
+            chunk_overlap=self.chunk_overlap,
+            length_function =len,
+            separators=["\n\n","\n","(?<=\. )\n",
+                        "(?<=[a-z0-9]\.)","(?<=\? )", "(?<=\! )",
+                        " ","" ] )
+        
+        print(f"Initialized PyMuPDFChunker for {os.path.basename(pdf_path)} with chunk_size = {chunk_size} and chunk overlap = {chunk_overlap}")
+
+    def _clean_text(self, text: str) -> str:
+        try:
+            text = text.replace('\u25cf','').replace('\u2022','')
+            text = text.replace('\u201c','').replace('\u201d','')
+            text = text.replace('\u2013','-').replace('\u2014','-').replace('\u2015','-')
+            text = re.sub(r'\n\s*\n',' ',text)
+            text = re.sub(r' {2,}', ' ',text)
+            text = text.replace('\\nb','')
+            text = '\n'.join([line.strip() for line in text.split('\n')])
+            text = re.sub(r'[^\x20-\x7E\t\n\r]', '', text)
+            return text.strip()
+        except Exception as ex:
+            traceback.print_exc()
+            return text.strip()
+
+
+    def hash_document(self):
+        try:
+            hasher = hashlib.sha256()
+            with open(self.pdf_path, 'rb') as fl:
+                while chunk:=fl.read(8192):
+                    hasher.update(chunk)
+            return hasher.hexdigest()
+        except Exception as ex:
+            print(f"Exception hashing PDF {self.pdf_path}: {ex}")
+            traceback.print_exc()
+            raise ValueError(f"Failed to hash PDF: {ex}")
+        
+    def chunk_documents(self) -> Tuple[str, List[Dict[str, Any]]]:
+        try:
+            doc_hash = self.hash_document()
+            doc_chunks =[]
+
+            with fitz.open(self.pdf_path) as document:
+                for idx, page in enumerate(document):
+                    page_text = page.get_text()
+                    if page_text.strip():
+                        section = self._extract_section(page_text)
+                        clause = self._extract_clause(page_text)
+                        page_text = self._clean_text(page_text)
+                        chunked_page = self.text_splitter.split_text(page_text)
+                        for chunk_idx, chunk_content in enumerate(chunked_page):
+                            if not chunk_content.strip():
+                                continue
+                            doc_chunks.append({
+                                "content":chunk_content.strip(),
+                                "metadata":{
+                                    "document_id":doc_hash,
+                                    "source_file":os.path.basename(self.pdf_path),
+                                    "page_number": idx+1,
+                                    "chunk_id":f"{doc_hash} - {idx + 1} -{chunk_idx}",
+                                    "section": section or "Unknown Section",
+                                    "clause": clause or "Unknown Clause",
+                                    "chunk_index_on_page": chunk_idx
+                                }
+                            })
+            if not doc_chunks:
+                print(f"No text or chunks extracted from {self.pdf_path} after cleaning")
+                return doc_hash,[]
+            else:
+                print(f"success Document chunked {self.pdf_path} into {len(doc_hash)} chunks")
+                return doc_hash,doc_chunks
+        
+                        
+
+        except Exception as ex:
+            print(f"Exception in document chunking {ex}")
+            traceback.print_exc()
+            return self.hash_document(), []
+        
+
+    def _extract_section(self, text: str) -> str:
+        match_major = re.search(r'^(?:[IVX]+\.?\s+|[A-Z]\.?\s+|[0-9]+\.?\s+)(.+)', text, re.MULTILINE)
+        if match_major:
+            return match_major.group(0)
+
+        match_firstline = re.search(r'^\s*([A-Za-z0-9][\w\s,&\'-]+?)\s*$', text, re.MULTILINE)
+        if match_firstline:
+            return match_firstline.group(1).strip()
+
+        return None
+
+
+    def _extract_clause(self,text) -> str:
+        match = re.search(r'^(?:(?:•|●|-|\*|\d+\.|\([a-z]\)|\([A-Z]\)|\w\))\s*)(.+?)(?=\n(?:•|●|-|\*|\d+\.|\([a-z]\)|\([A-Z]\)|\w\))|\n\n|\Z)', text, re.MULTILINE | re.DOTALL)
+        if match:
+            clause = match.group(1).strip()
+            if len(clause.split()) < 10:
+                next_match = re.search(
+                    r'^(?:(?:•|●|-|\*|\d+\.|\([a-z]\)|\([A-Z]\)|\w\))\s*)(.+?)(?=\n(?:•|●|-|\*|\d+\.|\([a-z]\)|\([A-Z]\)|\w\))|\n\n|\Z)',
+                    text[match.end():], re.MULTILINE | re.DOTALL)
+                if next_match:
+                    clause += " " + next_match.group(0).strip()
+            return clause
+
+        match_para = re.search(r'^(?!#|\s*$).*?\n\n',text,re.DOTALL)
+        if match_para:
+            return match_para.group(0).strip()
+        return None
+
+

rag_scripts/embedding/embedder.py

@@ -0,0 +1,53 @@
+import os
+import traceback
+from typing import List
+from sentence_transformers import SentenceTransformer
+from traits.trait_types import self
+
+from configuration import Configuration
+from rag_scripts.interfaces import IEmbedder
+
+class SentenceTransformerEmbedder(IEmbedder):
+    def __init__(self, model_name: str = Configuration.DEFAULT_SENTENCE_TRANSFORMER_MODEL):
+        self.model_name = model_name
+        try:
+            self.model = SentenceTransformer(self.model_name)
+            print(f'Sentence Transformer loaded {self.model_name}')
+        except Exception as ex:
+            print(f"Exception in loading Sentence Transformer {self.model_name}")
+            traceback.print_exc()
+            raise
+
+    
+    def embed_texts(self, texts: List[str]) -> List[List[float]]:
+        try:
+            embeddings = self.model.encode(texts).tolist()
+            return embeddings
+        except Exception as ex:
+            print(f"Exception in embedding: {ex}")
+            traceback.print_exc()
+            return [[] for _ in texts]
+        
+    def embed_query(self,query: str) -> List[float]:
+        try:
+            print(f"Embedding query: {query}")
+            embedding = self.model.encode(query).tolist()
+            return embedding
+        except Exception as ex:
+            print(f"Exception in query embedding: {ex}")
+            traceback.print_exc()
+            return []
+
+    def rank(selfself,Query: str, documents:List[str]) -> List[float]:
+        if not documents:
+            return []
+        try:
+            sentence_paris = [[Query, doc] for doc in documents]
+            scores = self.model.predict(sentence_paris)
+            return scores.tolist()
+        except Exception as ex:
+            print(f"Exception in ranking: {ex}")
+            traceback.print_exc()
+            return []
+
+

rag_scripts/embedding/vector_db/chroma_db.py

@@ -0,0 +1,103 @@
+import chromadb
+import traceback
+from typing import List, Dict, Any
+from chromadb.api.types import EmbeddingFunction
+from configuration import Configuration
+from rag_scripts.interfaces import IVector
+from rag_scripts.interfaces import IEmbedder
+from rag_scripts.embedding.embedder import SentenceTransformerEmbedder
+
+class chromaDBEmbeddingFunction(EmbeddingFunction):
+
+    def __init__(self, embedder: IEmbedder):
+        self.embedder = embedder
+    
+    def __call__(self, texts: List[str]) -> List[List[float]]:
+        return self.embedder.embed_texts(texts)
+
+class chromaDBVectorDB(IVector):
+    def __init__(self, embedder:IEmbedder, db_path: str = Configuration.CHROMA_DB_PATH, collection_name: str = Configuration.COLLECTION_NAME):
+        super().__init__(embedder, db_path, collection_name)
+        
+        self.client = chromadb.PersistentClient(path=self.db_path)
+        self.chroma_embed_function = chromaDBEmbeddingFunction(self.embedder)
+        self.collection = self._get_or_create_collection()
+        print(f"Chroma DB intialized path = {self.db_path}, collection = {self.collection_name}")
+
+    def _get_or_create_collection(self):
+        return self.client.get_or_create_collection(
+            name=self.collection_name,
+            embedding_function=self.chroma_embed_function)
+    
+    
+    def add_chunks(self,documents: List[Dict[str,Any]]) -> List[str]:
+        try:
+            if not documents:
+                return []
+            ids = [doc['metadata']['chunk_id'] for doc in documents]
+            contents = [doc['content'] for doc in documents]
+            metadatas = [doc['metadata'] for doc in documents]
+
+            self.collection.add(documents=contents, metadatas=metadatas,ids=ids)
+
+            print(f"Added {len(ids)} chunks to chroma db")
+            return ids
+        except Exception as ex:
+            print(f"Exception in adding chunks to chromaDB: {ex}")
+            traceback.print_exc
+            return[]
+    
+    def get_document_hash_ids(self, document_hash: str) -> List[str]:
+
+        try:
+            result = self.collection.get(where={"document_id":document_hash},limit=1)
+            return result['ids'] if result and result['ids'] else []
+        except Exception as ex:
+            print(f"Exception getting document hash {document_hash} from chromadb {ex}")
+            traceback.print_exc
+            return[]
+
+    def search(self, query: str, k: int=3) -> List[Dict[str, Any]]:
+        try:
+            search_result = self.collection.query(
+                query_texts = [query],
+                n_results=k,
+                include=['documents','metadatas','distances']
+            )
+            retrieved_documents =[]
+            if search_result and search_result['documents'] and search_result['metadatas']:
+                for indx in range(len(search_result['documents'][0])):
+                    document_content = search_result['documents'][0][indx]
+                    document_metadata = search_result['metadatas'][0][indx]
+                    document_distance = search_result['distances'][0][indx]
+
+                    retrieved_documents.append({
+                        "content": document_content,
+                        "metadata": document_metadata,
+                        "distance": document_distance })
+            print(f"Rettrieved {len(retrieved_documents)} documents from chroma DB for query: '{query}'")
+
+            return retrieved_documents
+        except Exception as ex:
+            print(f"Exception in Chroma db search: {ex}")
+            traceback.print_exc()
+            return []  
+        
+    def delete_collection(self, collection_name: str):
+        try:
+            self.client.delete_collection(name = collection_name)
+            print(f"Chroma DB collection {collection_name} deleted")
+            self.collection = self._get_or_create_collection()
+        except Exception as ex:
+            print(f"Exception in deleting the chroma db collection: {collection_name}")
+            traceback.print_exc()
+    
+    def count_documents(self) -> int:
+        try:
+            return self.collection.count()
+        except Exception as ex:
+            print(f"Exception in counting documents in chroma db: {ex}")
+            traceback.print_exc()
+            return 0
+
+        

rag_scripts/embedding/vector_db/faiss_db.py

@@ -0,0 +1,145 @@
+import os
+import faiss
+import pickle
+import traceback
+import numpy as np
+from typing import List, Dict, Any
+
+from configuration import Configuration
+from rag_scripts.interfaces import IVector, IEmbedder
+from rag_scripts.embedding.embedder import SentenceTransformerEmbedder
+
+class FAISSVectorDB(IVector):
+
+    def __init__(self, embedder: IEmbedder, db_path: str= Configuration.FAISS_DB_PATH,
+                 collection_name: str = Configuration.COLLECTION_NAME):
+        super().__init__(embedder,db_path,collection_name)
+        self.index = None
+        self.doc_store: List[Dict[str,Any]] =[]
+
+        self.collection_file = os.path.join(self.db_path,f"{self.collection_name}.faiss")
+        self.doc_store_file = os.path.join(self.db_path, f"{self.collection_name}_docs.pkl")
+
+        os.makedirs(self.db_path, exist_ok=True)
+
+        self._load_index()
+
+        print(f"FAISSDB Initalized: path ='{self.db_path}, collection ='{self.collection_name}")
+
+    
+    def _load_index(self):
+        try:
+            if os.path.exists(self.collection_file) and os.path.exists(self.doc_store_file):
+                self.index = faiss.read_index(self.collection_file)
+                with open(self.doc_store_file,'rb') as f:
+                    self.doc_store = pickle.load(f)
+                print(f"Loaded Existing FAISS index and doc store from {self.collection_file}")
+            else:
+                print(f"No Existing FAISS index found, need new collection")
+        except Exception as ex:
+           print(f"Exception in loading FAISS index: {ex}") 
+           traceback.print_exc()
+           self.index = None
+           self.doc_store = []
+    
+    def _save_index(self):
+        try:
+            if self.index is not None:
+                faiss.write_index(self.index, self.collection_file)
+                with open(self.doc_store_file, "wb") as f:
+                    pickle.dump(self.doc_store,f)
+                print(f"FAISS index and doc store saved to {self.collection_file}")
+
+        except Exception as ex:
+            print(f"Exception in saving FAISS index: {ex}")
+            traceback.print_exc()
+
+    def add_chunks(self, documents: List[Dict[str, Any]]) -> List[str]:
+        try:
+            if not documents:
+                return[]
+            
+            contents = [doc['content'] for doc in documents]
+            embeddings = np.array(self.embedder.embed_texts(contents),dtype='float32')
+
+            if self.index is None:
+                dimension = embeddings.shape[1]
+                self.index = faiss.IndexFlatL2(dimension)
+                print(f"Created New FAISS index with dimension {dimension}")
+
+            self.index.add(embeddings)
+
+            for doc in documents:
+                self.doc_store.append(doc)
+            
+            self._save_index()
+            chunk_ids = [doc['metadata']['chunk_id'] for doc in documents]
+            print(f"Added {len(chunk_ids)} chunks to FAISS index")
+            return chunk_ids
+        except Exception as ex:
+            print(f"Exception in adding chunks to FAISS: {ex}")
+            traceback.print_exc()
+            return []
+
+    def get_document_hash_ids(self, document_hash: str) ->List[str]:
+        try:
+            found_ids =[]
+            for doc in self.doc_store:
+                if doc['metadata'].get('document_id') == document_hash:
+                    found_ids.append(doc['metadata']['chunk_id'])
+        
+            return found_ids
+        except Exception as ex:
+            print(f"Exception in get document hash {ex}")
+
+    def search(self, query: str, k:int=3) -> List[Dict[str,Any]]:
+        try:
+            if self.index is None or self.index.ntotal ==0:
+                print(f"FAISS index is empty or not intialized")
+                return []
+            query_embedding = np.array(self.embedder.embed_query(query),dtype='float32').reshape(1,-1)
+            distances, indices = self.index.search(query_embedding,k)
+            
+            retrieved_documents = []
+            for dist, idx in zip(distances[0], indices[0]):
+                if 0 <= idx < len(self.doc_store):
+                    doc = self.doc_store[idx]
+                    retrieved_documents.append({
+                        "content":doc['content'],
+                        "metadata":doc['metadata'],
+                        "distance":float(dist) })
+            print(f"Retrieved {len(retrieved_documents)} documents from FAISS for query: {query}")
+
+            return retrieved_documents
+
+
+        except Exception as ex:
+            print(f"Exception in FAISS db search {ex}")
+            traceback.print_exc()
+            return []
+
+    def delete_collection(self,collection_name: str):
+        try:
+            if os.path.exists(self.collection_file):
+                os.remove(self.collection_file)
+            if os.path.exists(self.doc_store_file):
+                os.remove(self.doc_store_file)
+            self.index = None
+            self.doc_store = []
+            print(f"FAISS collection files for {collection_name} deleted")
+
+        except Exception as ex:
+            print(f"Error deleting FAISS collection files for {collection_name}: ex")
+            traceback.print_exc
+
+    def count_documents(self) -> int:
+        try:
+            return self.index.ntotal if self.index is not None else 0
+        except Exception as ex:
+            print(f"Exception in counting document in FAISS: {ex}")
+            traceback.print_exc()
+            return 0
+        
+         
+
+

rag_scripts/embedding/vector_db/pinecone_db.py

@@ -0,0 +1,202 @@
+import time
+import traceback
+from typing import List, Dict,Any
+import re
+from pinecone import Pinecone, ServerlessSpec
+from pinecone.exceptions import PineconeException
+
+from configuration import Configuration
+from rag_scripts.interfaces import IVector, IEmbedder
+
+class PineconeVectorDB(IVector):
+    def __init__(self, embedder:IEmbedder, db_path: str, collection_name: str="flykite"):
+        super().__init__(embedder,db_path,collection_name)
+
+        self.api_key = Configuration.PINECONE_API_KEY
+        self.cloud = Configuration.PINECONE_CLOUD
+        self.region = Configuration.PINECONE_REGION
+
+        if not self.api_key:
+            raise ValueError("Pinecone API KEY not provided in configuration")
+        
+        self.pc = Pinecone(api_key=self.api_key)
+        collection_name = collection_name.replace('_','-')
+        self.index_name=collection_name
+
+        print(f"Collection name: {collection_name}")
+
+        self.dim=len(self.embedder.embed_texts(["test"])[0])
+        self._create_index()
+        self.index=self.pc.Index(self.index_name)
+
+        print(f"Pinecone DB Initialized index= {self.index_name}, cloud= {self.cloud}, region = {self.region}")
+
+    def _create_index(self):
+        try:
+            existing_indexes = self.pc.list_indexes().names()
+            if self.index_name in existing_indexes:
+                index = self.pc.Index((self.index_name))
+                index_info = index.describe_index_stats()
+                if index_info.dimension!=self.dim:
+                    print(f"Pinecone DB Index already exists {self.index_name} "
+                          f"with dimension {index_info.dimension}"
+                          f"but the expected dimension is {self.dim}"
+                          " so deleting it and recreating again"
+                          )
+                    self.pc.delete_index(self.index_name)
+
+                    # while self.index_name in self.pc.list_indexes().names():
+                    #     time.sleep(2)
+                    for _ in range(30):
+                        if self.index_name not in self.pc.list_indexes().names():
+                            break
+                        else:
+                            time.sleep(2)
+
+
+                elif index_info.metric!='cosine':
+                    self.pc.delete_index(self.index_name)
+
+                    for _ in range(30):
+                        if self.index_name in self.pc.list_indexes().names():
+                            time.sleep(2)
+                        else:
+                            break
+                else:
+                    print(f"Pinecone index already exists {self.index_name} ")
+                    return
+
+            if self.index_name not in self.pc.list_indexes().names():
+                self.pc.create_index(name=self.index_name,
+                    dimension=self.dim,
+                    metric='cosine',
+                    spec=ServerlessSpec(cloud=self.cloud,region=self.region)
+                )
+                max_attempts = 90
+                for attempt in range(max_attempts):
+                    try:
+                        index = self.pc.Index(self.index_name)
+                        index_info = index.describe_index_stats()
+                        if index_info.dimension == self.dim:
+                            print(f"Pinecone index {self.index_name} already exists ")
+                            return
+                    except PineconeException:
+                        pass
+                    time.sleep(2)
+                raise TimeoutError(f"Pinecone index {self.index_name} does not exist")
+
+    
+        except PineconeException as ex:
+
+            print(f"Exception creating Pinecone index: {ex}")
+            traceback.print_exc()
+            raise
+        except Exception as ex:
+            print(f"Exception creating Pinecone index: {ex}")
+            traceback.print_exc()
+            raise
+
+
+
+    def add_chunks(self, documents: List[Dict[str,Any]]) -> List[str]:
+        try:
+            if not documents:
+                return[]
+            vectors_info = []
+            for doc in documents:
+                content = doc['content']
+                embedding = self.embedder.embed_texts([content])[0]
+                chunk_id = doc['metadata']['chunk_id']
+                metadata= doc['metadata'].copy()
+                metadata['content']=content
+
+                vectors_info.append({
+                    "id":chunk_id, "values":embedding, "metadata":metadata
+                })
+
+            print(f"Upserting {len(vectors_info)} to pincone index")
+            self.index.upsert(vectors=vectors_info)
+            chunk_ids = [vec['id'] for vec in vectors_info]
+            print(f"Added {len(chunk_ids)} chunks to pinecone index {self.index_name}")
+            print(f"Added chunk ids {chunk_ids[:5]}")
+            return chunk_ids
+
+        except PineconeException as ex:
+            print(f"Exception in adding chunks to pinecone db {ex}")
+            traceback.print_exc()
+            return []
+
+    def get_document_hash_ids(self, document_hash: str) ->List[str]:
+        try:
+            vector_sample = [0.0]*self.dim
+            result = self.index.query(
+                vector = vector_sample,
+                filter = {"document_id": {"$eq": document_hash}},
+                top_k = 10000,
+                include_metadata=False )
+            
+            return [match['id'] for match in result['matches']]
+        
+        except PineconeException as ex:
+            print(f"Exception getting document hash IDs from pinecone: {ex}")
+            traceback.print_exc()
+            return []
+    
+    def search(self,query: str, k:int=3) -> List[Dict[str,Any]]:
+        try:
+            print(f"searching pinecode index {self.index_name} for query {query}")
+            query_embedding = self.embedder.embed_query(query)
+            result = self.index.query(
+                vector=query_embedding,
+                top_k=k,
+                include_metadata=True
+            )
+
+            retrieved_documents = []
+            for match in result['matches']:
+                md= match['metadata']
+                content = md.pop('content','')
+
+                retrieved_documents.append({
+                    "content":content,
+                    "metadata":md,
+                    "distance":match['score']
+                })
+            
+            print(f"Retrieved {len(retrieved_documents)} documents from pinecone for query: {query}")
+
+            return retrieved_documents
+
+
+        except PineconeException as ex:
+            print(f"Exception in Pinecone search: {ex}")
+            traceback.print_exc()
+            return[]
+
+    def delete_collection(self, collection_name: str):
+        try:
+            
+            self.pc.delete_index(collection_name)
+            print(f"Pinecone index {collection_name} deleted")
+        except PineconeException as ex:
+            print(f"Exception in deleting pinecone index: {collection_name} {ex}")
+            traceback.print_exc()
+    
+    def count_documents(self) -> int:
+        try:
+            for attempt in range(19):
+                status = self.index.describe_index_stats()
+                count = status.get('total_vector_count',0)
+                if count > 0:
+                    return count
+                time.sleep(7)
+            print(f"No vectors found in index: {self.index_name}")
+            return 0
+            
+        except PineconeException as ex:
+            print(f"EXception in getting document counts: {ex}")
+            traceback.print_exc()
+            return 0
+        
+        
+        

rag_scripts/evaluation/evaluator.py

@@ -0,0 +1,525 @@
+import math
+import os
+import re
+import time
+import json
+import random
+import joblib
+import traceback
+import logging
+
+import torch
+import transformers
+from bert_score import score
+from itertools import product
+from datetime import datetime
+from typing import List, Dict, Any
+from configuration import Configuration
+from rag_scripts.llm.llmResponse import GROQLLM
+from pinecone import Pinecone, PineconeException
+from rag_scripts.rag_pipeline import RAGPipeline
+from sentence_transformers import SentenceTransformer, util
+from rag_scripts.embedding.vector_db.faiss_db import FAISSVectorDB
+from rag_scripts.documents_processing.chunking import PyMuPDFChunker
+from rag_scripts.embedding.embedder import SentenceTransformerEmbedder
+from rag_scripts.embedding.vector_db.chroma_db import chromaDBVectorDB
+from rag_scripts.embedding.vector_db.pinecone_db import PineconeVectorDB
+from rag_scripts.interfaces import IDocumentChunker, ILLM, IEmbedder, IRAGPipeline, IVector
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger(__name__)
+
+VECTOR_DB_CONSTRUCTORS = {
+    "chroma": chromaDBVectorDB,
+    "faiss": FAISSVectorDB,
+    "pinecone": PineconeVectorDB
+}
+
+class RAGEvaluator:
+    def __init__(self, eval_data_path: str = Configuration.EVAL_DATA_PATH,
+                 pdf_path: str = Configuration.FULL_PDF_PATH):
+        if not os.path.exists(eval_data_path):
+            raise FileNotFoundError(f"Evaluation data not found at: {eval_data_path}")
+        
+        if not os.path.exists(pdf_path):
+            raise FileNotFoundError(f"PDF document not found at: {pdf_path}")
+        
+        with open(eval_data_path, 'r') as f:
+            self.eval_queries = json.load(f)
+
+        self.pdf_path = pdf_path
+        self.embedder = SentenceTransformer(Configuration.DEFAULT_SENTENCE_TRANSFORMER_MODEL)
+        logger.info(f"RAG Evaluator initialized with {len(self.eval_queries)} evaluation queries")
+
+    def _sanitize_collection_name(self, name: str) -> str:
+        sanitized = re.sub(r'[^a-z0-9]', '-', name.lower())
+        sanitized = re.sub(r'-+', '-', sanitized).strip('-')
+        return sanitized[:45].rstrip('-') if len(sanitized) > 45 else sanitized
+    
+    def _calculate_retrieval_relevance(self, query: str, retrieved_docs: List[Dict[str, Any]],
+                                       expected_answer: str =None) -> float:
+        if not retrieved_docs:
+            logger.warning(f"No retrieved documents for query: {query}")
+            return 0.0
+        
+        query_embedding = self.embedder.encode(query, convert_to_tensor=True, normalize_embeddings=True)
+        reference_embedding = [query_embedding]
+        # if expected_answer.strip():
+        #     expected_answer_embedding = self.embedder.encode(expected_answer,convert_to_tensor=True,
+        #                                         normalize_embeddings=True,
+        #                                         normalize_retrieved=True)
+
+
+        score_cosine = []
+        for doc in retrieved_docs:
+            doc_content = doc.get('content','')
+            if not doc_content.strip():
+                continue
+            doc_embedding = self.embedder.encode(doc_content, convert_to_tensor=True, normalize_embeddings=True)
+            max_sim_to_reference = 0.0
+            for ref_embedding in reference_embedding:
+                sim = util.cos_sim(ref_embedding, doc_embedding)
+                if  isinstance(sim,torch.Tensor):
+                    sim_value = sim.item()
+                else:
+                    sim_value = sim
+                if sim_value > max_sim_to_reference:
+                    max_sim_to_reference = sim_value
+            score_cosine.append(max_sim_to_reference)
+        cosine_score = sum(score_cosine) / len(score_cosine) if score_cosine else 0.0
+        #cosine_score = max(score_cosine) if score_cosine else 0.0
+        logger.info(f"Max cosine score for query '{query}': {cosine_score}")
+        return cosine_score
+    
+    def _calculate_response_groundedness(self, response: Dict[str,Any], retrieved_docs: List[Dict[str, Any]]) -> float:
+        try:
+            if not retrieved_docs:
+                logger.warning(f"No retrieved documents for groundedness evaluation")
+                return 0.0
+            response_text = response.get("summary","")
+            if not isinstance(response_text,str):
+                response_text = str(response_text)
+
+            response_segments = re.split(r'[\u25cf\u25cb]\s*|\n\s*\n', response_text.strip())
+            response_segments = [seg.strip() for seg in response_segments if seg.strip()]
+
+            groundedness_scores = []
+            for doc in retrieved_docs:
+                doc_content = doc.get('content','')
+                if not doc_content.strip():
+                    continue
+                doc_embedding = self.embedder.encode(doc_content,
+                                            convert_to_tensor=True,
+                                            normalize_embeddings=True)
+                for segment in response_segments:
+                    if segment:
+                        segment_embedding = self.embedder.encode(segment,
+                                                    convert_to_tensor=True,
+                                                    normalize_embeddings=True)
+
+                        segment_similarity = util.cos_sim(segment_embedding,
+                                                          doc_embedding).item()
+                        groundedness_scores.append(segment_similarity)
+
+                full_response_embedding = self.embedder.encode(response_text,
+                                                        convert_to_tensor=True,
+                                                        normalize_embeddings=True)
+                full_response_similarity = util.cos_sim(full_response_embedding,doc_embedding).item()
+                groundedness_scores.append(full_response_similarity)
+
+            groundedness = max(
+                groundedness_scores) if groundedness_scores else 0.0  # Use max to handle structured responses
+            logger.info(f"Max groundedness score for response: {groundedness:.2f}")
+            return groundedness
+        except Exception as ex:
+            logger.error(f"Exception in calculating groundedness: {ex}")
+            traceback.print_exc()
+            return 0.0
+    
+    def _calculate_response_relevance(self, query: str, response: Dict[str,Any],expected_answer: str=None) -> float:
+        try:
+            response_text = response.get("summary","")
+            response_text = response_text.replace('\n','')
+            if not isinstance(response_text,str) or not response_text.strip():
+                return 0.0
+
+            reference_text = expected_answer if expected_answer is not None and  expected_answer.strip() else query
+
+            refernce_embedding = self.embedder.encode(reference_text,convert_to_tensor=True,normalize_embeddings=True)
+            #query_embedding = self.embedder.encode(query, convert_to_tensor=True, normalize_embeddings=True)
+            response_embedding = self.embedder.encode(response_text, convert_to_tensor=True, normalize_embeddings=True)
+            similarity = util.cos_sim(refernce_embedding, response_embedding).item()
+
+            response_length = len(response_text.split())
+            #query_length = len(query.split())
+            reference_length = len(reference_text.split())
+            length_penalty_factor = 1.0
+            if response_length > (reference_length*1.5) and response_length >50:
+                length_penalty_factor = max(0.8,1.0-(response_length-(reference_length*2))/100.0)
+            elif response_length < (reference_length*0.5) and response_length >20:
+                length_penalty_factor = max(0.5,response_length/(reference_length*0.5))
+
+            adjusted_relevance = similarity * length_penalty_factor
+            logger.info(
+                f"Relevance score for query '{query}': {adjusted_relevance:.2f} (similarity: {similarity:.2f}, penalty: {length_penalty_factor:.2f})")
+            return adjusted_relevance
+        except Exception as ex:
+            logger.error(f"Exception in relevance score calculation: {ex}")
+            traceback.print_exc()
+            return 0.0
+    
+    def _calculate_bert_score(self, response: Dict[str,Any], reference: str) -> float:
+        try:
+            response_text = response.get("summary", "")
+            if not isinstance(response_text, str):
+                logger.warning("Response text is not a string")
+                return 0.0
+            if not response_text:
+                logger.error(f"No retrieved documents for BERT score evaluation")
+                return 0.0
+            if not reference.strip():
+                logger.warning(f"No reference answer provided for BERTScore calculation")
+                return 0.0
+            transformers.utils.logging.set_verbosity_error()
+            _, _, f1 = score([response_text], [reference], lang="en", model_type="roberta-large")
+            transformers.utils.logging.set_verbosity_warning()
+            bert_score_value = f1.item()
+            logger.info(f"BERT score: {bert_score_value:.2f}")
+            return bert_score_value
+        except Exception as ex:
+            logger.error(f"Exception in BERT score calculation: {ex}")
+            traceback.print_exc()
+            return 0.0
+
+    def evaluate_response(self, query: str, response: Dict[str,Any], retrieved_docs: List[Dict[str, Any]],
+                         expected_keywords: List[str] = None, expected_answer: str = None) -> Dict[str, Any]:
+        try:
+            cosine_score = self._calculate_retrieval_relevance(query, retrieved_docs,expected_answer=expected_answer)
+            groundedness = self._calculate_response_groundedness(response, retrieved_docs)
+            relevance = self._calculate_response_relevance(query, response,expected_answer=expected_answer)
+            bert_score = self._calculate_bert_score(response, expected_answer) if expected_answer else 0.0
+
+            observations = []
+            if groundedness < 0.6:
+                observations.append(f"Response may contain ungrounded information (groundedness score: {groundedness:.2f})")
+            else:
+                observations.append(f"Response is well grounded (score: {groundedness:.2f})")
+            
+            if relevance < 0.7:
+                observations.append(f"Response may not fully address the question (relevance score: {relevance:.2f})")
+            else:
+                observations.append(f"Response is highly relevant (score: {relevance:.2f})")
+            
+            if cosine_score < 0.5:
+                observations.append(f"Low similarity between query and retrieved documents")
+            
+            if bert_score < 0.7 and expected_answer:
+                observations.append(f"Low BERT score, semantic mismatch with reference answer")
+            
+            return {
+                "cosine_score": round(cosine_score, 2),
+                "groundedness": round(groundedness, 2),
+                "relevance": round(relevance, 2),
+                "bert_score": round(bert_score, 2),
+                "observations": "; ".join(observations)
+            }
+        
+        except Exception as ex:
+            logger.error(f"Exception in calculating response scores: {ex}")
+            traceback.print_exc()
+            return {
+                "cosine_score": 0.0,
+                "groundedness": 0.0,
+                "relevance": 0.0,
+                "bert_score": 0.0,
+                "observations": f"Error in evaluation: {ex}"
+            }
+
+    def evaluate_combined_params_grid(self,
+                                     chunk_size_to_test: List[int],
+                                     chunk_overlap_to_test: List[int],
+                                     embedding_models_to_test: List[str],
+                                     vector_db_types_to_test: List[str],
+                                      re_ranker_model: List[str],
+                                     llm_model_name: str = Configuration.DEFAULT_GROQ_LLM_MODEL,
+                                     search_type: str = "grid",
+                                     n_iter: int = 50) -> Dict[str, Any]:
+        logger.info("\n--- Starting the evaluation of best parameters ---")
+        best_score = -1.0
+        best_params = {}
+        best_metrics = {}
+        results = []
+        
+        param_combination = [(c_size, c_overlap, embed_model, db_type,re_ranker)
+                             for c_size, c_overlap, embed_model, db_type,re_ranker in product(
+                                 chunk_size_to_test, chunk_overlap_to_test,
+                                 embedding_models_to_test, vector_db_types_to_test,re_ranker_model)
+                             if c_overlap < c_size]
+        
+        param_to_test = (random.sample(param_combination, min(n_iter, len(param_combination)))
+                         if search_type.lower() == 'random' else param_combination)
+        logger.info(f"Testing {len(param_to_test)} {'random' if search_type.lower() == 'random' else 'all'} "
+                    f"combinations out of {len(param_combination)}")
+
+        for idx, (c_size, c_overlap, embed_model, db_type,re_ranker) in enumerate(param_to_test, 1):
+            logger.info(f"\nIteration {idx}/{len(param_to_test)} \nchunk_size: {c_size} \nchunk_overlap: {c_overlap} "
+                        f"\nembed_model: {embed_model} \nvector_db: {db_type}")
+
+            current_params_str = f"Chunk: {c_size}-{c_overlap}- Embed- {embed_model}- DB-{db_type}-{re_ranker}"
+            embed_model = embed_model.replace('_', '-')
+            temp_collection_name = self._sanitize_collection_name(
+                f"{Configuration.COLLECTION_NAME}-{search_type}-{c_size}-{c_overlap}-{embed_model}-{db_type}")
+            temp_db_path = os.path.join(Configuration.DATA_DIR, f"{db_type}_temp_{search_type}_{c_size}_{c_overlap}_{embed_model}_{embed_model}")
+            os.makedirs(temp_db_path, exist_ok=True)
+
+            vector_db_instance = None
+            try:
+                embedder = SentenceTransformerEmbedder(model_name=embed_model)
+                db_constructor = VECTOR_DB_CONSTRUCTORS.get(db_type.lower())
+                if not db_constructor:
+                    logger.error(f"Unsupported vector DB type: {db_type}")
+                    continue
+                vector_db_instance = db_constructor(embedder=embedder,
+                                                    db_path=temp_db_path,
+                                                    collection_name=temp_collection_name)
+                                                    
+                
+                chunker_instance = PyMuPDFChunker(pdf_path=self.pdf_path,
+                                                 chunk_size=c_size,
+                                                 chunk_overlap=c_overlap)
+                
+                llm_instance = GROQLLM(model_name=llm_model_name)
+
+                pipeline = RAGPipeline(
+                    document_path=self.pdf_path,
+                    chunker=chunker_instance,
+                    embedder=embedder,
+                    vector_db=vector_db_instance,
+                    llm=llm_instance,
+                    chunk_size=c_size,
+                    chunk_overlap=c_overlap,
+                    embedding_model_name=embed_model,
+                    llm_model_name=llm_model_name,
+                    db_path=temp_db_path,
+                    collection_name=temp_collection_name,
+                    re_ranker_model_name=re_ranker
+                )
+                
+                pipeline.build_index()
+                if vector_db_instance.count_documents() == 0:
+                    logger.warning(f"No documents foundin vector DB after build for "
+                                   f"{current_params_str}. skipping evaluation for this combination.")
+                    continue
+
+                cosine_scores = []
+                groundedness_scores = []
+                relevance_scores = []
+                bert_scores = []
+
+                for eval_item in self.eval_queries:
+                    query = eval_item['query']
+                    expected_answer = eval_item.get('expected_answer_snippet', '')
+
+                    expected_keywords = eval_item.get('expected_keywords', [])
+                    retrieved_docs = pipeline.retrieve_raw_documents(query, k=3)
+                    response = pipeline.query(query, k=3)
+                    logger.debug(f"Query: {query} \n Response: {json.dumps(response,indent=2, ensure_ascii=False)}")
+                    if not expected_answer.strip():
+                        expected_answer = self._syntesize_raw_reference(retrieved_docs)
+
+
+                    eval_result = self.evaluate_response(query, response,
+                                            retrieved_docs,
+                                            expected_answer=expected_answer,
+                                            expected_keywords=expected_keywords)
+                    
+                    cosine_scores.append(eval_result['cosine_score'])
+                    groundedness_scores.append(eval_result['groundedness'])
+                    relevance_scores.append(eval_result['relevance'])
+                    bert_scores.append(eval_result['bert_score'])
+
+                average_cosine_score = round(sum(cosine_scores) / len(cosine_scores) if cosine_scores else 0.0, 2)
+                average_groundedness = round(sum(groundedness_scores) / len(groundedness_scores) if groundedness_scores else 0.0, 2)
+                average_relevance = round(sum(relevance_scores) / len(relevance_scores) if relevance_scores else 0.0, 2)
+                average_bert_score = round(sum(bert_scores) / len(bert_scores) if bert_scores and any(bert_scores) else 0.0, 2)
+
+                average_score = round((0.2* average_cosine_score +
+                                       0.35* average_groundedness +
+                                      0.35* average_relevance +
+                                       0.15*average_bert_score), 2)
+                
+                results.append({
+                    "iteration": idx,
+                    "chunk_size": c_size,
+                    "chunk_overlap": c_overlap,
+                    "embedding_model": embed_model,
+                    "vector_db_type": db_type,
+                    "average_retrieval_relevance_score": average_score,
+                    "average_cosine_score": average_cosine_score,
+                    "average_groundedness": average_groundedness,
+                    "average_relevance": average_relevance,
+                    "average_bert_score": average_bert_score
+                })
+                
+                logger.info(f"Average retrieval relevance score: {average_score}")
+                logger.info(f"Average cosine score: {average_cosine_score}")
+                logger.info(f"Average groundedness: {average_groundedness}")
+                logger.info(f"Average relevance: {average_relevance}")
+                logger.info(f"Average BERT score: {average_bert_score}")
+
+                logger.info(f"Comparing the average score {average_score} with best score {best_score}")
+                if average_score > best_score:
+                    best_score = average_score
+                    best_params = {
+                        "iteration":idx,
+                        "chunk_size": c_size,
+                        "chunk_overlap": c_overlap,
+                        "embedding_model": embed_model,
+                        "vector_db_type": db_type,
+                        "re_ranker_model":re_ranker
+                    }
+                    best_metrics = {
+                        "average_retrieval_relevance_score": average_score,
+                        "average_cosine_score": average_cosine_score,
+                        "average_groundedness": average_groundedness,
+                        "average_relevance": average_relevance,
+                        "average_bert_score": average_bert_score,
+                        "re_ranker_model":re_ranker
+                    }
+
+                    logger.info(f"Best score: {best_score}")
+                    logger.info(f"Best params: {best_params}")
+            
+            except (PineconeException, ValueError) as ex:
+                logger.error(f"Exception in grid search for chunk_size={c_size}, "
+                            f"chunk_overlap={c_overlap}, embed_model={embed_model}, vector_db={db_type}: {ex}")
+                traceback.print_exc()
+            finally:
+                if 'vector_db_instance' in locals() and vector_db_instance is not None:
+                    try:
+                        vector_db_instance.delete_collection(temp_collection_name)
+                        if isinstance(vector_db_instance, chromaDBVectorDB):
+                            del vector_db_instance.client
+                            time.sleep(5)
+                        if isinstance(vector_db_instance, PineconeVectorDB):
+                            pc = Pinecone(api_key=Configuration.PINECONE_API_KEY)
+                            if temp_collection_name in pc.list_indexes().names():
+                                pc.delete_index(temp_collection_name)
+                                logger.info(f"Pinecone index: {temp_collection_name} deleted")
+                    except Exception as cleanup_ex:
+                        logger.error(f"Error during cleanup: {cleanup_ex}")
+                        traceback.print_exc()
+
+        logger.info("\n---- Evaluation completed ----")
+        logger.info(f"Best parameters: {best_params}")
+        logger.info(f"Best score: {best_score:.2f}")
+        logger.info(f"Total iterations evaluated: {len(results)}")
+
+        pkl_directory = os.path.join(Configuration.DATA_DIR, "eval_results")
+        os.makedirs(pkl_directory, exist_ok=True)
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M")
+        pkl_file = os.path.join(pkl_directory, f"eval_results_{search_type}_{timestamp}.pkl")
+
+        try:
+            joblib.dump({
+                "best_params": best_params,
+                "best_score": best_score,
+                "results": results,
+                "best_metrics": best_metrics
+            }, pkl_file)
+            logger.info(f"Results saved to {pkl_file}")
+        except Exception as ex:
+            logger.error(f"Exception in saving results to {pkl_file}: {ex}")
+
+        return {
+            "best_params": best_params,
+            "best_score": best_score,
+            "results": results,
+            "best_metrics": best_metrics,
+            "pkl_file": pkl_file
+        }
+
+
+    def _evaluate_with_llm(self, query:str, response_summary: str,retrieved_docs_contents: List[str]) -> Dict[str,Any]:
+        try:
+            context = "\n".join(retrieved_docs_contents)
+            system_message = f"""
+                You are an expert, Impartial judge for evaluating Retrieval Augmented Generation (RAG) system message.
+                Your ONLY task is to output a JSON object containing the evaluation score and reasoning.
+                DO NOT include any other text, explanations, conversational remarks or markdown code blocks(```json).
+                Strictly adhere to the requested JSON format.
+                """
+            prompt = f"""
+                You are evaluating a RAG system's response.
+                
+                Query: "{query}"
+                Retrieved context: --- {context} ---
+                RAG Systems Response: "{response_summary}"
+                      
+                please provide a score for Groundedness and Relevance on a scale of 1 to 5,
+                where 5 is excellent and 1 is very poor.
+                
+                Groundedness: is how the response is supported *only* to the Retrieved context with  no hallucinations ?
+                1. Contains significant information not supported by the context or contradicts it.
+                2. Contains some unsupported information.
+                3. Mostly grounded, but might have minor deviations or additions.
+                4. Almost entirely grounded in the context.
+                5. Fully and accurately, using only information from the context.
+                
+                Relevance: how well the response is directly and comprehensively answers the query based on the context ?
+                1. Does not answer the query at all, or answers a different question.
+                2. Addresses the query partially but misses significant part or is off-topic.
+                3. Answer the query reasonably well, but could be more complete or focused.
+                4. Answer the query well, covering most relevant aspects.
+                5. Answer the query completely, accurately and concisely, directly addressing all aspects.
+                
+                output your assessment ONLY in the following JSON format. no other text.
+                {{
+                    "groundedness_score": <int> out of 5,
+                    "relevance_score": <int> out of 5,
+                    "reasoning": "Brief explanation for the scores."
+                }}                                              
+            """
+
+            #llm_response =  GROQLLM.generate_response(prompt=prompt,temperature=0.1,top_p=0.95,max_tokens=1500)
+            eval_with_llm = GROQLLM(model_name="llama-3.3-70b-versatile")
+            llm_response =  eval_with_llm.generate_response(
+                prompt=prompt,
+                system_message=system_message,
+                temperature=0.1,
+                top_p=0.95,
+                max_tokens=1500)
+
+            print("\n -- LLM Judge Raw Response --")
+            print(llm_response)
+            print('-'*50)
+
+            eval_scores = json.loads(llm_response)
+            return {
+                "Groundedness score": eval_scores.get("groundedness_score",0),
+                "Relevance score": eval_scores.get("relevance_score",0),
+                "Reasoning": eval_scores.get("reasoning","")
+            }
+        except Exception as ex:
+            traceback.print_exc()
+            #return ""
+            return {
+                "Groundedness score": 0,
+                "Relevance score": 0,
+                "Reasoning": f"Exception in LLM evaluation: {ex}"
+            }
+
+
+    def _syntesize_raw_reference(self, retrieved_docs: List[Dict[str, Any]]) -> str:
+        try:
+            if not retrieved_docs:
+                return  ""
+            raw_content_snippets = [doc.get('content','') for doc in retrieved_docs if doc.get('content')]
+            raw_answer = " ".join(raw_content_snippets)
+
+            raw_answer = " ".join(raw_answer.split()).join(sorted(list(set(raw_answer.split()))))
+
+            return raw_answer
+        except Exception as ex:
+            traceback.print_exc()
+            return ""

rag_scripts/interfaces.py

@@ -0,0 +1,129 @@
+from abc import ABC, abstractmethod
+from typing import List, Dict, Tuple, Any
+
+class IDocumentChunker(ABC):
+    @abstractmethod
+    def hash_document(self) -> str:
+        """This function will generate a unique hash for the document"""
+        pass
+    
+    @abstractmethod
+    def chunk_documents(self) -> Tuple[str,List[Dict[str,Any]]]:
+        '''
+        This function will be chunking the document
+        return: Tuple[str, List[Dict[str,Any]]]
+        this will return the hashed Document ID
+        and the dict which contains the content and metadata
+        '''
+        pass
+
+    
+
+class IEmbedder(ABC):
+    @abstractmethod
+    def embed_texts(self,texts: List[str]) -> List[List[float]]:
+        '''
+        This fucntion generates the embeddings for a list of text strings
+        Return: List[List[float]]: A list of embedding vectors
+        '''
+        pass
+
+    @abstractmethod
+    def embed_query(self, query: str) -> List[float]:
+        '''
+        This function generates a embedding for a single query string
+        returns: List[float]: A single embedding vector
+        '''
+        pass
+
+class IVector(ABC):
+    def __init__(self, embedder: IEmbedder, db_path: str, collection_name: str):
+        self.embedder = embedder
+        self.db_path = db_path
+        self.collection_name = collection_name
+    
+    @abstractmethod
+    def add_chunks(self, documents: List[Dict[str,Any]]) -> List[str]:
+        '''
+        This function add the list of documents chunk to the vector database
+        Argument:
+         documents: List of chunk dictionaries which contains content and metadata
+        Returns:
+         List[str]: List of Chunk id 
+        '''
+        pass
+
+    @abstractmethod
+    def get_document_hash_ids(self, document_hash: str) -> List[str]:
+
+        '''
+        This function check the document is already exist based ont he hash id
+        retunr: List[str]: List of chunk IDs associated with the document hash or empty list if not found
+        '''
+        pass
+
+    @abstractmethod
+    def search(self, query: str, k: int=3) -> List[Dict[str,Any]]:
+        '''
+        This function searches the vector database for relevant documents based on a query.
+        Argument:
+            query: user query string
+            k: The number of top-k results to retrieve
+        returns:
+            List[Dict]: A List of retrieved document chunk, including content, metadata and distance
+
+        '''
+        pass
+
+    @abstractmethod
+    def delete_collection(self, collection_name: str):
+        '''
+        Deletes a sepcified colelction or index from the vector database.
+        useful for clean up during evaluations
+        '''
+        pass
+
+    @abstractmethod
+    def count_documents(self) -> int:
+        '''
+        Returns the number of documents/chunks in the collection
+        '''
+        pass
+
+
+class ILLM(ABC):
+    @abstractmethod
+    def generate_response(self, prompt: str, system_message: str = None) -> str:
+        '''
+        Generates a response from the LLM based on a prompt
+
+        '''
+        pass
+
+class IRAGPipeline(ABC):
+    @abstractmethod
+    def build_index(self):
+        '''
+        Builds the Rag Pipeline for documents index
+        '''
+        pass
+
+    @abstractmethod
+    def query(self, user_query: str) -> str:
+        '''
+        Quering the RAG pipeline
+
+        '''
+        pass
+
+    @abstractmethod
+    def retrieve_raw_documents(self, user_query: str) -> List[Dict]:
+        '''
+        Retrives the raw documents without LLM generation
+        '''
+        pass
+
+
+
+
+

rag_scripts/llm/llmResponse.py

@@ -0,0 +1,63 @@
+import os
+import traceback
+from typing import List, Dict, Any, Optional
+from groq import Groq
+
+from rag_scripts.interfaces import ILLM
+from configuration import Configuration
+
+class GROQLLM(ILLM):
+    def __init__(self, api_key: str = Configuration.GROQ_API_KEY,
+                 model_name: str = Configuration.DEFAULT_GROQ_LLM_MODEL):
+        if not api_key:
+            raise ValueError("Groq API not provided in env file")
+        
+        self.api_key = api_key
+        self.model_name = model_name
+        self.client = Groq(api_key=self.api_key)
+        self.default_system_message = ("You are a helpful assistant for Flykite Airline HT Policy document queries."
+                                       "provide a concise and accurate answers based strictly on the provided context."
+                                       "Do not hallucinate or add ungrounded details.")
+
+        print(f"Initialized Groq LLM with model: {self.model_name}")
+
+    def generate_response(self, prompt, system_message: Optional[str] = None,
+                          context: Optional[List[Dict]] = None,
+                          temperature: float = 0.1,
+                          top_p: float = 0.95,
+                          max_tokens: int = 1000) -> str:
+        try:
+            complete_prompt = prompt
+            if context:
+                context_text ="\n".join([doc['content'] for doc in context])
+                complete_prompt = f"Context:\n{context_text}\n\nQuestion: {prompt}"
+            
+            message = [{"role": "system", "content":system_message or self.default_system_message},
+                       {"role":"user","content":complete_prompt} ]
+            
+            completion = self.client.chat.completions.create(
+                model = self.model_name,
+                messages= message,
+                temperature=temperature if temperature is not None else 0.1,
+                max_tokens=max_tokens if max_tokens is not None else 1000,
+                top_p=top_p if top_p is not None else 0.95,
+                stream=True, stop=None
+            )
+
+            response_content = ""
+            for chunk in completion:
+                if chunk.choices and chunk.choices[0].delta.content:
+                    response_content+=chunk.choices[0].delta.content
+            return response_content.strip()
+
+        except Exception as ex:
+            print(f"Exception in LLM response: {ex}")
+            traceback.print_exc()
+
+
+    def llm_judge(selfself, prompt:str, **kwargs) -> Dict[str,Any]:
+        try:
+            pass
+        except Exception as ex:
+            traceback.print_exc()
+

rag_scripts/rag_pipeline.py

@@ -0,0 +1,206 @@
+import os
+import json
+import traceback
+import math
+from typing import List, Dict, Any
+
+from configuration import Configuration
+from rag_scripts.interfaces import IDocumentChunker, IEmbedder, IVector, ILLM, IRAGPipeline
+from sentence_transformers import CrossEncoder
+from rag_scripts.documents_processing.chunking import PyMuPDFChunker
+from rag_scripts.embedding.embedder import SentenceTransformerEmbedder
+from rag_scripts.embedding.vector_db.chroma_db import chromaDBVectorDB
+from rag_scripts.embedding.vector_db.faiss_db import FAISSVectorDB
+from rag_scripts.embedding.vector_db.pinecone_db import PineconeVectorDB
+from rag_scripts.llm.llmResponse import GROQLLM
+
+class RAGPipeline(IRAGPipeline):
+
+    def __init__(self,
+                 document_path: str = Configuration.FULL_PDF_PATH,
+                 chunker: IDocumentChunker = None,
+                 embedder: IEmbedder = None,
+                 vector_db: IVector = None,
+                 llm: ILLM = None,
+                 chunk_size: int = Configuration.DEFAULT_CHUNK_SIZE,
+                 chunk_overlap: int = Configuration.DEFAULT_CHUNK_OVERLAP,
+                 embedding_model_name: str = Configuration.DEFAULT_SENTENCE_TRANSFORMER_MODEL,
+                 llm_model_name: str = Configuration.DEFAULT_GROQ_LLM_MODEL,
+                 vector_db_type: str = "chroma",
+                 db_path: str = None,
+                 collection_name: str = None,
+                 re_ranker_model_name: str = Configuration.DEFAULT_RERANKER
+                 ):
+        self.document_path = document_path
+
+        self.chunker = chunker if chunker else PyMuPDFChunker(
+            pdf_path = self.document_path,
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap
+        )
+
+        self.embedder = embedder if embedder else SentenceTransformerEmbedder(model_name = embedding_model_name)
+
+        if vector_db:
+            self.vector_db = vector_db
+        
+        else:
+            if not isinstance(vector_db_type, str):
+                raise ValueError("vector db type must be string")
+            db_path = db_path or (
+                Configuration.CHROMA_DB_PATH if vector_db_type.lower() == 'chroma'
+                else
+                Configuration.FAISS_DB_PATH if vector_db_type.lower() == 'faiss'
+                else "" )
+            collection_name = collection_name or Configuration.COLLECTION_NAME
+
+            if vector_db_type.lower() == "chroma":
+                self.vector_db = chromaDBVectorDB(
+                    embedder = self.embedder,
+                    db_path=db_path,
+                    collection_name=collection_name
+                )
+            elif vector_db_type.lower() == "faiss":
+                self.vector_db = FAISSVectorDB(
+                    embedder=self.embedder,
+                    db_path=db_path,
+                    collection_name=collection_name
+                )
+            elif vector_db_type.lower() == "pinecone":
+                self.vector_db = PineconeVectorDB(
+                    embedder=self.embedder,
+                    db_path=db_path,
+                    collection_name=collection_name
+                )
+            else:
+                raise ValueError("RAG application suppots chroma or faiss db")
+            
+        self.llm = llm if llm else GROQLLM(
+            api_key= Configuration.GROQ_API_KEY,
+            model_name=llm_model_name )
+
+        self.re_ranker = None
+        if re_ranker_model_name:
+            try:
+                self.re_ranker = CrossEncoder(re_ranker_model_name=re_ranker_model_name)
+                print(f"ReRanker model loaded: {re_ranker_model_name}")
+            except Exception as ex:
+                print(f"ReRanker model could not be loaded: {re_ranker_model_name}")
+                self.re_ranker = None
+        
+        print("RAG pipeline initialized")
+
+    def build_index(self):
+        print(f"building index for {self.document_path}")
+        try:
+            document_hash = self.chunker.hash_document()
+            print(f"{document_hash}")
+            existing_chunk_ids = self.vector_db.get_document_hash_ids(document_hash)
+
+            if existing_chunk_ids:
+                print(f"Documents {self.document_path} hash: {document_hash[:8]} already present in the vector DB with {len(existing_chunk_ids)} chunks.")
+                return
+            
+            print(f"Chunking starting")
+            document_hash, chunks = self.chunker.chunk_documents()
+
+            if not chunks:
+                print(f"No chunks generated for {self.document_path} index not built")
+                return
+            
+            self.vector_db.add_chunks(chunks)
+
+            print(f"Index built successfully for the {self.document_path} with {len(chunks)}")
+
+
+        except FileNotFoundError:
+            print(f"Exception Document not at {self.document_path}")
+            traceback.print_exc()
+        except Exception as ex:
+            print(f"Exception in build index {ex}")
+            traceback.print_exc()
+
+    def retrieve_raw_documents(self, user_query: str, k: int =5) -> List[Dict[str,Any]]:
+        if self.vector_db.count_documents() == 0:
+            print("vector database is empty please build index first")
+            return []
+        #query_embedding = self.embedder.embed_query(query=user_query)
+        query_embedding = user_query
+        initial_retrieval_k = k*3 if self.re_ranker else k
+        retrieved_docs_with_score = self.vector_db.search(query_embedding, k=initial_retrieval_k)
+        if not retrieved_docs_with_score:
+            print(f"Retrieval failed for {user_query}")
+            return []
+        retrieved_docs = [doc for doc in retrieved_docs_with_score]
+        if self.re_ranker:
+            document_content = [doc['content'] for doc in retrieved_docs]
+            rerank_score = self.re_ranker.rerank(document_content)
+            doc_with_reRank = []
+            for idx, doc in enumerate(retrieved_docs):
+                doc_with_reRank.append({
+                    'doc': doc,
+                    'rerank_score': rerank_score[idx]
+                })
+            ranked_docs = sorted(doc_with_reRank, key=lambda x: x['rerank_score'], reverse=True)
+            final_retrieved_docs = [item['doc'] for item in ranked_docs[:k]]
+        else:
+            final_retrieved_docs = retrieved_docs
+        return final_retrieved_docs
+
+    def query(self, user_query: str, k: int=3,
+              include_metadata: bool = True,
+              user_context: Dict[str,Any]=None) -> Dict[str,Any]:
+        if not user_query.strip():
+            return {"summary": "Enter the query", "sources": []}
+        
+        retrieved_docs = self.retrieve_raw_documents(user_query,k)
+
+        if not retrieved_docs:
+            return {"summary": "Unable to find relevant information in the documents for the query asked. "
+                               "Please refer contact HR department directly or refer HR policy document",
+                    "sources": []}
+
+
+
+        context_info =[]
+        metadata_info = []
+
+        for indx, doc in enumerate(retrieved_docs):
+            context_info.append(f"Document {indx+1} content: {doc['content']}")
+            if include_metadata:
+                metadata = doc['metadata']
+                metadata_info.append({
+                 "document_id": f"DOC {indx+1}",
+                 "page": str(metadata.get("page_number","NA")),
+                "section": metadata.get("section","NA"),
+                "clause": metadata.get("clause","NA") })
+        
+        context_string = "\n".join(context_info)
+        user_context = user_context or {"role":"general", "location":"chennai","department":"unknown"}
+        context_description = (f" for a {user_context['role']} in"
+                               f"{user_context['location']} and {user_context['department']}")
+
+
+
+        prompt = (
+            f"You are an expert assistant for Flykite Airlines HR Policy queries. "
+            f"Answer the question '{user_query}' based solely on the provided context from the Flykite Airlines HR Policy, "
+            f"Tailor the answer for a {user_context['role']} in {user_context['location']} and {user_context['department']}. "
+            f"Include only the criteria and details that directly address the question, ensuring all relevant points from the context are covered without adding unrelated information or assumptions. "
+            f"Present the answer in a concise format using bullet points, a table, or sections for readability, and cite specific sections and clauses from the sources where applicable. "
+            f"Cite specific sections and clauses from the sources using the format (soruce: DOC X, Page: Y, Section: Z, Clause: A) at the end of each relevant point or page"
+            f"If the query is ambiguous, ask for clarification. If the context does not fully address the question, state what is known and suggest consulting the full HR Policy or HR department. "
+            f"Context: \n{context_string}\n\n"
+            f"Answer: "  )
+        
+        llm_response = self.llm.generate_response(prompt)
+        final_response = {"summary": llm_response.strip(),
+                          "sources":metadata_info if include_metadata else []
+
+                          }
+
+
+
+        return final_response
+
+        

requirements.txt

@@ -0,0 +1,11 @@
+langchain-text-splitters
+sentence-transformers
+chromadb
+pymupdf
+regex
+groq
+streamlit
+huggingface_hub
+faiss-cpu
+pinecone
+bert-score