Spaces:
Running
on
CPU Upgrade
Running
on
CPU Upgrade
File size: 2,193 Bytes
63d1774 f2be657 63d1774 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 |
from httpx import Timeout
from langchain_openai import ChatOpenAI
from modules.nodes.state import ChatState
from tools import (
tool_format_scripture_answer,
tool_get_standardized_prabandham_names,
tool_search_db,
tool_search_web,
tool_push,
tool_get_standardized_azhwar_names,
tool_get_standardized_divya_desam_names,
)
tools = [
tool_search_db,
tool_get_standardized_azhwar_names,
tool_get_standardized_prabandham_names,
tool_get_standardized_divya_desam_names,
## disabling this tool as it is causing more latency
# tool_format_scripture_answer,
# tool_search_web,
# tool_push,
]
llm = ChatOpenAI(
model="gpt-4o-mini", temperature=0.2, max_retries=0, timeout=Timeout(60.0)
).bind_tools(tools)
def _truncate_messages_for_token_limit(messages, max_tokens=50000):
"""
Truncate messages to stay under token limit while preserving assistant-tool_call integrity.
"""
return messages
total_tokens = 0
result = []
# iterate oldest → newest to preserve conversation
for msg in messages:
content = getattr(msg, "content", "")
msg_tokens = len(content) // 4
# gather tool responses if any
tool_calls = getattr(msg, "additional_kwargs", {}).get("tool_calls", [])
group = [msg]
for call in tool_calls:
for m in messages:
if (
getattr(m, "additional_kwargs", {}).get("tool_call_id")
== call["id"]
):
group.append(m)
group_tokens = sum(len(getattr(m, "content", "")) // 4 for m in group)
# if this whole group would exceed the limit, stop
if total_tokens + group_tokens > max_tokens:
break
total_tokens += group_tokens
result.extend(group)
return result
def chatNode(state: ChatState) -> ChatState:
# logger.info("messages before LLM: %s", str(state["messages"]))
state["messages"] = _truncate_messages_for_token_limit(
messages=state["messages"]
)
response = llm.invoke(state["messages"])
state["messages"] = state["messages"] + [response]
return state
|