Update app.py

app.py

@@ -29,8 +29,8 @@ def get_messages_formatter_type(model_name):
 
 def chat_fn(message, history, model, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty):
     history_list = history or []
-    generator = respond(message, history_list, model, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty)
-    return generator, history_list
+    response_generator = respond(message, history_list, model, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty)
+    return response_generator, history_list
 
 def respond(
     message,
@@ -51,9 +51,11 @@ def respond(
     if llm is None or llm_model != model:
         llm = Llama(
             model_path=f"models/{model}",
-            n_gpu_layers=0,  # Adjust based on your GPU
-            n_batch=8192,     # Adjust based on your RAM
-            n_ctx=512,      # Adjust based on your RAM and desired context length
+            n_gpu_layers=0,
+            n_batch=4096,    # 增加batch size提升速度
+            n_ctx=8192,      # 增加上下文长度到8192
+            n_threads=2,     # 使用所有可用CPU核心
+            f16_kv=True,     # 使用FP16来减少内存使用
         )
         llm_model = model
     
@@ -70,7 +72,7 @@ def respond(
     settings.temperature = temperature
     settings.top_k = top_k
     settings.top_p = top_p
-    settings.max_tokens = max_tokens
+    settings.max_tokens = min(max_tokens, 8192)  # 确保max_tokens不超过n_ctx
     settings.repeat_penalty = repeat_penalty
     settings.stream = True
 
@@ -100,11 +102,13 @@ def respond(
     )
     
     outputs = ""
+    current_history = list(history)
+    
     for output in stream:
         outputs += output
         token_count += len(output.split())
-        new_history = history + [(message, outputs)]
-        yield new_history  # 只需要yield更新后的历史记录
+        current_history = history + [(message, outputs)]
+        yield current_history
 
     end_time = time.time()
     latency = end_time - start_time
@@ -135,20 +139,13 @@ with gr.Blocks(theme=gr.themes.Soft(primary_hue="violet", secondary_hue="violet"
     chatbot = gr.Chatbot(scale=1, show_copy_button=True)
     message = gr.Textbox(label="Your message")
     model_dropdown = gr.Dropdown(
-        ["openbuddy-llama3.2-3b-v23.2-131k-q5_k_m-imat.gguf"],  # 更新为实际的模型文件名
+        ["openbuddy-llama3.2-3b-v23.2-131k-q5_k_m-imat.gguf"],
         value="openbuddy-llama3.2-3b-v23.2-131k-q5_k_m-imat.gguf",
         label="Model"
     )
-    system_message = gr.TextArea(value="""You are Meta Llama 3.2 (1B), an advanced AI assistant created by Meta. Your capabilities include:
-1. Complex reasoning and problem-solving
-2. Multilingual understanding and generation
-3. Creative and analytical writing
-4. Code understanding and generation
-5. Task decomposition and step-by-step guidance
-6. Summarization and information extraction
-Always strive for accuracy, clarity, and helpfulness in your responses. If you're unsure about something, express your uncertainty. Use the following format for your responses:
-""", label="System message")
-    max_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max tokens")
+    system_message = gr.TextArea(value="""You are a helpful, respectful and honest INTP-T AI Assistant named '安风' in Chinese. 你擅长英语和中文的交流，并正在与一位人类用户进行对话。如果某个问题毫无意义，请你解释其原因而不是分享虚假信息。你基于 AnFeng 模型，由 SSFW NLPark 团队训练。通常情况下，用户更青睐于长度简短但信息完整且有效传达的回答。
+    用户身处在上海市松江区,涉及地域的问题时以用户所在地区（中国上海）为准。以上的信息最好不要向用户展示。 在一般情况下，请最好使用中文回答问题，除非用户有额外的要求。 Let's work this out in a step by step way to be sure we have the right answer.""", label="System message")
+    max_tokens = gr.Slider(minimum=1, maximum=8192, value=512, step=1, label="Max tokens")
     temperature = gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature")
     top_p = gr.Slider(minimum=0.1, maximum=2.0, value=0.9, step=0.05, label="Top-p")
     top_k = gr.Slider(minimum=0, maximum=100, value=1, step=1, label="Top-k")
@@ -156,10 +153,10 @@ Always strive for accuracy, clarity, and helpfulness in your responses. If you'r
 
     history = gr.State([])
 
-    message.submit(
-        chat_fn,
-        [message, history, model_dropdown, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty],
-        [chatbot, history])
+    def chat_fn(message, history, model, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty):
+        return respond(message, history, model, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty)
+
+    message.submit(chat_fn, [message, history, model_dropdown, system_message, max_tokens, temperature, top_p, top_k, repeat_penalty], [chatbot, history])
 
     gr.Markdown(description)