app.py

"""
cf https://huggingface.co/spaces/Nymbo/Qwen-2.5-72B-Instruct/blob/main/app.py
   https://huggingface.co/spaces/prithivMLmods/Llama-3.1-8B-Instruct/blob/main/app.py
https://github.com/huggingface/huggingface-llama-recipes/blob/main/api_inference/inference-api.ipynb
"""

import os 
import time
import gradio as gr

from openai import OpenAI
# from huggingface_hub import InferenceClient

os.environ.update(TZ='Asia/Shanghai')
time.tzset()

# ACCESS_TOKEN = os.getenv("HF_TOKEN")

# client = InferenceClient()
# _ = """
client = OpenAI(
    base_url="https://api-inference.huggingface.co/v1/",
    # api_key=ACCESS_TOKEN,
    api_key=os.getenv("HF_TOKEN", 'na')
)
# """


def respond(
    message,
    history: list[tuple[str, str]],
    system_message,
    max_tokens,
    temperature,
    top_p,
):
    messages = [{"role": "system", "content": system_message}]

    for val in history:
        if val[0]:
            messages.append({"role": "user", "content": val[0]})
        if val[1]:
            messages.append({"role": "assistant", "content": val[1]})

    messages.append({"role": "user", "content": message})

    response = ""
    try:
        _ = client.chat.completions.create(
        model="Qwen/Qwen2.5-72B-Instruct",
        max_tokens=max_tokens,
        stream=True,
        temperature=temperature,
        top_p=top_p,
        messages=messages,
        )
        for message in _:
            token = message.choices[0].delta.content
            response += token
            yield response
    except Exception as e:
        yield str(e)
        
chatbot = gr.Chatbot(height=600)

css = '''
.gradio-container{max-width: 1000px !important}
h1{text-align:center}
footer {
    visibility: hidden
}
'''

demo = gr.ChatInterface(
    respond,
    type='messages',
    # description='chatbox',
    additional_inputs=[
        gr.Textbox(value="", label="System message"),
        # gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
        gr.Slider(minimum=1, maximum=32768 // 2 - 500, value=32768 // 2 - 500, step=1, label="Max new tokens"),
        gr.Slider(minimum=0.1, maximum=4.0, value=0.3, step=0.1, label="Temperature"),
        gr.Slider(
            minimum=0.1,
            maximum=1.0,
            value=0.95,
            step=0.05,
            label="Top-P",
        ),
        
    ],
    fill_height=True,
    chatbot=chatbot,
    css=css,
    # examples=[{"role": "user", "content": "Define 'deep learning' in once sentence."}],
    # retry_btn="Retry",  # unexpected keyword argument 'retry_btn'
    # undo_btn="Undo",
    # clear_btn="Clear",
    # stop_btn='Cancel',
    # theme="allenai/gradio-theme",
    # theme="Nymbo/Alyx_Theme",
)
if __name__ == "__main__":
    demo.launch()  # ssr=False