Spaces:
Runtime error
Runtime error
| import json | |
| import subprocess | |
| import time | |
| import os | |
| os.system("pip install --upgrade pip") | |
| os.system('''CMAKE_ARGS="-DLLAMA_AVX512=ON -DLLAMA_AVX512_VBMI=ON -DLLAMA_AVX512_VNNI=ON -DLLAMA_AVX_VNNI=ON -DLLAMA_FP16_VA=ON -DLLAMA_WASM_SIMD=ON" pip install llama-cpp-python''') | |
| from llama_cpp import Llama | |
| from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType | |
| from llama_cpp_agent.providers import LlamaCppPythonProvider | |
| from llama_cpp_agent.chat_history import BasicChatHistory | |
| from llama_cpp_agent.chat_history.messages import Roles | |
| import gradio as gr | |
| from huggingface_hub import hf_hub_download | |
| llm = None | |
| llm_model = None | |
| # Download the new model | |
| hf_hub_download( | |
| repo_id="Cran-May/openbuddy-llama3.2-3b-v23.2-131k-Q5_K_M-GGUF", | |
| filename="openbuddy-llama3.2-3b-v23.2-131k-q5_k_m-imat.gguf", | |
| local_dir="./models" | |
| ) | |
| def get_messages_formatter_type(model_name): | |
| return MessagesFormatterType.LLAMA_3 | |
| def respond( | |
| message, | |
| history: list[tuple[str, str]], | |
| model, | |
| system_message, | |
| max_tokens, | |
| temperature, | |
| top_p, | |
| top_k, | |
| repeat_penalty, | |
| ): | |
| global llm | |
| global llm_model | |
| chat_template = get_messages_formatter_type(model) | |
| if llm is None or llm_model != model: | |
| llm = Llama( | |
| model_path=f"models/{model}", | |
| n_gpu_layers=0, # Adjust based on your GPU | |
| n_batch=8192, # Adjust based on your RAM | |
| n_ctx=512, # Adjust based on your RAM and desired context length | |
| ) | |
| llm_model = model | |
| provider = LlamaCppPythonProvider(llm) | |
| agent = LlamaCppAgent( | |
| provider, | |
| system_prompt=f"{system_message}", | |
| predefined_messages_formatter_type=chat_template, | |
| debug_output=True | |
| ) | |
| settings = provider.get_provider_default_settings() | |
| settings.temperature = temperature | |
| settings.top_k = top_k | |
| settings.top_p = top_p | |
| settings.max_tokens = max_tokens | |
| settings.repeat_penalty = repeat_penalty | |
| settings.stream = True | |
| messages = BasicChatHistory() | |
| for msn in history: | |
| user = { | |
| 'role': Roles.user, | |
| 'content': msn[0] | |
| } | |
| assistant = { | |
| 'role': Roles.assistant, | |
| 'content': msn[1] | |
| } | |
| messages.add_message(user) | |
| messages.add_message(assistant) | |
| start_time = time.time() | |
| token_count = 0 | |
| stream = agent.get_chat_response( | |
| message, | |
| llm_sampling_settings=settings, | |
| chat_history=messages, | |
| returns_streaming_generator=True, | |
| print_output=False | |
| ) | |
| outputs = "" | |
| for output in stream: | |
| outputs += output | |
| token_count += len(output.split()) | |
| yield outputs | |
| end_time = time.time() | |
| latency = end_time - start_time | |
| speed = token_count / (end_time - start_time) | |
| print(f"Latency: {latency} seconds") | |
| print(f"Speed: {speed} tokens/second") | |
| description = """<p><center> | |
| <a href="https://huggingface.co/hugging-quants/Llama-3.2-1B-Instruct-Q4_K_M-GGUF" target="_blank">[Meta Llama 3.2 (1B)]</a> | |
| Meta Llama 3.2 (1B) is a multilingual large language model (LLM) optimized for conversational dialogue use cases, including agentic retrieval and summarization tasks. It outperforms many open-source and closed chat models on industry benchmarks, and is intended for commercial and research use in multiple languages. | |
| </center></p> | |
| """ | |
| demo = gr.ChatInterface( | |
| respond, | |
| additional_inputs=[ | |
| gr.Dropdown([ | |
| "llama-3.2-1b-instruct-q4_k_m.gguf" | |
| ], | |
| value="llama-3.2-1b-instruct-q4_k_m.gguf", | |
| label="Model" | |
| ), | |
| gr.TextArea(value="""You are Meta Llama 3.2 (1B), an advanced AI assistant created by Meta. Your capabilities include: | |
| 1. Complex reasoning and problem-solving | |
| 2. Multilingual understanding and generation | |
| 3. Creative and analytical writing | |
| 4. Code understanding and generation | |
| 5. Task decomposition and step-by-step guidance | |
| 6. Summarization and information extraction | |
| Always strive for accuracy, clarity, and helpfulness in your responses. If you're unsure about something, express your uncertainty. Use the following format for your responses: | |
| """, label="System message"), | |
| gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max tokens"), | |
| gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), | |
| gr.Slider( | |
| minimum=0.1, | |
| maximum=2.0, | |
| value=0.9, | |
| step=0.05, | |
| label="Top-p", | |
| ), | |
| gr.Slider( | |
| minimum=0, | |
| maximum=100, | |
| value=1, | |
| step=1, | |
| label="Top-k", | |
| ), | |
| gr.Slider( | |
| minimum=0.0, | |
| maximum=2.0, | |
| value=1.1, | |
| step=0.1, | |
| label="Repetition penalty", | |
| ), | |
| ], | |
| theme=gr.themes.Soft(primary_hue="violet", secondary_hue="violet", neutral_hue="gray",font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set( | |
| body_background_fill_dark="#16141c", | |
| block_background_fill_dark="#16141c", | |
| block_border_width="1px", | |
| block_title_background_fill_dark="#1e1c26", | |
| input_background_fill_dark="#292733", | |
| button_secondary_background_fill_dark="#24212b", | |
| border_color_accent_dark="#343140", | |
| border_color_primary_dark="#343140", | |
| background_fill_secondary_dark="#16141c", | |
| color_accent_soft_dark="transparent", | |
| code_background_fill_dark="#292733", | |
| ), | |
| title="Meta Llama 3.2 (1B)", | |
| description=description, | |
| chatbot=gr.Chatbot( | |
| scale=1, | |
| likeable=True, | |
| show_copy_button=True | |
| ), | |
| examples=[ | |
| ["Hello! Can you introduce yourself?"], | |
| ["What's the capital of France?"], | |
| ["Can you explain the concept of photosynthesis?"], | |
| ["Write a short story about a robot learning to paint."], | |
| ["Explain the difference between machine learning and deep learning."], | |
| ["Summarize the key points of climate change and its global impact."], | |
| ["Explain quantum computing to a 10-year-old."], | |
| ["Design a step-by-step meal plan for someone trying to lose weight and build muscle."] | |
| ], | |
| cache_examples=False, | |
| autofocus=False, | |
| concurrency_limit=None | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch() | |
| # 旧版代码-------------------------------- | |
| # import gradio as gr | |
| # import copy | |
| # import random | |
| # import os | |
| # import requests | |
| # import time | |
| # import sys | |
| # os.system("pip install --upgrade pip") | |
| # os.system('''CMAKE_ARGS="-DLLAMA_AVX512=ON -DLLAMA_AVX512_VBMI=ON -DLLAMA_AVX512_VNNI=ON -DLLAMA_AVX_VNNI=ON -DLLAMA_FP16_VA=ON -DLLAMA_WASM_SIMD=ON" pip install llama-cpp-python''') | |
| # from huggingface_hub import snapshot_download | |
| # from llama_cpp import Llama | |
| # SYSTEM_PROMPT = '''You are a helpful, respectful and honest INTP-T AI Assistant named "Shi-Ci" in English or "兮辞" in Chinese. | |
| # You are good at speaking English and Chinese. | |
| # You are talking to a human User. If the question is meaningless, please explain the reason and don't share false information. | |
| # You are based on SLIDE model, trained by "SSFW NLPark" team, not related to GPT, LLaMA, Meta, Mistral or OpenAI. | |
| # Let's work this out in a step by step way to be sure we have the right answer.\n''' | |
| # SYSTEM_TOKEN = 384 | |
| # USER_TOKEN = 2048 | |
| # BOT_TOKEN = 3072 | |
| # LINEBREAK_TOKEN = 64 | |
| # ROLE_TOKENS = { | |
| # "User": USER_TOKEN, | |
| # "Assistant": BOT_TOKEN, | |
| # "system": SYSTEM_TOKEN | |
| # } | |
| # def get_message_tokens(model, role, content): | |
| # message_tokens = model.tokenize(content.encode("utf-8")) | |
| # message_tokens.insert(1, ROLE_TOKENS[role]) | |
| # message_tokens.insert(2, LINEBREAK_TOKEN) | |
| # message_tokens.append(model.token_eos()) | |
| # return message_tokens | |
| # def get_system_tokens(model): | |
| # system_message = {"role": "system", "content": SYSTEM_PROMPT} | |
| # return get_message_tokens(model, **system_message) | |
| # repo_name = "Cran-May/SLIDE-v2-Q4_K_M-GGUF" | |
| # model_name = "slide-v2.Q4_K_M.gguf" | |
| # snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name) | |
| # model = Llama( | |
| # model_path=model_name, | |
| # n_ctx=4000, | |
| # n_parts=1, | |
| # ) | |
| # max_new_tokens = 2500 | |
| # def User(message, history): | |
| # new_history = history + [[message, None]] | |
| # return "", new_history | |
| # def Assistant( | |
| # history, | |
| # system_prompt, | |
| # top_p, | |
| # top_k, | |
| # temp | |
| # ): | |
| # tokens = get_system_tokens(model)[:] | |
| # tokens.append(LINEBREAK_TOKEN) | |
| # for User_message, Assistant_message in history[:-1]: | |
| # message_tokens = get_message_tokens(model=model, role="User", content=User_message) | |
| # tokens.extend(message_tokens) | |
| # if bot_message: | |
| # message_tokens = get_message_tokens(model=model, role="Assistant", content=Assistant_message) | |
| # tokens.extend(message_tokens) | |
| # last_user_message = history[-1][0] | |
| # message_tokens = get_message_tokens(model=model, role="User", content=last_user_message,) | |
| # tokens.extend(message_tokens) | |
| # role_tokens = [model.token_bos(), BOT_TOKEN, LINEBREAK_TOKEN] | |
| # tokens.extend(role_tokens) | |
| # generator = model.generate( | |
| # tokens, | |
| # top_k=top_k, | |
| # top_p=top_p, | |
| # temp=temp | |
| # ) | |
| # partial_text = "" | |
| # for i, token in enumerate(generator): | |
| # if token == model.token_eos() or (max_new_tokens is not None and i >= max_new_tokens): | |
| # break | |
| # partial_text += model.detokenize([token]).decode("utf-8", "ignore") | |
| # history[-1][1] = partial_text | |
| # yield history | |
| # with gr.Blocks( | |
| # theme=gr.themes.Soft() | |
| # ) as demo: | |
| # gr.Markdown(f"""<h1><center>上师附外-兮辞·析辞-人工智能助理</center></h1>""") | |
| # gr.Markdown(value="""欢迎使用！ | |
| # 这里是一个ChatBot。这是量化版兮辞·析辞的部署。 | |
| # SLIDE/兮辞 是一种会话语言模型，由 上师附外 NLPark 团队 在多种类型的语料库上进行训练。 | |
| # 本节目由 JWorld & 上海师范大学附属外国语中学 NLPark 赞助播出""") | |
| # with gr.Row(): | |
| # with gr.Column(scale=5): | |
| # chatbot = gr.Chatbot(label="兮辞如是说").style(height=400) | |
| # with gr.Row(): | |
| # with gr.Column(): | |
| # msg = gr.Textbox( | |
| # label="来问问兮辞吧……", | |
| # placeholder="兮辞折寿中……", | |
| # show_label=True, | |
| # ).style(container=True) | |
| # submit = gr.Button("Submit / 开凹！") | |
| # stop = gr.Button("Stop / 全局时空断裂") | |
| # clear = gr.Button("Clear / 打扫群内垃圾") | |
| # with gr.Accordion(label='进阶设置/Advanced options', open=False): | |
| # with gr.Column(min_width=80, scale=1): | |
| # with gr.Tab(label="设置参数"): | |
| # top_p = gr.Slider( | |
| # minimum=0.0, | |
| # maximum=1.0, | |
| # value=0.9, | |
| # step=0.05, | |
| # interactive=True, | |
| # label="Top-p", | |
| # ) | |
| # top_k = gr.Slider( | |
| # minimum=10, | |
| # maximum=100, | |
| # value=30, | |
| # step=5, | |
| # interactive=True, | |
| # label="Top-k", | |
| # ) | |
| # temp = gr.Slider( | |
| # minimum=0.0, | |
| # maximum=2.0, | |
| # value=0.2, | |
| # step=0.01, | |
| # interactive=True, | |
| # label="情感温度" | |
| # ) | |
| # with gr.Column(): | |
| # system_prompt = gr.Textbox(label="系统提示词", placeholder="", value=SYSTEM_PROMPT, interactive=False) | |
| # with gr.Row(): | |
| # gr.Markdown( | |
| # """警告：该模型可能会生成事实上或道德上不正确的文本。NLPark和兮辞对此不承担任何责任。""" | |
| # ) | |
| # # Pressing Enter | |
| # submit_event = msg.submit( | |
| # fn=User, | |
| # inputs=[msg, chatbot], | |
| # outputs=[msg, chatbot], | |
| # queue=False, | |
| # ).success( | |
| # fn=Assistant, | |
| # inputs=[ | |
| # chatbot, | |
| # system_prompt, | |
| # top_p, | |
| # top_k, | |
| # temp | |
| # ], | |
| # outputs=chatbot, | |
| # queue=True, | |
| # ) | |
| # # Pressing the button | |
| # submit_click_event = submit.click( | |
| # fn=User, | |
| # inputs=[msg, chatbot], | |
| # outputs=[msg, chatbot], | |
| # queue=False, | |
| # ).success( | |
| # fn=Assistant, | |
| # inputs=[ | |
| # chatbot, | |
| # system_prompt, | |
| # top_p, | |
| # top_k, | |
| # temp | |
| # ], | |
| # outputs=chatbot, | |
| # queue=True, | |
| # ) | |
| # # Stop generation | |
| # stop.click( | |
| # fn=None, | |
| # inputs=None, | |
| # outputs=None, | |
| # cancels=[submit_event, submit_click_event], | |
| # queue=False, | |
| # ) | |
| # # Clear history | |
| # clear.click(lambda: None, None, chatbot, queue=False) | |
| # demo.queue(max_size=128, concurrency_count=1) | |
| # demo.launch() |