Update app.py

app.py

@@ -31,18 +31,18 @@ else:
     # torch_dtype = torch.bfloat16 if torch.cuda.get_device_capability()[0] == 8 else torch.float16
     # model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch_dtype, device_map="auto")
     model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", load_in_8bit=True)
-
 tokenizer = AutoTokenizer.from_pretrained(model_id)
 
 prompt_template = f"### Anweisung:\n{{input}}\n\n### Antwort:"
 
 
-def generate(instruction, temperature, max_new_tokens, top_p, length_penalty):
+def generate(instruction, temperature=1, max_new_tokens=256, top_p=0.9, length_penalty=1.0):
     formatted_instruction = prompt_template.format(input=instruction)
     # COMMENT IN FOR NON STREAMING
     # generation_config = GenerationConfig(
     #     do_sample=True,
     #     top_p=top_p,
+    #     top_k=0,
     #     temperature=temperature,
     #     max_new_tokens=max_new_tokens,
     #     early_stopping=True,
@@ -71,7 +71,9 @@ def generate(instruction, temperature, max_new_tokens, top_p, length_penalty):
 
     generate_kwargs = dict(
         top_p=top_p,
+        top_k=0,
         temperature=temperature,
+        do_sample=True,
         max_new_tokens=max_new_tokens,
         early_stopping=True,
         length_penalty=length_penalty,
@@ -142,7 +144,13 @@ with gr.Blocks(theme=theme) as demo:
                     placeholder="Hier Antwort erscheint...",
                 )
                 submit = gr.Button("Generate", variant="primary")
-                gr.Examples(examples=examples, inputs=[instruction])
+                gr.Examples(
+                    examples=examples,
+                    inputs=[instruction],
+                    # cache_examples=True,
+                    # fn=generate,
+                    # outputs=[output],
+                )
 
             with gr.Column(scale=1):
                 temperature = gr.Slider(