add tiktoken

app.py

@@ -2,7 +2,8 @@ import gradio as gr
 from transformers import AutoTokenizer, AutoModelForCausalLM
 
 tokenizer = AutoTokenizer.from_pretrained("Salesforce/codegen25-7b-instruct", trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained("Salesforce/codegen25-7b-instruct", cache_dir="models/")
+# model = AutoModelForCausalLM.from_pretrained("Salesforce/codegen25-7b-instruct", cache_dir="models/")
+model = AutoModelForCausalLM.from_pretrained("Salesforce/codegen25-7b-instruct")
 
 def code_gen(text):
     input_ids = tokenizer(text, return_tensors="pt").input_ids

requirements.txt

@@ -1 +1,2 @@
-transformers
+transformers
+tiktoken