load model first time

app.py

@@ -5,25 +5,27 @@ from pydantic import BaseModel, Field
 
 class RequestGenerate(BaseModel):
     prompt: str 
-    do_sample: bool = Field(default=True, example=True)
-    top_k: int = Field(default=1, example=1),
-    temperature: float = Field(default=0.9, example=0.9),
-    max_new_tokens: int = Field(default=500, example=500),
-    repetition_penalty: float = Field(default=1.5, example=1.5),
+    do_sample: bool = Field(default=bool(True), example=True)
+    top_k: int = Field(default=int(1), example=1),
+    temperature: float = Field(default=float(0.9), example=0.9),
+    max_new_tokens: int = Field(default=int(500), example=500),
+    repetition_penalty: float = Field(default=float(1.5), example=1.5),
 
 app = FastAPI()
 
+# model_name_or_id = "AI4Chem/ChemLLM-7B-Chat"
+model_name_or_id = "AI4Chem/CHEMLLM-2b-1_5"
+
+model = AutoModelForCausalLM.from_pretrained(model_name_or_id,trust_remote_code=True)
+tokenizer = AutoTokenizer.from_pretrained(model_name_or_id,trust_remote_code=True)
+
 @app.get("/")
 def greet_json():
     return {"Hello": "World!"}
 
 @app.post("/generate")
 def generate(req: RequestGenerate):
-    model_name_or_id = "AI4Chem/ChemLLM-7B-Chat"
-    # model_name_or_id = "AI4Chem/CHEMLLM-2b-1_5"
-
-    model = AutoModelForCausalLM.from_pretrained(model_name_or_id,trust_remote_code=True)
-    tokenizer = AutoTokenizer.from_pretrained(model_name_or_id,trust_remote_code=True)
+    
 
     inputs = tokenizer(req.prompt, return_tensors="pt")