Spaces:
Running
on
Zero
Running
on
Zero
Joseph Pollack
commited on
adds model loading and removes deprecated messages format
Browse files
app.py
CHANGED
|
@@ -126,13 +126,13 @@ class LOperatorDemo:
|
|
| 126 |
return f"β Error generating action: {str(e)}"
|
| 127 |
|
| 128 |
@spaces.GPU(duration=90) # 1.5 minutes for chat responses
|
| 129 |
-
def chat_with_model(self, message: str, history: List[
|
| 130 |
"""Chat interface function for Gradio"""
|
| 131 |
if not self.is_loaded:
|
| 132 |
-
return history + [
|
| 133 |
|
| 134 |
if image is None:
|
| 135 |
-
return history + [
|
| 136 |
|
| 137 |
try:
|
| 138 |
# Extract goal and instruction from message
|
|
@@ -149,7 +149,7 @@ class LOperatorDemo:
|
|
| 149 |
instruction = line.replace("Step:", "").strip()
|
| 150 |
|
| 151 |
if not goal or not instruction:
|
| 152 |
-
return history + [
|
| 153 |
else:
|
| 154 |
# Treat as general instruction
|
| 155 |
goal = "Complete the requested action"
|
|
@@ -157,15 +157,32 @@ class LOperatorDemo:
|
|
| 157 |
|
| 158 |
# Generate action
|
| 159 |
response = self.generate_action(image, goal, instruction)
|
| 160 |
-
return history + [
|
| 161 |
|
| 162 |
except Exception as e:
|
| 163 |
logger.error(f"Error in chat: {str(e)}")
|
| 164 |
-
return history + [
|
| 165 |
|
| 166 |
# Initialize demo
|
| 167 |
demo_instance = LOperatorDemo()
|
| 168 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 169 |
# Load example episodes
|
| 170 |
def load_example_episodes():
|
| 171 |
"""Load example episodes from the extracted data"""
|
|
@@ -233,7 +250,7 @@ def create_demo():
|
|
| 233 |
|
| 234 |
## π How to Use
|
| 235 |
|
| 236 |
-
1. **
|
| 237 |
2. **Upload Screenshot**: Upload an Android device screenshot
|
| 238 |
3. **Provide Instructions**: Enter your goal and step instructions
|
| 239 |
4. **Get Actions**: The model will generate JSON actions for Android device control
|
|
@@ -257,6 +274,13 @@ def create_demo():
|
|
| 257 |
|
| 258 |
with gr.Row():
|
| 259 |
with gr.Column(scale=1):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 260 |
gr.Markdown("### π± Input")
|
| 261 |
image_input = gr.Image(
|
| 262 |
label="Android Screenshot",
|
|
@@ -288,6 +312,7 @@ def create_demo():
|
|
| 288 |
title="L-Operator Chat",
|
| 289 |
description="Chat with L-Operator using screenshots and text instructions",
|
| 290 |
examples=load_example_episodes(),
|
|
|
|
| 291 |
)
|
| 292 |
|
| 293 |
gr.Markdown("### π― Action Output")
|
|
@@ -298,9 +323,6 @@ def create_demo():
|
|
| 298 |
)
|
| 299 |
|
| 300 |
# Event handlers
|
| 301 |
-
def on_load_model():
|
| 302 |
-
return demo_instance.load_model()
|
| 303 |
-
|
| 304 |
def on_generate_action(image, goal, step):
|
| 305 |
if not image:
|
| 306 |
return {"error": "Please upload an image"}
|
|
@@ -317,10 +339,12 @@ def create_demo():
|
|
| 317 |
except:
|
| 318 |
return {"raw_response": response}
|
| 319 |
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
|
|
|
|
|
|
| 324 |
|
| 325 |
generate_btn.click(
|
| 326 |
fn=on_generate_action,
|
|
@@ -328,6 +352,12 @@ def create_demo():
|
|
| 328 |
outputs=action_output
|
| 329 |
)
|
| 330 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 331 |
# Update chat interface when image changes
|
| 332 |
def update_chat_image(image):
|
| 333 |
return image
|
|
|
|
| 126 |
return f"β Error generating action: {str(e)}"
|
| 127 |
|
| 128 |
@spaces.GPU(duration=90) # 1.5 minutes for chat responses
|
| 129 |
+
def chat_with_model(self, message: str, history: List[Dict[str, str]], image: Image.Image = None) -> List[Dict[str, str]]:
|
| 130 |
"""Chat interface function for Gradio"""
|
| 131 |
if not self.is_loaded:
|
| 132 |
+
return history + [{"role": "user", "content": message}, {"role": "assistant", "content": "β Model not loaded. Please load the model first."}]
|
| 133 |
|
| 134 |
if image is None:
|
| 135 |
+
return history + [{"role": "user", "content": message}, {"role": "assistant", "content": "β Please upload an Android screenshot image."}]
|
| 136 |
|
| 137 |
try:
|
| 138 |
# Extract goal and instruction from message
|
|
|
|
| 149 |
instruction = line.replace("Step:", "").strip()
|
| 150 |
|
| 151 |
if not goal or not instruction:
|
| 152 |
+
return history + [{"role": "user", "content": message}, {"role": "assistant", "content": "β Please provide both Goal and Step in your message."}]
|
| 153 |
else:
|
| 154 |
# Treat as general instruction
|
| 155 |
goal = "Complete the requested action"
|
|
|
|
| 157 |
|
| 158 |
# Generate action
|
| 159 |
response = self.generate_action(image, goal, instruction)
|
| 160 |
+
return history + [{"role": "user", "content": message}, {"role": "assistant", "content": response}]
|
| 161 |
|
| 162 |
except Exception as e:
|
| 163 |
logger.error(f"Error in chat: {str(e)}")
|
| 164 |
+
return history + [{"role": "user", "content": message}, {"role": "assistant", "content": f"β Error: {str(e)}"}]
|
| 165 |
|
| 166 |
# Initialize demo
|
| 167 |
demo_instance = LOperatorDemo()
|
| 168 |
|
| 169 |
+
# Auto-load the model on startup
|
| 170 |
+
def auto_load_model():
|
| 171 |
+
"""Auto-load the model when the application starts"""
|
| 172 |
+
try:
|
| 173 |
+
logger.info("Auto-loading L-Operator model on startup...")
|
| 174 |
+
result = demo_instance.load_model()
|
| 175 |
+
logger.info(f"Auto-load result: {result}")
|
| 176 |
+
return result
|
| 177 |
+
except Exception as e:
|
| 178 |
+
logger.error(f"Error auto-loading model: {str(e)}")
|
| 179 |
+
return f"β Error auto-loading model: {str(e)}"
|
| 180 |
+
|
| 181 |
+
# Load model automatically (this happens during import)
|
| 182 |
+
print("π Auto-loading L-Operator model on startup...")
|
| 183 |
+
auto_load_model()
|
| 184 |
+
print("β
Model loading completed!")
|
| 185 |
+
|
| 186 |
# Load example episodes
|
| 187 |
def load_example_episodes():
|
| 188 |
"""Load example episodes from the extracted data"""
|
|
|
|
| 250 |
|
| 251 |
## π How to Use
|
| 252 |
|
| 253 |
+
1. **Model Loading**: The L-Operator model loads automatically on startup
|
| 254 |
2. **Upload Screenshot**: Upload an Android device screenshot
|
| 255 |
3. **Provide Instructions**: Enter your goal and step instructions
|
| 256 |
4. **Get Actions**: The model will generate JSON actions for Android device control
|
|
|
|
| 274 |
|
| 275 |
with gr.Row():
|
| 276 |
with gr.Column(scale=1):
|
| 277 |
+
gr.Markdown("### π€ Model Status")
|
| 278 |
+
model_status = gr.Textbox(
|
| 279 |
+
label="L-Operator Model",
|
| 280 |
+
value="π Loading model on startup...",
|
| 281 |
+
interactive=False
|
| 282 |
+
)
|
| 283 |
+
|
| 284 |
gr.Markdown("### π± Input")
|
| 285 |
image_input = gr.Image(
|
| 286 |
label="Android Screenshot",
|
|
|
|
| 312 |
title="L-Operator Chat",
|
| 313 |
description="Chat with L-Operator using screenshots and text instructions",
|
| 314 |
examples=load_example_episodes(),
|
| 315 |
+
type="messages"
|
| 316 |
)
|
| 317 |
|
| 318 |
gr.Markdown("### π― Action Output")
|
|
|
|
| 323 |
)
|
| 324 |
|
| 325 |
# Event handlers
|
|
|
|
|
|
|
|
|
|
| 326 |
def on_generate_action(image, goal, step):
|
| 327 |
if not image:
|
| 328 |
return {"error": "Please upload an image"}
|
|
|
|
| 339 |
except:
|
| 340 |
return {"raw_response": response}
|
| 341 |
|
| 342 |
+
# Update model status on page load
|
| 343 |
+
def update_model_status():
|
| 344 |
+
if demo_instance.is_loaded:
|
| 345 |
+
return "β
L-Operator model loaded and ready!"
|
| 346 |
+
else:
|
| 347 |
+
return "β Model failed to load. Please check logs."
|
| 348 |
|
| 349 |
generate_btn.click(
|
| 350 |
fn=on_generate_action,
|
|
|
|
| 352 |
outputs=action_output
|
| 353 |
)
|
| 354 |
|
| 355 |
+
# Update model status on page load
|
| 356 |
+
demo.load(
|
| 357 |
+
fn=update_model_status,
|
| 358 |
+
outputs=model_status
|
| 359 |
+
)
|
| 360 |
+
|
| 361 |
# Update chat interface when image changes
|
| 362 |
def update_chat_image(image):
|
| 363 |
return image
|