Joseph Pollack commited on
Commit
b7cacdf
Β·
unverified Β·
1 Parent(s): f4bce3e

adds model loading and removes deprecated messages format

Browse files
Files changed (1) hide show
  1. app.py +44 -14
app.py CHANGED
@@ -126,13 +126,13 @@ class LOperatorDemo:
126
  return f"❌ Error generating action: {str(e)}"
127
 
128
  @spaces.GPU(duration=90) # 1.5 minutes for chat responses
129
- def chat_with_model(self, message: str, history: List[List[str]], image: Image.Image = None) -> tuple:
130
  """Chat interface function for Gradio"""
131
  if not self.is_loaded:
132
- return history + [[message, "❌ Model not loaded. Please load the model first."]]
133
 
134
  if image is None:
135
- return history + [[message, "❌ Please upload an Android screenshot image."]]
136
 
137
  try:
138
  # Extract goal and instruction from message
@@ -149,7 +149,7 @@ class LOperatorDemo:
149
  instruction = line.replace("Step:", "").strip()
150
 
151
  if not goal or not instruction:
152
- return history + [[message, "❌ Please provide both Goal and Step in your message."]]
153
  else:
154
  # Treat as general instruction
155
  goal = "Complete the requested action"
@@ -157,15 +157,32 @@ class LOperatorDemo:
157
 
158
  # Generate action
159
  response = self.generate_action(image, goal, instruction)
160
- return history + [[message, response]]
161
 
162
  except Exception as e:
163
  logger.error(f"Error in chat: {str(e)}")
164
- return history + [[message, f"❌ Error: {str(e)}"]]
165
 
166
  # Initialize demo
167
  demo_instance = LOperatorDemo()
168
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
169
  # Load example episodes
170
  def load_example_episodes():
171
  """Load example episodes from the extracted data"""
@@ -233,7 +250,7 @@ def create_demo():
233
 
234
  ## πŸš€ How to Use
235
 
236
- 1. **Load the Model**: Click the "Load Model" button to initialize the L-Operator model
237
  2. **Upload Screenshot**: Upload an Android device screenshot
238
  3. **Provide Instructions**: Enter your goal and step instructions
239
  4. **Get Actions**: The model will generate JSON actions for Android device control
@@ -257,6 +274,13 @@ def create_demo():
257
 
258
  with gr.Row():
259
  with gr.Column(scale=1):
 
 
 
 
 
 
 
260
  gr.Markdown("### πŸ“± Input")
261
  image_input = gr.Image(
262
  label="Android Screenshot",
@@ -288,6 +312,7 @@ def create_demo():
288
  title="L-Operator Chat",
289
  description="Chat with L-Operator using screenshots and text instructions",
290
  examples=load_example_episodes(),
 
291
  )
292
 
293
  gr.Markdown("### 🎯 Action Output")
@@ -298,9 +323,6 @@ def create_demo():
298
  )
299
 
300
  # Event handlers
301
- def on_load_model():
302
- return demo_instance.load_model()
303
-
304
  def on_generate_action(image, goal, step):
305
  if not image:
306
  return {"error": "Please upload an image"}
@@ -317,10 +339,12 @@ def create_demo():
317
  except:
318
  return {"raw_response": response}
319
 
320
- load_btn.click(
321
- fn=on_load_model,
322
- outputs=load_status
323
- )
 
 
324
 
325
  generate_btn.click(
326
  fn=on_generate_action,
@@ -328,6 +352,12 @@ def create_demo():
328
  outputs=action_output
329
  )
330
 
 
 
 
 
 
 
331
  # Update chat interface when image changes
332
  def update_chat_image(image):
333
  return image
 
126
  return f"❌ Error generating action: {str(e)}"
127
 
128
  @spaces.GPU(duration=90) # 1.5 minutes for chat responses
129
+ def chat_with_model(self, message: str, history: List[Dict[str, str]], image: Image.Image = None) -> List[Dict[str, str]]:
130
  """Chat interface function for Gradio"""
131
  if not self.is_loaded:
132
+ return history + [{"role": "user", "content": message}, {"role": "assistant", "content": "❌ Model not loaded. Please load the model first."}]
133
 
134
  if image is None:
135
+ return history + [{"role": "user", "content": message}, {"role": "assistant", "content": "❌ Please upload an Android screenshot image."}]
136
 
137
  try:
138
  # Extract goal and instruction from message
 
149
  instruction = line.replace("Step:", "").strip()
150
 
151
  if not goal or not instruction:
152
+ return history + [{"role": "user", "content": message}, {"role": "assistant", "content": "❌ Please provide both Goal and Step in your message."}]
153
  else:
154
  # Treat as general instruction
155
  goal = "Complete the requested action"
 
157
 
158
  # Generate action
159
  response = self.generate_action(image, goal, instruction)
160
+ return history + [{"role": "user", "content": message}, {"role": "assistant", "content": response}]
161
 
162
  except Exception as e:
163
  logger.error(f"Error in chat: {str(e)}")
164
+ return history + [{"role": "user", "content": message}, {"role": "assistant", "content": f"❌ Error: {str(e)}"}]
165
 
166
  # Initialize demo
167
  demo_instance = LOperatorDemo()
168
 
169
+ # Auto-load the model on startup
170
+ def auto_load_model():
171
+ """Auto-load the model when the application starts"""
172
+ try:
173
+ logger.info("Auto-loading L-Operator model on startup...")
174
+ result = demo_instance.load_model()
175
+ logger.info(f"Auto-load result: {result}")
176
+ return result
177
+ except Exception as e:
178
+ logger.error(f"Error auto-loading model: {str(e)}")
179
+ return f"❌ Error auto-loading model: {str(e)}"
180
+
181
+ # Load model automatically (this happens during import)
182
+ print("πŸš€ Auto-loading L-Operator model on startup...")
183
+ auto_load_model()
184
+ print("βœ… Model loading completed!")
185
+
186
  # Load example episodes
187
  def load_example_episodes():
188
  """Load example episodes from the extracted data"""
 
250
 
251
  ## πŸš€ How to Use
252
 
253
+ 1. **Model Loading**: The L-Operator model loads automatically on startup
254
  2. **Upload Screenshot**: Upload an Android device screenshot
255
  3. **Provide Instructions**: Enter your goal and step instructions
256
  4. **Get Actions**: The model will generate JSON actions for Android device control
 
274
 
275
  with gr.Row():
276
  with gr.Column(scale=1):
277
+ gr.Markdown("### πŸ€– Model Status")
278
+ model_status = gr.Textbox(
279
+ label="L-Operator Model",
280
+ value="πŸ”„ Loading model on startup...",
281
+ interactive=False
282
+ )
283
+
284
  gr.Markdown("### πŸ“± Input")
285
  image_input = gr.Image(
286
  label="Android Screenshot",
 
312
  title="L-Operator Chat",
313
  description="Chat with L-Operator using screenshots and text instructions",
314
  examples=load_example_episodes(),
315
+ type="messages"
316
  )
317
 
318
  gr.Markdown("### 🎯 Action Output")
 
323
  )
324
 
325
  # Event handlers
 
 
 
326
  def on_generate_action(image, goal, step):
327
  if not image:
328
  return {"error": "Please upload an image"}
 
339
  except:
340
  return {"raw_response": response}
341
 
342
+ # Update model status on page load
343
+ def update_model_status():
344
+ if demo_instance.is_loaded:
345
+ return "βœ… L-Operator model loaded and ready!"
346
+ else:
347
+ return "❌ Model failed to load. Please check logs."
348
 
349
  generate_btn.click(
350
  fn=on_generate_action,
 
352
  outputs=action_output
353
  )
354
 
355
+ # Update model status on page load
356
+ demo.load(
357
+ fn=update_model_status,
358
+ outputs=model_status
359
+ )
360
+
361
  # Update chat interface when image changes
362
  def update_chat_image(image):
363
  return image