just use openrouter

app.py

@@ -1,39 +1,23 @@
 import gradio as gr
 import subprocess
-import spaces
-import torch
 import os
 import re
+from datetime import datetime
 
-zero = torch.Tensor([0]).cuda()
-print(zero.device)  # <-- 'cpu' 🤔
-
-@spaces.GPU
 def run_evaluation(model_name):
-    print(zero.device)  # <-- 'cuda:0' 🤗
-
     results = []
 
-    # Use the secret HF token from the Hugging Face space
-    if "HF_TOKEN" not in os.environ:
-        return "Error: HF_TOKEN not found in environment variables."
+    # Use the secret OpenRouter API key from the Hugging Face space
+    if "OPENROUTER_API_KEY" not in os.environ:
+        return "Error: OPENROUTER_API_KEY not found in environment variables."
 
-    manifest_process = None
     try:
-        # Start manifest server in background with explicit CUDA_VISIBLE_DEVICES
-        manifest_cmd = f"""
-        cd duckdb-nsql/ && 
-        CUDA_VISIBLE_DEVICES=0 HF_TOKEN={os.environ['HF_TOKEN']}  python -m manifest.api.app \
-        --model_type huggingface \
-        --model_generation_type text-generation \
-        --model_name_or_path {model_name} \
-        --fp16 \
-        --device 0
-        """
-        manifest_process = subprocess.Popen(manifest_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        results.append("Started manifest server in background.")
+        # Set up environment
+        env = os.environ.copy()
+        env["OPENROUTER_API_KEY"] = os.environ["OPENROUTER_API_KEY"]
 
         # Run inference
+        current_date = datetime.now().strftime("%Y%m%d")
         inference_cmd = f"""
         cd duckdb-nsql/ && 
         python eval/predict.py \
@@ -42,12 +26,13 @@ def run_evaluation(model_name):
         eval/data/tables.json \
         --output-dir output/ \
         --stop-tokens ';' \
+        --max-tokens 30000 \
         --overwrite-manifest \
-        --manifest-client huggingface \
-        --manifest-connection http://localhost:5000 \
+        --manifest-client openrouter \
+        --manifest-engine {model_name} \
         --prompt-format duckdbinstgraniteshort
         """
-        inference_result = subprocess.run(inference_cmd, shell=True, check=True, capture_output=True, text=True)
+        inference_result = subprocess.run(inference_cmd, shell=True, check=True, capture_output=True, text=True, env=env)
         results.append("Inference completed.")
 
         # Extract JSON file path from inference output
@@ -74,25 +59,20 @@ def run_evaluation(model_name):
         if metrics:
             results.append(f"Evaluation completed:\n{metrics}")
         else:
-            results.append("Evaluation completed, but get metrics.")
+            results.append("Evaluation completed, but couldn't get metrics.")
 
     except subprocess.CalledProcessError as e:
         results.append(f"Error occurred: {str(e)}")
         results.append(f"Command output: {e.output}")
     except Exception as e:
         results.append(f"An unexpected error occurred: {str(e)}")
-    finally:
-        # Terminate the background manifest server
-        if manifest_process:
-            manifest_process.terminate()
-            results.append("Terminated manifest server.")
 
     return "\n\n".join(results)
 
 with gr.Blocks() as demo:
-    gr.Markdown("# DuckDB SQL Evaluation App")
+    gr.Markdown("# DuckDB SQL Evaluation App (OpenRouter)")
 
-    model_name = gr.Textbox(label="Model Name (e.g., Qwen/Qwen2.5-7B-Instruct)")
+    model_name = gr.Textbox(label="Model Name (e.g., qwen/qwen-2.5-72b-instruct)")
     start_btn = gr.Button("Start Evaluation")
     output = gr.Textbox(label="Output", lines=20)
 

requirements.txt

@@ -22,7 +22,8 @@ ninja==1.11.1.1
 langchain
 pydantic
 packaging
-manifest-ml @ git+https://github.com/tdoehmen/manifest@till/added_openrouter
+-e duckdb-nsql/manifest
+# manifest-ml @ git+https://github.com/tdoehmen/manifest@till/added_openrouter
 flask
 diffusers
 deepspeed