Update config.py
Browse files
config.py
CHANGED
|
@@ -68,6 +68,16 @@ class AppConfig:
|
|
| 68 |
|
| 69 |
# Enhanced Inference Providers supporting GPT OSS models
|
| 70 |
INFERENCE_PROVIDERS = {
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 71 |
"cerebras": {
|
| 72 |
"name": "Cerebras",
|
| 73 |
"description": "World-record inference speeds (2-4k tokens/sec for GPT-OSS)",
|
|
@@ -151,6 +161,10 @@ class AppConfig:
|
|
| 151 |
provider = cls.INFERENCE_PROVIDERS[provider_id]
|
| 152 |
available_models = []
|
| 153 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 154 |
for model_id, model_info in cls.AVAILABLE_MODELS.items():
|
| 155 |
# If model declares explicit providers, honor that
|
| 156 |
if "providers" in model_info:
|
|
@@ -182,7 +196,11 @@ class AppConfig:
|
|
| 182 |
raise ValueError(f"Unknown provider: {provider_id}")
|
| 183 |
|
| 184 |
provider = cls.INFERENCE_PROVIDERS[provider_id]
|
| 185 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 186 |
|
| 187 |
@classmethod
|
| 188 |
def get_optimal_context_settings(cls, model_id: str, provider_id: str, mcp_servers_count: int = 0) -> Dict[str, int]:
|
|
|
|
| 68 |
|
| 69 |
# Enhanced Inference Providers supporting GPT OSS models
|
| 70 |
INFERENCE_PROVIDERS = {
|
| 71 |
+
"auto": {
|
| 72 |
+
"name": "Auto (Router)",
|
| 73 |
+
"description": "Let the HF router automatically choose the best provider",
|
| 74 |
+
"supports_120b": True,
|
| 75 |
+
"supports_20b": True,
|
| 76 |
+
"endpoint_suffix": None, # Omit suffix to use router auto selection
|
| 77 |
+
"speed": "Varies",
|
| 78 |
+
"recommended_for": ["getting-started", "automatic"],
|
| 79 |
+
"max_context_support": 128000
|
| 80 |
+
},
|
| 81 |
"cerebras": {
|
| 82 |
"name": "Cerebras",
|
| 83 |
"description": "World-record inference speeds (2-4k tokens/sec for GPT-OSS)",
|
|
|
|
| 161 |
provider = cls.INFERENCE_PROVIDERS[provider_id]
|
| 162 |
available_models = []
|
| 163 |
|
| 164 |
+
# Auto provider exposes all models (router decides)
|
| 165 |
+
if provider_id == "auto":
|
| 166 |
+
return list(cls.AVAILABLE_MODELS.keys())
|
| 167 |
+
|
| 168 |
for model_id, model_info in cls.AVAILABLE_MODELS.items():
|
| 169 |
# If model declares explicit providers, honor that
|
| 170 |
if "providers" in model_info:
|
|
|
|
| 196 |
raise ValueError(f"Unknown provider: {provider_id}")
|
| 197 |
|
| 198 |
provider = cls.INFERENCE_PROVIDERS[provider_id]
|
| 199 |
+
# For 'auto', omit provider suffix and let router choose
|
| 200 |
+
endpoint_suffix = provider.get('endpoint_suffix')
|
| 201 |
+
if provider_id == 'auto' or not endpoint_suffix:
|
| 202 |
+
return model_id
|
| 203 |
+
return f"{model_id}:{endpoint_suffix}"
|
| 204 |
|
| 205 |
@classmethod
|
| 206 |
def get_optimal_context_settings(cls, model_id: str, provider_id: str, mcp_servers_count: int = 0) -> Dict[str, int]:
|