Spaces:
Sleeping
Sleeping
File size: 9,552 Bytes
d28c36c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 |
"""
Évaluation finale comparative de tous les modèles pour MCP
Inclut le nouveau Gemma-3-270M plus petit et potentiellement plus rapide
"""
import sys
import os
import json
import time
# Ajouter le chemin pour les imports
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
def test_model_comprehensive(model_path, model_name):
"""Test complet d'un modèle pour les tâches MCP"""
try:
from llama_cpp import Llama
print(f"🔄 Test de {model_name}...")
# Initialiser avec des paramètres optimisés
llm = Llama(
model_path=model_path,
n_ctx=1024,
n_threads=1,
verbose=False
)
# Tests MCP variés
tests = [
{
"name": "Commande simple",
"prompt": """Tu es un assistant IA pour un jeu RTS via MCP.
Outils: get_game_state(), move_units(unit_ids, target_x, target_y)
Commande: "Montre-moi l'état du jeu"
Réponds avec JSON: {{"tool": "nom_outil", "args": {{}}}}""",
"expected": "get_game_state"
},
{
"name": "Action avec paramètres",
"prompt": """Outils: move_units(unit_ids, target_x, target_y)
Commande: "Déplace l'infanterie vers 100,200"
JSON: {{"tool": "move_units", "args": {{"unit_ids": ["infantry"], "target_x": 100, "target_y": 200}}}}""",
"expected": "move_units"
},
{
"name": "Vitesse de réponse",
"prompt": "Réponds simplement: OK",
"expected": "OK"
}
]
total_score = 0
total_time = 0
results = []
for test in tests:
start_time = time.time()
response = llm(
test['prompt'],
max_tokens=80,
temperature=0.1,
stop=["</s>", "<|im_end|>"]
)
response_time = time.time() - start_time
response_text = response['choices'][0]['text'].strip()
# Noter la réponse
score = 0
# JSON valide pour les tests MCP
if test['name'] != "Vitesse de réponse":
try:
json.loads(response_text)
score += 3
except:
pass
# Contenu attendu
if test['expected'] in response_text:
score += 4
# Format approprié
if "tool" in response_text and test['name'] != "Vitesse de réponse":
score += 2
# Cohérence
if any(word in response_text.lower() for word in ['game', 'move', 'state']):
score += 1
score = min(score, 10)
total_score += score
total_time += response_time
results.append({
'test': test['name'],
'score': score,
'time': response_time,
'response': response_text[:50] + "..." if len(response_text) > 50 else response_text
})
avg_score = total_score / len(tests)
avg_time = total_time / len(tests)
print(f"✅ {model_name}: {avg_score:.1f}/10 | Temps: {avg_time:.2f}s")
return {
'name': model_name,
'avg_score': avg_score,
'avg_time': avg_time,
'efficiency': avg_score / avg_time if avg_time > 0 else 0,
'tests': results
}
except Exception as e:
print(f"❌ {model_name}: Erreur - {e}")
return {
'name': model_name,
'avg_score': 0,
'avg_time': 0,
'efficiency': 0,
'error': str(e)
}
def main():
"""Évaluation finale comparative"""
print("🏁 ÉVALUATION FINALE COMPARATIVE MCP")
print("=" * 60)
# Tous les modèles à tester
models = [
{
'name': 'Qwen2.5-0.5B',
'path': 'qwen2.5-0.5b-instruct-q4_0.gguf'
},
{
'name': 'Qwen3-0.6B',
'path': 'Qwen3-0.6B-Q8_0.gguf'
},
{
'name': 'Gemma-3-1B',
'path': 'google_gemma-3-1b-it-qat-Q4_0.gguf'
},
{
'name': 'Gemma-3-270M',
'path': 'gemma-3-270m-it-qat-Q8_0.gguf'
}
]
results = []
for model in models:
if os.path.exists(model['path']):
result = test_model_comprehensive(model['path'], model['name'])
results.append(result)
else:
print(f"❌ Fichier non trouvé: {model['path']}")
# Analyse comparative
print("\n" + "=" * 60)
print("📊 RÉSULTATS FINAUX")
print("=" * 60)
successful_results = [r for r in results if 'error' not in r and r['avg_score'] > 0]
if successful_results:
# Classement par score
sorted_by_score = sorted(successful_results, key=lambda x: x['avg_score'], reverse=True)
print(f"\n🏆 CLASSEMENT PAR PERFORMANCE:")
for i, result in enumerate(sorted_by_score, 1):
file_size = os.path.getsize([m['path'] for m in models if m['name'] == result['name']][0]) / (1024*1024)
print(f" {i}. {result['name']}: {result['avg_score']:.1f}/10 | {result['avg_time']:.2f}s | {file_size:.0f}MB")
# Classement par efficacité (score/seconde)
sorted_by_efficiency = sorted(successful_results, key=lambda x: x['efficiency'], reverse=True)
print(f"\n⚡ CLASSEMENT PAR EFFICACITÉ:")
for i, result in enumerate(sorted_by_efficiency, 1):
file_size = os.path.getsize([m['path'] for m in models if m['name'] == result['name']][0]) / (1024*1024)
print(f" {i}. {result['name']}: {result['efficiency']:.2f} score/s | {file_size:.0f}MB")
# Meilleur modèle global
best_overall = sorted_by_score[0]
most_efficient = sorted_by_efficiency[0]
print(f"\n🎯 MEILLEUR MODÈLE GLOBAL: {best_overall['name']}")
print(f" Score: {best_overall['avg_score']:.1f}/10")
print(f" Temps: {best_overall['avg_time']:.2f}s")
print(f"\n⚡ MODÈLE LE PLUS EFFICACE: {most_efficient['name']}")
print(f" Efficacité: {most_efficient['efficiency']:.2f} score/s")
# Analyse détaillée
print(f"\n📈 ANALYSE DÉTAILLÉE:")
for result in successful_results:
file_size = os.path.getsize([m['path'] for m in models if m['name'] == result['name']][0]) / (1024*1024)
efficiency_per_mb = result['efficiency'] / file_size if file_size > 0 else 0
print(f"\n🔹 {result['name']}:")
print(f" Score moyen: {result['avg_score']:.1f}/10")
print(f" Temps moyen: {result['avg_time']:.2f}s")
print(f" Efficacité: {result['efficiency']:.2f} score/s")
print(f" Taille: {file_size:.0f}MB")
print(f" Efficacité/MB: {efficiency_per_mb:.4f}")
# Tests individuels
for test in result['tests']:
status = "✅" if test['score'] >= 6 else "⚠️" if test['score'] >= 4 else "❌"
print(f" {status} {test['test']}: {test['score']}/10 ({test['time']:.2f}s)")
# Recommandations finales
print(f"\n💡 RECOMMANDATIONS FINALES:")
if best_overall['avg_score'] >= 7:
print(f"✅ {best_overall['name']} est EXCELLENT pour la production MCP")
elif best_overall['avg_score'] >= 5:
print(f"👍 {best_overall['name']} est BON pour la production MCP")
else:
print(f"⚠️ {best_overall['name']} nécessite des améliorations")
# Recommandation basée sur l'usage
print(f"\n🎯 RECOMMANDATIONS SPÉCIFIQUES:")
if most_efficient['name'] != best_overall['name']:
print(f"⚡ Pour les réponses rapides: {most_efficient['name']}")
print(f"🏆 Pour la meilleure qualité: {best_overall['name']}")
else:
print(f"🎉 {best_overall['name']} est le meilleur choix pour la vitesse ET la qualité")
# Vérifier si Gemma-3-270M est surprenant
gemma_270m = next((r for r in successful_results if r['name'] == 'Gemma-3-270M'), None)
if gemma_270m and gemma_270m['avg_score'] >= 5:
print(f"🚀 Surprise: Gemma-3-270M offre un excellent rapport taille/performance!")
# Sauvegarder résultats complets
final_results = {
'all_results': results,
'successful_models': successful_results,
'ranking_by_score': sorted_by_score if successful_results else [],
'ranking_by_efficiency': sorted_by_efficiency if successful_results else [],
'best_overall': best_overall if successful_results else None,
'most_efficient': most_efficient if successful_results else None
}
with open("final_model_comparison.json", "w", encoding="utf-8") as f:
json.dump(final_results, f, indent=2, ensure_ascii=False)
print(f"\n📄 Résultats complets sauvegardés dans: final_model_comparison.json")
if __name__ == "__main__":
main() |