small tokenizer mod for bleu

generation_evaluator.py

@@ -174,9 +174,14 @@ class GenerationEvaluator(evaluate.Metric):
         )
 
         # Compute BLEU
-        bleu_results = self.bleu_scorer.compute(
-            predictions=predictions, references=references, tokenizer=tokenizer
-        )
+        if tokenizer is None:
+            bleu_results = self.bleu_scorer.compute(
+                predictions=predictions, references=references
+            )
+        else:
+            bleu_results = self.bleu_scorer.compute(
+                predictions=predictions, references=references, tokenizer=tokenizer
+            )
 
         # Compute Exact Match
         exact_match_results = self.exact_match_scorer.compute(