tokenizer included in rouge and bleu

generation_evaluator.py

@@ -167,15 +167,15 @@ class GenerationEvaluator(evaluate.Metric):
         self.bert_scorer = evaluate.load("bertscore")
         self.chrf_scorer = evaluate.load("chrf")
 
-    def _compute(self, predictions, references):
+    def _compute(self, predictions, references, tokenizer=None):
         # Compute ROUGE
         rouge_results = self.rouge_scorer.compute(
-            predictions=predictions, references=references
+            predictions=predictions, references=references, tokenizer=tokenizer
         )
 
         # Compute BLEU
         bleu_results = self.bleu_scorer.compute(
-            predictions=predictions, references=references
+            predictions=predictions, references=references, tokenizer=tokenizer
         )
 
         # Compute Exact Match