Disable dataloader multiprocessing

dataset.py

@@ -14,8 +14,6 @@ class DatasetReader(IterableDataset):
         self.current_line = 0
         self.total_lines = count_lines(filename)
         print(f"{self.total_lines} lines in {filename}")
-        file_itr = open(self.filename, "r", encoding="utf8")
-        self.mapped_itr = map(self.preprocess, file_itr)
 
     def preprocess(self, text: str):
         self.current_line += 1
@@ -31,7 +29,9 @@ class DatasetReader(IterableDataset):
         )
 
     def __iter__(self):
-        return self.mapped_itr
+        file_itr = open(self.filename, "r", encoding="utf8")
+        mapped_itr = map(self.preprocess, file_itr)
+        return mapped_itr
 
     def __len__(self):
         return self.total_lines
@@ -50,10 +50,6 @@ class ParallelTextReader(IterableDataset):
         self.num_sentences = gold_path_lines
         self.current_line = 0
 
-        pred_itr = open(self.pred_path, "r", encoding="utf8")
-        gold_itr = open(self.gold_path, "r", encoding="utf8")
-        self.mapped_itr = map(self.preprocess, pred_itr, gold_itr)
-
     def preprocess(self, pred: str, gold: str):
         self.current_line += 1
         pred = pred.rstrip().strip()
@@ -65,7 +61,10 @@ class ParallelTextReader(IterableDataset):
         return pred, [gold]
 
     def __iter__(self):
-        return self.mapped_itr
+        pred_itr = open(self.pred_path, "r", encoding="utf8")
+        gold_itr = open(self.gold_path, "r", encoding="utf8")
+        mapped_itr = map(self.preprocess, pred_itr, gold_itr)
+        return mapped_itr
 
     def __len__(self):
         return self.num_sentences

eval.py

@@ -19,7 +19,9 @@ def get_dataloader(pred_path: str, gold_path: str, batch_size: int):
         return list(map(list, zip(*batch)))
 
     reader = ParallelTextReader(pred_path=pred_path, gold_path=gold_path)
-    dataloader = DataLoader(reader, batch_size=batch_size, collate_fn=collate_fn)
+    dataloader = DataLoader(
+        reader, batch_size=batch_size, collate_fn=collate_fn, num_workers=0
+    )
     return dataloader
 
 

translate.py

@@ -19,10 +19,6 @@ from dataset import DatasetReader, count_lines
 
 from accelerate import Accelerator, DistributedType, find_executable_batch_size
 
-torch.multiprocessing.set_sharing_strategy(
-    "file_system"
-)  # FIXES RuntimeError: Too many open files.
-
 
 def get_dataloader(
     accelerator: Accelerator,
@@ -55,7 +51,7 @@ def get_dataloader(
         dataset,
         batch_size=batch_size,
         collate_fn=data_collator,
-        num_workers=1,
+        num_workers=0,  # Disable multiprocessing
     )