Make calls to "open" deterministic

They now use UTF-8 encoding.
This is the same as the old behaviour if (on Linux) environment variable LC_ALL=C.UTF-8 was set.
However, often this environment variable was not set and behaviour changed from environment to environment.

For CSV files, this also uses newline="".

Author: Rogier van Dalen

recipes/AISHELL-1/ASR/CTC/train_with_wav2vec.py

@@ -166,7 +166,7 @@ def on_stage_end(self, stage, stage_loss, epoch):
                 stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
                 test_stats=stage_stats,
             )
-            with open(self.hparams.cer_file, "w") as w:
+            with open(self.hparams.cer_file, "w", encoding="utf-8") as w:
                 self.cer_metric.write_stats(w)
 
     def init_optimizers(self):
@@ -318,7 +318,7 @@ def text_pipeline(wrd):
 if __name__ == "__main__":
     # CLI:
     hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # create ddp_group with the right communication protocol

recipes/AISHELL-1/ASR/seq2seq/train.py

@@ -141,7 +141,7 @@ def on_stage_end(self, stage, stage_loss, epoch):
                 stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
                 test_stats=stage_stats,
             )
-            with open(self.hparams.cer_file, "w") as w:
+            with open(self.hparams.cer_file, "w", encoding="utf-8") as w:
                 self.cer_metric.write_stats(w)
 
 
@@ -260,7 +260,7 @@ def text_pipeline(wrd):
 if __name__ == "__main__":
     # CLI:
     hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # create ddp_group with the right communication protocol

recipes/AISHELL-1/ASR/transformer/train.py

@@ -204,7 +204,7 @@ def on_stage_end(self, stage, stage_loss, epoch):
                 stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
                 test_stats=stage_stats,
             )
-            with open(self.hparams.cer_file, "w") as w:
+            with open(self.hparams.cer_file, "w", encoding="utf-8") as w:
                 self.cer_metric.write_stats(w)
 
             # save the averaged checkpoint at the end of the evaluation stage
@@ -382,7 +382,7 @@ def text_pipeline(wrd):
 if __name__ == "__main__":
     # CLI:
     hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # create ddp_group with the right communication protocol

recipes/AISHELL-1/ASR/transformer/train_with_wav2vect.py

@@ -187,7 +187,7 @@ def on_stage_end(self, stage, stage_loss, epoch):
                 stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
                 test_stats=stage_stats,
             )
-            with open(self.hparams.cer_file, "w") as w:
+            with open(self.hparams.cer_file, "w", encoding="utf-8") as w:
                 self.cer_metric.write_stats(w)
 
             # save the averaged checkpoint at the end of the evaluation stage
@@ -384,7 +384,7 @@ def text_pipeline(wrd):
 if __name__ == "__main__":
     # CLI:
     hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # create ddp_group with the right communication protocol

recipes/AISHELL-1/Tokenizer/train.py

@@ -23,7 +23,7 @@
 if __name__ == "__main__":
     # CLI:
     hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # create ddp_group with the right communication protocol

recipes/AISHELL-1/aishell_prepare.py

@@ -150,7 +150,7 @@ def prepare_aishell(
         data_folder, "transcript/aishell_transcript_v0.8.txt"
     )
 
-    with open(path_to_transcript, "r") as f:
+    with open(path_to_transcript, "r", encoding="utf-8") as f:
         lines = f.readlines()
         for line in lines:
             key = line.split()[0]
@@ -180,7 +180,7 @@ def prepare_aishell(
         total_line = 0
         total_duration = 0
         id = 0
-        with open(tmp_csv, mode="w", encoding="utf-8") as csv_f:
+        with open(tmp_csv, mode="w", newline="", encoding="utf-8") as csv_f:
             csv_writer = csv.writer(
                 csv_f, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL
             )

recipes/AMI/Diarization/experiment.py

@@ -154,7 +154,7 @@ def prepare_subset_json(full_meta_data, rec_id, out_meta_file):
         if k.startswith(rec_id):
             subset[key] = full_meta_data[key]
 
-    with open(out_meta_file, mode="w") as json_f:
+    with open(out_meta_file, mode="w", encoding="utf-8") as json_f:
         json.dump(subset, json_f, indent=2)
 
 
@@ -302,11 +302,11 @@ def diarize_dataset(full_meta, split_type, n_lambdas, pval, n_neighbors=10):
     # This is not needed but just staying with the standards.
     concate_rttm_file = out_rttm_dir + "/sys_output.rttm"
     logger.debug("Concatenating individual RTTM files...")
-    with open(concate_rttm_file, "w") as cat_file:
+    with open(concate_rttm_file, "w", encoding="utf-8") as cat_file:
         for f in glob.glob(out_rttm_dir + "/*.rttm"):
             if f == concate_rttm_file:
                 continue
-            with open(f, "r") as indi_rttm_file:
+            with open(f, "r", encoding="utf-8") as indi_rttm_file:
                 shutil.copyfileobj(indi_rttm_file, cat_file)
 
     msg = "The system generated RTTM file for %s set : %s" % (
@@ -507,7 +507,7 @@ def audio_pipeline(wav):
     # Load hyperparameters file with command-line overrides.
     params_file, run_opts, overrides = sb.core.parse_arguments(sys.argv[1:])
 
-    with open(params_file) as fin:
+    with open(params_file, encoding="utf-8") as fin:
         params = load_hyperpyyaml(fin, overrides)
 
     # Dataset prep (preparing metadata files)
@@ -558,7 +558,7 @@ def audio_pipeline(wav):
     # AMI Dev Set: Tune hyperparams on dev set.
     # Read the meta-data file for dev set generated during data_prep
     dev_meta_file = params["dev_meta_file"]
-    with open(dev_meta_file, "r") as f:
+    with open(dev_meta_file, "r", encoding="utf-8") as f:
         meta_dev = json.load(f)
 
     full_meta = meta_dev
@@ -600,7 +600,7 @@ def audio_pipeline(wav):
     # Load 'dev' and 'eval' metadata files.
     full_meta_dev = full_meta  # current full_meta is for 'dev'
     eval_meta_file = params["eval_meta_file"]
-    with open(eval_meta_file, "r") as f:
+    with open(eval_meta_file, "r", encoding="utf-8") as f:
         full_meta_eval = json.load(f)
 
     # Tag to be appended to final output DER files. Writing DER for individual files.

recipes/AMI/ami_prepare.py

@@ -294,7 +294,7 @@ def prepare_segs_for_RTTM(
             RTTM = RTTM + rttm_per_rec
 
     # Write one RTTM as groundtruth. For example, "fullref_eval.rttm"
-    with open(out_rttm_file, "w") as f:
+    with open(out_rttm_file, "w", encoding="utf-8") as f:
         for item in RTTM:
             f.write("%s\n" % item)
 
@@ -406,7 +406,7 @@ def prepare_metadata(
 
     # Read RTTM
     RTTM = []
-    with open(rttm_file, "r") as f:
+    with open(rttm_file, "r", encoding="utf-8") as f:
         for line in f:
             entry = line[:-1]
             RTTM.append(entry)
@@ -438,12 +438,12 @@ def prepare_metadata(
     segs_file = save_dir + "/" + filename + ".segments.rttm"
     subsegment_file = save_dir + "/" + filename + ".subsegments.rttm"
 
-    with open(segs_file, "w") as f:
+    with open(segs_file, "w", encoding="utf-8") as f:
         for row in MERGED_SEGMENTS:
             line_str = " ".join(row)
             f.write("%s\n" % line_str)
 
-    with open(subsegment_file, "w") as f:
+    with open(subsegment_file, "w", encoding="utf-8") as f:
         for row in SUBSEGMENTS:
             line_str = " ".join(row)
             f.write("%s\n" % line_str)
@@ -510,7 +510,7 @@ def prepare_metadata(
             }
 
     out_json_file = save_dir + "/" + filename + "." + mic_type + ".subsegs.json"
-    with open(out_json_file, mode="w") as json_f:
+    with open(out_json_file, mode="w", encoding="utf-8") as json_f:
         json.dump(json_dict, json_f, indent=2)
 
     msg = "%s JSON prepared" % (out_json_file)

recipes/Aishell1Mix/prepare_data.py

@@ -227,7 +227,10 @@ def create_aishell1mix2_csv(
         ]
 
         with open(
-            savepath + "/aishell1mix2_" + set_type + ".csv", "w"
+            savepath + "/aishell1mix2_" + set_type + ".csv",
+            "w",
+            newline="",
+            encoding="utf-8",
         ) as csvfile:
             writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
             writer.writeheader()
@@ -304,7 +307,10 @@ def create_aishell1mix3_csv(
         ]
 
         with open(
-            savepath + "/aishell1mix3_" + set_type + ".csv", "w"
+            savepath + "/aishell1mix3_" + set_type + ".csv",
+            "w",
+            newline="",
+            encoding="utf-8",
         ) as csvfile:
             writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
             writer.writeheader()

recipes/Aishell1Mix/separation/scripts/create_aishell1_metadata.py

@@ -38,6 +38,7 @@ def create_aishell1_metadata(aishell1_dir, md_dir):
     with open(
         os.path.join(aishell1_dir, "transcript/aishell_transcript_v0.8.txt"),
         "r",
+        encoding="utf-8",
     ) as f:
         lines = f.readlines()
         for line in lines:

recipes/Aishell1Mix/separation/train.py

@@ -356,7 +356,7 @@ def save_results(self, test_data):
             test_data, **self.hparams.dataloader_opts
         )
 
-        with open(save_file, "w") as results_csv:
+        with open(save_file, "w", newline="", encoding="utf-8") as results_csv:
             writer = csv.DictWriter(results_csv, fieldnames=csv_columns)
             writer.writeheader()
 
@@ -561,7 +561,7 @@ def audio_pipeline_noise(noise_wav):
 if __name__ == "__main__":
     # Load hyperparameters file with command-line overrides
     hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # Initialize ddp (useful only for multi-GPU DDP training)

recipes/AudioMNIST/audiomnist_prepare.py

@@ -278,7 +278,7 @@ def read_file_list(file_name):
     result: lists
         the file list
     """
-    with open(file_name) as list_file:
+    with open(file_name, encoding="utf-8") as list_file:
         return [line.strip() for line in list_file]
 
 
@@ -381,7 +381,7 @@ def read_meta(file_name):
     result: dict
         raw metadata
     """
-    with open(file_name) as meta_file:
+    with open(file_name, encoding="utf-8") as meta_file:
         return json.load(meta_file)
 
 
@@ -652,7 +652,7 @@ def convert_split(
         metadata[item_id].update(process_meta)
 
     logger.info(f"Saving metadata to {metadata_file_path}")
-    with open(metadata_file_path, "w") as metadata_file:
+    with open(metadata_file_path, "w", encoding="utf-8") as metadata_file:
         json.dump(metadata, metadata_file, indent=2)
 
 
@@ -805,7 +805,7 @@ def read_digit_lookup(file_name):
         }
 
     """
-    with open(file_name) as lookup_file:
+    with open(file_name, encoding="utf-8") as lookup_file:
         reader = csv.DictReader(lookup_file)
         lookup = {row["digit"]: row for row in reader}
         for value in lookup.values():

recipes/AudioMNIST/diffusion/train.py

@@ -1598,7 +1598,7 @@ def check_tensorboard(hparams):
     sb.utils.distributed.ddp_init_group(run_opts)
 
     # Load hyperparameters file with command-line overrides.
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # Check whether Tensorboard is available and enabled

recipes/BinauralWSJ0Mix/prepare_data.py

@@ -100,7 +100,10 @@ def create_binaural_wsj0mix2_csv(
         ]
 
         with open(
-            os.path.join(savepath, savename + set_type + ".csv"), "w"
+            os.path.join(savepath, savename + set_type + ".csv"),
+            "w",
+            newline="",
+            encoding="utf-8",
         ) as csvfile:
             writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
             writer.writeheader()
@@ -184,7 +187,10 @@ def create_binaural_wsj0mix3_csv(
         ]
 
         with open(
-            os.path.join(savepath, savename + set_type + ".csv"), "w"
+            os.path.join(savepath, savename + set_type + ".csv"),
+            "w",
+            newline="",
+            encoding="utf-8",
         ) as csvfile:
             writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
             writer.writeheader()
@@ -279,7 +285,10 @@ def create_binaural_wsj0mix2_noise_csv(
         ]
 
         with open(
-            os.path.join(savepath, savename + set_type + ".csv"), "w"
+            os.path.join(savepath, savename + set_type + ".csv"),
+            "w",
+            newline="",
+            encoding="utf-8",
         ) as csvfile:
             writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
             writer.writeheader()
@@ -359,7 +368,10 @@ def create_binaural_wsj0mix2_reverb_csv(
         ]
 
         with open(
-            os.path.join(savepath, savename + set_type + ".csv"), "w"
+            os.path.join(savepath, savename + set_type + ".csv"),
+            "w",
+            newline="",
+            encoding="utf-8",
         ) as csvfile:
             writer = csv.DictWriter(csvfile, fieldnames=csv_columns)
             writer.writeheader()

recipes/BinauralWSJ0Mix/separation/train.py

@@ -490,7 +490,7 @@ def save_results(self, test_data):
             test_data, **self.hparams.dataloader_opts
         )
 
-        with open(save_file, "w") as results_csv:
+        with open(save_file, "w", newline="", encoding="utf-8") as results_csv:
             writer = csv.DictWriter(results_csv, fieldnames=csv_columns)
             writer.writeheader()
 
@@ -680,7 +680,7 @@ def audio_pipeline_noise(noise_wav):
 if __name__ == "__main__":
     # Load hyperparameters file with command-line overrides
     hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # Initialize ddp (useful only for multi-GPU DDP training)

recipes/CVSS/S2ST/train.py

@@ -410,7 +410,7 @@ def _save_progress_sample(self, epoch):
             )
 
             sample_path = save_folder / f"{utt_id}.txt"
-            with open(sample_path, "w") as file:
+            with open(sample_path, "w", encoding="utf-8") as file:
                 file.write(f"pred: {transcript}\n")
                 file.write(f"ref: {tgt_transcript}\n")
 
@@ -421,7 +421,7 @@ def _save_progress_sample(self, epoch):
         )
 
         bleu_path = save_folder / "bleu.txt"
-        with open(bleu_path, "w") as file:
+        with open(bleu_path, "w", encoding="utf-8") as file:
             file.write(
                 f"BLEU score: {round(self.bleu_metric.summarize('BLEU'), 2)}\n"
             )
@@ -553,7 +553,7 @@ def unit_pipeline(utt_id):
     # Load hyperparameters file with command-line overrides
     hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
 
-    with open(hparams_file) as fin:
+    with open(hparams_file, encoding="utf-8") as fin:
         hparams = load_hyperpyyaml(fin, overrides)
 
     # If distributed_launch=True then