Update models

Author: abhik1505040

LaSE/LaSE/utils.py

@@ -69,7 +69,7 @@ def load_langid_model(cache_dir=None):
     "marathi": "mr",
     "spanish": "es",
     "scottish_gaelic": "gd",
-    "nepali": "np",
+    "nepali": "ne",
     "pashto": "ps",
     "persian": "fa",
     "pidgin": "pcm",

README.md

@@ -3,13 +3,15 @@ We use a modified fork of [huggingface transformers](https://github.com/huggingf
 ## Setup
 
 ```bash
-$ git clone https://github.com/abhik1505040/crossum
+$ git clone https://github.com/csebuetnlp/crossum
 $ cd crossum/seq2seq
 $ conda create python==3.7.9 pytorch==1.7.1 torchvision==0.8.2 torchaudio==0.7.2 cudatoolkit=10.2 -c pytorch -p ./env
 $ conda activate ./env # or source activate ./env (for older versions of anaconda)
 $ bash setup.sh 
 ```
 
+- **Note**: For newer NVIDIA GPUS such as ***A100*** or ***3090*** use `cudatoolkit=11.1`.
+
 ## Downloading data
 
 This script downloads the metadata-stripped version of the dataset required for training.
@@ -30,7 +32,7 @@ Some sample commands for training on a 8 GPU node are given below.
 For multi-node usage with SLURM, refer to [job.sh]().
 
 ```bash
-bash trainer.sh --ngpus 8 --training_type m2m # trains the many-to-many model
+bash trainer.sh --ngpus 8 --training_type m2m --sampling multistage # trains the many-to-many model with multistage sampling
 bash trainer.sh --ngpus 8 --training_type m2o --pivot_lang arabic # trains the many-to-one model using arabic as the target language
 bash trainer.sh --ngpus 8 --training_type o2m --pivot_lang english # trains the one-to-many model using english as the source language
 ```

figs/ar_tgt_lase.png

@@ -3,9 +3,9 @@
 FILE="dataset.tar.bz2"
 
 if [[ ! -d "dataset" ]]; then
-    id="1ywYJEEaFnXIWW5xBwp0cNuPinDwQjCxe"
-    curl -c ./cookie -s -L "https://drive.google.com/uc?export=download&id=${id}" > /dev/null
-    curl -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=${id}" -o ${FILE}
-    rm ./cookie
+    id="1bwURjAyQT6OkGRd_f9mwkWg9FABa_c6S"
+    cert="https://docs.google.com/uc?export=download&id=${id}"
+    wget --load-cookies /tmp/cookies.txt "https://docs.google.com/uc?export=download&confirm=$(wget --quiet --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate ${cert} -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=${id}" -O ${FILE}
+    rm -rf /tmp/cookies.txt
     tar -xvf ${FILE} && rm ${FILE}
 fi

figs/en_tgt_rouge2.png

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+ROOT_DATASET_DIR="dataset"
+ROOT_MODEL_DIR="output"
+RESULTS_DIR="evaluation_results"
+
+for model_dir in $ROOT_MODEL_DIR/*/; do
+
+    suffix=$(basename $model_dir)
+    read training_type pivot_lang rest <<< $(IFS="_"; echo $suffix)
+
+    if [[ "$training_type" = "m2o" ]]; then
+        required_str="--required_tgt_lang ${pivot_lang}"
+    elif [[ "$training_type" = "o2m" ]]; then
+        required_str="--required_src_lang ${pivot_lang}"
+    else
+        required_str=" "
+    fi
+
+    for data_type in "val" "test"; do
+        python evaluator.py \
+            --dataset_dir "${ROOT_DATASET_DIR}" \
+            --output_dir "${RESULTS_DIR}/${suffix}" \
+            --evaluation_type xlingual \
+            --data_type ${data_type} \
+            --xlingual_summarization_model_name_or_path $model_dir \
+            $required_str
+    done
+done

figs/hi_tgt_rouge2.png

@@ -231,7 +231,7 @@ def summarize_xlingual(
     tgt_lang,
     args
 ):
-    if os.path.isfile(os.path.join(output_dir, "test_generations.txt")):
+    if os.path.isfile(os.path.join(output_dir, f"{args.data_type}_generations.txt")):
         return
 
     script_path =  os.path.abspath("pipeline.py")
@@ -246,9 +246,10 @@ def summarize_xlingual(
         f"--no_repeat_ngram_size {args.no_repeat_ngram_size}",
         f"--eval_beams {args.beam_size}",
         f"--tgt_lang {tgt_lang}",
+        f"--rouge_lang {tgt_lang}",
         "--overwrite_output_dir",
         "--predict_with_generate",
-        "--do_predict" if args.data_type == "test" else "--do_eval",
+        "--do_predict",
         "--use_langid",
         "--seed 1234"        
     ]
@@ -272,8 +273,8 @@ def calculate_lase(
 
 
 def run(args):
-    root_output_dir = os.path.join(args.output_dir, args.data_type, args.evaluation_type, "outputs")
-    root_log_dir = os.path.join(args.output_dir, args.data_type, args.evaluation_type, "logs")
+    root_output_dir = os.path.join(args.output_dir, args.data_type, "outputs")
+    root_log_dir = os.path.join(args.output_dir, args.data_type, "logs")
 
     os.makedirs(root_output_dir, exist_ok=True)
     os.makedirs(root_log_dir, exist_ok=True)
@@ -346,21 +347,38 @@ def evaluate(lase_key):
                 pipeline_target_path
             )
 
+            # specially handly validation files
+            # since output file is generated for 
+            # test files only
+            if args.data_type == "val":
+                shutil.copy(
+                    pipeline_source_path,
+                    os.path.join(dir_name, "test.source")
+                )
+                shutil.copy(
+                    pipeline_source_path,
+                    os.path.join(dir_name, "test.target")
+                )
+
             if args.evaluation_type == "xlingual":
                 summarize_xlingual(dir_name, dir_name, tgt_lang, args)
+
+                if args.data_type == "val":
+                    shutil.move(
+                        os.path.join(dir_name, f"test_generations.txt"),
+                        os.path.join(dir_name, f"val_generations.txt")
+                    )
+
+                    os.remove(os.path.join(dir_name, "test.source"))
+                    os.remove(os.path.join(dir_name, "test.target"))
+
                 pred_lines = read_lines(
-                    os.path.join(dir_name, "test_generations.txt")
+                    os.path.join(dir_name, f"{args.data_type}_generations.txt")
                 )
                 ref_lines = read_lines(pipeline_target_path)
 
-                if lase_key == "LaSE_in_lang":
-                    scores.update(
-                        calculate_rouge(pred_lines, ref_lines, rouge_lang=tgt_lang)
-                    )
 
-                lase_scores = calculate_lase(pred_lines, ref_lines, tgt_lang)
-                scores[lase_key] = lase_scores["LaSE"]
-                
+
             elif args.evaluation_type == "baseline":
                 src_iso, tgt_iso = LANG2ISO.get(src_lang, None), LANG2ISO.get(tgt_lang, None)
                 if (
@@ -386,13 +404,13 @@ def evaluate(lase_key):
                 pred_lines = read_lines(translated_path)
                 ref_lines = read_lines(pipeline_target_path)
 
-                if lase_key == "LaSE_in_lang":
-                    scores.update(
-                        calculate_rouge(pred_lines, ref_lines, rouge_lang=tgt_lang)
-                    )
+            if lase_key == "LaSE_in_lang":
+                scores.update(
+                    calculate_rouge(pred_lines, ref_lines, rouge_lang=tgt_lang)
+                )
 
-                lase_scores = calculate_lase(pred_lines, ref_lines, tgt_lang)
-                scores[lase_key] = lase_scores["LaSE"]
+            lase_scores = calculate_lase(pred_lines, ref_lines, tgt_lang)
+            scores[lase_key] = lase_scores["LaSE"]
 
 
         # first do crossum evaluation (in lang LaSE)
@@ -411,7 +429,7 @@ def evaluate(lase_key):
         gc.collect()
 
     # aggregate results
-    combined_results_path = os.path.join(args.output_dir, args.data_type, args.evaluation_type, "combined_results.log")
+    combined_results_path = os.path.join(args.output_dir, args.data_type, "combined_results.log")
     logging.info("Writing the combined results to " + combined_results_path)
 
     with open(combined_results_path, 'w') as outf:

figs/ru_tgt_rouge2.png

@@ -602,7 +602,11 @@ def main():
         logger.info("*** Evaluate ***")
 
         metrics = trainer.evaluate(
-            metric_key_prefix="val", max_length=data_args.val_max_target_length, num_beams=data_args.eval_beams
+            metric_key_prefix="val", 
+            max_length=data_args.val_max_target_length, 
+            num_beams=data_args.eval_beams,
+            length_penalty=data_args.length_penalty,
+            no_repeat_ngram_size=data_args.no_repeat_ngram_size,
         )
         metrics["val_n_objs"] = data_args.n_val
         metrics["val_loss"] = round(metrics["val_loss"], 4)
@@ -618,7 +622,7 @@ def main():
         test_output = trainer.predict(
             test_dataset=test_dataset,
             metric_key_prefix="test",
-            max_length=data_args.val_max_target_length,
+            max_length=data_args.test_max_target_length,
             num_beams=data_args.eval_beams,
             length_penalty=data_args.length_penalty,
             no_repeat_ngram_size=data_args.no_repeat_ngram_size,

seq2seq/README.md

@@ -10,6 +10,8 @@ parser.add_argument('--training_type', type=str, choices=["m2m", "m2o", "o2m"],
                     required=True, help='Training type (many-to-many/many-to-one/one-to-many)')
 parser.add_argument('--pivot_lang', type=str, default="english",
                     help='Pivot language (Applicable for many-to-one and one-to-many)')
+parser.add_argument('--sampling', type=str, default="multistage", choices=["multistage", "unistage"],
+                    help='Sampling type (Applicable for many-to-many)')
 parser.add_argument('--exclude_native', action='store_true',
                     default=False, help='Exclude the native-to-native filepairs during training')
 EOF
@@ -22,10 +24,17 @@ export ROOT_OUTPUT_DIR="${BASE_DIR}/output"
 
 export PREFIX="${TRAINING_TYPE}_${PIVOT_LANG}"
 if [[ "$TRAINING_TYPE" = "m2m" ]]; then
-    PREFIX="${TRAINING_TYPE}"
+    PREFIX="${TRAINING_TYPE}_${SAMPLING}" 
     OPTIONAL_ARGS=(
         "--multistage_upsampling_factors 0.5 0.75"
     )
+    
+    if [[ "$SAMPLING" = "unistage" ]]; then
+        OPTIONAL_ARGS=(
+            "--upsampling_factor 0.25"
+        )   
+    fi
+    
 else
     OPTIONAL_ARGS=(
         "--upsampling_factor 0.75"
@@ -40,7 +49,7 @@ fi
 export BASENAME="${PREFIX}_${SUFFIX}"
 export INPUT_DIR="${ROOT_INPUT_DIR}/${BASENAME}"
 export OUTPUT_DIR="${ROOT_OUTPUT_DIR}/${BASENAME}"
-export MIN_EXAMPLE_COUNT=32
+export MIN_EXAMPLE_COUNT=30
 
 conda activate "${BASE_DIR}/env" || source activate "${BASE_DIR}/env"
 

seq2seq/download_data.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+bash trainer.sh --ngpus 8 --training_type m2o --pivot_lang english
+bash trainer.sh --ngpus 8 --training_type o2m --pivot_lang english
+bash trainer.sh --ngpus 8 --training_type m2o --pivot_lang hindi
+bash trainer.sh --ngpus 8 --training_type o2m --pivot_lang hindi
+bash trainer.sh --ngpus 8 --training_type m2o --pivot_lang russian
+bash trainer.sh --ngpus 8 --training_type o2m --pivot_lang russian
+bash trainer.sh --ngpus 8 --training_type m2o --pivot_lang arabic
+bash trainer.sh --ngpus 8 --training_type o2m --pivot_lang arabic